001    /**
002    * Licensed to the Apache Software Foundation (ASF) under one
003    * or more contributor license agreements.  See the NOTICE file
004    * distributed with this work for additional information
005    * regarding copyright ownership.  The ASF licenses this file
006    * to you under the Apache License, Version 2.0 (the
007    * "License"); you may not use this file except in compliance
008    * with the License.  You may obtain a copy of the License at
009    *
010    *     http://www.apache.org/licenses/LICENSE-2.0
011    *
012    * Unless required by applicable law or agreed to in writing, software
013    * distributed under the License is distributed on an "AS IS" BASIS,
014    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015    * See the License for the specific language governing permissions and
016    * limitations under the License.
017    */
018    
019    package org.apache.hadoop.yarn.logaggregation;
020    
021    import java.io.DataInputStream;
022    import java.io.EOFException;
023    import java.io.FileNotFoundException;
024    import java.io.IOException;
025    import java.io.PrintStream;
026    
027    import org.apache.commons.cli.CommandLine;
028    import org.apache.commons.cli.CommandLineParser;
029    import org.apache.commons.cli.GnuParser;
030    import org.apache.commons.cli.HelpFormatter;
031    import org.apache.commons.cli.Options;
032    import org.apache.commons.cli.ParseException;
033    import org.apache.commons.lang.StringUtils;
034    import org.apache.hadoop.classification.InterfaceAudience.Private;
035    import org.apache.hadoop.classification.InterfaceAudience.Public;
036    import org.apache.hadoop.classification.InterfaceStability.Evolving;
037    import org.apache.hadoop.conf.Configuration;
038    import org.apache.hadoop.conf.Configured;
039    import org.apache.hadoop.fs.FileContext;
040    import org.apache.hadoop.fs.FileStatus;
041    import org.apache.hadoop.fs.Path;
042    import org.apache.hadoop.fs.RemoteIterator;
043    import org.apache.hadoop.security.UserGroupInformation;
044    import org.apache.hadoop.util.Tool;
045    import org.apache.hadoop.yarn.api.records.ApplicationId;
046    import org.apache.hadoop.yarn.conf.YarnConfiguration;
047    import org.apache.hadoop.yarn.factories.RecordFactory;
048    import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
049    import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
050    import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader;
051    import org.apache.hadoop.yarn.util.ConverterUtils;
052    
053    import com.google.common.annotations.VisibleForTesting;
054    
055    @Public
056    @Evolving
057    public class LogDumper extends Configured implements Tool {
058    
059      private static final String CONTAINER_ID_OPTION = "containerId";
060      private static final String APPLICATION_ID_OPTION = "applicationId";
061      private static final String NODE_ADDRESS_OPTION = "nodeAddress";
062      private static final String APP_OWNER_OPTION = "appOwner";
063    
064      @Override
065      public int run(String[] args) throws Exception {
066    
067        Options opts = new Options();
068        opts.addOption(APPLICATION_ID_OPTION, true, "ApplicationId (required)");
069        opts.addOption(CONTAINER_ID_OPTION, true,
070          "ContainerId (must be specified if node address is specified)");
071        opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format "
072          + "nodename:port (must be specified if container id is specified)");
073        opts.addOption(APP_OWNER_OPTION, true,
074          "AppOwner (assumed to be current user if not specified)");
075    
076        if (args.length < 1) {
077          HelpFormatter formatter = new HelpFormatter();
078          formatter.printHelp("general options are: ", opts);
079          return -1;
080        }
081    
082        CommandLineParser parser = new GnuParser();
083        String appIdStr = null;
084        String containerIdStr = null;
085        String nodeAddress = null;
086        String appOwner = null;
087        try {
088          CommandLine commandLine = parser.parse(opts, args, true);
089          appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION);
090          containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION);
091          nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION);
092          appOwner = commandLine.getOptionValue(APP_OWNER_OPTION);
093        } catch (ParseException e) {
094          System.out.println("options parsing failed: " + e.getMessage());
095    
096          HelpFormatter formatter = new HelpFormatter();
097          formatter.printHelp("general options are: ", opts);
098          return -1;
099        }
100    
101        if (appIdStr == null) {
102          System.out.println("ApplicationId cannot be null!");
103          HelpFormatter formatter = new HelpFormatter();
104          formatter.printHelp("general options are: ", opts);
105          return -1;
106        }
107    
108        RecordFactory recordFactory =
109            RecordFactoryProvider.getRecordFactory(getConf());
110        ApplicationId appId =
111            ConverterUtils.toApplicationId(recordFactory, appIdStr);
112    
113        if (appOwner == null || appOwner.isEmpty()) {
114          appOwner = UserGroupInformation.getCurrentUser().getShortUserName();
115        }
116        int resultCode = 0;
117        if (containerIdStr == null && nodeAddress == null) {
118          resultCode = dumpAllContainersLogs(appId, appOwner, System.out);
119        } else if ((containerIdStr == null && nodeAddress != null)
120            || (containerIdStr != null && nodeAddress == null)) {
121          System.out.println("ContainerId or NodeAddress cannot be null!");
122          HelpFormatter formatter = new HelpFormatter();
123          formatter.printHelp("general options are: ", opts);
124          resultCode = -1;
125        } else {
126          Path remoteRootLogDir =
127            new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
128                YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
129          AggregatedLogFormat.LogReader reader =
130              new AggregatedLogFormat.LogReader(getConf(),
131                  LogAggregationUtils.getRemoteNodeLogFileForApp(
132                      remoteRootLogDir,
133                      appId,
134                      appOwner,
135                      ConverterUtils.toNodeId(nodeAddress),
136                      LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf())));
137          resultCode = dumpAContainerLogs(containerIdStr, reader, System.out);
138        }
139    
140        return resultCode;
141      }
142    
143      @Private
144      @VisibleForTesting
145      public int dumpAContainersLogs(String appId, String containerId,
146          String nodeId, String jobOwner) throws IOException {
147        Path remoteRootLogDir =
148            new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
149                YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
150        String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf());
151        Path logPath = LogAggregationUtils.getRemoteNodeLogFileForApp(
152            remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner,
153            ConverterUtils.toNodeId(nodeId), suffix);
154        AggregatedLogFormat.LogReader reader;
155        try {
156          reader = new AggregatedLogFormat.LogReader(getConf(), logPath);
157        } catch (FileNotFoundException fnfe) {
158          System.out.println("Logs not available at " + logPath.toString());
159          System.out.println(
160              "Log aggregation has not completed or is not enabled.");
161          return -1;
162        }
163        return dumpAContainerLogs(containerId, reader, System.out);
164      }
165    
166      private int dumpAContainerLogs(String containerIdStr,
167          AggregatedLogFormat.LogReader reader, PrintStream out)
168          throws IOException {
169        DataInputStream valueStream;
170        LogKey key = new LogKey();
171        valueStream = reader.next(key);
172    
173        while (valueStream != null && !key.toString().equals(containerIdStr)) {
174          // Next container
175          key = new LogKey();
176          valueStream = reader.next(key);
177        }
178    
179        if (valueStream == null) {
180          System.out.println("Logs for container " + containerIdStr
181              + " are not present in this log-file.");
182          return -1;
183        }
184    
185        while (true) {
186          try {
187            LogReader.readAContainerLogsForALogType(valueStream, out);
188          } catch (EOFException eof) {
189            break;
190          }
191        }
192        return 0;
193      }
194    
195      private int dumpAllContainersLogs(ApplicationId appId, String appOwner,
196          PrintStream out) throws IOException {
197        Path remoteRootLogDir =
198            new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
199                YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
200        String user = appOwner;
201        String logDirSuffix =
202            LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf());
203        //TODO Change this to get a list of files from the LAS.
204        Path remoteAppLogDir =
205            LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, user,
206                logDirSuffix);
207        RemoteIterator<FileStatus> nodeFiles;
208        try {
209          nodeFiles = FileContext.getFileContext().listStatus(remoteAppLogDir);
210        } catch (FileNotFoundException fnf) {
211          System.out.println("Logs not available at "
212              + remoteAppLogDir.toString());
213          System.out.println(
214              "Log aggregation has not completed or is not enabled.");
215          return -1;
216        }
217        while (nodeFiles.hasNext()) {
218          FileStatus thisNodeFile = nodeFiles.next();
219          AggregatedLogFormat.LogReader reader =
220              new AggregatedLogFormat.LogReader(getConf(),
221                  new Path(remoteAppLogDir, thisNodeFile.getPath().getName()));
222          try {
223    
224            DataInputStream valueStream;
225            LogKey key = new LogKey();
226            valueStream = reader.next(key);
227    
228            while (valueStream != null) {
229              String containerString = "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
230              out.println(containerString);
231              out.println(StringUtils.repeat("=", containerString.length()));
232              while (true) {
233                try {
234                  LogReader.readAContainerLogsForALogType(valueStream, out);
235                } catch (EOFException eof) {
236                  break;
237                }
238              }
239    
240              // Next container
241              key = new LogKey();
242              valueStream = reader.next(key);
243            }
244          } finally {
245            reader.close();
246          }
247        }
248        return 0;
249      }
250    
251      public static void main(String[] args) throws Exception {
252        Configuration conf = new YarnConfiguration();
253        LogDumper logDumper = new LogDumper();
254        logDumper.setConf(conf);
255        int exitCode = logDumper.run(args);
256        System.exit(exitCode);
257      }
258    }