001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.yarn.logaggregation; 020 021 import java.io.DataInputStream; 022 import java.io.EOFException; 023 import java.io.FileNotFoundException; 024 import java.io.IOException; 025 import java.io.PrintStream; 026 027 import org.apache.commons.cli.CommandLine; 028 import org.apache.commons.cli.CommandLineParser; 029 import org.apache.commons.cli.GnuParser; 030 import org.apache.commons.cli.HelpFormatter; 031 import org.apache.commons.cli.Options; 032 import org.apache.commons.cli.ParseException; 033 import org.apache.commons.lang.StringUtils; 034 import org.apache.hadoop.classification.InterfaceAudience.Private; 035 import org.apache.hadoop.classification.InterfaceAudience.Public; 036 import org.apache.hadoop.classification.InterfaceStability.Evolving; 037 import org.apache.hadoop.conf.Configuration; 038 import org.apache.hadoop.conf.Configured; 039 import org.apache.hadoop.fs.FileContext; 040 import org.apache.hadoop.fs.FileStatus; 041 import org.apache.hadoop.fs.Path; 042 import org.apache.hadoop.fs.RemoteIterator; 043 import org.apache.hadoop.security.UserGroupInformation; 044 import org.apache.hadoop.util.Tool; 045 import org.apache.hadoop.yarn.api.records.ApplicationId; 046 import org.apache.hadoop.yarn.conf.YarnConfiguration; 047 import org.apache.hadoop.yarn.factories.RecordFactory; 048 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; 049 import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey; 050 import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader; 051 import org.apache.hadoop.yarn.util.ConverterUtils; 052 053 import com.google.common.annotations.VisibleForTesting; 054 055 @Public 056 @Evolving 057 public class LogDumper extends Configured implements Tool { 058 059 private static final String CONTAINER_ID_OPTION = "containerId"; 060 private static final String APPLICATION_ID_OPTION = "applicationId"; 061 private static final String NODE_ADDRESS_OPTION = "nodeAddress"; 062 private static final String APP_OWNER_OPTION = "appOwner"; 063 064 @Override 065 public int run(String[] args) throws Exception { 066 067 Options opts = new Options(); 068 opts.addOption(APPLICATION_ID_OPTION, true, "ApplicationId (required)"); 069 opts.addOption(CONTAINER_ID_OPTION, true, 070 "ContainerId (must be specified if node address is specified)"); 071 opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format " 072 + "nodename:port (must be specified if container id is specified)"); 073 opts.addOption(APP_OWNER_OPTION, true, 074 "AppOwner (assumed to be current user if not specified)"); 075 076 if (args.length < 1) { 077 HelpFormatter formatter = new HelpFormatter(); 078 formatter.printHelp("general options are: ", opts); 079 return -1; 080 } 081 082 CommandLineParser parser = new GnuParser(); 083 String appIdStr = null; 084 String containerIdStr = null; 085 String nodeAddress = null; 086 String appOwner = null; 087 try { 088 CommandLine commandLine = parser.parse(opts, args, true); 089 appIdStr = commandLine.getOptionValue(APPLICATION_ID_OPTION); 090 containerIdStr = commandLine.getOptionValue(CONTAINER_ID_OPTION); 091 nodeAddress = commandLine.getOptionValue(NODE_ADDRESS_OPTION); 092 appOwner = commandLine.getOptionValue(APP_OWNER_OPTION); 093 } catch (ParseException e) { 094 System.out.println("options parsing failed: " + e.getMessage()); 095 096 HelpFormatter formatter = new HelpFormatter(); 097 formatter.printHelp("general options are: ", opts); 098 return -1; 099 } 100 101 if (appIdStr == null) { 102 System.out.println("ApplicationId cannot be null!"); 103 HelpFormatter formatter = new HelpFormatter(); 104 formatter.printHelp("general options are: ", opts); 105 return -1; 106 } 107 108 RecordFactory recordFactory = 109 RecordFactoryProvider.getRecordFactory(getConf()); 110 ApplicationId appId = 111 ConverterUtils.toApplicationId(recordFactory, appIdStr); 112 113 if (appOwner == null || appOwner.isEmpty()) { 114 appOwner = UserGroupInformation.getCurrentUser().getShortUserName(); 115 } 116 int resultCode = 0; 117 if (containerIdStr == null && nodeAddress == null) { 118 resultCode = dumpAllContainersLogs(appId, appOwner, System.out); 119 } else if ((containerIdStr == null && nodeAddress != null) 120 || (containerIdStr != null && nodeAddress == null)) { 121 System.out.println("ContainerId or NodeAddress cannot be null!"); 122 HelpFormatter formatter = new HelpFormatter(); 123 formatter.printHelp("general options are: ", opts); 124 resultCode = -1; 125 } else { 126 Path remoteRootLogDir = 127 new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, 128 YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); 129 AggregatedLogFormat.LogReader reader = 130 new AggregatedLogFormat.LogReader(getConf(), 131 LogAggregationUtils.getRemoteNodeLogFileForApp( 132 remoteRootLogDir, 133 appId, 134 appOwner, 135 ConverterUtils.toNodeId(nodeAddress), 136 LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()))); 137 resultCode = dumpAContainerLogs(containerIdStr, reader, System.out); 138 } 139 140 return resultCode; 141 } 142 143 @Private 144 @VisibleForTesting 145 public int dumpAContainersLogs(String appId, String containerId, 146 String nodeId, String jobOwner) throws IOException { 147 Path remoteRootLogDir = 148 new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, 149 YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); 150 String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); 151 Path logPath = LogAggregationUtils.getRemoteNodeLogFileForApp( 152 remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, 153 ConverterUtils.toNodeId(nodeId), suffix); 154 AggregatedLogFormat.LogReader reader; 155 try { 156 reader = new AggregatedLogFormat.LogReader(getConf(), logPath); 157 } catch (FileNotFoundException fnfe) { 158 System.out.println("Logs not available at " + logPath.toString()); 159 System.out.println( 160 "Log aggregation has not completed or is not enabled."); 161 return -1; 162 } 163 return dumpAContainerLogs(containerId, reader, System.out); 164 } 165 166 private int dumpAContainerLogs(String containerIdStr, 167 AggregatedLogFormat.LogReader reader, PrintStream out) 168 throws IOException { 169 DataInputStream valueStream; 170 LogKey key = new LogKey(); 171 valueStream = reader.next(key); 172 173 while (valueStream != null && !key.toString().equals(containerIdStr)) { 174 // Next container 175 key = new LogKey(); 176 valueStream = reader.next(key); 177 } 178 179 if (valueStream == null) { 180 System.out.println("Logs for container " + containerIdStr 181 + " are not present in this log-file."); 182 return -1; 183 } 184 185 while (true) { 186 try { 187 LogReader.readAContainerLogsForALogType(valueStream, out); 188 } catch (EOFException eof) { 189 break; 190 } 191 } 192 return 0; 193 } 194 195 private int dumpAllContainersLogs(ApplicationId appId, String appOwner, 196 PrintStream out) throws IOException { 197 Path remoteRootLogDir = 198 new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, 199 YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); 200 String user = appOwner; 201 String logDirSuffix = 202 LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); 203 //TODO Change this to get a list of files from the LAS. 204 Path remoteAppLogDir = 205 LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, user, 206 logDirSuffix); 207 RemoteIterator<FileStatus> nodeFiles; 208 try { 209 nodeFiles = FileContext.getFileContext().listStatus(remoteAppLogDir); 210 } catch (FileNotFoundException fnf) { 211 System.out.println("Logs not available at " 212 + remoteAppLogDir.toString()); 213 System.out.println( 214 "Log aggregation has not completed or is not enabled."); 215 return -1; 216 } 217 while (nodeFiles.hasNext()) { 218 FileStatus thisNodeFile = nodeFiles.next(); 219 AggregatedLogFormat.LogReader reader = 220 new AggregatedLogFormat.LogReader(getConf(), 221 new Path(remoteAppLogDir, thisNodeFile.getPath().getName())); 222 try { 223 224 DataInputStream valueStream; 225 LogKey key = new LogKey(); 226 valueStream = reader.next(key); 227 228 while (valueStream != null) { 229 String containerString = "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName(); 230 out.println(containerString); 231 out.println(StringUtils.repeat("=", containerString.length())); 232 while (true) { 233 try { 234 LogReader.readAContainerLogsForALogType(valueStream, out); 235 } catch (EOFException eof) { 236 break; 237 } 238 } 239 240 // Next container 241 key = new LogKey(); 242 valueStream = reader.next(key); 243 } 244 } finally { 245 reader.close(); 246 } 247 } 248 return 0; 249 } 250 251 public static void main(String[] args) throws Exception { 252 Configuration conf = new YarnConfiguration(); 253 LogDumper logDumper = new LogDumper(); 254 logDumper.setConf(conf); 255 int exitCode = logDumper.run(args); 256 System.exit(exitCode); 257 } 258 }