001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.yarn.client.api; 020 021 import java.io.IOException; 022 import java.util.Collection; 023 import java.util.List; 024 025 import org.apache.hadoop.classification.InterfaceAudience; 026 import org.apache.hadoop.classification.InterfaceAudience.Private; 027 import org.apache.hadoop.classification.InterfaceAudience.Public; 028 import org.apache.hadoop.classification.InterfaceStability; 029 import org.apache.hadoop.service.AbstractService; 030 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; 031 import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; 032 import org.apache.hadoop.yarn.api.records.ContainerId; 033 import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; 034 import org.apache.hadoop.yarn.api.records.Priority; 035 import org.apache.hadoop.yarn.api.records.Resource; 036 import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; 037 import org.apache.hadoop.yarn.exceptions.YarnException; 038 039 import com.google.common.base.Preconditions; 040 import com.google.common.collect.ImmutableList; 041 042 @InterfaceAudience.Public 043 @InterfaceStability.Stable 044 public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends 045 AbstractService { 046 047 /** 048 * Create a new instance of AMRMClient. 049 * For usage: 050 * <pre> 051 * {@code 052 * AMRMClient.<T>createAMRMClientContainerRequest() 053 * }</pre> 054 * @return the newly create AMRMClient instance. 055 */ 056 @Public 057 public static <T extends ContainerRequest> AMRMClient<T> createAMRMClient() { 058 AMRMClient<T> client = new AMRMClientImpl<T>(); 059 return client; 060 } 061 062 @Private 063 protected AMRMClient(String name) { 064 super(name); 065 } 066 067 /** 068 * Object to represent a single container request for resources. Scheduler 069 * documentation should be consulted for the specifics of how the parameters 070 * are honored. 071 * 072 * By default, YARN schedulers try to allocate containers at the requested 073 * locations but they may relax the constraints in order to expedite meeting 074 * allocations limits. They first relax the constraint to the same rack as the 075 * requested node and then to anywhere in the cluster. The relaxLocality flag 076 * may be used to disable locality relaxation and request containers at only 077 * specific locations. The following conditions apply. 078 * <ul> 079 * <li>Within a priority, all container requests must have the same value for 080 * locality relaxation. Either enabled or disabled.</li> 081 * <li>If locality relaxation is disabled, then across requests, locations at 082 * different network levels may not be specified. E.g. its invalid to make a 083 * request for a specific node and another request for a specific rack.</li> 084 * <li>If locality relaxation is disabled, then only within the same request, 085 * a node and its rack may be specified together. This allows for a specific 086 * rack with a preference for a specific node within that rack.</li> 087 * <li></li> 088 * </ul> 089 * To re-enable locality relaxation at a given priority, all pending requests 090 * with locality relaxation disabled must be first removed. Then they can be 091 * added back with locality relaxation enabled. 092 * 093 * All getters return immutable values. 094 */ 095 public static class ContainerRequest { 096 final Resource capability; 097 final List<String> nodes; 098 final List<String> racks; 099 final Priority priority; 100 final boolean relaxLocality; 101 102 /** 103 * Instantiates a {@link ContainerRequest} with the given constraints and 104 * locality relaxation enabled. 105 * 106 * @param capability 107 * The {@link Resource} to be requested for each container. 108 * @param nodes 109 * Any hosts to request that the containers are placed on. 110 * @param racks 111 * Any racks to request that the containers are placed on. The 112 * racks corresponding to any hosts requested will be automatically 113 * added to this list. 114 * @param priority 115 * The priority at which to request the containers. Higher 116 * priorities have lower numerical values. 117 */ 118 public ContainerRequest(Resource capability, String[] nodes, 119 String[] racks, Priority priority) { 120 this(capability, nodes, racks, priority, true); 121 } 122 123 /** 124 * Instantiates a {@link ContainerRequest} with the given constraints. 125 * 126 * @param capability 127 * The {@link Resource} to be requested for each container. 128 * @param nodes 129 * Any hosts to request that the containers are placed on. 130 * @param racks 131 * Any racks to request that the containers are placed on. The 132 * racks corresponding to any hosts requested will be automatically 133 * added to this list. 134 * @param priority 135 * The priority at which to request the containers. Higher 136 * priorities have lower numerical values. 137 * @param relaxLocality 138 * If true, containers for this request may be assigned on hosts 139 * and racks other than the ones explicitly requested. 140 */ 141 public ContainerRequest(Resource capability, String[] nodes, 142 String[] racks, Priority priority, boolean relaxLocality) { 143 // Validate request 144 Preconditions.checkArgument(capability != null, 145 "The Resource to be requested for each container " + 146 "should not be null "); 147 Preconditions.checkArgument(priority != null, 148 "The priority at which to request containers should not be null "); 149 Preconditions.checkArgument( 150 !(!relaxLocality && (racks == null || racks.length == 0) 151 && (nodes == null || nodes.length == 0)), 152 "Can't turn off locality relaxation on a " + 153 "request with no location constraints"); 154 this.capability = capability; 155 this.nodes = (nodes != null ? ImmutableList.copyOf(nodes) : null); 156 this.racks = (racks != null ? ImmutableList.copyOf(racks) : null); 157 this.priority = priority; 158 this.relaxLocality = relaxLocality; 159 } 160 161 public Resource getCapability() { 162 return capability; 163 } 164 165 public List<String> getNodes() { 166 return nodes; 167 } 168 169 public List<String> getRacks() { 170 return racks; 171 } 172 173 public Priority getPriority() { 174 return priority; 175 } 176 177 public boolean getRelaxLocality() { 178 return relaxLocality; 179 } 180 181 public String toString() { 182 StringBuilder sb = new StringBuilder(); 183 sb.append("Capability[").append(capability).append("]"); 184 sb.append("Priority[").append(priority).append("]"); 185 return sb.toString(); 186 } 187 } 188 189 /** 190 * Register the application master. This must be called before any 191 * other interaction 192 * @param appHostName Name of the host on which master is running 193 * @param appHostPort Port master is listening on 194 * @param appTrackingUrl URL at which the master info can be seen 195 * @return <code>RegisterApplicationMasterResponse</code> 196 * @throws YarnException 197 * @throws IOException 198 */ 199 public abstract RegisterApplicationMasterResponse 200 registerApplicationMaster(String appHostName, 201 int appHostPort, 202 String appTrackingUrl) 203 throws YarnException, IOException; 204 205 /** 206 * Request additional containers and receive new container allocations. 207 * Requests made via <code>addContainerRequest</code> are sent to the 208 * <code>ResourceManager</code>. New containers assigned to the master are 209 * retrieved. Status of completed containers and node health updates are 210 * also retrieved. 211 * This also doubles up as a heartbeat to the ResourceManager and must be 212 * made periodically. 213 * The call may not always return any new allocations of containers. 214 * App should not make concurrent allocate requests. May cause request loss. 215 * @param progressIndicator Indicates progress made by the master 216 * @return the response of the allocate request 217 * @throws YarnException 218 * @throws IOException 219 */ 220 public abstract AllocateResponse allocate(float progressIndicator) 221 throws YarnException, IOException; 222 223 /** 224 * Unregister the application master. This must be called in the end. 225 * @param appStatus Success/Failure status of the master 226 * @param appMessage Diagnostics message on failure 227 * @param appTrackingUrl New URL to get master info 228 * @throws YarnException 229 * @throws IOException 230 */ 231 public abstract void unregisterApplicationMaster(FinalApplicationStatus appStatus, 232 String appMessage, 233 String appTrackingUrl) 234 throws YarnException, IOException; 235 236 /** 237 * Request containers for resources before calling <code>allocate</code> 238 * @param req Resource request 239 */ 240 public abstract void addContainerRequest(T req); 241 242 /** 243 * Remove previous container request. The previous container request may have 244 * already been sent to the ResourceManager. So even after the remove request 245 * the app must be prepared to receive an allocation for the previous request 246 * even after the remove request 247 * @param req Resource request 248 */ 249 public abstract void removeContainerRequest(T req); 250 251 /** 252 * Release containers assigned by the Resource Manager. If the app cannot use 253 * the container or wants to give up the container then it can release them. 254 * The app needs to make new requests for the released resource capability if 255 * it still needs it. eg. it released non-local resources 256 * @param containerId 257 */ 258 public abstract void releaseAssignedContainer(ContainerId containerId); 259 260 /** 261 * Get the currently available resources in the cluster. 262 * A valid value is available after a call to allocate has been made 263 * @return Currently available resources 264 */ 265 public abstract Resource getAvailableResources(); 266 267 /** 268 * Get the current number of nodes in the cluster. 269 * A valid values is available after a call to allocate has been made 270 * @return Current number of nodes in the cluster 271 */ 272 public abstract int getClusterNodeCount(); 273 274 /** 275 * Get outstanding <code>ContainerRequest</code>s matching the given 276 * parameters. These ContainerRequests should have been added via 277 * <code>addContainerRequest</code> earlier in the lifecycle. For performance, 278 * the AMRMClient may return its internal collection directly without creating 279 * a copy. Users should not perform mutable operations on the return value. 280 * Each collection in the list contains requests with identical 281 * <code>Resource</code> size that fit in the given capability. In a 282 * collection, requests will be returned in the same order as they were added. 283 * @return Collection of request matching the parameters 284 */ 285 public abstract List<? extends Collection<T>> getMatchingRequests( 286 Priority priority, 287 String resourceName, 288 Resource capability); 289 }