001    /**
002    * Licensed to the Apache Software Foundation (ASF) under one
003    * or more contributor license agreements.  See the NOTICE file
004    * distributed with this work for additional information
005    * regarding copyright ownership.  The ASF licenses this file
006    * to you under the Apache License, Version 2.0 (the
007    * "License"); you may not use this file except in compliance
008    * with the License.  You may obtain a copy of the License at
009    *
010    *     http://www.apache.org/licenses/LICENSE-2.0
011    *
012    * Unless required by applicable law or agreed to in writing, software
013    * distributed under the License is distributed on an "AS IS" BASIS,
014    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015    * See the License for the specific language governing permissions and
016    * limitations under the License.
017    */
018    
019    package org.apache.hadoop.yarn.client.api;
020    
021    import java.io.IOException;
022    import java.util.Collection;
023    import java.util.List;
024    
025    import org.apache.hadoop.classification.InterfaceAudience;
026    import org.apache.hadoop.classification.InterfaceAudience.Private;
027    import org.apache.hadoop.classification.InterfaceAudience.Public;
028    import org.apache.hadoop.classification.InterfaceStability;
029    import org.apache.hadoop.service.AbstractService;
030    import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
031    import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
032    import org.apache.hadoop.yarn.api.records.ContainerId;
033    import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
034    import org.apache.hadoop.yarn.api.records.Priority;
035    import org.apache.hadoop.yarn.api.records.Resource;
036    import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
037    import org.apache.hadoop.yarn.exceptions.YarnException;
038    
039    import com.google.common.base.Preconditions;
040    import com.google.common.collect.ImmutableList;
041    
042    @InterfaceAudience.Public
043    @InterfaceStability.Stable
044    public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
045        AbstractService {
046    
047      /**
048       * Create a new instance of AMRMClient.
049       * For usage:
050       * <pre>
051       * {@code
052       * AMRMClient.<T>createAMRMClientContainerRequest()
053       * }</pre>
054       * @return the newly create AMRMClient instance.
055       */
056      @Public
057      public static <T extends ContainerRequest> AMRMClient<T> createAMRMClient() {
058        AMRMClient<T> client = new AMRMClientImpl<T>();
059        return client;
060      }
061    
062      @Private
063      protected AMRMClient(String name) {
064        super(name);
065      }
066    
067      /**
068       * Object to represent a single container request for resources. Scheduler
069       * documentation should be consulted for the specifics of how the parameters
070       * are honored.
071       * 
072       * By default, YARN schedulers try to allocate containers at the requested
073       * locations but they may relax the constraints in order to expedite meeting
074       * allocations limits. They first relax the constraint to the same rack as the
075       * requested node and then to anywhere in the cluster. The relaxLocality flag
076       * may be used to disable locality relaxation and request containers at only 
077       * specific locations. The following conditions apply.
078       * <ul>
079       * <li>Within a priority, all container requests must have the same value for
080       * locality relaxation. Either enabled or disabled.</li>
081       * <li>If locality relaxation is disabled, then across requests, locations at
082       * different network levels may not be specified. E.g. its invalid to make a
083       * request for a specific node and another request for a specific rack.</li>
084       * <li>If locality relaxation is disabled, then only within the same request,  
085       * a node and its rack may be specified together. This allows for a specific   
086       * rack with a preference for a specific node within that rack.</li>
087       * <li></li>
088       * </ul>
089       * To re-enable locality relaxation at a given priority, all pending requests 
090       * with locality relaxation disabled must be first removed. Then they can be 
091       * added back with locality relaxation enabled.
092       * 
093       * All getters return immutable values.
094       */
095      public static class ContainerRequest {
096        final Resource capability;
097        final List<String> nodes;
098        final List<String> racks;
099        final Priority priority;
100        final boolean relaxLocality;
101        
102        /**
103         * Instantiates a {@link ContainerRequest} with the given constraints and
104         * locality relaxation enabled.
105         * 
106         * @param capability
107         *          The {@link Resource} to be requested for each container.
108         * @param nodes
109         *          Any hosts to request that the containers are placed on.
110         * @param racks
111         *          Any racks to request that the containers are placed on. The
112         *          racks corresponding to any hosts requested will be automatically
113         *          added to this list.
114         * @param priority
115         *          The priority at which to request the containers. Higher
116         *          priorities have lower numerical values.
117         */
118        public ContainerRequest(Resource capability, String[] nodes,
119            String[] racks, Priority priority) {
120          this(capability, nodes, racks, priority, true);
121        }
122              
123        /**
124         * Instantiates a {@link ContainerRequest} with the given constraints.
125         * 
126         * @param capability
127         *          The {@link Resource} to be requested for each container.
128         * @param nodes
129         *          Any hosts to request that the containers are placed on.
130         * @param racks
131         *          Any racks to request that the containers are placed on. The
132         *          racks corresponding to any hosts requested will be automatically
133         *          added to this list.
134         * @param priority
135         *          The priority at which to request the containers. Higher
136         *          priorities have lower numerical values.
137         * @param relaxLocality
138         *          If true, containers for this request may be assigned on hosts
139         *          and racks other than the ones explicitly requested.
140         */
141        public ContainerRequest(Resource capability, String[] nodes,
142            String[] racks, Priority priority, boolean relaxLocality) {
143          // Validate request
144          Preconditions.checkArgument(capability != null,
145              "The Resource to be requested for each container " +
146                  "should not be null ");
147          Preconditions.checkArgument(priority != null,
148              "The priority at which to request containers should not be null ");
149          Preconditions.checkArgument(
150                  !(!relaxLocality && (racks == null || racks.length == 0) 
151                      && (nodes == null || nodes.length == 0)),
152                  "Can't turn off locality relaxation on a " + 
153                  "request with no location constraints");
154          this.capability = capability;
155          this.nodes = (nodes != null ? ImmutableList.copyOf(nodes) : null);
156          this.racks = (racks != null ? ImmutableList.copyOf(racks) : null);
157          this.priority = priority;
158          this.relaxLocality = relaxLocality;
159        }
160        
161        public Resource getCapability() {
162          return capability;
163        }
164        
165        public List<String> getNodes() {
166          return nodes;
167        }
168        
169        public List<String> getRacks() {
170          return racks;
171        }
172        
173        public Priority getPriority() {
174          return priority;
175        }
176        
177        public boolean getRelaxLocality() {
178          return relaxLocality;
179        }
180        
181        public String toString() {
182          StringBuilder sb = new StringBuilder();
183          sb.append("Capability[").append(capability).append("]");
184          sb.append("Priority[").append(priority).append("]");
185          return sb.toString();
186        }
187      }
188     
189      /**
190       * Register the application master. This must be called before any 
191       * other interaction
192       * @param appHostName Name of the host on which master is running
193       * @param appHostPort Port master is listening on
194       * @param appTrackingUrl URL at which the master info can be seen
195       * @return <code>RegisterApplicationMasterResponse</code>
196       * @throws YarnException
197       * @throws IOException
198       */
199      public abstract RegisterApplicationMasterResponse 
200                   registerApplicationMaster(String appHostName,
201                                             int appHostPort,
202                                             String appTrackingUrl) 
203                   throws YarnException, IOException;
204      
205      /**
206       * Request additional containers and receive new container allocations.
207       * Requests made via <code>addContainerRequest</code> are sent to the 
208       * <code>ResourceManager</code>. New containers assigned to the master are 
209       * retrieved. Status of completed containers and node health updates are 
210       * also retrieved.
211       * This also doubles up as a heartbeat to the ResourceManager and must be 
212       * made periodically.
213       * The call may not always return any new allocations of containers.
214       * App should not make concurrent allocate requests. May cause request loss.
215       * @param progressIndicator Indicates progress made by the master
216       * @return the response of the allocate request
217       * @throws YarnException
218       * @throws IOException
219       */
220      public abstract AllocateResponse allocate(float progressIndicator) 
221                               throws YarnException, IOException;
222      
223      /**
224       * Unregister the application master. This must be called in the end.
225       * @param appStatus Success/Failure status of the master
226       * @param appMessage Diagnostics message on failure
227       * @param appTrackingUrl New URL to get master info
228       * @throws YarnException
229       * @throws IOException
230       */
231      public abstract void unregisterApplicationMaster(FinalApplicationStatus appStatus,
232                                               String appMessage,
233                                               String appTrackingUrl) 
234                   throws YarnException, IOException;
235      
236      /**
237       * Request containers for resources before calling <code>allocate</code>
238       * @param req Resource request
239       */
240      public abstract void addContainerRequest(T req);
241      
242      /**
243       * Remove previous container request. The previous container request may have 
244       * already been sent to the ResourceManager. So even after the remove request 
245       * the app must be prepared to receive an allocation for the previous request 
246       * even after the remove request
247       * @param req Resource request
248       */
249      public abstract void removeContainerRequest(T req);
250      
251      /**
252       * Release containers assigned by the Resource Manager. If the app cannot use
253       * the container or wants to give up the container then it can release them.
254       * The app needs to make new requests for the released resource capability if
255       * it still needs it. eg. it released non-local resources
256       * @param containerId
257       */
258      public abstract void releaseAssignedContainer(ContainerId containerId);
259      
260      /**
261       * Get the currently available resources in the cluster.
262       * A valid value is available after a call to allocate has been made
263       * @return Currently available resources
264       */
265      public abstract Resource getAvailableResources();
266      
267      /**
268       * Get the current number of nodes in the cluster.
269       * A valid values is available after a call to allocate has been made
270       * @return Current number of nodes in the cluster
271       */
272      public abstract int getClusterNodeCount();
273    
274      /**
275       * Get outstanding <code>ContainerRequest</code>s matching the given 
276       * parameters. These ContainerRequests should have been added via
277       * <code>addContainerRequest</code> earlier in the lifecycle. For performance,
278       * the AMRMClient may return its internal collection directly without creating 
279       * a copy. Users should not perform mutable operations on the return value.
280       * Each collection in the list contains requests with identical 
281       * <code>Resource</code> size that fit in the given capability. In a 
282       * collection, requests will be returned in the same order as they were added.
283       * @return Collection of request matching the parameters
284       */
285      public abstract List<? extends Collection<T>> getMatchingRequests(
286                                               Priority priority, 
287                                               String resourceName, 
288                                               Resource capability);
289    }