public static ResourceRequest newResourceRequest(ResourceRequest r) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
request.setPriority(r.getPriority());
request.setResourceName(r.getResourceName());
request.setCapability(r.getCapability());
request.setNumContainers(r.getNumContainers());
return request;
}
public static ResourceRequest newResourceRequest(
Priority priority, String hostName, Resource capability, int numContainers) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
return request;
}
private void addResourceRequest(Priority priority, String resourceName, Resource capability) {
Map<String, Map<Resource, ResourceRequest>> remoteRequests =
this.remoteRequestsTable.get(priority);
if (remoteRequests == null) {
remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
this.remoteRequestsTable.put(priority, remoteRequests);
if (LOG.isDebugEnabled()) {
LOG.debug("Added priority=" + priority);
}
}
Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
if (reqMap == null) {
reqMap = new HashMap<Resource, ResourceRequest>();
remoteRequests.put(resourceName, reqMap);
}
ResourceRequest remoteRequest = reqMap.get(capability);
if (remoteRequest == null) {
remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
remoteRequest.setPriority(priority);
remoteRequest.setResourceName(resourceName);
remoteRequest.setCapability(capability);
remoteRequest.setNumContainers(0);
reqMap.put(capability, remoteRequest);
}
remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);
// Note this down for next interaction with ResourceManager
addResourceRequestToAsk(remoteRequest);
if (LOG.isDebugEnabled()) {
LOG.debug(
"addResourceRequest:"
+ " applicationId="
+ applicationId.getId()
+ " priority="
+ priority.getPriority()
+ " resourceName="
+ resourceName
+ " numContainers="
+ remoteRequest.getNumContainers()
+ " #asks="
+ ask.size());
}
}
/**
* Setup a container request on specified node
*
* @param node the specified node
* @return ResourceRequest sent to RM
*/
private ResourceRequest setupAContainerAskForRM(String node) {
ResourceRequest request = Records.newRecord(ResourceRequest.class);
request.setResourceName(node);
request.setNumContainers(1); // important
Priority priority = Records.newRecord(Priority.class);
priority.setPriority(requestPriority);
request.setPriority(priority);
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(containerMemory);
request.setCapability(capability);
return request;
}
/**
* Utility method creating a {@link ResourceRequest}.
*
* @param numContainers number of containers to request
* @return request to be sent to resource manager
*/
private ResourceRequest getContainerResourceRequest(
int numContainers, String hostName, boolean relaxLocality) {
ResourceRequest request = Records.newRecord(ResourceRequest.class);
request.setRelaxLocality(relaxLocality);
request.setResourceName(hostName);
request.setNumContainers(numContainers);
Priority pri = Records.newRecord(Priority.class);
pri.setPriority(priority);
request.setPriority(pri);
Resource capability = Records.newRecord(Resource.class);
capability.setMemory(memory);
ResourceCompat.setVirtualCores(capability, virtualcores);
request.setCapability(capability);
return request;
}
private void decResourceRequest(Priority priority, String resourceName, Resource capability) {
Map<String, Map<Resource, ResourceRequest>> remoteRequests =
this.remoteRequestsTable.get(priority);
Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
if (reqMap == null) {
// as we modify the resource requests by filtering out blacklisted hosts
// when they are added, this value may be null when being
// decremented
if (LOG.isDebugEnabled()) {
LOG.debug(
"Not decrementing resource as " + resourceName + " is not present in request table");
}
return;
}
ResourceRequest remoteRequest = reqMap.get(capability);
if (LOG.isDebugEnabled()) {
LOG.debug(
"BEFORE decResourceRequest:"
+ " applicationId="
+ applicationId.getId()
+ " priority="
+ priority.getPriority()
+ " resourceName="
+ resourceName
+ " numContainers="
+ remoteRequest.getNumContainers()
+ " #asks="
+ ask.size());
}
if (remoteRequest.getNumContainers() > 0) {
// based on blacklisting comments above we can end up decrementing more
// than requested. so guard for that.
remoteRequest.setNumContainers(remoteRequest.getNumContainers() - 1);
}
if (remoteRequest.getNumContainers() == 0) {
reqMap.remove(capability);
if (reqMap.size() == 0) {
remoteRequests.remove(resourceName);
}
if (remoteRequests.size() == 0) {
remoteRequestsTable.remove(priority);
}
}
// send the updated resource request to RM
// send 0 container count requests also to cancel previous requests
addResourceRequestToAsk(remoteRequest);
if (LOG.isDebugEnabled()) {
LOG.info(
"AFTER decResourceRequest:"
+ " applicationId="
+ applicationId.getId()
+ " priority="
+ priority.getPriority()
+ " resourceName="
+ resourceName
+ " numContainers="
+ remoteRequest.getNumContainers()
+ " #asks="
+ ask.size());
}
}
protected void containerFailedOnHost(String hostName) {
if (!nodeBlacklistingEnabled) {
return;
}
if (blacklistedNodes.contains(hostName)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Host " + hostName + " is already blacklisted.");
}
return; // already blacklisted
}
Integer failures = nodeFailures.remove(hostName);
failures = failures == null ? Integer.valueOf(0) : failures;
failures++;
LOG.info(failures + " failures on node " + hostName);
if (failures >= maxTaskFailuresPerNode) {
blacklistedNodes.add(hostName);
// Even if blacklisting is ignored, continue to remove the host from
// the request table. The RM may have additional nodes it can allocate on.
LOG.info("Blacklisted host " + hostName);
// remove all the requests corresponding to this hostname
for (Map<String, Map<Resource, ResourceRequest>> remoteRequests :
remoteRequestsTable.values()) {
// remove from host if no pending allocations
boolean foundAll = true;
Map<Resource, ResourceRequest> reqMap = remoteRequests.get(hostName);
if (reqMap != null) {
for (ResourceRequest req : reqMap.values()) {
if (!ask.remove(req)) {
foundAll = false;
// if ask already sent to RM, we can try and overwrite it if possible.
// send a new ask to RM with numContainers
// specified for the blacklisted host to be 0.
ResourceRequest zeroedRequest =
ResourceRequest.newInstance(
req.getPriority(),
req.getResourceName(),
req.getCapability(),
req.getNumContainers(),
req.getRelaxLocality());
zeroedRequest.setNumContainers(0);
// to be sent to RM on next heartbeat
addResourceRequestToAsk(zeroedRequest);
}
}
// if all requests were still in ask queue
// we can remove this request
if (foundAll) {
remoteRequests.remove(hostName);
}
}
// TODO handling of rack blacklisting
// Removing from rack should be dependent on no. of failures within the rack
// Blacklisting a rack on the basis of a single node's blacklisting
// may be overly aggressive.
// Node failures could be co-related with other failures on the same rack
// but we probably need a better approach at trying to decide how and when
// to blacklist a rack
}
} else {
nodeFailures.put(hostName, failures);
}
}
