/**
* Retrieves, but does not remove, the first allocated container on the specified host.
*
* @param host the host for which a container is needed.
* @return the first {@link Container} allocated for the specified host or {@code null} if there
* isn't one.
*/
protected Container peekAllocatedContainer(String host) {
List<Container> allocatedContainers = containerRequestState.getContainersOnAHost(host);
if (allocatedContainers == null || allocatedContainers.isEmpty()) {
return null;
}
return allocatedContainers.get(0);
}
/**
* Method to request a container resource from yarn
*
* @param expectedContainerId Identifier of the container that will be run when a container
* resource is allocated for this request
* @param preferredHost Name of the host that you prefer to run the container on
*/
public final void requestContainer(int expectedContainerId, String preferredHost) {
SamzaContainerRequest request =
new SamzaContainerRequest(
containerMaxMemoryMb,
containerMaxCpuCore,
DEFAULT_PRIORITY,
expectedContainerId,
preferredHost);
containerRequestState.updateRequestState(request);
containerUtil.incrementContainerRequests();
}
/**
* Updates the request state and runs the container on the specified host. Assumes a container is
* available on the preferred host, so the caller must verify that before invoking this method.
*
* @param request the {@link SamzaContainerRequest} which is being handled.
* @param preferredHost the preferred host on which the container should be run or {@link
* ContainerRequestState#ANY_HOST} if there is no host preference.
*/
protected void runContainer(SamzaContainerRequest request, String preferredHost) {
// Get the available container
Container container = peekAllocatedContainer(preferredHost);
if (container == null)
throw new SamzaException("Expected container was unavailable on host " + preferredHost);
// Update state
containerRequestState.updateStateAfterAssignment(request, preferredHost, container);
int expectedContainerId = request.expectedContainerId;
// Cancel request and run container
log.info(
"Found available containers on {}. Assigning request for container_id {} with "
+ "timestamp {} to container {}",
new Object[] {
preferredHost,
String.valueOf(expectedContainerId),
request.getRequestTimestamp(),
container.getId()
});
try {
if (preferredHost.equals(ANY_HOST)) {
containerUtil.runContainer(expectedContainerId, container);
} else {
containerUtil.runMatchedContainer(expectedContainerId, container);
}
} catch (SamzaContainerLaunchException e) {
log.warn(
String.format(
"Got exception while starting container %s. Requesting a new container on any host",
container),
e);
containerRequestState.releaseUnstartableContainer(container);
requestContainer(expectedContainerId, ContainerAllocator.ANY_HOST);
}
}
/**
* Continuously assigns requested containers to the allocated containers provided by the cluster
* manager. The loop frequency is governed by thread sleeps for ALLOCATOR_SLEEP_TIME ms.
*
* <p>Terminates when the isRunning flag is cleared.
*/
@Override
public void run() {
while (isRunning.get()) {
try {
assignContainerRequests();
// Release extra containers and update the entire system's state
containerRequestState.releaseExtraContainers();
Thread.sleep(ALLOCATOR_SLEEP_TIME);
} catch (InterruptedException e) {
log.info("Got InterruptedException in AllocatorThread.", e);
} catch (Exception e) {
log.error("Got unknown Exception in AllocatorThread.", e);
}
}
}
/**
* Method that adds allocated container to a synchronized buffer of allocated containers list See
* allocatedContainers in {@link org.apache.samza.job.yarn.ContainerRequestState}
*
* @param container Container resource returned by the RM
*/
public final void addContainer(Container container) {
containerRequestState.addContainer(container);
}
/**
* Retrieves, but does not remove, the next pending request in the queue.
*
* @return the pending request or {@code null} if there is no pending request.
*/
protected SamzaContainerRequest peekPendingRequest() {
return containerRequestState.getRequestsQueue().peek();
}