@Test
public void testDefaultSingleQueueMetrics() {
String queueName = "single";
String user = "******";
QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, conf);
MetricsSource queueSource = queueSource(ms, queueName);
AppSchedulingInfo app = mockApp(user);
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, queueName, user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0);
metrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB));
metrics.incrPendingResources(user, 5, Resources.createResource(15 * GB));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
checkResources(queueSource, 0, 0, 0, 0, 100 * GB, 15 * GB, 5, 0, 0);
metrics.incrAppsRunning(app, user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
metrics.allocateResources(user, 3, Resources.createResource(2 * GB));
checkResources(queueSource, 6 * GB, 3, 3, 0, 100 * GB, 9 * GB, 2, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2 * GB));
checkResources(queueSource, 4 * GB, 2, 3, 1, 100 * GB, 9 * GB, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
assertNull(userSource);
}
@Test
public void testTwoLevelWithUserMetrics() {
String parentQueueName = "root";
String leafQueueName = "root.leaf";
String user = "******";
QueueMetrics parentMetrics = QueueMetrics.forQueue(ms, parentQueueName, null, true, conf);
Queue parentQueue = make(stub(Queue.class).returning(parentMetrics).from.getMetrics());
QueueMetrics metrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue, true, conf);
MetricsSource parentQueueSource = queueSource(ms, parentQueueName);
MetricsSource queueSource = queueSource(ms, leafQueueName);
AppSchedulingInfo app = mockApp(user);
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, leafQueueName, user);
MetricsSource parentUserSource = userSource(ms, parentQueueName, user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0);
checkApps(parentQueueSource, 1, 1, 0, 0, 0, 0);
checkApps(userSource, 1, 1, 0, 0, 0, 0);
checkApps(parentUserSource, 1, 1, 0, 0, 0, 0);
parentMetrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB));
metrics.setAvailableResourcesToQueue(Resources.createResource(100 * GB));
parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10 * GB));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10 * GB));
metrics.incrPendingResources(user, 5, Resources.createResource(15 * GB));
checkResources(queueSource, 0, 0, 0, 0, 100 * GB, 15 * GB, 5, 0, 0);
checkResources(parentQueueSource, 0, 0, 0, 0, 100 * GB, 15 * GB, 5, 0, 0);
checkResources(userSource, 0, 0, 0, 0, 10 * GB, 15 * GB, 5, 0, 0);
checkResources(parentUserSource, 0, 0, 0, 0, 10 * GB, 15 * GB, 5, 0, 0);
metrics.incrAppsRunning(app, user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
checkApps(userSource, 1, 0, 1, 0, 0, 0);
metrics.allocateResources(user, 3, Resources.createResource(2 * GB));
metrics.reserveResource(user, Resources.createResource(3 * GB));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
checkResources(queueSource, 6 * GB, 3, 3, 0, 100 * GB, 9 * GB, 2, 3 * GB, 1);
checkResources(parentQueueSource, 6 * GB, 3, 3, 0, 100 * GB, 9 * GB, 2, 3 * GB, 1);
checkResources(userSource, 6 * GB, 3, 3, 0, 10 * GB, 9 * GB, 2, 3 * GB, 1);
checkResources(parentUserSource, 6 * GB, 3, 3, 0, 10 * GB, 9 * GB, 2, 3 * GB, 1);
metrics.releaseResources(user, 1, Resources.createResource(2 * GB));
metrics.unreserveResource(user, Resources.createResource(3 * GB));
checkResources(queueSource, 4 * GB, 2, 3, 1, 100 * GB, 9 * GB, 2, 0, 0);
checkResources(parentQueueSource, 4 * GB, 2, 3, 1, 100 * GB, 9 * GB, 2, 0, 0);
checkResources(userSource, 4 * GB, 2, 3, 1, 10 * GB, 9 * GB, 2, 0, 0);
checkResources(parentUserSource, 4 * GB, 2, 3, 1, 10 * GB, 9 * GB, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
checkApps(parentQueueSource, 1, 0, 0, 1, 0, 0);
checkApps(userSource, 1, 0, 0, 1, 0, 0);
checkApps(parentUserSource, 1, 0, 0, 1, 0, 0);
}
@Test
public void testQueueAppMetricsForMultipleFailures() {
String queueName = "single";
String user = "******";
QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, new Configuration());
MetricsSource queueSource = queueSource(ms, queueName);
AppSchedulingInfo app = mockApp(user);
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, queueName, user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0);
metrics.incrAppsRunning(app, user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FAILED);
checkApps(queueSource, 1, 0, 0, 0, 1, 0);
// As the application has failed, framework retries the same application
// based on configuration
metrics.submitApp(user, 2);
checkApps(queueSource, 1, 1, 0, 0, 0, 0);
metrics.incrAppsRunning(app, user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
// Suppose say application has failed this time as well.
metrics.finishApp(app, RMAppAttemptState.FAILED);
checkApps(queueSource, 1, 0, 0, 0, 1, 0);
// As the application has failed, framework retries the same application
// based on configuration
metrics.submitApp(user, 3);
checkApps(queueSource, 1, 1, 0, 0, 0, 0);
metrics.incrAppsRunning(app, user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
// Suppose say application has finished.
metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
assertNull(userSource);
}
@Test
public void testMove() {
final String user = "******";
Queue parentQueue = createQueue("parent", null);
Queue oldQueue = createQueue("old", parentQueue);
Queue newQueue = createQueue("new", parentQueue);
QueueMetrics parentMetrics = parentQueue.getMetrics();
QueueMetrics oldMetrics = oldQueue.getMetrics();
QueueMetrics newMetrics = newQueue.getMetrics();
ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
SchedulerApplicationAttempt app =
new SchedulerApplicationAttempt(
appAttId, user, oldQueue, oldQueue.getActiveUsersManager(), null);
oldMetrics.submitApp(user);
// Resource request
Resource requestedResource = Resource.newInstance(1536, 2);
Priority requestedPriority = Priority.newInstance(2);
ResourceRequest request =
ResourceRequest.newInstance(requestedPriority, ResourceRequest.ANY, requestedResource, 3);
app.updateResourceRequests(Arrays.asList(request));
// Allocated container
RMContainer container1 = createRMContainer(appAttId, 1, requestedResource);
app.liveContainers.put(container1.getContainerId(), container1);
SchedulerNode node = createNode();
app.appSchedulingInfo.allocate(
NodeType.OFF_SWITCH, node, requestedPriority, request, container1.getContainer());
// Reserved container
Priority prio1 = Priority.newInstance(1);
Resource reservedResource = Resource.newInstance(2048, 3);
RMContainer container2 =
createReservedRMContainer(appAttId, 1, reservedResource, node.getNodeID(), prio1);
Map<NodeId, RMContainer> reservations = new HashMap<NodeId, RMContainer>();
reservations.put(node.getNodeID(), container2);
app.reservedContainers.put(prio1, reservations);
oldMetrics.reserveResource(user, reservedResource);
checkQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
checkQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
app.move(newQueue);
checkQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
checkQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
}