/**
* Test serialization of the container manager with mock execution layer.
*
* @throws Exception
*/
private void testContainerManager(StorageAgent agent) throws Exception {
FileUtils.deleteDirectory(new File(testMeta.dir)); // clean any state from previous run
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir);
dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
StatsListeningOperator o1 = dag.addOperator("o1", StatsListeningOperator.class);
FSRecoveryHandler recoveryHandler =
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false));
StreamingContainerManager scm =
StreamingContainerManager.getInstance(recoveryHandler, dag, false);
File expFile = new File(recoveryHandler.getDir(), FSRecoveryHandler.FILE_SNAPSHOT);
Assert.assertTrue("snapshot file " + expFile, expFile.exists());
PhysicalPlan plan = scm.getPhysicalPlan();
assertEquals("number required containers", 1, plan.getContainers().size());
PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
@SuppressWarnings(
"UnusedAssignment") /* sneaky: the constructor does some changes to the container */
MockContainer mc = new MockContainer(scm, o1p1.getContainer());
PTContainer originalContainer = o1p1.getContainer();
Assert.assertNotNull(o1p1.getContainer().bufferServerAddress);
assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
assertEquals("state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
// test restore initial snapshot + log
dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir);
scm =
StreamingContainerManager.getInstance(
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false)), dag, false);
dag = scm.getLogicalPlan();
plan = scm.getPhysicalPlan();
o1p1 = plan.getOperators(dag.getOperatorMeta("o1")).get(0);
assertEquals("post restore state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
o1 = (StatsListeningOperator) o1p1.getOperatorMeta().getOperator();
assertEquals(
"containerId", originalContainer.getExternalId(), o1p1.getContainer().getExternalId());
assertEquals("stats listener", 1, o1p1.statsListeners.size());
assertEquals("number stats calls", 0, o1.processStatsCnt); // stats are not logged
assertEquals("post restore 1", PTContainer.State.ALLOCATED, o1p1.getContainer().getState());
assertEquals(
"post restore 1",
originalContainer.bufferServerAddress,
o1p1.getContainer().bufferServerAddress);
StreamingContainerAgent sca = scm.getContainerAgent(originalContainer.getExternalId());
Assert.assertNotNull("allocated container restored " + originalContainer, sca);
assertEquals(
"memory usage allocated container",
(int) OperatorContext.MEMORY_MB.defaultValue,
sca.container.getAllocatedMemoryMB());
// YARN-1490 - simulate container terminated on AM recovery
scm.scheduleContainerRestart(originalContainer.getExternalId());
assertEquals("memory usage of failed container", 0, sca.container.getAllocatedMemoryMB());
Checkpoint firstCheckpoint = new Checkpoint(3, 0, 0);
mc = new MockContainer(scm, o1p1.getContainer());
checkpoint(scm, o1p1, firstCheckpoint);
mc.stats(o1p1.getId())
.deployState(OperatorHeartbeat.DeployState.ACTIVE)
.currentWindowId(3)
.checkpointWindowId(3);
mc.sendHeartbeat();
assertEquals("state " + o1p1, PTOperator.State.ACTIVE, o1p1.getState());
// logical plan modification triggers snapshot
CreateOperatorRequest cor = new CreateOperatorRequest();
cor.setOperatorFQCN(GenericTestOperator.class.getName());
cor.setOperatorName("o2");
CreateStreamRequest csr = new CreateStreamRequest();
csr.setSourceOperatorName("o1");
csr.setSourceOperatorPortName("outport");
csr.setSinkOperatorName("o2");
csr.setSinkOperatorPortName("inport1");
FutureTask<?> lpmf = scm.logicalPlanModification(Lists.newArrayList(cor, csr));
while (!lpmf.isDone()) {
scm.monitorHeartbeat();
}
Assert.assertNull(lpmf.get()); // unmask exception, if any
Assert.assertSame("dag references", dag, scm.getLogicalPlan());
assertEquals("number operators after plan modification", 2, dag.getAllOperators().size());
// set operator state triggers journal write
o1p1.setState(PTOperator.State.INACTIVE);
Checkpoint offlineCheckpoint = new Checkpoint(10, 0, 0);
// write checkpoint while AM is out,
// it needs to be picked up as part of restore
checkpoint(scm, o1p1, offlineCheckpoint);
// test restore
dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir);
scm =
StreamingContainerManager.getInstance(
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false)), dag, false);
Assert.assertNotSame("dag references", dag, scm.getLogicalPlan());
assertEquals(
"number operators after restore", 2, scm.getLogicalPlan().getAllOperators().size());
dag = scm.getLogicalPlan();
plan = scm.getPhysicalPlan();
o1p1 = plan.getOperators(dag.getOperatorMeta("o1")).get(0);
assertEquals("post restore state " + o1p1, PTOperator.State.INACTIVE, o1p1.getState());
o1 = (StatsListeningOperator) o1p1.getOperatorMeta().getOperator();
assertEquals("stats listener", 1, o1p1.statsListeners.size());
assertEquals("number stats calls post restore", 1, o1.processStatsCnt);
assertEquals("post restore 1", PTContainer.State.ACTIVE, o1p1.getContainer().getState());
assertEquals(
"post restore 1",
originalContainer.bufferServerAddress,
o1p1.getContainer().bufferServerAddress);
// offline checkpoint detection
assertEquals(
"checkpoints after recovery",
Lists.newArrayList(firstCheckpoint, offlineCheckpoint),
o1p1.checkpoints);
}
@Test
public void testGenerateDeployInfo() {
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
dag.setOutputPortAttribute(o1.outport, PortContext.BUFFER_MEMORY_MB, 256);
dag.addStream("o1.outport", o1.outport, o2.inport1);
dag.setOutputPortAttribute(o1.outport, PortContext.SPIN_MILLIS, 99);
dag.addStream("o2.outport1", o2.outport1, o3.inport1).setLocality(Locality.CONTAINER_LOCAL);
dag.addStream("o3.outport1", o3.outport1, o4.inport1).setLocality(Locality.THREAD_LOCAL);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
Assert.assertEquals("number operators", 4, dag.getAllOperators().size());
Assert.assertEquals("number root operators", 1, dag.getRootOperators().size());
StreamingContainerManager dnm = new StreamingContainerManager(dag);
Assert.assertEquals("number containers", 2, dnm.getPhysicalPlan().getContainers().size());
dnm.assignContainer(
new ContainerResource(0, "container1Id", "host1", 1024, 0, null),
InetSocketAddress.createUnresolved("host1", 9001));
dnm.assignContainer(
new ContainerResource(0, "container2Id", "host2", 1024, 0, null),
InetSocketAddress.createUnresolved("host2", 9002));
StreamingContainerAgent sca1 =
dnm.getContainerAgent(dnm.getPhysicalPlan().getContainers().get(0).getExternalId());
StreamingContainerAgent sca2 =
dnm.getContainerAgent(dnm.getPhysicalPlan().getContainers().get(1).getExternalId());
Assert.assertEquals("", dnm.getPhysicalPlan().getContainers().get(0), sca1.container);
Assert.assertEquals("", PTContainer.State.ALLOCATED, sca1.container.getState());
List<OperatorDeployInfo> c1 = sca1.getDeployInfoList(sca1.container.getOperators());
Assert.assertEquals("number operators assigned to c1", 1, c1.size());
OperatorDeployInfo o1DI = getNodeDeployInfo(c1, dag.getMeta(o1));
Assert.assertNotNull(o1 + " assigned to " + sca1.container.getExternalId(), o1DI);
Assert.assertEquals("type " + o1DI, OperatorDeployInfo.OperatorType.INPUT, o1DI.type);
Assert.assertEquals("inputs " + o1DI.name, 0, o1DI.inputs.size());
Assert.assertEquals("outputs " + o1DI.name, 1, o1DI.outputs.size());
Assert.assertNotNull("contextAttributes " + o1DI.name, o1DI.contextAttributes);
OutputDeployInfo c1o1outport = o1DI.outputs.get(0);
Assert.assertNotNull("stream connection for container1", c1o1outport);
Assert.assertEquals(
"stream connection for container1", "o1.outport", c1o1outport.declaredStreamId);
Assert.assertEquals(
"stream connects to upstream host", sca1.container.host, c1o1outport.bufferServerHost);
Assert.assertEquals(
"stream connects to upstream port",
sca1.container.bufferServerAddress.getPort(),
c1o1outport.bufferServerPort);
Assert.assertNotNull("contextAttributes " + c1o1outport, c1o1outport.contextAttributes);
Assert.assertEquals(
"contextAttributes " + c1o1outport,
Integer.valueOf(99),
c1o1outport.contextAttributes.get(PortContext.SPIN_MILLIS));
List<OperatorDeployInfo> c2 = sca2.getDeployInfoList(sca2.container.getOperators());
Assert.assertEquals("number operators assigned to container", 3, c2.size());
OperatorDeployInfo o2DI = getNodeDeployInfo(c2, dag.getMeta(o2));
OperatorDeployInfo o3DI = getNodeDeployInfo(c2, dag.getMeta(o3));
Assert.assertNotNull(dag.getMeta(o2) + " assigned to " + sca2.container.getExternalId(), o2DI);
Assert.assertNotNull(dag.getMeta(o3) + " assigned to " + sca2.container.getExternalId(), o3DI);
Assert.assertTrue(
"The buffer server memory for container 1",
256 == sca1.getInitContext().getValue(ContainerContext.BUFFER_SERVER_MB));
Assert.assertTrue(
"The buffer server memory for container 2",
0 == sca2.getInitContext().getValue(ContainerContext.BUFFER_SERVER_MB));
// buffer server input o2 from o1
InputDeployInfo c2o2i1 = getInputDeployInfo(o2DI, "o1.outport");
Assert.assertNotNull("stream connection for container2", c2o2i1);
Assert.assertEquals(
"stream connects to upstream host", sca1.container.host, c2o2i1.bufferServerHost);
Assert.assertEquals(
"stream connects to upstream port",
sca1.container.bufferServerAddress.getPort(),
c2o2i1.bufferServerPort);
Assert.assertEquals(
"portName " + c2o2i1, dag.getMeta(o2).getMeta(o2.inport1).getPortName(), c2o2i1.portName);
Assert.assertNull("partitionKeys " + c2o2i1, c2o2i1.partitionKeys);
Assert.assertEquals("sourceNodeId " + c2o2i1, o1DI.id, c2o2i1.sourceNodeId);
Assert.assertEquals(
"sourcePortName " + c2o2i1, TestGeneratorInputOperator.OUTPUT_PORT, c2o2i1.sourcePortName);
Assert.assertNotNull("contextAttributes " + c2o2i1, c2o2i1.contextAttributes);
// inline input o3 from o2
InputDeployInfo c2o3i1 = getInputDeployInfo(o3DI, "o2.outport1");
Assert.assertNotNull("input from o2.outport1", c2o3i1);
Assert.assertEquals("portName " + c2o3i1, GenericTestOperator.IPORT1, c2o3i1.portName);
Assert.assertNotNull("stream connection for container2", c2o3i1);
Assert.assertNull("bufferServerHost " + c2o3i1, c2o3i1.bufferServerHost);
Assert.assertEquals("bufferServerPort " + c2o3i1, 0, c2o3i1.bufferServerPort);
Assert.assertNull("partitionKeys " + c2o3i1, c2o3i1.partitionKeys);
Assert.assertEquals("sourceNodeId " + c2o3i1, o2DI.id, c2o3i1.sourceNodeId);
Assert.assertEquals(
"sourcePortName " + c2o3i1, GenericTestOperator.OPORT1, c2o3i1.sourcePortName);
Assert.assertEquals("locality " + c2o3i1, Locality.CONTAINER_LOCAL, c2o3i1.locality);
// THREAD_LOCAL o4.inport1
OperatorDeployInfo o4DI = getNodeDeployInfo(c2, dag.getMeta(o4));
Assert.assertNotNull(dag.getMeta(o4) + " assigned to " + sca2.container.getExternalId(), o4DI);
InputDeployInfo c2o4i1 = getInputDeployInfo(o4DI, "o3.outport1");
Assert.assertNotNull("input from o3.outport1", c2o4i1);
Assert.assertEquals("portName " + c2o4i1, GenericTestOperator.IPORT1, c2o4i1.portName);
Assert.assertNotNull("stream connection for container2", c2o4i1);
Assert.assertNull("bufferServerHost " + c2o4i1, c2o4i1.bufferServerHost);
Assert.assertEquals("bufferServerPort " + c2o4i1, 0, c2o4i1.bufferServerPort);
Assert.assertNull("partitionKeys " + c2o4i1, c2o4i1.partitionKeys);
Assert.assertEquals("sourceNodeId " + c2o4i1, o3DI.id, c2o4i1.sourceNodeId);
Assert.assertEquals(
"sourcePortName " + c2o4i1, GenericTestOperator.OPORT1, c2o4i1.sourcePortName);
Assert.assertEquals("locality " + c2o4i1, Locality.THREAD_LOCAL, c2o4i1.locality);
}