@Test
public void testValidInputOperatorDeployInfoType() {
TestGeneratorInputOperator.ValidInputOperator o1 =
dag.addOperator("o1", TestGeneratorInputOperator.ValidInputOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.addStream("stream1", o1.outport, o2.inport1);
dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
StreamingContainerManager scm = new StreamingContainerManager(dag);
PhysicalPlan physicalPlan = scm.getPhysicalPlan();
List<PTContainer> containers = physicalPlan.getContainers();
for (int i = 0; i < containers.size(); ++i) {
assignContainer(scm, "container" + (i + 1));
}
OperatorMeta o1Meta = dag.getMeta(o1);
PTOperator o1Physical = physicalPlan.getOperators(o1Meta).get(0);
String containerId = o1Physical.getContainer().getExternalId();
OperatorDeployInfo o1DeployInfo = getDeployInfo(scm.getContainerAgent(containerId)).get(0);
Assert.assertEquals(
"type " + o1DeployInfo, OperatorDeployInfo.OperatorType.INPUT, o1DeployInfo.type);
}
public static void checkpoint(
StreamingContainerManager scm, PTOperator oper, Checkpoint checkpoint) throws Exception {
// write checkpoint while AM is out,
// it needs to be picked up as part of restore
StorageAgent sa = oper.getOperatorMeta().getValue(OperatorContext.STORAGE_AGENT);
sa.save(oper.getOperatorMeta().getOperator(), oper.getId(), checkpoint.windowId);
}
public boolean hasPendingWork() {
for (PTOperator oper : container.getOperators()) {
if (oper.getState() == PTOperator.State.PENDING_DEPLOY) {
return true;
}
}
return false;
}
private int countState(PTContainer c, PTOperator.State state) {
int count = 0;
for (PTOperator o : c.getOperators()) {
if (o.getState() == state) {
count++;
}
}
return count;
}
public static InputPortMeta getIdentifyingInputPortMeta(PTOperator.PTInput input) {
InputPortMeta inputPortMeta;
PTOperator inputTarget = input.target;
StreamMeta streamMeta = input.logicalStream;
if (!inputTarget.isUnifier()) {
inputPortMeta = getInputPortMeta(inputTarget.getOperatorMeta(), streamMeta);
} else {
PTOperator destTarget = getIdentifyingOperator(inputTarget);
inputPortMeta = getInputPortMeta(destTarget.getOperatorMeta(), streamMeta);
}
return inputPortMeta;
}
private void testPhysicalPlanSerialization(StorageAgent agent) throws Exception {
LogicalPlan dag = new LogicalPlan();
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
PartitioningTestOperator o2 = dag.addOperator("o2", PartitioningTestOperator.class);
o2.setPartitionCount(3);
GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
dag.addStream("o1.outport1", o1.outport1, o2.inport1, o2.inportWithCodec);
dag.addStream("mergeStream", o2.outport1, o3.inport1);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
TestPlanContext ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
LogicalPlan.write(dag, bos);
LOG.debug("logicalPlan size: " + bos.toByteArray().length);
bos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(bos);
oos.writeObject(plan);
LOG.debug("physicalPlan size: " + bos.toByteArray().length);
ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
plan = (PhysicalPlan) new ObjectInputStream(bis).readObject();
dag = plan.getLogicalPlan();
Field f = PhysicalPlan.class.getDeclaredField("ctx");
f.setAccessible(true);
f.set(plan, ctx);
f.setAccessible(false);
OperatorMeta o2Meta = dag.getOperatorMeta("o2");
List<PTOperator> o2Partitions = plan.getOperators(o2Meta);
assertEquals(3, o2Partitions.size());
for (PTOperator o : o2Partitions) {
Assert.assertNotNull("partition null " + o, o.getPartitionKeys());
assertEquals(
"partition keys " + o + " " + o.getPartitionKeys(), 2, o.getPartitionKeys().size());
PartitioningTestOperator partitionedInstance =
(PartitioningTestOperator) plan.loadOperator(o);
assertEquals(
"instance per partition",
o.getPartitionKeys().values().toString(),
partitionedInstance.pks);
Assert.assertNotNull("partition stats null " + o, o.stats);
}
}
public static PTOperator getIdentifyingOperator(PTOperator operator) {
while ((operator != null) && operator.isUnifier()) {
PTOperator idOperator = null;
List<PTOperator.PTOutput> outputs = operator.getOutputs();
// Since it is a unifier, getting the downstream operator it is connected to which is on the
// first port
if (outputs.size() > 0) {
List<PTOperator.PTInput> sinks = outputs.get(0).sinks;
if (sinks.size() > 0) {
PTOperator.PTInput sink = sinks.get(0);
idOperator = sink.target;
}
}
operator = idOperator;
}
return operator;
}
@Test
public void testRecoveryUpstreamInline() throws Exception {
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
dag.addStream("o1o3", o1.outport1, o3.inport1);
dag.addStream("o2o3", o2.outport1, o3.inport2);
dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
StreamingContainerManager scm = new StreamingContainerManager(dag);
PhysicalPlan plan = scm.getPhysicalPlan();
Assert.assertEquals(2, plan.getContainers().size());
plan.getOperators(dag.getMeta(o1)).get(0);
Assert.assertEquals(2, plan.getContainers().size());
PTContainer c1 = plan.getContainers().get(0);
Assert.assertEquals(
Sets.newHashSet(
plan.getOperators(dag.getMeta(o1)).get(0), plan.getOperators(dag.getMeta(o3)).get(0)),
Sets.newHashSet(c1.getOperators()));
PTContainer c2 = plan.getContainers().get(1);
assignContainer(scm, "c1");
assignContainer(scm, "c2");
for (PTOperator oper : c1.getOperators()) {
Assert.assertEquals("state " + oper, PTOperator.State.PENDING_DEPLOY, oper.getState());
}
scm.scheduleContainerRestart(c2.getExternalId());
for (PTOperator oper : c1.getOperators()) {
Assert.assertEquals("state " + oper, PTOperator.State.PENDING_UNDEPLOY, oper.getState());
}
}
@Test
public void testWriteAheadLog() throws Exception {
final MutableInt flushCount = new MutableInt();
final MutableBoolean isClosed = new MutableBoolean(false);
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir);
dag.setAttribute(OperatorContext.STORAGE_AGENT, new FSStorageAgent(testMeta.dir, null));
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
StreamingContainerManager scm = new StreamingContainerManager(dag);
PhysicalPlan plan = scm.getPhysicalPlan();
Journal j = scm.getJournal();
ByteArrayOutputStream bos =
new ByteArrayOutputStream() {
@Override
public void flush() throws IOException {
super.flush();
flushCount.increment();
}
@Override
public void close() throws IOException {
super.close();
isClosed.setValue(true);
}
};
j.setOutputStream(new DataOutputStream(bos));
PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
String externalId = new MockContainer(scm, o1p1.getContainer()).container.getExternalId();
assertEquals("flush count", 1, flushCount.intValue());
o1p1.setState(PTOperator.State.ACTIVE);
assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
assertEquals("flush count", 2, flushCount.intValue());
assertEquals("is closed", false, isClosed.booleanValue());
// this will close the stream. There are 2 calls to flush() during the close() - one in Kryo
// Output and one
// in FilterOutputStream
j.setOutputStream(null);
assertEquals("flush count", 4, flushCount.intValue());
assertEquals("is closed", true, isClosed.booleanValue());
// output stream is closed, so state will be changed without recording it in the journal
o1p1.setState(PTOperator.State.INACTIVE);
assertEquals(PTOperator.State.INACTIVE, o1p1.getState());
assertEquals("flush count", 4, flushCount.intValue());
ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
j.replay(new DataInputStream(bis));
assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
InetSocketAddress addr1 = InetSocketAddress.createUnresolved("host1", 1);
PTContainer c1 = plan.getContainers().get(0);
c1.setState(PTContainer.State.ALLOCATED);
c1.host = "host1";
c1.bufferServerAddress = addr1;
c1.setAllocatedMemoryMB(2);
c1.setRequiredMemoryMB(1);
c1.setAllocatedVCores(3);
c1.setRequiredVCores(4);
j.setOutputStream(new DataOutputStream(bos));
j.write(c1.getSetContainerState());
c1.setExternalId(null);
c1.setState(PTContainer.State.NEW);
c1.setExternalId(null);
c1.host = null;
c1.bufferServerAddress = null;
bis = new ByteArrayInputStream(bos.toByteArray());
j.replay(new DataInputStream(bis));
assertEquals(externalId, c1.getExternalId());
assertEquals(PTContainer.State.ALLOCATED, c1.getState());
assertEquals("host1", c1.host);
assertEquals(addr1, c1.bufferServerAddress);
assertEquals(1, c1.getRequiredMemoryMB());
assertEquals(2, c1.getAllocatedMemoryMB());
assertEquals(3, c1.getAllocatedVCores());
assertEquals(4, c1.getRequiredVCores());
j.write(scm.getSetOperatorProperty("o1", "maxTuples", "100"));
o1.setMaxTuples(10);
j.setOutputStream(null);
bis = new ByteArrayInputStream(bos.toByteArray());
j.replay(new DataInputStream(bis));
assertEquals(100, o1.getMaxTuples());
j.setOutputStream(new DataOutputStream(bos));
scm.setOperatorProperty("o1", "maxTuples", "10");
assertEquals(10, o1.getMaxTuples());
o1.setMaxTuples(100);
assertEquals(100, o1.getMaxTuples());
j.setOutputStream(null);
bis = new ByteArrayInputStream(bos.toByteArray());
j.replay(new DataInputStream(bis));
assertEquals(10, o1.getMaxTuples());
j.setOutputStream(new DataOutputStream(bos));
scm.setPhysicalOperatorProperty(o1p1.getId(), "maxTuples", "50");
}
/**
* Test serialization of the container manager with mock execution layer.
*
* @throws Exception
*/
private void testContainerManager(StorageAgent agent) throws Exception {
FileUtils.deleteDirectory(new File(testMeta.dir)); // clean any state from previous run
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir);
dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
StatsListeningOperator o1 = dag.addOperator("o1", StatsListeningOperator.class);
FSRecoveryHandler recoveryHandler =
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false));
StreamingContainerManager scm =
StreamingContainerManager.getInstance(recoveryHandler, dag, false);
File expFile = new File(recoveryHandler.getDir(), FSRecoveryHandler.FILE_SNAPSHOT);
Assert.assertTrue("snapshot file " + expFile, expFile.exists());
PhysicalPlan plan = scm.getPhysicalPlan();
assertEquals("number required containers", 1, plan.getContainers().size());
PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
@SuppressWarnings(
"UnusedAssignment") /* sneaky: the constructor does some changes to the container */
MockContainer mc = new MockContainer(scm, o1p1.getContainer());
PTContainer originalContainer = o1p1.getContainer();
Assert.assertNotNull(o1p1.getContainer().bufferServerAddress);
assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
assertEquals("state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
// test restore initial snapshot + log
dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir);
scm =
StreamingContainerManager.getInstance(
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false)), dag, false);
dag = scm.getLogicalPlan();
plan = scm.getPhysicalPlan();
o1p1 = plan.getOperators(dag.getOperatorMeta("o1")).get(0);
assertEquals("post restore state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
o1 = (StatsListeningOperator) o1p1.getOperatorMeta().getOperator();
assertEquals(
"containerId", originalContainer.getExternalId(), o1p1.getContainer().getExternalId());
assertEquals("stats listener", 1, o1p1.statsListeners.size());
assertEquals("number stats calls", 0, o1.processStatsCnt); // stats are not logged
assertEquals("post restore 1", PTContainer.State.ALLOCATED, o1p1.getContainer().getState());
assertEquals(
"post restore 1",
originalContainer.bufferServerAddress,
o1p1.getContainer().bufferServerAddress);
StreamingContainerAgent sca = scm.getContainerAgent(originalContainer.getExternalId());
Assert.assertNotNull("allocated container restored " + originalContainer, sca);
assertEquals(
"memory usage allocated container",
(int) OperatorContext.MEMORY_MB.defaultValue,
sca.container.getAllocatedMemoryMB());
// YARN-1490 - simulate container terminated on AM recovery
scm.scheduleContainerRestart(originalContainer.getExternalId());
assertEquals("memory usage of failed container", 0, sca.container.getAllocatedMemoryMB());
Checkpoint firstCheckpoint = new Checkpoint(3, 0, 0);
mc = new MockContainer(scm, o1p1.getContainer());
checkpoint(scm, o1p1, firstCheckpoint);
mc.stats(o1p1.getId())
.deployState(OperatorHeartbeat.DeployState.ACTIVE)
.currentWindowId(3)
.checkpointWindowId(3);
mc.sendHeartbeat();
assertEquals("state " + o1p1, PTOperator.State.ACTIVE, o1p1.getState());
// logical plan modification triggers snapshot
CreateOperatorRequest cor = new CreateOperatorRequest();
cor.setOperatorFQCN(GenericTestOperator.class.getName());
cor.setOperatorName("o2");
CreateStreamRequest csr = new CreateStreamRequest();
csr.setSourceOperatorName("o1");
csr.setSourceOperatorPortName("outport");
csr.setSinkOperatorName("o2");
csr.setSinkOperatorPortName("inport1");
FutureTask<?> lpmf = scm.logicalPlanModification(Lists.newArrayList(cor, csr));
while (!lpmf.isDone()) {
scm.monitorHeartbeat();
}
Assert.assertNull(lpmf.get()); // unmask exception, if any
Assert.assertSame("dag references", dag, scm.getLogicalPlan());
assertEquals("number operators after plan modification", 2, dag.getAllOperators().size());
// set operator state triggers journal write
o1p1.setState(PTOperator.State.INACTIVE);
Checkpoint offlineCheckpoint = new Checkpoint(10, 0, 0);
// write checkpoint while AM is out,
// it needs to be picked up as part of restore
checkpoint(scm, o1p1, offlineCheckpoint);
// test restore
dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.dir);
scm =
StreamingContainerManager.getInstance(
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false)), dag, false);
Assert.assertNotSame("dag references", dag, scm.getLogicalPlan());
assertEquals(
"number operators after restore", 2, scm.getLogicalPlan().getAllOperators().size());
dag = scm.getLogicalPlan();
plan = scm.getPhysicalPlan();
o1p1 = plan.getOperators(dag.getOperatorMeta("o1")).get(0);
assertEquals("post restore state " + o1p1, PTOperator.State.INACTIVE, o1p1.getState());
o1 = (StatsListeningOperator) o1p1.getOperatorMeta().getOperator();
assertEquals("stats listener", 1, o1p1.statsListeners.size());
assertEquals("number stats calls post restore", 1, o1.processStatsCnt);
assertEquals("post restore 1", PTContainer.State.ACTIVE, o1p1.getContainer().getState());
assertEquals(
"post restore 1",
originalContainer.bufferServerAddress,
o1p1.getContainer().bufferServerAddress);
// offline checkpoint detection
assertEquals(
"checkpoints after recovery",
Lists.newArrayList(firstCheckpoint, offlineCheckpoint),
o1p1.checkpoints);
}
/**
* Create deploy info for operator.
*
* <p>
*
* @return {@link com.datatorrent.stram.api.OperatorDeployInfo}
*/
private OperatorDeployInfo createOperatorDeployInfo(PTOperator oper) {
OperatorDeployInfo ndi;
if (oper.isUnifier()) {
UnifierDeployInfo udi = new UnifierDeployInfo(); /* the constructor auto sets the type */
try {
udi.operatorAttributes = oper.getUnifiedOperatorMeta().getAttributes().clone();
} catch (CloneNotSupportedException ex) {
throw new RuntimeException("Cannot clone unifier attributes", ex);
}
ndi = udi;
} else {
ndi = new OperatorDeployInfo();
Operator operator = oper.getOperatorMeta().getOperator();
if (operator instanceof InputOperator) {
ndi.type = OperatorType.INPUT;
if (!oper.getInputs().isEmpty()) {
// If there are no input ports then it has to be an input operator. But if there are input
// ports then
// we check if any input port is connected which would make it a Generic operator.
for (PTOperator.PTInput ptInput : oper.getInputs()) {
if (ptInput.logicalStream != null && ptInput.logicalStream.getSource() != null) {
ndi.type = OperatorType.GENERIC;
break;
}
}
}
} else {
ndi.type = OperatorType.GENERIC;
}
}
Checkpoint checkpoint = oper.getRecoveryCheckpoint();
ProcessingMode pm = oper.getOperatorMeta().getValue(OperatorContext.PROCESSING_MODE);
if (pm == ProcessingMode.AT_MOST_ONCE || pm == ProcessingMode.EXACTLY_ONCE) {
// TODO: following should be handled in the container at deploy time
// for exactly once container should also purge previous checkpoint
// whenever new checkpoint is written.
StorageAgent agent =
oper.getOperatorMeta().getAttributes().get(OperatorContext.STORAGE_AGENT);
if (agent == null) {
agent = initCtx.getValue(OperatorContext.STORAGE_AGENT);
}
// pick checkpoint most recently written
try {
long[] windowIds = agent.getWindowIds(oper.getId());
long checkpointId = Stateless.WINDOW_ID;
for (long windowId : windowIds) {
if (windowId > checkpointId) {
checkpointId = windowId;
}
}
if (checkpoint == null || checkpoint.windowId != checkpointId) {
checkpoint = new Checkpoint(checkpointId, 0, 0);
}
} catch (Exception e) {
throw new RuntimeException("Failed to determine checkpoint window id " + oper, e);
}
}
LOG.debug("{} recovery checkpoint {}", oper, checkpoint);
ndi.checkpoint = checkpoint;
ndi.name = oper.getOperatorMeta().getName();
ndi.id = oper.getId();
try {
// clone map before modifying it
ndi.contextAttributes = oper.getOperatorMeta().getAttributes().clone();
} catch (CloneNotSupportedException ex) {
throw new RuntimeException("Cannot clone operator attributes", ex);
}
if (oper.isOperatorStateLess()) {
ndi.contextAttributes.put(OperatorContext.STATELESS, true);
}
return ndi;
}
@Override
public int getOperatorId() {
return oper.getId();
}
@Test
public void testOperatorShutdown() {
dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
dag.addStream("stream1", o1.outport1, o2.inport1);
dag.addStream("stream2", o2.outport1, o3.inport1);
dag.setAttribute(
o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
StreamingContainerManager scm = new StreamingContainerManager(dag);
PhysicalPlan physicalPlan = scm.getPhysicalPlan();
Map<PTContainer, MockContainer> mockContainers = new HashMap<>();
for (PTContainer c : physicalPlan.getContainers()) {
MockContainer mc = new MockContainer(scm, c);
mockContainers.put(c, mc);
}
// deploy all containers
for (Map.Entry<PTContainer, MockContainer> ce : mockContainers.entrySet()) {
ce.getValue().deploy();
}
for (Map.Entry<PTContainer, MockContainer> ce : mockContainers.entrySet()) {
// skip buffer server purge in monitorHeartbeat
ce.getKey().bufferServerAddress = null;
}
PTOperator o1p1 = physicalPlan.getOperators(dag.getMeta(o1)).get(0);
MockContainer mc1 = mockContainers.get(o1p1.getContainer());
MockOperatorStats o1p1mos = mc1.stats(o1p1.getId());
o1p1mos.currentWindowId(1).checkpointWindowId(1).deployState(DeployState.ACTIVE);
mc1.sendHeartbeat();
PTOperator o2p1 = physicalPlan.getOperators(dag.getMeta(o2)).get(0);
MockContainer mc2 = mockContainers.get(o2p1.getContainer());
MockOperatorStats o2p1mos = mc2.stats(o2p1.getId());
o2p1mos.currentWindowId(1).checkpointWindowId(1).deployState(DeployState.ACTIVE);
mc2.sendHeartbeat();
Assert.assertEquals("2 partitions", 2, physicalPlan.getOperators(dag.getMeta(o2)).size());
PTOperator o2p2 = physicalPlan.getOperators(dag.getMeta(o2)).get(1);
MockContainer mc3 = mockContainers.get(o2p2.getContainer());
MockOperatorStats o2p2mos = mc3.stats(o2p2.getId());
o2p2mos.currentWindowId(1).checkpointWindowId(1).deployState(DeployState.ACTIVE);
mc3.sendHeartbeat();
PTOperator o3p1 = physicalPlan.getOperators(dag.getMeta(o3)).get(0);
MockContainer mc4 = mockContainers.get(o3p1.getContainer());
MockOperatorStats o3p1mos = mc4.stats(o3p1.getId());
o3p1mos.currentWindowId(1).checkpointWindowId(1).deployState(DeployState.ACTIVE);
mc4.sendHeartbeat();
// unifier
PTOperator unifier = physicalPlan.getMergeOperators(dag.getMeta(o2)).get(0);
MockContainer mc5 = mockContainers.get(unifier.getContainer());
MockOperatorStats unifierp1mos = mc5.stats(unifier.getId());
unifierp1mos.currentWindowId(1).checkpointWindowId(1).deployState(DeployState.ACTIVE);
mc5.sendHeartbeat();
o1p1mos.currentWindowId(2).deployState(DeployState.SHUTDOWN);
mc1.sendHeartbeat();
scm.monitorHeartbeat();
Assert.assertEquals("committedWindowId", -1, scm.getCommittedWindowId());
scm.monitorHeartbeat(); // committedWindowId updated in next cycle
Assert.assertEquals("committedWindowId", 1, scm.getCommittedWindowId());
scm.processEvents();
Assert.assertEquals(
"containers at committedWindowId=1", 5, physicalPlan.getContainers().size());
// checkpoint window 2
o1p1mos.checkpointWindowId(2);
mc1.sendHeartbeat();
scm.monitorHeartbeat();
Assert.assertEquals("committedWindowId", 1, scm.getCommittedWindowId());
o2p1mos.currentWindowId(2).checkpointWindowId(2);
o2p2mos.currentWindowId(2).checkpointWindowId(2);
o3p1mos.currentWindowId(2).checkpointWindowId(2);
unifierp1mos.currentWindowId(2).checkpointWindowId(2);
mc2.sendHeartbeat();
mc3.sendHeartbeat();
mc4.sendHeartbeat();
mc5.sendHeartbeat();
scm.monitorHeartbeat();
// Operators are shutdown when both operators reach window Id 2
Assert.assertEquals(0, o1p1.getContainer().getOperators().size());
Assert.assertEquals(0, o2p1.getContainer().getOperators().size());
Assert.assertEquals(0, physicalPlan.getContainers().size());
}
@Test
public void testProcessHeartbeat() throws Exception {
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
dag.setAttribute(
o1,
OperatorContext.STATS_LISTENERS,
Arrays.asList(new StatsListener[] {new PartitioningTest.PartitionLoadWatch()}));
dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
StreamingContainerManager scm = new StreamingContainerManager(dag);
PhysicalPlan plan = scm.getPhysicalPlan();
Assert.assertEquals("number required containers", 1, plan.getContainers().size());
PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
// assign container
String containerId = "container1";
StreamingContainerAgent sca =
scm.assignContainer(
new ContainerResource(0, containerId, "localhost", 512, 0, null),
InetSocketAddress.createUnresolved("localhost", 0));
Assert.assertNotNull(sca);
Assert.assertEquals(PTContainer.State.ALLOCATED, o1p1.getContainer().getState());
Assert.assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
ContainerStats cstats = new ContainerStats(containerId);
ContainerHeartbeat hb = new ContainerHeartbeat();
hb.setContainerStats(cstats);
ContainerHeartbeatResponse chr = scm.processHeartbeat(hb); // get deploy request
Assert.assertNotNull(chr.deployRequest);
Assert.assertEquals("" + chr.deployRequest, 1, chr.deployRequest.size());
Assert.assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
Assert.assertEquals("state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
// first operator heartbeat
OperatorHeartbeat ohb = new OperatorHeartbeat();
ohb.setNodeId(o1p1.getId());
ohb.setState(OperatorHeartbeat.DeployState.ACTIVE);
OperatorStats stats = new OperatorStats();
stats.checkpoint = new Checkpoint(2, 0, 0);
stats.windowId = 3;
stats.outputPorts = Lists.newArrayList();
PortStats ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
ps.bufferServerBytes = 101;
ps.tupleCount = 1;
stats.outputPorts.add(ps);
ohb.windowStats = Lists.newArrayList(stats);
cstats.operators.add(ohb);
scm.processHeartbeat(hb); // activate operator
Assert.assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
Assert.assertEquals("state " + o1p1, PTOperator.State.ACTIVE, o1p1.getState());
Assert.assertEquals("tuples " + o1p1, 1, o1p1.stats.totalTuplesEmitted.get());
Assert.assertEquals("tuples " + o1p1, 0, o1p1.stats.totalTuplesProcessed.get());
Assert.assertEquals("window " + o1p1, 3, o1p1.stats.currentWindowId.get());
Assert.assertEquals("port stats", 1, o1p1.stats.outputPortStatusList.size());
PortStatus o1p1ps = o1p1.stats.outputPortStatusList.get(TestGeneratorInputOperator.OUTPUT_PORT);
Assert.assertNotNull("port stats", o1p1ps);
Assert.assertEquals("port stats", 1, o1p1ps.totalTuples);
// second operator heartbeat
stats = new OperatorStats();
stats.checkpoint = new Checkpoint(2, 0, 0);
stats.windowId = 4;
stats.outputPorts = Lists.newArrayList();
ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
ps.bufferServerBytes = 1;
ps.tupleCount = 1;
stats.outputPorts.add(ps);
ohb.windowStats = Lists.newArrayList(stats);
cstats.operators.clear();
cstats.operators.add(ohb);
scm.processHeartbeat(hb);
Assert.assertEquals("tuples " + o1p1, 2, o1p1.stats.totalTuplesEmitted.get());
Assert.assertEquals("window " + o1p1, 4, o1p1.stats.currentWindowId.get());
Assert.assertEquals("statsQueue " + o1p1, 2, o1p1.stats.listenerStats.size());
scm.processEvents();
Assert.assertEquals("statsQueue " + o1p1, 0, o1p1.stats.listenerStats.size());
Assert.assertEquals("lastStats " + o1p1, 2, o1p1.stats.lastWindowedStats.size());
}
private void testRestartApp(StorageAgent agent, String appPath1) throws Exception {
FileUtils.deleteDirectory(new File(testMeta.dir)); // clean any state from previous run
String appId1 = "app1";
String appId2 = "app2";
String appPath2 = testMeta.dir + "/" + appId2;
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_ID, appId1);
dag.setAttribute(LogicalPlan.APPLICATION_PATH, appPath1);
dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, 1);
dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
dag.addOperator("o1", StatsListeningOperator.class);
FSRecoveryHandler recoveryHandler =
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false));
StreamingContainerManager.getInstance(recoveryHandler, dag, false);
// test restore initial snapshot + log
dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, appPath1);
StreamingContainerManager scm =
StreamingContainerManager.getInstance(
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false)), dag, false);
PhysicalPlan plan = scm.getPhysicalPlan();
dag = plan.getLogicalPlan(); // original plan
Assert.assertNotNull("operator", dag.getOperatorMeta("o1"));
PTOperator o1p1 = plan.getOperators(dag.getOperatorMeta("o1")).get(0);
long[] ids =
new FSStorageAgent(appPath1 + "/" + LogicalPlan.SUBDIR_CHECKPOINTS, new Configuration())
.getWindowIds(o1p1.getId());
Assert.assertArrayEquals(new long[] {o1p1.getRecoveryCheckpoint().getWindowId()}, ids);
Assert.assertNull(o1p1.getContainer().getExternalId());
// trigger journal write
o1p1.getContainer().setExternalId("cid1");
scm.writeJournal(o1p1.getContainer().getSetContainerState());
dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, appPath2);
dag.setAttribute(LogicalPlan.APPLICATION_ID, appId2);
StramClient sc = new StramClient(new Configuration(), dag);
try {
sc.start();
sc.copyInitialState(new Path(appPath1));
} finally {
sc.stop();
}
scm =
StreamingContainerManager.getInstance(
new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false)), dag, false);
plan = scm.getPhysicalPlan();
dag = plan.getLogicalPlan();
assertEquals("modified appId", appId2, dag.getValue(LogicalPlan.APPLICATION_ID));
assertEquals("modified appPath", appPath2, dag.getValue(LogicalPlan.APPLICATION_PATH));
Assert.assertNotNull("operator", dag.getOperatorMeta("o1"));
o1p1 = plan.getOperators(dag.getOperatorMeta("o1")).get(0);
assertEquals("journal copied", "cid1", o1p1.getContainer().getExternalId());
ids =
new FSStorageAgent(appPath2 + "/" + LogicalPlan.SUBDIR_CHECKPOINTS, new Configuration())
.getWindowIds(o1p1.getId());
Assert.assertArrayEquals(
"checkpoints copied", new long[] {o1p1.getRecoveryCheckpoint().getWindowId()}, ids);
}
/**
* Create deploy info for StramChild.
*
* @param operators
* @return StreamingContainerContext
*/
public List<OperatorDeployInfo> getDeployInfoList(Collection<PTOperator> operators) {
if (container.bufferServerAddress == null) {
throw new AssertionError("No buffer server address assigned");
}
Map<OperatorDeployInfo, PTOperator> nodes = new LinkedHashMap<>();
HashSet<PTOperator.PTOutput> publishers = new HashSet<>();
PhysicalPlan physicalPlan = dnmgr.getPhysicalPlan();
for (PTOperator oper : operators) {
if (oper.getState() != State.PENDING_DEPLOY) {
LOG.debug("Skipping deploy for operator {} state {}", oper, oper.getState());
continue;
}
OperatorDeployInfo ndi = createOperatorDeployInfo(oper);
nodes.put(ndi, oper);
ndi.inputs = new ArrayList<>(oper.getInputs().size());
ndi.outputs = new ArrayList<>(oper.getOutputs().size());
for (PTOperator.PTOutput out : oper.getOutputs()) {
final StreamMeta streamMeta = out.logicalStream;
// buffer server or inline publisher
OutputDeployInfo portInfo = new OutputDeployInfo();
portInfo.declaredStreamId = streamMeta.getName();
portInfo.portName = out.portName;
try {
portInfo.contextAttributes = streamMeta.getSource().getAttributes().clone();
} catch (CloneNotSupportedException ex) {
throw new RuntimeException("Cannot clone attributes", ex);
}
boolean outputUnified = false;
for (PTOperator.PTInput input : out.sinks) {
if (input.target.isUnifier()) {
outputUnified = true;
break;
}
}
portInfo.contextAttributes.put(PortContext.IS_OUTPUT_UNIFIED, outputUnified);
if (ndi.type == OperatorDeployInfo.OperatorType.UNIFIER) {
// input attributes of the downstream operator
for (InputPortMeta sink : streamMeta.getSinks()) {
portInfo.contextAttributes = sink.getAttributes();
break;
}
}
if (!out.isDownStreamInline()) {
portInfo.bufferServerHost = oper.getContainer().bufferServerAddress.getHostName();
portInfo.bufferServerPort = oper.getContainer().bufferServerAddress.getPort();
portInfo.bufferServerToken = oper.getContainer().getBufferServerToken();
// Build the stream codec configuration of all sinks connected to this port
for (PTOperator.PTInput input : out.sinks) {
// Create mappings for all non-inline operators
if (input.target.getContainer() != out.source.getContainer()) {
InputPortMeta inputPortMeta = getIdentifyingInputPortMeta(input);
StreamCodec<?> streamCodecInfo = getStreamCodec(inputPortMeta);
Integer id = physicalPlan.getStreamCodecIdentifier(streamCodecInfo);
if (!portInfo.streamCodecs.containsKey(id)) {
portInfo.streamCodecs.put(id, streamCodecInfo);
}
}
}
}
ndi.outputs.add(portInfo);
publishers.add(out);
}
}
// after we know all publishers within container, determine subscribers
for (Map.Entry<OperatorDeployInfo, PTOperator> operEntry : nodes.entrySet()) {
OperatorDeployInfo ndi = operEntry.getKey();
PTOperator oper = operEntry.getValue();
for (PTOperator.PTInput in : oper.getInputs()) {
final StreamMeta streamMeta = in.logicalStream;
if (streamMeta.getSource() == null) {
throw new AssertionError("source is null: " + in);
}
PTOperator.PTOutput sourceOutput = in.source;
InputDeployInfo inputInfo = new InputDeployInfo();
inputInfo.declaredStreamId = streamMeta.getName();
inputInfo.portName = in.portName;
InputPortMeta inputPortMeta = getInputPortMeta(oper.getOperatorMeta(), streamMeta);
if (inputPortMeta != null) {
inputInfo.contextAttributes = inputPortMeta.getAttributes();
}
if (inputInfo.contextAttributes == null
&& ndi.type == OperatorDeployInfo.OperatorType.UNIFIER) {
inputInfo.contextAttributes = in.source.logicalStream.getSource().getAttributes();
}
inputInfo.sourceNodeId = sourceOutput.source.getId();
inputInfo.sourcePortName = sourceOutput.portName;
if (in.partitions != null && in.partitions.mask != 0) {
inputInfo.partitionMask = in.partitions.mask;
inputInfo.partitionKeys = in.partitions.partitions;
}
if (sourceOutput.source.getContainer() == oper.getContainer()) {
// both operators in same container
if (!publishers.contains(sourceOutput)) {
throw new AssertionError(
"Source not deployed for container local stream " + sourceOutput + " " + in);
}
if (streamMeta.getLocality() == Locality.THREAD_LOCAL) {
inputInfo.locality = Locality.THREAD_LOCAL;
ndi.type = OperatorType.OIO;
} else {
inputInfo.locality = Locality.CONTAINER_LOCAL;
}
} else {
// buffer server input
PTContainer container = sourceOutput.source.getContainer();
InetSocketAddress addr = container.bufferServerAddress;
if (addr == null) {
throw new AssertionError("upstream address not assigned: " + sourceOutput);
}
inputInfo.bufferServerHost = addr.getHostName();
inputInfo.bufferServerPort = addr.getPort();
inputInfo.bufferServerToken = container.getBufferServerToken();
}
// On the input side there is a unlikely scenario of partitions even for inline stream that
// is being
// handled. Always specifying a stream codec configuration in case that scenario happens.
InputPortMeta idInputPortMeta = getIdentifyingInputPortMeta(in);
StreamCodec<?> streamCodecInfo = getStreamCodec(idInputPortMeta);
Integer id = physicalPlan.getStreamCodecIdentifier(streamCodecInfo);
inputInfo.streamCodecs.put(id, streamCodecInfo);
ndi.inputs.add(inputInfo);
}
}
return new ArrayList<>(nodes.keySet());
}
@Test
public void testStaticPartitioning() {
//
// ,---> node2----,
// | |
// node1---+---> node2----+--->unifier--->node3
// | |
// '---> node2----'
//
GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
PhysicalPlanTest.PartitioningTestOperator node2 =
dag.addOperator("node2", PhysicalPlanTest.PartitioningTestOperator.class);
node2.setPartitionCount(3);
dag.setAttribute(
node2, OperatorContext.SPIN_MILLIS, 10); /* this should not affect anything materially */
dag.setOutputPortAttribute(node2.outport1, PortContext.QUEUE_CAPACITY, 1111);
GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
dag.setInputPortAttribute(node3.inport1, PortContext.QUEUE_CAPACITY, 2222);
LogicalPlan.StreamMeta n1n2 = dag.addStream("n1n2", node1.outport1, node2.inport1);
LogicalPlan.StreamMeta n2n3 = dag.addStream("n2n3", node2.outport1, node3.inport1);
dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
MemoryStorageAgent msa = new MemoryStorageAgent();
dag.setAttribute(OperatorContext.STORAGE_AGENT, msa);
StreamingContainerManager dnm = new StreamingContainerManager(dag);
PhysicalPlan plan = dnm.getPhysicalPlan();
Assert.assertEquals("number containers", 6, plan.getContainers().size());
List<StreamingContainerAgent> containerAgents = Lists.newArrayList();
for (int i = 0; i < plan.getContainers().size(); i++) {
containerAgents.add(assignContainer(dnm, "container" + (i + 1)));
}
PTContainer c = plan.getOperators(dag.getMeta(node1)).get(0).getContainer();
StreamingContainerAgent sca1 = dnm.getContainerAgent(c.getExternalId());
List<OperatorDeployInfo> c1 = getDeployInfo(sca1);
Assert.assertEquals("number operators assigned to container", 1, c1.size());
Assert.assertTrue(
dag.getMeta(node2) + " assigned to " + sca1.container.getExternalId(),
containsNodeContext(c1, dag.getMeta(node1)));
List<PTOperator> o2Partitions = plan.getOperators(dag.getMeta(node2));
Assert.assertEquals(
"number partitions", TestStaticPartitioningSerDe.partitions.length, o2Partitions.size());
for (int i = 0; i < o2Partitions.size(); i++) {
String containerId = o2Partitions.get(i).getContainer().getExternalId();
List<OperatorDeployInfo> cc = getDeployInfo(dnm.getContainerAgent(containerId));
Assert.assertEquals("number operators assigned to container", 1, cc.size());
Assert.assertTrue(
dag.getMeta(node2) + " assigned to " + containerId,
containsNodeContext(cc, dag.getMeta(node2)));
// n1n2 in, mergeStream out
OperatorDeployInfo ndi = cc.get(0);
Assert.assertEquals("type " + ndi, OperatorDeployInfo.OperatorType.GENERIC, ndi.type);
Assert.assertEquals("inputs " + ndi, 1, ndi.inputs.size());
Assert.assertEquals("outputs " + ndi, 1, ndi.outputs.size());
InputDeployInfo nidi = ndi.inputs.get(0);
Assert.assertEquals("stream " + nidi, n1n2.getName(), nidi.declaredStreamId);
Assert.assertEquals(
"partition for " + containerId,
Sets.newHashSet(node2.partitionKeys[i]),
nidi.partitionKeys);
Assert.assertEquals("number stream codecs for " + nidi, 1, nidi.streamCodecs.size());
}
// unifier
List<PTOperator> o2Unifiers = plan.getMergeOperators(dag.getMeta(node2));
Assert.assertEquals("number unifiers", 1, o2Unifiers.size());
List<OperatorDeployInfo> cUnifier =
getDeployInfo(dnm.getContainerAgent(o2Unifiers.get(0).getContainer().getExternalId()));
Assert.assertEquals("number operators " + cUnifier, 1, cUnifier.size());
OperatorDeployInfo mergeNodeDI =
getNodeDeployInfo(cUnifier, dag.getMeta(node2).getMeta(node2.outport1).getUnifierMeta());
Assert.assertNotNull("unifier for " + node2, mergeNodeDI);
Assert.assertEquals(
"type " + mergeNodeDI, OperatorDeployInfo.OperatorType.UNIFIER, mergeNodeDI.type);
Assert.assertEquals("inputs " + mergeNodeDI, 3, mergeNodeDI.inputs.size());
List<Integer> sourceNodeIds = Lists.newArrayList();
for (InputDeployInfo nidi : mergeNodeDI.inputs) {
Assert.assertEquals("streamName " + nidi, n2n3.getName(), nidi.declaredStreamId);
String mergePortName =
"<merge#" + dag.getMeta(node2).getMeta(node2.outport1).getPortName() + ">";
Assert.assertEquals("portName " + nidi, mergePortName, nidi.portName);
Assert.assertNotNull("sourceNodeId " + nidi, nidi.sourceNodeId);
Assert.assertNotNull("contextAttributes " + nidi, nidi.contextAttributes);
Assert.assertEquals(
"contextAttributes ", new Integer(1111), nidi.getValue(PortContext.QUEUE_CAPACITY));
sourceNodeIds.add(nidi.sourceNodeId);
}
for (PTOperator node : dnm.getPhysicalPlan().getOperators(dag.getMeta(node2))) {
Assert.assertTrue(
sourceNodeIds + " contains " + node.getId(), sourceNodeIds.contains(node.getId()));
}
Assert.assertEquals("outputs " + mergeNodeDI, 1, mergeNodeDI.outputs.size());
for (OutputDeployInfo odi : mergeNodeDI.outputs) {
Assert.assertNotNull("contextAttributes " + odi, odi.contextAttributes);
Assert.assertEquals(
"contextAttributes ", new Integer(2222), odi.getValue(PortContext.QUEUE_CAPACITY));
}
try {
Object operator = msa.load(mergeNodeDI.id, Stateless.WINDOW_ID);
Assert.assertTrue("" + operator, operator instanceof DefaultUnifier);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
// node3 container
c = plan.getOperators(dag.getMeta(node3)).get(0).getContainer();
List<OperatorDeployInfo> cmerge = getDeployInfo(dnm.getContainerAgent(c.getExternalId()));
Assert.assertEquals("number operators " + cmerge, 1, cmerge.size());
OperatorDeployInfo node3DI = getNodeDeployInfo(cmerge, dag.getMeta(node3));
Assert.assertNotNull(dag.getMeta(node3) + " assigned", node3DI);
Assert.assertEquals("inputs " + node3DI, 1, node3DI.inputs.size());
InputDeployInfo node3In = node3DI.inputs.get(0);
Assert.assertEquals("streamName " + node3In, n2n3.getName(), node3In.declaredStreamId);
Assert.assertEquals(
"portName " + node3In,
dag.getMeta(node3).getMeta(node3.inport1).getPortName(),
node3In.portName);
Assert.assertNotNull("sourceNodeId " + node3DI, node3In.sourceNodeId);
Assert.assertEquals(
"sourcePortName " + node3DI, mergeNodeDI.outputs.get(0).portName, node3In.sourcePortName);
}