@Test
public void testDeployInfoSerialization() throws Exception {
OperatorDeployInfo ndi = new OperatorDeployInfo();
ndi.name = "node1";
ndi.type = OperatorDeployInfo.OperatorType.GENERIC;
ndi.id = 1;
ndi.contextAttributes = new com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap();
ndi.contextAttributes.put(OperatorContext.SPIN_MILLIS, 100);
OperatorDeployInfo.InputDeployInfo input = new OperatorDeployInfo.InputDeployInfo();
input.declaredStreamId = "streamToNode";
input.portName = "inputPortNameOnNode";
input.sourceNodeId = 99;
ndi.inputs = new ArrayList<OperatorDeployInfo.InputDeployInfo>();
ndi.inputs.add(input);
OperatorDeployInfo.OutputDeployInfo output = new OperatorDeployInfo.OutputDeployInfo();
output.declaredStreamId = "streamFromNode";
output.portName = "outputPortNameOnNode";
ndi.outputs = new ArrayList<OperatorDeployInfo.OutputDeployInfo>();
ndi.outputs.add(output);
ContainerHeartbeatResponse scc = new ContainerHeartbeatResponse();
scc.deployRequest = Collections.singletonList(ndi);
DataOutputByteBuffer out = new DataOutputByteBuffer();
scc.write(out);
DataInputByteBuffer in = new DataInputByteBuffer();
in.reset(out.getData());
ContainerHeartbeatResponse clone = new ContainerHeartbeatResponse();
clone.readFields(in);
Assert.assertNotNull(clone.deployRequest);
Assert.assertEquals(1, clone.deployRequest.size());
OperatorDeployInfo ndiClone = clone.deployRequest.get(0);
Assert.assertEquals("name", ndi.name, ndiClone.name);
Assert.assertEquals("type", ndi.type, ndiClone.type);
String nodeToString = ndi.toString();
Assert.assertTrue(nodeToString.contains(input.portName));
Assert.assertTrue(nodeToString.contains(output.portName));
Assert.assertEquals(
"contextAttributes " + ndiClone.contextAttributes,
Integer.valueOf(100),
ndiClone.contextAttributes.get(OperatorContext.SPIN_MILLIS));
}
/**
* Create deploy info for operator.
*
* <p>
*
* @return {@link com.datatorrent.stram.api.OperatorDeployInfo}
*/
private OperatorDeployInfo createOperatorDeployInfo(PTOperator oper) {
OperatorDeployInfo ndi;
if (oper.isUnifier()) {
UnifierDeployInfo udi = new UnifierDeployInfo(); /* the constructor auto sets the type */
try {
udi.operatorAttributes = oper.getUnifiedOperatorMeta().getAttributes().clone();
} catch (CloneNotSupportedException ex) {
throw new RuntimeException("Cannot clone unifier attributes", ex);
}
ndi = udi;
} else {
ndi = new OperatorDeployInfo();
Operator operator = oper.getOperatorMeta().getOperator();
if (operator instanceof InputOperator) {
ndi.type = OperatorType.INPUT;
if (!oper.getInputs().isEmpty()) {
// If there are no input ports then it has to be an input operator. But if there are input
// ports then
// we check if any input port is connected which would make it a Generic operator.
for (PTOperator.PTInput ptInput : oper.getInputs()) {
if (ptInput.logicalStream != null && ptInput.logicalStream.getSource() != null) {
ndi.type = OperatorType.GENERIC;
break;
}
}
}
} else {
ndi.type = OperatorType.GENERIC;
}
}
Checkpoint checkpoint = oper.getRecoveryCheckpoint();
ProcessingMode pm = oper.getOperatorMeta().getValue(OperatorContext.PROCESSING_MODE);
if (pm == ProcessingMode.AT_MOST_ONCE || pm == ProcessingMode.EXACTLY_ONCE) {
// TODO: following should be handled in the container at deploy time
// for exactly once container should also purge previous checkpoint
// whenever new checkpoint is written.
StorageAgent agent =
oper.getOperatorMeta().getAttributes().get(OperatorContext.STORAGE_AGENT);
if (agent == null) {
agent = initCtx.getValue(OperatorContext.STORAGE_AGENT);
}
// pick checkpoint most recently written
try {
long[] windowIds = agent.getWindowIds(oper.getId());
long checkpointId = Stateless.WINDOW_ID;
for (long windowId : windowIds) {
if (windowId > checkpointId) {
checkpointId = windowId;
}
}
if (checkpoint == null || checkpoint.windowId != checkpointId) {
checkpoint = new Checkpoint(checkpointId, 0, 0);
}
} catch (Exception e) {
throw new RuntimeException("Failed to determine checkpoint window id " + oper, e);
}
}
LOG.debug("{} recovery checkpoint {}", oper, checkpoint);
ndi.checkpoint = checkpoint;
ndi.name = oper.getOperatorMeta().getName();
ndi.id = oper.getId();
try {
// clone map before modifying it
ndi.contextAttributes = oper.getOperatorMeta().getAttributes().clone();
} catch (CloneNotSupportedException ex) {
throw new RuntimeException("Cannot clone operator attributes", ex);
}
if (oper.isOperatorStateLess()) {
ndi.contextAttributes.put(OperatorContext.STATELESS, true);
}
return ndi;
}
