private void testDownStreamPartition(Locality locality) throws Exception {
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.setAttribute(
o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
dag.addStream("o1Output1", o1.outport, o2.inport1).setLocality(locality);
int maxContainers = 5;
dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);
dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
dag.validate();
PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
Assert.assertEquals("number of containers", 1, plan.getContainers().size());
PTContainer container1 = plan.getContainers().get(0);
Assert.assertEquals("number operators " + container1, 3, container1.getOperators().size());
StramLocalCluster slc = new StramLocalCluster(dag);
slc.run(5000);
}
private void testAppDataSources(boolean appendQIDToTopic) throws Exception {
StramLocalCluster lc = new StramLocalCluster(dag);
lc.runAsync();
StreamingContainerManager dnmgr = lc.dnmgr;
List<AppDataSource> appDataSources = dnmgr.getAppDataSources();
Assert.assertEquals("There should be exactly one data source", 1, appDataSources.size());
AppDataSource ads = appDataSources.get(0);
Assert.assertEquals("Data Source name verification", "ds.result", ads.getName());
AppDataSource.QueryInfo query = ads.getQuery();
Assert.assertEquals("Query operator name verification", "q", query.operatorName);
Assert.assertEquals("Query topic verification", "xyz.query", query.topic);
Assert.assertEquals("Query URL verification", "ws://123.123.123.123:9090/pubsub", query.url);
AppDataSource.ResultInfo result = ads.getResult();
Assert.assertEquals("Result operator name verification", "r", result.operatorName);
Assert.assertEquals("Result topic verification", "xyz.result", result.topic);
Assert.assertEquals("Result URL verification", "ws://123.123.123.124:9090/pubsub", result.url);
Assert.assertEquals(
"Result QID append verification", appendQIDToTopic, result.appendQIDToTopic);
lc.shutdown();
}
@Test
public void testLatency() throws Exception {
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
HighLatencyTestOperator o3 = dag.addOperator("o3", HighLatencyTestOperator.class);
GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
long latency = 5000; // 5 seconds
o3.setLatency(latency);
dag.addStream("o1.outport", o1.outport, o2.inport1, o3.inport1);
dag.addStream("o2.outport1", o2.outport1, o4.inport1);
dag.addStream("o3.outport1", o3.outport1, o4.inport2);
dag.setAttribute(Context.DAGContext.STATS_MAX_ALLOWABLE_WINDOWS_LAG, 2); // 1 second
StramLocalCluster lc = new StramLocalCluster(dag);
StreamingContainerManager dnmgr = lc.dnmgr;
lc.runAsync();
Thread.sleep(10000);
LogicalOperatorInfo o1Info = dnmgr.getLogicalOperatorInfo("o1");
LogicalOperatorInfo o2Info = dnmgr.getLogicalOperatorInfo("o2");
LogicalOperatorInfo o3Info = dnmgr.getLogicalOperatorInfo("o3");
LogicalOperatorInfo o4Info = dnmgr.getLogicalOperatorInfo("o4");
Assert.assertEquals("Input operator latency must be zero", 0, o1Info.latencyMA);
Assert.assertTrue("Latency must be greater than or equal to zero", o2Info.latencyMA >= 0);
Assert.assertTrue(
"Actual latency must be greater than the artificially introduced latency",
o3Info.latencyMA > latency);
Assert.assertTrue("Latency must be greater than or equal to zero", o4Info.latencyMA >= 0);
StreamingContainerManager.CriticalPathInfo criticalPathInfo = dnmgr.getCriticalPathInfo();
Assert.assertArrayEquals(
"Critical Path must be the path in the DAG that includes the HighLatencyTestOperator",
new Integer[] {
o1Info.partitions.iterator().next(),
o3Info.partitions.iterator().next(),
o4Info.partitions.iterator().next()
},
criticalPathInfo.path.toArray());
Assert.assertTrue(
"Whole DAG latency must be greater than the artificially introduced latency",
criticalPathInfo.latency > latency);
lc.shutdown();
}
@Test
public void testPhysicalPropertyUpdate() throws Exception {
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.addStream("o1.outport", o1.outport, o2.inport1);
StramLocalCluster lc = new StramLocalCluster(dag);
lc.runAsync();
StreamingContainerManager dnmgr = lc.dnmgr;
Map<Integer, PTOperator> operatorMap = dnmgr.getPhysicalPlan().getAllOperators();
for (PTOperator p : operatorMap.values()) {
StramTestSupport.waitForActivation(lc, p);
}
dnmgr.setPhysicalOperatorProperty(
lc.getPlanOperators(dag.getMeta(o1)).get(0).getId(), "maxTuples", "2");
Future<?> future =
dnmgr.getPhysicalOperatorProperty(
lc.getPlanOperators(dag.getMeta(o1)).get(0).getId(), "maxTuples", 10000);
Object object = future.get(10000, TimeUnit.MILLISECONDS);
Assert.assertNotNull(object);
@SuppressWarnings("unchecked")
Map<String, Object> propertyValue = (Map<String, Object>) object;
Assert.assertEquals(2, propertyValue.get("maxTuples"));
lc.shutdown();
}
