private LogicalPlan createDAG(LogicalPlanConfiguration lpc) throws Exception {
LogicalPlan dag = new LogicalPlan();
dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.toURI().toString());
lpc.prepareDAG(dag, null, "testApp");
dag.validate();
Assert.assertEquals("", Integer.valueOf(128), dag.getValue(DAG.MASTER_MEMORY_MB));
Assert.assertEquals(
"", "-Dlog4j.properties=custom_log4j.properties", dag.getValue(DAG.CONTAINER_JVM_OPTIONS));
return dag;
}
/**
* Verify the web service deployment and lifecycle functionality
*
* @throws Exception
*/
@Ignore // disabled due to web service init delay issue
@Test
public void testWebService() throws Exception {
// single container topology of inline input and module
Properties props = new Properties();
props.put(
StreamingApplication.DT_PREFIX + "stream.input.classname",
TestGeneratorInputOperator.class.getName());
props.put(StreamingApplication.DT_PREFIX + "stream.input.outputNode", "module1");
props.put(
StreamingApplication.DT_PREFIX + "module.module1.classname",
GenericTestOperator.class.getName());
LOG.info("Initializing Client");
LogicalPlanConfiguration tb = new LogicalPlanConfiguration(new Configuration(false));
tb.addFromProperties(props, null);
StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb));
if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
}
try {
client.start();
client.startApplication();
// attempt web service connection
ApplicationReport appReport = client.getApplicationReport();
Thread.sleep(5000); // delay to give web service time to fully initialize
Client wsClient = Client.create();
wsClient.setFollowRedirects(true);
WebResource r =
wsClient
.resource("http://" + appReport.getTrackingUrl())
.path(StramWebServices.PATH)
.path(StramWebServices.PATH_INFO);
LOG.info("Requesting: " + r.getURI());
ClientResponse response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
LOG.info("Got response: " + json.toString());
assertEquals("incorrect number of elements", 1, json.length());
assertEquals("appId", appReport.getApplicationId().toString(), json.get("id"));
r =
wsClient
.resource("http://" + appReport.getTrackingUrl())
.path(StramWebServices.PATH)
.path(StramWebServices.PATH_PHYSICAL_PLAN_OPERATORS);
LOG.info("Requesting: " + r.getURI());
response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
LOG.info("Got response: " + json.toString());
} finally {
// LOG.info("waiting...");
// synchronized (this) {
// this.wait();
// }
// boolean result = client.monitorApplication();
client.killApplication();
client.stop();
}
}
@Test
public void testSetupShutdown() throws Exception {
GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
List<NodeReport> nodeReports = response.getNodeReports();
LOG.info("{}", nodeReports);
for (NodeReport nr : nodeReports) {
LOG.info("Node: {}", nr.getNodeId());
LOG.info("Total memory: {}", nr.getCapability());
LOG.info("Used memory: {}", nr.getUsed());
LOG.info("Number containers: {}", nr.getNumContainers());
}
String appMasterJar = JarFinder.getJar(StreamingAppMaster.class);
LOG.info("appmaster jar: " + appMasterJar);
String testJar = JarFinder.getJar(StramMiniClusterTest.class);
LOG.info("testJar: " + testJar);
// create test application
Properties dagProps = new Properties();
// input module (ensure shutdown works while windows are generated)
dagProps.put(
StreamingApplication.DT_PREFIX + "operator.numGen.classname",
TestGeneratorInputOperator.class.getName());
dagProps.put(StreamingApplication.DT_PREFIX + "operator.numGen.maxTuples", "1");
// fake output adapter - to be ignored when determine shutdown
// props.put(DAGContext.DT_PREFIX + "stream.output.classname",
// HDFSOutputStream.class.getName());
// props.put(DAGContext.DT_PREFIX + "stream.output.inputNode", "module2");
// props.put(DAGContext.DT_PREFIX + "stream.output.filepath",
// "miniclustertest-testSetupShutdown.out");
dagProps.put(
StreamingApplication.DT_PREFIX + "operator.module1.classname",
GenericTestOperator.class.getName());
dagProps.put(
StreamingApplication.DT_PREFIX + "operator.module2.classname",
GenericTestOperator.class.getName());
dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.source", "numGen.outport");
dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.sinks", "module1.inport1");
dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.source", "module1.outport1");
dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.sinks", "module2.inport1");
dagProps.setProperty(
StreamingApplication.DT_PREFIX + LogicalPlan.MASTER_MEMORY_MB.getName(), "128");
dagProps.setProperty(
StreamingApplication.DT_PREFIX + LogicalPlan.CONTAINER_JVM_OPTIONS.getName(),
"-Dlog4j.properties=custom_log4j.properties");
dagProps.setProperty(
StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.MEMORY_MB.getName(), "64");
dagProps.setProperty(
StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.VCORES.getName(), "1");
dagProps.setProperty(
StreamingApplication.DT_PREFIX
+ "operator.*.port.*."
+ Context.PortContext.BUFFER_MEMORY_MB.getName(),
"32");
dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.DEBUG.getName(), "true");
// dagProps.setProperty(StreamingApplication.DT_PREFIX +
// LogicalPlan.CONTAINERS_MAX_COUNT.getName(), "2");
LOG.info("dag properties: {}", dagProps);
LOG.info("Initializing Client");
LogicalPlanConfiguration tb = new LogicalPlanConfiguration(conf);
tb.addFromProperties(dagProps, null);
LogicalPlan dag = createDAG(tb);
Configuration yarnConf = new Configuration(yarnCluster.getConfig());
StramClient client = new StramClient(yarnConf, dag);
try {
client.start();
if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
}
LOG.info("Running client");
client.startApplication();
boolean result = client.monitorApplication();
LOG.info("Client run completed. Result=" + result);
Assert.assertTrue(result);
} finally {
client.stop();
}
}