private LogicalPlan createDAG(LogicalPlanConfiguration lpc) throws Exception { LogicalPlan dag = new LogicalPlan(); dag.setAttribute(LogicalPlan.APPLICATION_PATH, testMeta.toURI().toString()); lpc.prepareDAG(dag, null, "testApp"); dag.validate(); Assert.assertEquals("", Integer.valueOf(128), dag.getValue(DAG.MASTER_MEMORY_MB)); Assert.assertEquals( "", "-Dlog4j.properties=custom_log4j.properties", dag.getValue(DAG.CONTAINER_JVM_OPTIONS)); return dag; }
/** * Verify the web service deployment and lifecycle functionality * * @throws Exception */ @Ignore // disabled due to web service init delay issue @Test public void testWebService() throws Exception { // single container topology of inline input and module Properties props = new Properties(); props.put( StreamingApplication.DT_PREFIX + "stream.input.classname", TestGeneratorInputOperator.class.getName()); props.put(StreamingApplication.DT_PREFIX + "stream.input.outputNode", "module1"); props.put( StreamingApplication.DT_PREFIX + "module.module1.classname", GenericTestOperator.class.getName()); LOG.info("Initializing Client"); LogicalPlanConfiguration tb = new LogicalPlanConfiguration(new Configuration(false)); tb.addFromProperties(props, null); StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb)); if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) { client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster } try { client.start(); client.startApplication(); // attempt web service connection ApplicationReport appReport = client.getApplicationReport(); Thread.sleep(5000); // delay to give web service time to fully initialize Client wsClient = Client.create(); wsClient.setFollowRedirects(true); WebResource r = wsClient .resource("http://" + appReport.getTrackingUrl()) .path(StramWebServices.PATH) .path(StramWebServices.PATH_INFO); LOG.info("Requesting: " + r.getURI()); ClientResponse response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); LOG.info("Got response: " + json.toString()); assertEquals("incorrect number of elements", 1, json.length()); assertEquals("appId", appReport.getApplicationId().toString(), json.get("id")); r = wsClient .resource("http://" + appReport.getTrackingUrl()) .path(StramWebServices.PATH) .path(StramWebServices.PATH_PHYSICAL_PLAN_OPERATORS); LOG.info("Requesting: " + r.getURI()); response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); json = response.getEntity(JSONObject.class); LOG.info("Got response: " + json.toString()); } finally { // LOG.info("waiting..."); // synchronized (this) { // this.wait(); // } // boolean result = client.monitorApplication(); client.killApplication(); client.stop(); } }
@Test public void testSetupShutdown() throws Exception { GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class); ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService(); GetClusterNodesResponse response = clientRMService.getClusterNodes(request); List<NodeReport> nodeReports = response.getNodeReports(); LOG.info("{}", nodeReports); for (NodeReport nr : nodeReports) { LOG.info("Node: {}", nr.getNodeId()); LOG.info("Total memory: {}", nr.getCapability()); LOG.info("Used memory: {}", nr.getUsed()); LOG.info("Number containers: {}", nr.getNumContainers()); } String appMasterJar = JarFinder.getJar(StreamingAppMaster.class); LOG.info("appmaster jar: " + appMasterJar); String testJar = JarFinder.getJar(StramMiniClusterTest.class); LOG.info("testJar: " + testJar); // create test application Properties dagProps = new Properties(); // input module (ensure shutdown works while windows are generated) dagProps.put( StreamingApplication.DT_PREFIX + "operator.numGen.classname", TestGeneratorInputOperator.class.getName()); dagProps.put(StreamingApplication.DT_PREFIX + "operator.numGen.maxTuples", "1"); // fake output adapter - to be ignored when determine shutdown // props.put(DAGContext.DT_PREFIX + "stream.output.classname", // HDFSOutputStream.class.getName()); // props.put(DAGContext.DT_PREFIX + "stream.output.inputNode", "module2"); // props.put(DAGContext.DT_PREFIX + "stream.output.filepath", // "miniclustertest-testSetupShutdown.out"); dagProps.put( StreamingApplication.DT_PREFIX + "operator.module1.classname", GenericTestOperator.class.getName()); dagProps.put( StreamingApplication.DT_PREFIX + "operator.module2.classname", GenericTestOperator.class.getName()); dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.source", "numGen.outport"); dagProps.put(StreamingApplication.DT_PREFIX + "stream.fromNumGen.sinks", "module1.inport1"); dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.source", "module1.outport1"); dagProps.put(StreamingApplication.DT_PREFIX + "stream.n1n2.sinks", "module2.inport1"); dagProps.setProperty( StreamingApplication.DT_PREFIX + LogicalPlan.MASTER_MEMORY_MB.getName(), "128"); dagProps.setProperty( StreamingApplication.DT_PREFIX + LogicalPlan.CONTAINER_JVM_OPTIONS.getName(), "-Dlog4j.properties=custom_log4j.properties"); dagProps.setProperty( StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.MEMORY_MB.getName(), "64"); dagProps.setProperty( StreamingApplication.DT_PREFIX + "operator.*." + OperatorContext.VCORES.getName(), "1"); dagProps.setProperty( StreamingApplication.DT_PREFIX + "operator.*.port.*." + Context.PortContext.BUFFER_MEMORY_MB.getName(), "32"); dagProps.setProperty(StreamingApplication.DT_PREFIX + LogicalPlan.DEBUG.getName(), "true"); // dagProps.setProperty(StreamingApplication.DT_PREFIX + // LogicalPlan.CONTAINERS_MAX_COUNT.getName(), "2"); LOG.info("dag properties: {}", dagProps); LOG.info("Initializing Client"); LogicalPlanConfiguration tb = new LogicalPlanConfiguration(conf); tb.addFromProperties(dagProps, null); LogicalPlan dag = createDAG(tb); Configuration yarnConf = new Configuration(yarnCluster.getConfig()); StramClient client = new StramClient(yarnConf, dag); try { client.start(); if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) { client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster } LOG.info("Running client"); client.startApplication(); boolean result = client.monitorApplication(); LOG.info("Client run completed. Result=" + result); Assert.assertTrue(result); } finally { client.stop(); } }