Exemplo n.º 1
0
  /** Test regular operation, including command line parameter parsing. */
  @Test(timeout = 60000) // timeout after a minute.
  public void testDetachedMode() {
    LOG.info("Starting testDetachedMode()");
    addTestAppender(FlinkYarnSessionCli.class, Level.INFO);
    Runner runner =
        startWithArgs(
            new String[] {
              "-j",
              flinkUberjar.getAbsolutePath(),
              "-t",
              flinkLibFolder.getAbsolutePath(),
              "-n",
              "1",
              "-jm",
              "768",
              "-tm",
              "1024",
              "--name",
              "MyCustomName", // test setting a custom name
              "--detached"
            },
            "Flink JobManager is now running on",
            RunTypes.YARN_SESSION);

    checkForLogString("The Flink YARN client has been started in detached mode");

    Assert.assertFalse("The runner should detach.", runner.isAlive());

    LOG.info("Waiting until two containers are running");
    // wait until two containers are running
    while (getRunningContainers() < 2) {
      sleep(500);
    }
    LOG.info("Two containers are running. Killing the application");

    // kill application "externally".
    try {
      YarnClient yc = YarnClient.createYarnClient();
      yc.init(yarnConfiguration);
      yc.start();
      List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
      Assert.assertEquals(1, apps.size()); // Only one running
      ApplicationReport app = apps.get(0);

      Assert.assertEquals("MyCustomName", app.getName());
      ApplicationId id = app.getApplicationId();
      yc.killApplication(id);

      while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0) {
        sleep(500);
      }
    } catch (Throwable t) {
      LOG.warn("Killing failed", t);
      Assert.fail();
    }

    LOG.info("Finished testDetachedMode()");
  }
Exemplo n.º 2
0
  /**
   * convert an AM report to a string for diagnostics
   *
   * @param report the report
   * @return the string value
   */
  public static String reportToString(ApplicationReport report) {
    if (report == null) {
      return "Null application report";
    }

    return "App "
        + report.getName()
        + "/"
        + report.getApplicationType()
        + "# "
        + report.getApplicationId()
        + " user "
        + report.getUser()
        + " is in state "
        + report.getYarnApplicationState()
        + "RPC: "
        + report.getHost()
        + ":"
        + report.getRpcPort();
  }
Exemplo n.º 3
0
 @Test(timeout = 20000)
 public void testJobSubmissionFailure() throws Exception {
   when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class)))
       .thenReturn(appId);
   ApplicationReport report = mock(ApplicationReport.class);
   when(report.getApplicationId()).thenReturn(appId);
   when(report.getDiagnostics()).thenReturn(failString);
   when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
   when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
   Credentials credentials = new Credentials();
   File jobxml = new File(testWorkDir, "job.xml");
   OutputStream out = new FileOutputStream(jobxml);
   conf.writeXml(out);
   out.close();
   try {
     yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
   } catch (IOException io) {
     LOG.info("Logging exception:", io);
     assertTrue(io.getLocalizedMessage().contains(failString));
   }
 }
Exemplo n.º 4
0
  @Before
  public void checkClusterEmpty() throws IOException, YarnException {
    if (yarnClient == null) {
      yarnClient = YarnClient.createYarnClient();
      yarnClient.init(yarnConfiguration);
      yarnClient.start();
    }

    List<ApplicationReport> apps = yarnClient.getApplications();
    for (ApplicationReport app : apps) {
      if (app.getYarnApplicationState() != YarnApplicationState.FINISHED
          && app.getYarnApplicationState() != YarnApplicationState.KILLED
          && app.getYarnApplicationState() != YarnApplicationState.FAILED) {
        Assert.fail(
            "There is at least one application on the cluster is not finished."
                + "App "
                + app.getApplicationId()
                + " is in state "
                + app.getYarnApplicationState());
      }
    }
  }
Exemplo n.º 5
0
  /**
   * Verify the web service deployment and lifecycle functionality
   *
   * @throws Exception
   */
  @Ignore // disabled due to web service init delay issue
  @Test
  public void testWebService() throws Exception {

    // single container topology of inline input and module
    Properties props = new Properties();
    props.put(
        StreamingApplication.DT_PREFIX + "stream.input.classname",
        TestGeneratorInputOperator.class.getName());
    props.put(StreamingApplication.DT_PREFIX + "stream.input.outputNode", "module1");
    props.put(
        StreamingApplication.DT_PREFIX + "module.module1.classname",
        GenericTestOperator.class.getName());

    LOG.info("Initializing Client");
    LogicalPlanConfiguration tb = new LogicalPlanConfiguration(new Configuration(false));
    tb.addFromProperties(props, null);

    StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb));
    if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
      client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
    }
    try {
      client.start();
      client.startApplication();

      // attempt web service connection
      ApplicationReport appReport = client.getApplicationReport();
      Thread.sleep(5000); // delay to give web service time to fully initialize
      Client wsClient = Client.create();
      wsClient.setFollowRedirects(true);
      WebResource r =
          wsClient
              .resource("http://" + appReport.getTrackingUrl())
              .path(StramWebServices.PATH)
              .path(StramWebServices.PATH_INFO);
      LOG.info("Requesting: " + r.getURI());
      ClientResponse response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      JSONObject json = response.getEntity(JSONObject.class);
      LOG.info("Got response: " + json.toString());
      assertEquals("incorrect number of elements", 1, json.length());
      assertEquals("appId", appReport.getApplicationId().toString(), json.get("id"));
      r =
          wsClient
              .resource("http://" + appReport.getTrackingUrl())
              .path(StramWebServices.PATH)
              .path(StramWebServices.PATH_PHYSICAL_PLAN_OPERATORS);
      LOG.info("Requesting: " + r.getURI());
      response = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
      assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
      json = response.getEntity(JSONObject.class);
      LOG.info("Got response: " + json.toString());

    } finally {
      // LOG.info("waiting...");
      // synchronized (this) {
      //  this.wait();
      // }
      // boolean result = client.monitorApplication();
      client.killApplication();
      client.stop();
    }
  }