public AddJobFlowStepsResult unmarshall(JsonUnmarshallerContext context) throws Exception {
    AddJobFlowStepsResult addJobFlowStepsResult = new AddJobFlowStepsResult();

    int originalDepth = context.getCurrentDepth();
    String currentParentElement = context.getCurrentParentElement();
    int targetDepth = originalDepth + 1;

    JsonToken token = context.getCurrentToken();
    if (token == null) token = context.nextToken();
    if (token == VALUE_NULL) return null;

    while (true) {
      if (token == null) break;

      if (token == FIELD_NAME || token == START_OBJECT) {
        if (context.testExpression("StepIds", targetDepth)) {
          context.nextToken();
          addJobFlowStepsResult.setStepIds(
              new ListUnmarshaller<String>(StringJsonUnmarshaller.getInstance())
                  .unmarshall(context));
        }
      } else if (token == END_ARRAY || token == END_OBJECT) {
        if (context.getLastParsedParentElement() == null
            || context.getLastParsedParentElement().equals(currentParentElement)) {
          if (context.getCurrentDepth() <= originalDepth) break;
        }
      }

      token = context.nextToken();
    }

    return addJobFlowStepsResult;
  }
예제 #2
0
  public void createClusterWithStep(String inputPath, String outputPath, String logBucket) {

    String logUri = "s3://" + logBucket + "/";

    RunJobFlowRequest request =
        new RunJobFlowRequest()
            .withName("Create cluster with ReleaseLabel")
            .withReleaseLabel("emr-4.2.0")
            // .withSteps(customStep)
            .withServiceRole("EMR_DefaultRole")
            .withJobFlowRole("EMR_EC2_DefaultRole")
            .withInstances(
                new JobFlowInstancesConfig()
                    .withInstanceCount(3)
                    .withKeepJobFlowAliveWhenNoSteps(true)
                    .withMasterInstanceType("m3.xlarge")
                    .withSlaveInstanceType("m3.xlarge"))
            .withVisibleToAllUsers(true)
            .withLogUri(logUri)
            .withLogUri(logUri);

    RunJobFlowResult result = emr.runJobFlow(request);

    // --------------------------------------------------------------------------

    ClusterSummary cluster = emr.listClusters().getClusters().get(0);

    HadoopJarStepConfig hadoopConfigAdd =
        new HadoopJarStepConfig()
            .withJar("s3://hadoop-lyy/code/hadoop-mapreduce-examples-2.6.0.jar")
            .withMainClass("wordcount")
            .withArgs("-Dfs.s3.canned.acl=BucketOwnerFullControl", inputPath, outputPath);

    StepConfig customStepAdd =
        new StepConfig("Step2", hadoopConfigAdd).withActionOnFailure(ActionOnFailure.CONTINUE);
    AddJobFlowStepsResult resultAdd =
        emr.addJobFlowSteps(
            new AddJobFlowStepsRequest().withJobFlowId(cluster.getId()).withSteps(customStepAdd));

    System.out.println(resultAdd.getStepIds());

    //

    DescribeStepRequest describe =
        new DescribeStepRequest().withStepId(resultAdd.getStepIds().get(0));

    describe.setClusterId(cluster.getId());
    describe.setRequestCredentials(credentials);

    DescribeStepResult res = emr.describeStep(describe);
    StepStatus status = res.getStep().getStatus();
    String stas = status.getState();

    while (stas.equals(StepExecutionState.PENDING.name())
        || stas.equals(StepExecutionState.RUNNING.name())) {
      try {
        Thread.sleep(5000);
        res = emr.describeStep(describe);
        status = res.getStep().getStatus();
        stas = status.getState();
        System.out.print(".");
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }

    if (stas.equals(StepExecutionState.COMPLETED.name())) {
      System.out.println("\n step complete");
    } else if (stas.equals(StepExecutionState.FAILED.name())
        || stas.equals(StepExecutionState.CANCELLED.name())) {
      System.out.println("\n step failed");
    }
  }