Example #1
0
 public static void printDAGStatus(
     DAGClient dagClient,
     String[] vertexNames,
     boolean displayDAGCounters,
     boolean displayVertexCounters)
     throws IOException, TezException {
   Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
   DAGStatus dagStatus = dagClient.getDAGStatus((displayDAGCounters ? opts : null));
   Progress progress = dagStatus.getDAGProgress();
   double vProgressFloat = 0.0f;
   if (progress != null) {
     System.out.println("");
     System.out.println(
         "DAG: State: "
             + dagStatus.getState()
             + " Progress: "
             + (progress.getTotalTaskCount() < 0
                 ? formatter.format(0.0f)
                 : formatter.format(
                     (double) (progress.getSucceededTaskCount()) / progress.getTotalTaskCount())));
     for (String vertexName : vertexNames) {
       VertexStatus vStatus =
           dagClient.getVertexStatus(vertexName, (displayVertexCounters ? opts : null));
       if (vStatus == null) {
         System.out.println("Could not retrieve status for vertex: " + vertexName);
         continue;
       }
       Progress vProgress = vStatus.getProgress();
       if (vProgress != null) {
         vProgressFloat = 0.0f;
         if (vProgress.getTotalTaskCount() == 0) {
           vProgressFloat = 1.0f;
         } else if (vProgress.getTotalTaskCount() > 0) {
           vProgressFloat =
               (double) vProgress.getSucceededTaskCount() / vProgress.getTotalTaskCount();
         }
         System.out.println(
             "VertexStatus:"
                 + " VertexName: "
                 + (vertexName.equals("ivertex1") ? "intermediate-reducer" : vertexName)
                 + " Progress: "
                 + formatter.format(vProgressFloat));
       }
       if (displayVertexCounters) {
         TezCounters counters = vStatus.getVertexCounters();
         if (counters != null) {
           System.out.println("Vertex Counters for " + vertexName + ": " + counters);
         }
       }
     }
   }
   if (displayDAGCounters) {
     TezCounters counters = dagStatus.getDAGCounters();
     if (counters != null) {
       System.out.println("DAG Counters: " + counters);
     }
   }
 }
Example #2
0
  @Test(timeout = 10000)
  public void testBasicSpeculationPerVertexConf() throws Exception {
    DAG dag = DAG.create("test");
    String vNameNoSpec = "A";
    String vNameSpec = "B";
    Vertex vA = Vertex.create(vNameNoSpec, ProcessorDescriptor.create("Proc.class"), 5);
    Vertex vB = Vertex.create(vNameSpec, ProcessorDescriptor.create("Proc.class"), 5);
    vA.setConf(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, "false");
    dag.addVertex(vA);
    dag.addVertex(vB);
    // min/max src fraction is set to 1. So vertices will run sequentially
    dag.addEdge(
        Edge.create(
            vA,
            vB,
            EdgeProperty.create(
                DataMovementType.SCATTER_GATHER,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                OutputDescriptor.create("O"),
                InputDescriptor.create("I"))));

    MockTezClient tezClient = createTezSession();

    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = dagImpl.getVertex(vNameSpec).getVertexId();
    TezVertexID vertexIdNoSpec = dagImpl.getVertex(vNameNoSpec).getVertexId();
    // original attempt is killed and speculative one is successful
    TezTaskAttemptID killedTaId =
        TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TezTaskAttemptID noSpecTaId =
        TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexIdNoSpec, 0), 0);

    // cause speculation trigger for both
    mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
    mockLauncher.setStatusUpdatesForTask(noSpecTaId, 100);

    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    org.apache.tez.dag.app.dag.Vertex vSpec = dagImpl.getVertex(vertexId);
    org.apache.tez.dag.app.dag.Vertex vNoSpec = dagImpl.getVertex(vertexIdNoSpec);
    // speculation for vA but not for vB
    Assert.assertTrue(
        vSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue() > 0);
    Assert.assertEquals(
        0, vNoSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());

    tezClient.stop();
  }
Example #3
0
  public void testBasicSpeculation(boolean withProgress) throws Exception {
    DAG dag = DAG.create("test");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
    dag.addVertex(vA);

    MockTezClient tezClient = createTezSession();

    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
    // original attempt is killed and speculative one is successful
    TezTaskAttemptID killedTaId =
        TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TezTaskAttemptID successTaId =
        TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 1);

    mockLauncher.updateProgress(withProgress);
    // cause speculation trigger
    mockLauncher.setStatusUpdatesForTask(killedTaId, 100);

    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    Task task = dagImpl.getTask(killedTaId.getTaskID());
    Assert.assertEquals(2, task.getAttempts().size());
    Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getID());
    TaskAttempt killedAttempt = task.getAttempt(killedTaId);
    Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt");
    Assert.assertEquals(
        TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION,
        killedAttempt.getTerminationCause());
    if (withProgress) {
      // without progress updates occasionally more than 1 task speculates
      Assert.assertEquals(
          1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
      Assert.assertEquals(
          1, dagImpl.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
      org.apache.tez.dag.app.dag.Vertex v = dagImpl.getVertex(killedTaId.getTaskID().getVertexID());
      Assert.assertEquals(
          1, v.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
    }
    tezClient.stop();
  }
Example #4
0
  public int run(String[] args) throws Exception {

    if (args.length < 1) {
      System.err.println(
          "MRRSleepJob [-m numMapper] [-r numReducer]"
              + " [-ir numIntermediateReducer]"
              + " [-irs numIntermediateReducerStages]"
              + " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]"
              + " [-irt intermediateReduceSleepTime]"
              + " [-recordt recordSleepTime (msec)]"
              + " [-generateSplitsInAM (false)/true]"
              + " [-writeSplitsToDfs (false)/true]");
      ToolRunner.printGenericCommandUsage(System.err);
      return 2;
    }

    int numMapper = 1, numReducer = 1, numIReducer = 1;
    long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100, iReduceSleepTime = 1;
    int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
    int iReduceStagesCount = 1;
    boolean writeSplitsToDfs = false;
    boolean generateSplitsInAM = false;
    boolean splitsOptionFound = false;

    for (int i = 0; i < args.length; i++) {
      if (args[i].equals("-m")) {
        numMapper = Integer.parseInt(args[++i]);
      } else if (args[i].equals("-r")) {
        numReducer = Integer.parseInt(args[++i]);
      } else if (args[i].equals("-ir")) {
        numIReducer = Integer.parseInt(args[++i]);
      } else if (args[i].equals("-mt")) {
        mapSleepTime = Long.parseLong(args[++i]);
      } else if (args[i].equals("-rt")) {
        reduceSleepTime = Long.parseLong(args[++i]);
      } else if (args[i].equals("-irt")) {
        iReduceSleepTime = Long.parseLong(args[++i]);
      } else if (args[i].equals("-irs")) {
        iReduceStagesCount = Integer.parseInt(args[++i]);
      } else if (args[i].equals("-recordt")) {
        recSleepTime = Long.parseLong(args[++i]);
      } else if (args[i].equals("-generateSplitsInAM")) {
        if (splitsOptionFound) {
          throw new RuntimeException(
              "Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
        }
        splitsOptionFound = true;
        generateSplitsInAM = Boolean.parseBoolean(args[++i]);

      } else if (args[i].equals("-writeSplitsToDfs")) {
        if (splitsOptionFound) {
          throw new RuntimeException(
              "Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
        }
        splitsOptionFound = true;
        writeSplitsToDfs = Boolean.parseBoolean(args[++i]);
      }
    }

    if (numIReducer > 0 && numReducer <= 0) {
      throw new RuntimeException("Cannot have intermediate reduces without" + " a final reduce");
    }

    // sleep for *SleepTime duration in Task by recSleepTime per record
    mapSleepCount = (int) Math.ceil(mapSleepTime / ((double) recSleepTime));
    reduceSleepCount = (int) Math.ceil(reduceSleepTime / ((double) recSleepTime));
    iReduceSleepCount = (int) Math.ceil(iReduceSleepTime / ((double) recSleepTime));

    TezConfiguration conf = new TezConfiguration(getConf());
    FileSystem remoteFs = FileSystem.get(conf);

    conf.set(
        TezConfiguration.TEZ_AM_STAGING_DIR,
        conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));

    Path remoteStagingDir =
        remoteFs.makeQualified(
            new Path(
                conf.get(
                    TezConfiguration.TEZ_AM_STAGING_DIR,
                    TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
                Long.toString(System.currentTimeMillis())));
    TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);

    DAG dag =
        createDAG(
            remoteFs,
            conf,
            remoteStagingDir,
            numMapper,
            numReducer,
            iReduceStagesCount,
            numIReducer,
            mapSleepTime,
            mapSleepCount,
            reduceSleepTime,
            reduceSleepCount,
            iReduceSleepTime,
            iReduceSleepCount,
            writeSplitsToDfs,
            generateSplitsInAM);

    TezClient tezSession = new TezClient("MRRSleep", conf, false, null, credentials);
    tezSession.start();
    DAGClient dagClient = tezSession.submitDAG(dag);

    while (true) {
      DAGStatus status = dagClient.getDAGStatus(null);
      LOG.info("DAG Status: " + status);
      if (status.isCompleted()) {
        break;
      }
      try {
        Thread.sleep(1000);
      } catch (InterruptedException e) {
        // do nothing
      }
    }
    tezSession.stop();

    return dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED) ? 0 : 1;
  }
  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    // Configure intermediate reduces
    conf.setInt(MRJobConfig.MRR_INTERMEDIATE_STAGES, 1);

    // Set reducer class for intermediate reduce
    conf.setClass(
        MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduce.class"),
        MyGroupByReducer.class,
        Reducer.class);
    // Set reducer output key class
    conf.setClass(
        MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(
            1, "mapreduce.map.output.key.class"),
        IntWritable.class,
        Object.class);
    // Set reducer output value class
    conf.setClass(
        MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(
            1, "mapreduce.map.output.value.class"),
        Text.class,
        Object.class);
    conf.setInt(
        MultiStageMRConfigUtil.getPropertyNameForIntermediateStage(1, "mapreduce.job.reduces"), 2);

    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length != 2) {
      System.err.println("Usage: groupbyorderbymrrtest <in> <out>");
      System.exit(2);
    }

    @SuppressWarnings("deprecation")
    Job job = new Job(conf, "groupbyorderbymrrtest");

    job.setJarByClass(GroupByOrderByMRRTest.class);

    // Configure map
    job.setMapperClass(MyMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);

    // Configure reduce
    job.setReducerClass(MyOrderByNoOpReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    job.setNumReduceTasks(1);

    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

    TezClient tezClient = new TezClient(new TezConfiguration(conf));

    job.submit();
    JobID jobId = job.getJobID();
    ApplicationId appId = TypeConverter.toYarn(jobId).getAppId();

    DAGClient dagClient = tezClient.getDAGClient(appId);
    DAGStatus dagStatus = null;
    while (true) {
      dagStatus = dagClient.getDAGStatus();
      if (dagStatus.getState() == DAGStatus.State.RUNNING
          || dagStatus.getState() == DAGStatus.State.SUCCEEDED
          || dagStatus.getState() == DAGStatus.State.FAILED
          || dagStatus.getState() == DAGStatus.State.KILLED
          || dagStatus.getState() == DAGStatus.State.ERROR) {
        break;
      }
      try {
        Thread.sleep(500);
      } catch (InterruptedException e) {
        // continue;
      }
    }

    while (dagStatus.getState() == DAGStatus.State.RUNNING) {
      try {
        ExampleDriver.printMRRDAGStatus(dagStatus);
        try {
          Thread.sleep(1000);
        } catch (InterruptedException e) {
          // continue;
        }
        dagStatus = dagClient.getDAGStatus();
      } catch (TezException e) {
        LOG.fatal("Failed to get application progress. Exiting");
        System.exit(-1);
      }
    }

    ExampleDriver.printMRRDAGStatus(dagStatus);
    LOG.info("Application completed. " + "FinalState=" + dagStatus.getState());
    System.exit(dagStatus.getState() == DAGStatus.State.SUCCEEDED ? 0 : 1);
  }