コード例 #1
0
 private String getHistoryFile(Configuration conf, String jobId) throws IOException {
   String jtAddress = "scheme://" + conf.get("mapred.job.tracker");
   String jtHttpAddr = "scheme://" + conf.get("mapred.job.tracker.http.address");
   try {
     String host = new URI(jtAddress).getHost();
     int port = new URI(jtHttpAddr).getPort();
     HttpClient client = new HttpClient();
     String jobUrl = "http://" + host + ":" + port + "/jobdetails.jsp";
     GetMethod get = new GetMethod(jobUrl);
     get.setQueryString("jobid=" + jobId);
     get.setFollowRedirects(false);
     int status = client.executeMethod(get);
     String file = null;
     if (status == HttpStatus.SC_MOVED_PERMANENTLY || status == HttpStatus.SC_MOVED_TEMPORARILY) {
       file = get.getResponseHeader("Location").toString();
       file = file.substring(file.lastIndexOf('=') + 1);
       file = JobHistory.JobInfo.decodeJobHistoryFileName(file);
     } else {
       LOG.warn("JobURL {} for id: {} returned {}", jobUrl, jobId, status);
     }
     return file;
   } catch (URISyntaxException e) {
     throw new IOException("JT Address: " + jtAddress + ", http Address: " + jtHttpAddr, e);
   }
 }
コード例 #2
0
 @Override
 public String getFromHistory(String jobId) throws IOException {
   Configuration conf = getConf();
   String file = getHistoryFile(conf, jobId);
   if (file == null) return null;
   JobHistory.JobInfo jobInfo = new JobHistory.JobInfo(jobId);
   DefaultJobHistoryParser.parseJobTasks(file, jobInfo, new Path(file).getFileSystem(conf));
   LOG.info("History file: {}", file);
   LOG.debug("Number of tasks in the history file: {}", jobInfo.getAllTasks().size());
   for (JobHistory.Task task : jobInfo.getAllTasks().values()) {
     if (task.get(JobHistory.Keys.TASK_TYPE).equals(JobHistory.Values.MAP.name())
         && task.get(JobHistory.Keys.TASK_STATUS).equals(JobHistory.Values.SUCCESS.name())) {
       for (JobHistory.TaskAttempt attempt : task.getTaskAttempts().values()) {
         if (attempt.get(JobHistory.Keys.TASK_STATUS).equals(JobHistory.Values.SUCCESS.name())) {
           return JobHistory.getTaskLogsUrl(attempt);
         }
       }
     }
   }
   LOG.warn("Unable to find successful map task attempt");
   return null;
 }
コード例 #3
0
  public void test(
      String workflowId,
      String workflowName,
      String workflowNodeName,
      Map<String, String[]> adjacencies) {
    Configuration conf = new Configuration();
    setProperties(conf, workflowId, workflowName, workflowNodeName, adjacencies);
    String log =
        log(
            "JOB",
            new String[] {ID, NAME, NODE, ADJ},
            new String[] {
              conf.get(ID_PROP),
              conf.get(NAME_PROP),
              conf.get(NODE_PROP),
              JobHistory.JobInfo.getWorkflowAdjacencies(conf)
            });
    ParsedLine line = new ParsedLine(log);
    JobID jobid = new JobID("id", 1);
    JobSubmittedEvent event =
        new JobSubmittedEvent(
            jobid,
            workflowName,
            "",
            0l,
            "",
            null,
            "",
            line.get(ID),
            line.get(NAME),
            line.get(NODE),
            line.get(ADJ));
    WorkflowContext context = MapReduceJobHistoryUpdater.buildWorkflowContext(event);

    String resultingWorkflowId = workflowId;
    if (workflowId.isEmpty()) resultingWorkflowId = jobid.toString().replace("job_", "mr_");
    assertEquals("Didn't recover workflowId", resultingWorkflowId, context.getWorkflowId());
    assertEquals("Didn't recover workflowName", workflowName, context.getWorkflowName());
    assertEquals(
        "Didn't recover workflowNodeName", workflowNodeName, context.getWorkflowEntityName());

    Map<String, String[]> resultingAdjacencies = adjacencies;
    if (resultingAdjacencies.size() == 0) {
      resultingAdjacencies = new HashMap<String, String[]>();
      resultingAdjacencies.put(workflowNodeName, new String[] {});
    }
    assertEquals(
        "Got incorrect number of adjacencies",
        resultingAdjacencies.size(),
        context.getWorkflowDag().getEntries().size());
    for (WorkflowDagEntry entry : context.getWorkflowDag().getEntries()) {
      String[] sTargets = resultingAdjacencies.get(entry.getSource());
      assertNotNull("No original targets for " + entry.getSource(), sTargets);
      List<String> dTargets = entry.getTargets();
      assertEquals(
          "Got incorrect number of targets for " + entry.getSource(),
          sTargets.length,
          dTargets.size());
      for (int i = 0; i < sTargets.length; i++) {
        assertEquals("Got incorrect target for " + entry.getSource(), sTargets[i], dTargets.get(i));
      }
    }
  }