예제 #1
0
 @SuppressWarnings("resource")
 private void testDeletionofStagingOnUnregistrationFailure(
     int maxAttempts, boolean shouldHaveDeleted) throws IOException {
   conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
   fs = mock(FileSystem.class);
   when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
   // Staging Dir exists
   String user = UserGroupInformation.getCurrentUser().getShortUserName();
   Path stagingDir = MRApps.getStagingAreaDir(conf, user);
   when(fs.exists(stagingDir)).thenReturn(true);
   ApplicationId appId = ApplicationId.newInstance(0, 1);
   ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
   JobId jobid = recordFactory.newRecordInstance(JobId.class);
   jobid.setAppId(appId);
   TestMRApp appMaster = new TestMRApp(attemptId, null, JobStateInternal.RUNNING, maxAttempts);
   appMaster.crushUnregistration = true;
   appMaster.init(conf);
   appMaster.start();
   appMaster.shutDownJob();
   ((RunningAppContext) appMaster.getContext()).computeIsLastAMRetry();
   if (shouldHaveDeleted) {
     Assert.assertEquals(new Boolean(true), appMaster.isLastAMRetry());
     verify(fs).delete(stagingJobPath, true);
   } else {
     Assert.assertEquals(new Boolean(false), appMaster.isLastAMRetry());
     verify(fs, never()).delete(stagingJobPath, true);
   }
 }
예제 #2
0
 @Test
 public void testDeletionofStaging() throws IOException {
   conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
   fs = mock(FileSystem.class);
   when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
   // Staging Dir exists
   String user = UserGroupInformation.getCurrentUser().getShortUserName();
   Path stagingDir = MRApps.getStagingAreaDir(conf, user);
   when(fs.exists(stagingDir)).thenReturn(true);
   ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
   ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
   JobId jobid = recordFactory.newRecordInstance(JobId.class);
   jobid.setAppId(appId);
   ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
   Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
   MRAppMaster appMaster =
       new TestMRApp(
           attemptId, mockAlloc, JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
   appMaster.init(conf);
   appMaster.start();
   appMaster.shutDownJob();
   // test whether notifyIsLastAMRetry called
   Assert.assertEquals(true, ((TestMRApp) appMaster).getTestIsLastAMRetry());
   verify(fs).delete(stagingJobPath, true);
 }
예제 #3
0
 // mostly useful for parsing task/attempt id like strings
 public static JobId toJobID(String prefix, String s, Iterator<String> it) {
   ApplicationId appId = toAppID(prefix, s, it);
   shouldHaveNext(prefix, s, it);
   JobId jobId = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class);
   jobId.setAppId(appId);
   jobId.setId(Integer.parseInt(it.next()));
   return jobId;
 }
 private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries)
     throws IOException {
   conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
   conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
   MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
   ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
   try {
     when(rmDelegate.getApplicationReport(jobId.getAppId()))
         .thenThrow(
             new java.lang.reflect.UndeclaredThrowableException(
                 new IOException("Connection refuced1")))
         .thenThrow(
             new java.lang.reflect.UndeclaredThrowableException(
                 new IOException("Connection refuced2")))
         .thenThrow(
             new java.lang.reflect.UndeclaredThrowableException(
                 new IOException("Connection refuced3")));
     ClientServiceDelegate clientServiceDelegate =
         new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
     try {
       clientServiceDelegate.getJobStatus(oldJobId);
       Assert.fail("It should throw exception after retries");
     } catch (IOException e) {
       System.out.println("fail to get job status,and e=" + e.toString());
     }
     verify(rmDelegate, times(noOfRetries)).getApplicationReport(any(ApplicationId.class));
   } catch (YarnException e) {
     throw new IOException(e);
   }
 }
  @Test
  public void testRMDownRestoreForJobStatusBeforeGetAMReport() throws IOException {
    Configuration conf = new YarnConfiguration();
    conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);

    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(getJobReportResponse());
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced1")))
          .thenThrow(
              new java.lang.reflect.UndeclaredThrowableException(
                  new IOException("Connection refuced2")))
          .thenReturn(getFinishedApplicationReport());
      ClientServiceDelegate clientServiceDelegate =
          new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
      JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
      verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class));
      Assert.assertNotNull(jobStatus);
    } catch (YarnException e) {
      throw new IOException(e);
    }
  }
 public HistoryFileInfo addIfAbsent(HistoryFileInfo fileInfo) {
   JobId jobId = fileInfo.getJobId();
   if (LogGlobal.isDebugEnabled()) {
     /* LOG.debug("Adding "+jobId+" to job list cache with "+fileInfo.getJobIndexInfo()) */
     LOG.adding_job_list_cache_with(jobId.toString(), String.valueOf(fileInfo.getJobIndexInfo()))
         .tag("methodCall")
         .debug();
   }
   HistoryFileInfo old = cache.putIfAbsent(jobId, fileInfo);
   if (cache.size() > maxSize) {
     // There is a race here, where more then one thread could be trying to
     // remove entries.  This could result in too many entries being removed
     // from the cache.  This is considered OK as the size of the cache
     // should be rather large, and we would rather have performance over
     // keeping the cache size exactly at the maximum.
     Iterator<JobId> keys = cache.navigableKeySet().iterator();
     long cutoff = System.currentTimeMillis() - maxAge;
     while (cache.size() > maxSize && keys.hasNext()) {
       JobId key = keys.next();
       HistoryFileInfo firstValue = cache.get(key);
       if (firstValue != null) {
         synchronized (firstValue) {
           if (firstValue.isMovePending()) {
             if (firstValue.didMoveFail() && firstValue.jobIndexInfo.getFinishTime() <= cutoff) {
               cache.remove(key);
               // Now lets try to delete it
               try {
                 firstValue.delete();
               } catch (IOException e) {
                 /* LOG.error("Error while trying to delete history files"+" that could not be moved to done.",e) */
                 LOG.error_while_trying_delete_history_files_(e.toString()).error();
               }
             } else {
               /* LOG.warn("Waiting to remove "+key+" from JobListCache because it is not in done yet.") */
               LOG.waiting_remove_from_joblistcache_because(key.toString()).warn();
             }
           } else {
             cache.remove(key);
           }
         }
       }
     }
   }
   return old;
 }
 private ResourceMgrDelegate getRMDelegate() throws IOException {
   ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
   try {
     ApplicationId appId = jobId.getAppId();
     when(rm.getApplicationReport(appId))
         .thenThrow(new ApplicationNotFoundException(appId + " not found"));
   } catch (YarnException e) {
     throw new IOException(e);
   }
   return rm;
 }
예제 #8
0
 @Test(timeout = 30000)
 public void testDeletionofStagingOnKill() throws IOException {
   conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
   fs = mock(FileSystem.class);
   when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
   // Staging Dir exists
   String user = UserGroupInformation.getCurrentUser().getShortUserName();
   Path stagingDir = MRApps.getStagingAreaDir(conf, user);
   when(fs.exists(stagingDir)).thenReturn(true);
   ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
   ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
   JobId jobid = recordFactory.newRecordInstance(JobId.class);
   jobid.setAppId(appId);
   ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
   MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, 4);
   appMaster.init(conf);
   // simulate the process being killed
   MRAppMaster.MRAppMasterShutdownHook hook = new MRAppMaster.MRAppMasterShutdownHook(appMaster);
   hook.run();
   verify(fs, times(0)).delete(stagingJobPath, true);
 }
  @Test
  public void testAMAccessDisabled() throws IOException {
    // test only applicable when AM not reachable
    if (isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
    when(historyServerProxy.getJobReport(getJobReportRequest()))
        .thenReturn(getJobReportResponseFromHistoryServer());

    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getFinishedApplicationReport());
    } catch (YarnException e) {
      throw new IOException(e);
    }

    ClientServiceDelegate clientServiceDelegate =
        spy(getClientServiceDelegate(historyServerProxy, rmDelegate));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());

    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));

    // Should not reach AM even for second and third times too.
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("N/A", jobStatus.getJobName());
    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));

    // The third time around, app is completed, so should go to JHS
    JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus1);
    Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());
    Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());
    Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f);
    Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f);

    verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
  }
  @Test
  public void testHistoryServerNotConfigured() throws Exception {
    // RM doesn't have app report and job History Server is not configured
    ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(null, getRMDelegate());
    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals("N/A", jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());

    // RM has app report and job History Server is not configured
    ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
    ApplicationReport applicationReport = getFinishedApplicationReport();
    when(rm.getApplicationReport(jobId.getAppId())).thenReturn(applicationReport);

    clientServiceDelegate = getClientServiceDelegate(null, rm);
    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
    Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
  }
  @Test
  public void testReconnectOnAMRestart() throws IOException {
    // test not applicable when AM not reachable
    // as instantiateAMProxy is not called at all
    if (!isAMReachableFromClient) {
      return;
    }

    MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);

    // RM returns AM1 url, null, null and AM2 url on invocations.
    // Nulls simulate the time when AM2 is in the process of restarting.
    ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
    try {
      when(rmDelegate.getApplicationReport(jobId.getAppId()))
          .thenReturn(getRunningApplicationReport("am1", 78))
          .thenReturn(getRunningApplicationReport(null, 0))
          .thenReturn(getRunningApplicationReport(null, 0))
          .thenReturn(getRunningApplicationReport("am2", 90));
    } catch (YarnException e) {
      throw new IOException(e);
    }

    GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
    when(jobReportResponse1.getJobReport())
        .thenReturn(
            MRBuilderUtils.newJobReport(
                jobId,
                "jobName-firstGen",
                "user",
                JobState.RUNNING,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                "anything",
                null,
                false,
                ""));

    // First AM returns a report with jobName firstGen and simulates AM shutdown
    // on second invocation.
    MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
    when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse1)
        .thenThrow(new RuntimeException("AM is down!"));

    GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
    when(jobReportResponse2.getJobReport())
        .thenReturn(
            MRBuilderUtils.newJobReport(
                jobId,
                "jobName-secondGen",
                "user",
                JobState.RUNNING,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                "anything",
                null,
                false,
                ""));

    // Second AM generation returns a report with jobName secondGen
    MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
    when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
        .thenReturn(jobReportResponse2);

    ClientServiceDelegate clientServiceDelegate =
        spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
    // First time, connection should be to AM1, then to AM2. Further requests
    // should use the same proxy to AM2 and so instantiateProxy shouldn't be
    // called.
    doReturn(firstGenAMProxy)
        .doReturn(secondGenAMProxy)
        .when(clientServiceDelegate)
        .instantiateAMProxy(any(InetSocketAddress.class));

    JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
    Assert.assertNotNull(jobStatus);
    Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());

    verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
  }
예제 #12
0
 public static String toString(JobId jid) {
   return _join(JOB, jid.getAppId().getClusterTimestamp(), jid.getAppId().getId(), jid.getId());
 }
예제 #13
0
  private void checkHistoryParsing(
      final int numMaps, final int numReduces, final int numSuccessfulMaps) throws Exception {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
    long amStartTimeEst = System.currentTimeMillis();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class,
        DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(numMaps, numReduces, true, this.getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    String jobhistoryDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);

    FileContext fc = null;
    try {
      fc = FileContext.getFileContext(conf);
    } catch (IOException ioe) {
      LOG.info("Can not get FileContext", ioe);
      throw (new Exception("Can not get File Context"));
    }

    if (numMaps == numSuccessfulMaps) {
      String summaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(jobId);
      Path summaryFile = new Path(jobhistoryDir, summaryFileName);
      String jobSummaryString = getJobSummary(fc, summaryFile);
      Assert.assertNotNull(jobSummaryString);
      Assert.assertTrue(jobSummaryString.contains("resourcesPerMap=100"));
      Assert.assertTrue(jobSummaryString.contains("resourcesPerReduce=100"));

      Map<String, String> jobSummaryElements = new HashMap<String, String>();
      StringTokenizer strToken = new StringTokenizer(jobSummaryString, ",");
      while (strToken.hasMoreTokens()) {
        String keypair = strToken.nextToken();
        jobSummaryElements.put(keypair.split("=")[0], keypair.split("=")[1]);
      }

      Assert.assertEquals(
          "JobId does not match", jobId.toString(), jobSummaryElements.get("jobId"));
      Assert.assertEquals("JobName does not match", "test", jobSummaryElements.get("jobName"));
      Assert.assertTrue(
          "submitTime should not be 0", Long.parseLong(jobSummaryElements.get("submitTime")) != 0);
      Assert.assertTrue(
          "launchTime should not be 0", Long.parseLong(jobSummaryElements.get("launchTime")) != 0);
      Assert.assertTrue(
          "firstMapTaskLaunchTime should not be 0",
          Long.parseLong(jobSummaryElements.get("firstMapTaskLaunchTime")) != 0);
      Assert.assertTrue(
          "firstReduceTaskLaunchTime should not be 0",
          Long.parseLong(jobSummaryElements.get("firstReduceTaskLaunchTime")) != 0);
      Assert.assertTrue(
          "finishTime should not be 0", Long.parseLong(jobSummaryElements.get("finishTime")) != 0);
      Assert.assertEquals(
          "Mismatch in num map slots",
          numSuccessfulMaps,
          Integer.parseInt(jobSummaryElements.get("numMaps")));
      Assert.assertEquals(
          "Mismatch in num reduce slots",
          numReduces,
          Integer.parseInt(jobSummaryElements.get("numReduces")));
      Assert.assertEquals(
          "User does not match", System.getProperty("user.name"), jobSummaryElements.get("user"));
      Assert.assertEquals("Queue does not match", "default", jobSummaryElements.get("queue"));
      Assert.assertEquals("Status does not match", "SUCCEEDED", jobSummaryElements.get("status"));
    }

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);
    HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
    JobInfo jobInfo;
    long numFinishedMaps;

    synchronized (fileInfo) {
      Path historyFilePath = fileInfo.getHistoryFile();
      FSDataInputStream in = null;
      LOG.info("JobHistoryFile is: " + historyFilePath);
      try {
        in = fc.open(fc.makeQualified(historyFilePath));
      } catch (IOException ioe) {
        LOG.info("Can not open history file: " + historyFilePath, ioe);
        throw (new Exception("Can not open History File"));
      }

      JobHistoryParser parser = new JobHistoryParser(in);
      final EventReader realReader = new EventReader(in);
      EventReader reader = Mockito.mock(EventReader.class);
      if (numMaps == numSuccessfulMaps) {
        reader = realReader;
      } else {
        final AtomicInteger numFinishedEvents = new AtomicInteger(0); // Hack!
        Mockito.when(reader.getNextEvent())
            .thenAnswer(
                new Answer<HistoryEvent>() {
                  public HistoryEvent answer(InvocationOnMock invocation) throws IOException {
                    HistoryEvent event = realReader.getNextEvent();
                    if (event instanceof TaskFinishedEvent) {
                      numFinishedEvents.incrementAndGet();
                    }

                    if (numFinishedEvents.get() <= numSuccessfulMaps) {
                      return event;
                    } else {
                      throw new IOException("test");
                    }
                  }
                });
      }

      jobInfo = parser.parse(reader);

      numFinishedMaps = computeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);

      if (numFinishedMaps != numMaps) {
        Exception parseException = parser.getParseException();
        Assert.assertNotNull("Didn't get expected parse exception", parseException);
      }
    }

    Assert.assertEquals(
        "Incorrect username ", System.getProperty("user.name"), jobInfo.getUsername());
    Assert.assertEquals("Incorrect jobName ", "test", jobInfo.getJobname());
    Assert.assertEquals("Incorrect queuename ", "default", jobInfo.getJobQueueName());
    Assert.assertEquals("incorrect conf path", "test", jobInfo.getJobConfPath());
    Assert.assertEquals("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps);
    Assert.assertEquals("incorrect finishedReduces ", numReduces, jobInfo.getFinishedReduces());
    Assert.assertEquals("incorrect uberized ", job.isUber(), jobInfo.getUberized());
    Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
    int totalTasks = allTasks.size();
    Assert.assertEquals("total number of tasks is incorrect  ", (numMaps + numReduces), totalTasks);

    // Verify aminfo
    Assert.assertEquals(1, jobInfo.getAMInfos().size());
    Assert.assertEquals(MRApp.NM_HOST, jobInfo.getAMInfos().get(0).getNodeManagerHost());
    AMInfo amInfo = jobInfo.getAMInfos().get(0);
    Assert.assertEquals(MRApp.NM_PORT, amInfo.getNodeManagerPort());
    Assert.assertEquals(MRApp.NM_HTTP_PORT, amInfo.getNodeManagerHttpPort());
    Assert.assertEquals(1, amInfo.getAppAttemptId().getAttemptId());
    Assert.assertEquals(
        amInfo.getAppAttemptId(), amInfo.getContainerId().getApplicationAttemptId());
    Assert.assertTrue(
        amInfo.getStartTime() <= System.currentTimeMillis()
            && amInfo.getStartTime() >= amStartTimeEst);

    ContainerId fakeCid = BuilderUtils.newContainerId(-1, -1, -1, -1);
    // Assert at taskAttempt level
    for (TaskInfo taskInfo : allTasks.values()) {
      int taskAttemptCount = taskInfo.getAllTaskAttempts().size();
      Assert.assertEquals("total number of task attempts ", 1, taskAttemptCount);
      TaskAttemptInfo taInfo = taskInfo.getAllTaskAttempts().values().iterator().next();
      Assert.assertNotNull(taInfo.getContainerId());
      // Verify the wrong ctor is not being used. Remove after mrv1 is removed.
      Assert.assertFalse(taInfo.getContainerId().equals(fakeCid));
    }

    // Deep compare Job and JobInfo
    for (Task task : job.getTasks().values()) {
      TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
      Assert.assertNotNull("TaskInfo not found", taskInfo);
      for (TaskAttempt taskAttempt : task.getAttempts().values()) {
        TaskAttemptInfo taskAttemptInfo =
            taskInfo.getAllTaskAttempts().get(TypeConverter.fromYarn((taskAttempt.getID())));
        Assert.assertNotNull("TaskAttemptInfo not found", taskAttemptInfo);
        Assert.assertEquals(
            "Incorrect shuffle port for task attempt",
            taskAttempt.getShufflePort(),
            taskAttemptInfo.getShufflePort());
        if (numMaps == numSuccessfulMaps) {
          Assert.assertEquals(MRApp.NM_HOST, taskAttemptInfo.getHostname());
          Assert.assertEquals(MRApp.NM_PORT, taskAttemptInfo.getPort());

          // Verify rack-name
          Assert.assertEquals("rack-name is incorrect", taskAttemptInfo.getRackname(), RACK_NAME);
        }
      }
    }
  }
    private synchronized void moveToDone() throws IOException {
      if (LogGlobal.isDebugEnabled()) {
        /* LOG.debug("moveToDone: "+historyFile) */
        LOG.movetodone(historyFile.toString()).debug();
      }
      if (!isMovePending()) {
        // It was either deleted or is already in done. Either way do nothing
        if (LogGlobal.isDebugEnabled()) {
          /* LOG.debug("Move no longer pending") */
          LOG.move_longer_pending().debug();
        }
        return;
      }
      try {
        long completeTime = jobIndexInfo.getFinishTime();
        if (completeTime == 0) {
          completeTime = System.currentTimeMillis();
        }
        JobId jobId = jobIndexInfo.getJobId();

        List<Path> paths = new ArrayList<Path>(2);
        if (historyFile == null) {
          /* LOG.info("No file for job-history with "+jobId+" found in cache!") */
          LOG.file_for_job_history_with_found(jobId.toString()).info();
        } else {
          paths.add(historyFile);
        }

        if (confFile == null) {
          /* LOG.info("No file for jobConf with "+jobId+" found in cache!") */
          LOG.file_for_jobconf_with_found_cache(jobId.toString()).info();
        } else {
          paths.add(confFile);
        }

        if (summaryFile == null) {
          /* LOG.info("No summary file for job: "+jobId) */
          LOG.summary_file_for_job(jobId.toString()).info();
        } else {
          String jobSummaryString = getJobSummary(intermediateDoneDirFc, summaryFile);
          LOG.summary_file_for_job(jobSummaryString);
          /* LOG.info("Deleting JobSummary file: ["+summaryFile+"]") */
          LOG.deleting_jobsummary_file(summaryFile.toString()).info();
          intermediateDoneDirFc.delete(summaryFile, false);
          summaryFile = null;
        }

        Path targetDir = canonicalHistoryLogPath(jobId, completeTime);
        addDirectoryToSerialNumberIndex(targetDir);
        makeDoneSubdir(targetDir);
        if (historyFile != null) {
          Path toPath = doneDirFc.makeQualified(new Path(targetDir, historyFile.getName()));
          if (!toPath.equals(historyFile)) {
            moveToDoneNow(historyFile, toPath);
            historyFile = toPath;
          }
        }
        if (confFile != null) {
          Path toPath = doneDirFc.makeQualified(new Path(targetDir, confFile.getName()));
          if (!toPath.equals(confFile)) {
            moveToDoneNow(confFile, toPath);
            confFile = toPath;
          }
        }
        state = HistoryInfoState.IN_DONE;
      } catch (Throwable t) {
        /* LOG.error("Error while trying to move a job to done",t) */
        LOG.error_while_trying_move_job_done(t.toString()).error();
        this.state = HistoryInfoState.MOVE_FAILED;
      }
    }
예제 #15
0
 /**
  * Computes a serial number used as part of directory naming for the given jobId.
  *
  * @param id the jobId.
  * @return the serial number used as part of directory naming for the given jobid
  */
 public static int jobSerialNumber(JobId id) {
   return id.getId();
 }