Ejemplo n.º 1
0
 public static Path getPreviousJobHistoryPath(
     Configuration conf, ApplicationAttemptId applicationAttemptId) throws IOException {
   String jobId = TypeConverter.fromYarn(applicationAttemptId.getApplicationId()).toString();
   String jobhistoryDir = JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId);
   Path histDirPath = FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir));
   FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf);
   return fc.makeQualified(
       JobHistoryUtils.getStagingJobHistoryFile(
           histDirPath, jobId, (applicationAttemptId.getAttemptId() - 1)));
 }
Ejemplo n.º 2
0
  public HdfsDirectory(Path hdfsDirPath, LockFactory lockFactory, Configuration configuration)
      throws IOException {
    super(lockFactory);
    this.hdfsDirPath = hdfsDirPath;
    this.configuration = configuration;
    fileSystem = FileSystem.get(hdfsDirPath.toUri(), configuration);
    fileContext = FileContext.getFileContext(hdfsDirPath.toUri(), configuration);

    if (fileSystem instanceof DistributedFileSystem) {
      // Make sure dfs is not in safe mode
      while (((DistributedFileSystem) fileSystem).setSafeMode(SafeModeAction.SAFEMODE_GET, true)) {
        LOG.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
        try {
          Thread.sleep(5000);
        } catch (InterruptedException e) {
          Thread.interrupted();
          // continue
        }
      }
    }

    try {
      if (!fileSystem.exists(hdfsDirPath)) {
        boolean success = fileSystem.mkdirs(hdfsDirPath);
        if (!success) {
          throw new RuntimeException("Could not create directory: " + hdfsDirPath);
        }
      }
    } catch (Exception e) {
      org.apache.solr.common.util.IOUtils.closeQuietly(fileSystem);
      throw new RuntimeException("Problem creating directory: " + hdfsDirPath, e);
    }
  }
Ejemplo n.º 3
0
 protected static void printResults(PrintStream out) throws UnsupportedFileSystemException {
   out.println(
       "Result of running LoadGenerator against fileSystem: "
           + FileContext.getFileContext().getDefaultFileSystem().getUri());
   if (numOfOps[OPEN] != 0) {
     out.println(
         "Average open execution time: " + (double) executionTime[OPEN] / numOfOps[OPEN] + "ms");
   }
   if (numOfOps[LIST] != 0) {
     out.println(
         "Average list execution time: " + (double) executionTime[LIST] / numOfOps[LIST] + "ms");
   }
   if (numOfOps[DELETE] != 0) {
     out.println(
         "Average deletion execution time: "
             + (double) executionTime[DELETE] / numOfOps[DELETE]
             + "ms");
     out.println(
         "Average create execution time: "
             + (double) executionTime[CREATE] / numOfOps[CREATE]
             + "ms");
     out.println(
         "Average write_close execution time: "
             + (double) executionTime[WRITE_CLOSE] / numOfOps[WRITE_CLOSE]
             + "ms");
   }
   if (totalTime != 0) {
     out.println("Average operations per second: " + (double) totalOps / totalTime + "ops/s");
   }
   out.println();
 }
Ejemplo n.º 4
0
 public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException {
   FileContext fileContext = FileContext.getFileContext(conf);
   this.fsDataIStream = fileContext.open(remoteAppLogFile);
   reader =
       new TFile.Reader(
           this.fsDataIStream, fileContext.getFileStatus(remoteAppLogFile).getLen(), conf);
   this.scanner = reader.createScanner();
 }
Ejemplo n.º 5
0
  /**
   * Read a script file of the form: lines of text with duration in seconds, read probability and
   * write probability, separated by white space.
   *
   * @param filename Script file
   * @return 0 if successful, -1 if not
   * @throws IOException if errors with file IO
   */
  protected static int loadScriptFile(String filename, boolean readLocally) throws IOException {

    FileContext fc;
    if (readLocally) { // read locally - program is run without MR
      fc = FileContext.getLocalFSFileContext();
    } else {
      fc = FileContext.getFileContext(); // use default file system
    }
    DataInputStream in = null;
    try {
      in = fc.open(new Path(filename));
    } catch (IOException e) {
      System.err.println("Unable to open scriptFile: " + filename);

      System.exit(-1);
    }
    InputStreamReader inr = new InputStreamReader(in);

    BufferedReader br = new BufferedReader(inr);
    ArrayList<Long> duration = new ArrayList<Long>();
    ArrayList<Double> readProb = new ArrayList<Double>();
    ArrayList<Double> writeProb = new ArrayList<Double>();
    int lineNum = 0;

    String line;
    // Read script, parse values, build array of duration, read and write probs

    try {
      while ((line = br.readLine()) != null) {
        lineNum++;
        if (line.startsWith("#") || line.isEmpty()) // skip comments and blanks
        continue;

        parseScriptLine(line, duration, readProb, writeProb);
      }
    } catch (IllegalArgumentException e) {
      System.err.println("Line: " + lineNum + ", " + e.getMessage());
      return -1;
    } finally {
      IOUtils.cleanup(LOG, br);
    }

    // Copy vectors to arrays of values, to avoid autoboxing overhead later
    durations = new long[duration.size()];
    readProbs = new double[readProb.size()];
    writeProbs = new double[writeProb.size()];

    for (int i = 0; i < durations.length; i++) {
      durations[i] = duration.get(i);
      readProbs[i] = readProb.get(i);
      writeProbs[i] = writeProb.get(i);
    }

    if (durations[0] == 0)
      System.err.println("Initial duration set to 0.  " + "Will loop until stopped manually.");

    return 0;
  }
  // Equivalence of @Before for cluster mode testing.
  private void initClusterModeTest() throws IOException {

    LOG = LogFactory.getLog(TestWriteRead.class);
    LOG.info("initClusterModeTest");

    conf = new Configuration();
    mfc = FileContext.getFileContext();
    mfs = FileSystem.get(conf);
  }
Ejemplo n.º 7
0
 /**
  * Main function called by tool runner. It first initializes data by parsing the command line
  * arguments. It then calls the loadGenerator
  */
 @Override
 public int run(String[] args) throws Exception {
   int exitCode = parseArgs(false, args);
   if (exitCode != 0) {
     return exitCode;
   }
   System.out.println(
       "Running LoadGenerator against fileSystem: "
           + FileContext.getFileContext().getDefaultFileSystem().getUri());
   exitCode = generateLoadOnNN();
   printResults(System.out);
   return exitCode;
 }
Ejemplo n.º 8
0
  /** Creates a LocalResource instance for the JAR file referenced by the given Path. */
  public LocalResource getLocalResourceForPath(final Path jarPath, final LocalResourceType type)
      throws IOException {

    final FileStatus status =
        FileContext.getFileContext(fileSystem.getUri()).getFileStatus(jarPath);

    final LocalResource localResource = Records.newRecord(LocalResource.class);
    localResource.setType(type);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
    localResource.setTimestamp(status.getModificationTime());
    localResource.setSize(status.getLen());

    return localResource;
  }
  @Before
  public void initJunitModeTest() throws Exception {
    LOG.info("initJunitModeTest");

    conf = new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); // 100K
    // blocksize

    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    cluster.waitActive();

    mfs = cluster.getFileSystem();
    mfc = FileContext.getFileContext();

    Path rootdir = new Path(ROOT_DIR);
    mfs.mkdirs(rootdir);
  }
  @Override
  protected void serviceInit(Configuration conf) throws Exception {
    this.conf = conf;

    int serialNumberLowDigits = 3;
    serialNumberFormat =
        ("%0" + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits) + "d");

    String doneDirPrefix = null;
    doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    try {
      doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
      doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
      doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
      mkdir(
          doneDirFc,
          doneDirPrefixPath,
          new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
    } catch (IOException e) {
      throw new YarnRuntimeException(
          "Error creating done directory: [" + doneDirPrefixPath + "]", e);
    }

    String intermediateDoneDirPrefix = null;
    intermediateDoneDirPrefix = JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
    try {
      intermediateDoneDirPath =
          FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
      intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
      mkdir(
          intermediateDoneDirFc,
          intermediateDoneDirPath,
          new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
    } catch (IOException e) {
      LOG.info("error creating done directory on dfs " + e);
      throw new YarnRuntimeException(
          "Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
    }

    this.aclsMgr = new JobACLsManager(conf);

    maxHistoryAge =
        conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);

    jobListCache =
        new JobListCache(
            conf.getInt(
                JHAdminConfig.MR_HISTORY_JOBLIST_CACHE_SIZE,
                JHAdminConfig.DEFAULT_MR_HISTORY_JOBLIST_CACHE_SIZE),
            maxHistoryAge);

    serialNumberIndex =
        new SerialNumberIndex(
            conf.getInt(
                JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
                JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

    int numMoveThreads =
        conf.getInt(
            JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
            JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
    ThreadFactory tf =
        new ThreadFactoryBuilder().setNameFormat("MoveIntermediateToDone Thread #%d").build();
    moveToDoneExecutor =
        new ThreadPoolExecutor(
            numMoveThreads,
            numMoveThreads,
            1,
            TimeUnit.HOURS,
            new LinkedBlockingQueue<Runnable>(),
            tf);

    super.serviceInit(conf);
  }
 public synchronized Configuration loadConfFile() throws IOException {
   FileContext fc = FileContext.getFileContext(confFile.toUri(), conf);
   Configuration jobConf = new Configuration(false);
   jobConf.addResource(fc.open(confFile), confFile.toString());
   return jobConf;
 }
Ejemplo n.º 12
0
  @Override
  public void serviceInit(Configuration conf) throws Exception {
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
    // blacklisting disabled to prevent scheduling issues
    conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
    if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
      conf.set(
          MRJobConfig.MR_AM_STAGING_DIR,
          new File(getTestWorkDir(), "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
    }

    if (conf.get(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC) == null) {
      // nothing defined. set quick delete value
      conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
    }

    File appJarLocalFile = new File(MiniTezCluster.APPJAR);

    if (!appJarLocalFile.exists()) {
      String message = "TezAppJar " + MiniTezCluster.APPJAR + " not found. Exiting.";
      LOG.info(message);
      throw new TezUncheckedException(message);
    }

    FileSystem fs = FileSystem.get(conf);
    Path testRootDir = fs.makeQualified(new Path("target", getName() + "-tmpDir"));
    Path appRemoteJar = new Path(testRootDir, "TezAppJar.jar");
    // Copy AppJar and make it public.
    Path appMasterJar = new Path(MiniTezCluster.APPJAR);
    fs.copyFromLocalFile(appMasterJar, appRemoteJar);
    fs.setPermission(appRemoteJar, new FsPermission("777"));

    conf.set(TezConfiguration.TEZ_LIB_URIS, appRemoteJar.toUri().toString());
    LOG.info("Set TEZ-LIB-URI to: " + conf.get(TezConfiguration.TEZ_LIB_URIS));

    // VMEM monitoring disabled, PMEM monitoring enabled.
    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);

    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");

    try {
      Path stagingPath =
          FileContext.getFileContext(conf)
              .makeQualified(new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
      /*
       * Re-configure the staging path on Windows if the file system is localFs.
       * We need to use a absolute path that contains the drive letter. The unit
       * test could run on a different drive than the AM. We can run into the
       * issue that job files are localized to the drive where the test runs on,
       * while the AM starts on a different drive and fails to find the job
       * metafiles. Using absolute path can avoid this ambiguity.
       */
      if (Path.WINDOWS) {
        if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
          conf.set(
              MRJobConfig.MR_AM_STAGING_DIR,
              new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR)).getAbsolutePath());
        }
      }
      FileContext fc = FileContext.getFileContext(stagingPath.toUri(), conf);
      if (fc.util().exists(stagingPath)) {
        LOG.info(stagingPath + " exists! deleting...");
        fc.delete(stagingPath, true);
      }
      LOG.info("mkdir: " + stagingPath);
      fc.mkdir(stagingPath, null, true);

      // mkdir done directory as well
      String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
      Path doneDirPath = fc.makeQualified(new Path(doneDir));
      fc.mkdir(doneDirPath, null, true);
    } catch (IOException e) {
      throw new TezUncheckedException("Could not create staging directory. ", e);
    }
    conf.set(MRConfig.MASTER_ADDRESS, "test");

    // configure the shuffle service in NM
    conf.setStrings(
        YarnConfiguration.NM_AUX_SERVICES,
        new String[] {ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID});
    conf.setClass(
        String.format(
            YarnConfiguration.NM_AUX_SERVICE_FMT, ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID),
        ShuffleHandler.class,
        Service.class);

    // Non-standard shuffle port
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

    conf.setClass(
        YarnConfiguration.NM_CONTAINER_EXECUTOR,
        DefaultContainerExecutor.class,
        ContainerExecutor.class);

    // TestMRJobs is for testing non-uberized operation only; see TestUberAM
    // for corresponding uberized tests.
    conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
    super.serviceInit(conf);
  }
Ejemplo n.º 13
0
  /**
   * This is the main function - run threads to generate load on NN It starts the number of
   * DFSClient threads as specified by the user. It stops all the threads when the specified elapsed
   * time is passed.
   */
  protected int generateLoadOnNN() throws InterruptedException {
    int hostHashCode = hostname.hashCode();
    if (seed == 0) {
      r = new Random(System.currentTimeMillis() + hostHashCode);
    } else {
      r = new Random(seed + hostHashCode);
    }
    try {
      fc = FileContext.getFileContext(getConf());
    } catch (IOException ioe) {
      System.err.println("Can not initialize the file system: " + ioe.getLocalizedMessage());
      return -1;
    }

    int status = initFileDirTables();
    if (status != 0) {
      return status;
    }
    barrier();

    DFSClientThread[] threads = new DFSClientThread[numOfThreads];
    for (int i = 0; i < numOfThreads; i++) {
      threads[i] = new DFSClientThread(i);
      threads[i].start();
    }

    if (durations[0] > 0) {
      if (durations.length == 1) { // There is a fixed run time
        while (shouldRun) {
          Thread.sleep(2000);
          totalTime += 2;
          if (totalTime >= durations[0] || stopFileCreated()) {
            shouldRun = false;
          }
        }
      } else {
        // script run

        while (shouldRun) {
          Thread.sleep(durations[currentIndex] * 1000);
          totalTime += durations[currentIndex];
          // Are we on the final line of the script?
          if ((currentIndex + 1) == durations.length || stopFileCreated()) {
            shouldRun = false;
          } else {
            if (LOG.isDebugEnabled()) {
              LOG.debug(
                  "Moving to index "
                      + currentIndex
                      + ": r = "
                      + readProbs[currentIndex]
                      + ", w = "
                      + writeProbs
                      + " for duration "
                      + durations[currentIndex]);
            }
            currentIndex++;
          }
        }
      }
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("Done with testing.  Waiting for threads to finish.");
    }

    boolean failed = false;
    for (DFSClientThread thread : threads) {
      thread.join();
      for (int i = 0; i < TOTAL_OP_TYPES; i++) {
        executionTime[i] += thread.executionTime[i];
        numOfOps[i] += thread.totalNumOfOps[i];
      }
      failed = failed || thread.failed;
    }
    int exitCode = 0;
    if (failed) {
      exitCode = -ERR_TEST_FAILED;
    }

    totalOps = 0;
    for (int i = 0; i < TOTAL_OP_TYPES; i++) {
      totalOps += numOfOps[i];
    }
    return exitCode;
  }
Ejemplo n.º 14
0
  @Test
  public void testHistoryParsingForFailedAttempts() throws Exception {
    LOG.info("STARTING testHistoryParsingForFailedAttempts");
    try {
      Configuration conf = new Configuration();
      conf.setClass(
          CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
          MyResolver.class,
          DNSToSwitchMapping.class);
      RackResolver.init(conf);
      MRApp app =
          new MRAppWithHistoryWithFailedAttempt(2, 1, true, this.getClass().getName(), true);
      app.submit(conf);
      Job job = app.getContext().getAllJobs().values().iterator().next();
      JobId jobId = job.getID();
      app.waitForState(job, JobState.SUCCEEDED);

      // make sure all events are flushed
      app.waitForState(Service.STATE.STOPPED);

      String jobhistoryDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
      JobHistory jobHistory = new JobHistory();
      jobHistory.init(conf);

      JobIndexInfo jobIndexInfo = jobHistory.getJobFileInfo(jobId).getJobIndexInfo();
      String jobhistoryFileName = FileNameIndexUtils.getDoneFileName(jobIndexInfo);

      Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);
      FSDataInputStream in = null;
      FileContext fc = null;
      try {
        fc = FileContext.getFileContext(conf);
        in = fc.open(fc.makeQualified(historyFilePath));
      } catch (IOException ioe) {
        LOG.info("Can not open history file: " + historyFilePath, ioe);
        throw (new Exception("Can not open History File"));
      }

      JobHistoryParser parser = new JobHistoryParser(in);
      JobInfo jobInfo = parser.parse();
      Exception parseException = parser.getParseException();
      Assert.assertNull("Caught an expected exception " + parseException, parseException);
      int noOffailedAttempts = 0;
      Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
      for (Task task : job.getTasks().values()) {
        TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
        for (TaskAttempt taskAttempt : task.getAttempts().values()) {
          TaskAttemptInfo taskAttemptInfo =
              taskInfo.getAllTaskAttempts().get(TypeConverter.fromYarn((taskAttempt.getID())));
          // Verify rack-name for all task attempts
          Assert.assertEquals("rack-name is incorrect", taskAttemptInfo.getRackname(), RACK_NAME);
          if (taskAttemptInfo.getTaskStatus().equals("FAILED")) {
            noOffailedAttempts++;
          }
        }
      }
      Assert.assertEquals("No of Failed tasks doesn't match.", 2, noOffailedAttempts);
    } finally {
      LOG.info("FINISHED testHistoryParsingForFailedAttempts");
    }
  }
Ejemplo n.º 15
0
  private void checkHistoryParsing(
      final int numMaps, final int numReduces, final int numSuccessfulMaps) throws Exception {
    Configuration conf = new Configuration();
    conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
    long amStartTimeEst = System.currentTimeMillis();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class,
        DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(numMaps, numReduces, true, this.getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    String jobhistoryDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);

    FileContext fc = null;
    try {
      fc = FileContext.getFileContext(conf);
    } catch (IOException ioe) {
      LOG.info("Can not get FileContext", ioe);
      throw (new Exception("Can not get File Context"));
    }

    if (numMaps == numSuccessfulMaps) {
      String summaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(jobId);
      Path summaryFile = new Path(jobhistoryDir, summaryFileName);
      String jobSummaryString = getJobSummary(fc, summaryFile);
      Assert.assertNotNull(jobSummaryString);
      Assert.assertTrue(jobSummaryString.contains("resourcesPerMap=100"));
      Assert.assertTrue(jobSummaryString.contains("resourcesPerReduce=100"));

      Map<String, String> jobSummaryElements = new HashMap<String, String>();
      StringTokenizer strToken = new StringTokenizer(jobSummaryString, ",");
      while (strToken.hasMoreTokens()) {
        String keypair = strToken.nextToken();
        jobSummaryElements.put(keypair.split("=")[0], keypair.split("=")[1]);
      }

      Assert.assertEquals(
          "JobId does not match", jobId.toString(), jobSummaryElements.get("jobId"));
      Assert.assertEquals("JobName does not match", "test", jobSummaryElements.get("jobName"));
      Assert.assertTrue(
          "submitTime should not be 0", Long.parseLong(jobSummaryElements.get("submitTime")) != 0);
      Assert.assertTrue(
          "launchTime should not be 0", Long.parseLong(jobSummaryElements.get("launchTime")) != 0);
      Assert.assertTrue(
          "firstMapTaskLaunchTime should not be 0",
          Long.parseLong(jobSummaryElements.get("firstMapTaskLaunchTime")) != 0);
      Assert.assertTrue(
          "firstReduceTaskLaunchTime should not be 0",
          Long.parseLong(jobSummaryElements.get("firstReduceTaskLaunchTime")) != 0);
      Assert.assertTrue(
          "finishTime should not be 0", Long.parseLong(jobSummaryElements.get("finishTime")) != 0);
      Assert.assertEquals(
          "Mismatch in num map slots",
          numSuccessfulMaps,
          Integer.parseInt(jobSummaryElements.get("numMaps")));
      Assert.assertEquals(
          "Mismatch in num reduce slots",
          numReduces,
          Integer.parseInt(jobSummaryElements.get("numReduces")));
      Assert.assertEquals(
          "User does not match", System.getProperty("user.name"), jobSummaryElements.get("user"));
      Assert.assertEquals("Queue does not match", "default", jobSummaryElements.get("queue"));
      Assert.assertEquals("Status does not match", "SUCCEEDED", jobSummaryElements.get("status"));
    }

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);
    HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
    JobInfo jobInfo;
    long numFinishedMaps;

    synchronized (fileInfo) {
      Path historyFilePath = fileInfo.getHistoryFile();
      FSDataInputStream in = null;
      LOG.info("JobHistoryFile is: " + historyFilePath);
      try {
        in = fc.open(fc.makeQualified(historyFilePath));
      } catch (IOException ioe) {
        LOG.info("Can not open history file: " + historyFilePath, ioe);
        throw (new Exception("Can not open History File"));
      }

      JobHistoryParser parser = new JobHistoryParser(in);
      final EventReader realReader = new EventReader(in);
      EventReader reader = Mockito.mock(EventReader.class);
      if (numMaps == numSuccessfulMaps) {
        reader = realReader;
      } else {
        final AtomicInteger numFinishedEvents = new AtomicInteger(0); // Hack!
        Mockito.when(reader.getNextEvent())
            .thenAnswer(
                new Answer<HistoryEvent>() {
                  public HistoryEvent answer(InvocationOnMock invocation) throws IOException {
                    HistoryEvent event = realReader.getNextEvent();
                    if (event instanceof TaskFinishedEvent) {
                      numFinishedEvents.incrementAndGet();
                    }

                    if (numFinishedEvents.get() <= numSuccessfulMaps) {
                      return event;
                    } else {
                      throw new IOException("test");
                    }
                  }
                });
      }

      jobInfo = parser.parse(reader);

      numFinishedMaps = computeFinishedMaps(jobInfo, numMaps, numSuccessfulMaps);

      if (numFinishedMaps != numMaps) {
        Exception parseException = parser.getParseException();
        Assert.assertNotNull("Didn't get expected parse exception", parseException);
      }
    }

    Assert.assertEquals(
        "Incorrect username ", System.getProperty("user.name"), jobInfo.getUsername());
    Assert.assertEquals("Incorrect jobName ", "test", jobInfo.getJobname());
    Assert.assertEquals("Incorrect queuename ", "default", jobInfo.getJobQueueName());
    Assert.assertEquals("incorrect conf path", "test", jobInfo.getJobConfPath());
    Assert.assertEquals("incorrect finishedMap ", numSuccessfulMaps, numFinishedMaps);
    Assert.assertEquals("incorrect finishedReduces ", numReduces, jobInfo.getFinishedReduces());
    Assert.assertEquals("incorrect uberized ", job.isUber(), jobInfo.getUberized());
    Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
    int totalTasks = allTasks.size();
    Assert.assertEquals("total number of tasks is incorrect  ", (numMaps + numReduces), totalTasks);

    // Verify aminfo
    Assert.assertEquals(1, jobInfo.getAMInfos().size());
    Assert.assertEquals(MRApp.NM_HOST, jobInfo.getAMInfos().get(0).getNodeManagerHost());
    AMInfo amInfo = jobInfo.getAMInfos().get(0);
    Assert.assertEquals(MRApp.NM_PORT, amInfo.getNodeManagerPort());
    Assert.assertEquals(MRApp.NM_HTTP_PORT, amInfo.getNodeManagerHttpPort());
    Assert.assertEquals(1, amInfo.getAppAttemptId().getAttemptId());
    Assert.assertEquals(
        amInfo.getAppAttemptId(), amInfo.getContainerId().getApplicationAttemptId());
    Assert.assertTrue(
        amInfo.getStartTime() <= System.currentTimeMillis()
            && amInfo.getStartTime() >= amStartTimeEst);

    ContainerId fakeCid = BuilderUtils.newContainerId(-1, -1, -1, -1);
    // Assert at taskAttempt level
    for (TaskInfo taskInfo : allTasks.values()) {
      int taskAttemptCount = taskInfo.getAllTaskAttempts().size();
      Assert.assertEquals("total number of task attempts ", 1, taskAttemptCount);
      TaskAttemptInfo taInfo = taskInfo.getAllTaskAttempts().values().iterator().next();
      Assert.assertNotNull(taInfo.getContainerId());
      // Verify the wrong ctor is not being used. Remove after mrv1 is removed.
      Assert.assertFalse(taInfo.getContainerId().equals(fakeCid));
    }

    // Deep compare Job and JobInfo
    for (Task task : job.getTasks().values()) {
      TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
      Assert.assertNotNull("TaskInfo not found", taskInfo);
      for (TaskAttempt taskAttempt : task.getAttempts().values()) {
        TaskAttemptInfo taskAttemptInfo =
            taskInfo.getAllTaskAttempts().get(TypeConverter.fromYarn((taskAttempt.getID())));
        Assert.assertNotNull("TaskAttemptInfo not found", taskAttemptInfo);
        Assert.assertEquals(
            "Incorrect shuffle port for task attempt",
            taskAttempt.getShufflePort(),
            taskAttemptInfo.getShufflePort());
        if (numMaps == numSuccessfulMaps) {
          Assert.assertEquals(MRApp.NM_HOST, taskAttemptInfo.getHostname());
          Assert.assertEquals(MRApp.NM_PORT, taskAttemptInfo.getPort());

          // Verify rack-name
          Assert.assertEquals("rack-name is incorrect", taskAttemptInfo.getRackname(), RACK_NAME);
        }
      }
    }
  }
Ejemplo n.º 16
0
  @Override
  protected void render(Block html) {
    ContainerId containerId = verifyAndGetContainerId(html);
    NodeId nodeId = verifyAndGetNodeId(html);
    String appOwner = verifyAndGetAppOwner(html);
    LogLimits logLimits = verifyAndGetLogLimits(html);
    if (containerId == null || nodeId == null || appOwner == null
        || appOwner.isEmpty() || logLimits == null) {
      return;
    }

    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    String logEntity = $(ENTITY_STRING);
    if (logEntity == null || logEntity.isEmpty()) {
      logEntity = containerId.toString();
    }

    if (!conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
        YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
      html.h1()
          ._("Aggregation is not enabled. Try the nodemanager at " + nodeId)
          ._();
      return;
    }

    Path remoteRootLogDir = new Path(conf.get(
        YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
        YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
    Path remoteAppDir = LogAggregationUtils.getRemoteAppLogDir(
        remoteRootLogDir, applicationId, appOwner,
        LogAggregationUtils.getRemoteNodeLogDirSuffix(conf));
    RemoteIterator<FileStatus> nodeFiles;
    try {
      Path qualifiedLogDir =
          FileContext.getFileContext(conf).makeQualified(
            remoteAppDir);
      nodeFiles =
          FileContext.getFileContext(qualifiedLogDir.toUri(), conf)
            .listStatus(remoteAppDir);
    } catch (FileNotFoundException fnf) {
      html.h1()
          ._("Logs not available for " + logEntity
              + ". Aggregation may not be complete, "
              + "Check back later or try the nodemanager at " + nodeId)._();
      return;
    } catch (Exception ex) {
      html.h1()
          ._("Error getting logs at " + nodeId)._();
      return;
    }

    boolean foundLog = false;
    String desiredLogType = $(CONTAINER_LOG_TYPE);
    try {
      while (nodeFiles.hasNext()) {
        AggregatedLogFormat.LogReader reader = null;
        try {
          FileStatus thisNodeFile = nodeFiles.next();
          if (!thisNodeFile.getPath().getName()
            .contains(LogAggregationUtils.getNodeString(nodeId))
              || thisNodeFile.getPath().getName()
                .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) {
            continue;
          }
          long logUploadedTime = thisNodeFile.getModificationTime();
          reader =
              new AggregatedLogFormat.LogReader(conf, thisNodeFile.getPath());

          String owner = null;
          Map<ApplicationAccessType, String> appAcls = null;
          try {
            owner = reader.getApplicationOwner();
            appAcls = reader.getApplicationAcls();
          } catch (IOException e) {
            LOG.error("Error getting logs for " + logEntity, e);
            continue;
          }
          ApplicationACLsManager aclsManager = new ApplicationACLsManager(conf);
          aclsManager.addApplication(applicationId, appAcls);

          String remoteUser = request().getRemoteUser();
          UserGroupInformation callerUGI = null;
          if (remoteUser != null) {
            callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
          }
          if (callerUGI != null && !aclsManager.checkAccess(callerUGI,
              ApplicationAccessType.VIEW_APP, owner, applicationId)) {
            html.h1()
                ._("User [" + remoteUser
                    + "] is not authorized to view the logs for " + logEntity
                    + " in log file [" + thisNodeFile.getPath().getName() + "]")._();
            LOG.error("User [" + remoteUser
              + "] is not authorized to view the logs for " + logEntity);
            continue;
          }

          AggregatedLogFormat.ContainerLogsReader logReader = reader
            .getContainerLogsReader(containerId);
          if (logReader == null) {
            continue;
          }

          foundLog = readContainerLogs(html, logReader, logLimits,
              desiredLogType, logUploadedTime);
        } catch (IOException ex) {
          LOG.error("Error getting logs for " + logEntity, ex);
          continue;
        } finally {
          if (reader != null)
            reader.close();
        }
      }
      if (!foundLog) {
        if (desiredLogType.isEmpty()) {
          html.h1("No logs available for container " + containerId.toString());
        } else {
          html.h1("Unable to locate '" + desiredLogType
              + "' log for container " + containerId.toString());
        }
      }
    } catch (IOException e) {
      html.h1()._("Error getting logs for " + logEntity)._();
      LOG.error("Error getting logs for " + logEntity, e);
    }
  }
 /**
  * Returns TRUE if the history dirs were created, FALSE if they could not be created because the
  * FileSystem is not reachable or in safe mode and throws and exception otherwise.
  */
 @VisibleForTesting
 boolean tryCreatingHistoryDirs(boolean logWait) throws IOException {
   boolean succeeded = true;
   String doneDirPrefix = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
   try {
     doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(doneDirPrefix));
     doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
     doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
     mkdir(
         doneDirFc,
         doneDirPrefixPath,
         new FsPermission(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
   } catch (ConnectException ex) {
     if (logWait) {
       /* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be available") */
       LOG.waiting_for_filesystem_available(
               String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
           .tag("methodCall")
           .info();
     }
     succeeded = false;
   } catch (IOException e) {
     if (isBecauseSafeMode(e)) {
       succeeded = false;
       if (logWait) {
         /* LOG.info("Waiting for FileSystem at "+doneDirPrefixPath.toUri().getAuthority()+"to be out of safe mode") */
         LOG.waiting_for_filesystem_out_safe_mode(
                 String.valueOf(doneDirPrefixPath.toUri().getAuthority()))
             .tag("methodCall")
             .info();
       }
     } else {
       throw new YarnRuntimeException(
           "Error creating done directory: [" + doneDirPrefixPath + "]", e);
     }
   }
   if (succeeded) {
     String intermediateDoneDirPrefix =
         JobHistoryUtils.getConfiguredHistoryIntermediateDoneDirPrefix(conf);
     try {
       intermediateDoneDirPath =
           FileContext.getFileContext(conf).makeQualified(new Path(intermediateDoneDirPrefix));
       intermediateDoneDirFc = FileContext.getFileContext(intermediateDoneDirPath.toUri(), conf);
       mkdir(
           intermediateDoneDirFc,
           intermediateDoneDirPath,
           new FsPermission(JobHistoryUtils.HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS.toShort()));
     } catch (ConnectException ex) {
       succeeded = false;
       if (logWait) {
         /* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be available") */
         LOG.waiting_for_filesystem_available(
                 String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
             .tag("methodCall")
             .info();
       }
     } catch (IOException e) {
       if (isBecauseSafeMode(e)) {
         succeeded = false;
         if (logWait) {
           /* LOG.info("Waiting for FileSystem at "+intermediateDoneDirPath.toUri().getAuthority()+"to be out of safe mode") */
           LOG.waiting_for_filesystem_out_safe_mode(
                   String.valueOf(intermediateDoneDirPath.toUri().getAuthority()))
               .tag("methodCall")
               .info();
         }
       } else {
         throw new YarnRuntimeException(
             "Error creating intermediate done directory: [" + intermediateDoneDirPath + "]", e);
       }
     }
   }
   return succeeded;
 }