Пример #1
0
  /**
   * Create each component in the pipeline and start it.
   *
   * @param conf Configuration data, no keys specific to this context
   * @param traceIn Either a Path to the trace data or "-" for stdin
   * @param ioPath <ioPath>/input/ is the dir from which input data is read and
   *     <ioPath>/distributedCache/ is the gridmix distributed cache directory.
   * @param scratchDir Path into which job output is written
   * @param startFlag Semaphore for starting job trace pipeline
   */
  private void startThreads(
      Configuration conf,
      String traceIn,
      Path ioPath,
      Path scratchDir,
      CountDownLatch startFlag,
      UserResolver userResolver)
      throws IOException {
    try {
      Path inputDir = getGridmixInputDataPath(ioPath);
      GridmixJobSubmissionPolicy policy = getJobSubmissionPolicy(conf);
      LOG.info(" Submission policy is " + policy.name());
      statistics = new Statistics(conf, policy.getPollingInterval(), startFlag);
      monitor = createJobMonitor(statistics, conf);
      int noOfSubmitterThreads =
          (policy == GridmixJobSubmissionPolicy.SERIAL)
              ? 1
              : Runtime.getRuntime().availableProcessors() + 1;

      int numThreads = conf.getInt(GRIDMIX_SUB_THR, noOfSubmitterThreads);
      int queueDep = conf.getInt(GRIDMIX_QUE_DEP, 5);
      submitter =
          createJobSubmitter(
              monitor,
              numThreads,
              queueDep,
              new FilePool(conf, inputDir),
              userResolver,
              statistics);
      distCacheEmulator = new DistributedCacheEmulator(conf, ioPath);

      factory = createJobFactory(submitter, traceIn, scratchDir, conf, startFlag, userResolver);
      factory.jobCreator.setDistCacheEmulator(distCacheEmulator);

      if (policy == GridmixJobSubmissionPolicy.SERIAL) {
        statistics.addJobStatsListeners(factory);
      } else {
        statistics.addClusterStatsObservers(factory);
      }

      // add the gridmix run summarizer to the statistics
      statistics.addJobStatsListeners(summarizer.getExecutionSummarizer());
      statistics.addClusterStatsObservers(summarizer.getClusterSummarizer());

      monitor.start();
      submitter.start();
    } catch (Exception e) {
      LOG.error(" Exception at start ", e);
      throw new IOException(e);
    }
  }
Пример #2
0
  /**
   * Create each component in the pipeline and start it.
   *
   * @param conf Configuration data, no keys specific to this context
   * @param traceIn Either a Path to the trace data or "-" for stdin
   * @param ioPath Path from which input data is read
   * @param scratchDir Path into which job output is written
   * @param startFlag Semaphore for starting job trace pipeline
   */
  private void startThreads(
      Configuration conf,
      String traceIn,
      Path ioPath,
      Path scratchDir,
      CountDownLatch startFlag,
      UserResolver userResolver)
      throws IOException {
    try {
      GridmixJobSubmissionPolicy policy =
          GridmixJobSubmissionPolicy.getPolicy(conf, GridmixJobSubmissionPolicy.STRESS);
      LOG.info(" Submission policy is " + policy.name());
      statistics = new Statistics(conf, policy.getPollingInterval(), startFlag);
      monitor = createJobMonitor(statistics);
      int noOfSubmitterThreads =
          (policy == GridmixJobSubmissionPolicy.SERIAL)
              ? 1
              : Runtime.getRuntime().availableProcessors() + 1;

      submitter =
          createJobSubmitter(
              monitor,
              conf.getInt(GRIDMIX_SUB_THR, noOfSubmitterThreads),
              conf.getInt(GRIDMIX_QUE_DEP, 5),
              new FilePool(conf, ioPath),
              userResolver,
              statistics);

      factory = createJobFactory(submitter, traceIn, scratchDir, conf, startFlag, userResolver);
      if (policy == GridmixJobSubmissionPolicy.SERIAL) {
        statistics.addJobStatsListeners(factory);
      } else {
        statistics.addClusterStatsObservers(factory);
      }

      monitor.start();
      submitter.start();
    } catch (Exception e) {
      LOG.error(" Exception at start ", e);
      throw new IOException(e);
    }
  }