Example #1
0
  /**
   * Create each component in the pipeline and start it.
   *
   * @param conf Configuration data, no keys specific to this context
   * @param traceIn Either a Path to the trace data or "-" for stdin
   * @param ioPath <ioPath>/input/ is the dir from which input data is read and
   *     <ioPath>/distributedCache/ is the gridmix distributed cache directory.
   * @param scratchDir Path into which job output is written
   * @param startFlag Semaphore for starting job trace pipeline
   */
  private void startThreads(
      Configuration conf,
      String traceIn,
      Path ioPath,
      Path scratchDir,
      CountDownLatch startFlag,
      UserResolver userResolver)
      throws IOException {
    try {
      Path inputDir = getGridmixInputDataPath(ioPath);
      GridmixJobSubmissionPolicy policy = getJobSubmissionPolicy(conf);
      LOG.info(" Submission policy is " + policy.name());
      statistics = new Statistics(conf, policy.getPollingInterval(), startFlag);
      monitor = createJobMonitor(statistics, conf);
      int noOfSubmitterThreads =
          (policy == GridmixJobSubmissionPolicy.SERIAL)
              ? 1
              : Runtime.getRuntime().availableProcessors() + 1;

      int numThreads = conf.getInt(GRIDMIX_SUB_THR, noOfSubmitterThreads);
      int queueDep = conf.getInt(GRIDMIX_QUE_DEP, 5);
      submitter =
          createJobSubmitter(
              monitor,
              numThreads,
              queueDep,
              new FilePool(conf, inputDir),
              userResolver,
              statistics);
      distCacheEmulator = new DistributedCacheEmulator(conf, ioPath);

      factory = createJobFactory(submitter, traceIn, scratchDir, conf, startFlag, userResolver);
      factory.jobCreator.setDistCacheEmulator(distCacheEmulator);

      if (policy == GridmixJobSubmissionPolicy.SERIAL) {
        statistics.addJobStatsListeners(factory);
      } else {
        statistics.addClusterStatsObservers(factory);
      }

      // add the gridmix run summarizer to the statistics
      statistics.addJobStatsListeners(summarizer.getExecutionSummarizer());
      statistics.addClusterStatsObservers(summarizer.getClusterSummarizer());

      monitor.start();
      submitter.start();
    } catch (Exception e) {
      LOG.error(" Exception at start ", e);
      throw new IOException(e);
    }
  }
Example #2
0
  public int run(final String[] argv) throws IOException, InterruptedException {
    int val = -1;
    final Configuration conf = getConf();
    UserGroupInformation.setConfiguration(conf);
    UserGroupInformation ugi = UserGroupInformation.getLoginUser();

    val =
        ugi.doAs(
            new PrivilegedExceptionAction<Integer>() {
              public Integer run() throws Exception {
                return runJob(conf, argv);
              }
            });

    // print the gridmix summary if the run was successful
    if (val == 0) {
      // print the run summary
      System.out.print("\n\n");
      System.out.println(summarizer.toString());
    }

    return val;
  }
Example #3
0
  /**
   * @param conf gridmix configuration
   * @param traceIn trace file path(if it is '-', then trace comes from the stream stdin)
   * @param ioPath Working directory for gridmix. GenerateData job will generate data in the
   *     directory &lt;ioPath&gt;/input/ and distributed cache data is generated in the directory
   *     &lt;ioPath&gt;/distributedCache/, if -generate option is specified.
   * @param genbytes size of input data to be generated under the directory &lt;ioPath&gt;/input/
   * @param userResolver gridmix user resolver
   * @param generate true if -generate option was specified
   * @return exit code
   * @throws IOException
   * @throws InterruptedException
   */
  int start(
      Configuration conf,
      String traceIn,
      Path ioPath,
      long genbytes,
      UserResolver userResolver,
      boolean generate)
      throws IOException, InterruptedException {
    DataStatistics stats = null;
    InputStream trace = null;
    ioPath = ioPath.makeQualified(ioPath.getFileSystem(conf));

    try {
      Path scratchDir = new Path(ioPath, conf.get(GRIDMIX_OUT_DIR, "gridmix"));

      // add shutdown hook for SIGINT, etc.
      Runtime.getRuntime().addShutdownHook(sdh);
      CountDownLatch startFlag = new CountDownLatch(1);
      try {
        // Create, start job submission threads
        startThreads(conf, traceIn, ioPath, scratchDir, startFlag, userResolver);

        Path inputDir = getGridmixInputDataPath(ioPath);

        // Write input data if specified
        if (genbytes > 0) {
          writeInputData(genbytes, inputDir);
        }

        // publish the data statistics
        stats = GenerateData.publishDataStatistics(inputDir, genbytes, conf);

        // scan input dir contents
        submitter.refreshFilePool();

        // set up the needed things for emulation of various loads
        int exitCode = setupEmulation(conf, traceIn, scratchDir, ioPath, generate);
        if (exitCode != 0) {
          return exitCode;
        }

        // start the summarizer
        summarizer.start(conf);

        factory.start();
        statistics.start();
      } catch (Throwable e) {
        LOG.error("Startup failed", e);
        if (factory != null) factory.abort(); // abort pipeline
      } finally {
        // signal for factory to start; sets start time
        startFlag.countDown();
      }
      if (factory != null) {
        // wait for input exhaustion
        factory.join(Long.MAX_VALUE);
        final Throwable badTraceException = factory.error();
        if (null != badTraceException) {
          LOG.error("Error in trace", badTraceException);
          throw new IOException("Error in trace", badTraceException);
        }
        // wait for pending tasks to be submitted
        submitter.shutdown();
        submitter.join(Long.MAX_VALUE);
        // wait for running tasks to complete
        monitor.shutdown();
        monitor.join(Long.MAX_VALUE);

        statistics.shutdown();
        statistics.join(Long.MAX_VALUE);
      }
    } finally {
      if (factory != null) {
        summarizer.finalize(factory, traceIn, genbytes, userResolver, stats, conf);
      }
      IOUtils.cleanup(LOG, trace);
    }
    return 0;
  }