Esempio n. 1
0
  int start(
      Configuration conf, String traceIn, Path ioPath, long genbytes, UserResolver userResolver)
      throws IOException, InterruptedException {
    InputStream trace = null;
    try {
      Path scratchDir = new Path(ioPath, conf.get(GRIDMIX_OUT_DIR, "gridmix"));
      final FileSystem scratchFs = scratchDir.getFileSystem(conf);
      scratchFs.mkdirs(scratchDir, new FsPermission((short) 0777));
      scratchFs.setPermission(scratchDir, new FsPermission((short) 0777));
      // add shutdown hook for SIGINT, etc.
      Runtime.getRuntime().addShutdownHook(sdh);
      CountDownLatch startFlag = new CountDownLatch(1);
      try {
        // Create, start job submission threads
        startThreads(conf, traceIn, ioPath, scratchDir, startFlag, userResolver);
        // Write input data if specified
        if (genbytes > 0) {
          writeInputData(genbytes, ioPath);
        }
        // scan input dir contents
        submitter.refreshFilePool();
        factory.start();
        statistics.start();
      } catch (Throwable e) {
        LOG.error("Startup failed", e);
        if (factory != null) factory.abort(); // abort pipeline
      } finally {
        // signal for factory to start; sets start time
        startFlag.countDown();
      }
      if (factory != null) {
        // wait for input exhaustion
        factory.join(Long.MAX_VALUE);
        final Throwable badTraceException = factory.error();
        if (null != badTraceException) {
          LOG.error("Error in trace", badTraceException);
          throw new IOException("Error in trace", badTraceException);
        }
        // wait for pending tasks to be submitted
        submitter.shutdown();
        submitter.join(Long.MAX_VALUE);
        // wait for running tasks to complete
        monitor.shutdown();
        monitor.join(Long.MAX_VALUE);

        statistics.shutdown();
        statistics.join(Long.MAX_VALUE);
      }
    } finally {
      IOUtils.cleanup(LOG, trace);
    }
    return 0;
  }
Esempio n. 2
0
  /**
   * @param conf gridmix configuration
   * @param traceIn trace file path(if it is '-', then trace comes from the stream stdin)
   * @param ioPath Working directory for gridmix. GenerateData job will generate data in the
   *     directory <ioPath>/input/ and distributed cache data is generated in the directory
   *     <ioPath>/distributedCache/, if -generate option is specified.
   * @param genbytes size of input data to be generated under the directory <ioPath>/input/
   * @param userResolver gridmix user resolver
   * @param generate true if -generate option was specified
   * @return exit code
   * @throws IOException
   * @throws InterruptedException
   */
  int start(
      Configuration conf,
      String traceIn,
      Path ioPath,
      long genbytes,
      UserResolver userResolver,
      boolean generate)
      throws IOException, InterruptedException {
    DataStatistics stats = null;
    InputStream trace = null;
    ioPath = ioPath.makeQualified(ioPath.getFileSystem(conf));

    try {
      Path scratchDir = new Path(ioPath, conf.get(GRIDMIX_OUT_DIR, "gridmix"));

      // add shutdown hook for SIGINT, etc.
      Runtime.getRuntime().addShutdownHook(sdh);
      CountDownLatch startFlag = new CountDownLatch(1);
      try {
        // Create, start job submission threads
        startThreads(conf, traceIn, ioPath, scratchDir, startFlag, userResolver);

        Path inputDir = getGridmixInputDataPath(ioPath);

        // Write input data if specified
        if (genbytes > 0) {
          writeInputData(genbytes, inputDir);
        }

        // publish the data statistics
        stats = GenerateData.publishDataStatistics(inputDir, genbytes, conf);

        // scan input dir contents
        submitter.refreshFilePool();

        // set up the needed things for emulation of various loads
        int exitCode = setupEmulation(conf, traceIn, scratchDir, ioPath, generate);
        if (exitCode != 0) {
          return exitCode;
        }

        // start the summarizer
        summarizer.start(conf);

        factory.start();
        statistics.start();
      } catch (Throwable e) {
        LOG.error("Startup failed", e);
        if (factory != null) factory.abort(); // abort pipeline
      } finally {
        // signal for factory to start; sets start time
        startFlag.countDown();
      }
      if (factory != null) {
        // wait for input exhaustion
        factory.join(Long.MAX_VALUE);
        final Throwable badTraceException = factory.error();
        if (null != badTraceException) {
          LOG.error("Error in trace", badTraceException);
          throw new IOException("Error in trace", badTraceException);
        }
        // wait for pending tasks to be submitted
        submitter.shutdown();
        submitter.join(Long.MAX_VALUE);
        // wait for running tasks to complete
        monitor.shutdown();
        monitor.join(Long.MAX_VALUE);

        statistics.shutdown();
        statistics.join(Long.MAX_VALUE);
      }
    } finally {
      if (factory != null) {
        summarizer.finalize(factory, traceIn, genbytes, userResolver, stats, conf);
      }
      IOUtils.cleanup(LOG, trace);
    }
    return 0;
  }