Beispiel #1
0
  /**
   * Write random bytes at the path provided.
   *
   * @see org.apache.hadoop.mapred.gridmix.GenerateData
   */
  protected void writeInputData(long genbytes, Path ioPath)
      throws IOException, InterruptedException {
    final Configuration conf = getConf();
    final GridmixJob genData = new GenerateData(conf, ioPath, genbytes);
    submitter.add(genData);
    LOG.info("Generating " + StringUtils.humanReadableInt(genbytes) + " of test data...");
    // TODO add listeners, use for job dependencies
    TimeUnit.SECONDS.sleep(10);
    try {
      genData.getJob().waitForCompletion(false);
    } catch (ClassNotFoundException e) {
      throw new IOException("Internal error", e);
    }
    if (!genData.getJob().isSuccessful()) {
      throw new IOException("Data generation failed!");
    }

    FsShell shell = new FsShell(conf);
    try {
      LOG.info("Changing the permissions for inputPath " + ioPath.toString());
      shell.run(new String[] {"-chmod", "-R", "777", ioPath.toString()});
    } catch (Exception e) {
      LOG.error("Couldnt change the file permissions ", e);
      throw new IOException(e);
    }
    LOG.info("Done.");
  }
Beispiel #2
0
  // Launch Input/DistCache Data Generation job and wait for completion
  void launchGridmixJob(GridmixJob job) throws IOException, InterruptedException {
    submitter.add(job);

    // TODO add listeners, use for job dependencies
    try {
      while (!job.isSubmitted()) {
        try {
          Thread.sleep(100); // sleep
        } catch (InterruptedException ie) {
        }
      }
      // wait for completion
      job.getJob().waitForCompletion(false);
    } catch (ClassNotFoundException e) {
      throw new IOException("Internal error", e);
    }
    if (!job.getJob().isSuccessful()) {
      throw new IOException(job.getJob().getJobName() + " job failed!");
    }
  }