Exemplo n.º 1
0
 /**
  * This is the main driver for recursively copying directories across file systems. It takes at
  * least two cmdline parameters. A source URL and a destination URL. It then essentially does an
  * "ls -lR" on the source URL, and writes the output in a round-robin manner to all the map input
  * files. The mapper actually copies the files allotted to it. The reduce is empty.
  */
 public int run(String[] args) {
   try {
     copy(conf, Arguments.valueOf(args, conf));
     return 0;
   } catch (IllegalArgumentException e) {
     System.err.println(StringUtils.stringifyException(e) + "\n" + usage);
     ToolRunner.printGenericCommandUsage(System.err);
     return -1;
   } catch (DuplicationException e) {
     System.err.println(StringUtils.stringifyException(e));
     return DuplicationException.ERROR_CODE;
   } catch (RemoteException e) {
     final IOException unwrapped =
         e.unwrapRemoteException(
             FileNotFoundException.class,
             AccessControlException.class,
             QuotaExceededException.class);
     System.err.println(StringUtils.stringifyException(unwrapped));
     return -3;
   } catch (Exception e) {
     System.err.println(
         "With failures, global counters are inaccurate; " + "consider running with -i");
     System.err.println("Copy failed: " + StringUtils.stringifyException(e));
     return -999;
   }
 }
Exemplo n.º 2
0
  public int run(String[] args) throws Exception {

    if (args.length != 2) {
      System.out.printf(
          "Usage: %s [generic options] <input dir> <output dir>\n", getClass().getSimpleName());
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    Job job = new Job(getConf());
    job.setJarByClass(RowFilter.class);
    job.setJobName(this.getClass().getName());

    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(RowMapper.class);
    job.setNumReduceTasks(0);

    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);

    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);

    if (job.waitForCompletion(true)) {
      return 0;
    }
    return 1;
  }
  @Override
  public int run(String[] args) throws Exception {
    if (args.length < 2) {
      System.out.println("hitsperurl.DriverHitsPerUrl <inDir> <outDir>");
      ToolRunner.printGenericCommandUsage(System.out);
      System.out.println("");
      return -1;
    }

    System.out.println(Arrays.toString(args));

    Job job = new Job(getConf(), "number of hits per url");
    job.setJarByClass(DriverHitsPerUrl.class);
    job.setMapperClass(ParserMapper.class);
    // Uncomment this to set combiner
    // job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(AggregateURLReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.waitForCompletion(true);

    return 0;
  }
Exemplo n.º 4
0
  @Override
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.println("Usage: WhiteHouseVisitorDriver <input path> <output path>");
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    Configuration conf = new Configuration();
    Job job = new Job(conf);
    job.setJarByClass(WhiteHouseVisitorDriver.class);

    // input file format
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    // map/combine/reduce class definition
    job.setMapperClass(WhiteHouseVisitorMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);

    // Key, Value set type definition
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    return job.waitForCompletion(true) ? 0 : 1;
  }
  @Override
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.printf(
          "Usage: %s [generic options] <input> <output>\n", getClass().getSimpleName());
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    JobConf conf = new JobConf(getConf(), getClass());
    conf.setJobName("Max temperature");

    FileInputFormat.addInputPath(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(IntWritable.class);

    conf.setMapperClass(MaxTemperatureMapper.class);
    conf.setCombinerClass(MaxTemperatureReducer.class);
    conf.setReducerClass(MaxTemperatureReducer.class);

    // vv MaxTemperatureDriverV6
    conf.setProfileEnabled(true);
    conf.setProfileParams(
        "-agentlib:hprof=cpu=samples,heap=sites,depth=6," + "force=n,thread=y,verbose=n,file=%s");
    conf.setProfileTaskRange(true, "0-2");
    // ^^ MaxTemperatureDriverV6

    JobClient.runJob(conf);
    return 0;
  }
  /** Print help. */
  private void printHelp() {
    String summary =
        "Usage: bin/hdfs oev [OPTIONS] -i INPUT_FILE -o OUTPUT_FILE\n"
            + "Offline edits viewer\n"
            + "Parse a Hadoop edits log file INPUT_FILE and save results\n"
            + "in OUTPUT_FILE.\n"
            + "Required command line arguments:\n"
            + "-i,--inputFile <arg>   edits file to process, xml (case\n"
            + "                       insensitive) extension means XML format,\n"
            + "                       any other filename means binary format\n"
            + "-o,--outputFile <arg>  Name of output file. If the specified\n"
            + "                       file exists, it will be overwritten,\n"
            + "                       format of the file is determined\n"
            + "                       by -p option\n"
            + "\n"
            + "Optional command line arguments:\n"
            + "-p,--processor <arg>   Select which type of processor to apply\n"
            + "                       against image file, currently supported\n"
            + "                       processors are: binary (native binary format\n"
            + "                       that Hadoop uses), xml (default, XML\n"
            + "                       format), stats (prints statistics about\n"
            + "                       edits file)\n"
            + "-h,--help              Display usage information and exit\n"
            + "-v,--verbose           More verbose output, prints the input and\n"
            + "                       output filenames, for processors that write\n"
            + "                       to a file, also output to screen. On large\n"
            + "                       image files this will dramatically increase\n"
            + "                       processing time (default is false).\n";

    System.out.println(summary);
    System.out.println();
    ToolRunner.printGenericCommandUsage(System.out);
  }
Exemplo n.º 7
0
  protected void printUsage(PrintStream out) {
    ToolRunner.printGenericCommandUsage(out);
    out.println("Usage: gridmix [-generate <MiB>] [-users URI] <iopath> <trace>");
    out.println("  e.g. gridmix -generate 100m foo -");
    out.println("Configuration parameters:");
    out.printf("       %-42s : Output directory\n", GRIDMIX_OUT_DIR);
    out.printf("       %-42s : Submitting threads\n", GRIDMIX_SUB_THR);
    out.printf("       %-42s : Queued job desc\n", GRIDMIX_QUE_DEP);
    out.printf("       %-42s : Key fraction of rec\n", AvgRecordFactory.GRIDMIX_KEY_FRC);
    out.printf("       %-42s : User resolution class\n", GRIDMIX_USR_RSV);
    out.printf(
        "       %-42s : Enable/disable using queues in trace\n",
        GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
    out.printf("       %-42s : Default queue\n", GridmixJob.GRIDMIX_DEFAULT_QUEUE);

    StringBuilder sb = new StringBuilder();
    String sep = "";
    for (GridmixJobSubmissionPolicy policy : GridmixJobSubmissionPolicy.values()) {
      sb.append(sep);
      sb.append(policy.name());
      sep = "|";
    }
    out.printf(
        "       %-40s : Job submission policy (%s)\n",
        GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, sb.toString());
  }
Exemplo n.º 8
0
  /**
   * This is the main routine for launching a distributed random write job. It runs 10 maps/node and
   * each node writes 1 gig of data to a DFS file. The reduce doesn't do anything.
   *
   * @throws IOException
   */
  public int run(String[] args) throws Exception {
    if (args.length == 0) {
      System.out.println("Usage: writer <out-dir>");
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    Path outDir = new Path(args[0]);
    JobConf job = new JobConf(getConf());

    job.setJarByClass(RandomWriter.class);
    job.setJobName("random-writer");
    FileOutputFormat.setOutputPath(job, outDir);

    job.setOutputKeyClass(BytesWritable.class);
    job.setOutputValueClass(BytesWritable.class);

    job.setInputFormat(RandomInputFormat.class);
    job.setMapperClass(Map.class);
    job.setReducerClass(IdentityReducer.class);
    job.setOutputFormat(SequenceFileOutputFormat.class);

    JobClient client = new JobClient(job);
    ClusterStatus cluster = client.getClusterStatus();
    /** 如果属性不存在 则返回默认的值 * */
    int numMapsPerHost = job.getInt("test.randomwriter.maps_per_host", 10);
    long numBytesToWritePerMap =
        job.getLong("test.randomwrite.bytes_per_map", 1 * 1024 * 1024 * 1024);
    if (numBytesToWritePerMap == 0) {
      System.err.println("Cannot have test.randomwrite.bytes_per_map set to 0");
      return -2;
    }
    long totalBytesToWrite =
        job.getLong(
            "test.randomwrite.total_bytes",
            numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
    int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
    if (numMaps == 0 && totalBytesToWrite > 0) {
      numMaps = 1;
      job.setLong("test.randomwrite.bytes_per_map", totalBytesToWrite);
    }

    job.setNumMapTasks(numMaps);
    /** 建议型的 * */
    System.out.println("Running " + numMaps + " maps.");

    // reducer NONE
    job.setNumReduceTasks(0);

    Date startTime = new Date();
    System.out.println("Job started: " + startTime);
    JobClient.runJob(job);
    Date endTime = new Date();
    System.out.println("Job ended: " + endTime);
    System.out.println(
        "The job took " + (endTime.getTime() - startTime.getTime()) / 1000 + " seconds.");

    return 0;
  }
Exemplo n.º 9
0
 /** Print usage messages */
 public static int printUsage(String[] args, String usage) {
   err.println("args = " + Arrays.asList(args));
   err.println();
   err.println("Usage: java " + usage);
   err.println();
   ToolRunner.printGenericCommandUsage(err);
   return -1;
 }
Exemplo n.º 10
0
  @Override
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.printf(
          "Usage: %s [generic options] <input> <output>\n", getClass().getSimpleName());
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    // get configuration
    Configuration conf = getConf();
    CommonUtility.printConfiguration(conf);

    // create a JobConf
    JobConf jobconf = new JobConf(conf);

    // set name
    jobconf.setJobName("CommonTestJob");

    // set input/output path
    Path in = new Path(args[0]);
    Path out = new Path(args[1]);
    FileInputFormat.setInputPaths(jobconf, in);
    FileOutputFormat.setOutputPath(jobconf, out);

    // set input/output format
    jobconf.setInputFormat(SequenceFileInputFormat.class);
    jobconf.setOutputFormat(SequenceFileOutputFormat.class);

    // set output key/value
    jobconf.setOutputKeyClass(Text.class);
    jobconf.setOutputValueClass(RRIntervalWritable.class);

    // set mapper/reducer class
    jobconf.setMapperClass(CommonTestMapper.class);
    jobconf.setReducerClass(IdentityReducer.class);

    // MultipleInputs.addInputPath(jobconf, new
    // Path("hdfs://localhost/work/lab/ecg/rrSeqMulti"),
    // SequenceFileInputFormat.class, CommonTestMapper.class);
    //
    // MultipleInputs.addInputPath(jobconf, new
    // Path("hdfs://localhost/work/lab/ecg/rrSeqSingle"),
    // SequenceFileInputFormat.class, IdentityMapper.class);

    jobconf.setNumReduceTasks(0);

    JobClient.runJob(jobconf);

    // --- end---

    CommonUtility.printConfiguration(jobconf);

    return 0;
  }
Exemplo n.º 11
0
 protected void printUsage(PrintStream out) {
   ToolRunner.printGenericCommandUsage(out);
   out.println(
       "Usage: gridmix [-generate <MiB>] [-users URI] [-Dname=value ...] <iopath> <trace>");
   out.println("  e.g. gridmix -generate 100m foo -");
   out.println("Options:");
   out.println(
       "   -generate <MiB> : Generate input data of size MiB under "
           + "<iopath>/input/ and generate\n\t\t     distributed cache data under "
           + "<iopath>/distributedCache/.");
   out.println("   -users <usersResourceURI> : URI that contains the users list.");
   out.println("Configuration parameters:");
   out.println("   General parameters:");
   out.printf("       %-48s : Output directory\n", GRIDMIX_OUT_DIR);
   out.printf("       %-48s : Submitting threads\n", GRIDMIX_SUB_THR);
   out.printf("       %-48s : Queued job desc\n", GRIDMIX_QUE_DEP);
   out.printf("       %-48s : User resolution class\n", GRIDMIX_USR_RSV);
   out.printf("       %-48s : Job types (%s)\n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes());
   out.println("   Parameters related to job submission:");
   out.printf("       %-48s : Default queue\n", GridmixJob.GRIDMIX_DEFAULT_QUEUE);
   out.printf(
       "       %-48s : Enable/disable using queues in trace\n",
       GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
   out.printf(
       "       %-48s : Job submission policy (%s)\n",
       GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, getSubmissionPolicies());
   out.println("   Parameters specific for LOADJOB:");
   out.printf("       %-48s : Key fraction of rec\n", AvgRecordFactory.GRIDMIX_KEY_FRC);
   out.println("   Parameters specific for SLEEPJOB:");
   out.printf("       %-48s : Whether to ignore reduce tasks\n", SleepJob.SLEEPJOB_MAPTASK_ONLY);
   out.printf(
       "       %-48s : Number of fake locations for map tasks\n",
       JobCreator.SLEEPJOB_RANDOM_LOCATIONS);
   out.printf(
       "       %-48s : Maximum map task runtime in mili-sec\n",
       SleepJob.GRIDMIX_SLEEP_MAX_MAP_TIME);
   out.printf(
       "       %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)\n",
       SleepJob.GRIDMIX_SLEEP_MAX_REDUCE_TIME);
   out.println("   Parameters specific for STRESS submission throttling policy:");
   out.printf(
       "       %-48s : jobs vs task-tracker ratio\n", StressJobFactory.CONF_MAX_JOB_TRACKER_RATIO);
   out.printf(
       "       %-48s : maps vs map-slot ratio\n",
       StressJobFactory.CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO);
   out.printf(
       "       %-48s : reduces vs reduce-slot ratio\n",
       StressJobFactory.CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO);
   out.printf(
       "       %-48s : map-slot share per job\n", StressJobFactory.CONF_MAX_MAPSLOT_SHARE_PER_JOB);
   out.printf(
       "       %-48s : reduce-slot share per job\n",
       StressJobFactory.CONF_MAX_REDUCESLOT_SHARE_PER_JOB);
 }
Exemplo n.º 12
0
 static int printUsage() {
   System.out.println(
       "join [-r <reduces>] "
           + "[-inFormat <input format class>] "
           + "[-outFormat <output format class>] "
           + "[-outKey <output key class>] "
           + "[-outValue <output value class>] "
           + "[-joinOp <inner|outer|override>] "
           + "[input]* <input> <output>");
   ToolRunner.printGenericCommandUsage(System.out);
   return 2;
 }
Exemplo n.º 13
0
 private void displayUsage(String cmd) {
   String prefix = "Usage: JobQueueClient ";
   if ("-queueinfo".equals(cmd)) {
     System.err.println(prefix + "[" + cmd + "<job-queue-name> [-showJobs]]");
   } else {
     System.err.printf(prefix + "<command> <args>\n");
     System.err.printf("\t[-list]\n");
     System.err.printf("\t[-info <job-queue-name> [-showJobs]]\n");
     System.err.printf("\t[-showacls] \n\n");
     ToolRunner.printGenericCommandUsage(System.out);
   }
 }
Exemplo n.º 14
0
  @Override
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.printf("Usage: %s <input path> <output path>", getClass().getSimpleName());
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    Job job = new Job();
    job.setJarByClass(PopulationsDriver.class);
    // TODO add configuration for map and reduce
    return job.waitForCompletion(true) ? 1 : 0;
  }
Exemplo n.º 15
0
 private static int printUsage() {
   System.out.println(
       "SAMMEPL's parameters are:\n"
           + "  -train|-eval\n"
           + "  -d|-data <data path>\n"
           + "  -m|-model <model path>\n"
           + "  -n|-num <num instances per map>\n"
           + "  -i|-iteration <num iterations>\n"
           + "  [-md|-metadata <metadata path>]\n"
           + "  [-o|-output <msg output folder>]\n\n");
   ToolRunner.printGenericCommandUsage(System.out);
   return -1;
 }
Exemplo n.º 16
0
 public static void main(String[] args) throws Exception {
   // -files option is also used by GenericOptionsParser
   // Make sure that is not the first argument for fsck
   int res = -1;
   if ((args.length == 0) || ("-files".equals(args[0]))) {
     printUsage(System.err);
     ToolRunner.printGenericCommandUsage(System.err);
   } else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
     res = 0;
   } else {
     res = ToolRunner.run(new DFSck(new HdfsConfiguration()), args);
   }
   System.exit(res);
 }
  /* (non-Javadoc)
   * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
   */
  @Override
  public int run(String[] args) throws Exception {
    // arguments
    if (args.length < 2) {
      System.err.printf(
          "Usage: %s [generic options] <wikidump path> <output path> <start job(optional)>\n",
          getClass().getSimpleName());

      System.err.println("<start job>: parse, degree, graph, rank, list");

      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    // Configuration
    Configuration conf = getConf();
    conf.set("wikidump_path", args[0]);
    conf.set("output_path", args[1]);

    // Job selection
    JobName startJob = JobName.PARSE;
    if (args.length >= 3) {
      try {
        startJob = JobName.valueOf(args[2].toUpperCase());
      } catch (IllegalArgumentException e) {

      }
    }

    int K = 20;
    switch (startJob) {
      case PARSE:
        JobParse.run(conf);
      case DEGREE:
        JobOutDegree.run(conf);
        JobInDegree.run(conf);
      case GRAPH:
        JobGraph.run(conf);
      case RANK:
        JobRank.run(conf, K);
      case RESULT:
        JobResult.run(conf, K);
        break;
      default:
        break;
    }

    return 0;
  }
Exemplo n.º 18
0
 /** Launches all the tasks in order. */
 @Override
 public int run(String[] args) throws Exception {
   if (args.length < 10) {
     System.err.println(
         "Usage: edu.iu.kmeans.KMeansMapCollective <num Of DataPoints> <num of Centroids> <vector size> "
             + "<number of map tasks> <partition per worker> <number of iteration> <iteration per job> <start Job ID>"
             + "<work dir> <local points dir>");
     ToolRunner.printGenericCommandUsage(System.err);
     return -1;
   }
   int numOfDataPoints = Integer.parseInt(args[0]);
   int numCentroids = Integer.parseInt(args[1]);
   int vectorSize = Integer.parseInt(args[2]);
   int numMapTasks = Integer.parseInt(args[3]);
   int partitionPerWorker = Integer.parseInt(args[4]);
   int numIteration = Integer.parseInt(args[5]);
   int iterationPerJob = Integer.parseInt(args[6]);
   int startJobID = Integer.parseInt(args[7]);
   String workDir = args[8];
   String localPointFilesDir = args[9];
   boolean regenerateData = true;
   if (args.length == 11) {
     regenerateData = Boolean.parseBoolean(args[10]);
   }
   System.out.println("Number of Map Tasks = " + numMapTasks);
   int numPointFiles = numMapTasks * partitionPerWorker;
   if (numOfDataPoints / numPointFiles == 0 || numCentroids / numMapTasks == 0) {
     return -1;
   }
   if (numIteration == 0) {
     numIteration = 1;
   }
   if (iterationPerJob == 0) {
     iterationPerJob = 1;
   }
   launch(
       numOfDataPoints,
       numCentroids,
       vectorSize,
       numPointFiles,
       numMapTasks,
       numIteration,
       iterationPerJob,
       startJobID,
       workDir,
       localPointFilesDir,
       regenerateData);
   return 0;
 }
Exemplo n.º 19
0
 static int printUsage() {
   System.out.println(
       "sampler -r <reduces>\n"
           + "      [-inFormat <input format class>]\n"
           + "      [-keyClass <map input & output key class>]\n"
           + "      [-splitRandom <double pcnt> <numSamples> <maxsplits> | "
           + "// Sample from random splits at random (general)\n"
           + "       -splitSample <numSamples> <maxsplits> | "
           + "             // Sample from first records in splits (random data)\n"
           + "       -splitInterval <double pcnt> <maxsplits>]"
           + "             // Sample from splits at intervals (sorted data)");
   System.out.println("Default sampler: -splitRandom 0.1 10000 10");
   ToolRunner.printGenericCommandUsage(System.out);
   return -1;
 }
Exemplo n.º 20
0
  public int run(String[] args) throws Exception {

    if (args.length == 0) {
      System.err.println("Usage: MigrationTool <S3 file system URI>");
      System.err.println("\t<S3 file system URI>\tfilesystem to migrate");
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    URI uri = URI.create(args[0]);

    initialize(uri);

    FileSystemStore newStore = new Jets3tFileSystemStore();
    newStore.initialize(uri, getConf());

    if (get("%2F") != null) {
      System.err.println("Current version number is [unversioned].");
      System.err.println("Target version number is " + newStore.getVersion() + ".");
      Store oldStore = new UnversionedStore();
      migrate(oldStore, newStore);
      return 0;
    } else {
      S3Object root = get("/");
      if (root != null) {
        String version = (String) root.getMetadata("fs-version");
        if (version == null) {
          System.err.println("Can't detect version - exiting.");
        } else {
          String newVersion = newStore.getVersion();
          System.err.println("Current version number is " + version + ".");
          System.err.println("Target version number is " + newVersion + ".");
          if (version.equals(newStore.getVersion())) {
            System.err.println("No migration required.");
            return 0;
          }
          // use version number to create Store
          // Store oldStore = ...
          // migrate(oldStore, newStore);
          System.err.println("Not currently implemented.");
          return 0;
        }
      }
      System.err.println("Can't detect version - exiting.");
      return 0;
    }
  }
Exemplo n.º 21
0
  /**
   * Parse arguments and then runs a map/reduce job. Print output in standard out.
   *
   * @return a non-zero if there is an error. Otherwise, return 0.
   */
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.println("Usage: " + getClass().getName() + " <nMaps> <nSamples>");
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    final int nMaps = Integer.parseInt(args[0]);
    final long nSamples = Long.parseLong(args[1]);

    System.out.println("Number of Maps  = " + nMaps);
    System.out.println("Samples per Map = " + nSamples);

    final JobConf jobConf = new JobConf(getConf(), getClass());
    System.out.println("Estimated value of Pi is " + estimate(nMaps, nSamples, jobConf));
    return 0;
  }
Exemplo n.º 22
0
 /**
  * Displays format of commands.
  *
  * @param cmd The command that is being executed.
  */
 private static void printUsage(String cmd) {
   if ("-report".equals(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [-report]");
   } else if ("-safemode".equals(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [-safemode enter | leave | get | wait]");
   } else if ("-saveNamespace".equals(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [-saveNamespace]");
   } else if ("-refreshNodes".equals(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [-refreshNodes]");
   } else if ("-finalizeUpgrade".equals(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [-finalizeUpgrade]");
   } else if ("-upgradeProgress".equals(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [-upgradeProgress status | details | force]");
   } else if ("-metasave".equals(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [-metasave filename]");
   } else if (SetQuotaCommand.matches(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [" + SetQuotaCommand.USAGE + "]");
   } else if (ClearQuotaCommand.matches(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [" + ClearQuotaCommand.USAGE + "]");
   } else if (SetSpaceQuotaCommand.matches(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [" + SetSpaceQuotaCommand.USAGE + "]");
   } else if (ClearSpaceQuotaCommand.matches(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [" + ClearSpaceQuotaCommand.USAGE + "]");
   } else if ("-refreshServiceAcl".equals(cmd)) {
     System.err.println("Usage: java DFSAdmin" + " [-refreshServiceAcl]");
   } else {
     System.err.println("Usage: java DFSAdmin");
     System.err.println("           [-report]");
     System.err.println("           [-safemode enter | leave | get | wait]");
     System.err.println("           [-saveNamespace]");
     System.err.println("           [-refreshNodes]");
     System.err.println("           [-finalizeUpgrade]");
     System.err.println("           [-upgradeProgress status | details | force]");
     System.err.println("           [-metasave filename]");
     System.err.println("           [-refreshServiceAcl]");
     System.err.println("           [" + SetQuotaCommand.USAGE + "]");
     System.err.println("           [" + ClearQuotaCommand.USAGE + "]");
     System.err.println("           [" + SetSpaceQuotaCommand.USAGE + "]");
     System.err.println("           [" + ClearSpaceQuotaCommand.USAGE + "]");
     System.err.println("           [-help [cmd]]");
     System.err.println();
     ToolRunner.printGenericCommandUsage(System.err);
   }
 }
Exemplo n.º 23
0
  /**
   * Parse arguments and then runs a map/reduce job. Print output in standard out.
   *
   * @return a non-zero if there is an error. Otherwise, return 0.
   */
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.println("Usage: " + getClass().getName() + " <nMaps> <nSamples>");
      ToolRunner.printGenericCommandUsage(System.err);
      return 2;
    }

    final int nMaps = Integer.parseInt(args[0]);
    final long nSamples = Long.parseLong(args[1]);
    long now = System.currentTimeMillis();
    int rand = new Random().nextInt(Integer.MAX_VALUE);
    final Path tmpDir = new Path(TMP_DIR_PREFIX + "_" + now + "_" + rand);

    System.out.println("Number of Maps  = " + nMaps);
    System.out.println("Samples per Map = " + nSamples);

    System.out.println(
        "Estimated value of Pi is " + estimatePi(nMaps, nSamples, tmpDir, getConf()));
    return 0;
  }
  @SuppressWarnings("deprecation")
  @Override
  public int run(String[] args) throws Exception {

    if (args.length != 2) {
      System.err.printf(
          "Usage: %s [generic options] <input> <output>\n", getClass().getSimpleName());
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    // create JobConf
    JobConf jobConf = new JobConf(getConf(), this.getClass());
    CommonUtility.printConfiguration(jobConf);

    // set path for input and output
    FileInputFormat.addInputPath(jobConf, new Path(args[0]));
    FileOutputFormat.setOutputPath(jobConf, new Path(args[1]));

    // set format for input and output
    jobConf.setInputFormat(WholeFileInputFormat.class);
    jobConf.setOutputFormat(SequenceFileOutputFormat.class);

    // set class of output key and value
    jobConf.setOutputKeyClass(Text.class);
    jobConf.setOutputValueClass(BytesWritable.class);

    // set special attributes
    jobConf.setNumReduceTasks(2);

    // set mapper and reducer
    jobConf.setMapperClass(SmallFiles2SequenceFileMapper.class);
    jobConf.setReducerClass(IdentityReducer.class);

    jobConf.setNumReduceTasks(0);

    // run the job
    JobClient.runJob(jobConf);

    return 0;
  }
Exemplo n.º 25
0
  @Override
  public int run(String[] args) throws Exception {
    Configuration conf = new Configuration();
    GenericOptionsParser parser = new GenericOptionsParser(conf, args);
    String[] otherArgs = parser.getRemainingArgs();
    if (otherArgs.length != 3) {
      System.err.println("Usage: DistributedGrep <regex> <in> <out>");
      ToolRunner.printGenericCommandUsage(System.err);
      System.exit(2);
    }
    Job job = new Job(conf, "Distributed Grep");
    job.setJarByClass(DistributedGrep.class);
    job.setMapperClass(GrepMapper.class);
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(Text.class);
    job.getConfiguration().set(REGEX_KEY, otherArgs[0]);
    FileInputFormat.addInputPath(job, new Path(otherArgs[1]));
    FileOutputFormat.setOutputPath(job, new Path(otherArgs[2]));
    boolean success = job.waitForCompletion(true);

    return success ? 0 : 1;
  }
  @Override
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.printf(
          "Usage: %s [generic options] <input> <output>\n", getClass().getSimpleName());
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    Job job = new Job(getConf(), "Max temperature");
    job.setJarByClass(getClass());

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(MaxTemperatureMapper.class);
    job.setCombinerClass(MaxTemperatureReducer.class);
    job.setReducerClass(MaxTemperatureReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    return job.waitForCompletion(true) ? 0 : 1;
  }
  /** Runs this tool. */
  @SuppressWarnings({"static-access"})
  public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(
        OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INDEX));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("output path")
            .create(COLLECTION));

    CommandLine cmdline = null;
    CommandLineParser parser = new GnuParser();

    try {
      cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
      System.err.println("Error parsing command line: " + exp.getMessage());
      System.exit(-1);
    }

    if (!cmdline.hasOption(INDEX) || !cmdline.hasOption(COLLECTION)) {
      System.out.println("args: " + Arrays.toString(args));
      HelpFormatter formatter = new HelpFormatter();
      formatter.setWidth(120);
      formatter.printHelp(LookupPostingsCompressed.class.getName(), options);
      ToolRunner.printGenericCommandUsage(System.out);
      System.exit(-1);
    }

    String indexPath = cmdline.getOptionValue(INDEX);
    String collectionPath = cmdline.getOptionValue(COLLECTION);

    if (collectionPath.endsWith(".gz")) {
      System.out.println("gzipped collection is not seekable: use compressed version!");
      System.exit(-1);
    }

    Configuration config = new Configuration();
    FileSystem fs = FileSystem.get(config);
    MapFile.Reader reader = new MapFile.Reader(new Path(indexPath + "/part-r-00000"), config);

    FSDataInputStream collection = fs.open(new Path(collectionPath));
    BufferedReader d = new BufferedReader(new InputStreamReader(collection));

    Text key = new Text();
    ArrayListWritable<PairOfInts> postings;
    BytesWritable bytesValue = new BytesWritable();

    System.out.println("Looking up postings for the term \"starcross'd\"");
    key.set("starcross'd");

    reader.get(key, bytesValue);
    postings = deserializePosting(bytesValue);

    // ArrayListWritable<PairOfVInts> postings = value;
    for (PairOfInts pair : postings) {
      System.out.println(pair);
      collection.seek(pair.getLeftElement());
      System.out.println(d.readLine());
    }

    bytesValue = new BytesWritable();
    key.set("gold");
    reader.get(key, bytesValue);
    postings = deserializePosting(bytesValue);
    System.out.println(
        "Complete postings list for 'gold': (" + postings.size() + ", " + postings + ")");

    Int2IntFrequencyDistribution goldHist = new Int2IntFrequencyDistributionEntry();
    // postings = value;
    for (PairOfInts pair : postings) {
      goldHist.increment(pair.getRightElement());
    }

    System.out.println("histogram of tf values for gold");
    for (PairOfInts pair : goldHist) {
      System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
    }

    bytesValue = new BytesWritable();
    key.set("silver");
    reader.get(key, bytesValue);
    postings = deserializePosting(bytesValue);
    System.out.println(
        "Complete postings list for 'silver': (" + postings.size() + ", " + postings + ")");

    Int2IntFrequencyDistribution silverHist = new Int2IntFrequencyDistributionEntry();
    // postings = value;
    for (PairOfInts pair : postings) {
      silverHist.increment(pair.getRightElement());
    }

    System.out.println("histogram of tf values for silver");
    for (PairOfInts pair : silverHist) {
      System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
    }

    bytesValue = new BytesWritable();
    key.set("bronze");
    Writable w = reader.get(key, bytesValue);

    if (w == null) {
      System.out.println("the term bronze does not appear in the collection");
    }

    collection.close();
    reader.close();

    return 0;
  }
Exemplo n.º 28
0
  @SuppressWarnings("static-access")
  private int parseArgs(String[] args) {
    Options options = new Options();
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("Tfidf vectors")
            .create(Fields.INPUT.get()));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("Vectors' length")
            .create(Fields.BASIS.get()));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("Near documents")
            .create(Fields.OUTPUT.get()));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("Document dates")
            .create(Fields.DOC_DATES.get()));
    options.addOption(
        OptionBuilder.withDescription("Ignore docs without NN").create(Fields.IGNORE.get()));
    options.addOption(
        OptionBuilder.withDescription("Output buckets").create(Fields.OUTPUT_BUCKETS.get()));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("Number of bands")
            .create(Fields.BANDS.get()));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("Number of rows")
            .create(Fields.ROWS.get()));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();
    try {
      cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
      System.err.println("Error parsing command line: " + exp.getMessage());
      return -1;
    }

    if (!cmdline.hasOption(Fields.INPUT.get())
        || !cmdline.hasOption(Fields.OUTPUT.get())
        || !cmdline.hasOption(Fields.BASIS.get())
        || (!cmdline.hasOption(Fields.DOC_DATES.get())
            && !cmdline.hasOption(Fields.OUTPUT_BUCKETS.get()))) {
      HelpFormatter formatter = new HelpFormatter();
      formatter.printHelp(this.getClass().getName(), options);
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    inputPath = cmdline.getOptionValue(Fields.INPUT.get());
    outputPath = cmdline.getOptionValue(Fields.OUTPUT.get());
    basisPath = cmdline.getOptionValue(Fields.BASIS.get());
    datesPath = cmdline.getOptionValue(Fields.DOC_DATES.get());

    ignoreDocs = false;
    if (cmdline.hasOption(Fields.IGNORE.get())) {
      ignoreDocs = true;
    }
    outputBuckets = false;
    if (cmdline.hasOption(Fields.OUTPUT_BUCKETS.get())) {
      outputBuckets = true;
    }
    bands = -1;
    if (cmdline.hasOption(Fields.BANDS.get())) {
      bands = Integer.parseInt(cmdline.getOptionValue(Fields.BANDS.get()));
    }
    rows = -1;
    if (cmdline.hasOption(Fields.ROWS.get())) {
      rows = Integer.parseInt(cmdline.getOptionValue(Fields.ROWS.get()));
    }

    logger.info("Tool name: " + this.getClass().getName());
    logger.info(" - input: " + inputPath);
    logger.info(" - basis: " + basisPath);
    logger.info(" - output: " + outputPath);
    logger.info(" - dates: " + datesPath);
    logger.info(" - ignore: " + ignoreDocs);
    logger.info(" - outputBuckets: " + outputBuckets);
    logger.info(" - bands: " + bands);
    logger.info(" - rows: " + rows);

    return 0;
  }
Exemplo n.º 29
0
  @SuppressWarnings("static-access")
  private int parseArgs(String[] args) {
    Options options = new Options();
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("Tfidf vectors")
            .create(Fields.INPUT.get()));
    options.addOption(
        OptionBuilder.withArgName("path")
            .hasArg()
            .withDescription("Selected articles")
            .create(Fields.OUTPUT.get()));
    options.addOption(
        OptionBuilder.withArgName("integer")
            .hasArg()
            .withDescription("Partition count")
            .create(Fields.PARTITION_COUNT.get()));
    options.addOption(
        OptionBuilder.withArgName("integer")
            .hasArg()
            .withDescription("Select count")
            .create(Fields.SELECT_COUNT.get()));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();
    try {
      cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
      logger.fatal("Error parsing command line: " + exp.getMessage());
      return -1;
    }

    if (!cmdline.hasOption(Fields.INPUT.get()) || !cmdline.hasOption(Fields.OUTPUT.get())) {
      HelpFormatter formatter = new HelpFormatter();
      formatter.printHelp(this.getClass().getName(), options);
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    inputPath = cmdline.getOptionValue(Fields.INPUT.get());
    outputPath = cmdline.getOptionValue(Fields.OUTPUT.get());

    partitionCount = Defaults.PARTITION_COUNT.get();
    if (cmdline.hasOption(Fields.PARTITION_COUNT.get())) {
      partitionCount = Integer.parseInt(cmdline.getOptionValue(Fields.PARTITION_COUNT.get()));
      if (partitionCount <= 0) {
        System.err.println("Error: \"" + partitionCount + "\" has to be positive!");
        return -1;
      }
    }
    selectCount = Integer.parseInt(cmdline.getOptionValue(Fields.SELECT_COUNT.get()));

    logger.info("Tool name: " + this.getClass().getName());
    logger.info(" - input: " + inputPath);
    logger.info(" - output: " + outputPath);
    logger.info(" - partitions: " + partitionCount);
    logger.info(" - select: " + selectCount);

    return 0;
  }
 private static int printUsage() {
   System.out.println("usage: [inputDir] [outputDir] [numNodes]");
   ToolRunner.printGenericCommandUsage(System.out);
   return -1;
 }