Beispiel #1
0
  public static void main(String[] args) throws Exception {

    final String NAME_NODE = "hdfs://sandbox.hortonworks.com:8020";
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf);
    job.setJarByClass(WordCount.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(NullWritable.class);
    if (args.length > 2) {
      job.setNumReduceTasks(Integer.parseInt(args[2]));
    }

    job.setMapperClass(CountMapper.class);
    job.setReducerClass(CountReducer.class);
    job.setJarByClass(WordCount.class);
    job.setNumReduceTasks(1);

    FileInputFormat.addInputPath(job, new Path(args[0] + "data/plot_summaries.txt"));
    FileSystem fs = FileSystem.get(conf);
    // handle (e.g. delete) existing output path
    Path outputDestination = new Path(args[0] + args[1]);
    if (fs.exists(outputDestination)) {
      fs.delete(outputDestination, true);
    }

    // set output path & start job1
    FileOutputFormat.setOutputPath(job, outputDestination);
    int jobCompletionStatus = job.waitForCompletion(true) ? 0 : 1;
  }
  public void testFormat() throws Exception {
    localFs = FileSystem.getLocal(defaultConf);
    localFs.delete(workDir, true);

    Job job = new Job(new Configuration(defaultConf));
    Path file = new Path(workDir, "test.txt");

    int seed = new Random().nextInt();
    Random random = new Random(seed);

    // for a variety of lengths
    for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
      // create a file with length entries
      Writer writer = new OutputStreamWriter(localFs.create(file));
      try {
        MyClass mc = new MyClass();
        for (int i = 0; i < length; i++) {
          mc.s = Integer.toString(i);
          mc.v = i;
          byte[] raw = MessagePack.pack(mc);
          byte[] b64e = base64_.encodeBase64(raw);
          byte[] b64d = base64_.decode(b64e);
          MyClass mc2 = MessagePack.unpack(b64d, mc.getClass());
          assertEquals(mc.s, mc2.s);
          assertEquals(mc.v, mc2.v);

          writer.write(base64_.encodeToString(raw));
        }
      } finally {
        writer.close();
      }
      checkFormat(job);
    }
  }
 private int getNumSecrets(Path secretsPath) throws Exception {
   FileSystem fileSystem = FileSystem.get(secretsPath.toUri(), getConf());
   FSDataInputStream inputStream = fileSystem.open(secretsPath);
   BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
   String currentLine;
   int numLines = 0;
   while ((currentLine = reader.readLine()) != null) {
     if (!currentLine.isEmpty()) {
       numLines++;
     }
   }
   reader.close();
   return numLines;
 }
 private void generateSaltIfNeeded(Path saltFilePath, Path secretsPath) throws Exception {
   FileSystem fileSystem = FileSystem.get(saltFilePath.toUri(), getConf());
   if (!fileSystem.exists(saltFilePath)) {
     FSDataOutputStream outputStream = fileSystem.create(saltFilePath);
     BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream));
     int numSaltsToGenerate = getNumSecrets(secretsPath);
     System.out.printf("Generating %d salts\n", numSaltsToGenerate);
     for (int i = 0; i < numSaltsToGenerate; i++) {
       writer.write(BCrypt.gensalt());
       writer.newLine();
     }
     writer.close();
     outputStream.close();
   }
 }
 private Job configureJob(Path secretsPath, Path saltFilePath, Path inputPath, Path outputPath)
     throws Exception {
   Job job = Job.getInstance(getConf());
   FileInputFormat.setInputPaths(job, inputPath);
   FileOutputFormat.setOutputPath(job, outputPath);
   job.getConfiguration().set(ObfuscateMapper.SECRET_WORDS_FILE_KEY, secretsPath.toString());
   job.getConfiguration().set(ObfuscateMapper.SALT_FILE_KEY, saltFilePath.toString());
   job.setInputFormatClass(TextInputFormat.class);
   job.setMapperClass(ObfuscateMapper.class);
   job.setNumReduceTasks(0);
   job.setJarByClass(getClass());
   FileSystem.get(outputPath.toUri(), getConf()).delete(outputPath, true);
   return job;
 }