private static void setRecordCount(State state, Job job) {

    Counters counters = null;
    try {
      counters = job.getCounters();
    } catch (IOException e) {
      LOG.info("Failed to get job counters. Record count will not be set. ", e);
      return;
    }

    Counter recordCounter = counters.findCounter(AvroKeyDedupReducer.EVENT_COUNTER.RECORD_COUNT);

    if (recordCounter != null && recordCounter.getValue() != 0) {
      state.setProp(SlaEventKeys.RECORD_COUNT_KEY, Long.toString(recordCounter.getValue()));
      return;
    }

    recordCounter = counters.findCounter(AvroKeyMapper.EVENT_COUNTER.RECORD_COUNT);

    if (recordCounter != null && recordCounter.getValue() != 0) {
      state.setProp(SlaEventKeys.RECORD_COUNT_KEY, Long.toString(recordCounter.getValue()));
      return;
    }

    LOG.info("Non zero record count not found in both mapper and reducer counters");
  }
Exemple #2
0
  private void validateFileCounters(
      org.apache.hadoop.mapreduce.Counters counter,
      long fileBytesRead,
      long fileBytesWritten,
      long mapOutputBytes,
      long mapOutputMaterializedBytes) {
    assertTrue(
        counter
                .findCounter(
                    org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.BYTES_READ)
                .getValue()
            != 0);
    assertEquals(
        fileBytesRead,
        counter
            .findCounter(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.BYTES_READ)
            .getValue());

    assertTrue(
        counter
                .findCounter(
                    org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.BYTES_WRITTEN)
                .getValue()
            != 0);

    if (mapOutputBytes >= 0) {
      assertTrue(counter.findCounter(MAP_OUTPUT_BYTES).getValue() != 0);
    }
    if (mapOutputMaterializedBytes >= 0) {
      assertTrue(counter.findCounter(MAP_OUTPUT_MATERIALIZED_BYTES).getValue() != 0);
    }
  }
Exemple #3
0
  /**
   * Performs an HDF to text operation as a MapReduce job and returns total number of points
   * generated.
   *
   * @param inPath
   * @param outPath
   * @param datasetName
   * @param skipFillValue
   * @return
   * @throws IOException
   * @throws ClassNotFoundException
   * @throws InterruptedException
   */
  public static long HDFToTextMapReduce(
      Path inPath, Path outPath, String datasetName, boolean skipFillValue, OperationsParams params)
      throws IOException, InterruptedException, ClassNotFoundException {
    Job job = new Job(params, "HDFToText");
    Configuration conf = job.getConfiguration();
    job.setJarByClass(HDFToText.class);
    job.setJobName("HDFToText");

    // Set Map function details
    job.setMapperClass(HDFToTextMap.class);
    job.setNumReduceTasks(0);

    // Set input information
    job.setInputFormatClass(SpatialInputFormat3.class);
    SpatialInputFormat3.setInputPaths(job, inPath);
    if (conf.get("shape") == null) conf.setClass("shape", NASAPoint.class, Shape.class);
    conf.set("dataset", datasetName);
    conf.setBoolean("skipfillvalue", skipFillValue);

    // Set output information
    job.setOutputFormatClass(TextOutputFormat3.class);
    TextOutputFormat3.setOutputPath(job, outPath);

    // Run the job
    boolean verbose = conf.getBoolean("verbose", false);
    job.waitForCompletion(verbose);
    Counters counters = job.getCounters();
    Counter outputRecordCounter = counters.findCounter(Task.Counter.MAP_OUTPUT_RECORDS);
    final long resultCount = outputRecordCounter.getValue();

    return resultCount;
  }
  @Test
  public void testJSONBulkImporter() throws Exception {
    // Prepare input file:
    File inputFile = File.createTempFile("TestJSONImportInput", ".txt", getLocalTempDir());
    TestingResources.writeTextFile(
        inputFile, TestingResources.get(BulkImporterTestUtils.JSON_IMPORT_DATA));

    Configuration conf = getConf();
    conf.set(
        DescribedInputTextBulkImporter.CONF_FILE,
        BulkImporterTestUtils.localResource(BulkImporterTestUtils.FOO_IMPORT_DESCRIPTOR));

    // Run the bulk-import:
    final KijiMapReduceJob job =
        KijiBulkImportJobBuilder.create()
            .withConf(conf)
            .withBulkImporter(JSONBulkImporter.class)
            .withInput(MapReduceJobInputs.newTextMapReduceJobInput(new Path(inputFile.toString())))
            .withOutput(new DirectKijiTableMapReduceJobOutput(mTable.getURI()))
            .build();
    assertTrue(job.run());

    final Counters counters = job.getHadoopJob().getCounters();
    assertEquals(
        3, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_PROCESSED).getValue());
    assertEquals(
        1, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_INCOMPLETE).getValue());
    assertEquals(
        0, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_REJECTED).getValue());

    // Validate output:
    final KijiRowScanner scanner = mReader.getScanner(KijiDataRequest.create("info"));
    BulkImporterTestUtils.validateImportedRows(scanner, false);
    scanner.close();
  }
Exemple #5
0
 private void validateCounters(
     org.apache.hadoop.mapreduce.Counters counter,
     long spillRecCnt,
     long mapInputRecords,
     long mapOutputRecords) {
   // Check if the numer of Spilled Records is same as expected
   assertEquals(spillRecCnt, counter.findCounter(SPILLED_RECORDS).getValue());
   assertEquals(mapInputRecords, counter.findCounter(MAP_INPUT_RECORDS).getValue());
   assertEquals(mapOutputRecords, counter.findCounter(MAP_OUTPUT_RECORDS).getValue());
 }
 @Test(expected = IOException.class)
 public final void testMapperForNullKeyValue() throws IOException, InterruptedException {
   Mapper.Context context = mock(Mapper.Context.class);
   Counters counters = new Counters();
   Counter counter = counters.findCounter(MergeRecordCounter.BAD_RECORD);
   when(context.getCounter(MergeRecordCounter.BAD_RECORD)).thenReturn(counter);
   MergeKeyMapper mapper = new MergeKeyMapper();
   Text val = new Text("valueOfKey");
   mapper.map(null, val, context);
 }
Exemple #7
0
 private Counters getCounters() {
   Counters c = new Counters();
   Map<String, Map<String, Long>> values = counters.value();
   for (Map.Entry<String, Map<String, Long>> e : values.entrySet()) {
     CounterGroup cg = c.getGroup(e.getKey());
     for (Map.Entry<String, Long> f : e.getValue().entrySet()) {
       cg.findCounter(f.getKey()).setValue(f.getValue());
     }
   }
   return c;
 }
 private void validateCounters(org.apache.hadoop.mapreduce.Counters counters) {
   Iterator<org.apache.hadoop.mapreduce.CounterGroup> it = counters.iterator();
   while (it.hasNext()) {
     org.apache.hadoop.mapreduce.CounterGroup group = it.next();
     LOG.info("Group " + group.getDisplayName());
     Iterator<org.apache.hadoop.mapreduce.Counter> itc = group.iterator();
     while (itc.hasNext()) {
       LOG.info("Counter is " + itc.next().getDisplayName());
     }
   }
   Assert.assertEquals(1, counters.countCounters());
 }
  private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {

    Counter slotMillisMapCounter = allCounters.findCounter(JobCounter.SLOTS_MILLIS_MAPS);
    if (slotMillisMapCounter != null) {
      summary.setMapSlotSeconds(slotMillisMapCounter.getValue() / 1000);
    }

    Counter slotMillisReduceCounter = allCounters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES);
    if (slotMillisReduceCounter != null) {
      summary.setReduceSlotSeconds(slotMillisReduceCounter.getValue() / 1000);
    }
  }
Exemple #10
0
 static Counters fromAvro(JhCounters counters) {
   Counters result = new Counters();
   for (JhCounterGroup g : counters.groups) {
     CounterGroup group =
         result.addGroup(
             StringInterner.weakIntern(g.name.toString()),
             StringInterner.weakIntern(g.displayName.toString()));
     for (JhCounter c : g.counts) {
       group.addCounter(
           StringInterner.weakIntern(c.name.toString()),
           StringInterner.weakIntern(c.displayName.toString()),
           c.value);
     }
   }
   return result;
 }
 @Test
 public void parsesMalformedTemperature() throws IOException, InterruptedException {
   Text value =
       new Text(
           "0335999999433181957042302005+37950+139117SAO +0004"
               + // Year ^^^^
               "RJSN V02011359003150070356999999433201957010100005+353"); // Temperature ^^^^^
   Counters counters = new Counters();
   new MapDriver<LongWritable, Text, Text, IntWritable>()
       .withMapper(new MaxTemperatureMapper())
       .withInput(new LongWritable(0), value)
       .withOutput(new Text("1957"), new IntWritable(19570))
       .withCounters(counters)
       .runTest();
   Counter c = counters.findCounter(MaxTemperatureMapper.Temperature.OVER_100);
   assertThat(c.getValue(), is(1L));
 }
 @Test(dataProvider = "simpleTest")
 public void countersTest(String inString, String[] splits, int lineNum, int wordCount)
     throws IOException {
   mapDriver.withInput(new LongWritable(), new Text(inString));
   for (String split : splits) {
     mapDriver.withOutput(new Text(split), new IntWritable(1));
   }
   mapDriver.runTest();
   Counters counters = mapDriver.getCounters();
   assertEquals(
       counters.findCounter(WordCountWithTestsMapper.Status.LINES_NUM).getValue(),
       lineNum,
       "Wrong lines num!");
   assertEquals(
       counters.findCounter(WordCountWithTestsMapper.Status.WORD_COUNT).getValue(),
       wordCount,
       "Wrong word count!");
 }
 protected Set<Integer> getNeeds(String type, Counters counts) {
   CounterGroup group = counts.getGroup(type);
   HashSet<Integer> result = new HashSet<Integer>(group.size());
   for (Counter counter : group) {
     String name = counter.getName();
     assert name.startsWith("t");
     result.add(Integer.parseInt(name.substring(1)));
   }
   return result;
 }
  @Test
  public final void testMapperValidValues() throws IOException, InterruptedException {
    Mapper.Context context = mock(Mapper.Context.class);
    Counters counters = new Counters();
    Counter counter = counters.findCounter(MergeRecordCounter.TOTAL_RECORDS_NEW);
    when(context.getCounter(MergeRecordCounter.TOTAL_RECORDS_NEW)).thenReturn(counter);

    MergeKeyMapper mapper = new MergeKeyMapper();
    Text key = new Text("abc123");
    Text val = new Text("valueOfKey");
    mapper.isOld = false;
    mapper.map(key, val, context);

    HihoValue hihoValue = new HihoValue();
    hihoValue.setVal(val);
    hihoValue.setIsOld(false);
    HihoTuple hihoTuple = new HihoTuple();
    hihoTuple.setKey(key);
    verify(context).write(hihoTuple, hihoValue);
    assertEquals(1, context.getCounter(MergeRecordCounter.TOTAL_RECORDS_NEW).getValue());
  }
Exemple #15
0
  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    Job job = new Job(conf, "Join");

    job.setJarByClass(Join.class);
    job.setMapperClass(MapClass.class);
    job.setPartitionerClass(PartitionerClass.class);
    job.setReducerClass(ReduceClass.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    // NOTE we use two different input paths in here.
    FileInputFormat.addInputPath(job, new Path(args[Utils.argInIndex]));
    FileInputFormat.addInputPath(job, new Path(args[Utils.argInIndex + 1]));

    FileOutputFormat.setOutputPath(job, new Path(args[Utils.argInIndex + 2]));

    boolean result = job.waitForCompletion(true);

    // Write totalRecord and totalBytes from task counter to file.
    if (result) {
      Counters counters = job.getCounters();

      long totalRecords =
          counters
              .findCounter("org.apache.hadoop.mapred.Task$Counter", "MAP_OUTPUT_RECORDS")
              .getValue();
      long totalBytes =
          counters
              .findCounter("org.apache.hadoop.mapred.Task$Counter", "MAP_OUTPUT_BYTES")
              .getValue();

      Utils.uploadCountersToS3(totalRecords, totalBytes, "Join");
    }

    System.exit(result ? 0 : 1);
  }
 private void validateCounters(
     Counters counters,
     long mapInputRecords,
     long mapOutputRecords,
     long reduceInputGroups,
     long reduceOutputRecords) {
   assertEquals(
       "MapInputRecords",
       mapInputRecords,
       counters.findCounter("MyCounterGroup", "MAP_INPUT_RECORDS").getValue());
   assertEquals(
       "MapOutputRecords",
       mapOutputRecords,
       counters.findCounter("MyCounterGroup", "MAP_OUTPUT_RECORDS").getValue());
   assertEquals(
       "ReduceInputGroups",
       reduceInputGroups,
       counters.findCounter("MyCounterGroup", "REDUCE_INPUT_GROUPS").getValue());
   assertEquals(
       "ReduceOutputRecords",
       reduceOutputRecords,
       counters.findCounter("MyCounterGroup", "REDUCE_OUTPUT_RECORDS").getValue());
 }
  public boolean verify(long expectedReferenced) throws Exception {
    if (job == null) {
      throw new IllegalStateException("You should call run() first");
    }

    Counters counters = job.getCounters();

    Counter referenced = counters.findCounter(Counts.REFERENCED);
    Counter unreferenced = counters.findCounter(Counts.UNREFERENCED);
    Counter undefined = counters.findCounter(Counts.UNDEFINED);

    boolean success = true;
    // assert
    if (expectedReferenced != referenced.getValue()) {
      LOG.error(
          "Expected referenced count does not match with actual referenced count. "
              + "expected referenced="
              + expectedReferenced
              + " ,actual="
              + referenced.getValue());
      success = false;
    }

    if (unreferenced.getValue() > 0) {
      LOG.error(
          "Unreferenced nodes were not expected. Unreferenced count=" + unreferenced.getValue());
      success = false;
    }

    if (undefined.getValue() > 0) {
      LOG.error("Found an undefined node. Undefined count=" + undefined.getValue());
      success = false;
    }

    return success;
  }
Exemple #18
0
  /**
   * Saves the centroids between K-means iterations.
   *
   * @param counters - the counters containing the centroids
   */
  public static void setCentroids(Counters counters) throws Exception {
    Configuration conf = setupConf();
    FileSystem fs = FileSystem.get(conf);
    Path path = new Path(BASE_OUTPUT);
    Path tempPath = new Path(BASE_OUTPUT + TEMP_FILE);
    FSDataOutputStream out = null;

    if (!fs.exists(path)) fs.mkdirs(path);
    path = new Path(BASE_OUTPUT + CENTROID_FILE);

    if (fs.exists(path)) fs.rename(path, tempPath);
    out = fs.create(path);
    out.writeUTF(new Long(counters.findCounter(Common.Centroids.ITERATION).getValue()).toString());
    out.writeChar(Common.SEPARATOR.charAt(0));

    out.writeUTF(Common.Centroids.LOW.toString());
    out.writeChar(Common.SEPARATOR.charAt(0));
    out.writeUTF(new Long(counters.findCounter(Common.Centroids.LOW).getValue()).toString());
    out.writeChar(Common.SEPARATOR.charAt(0));
    out.writeUTF(new Long(counters.findCounter(Common.Centroids.COUNT_LOW).getValue()).toString());
    out.writeChar(Common.SEPARATOR.charAt(0));

    out.writeUTF(Common.Centroids.MEDIUM.toString());
    out.writeChar(Common.SEPARATOR.charAt(0));
    out.writeUTF(new Long(counters.findCounter(Common.Centroids.MEDIUM).getValue()).toString());
    out.writeChar(Common.SEPARATOR.charAt(0));
    out.writeUTF(
        new Long(counters.findCounter(Common.Centroids.COUNT_MEDIUM).getValue()).toString());
    out.writeChar(Common.SEPARATOR.charAt(0));

    out.writeUTF(Common.Centroids.HIGH.toString());
    out.writeChar(Common.SEPARATOR.charAt(0));
    out.writeUTF(new Long(counters.findCounter(Common.Centroids.HIGH).getValue()).toString());
    out.writeChar(Common.SEPARATOR.charAt(0));
    out.writeUTF(new Long(counters.findCounter(Common.Centroids.COUNT_HIGH).getValue()).toString());
    out.writeChar('\n');

    if (fs.exists(tempPath)) {
      FSDataInputStream in = fs.open(tempPath);
      int i = 0;
      while ((i = in.read()) != -1) out.write(i);
      in.close();
      fs.delete(tempPath, false);
    }

    out.close();
  }
Exemple #19
0
 public static Counter getCounter(String group, String name) {
   return COUNTERS.findCounter(group, name);
 }
Exemple #20
0
  public static void main(String args[]) throws IOException {
    String inputFile = "/corpus/pagerank";
    long start = System.currentTimeMillis();
    long ncPageCount = 0;
    long pageCount = 0;
    int iterationCount = 0;
    try {
      do {
        log.info("第" + iterationCount + "轮迭代开始");
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "PageRank-Job-" + iterationCount);
        job.setJarByClass(PageRankJob.class);
        job.setInputFormatClass(TextInputFormat.class);
        FileInputFormat.addInputPath(job, new Path(inputFile));
        job.setMapperClass(PageRankMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(PRWritable.class);

        job.setReducerClass(PageRankReducer.class);
        job.setOutputFormatClass(TextOutputFormat.class);
        String outputFile = "/output/" + System.currentTimeMillis();
        FileOutputFormat.setOutputPath(job, new Path(outputFile));
        if (!job.waitForCompletion(true)) {
          System.err.println("PageRank Job Failed");
        }
        inputFile = outputFile + "/part-r-00000";
        Counters counters = job.getCounters();
        long danglingPagePr = counters.findCounter(PRCounters.DANGLING_PAGE_PR).getValue();
        pageCount = counters.findCounter(PRCounters.PAGES_COUNT).getValue();

        /** 第二阶段任务开始 */
        conf = new Configuration();
        conf.set("DANGLING_PAGE_PR", String.valueOf(danglingPagePr));
        conf.set("PAGES_COUNT", String.valueOf(pageCount));
        Job prCompleteJob = Job.getInstance(conf, "PRComplete-Job-" + iterationCount);
        prCompleteJob.setJarByClass(PageRankJob.class);
        prCompleteJob.setInputFormatClass(TextInputFormat.class);
        FileInputFormat.addInputPath(prCompleteJob, new Path(inputFile));
        prCompleteJob.setMapperClass(PRCompleteMapper.class);
        prCompleteJob.setMapOutputKeyClass(NullWritable.class);
        prCompleteJob.setMapOutputValueClass(Text.class);
        prCompleteJob.setNumReduceTasks(0);
        outputFile = "/output/" + System.currentTimeMillis();
        FileOutputFormat.setOutputPath(prCompleteJob, new Path(outputFile));
        if (!prCompleteJob.waitForCompletion(true)) {
          System.err.println("PRComplete Job Failed");
        }
        inputFile = outputFile + "/part-m-00000";
        counters = prCompleteJob.getCounters();
        ncPageCount = counters.findCounter(PRCounters.NON_CONVERGING_PAGES).getValue();
        iterationCount++;
      } while (ncPageCount > (long) (pageCount * THRESHOLD));
      log.info("经过" + iterationCount + "轮迭代,计算结束");
      log.info("总用时:" + (System.currentTimeMillis() - start) + "ms");
    } catch (IOException e) {
      e.printStackTrace();
    } catch (InterruptedException e) {
      e.printStackTrace();
    } catch (ClassNotFoundException e) {
      e.printStackTrace();
    }
  }
  @Override
  public int run(String[] args) throws Exception {
    addInputOption();
    addOutputOption();

    addOption("inputDelimiter", "id", "입력 컬럼 구분자", Delimiter.COMMA.getDelimiter());
    addOption("outputDelimiter", "od", "출력 컬럼 구분자", Delimiter.COMMA.getDelimiter());
    addOption("sequenceIndex", "si", "시퀀스를 삽입할 컬럼의 인덱스(0부터 시작, 기본값 0)", "0");
    addOption("startNumber", "sn", "시퀀스 번호의 시작값(기본값 0)", "0");
    addOption("columnSize", "cs", "컬럼의 개수", true);
    addOption(
        "generateType", "gt", "생성할 시퀀스의 유형(SEQUENCE, TIMESTAMP)", GenerateType.SEQUENCE.getType());
    addOption("dateFormat", "df", "날짜 패턴(SimpleDateFormat)", /*FIXME*/ "yyyyMMdd");

    Map<String, String> parsedArgs = parseArguments(args);
    if (parsedArgs == null) {
      return JOB_FAIL;
    }

    ////////////////////////////////////////
    // Line Count Hadoop Job
    ///////////////////////////////////////

    // 임시 디렉토리를 가져온다. flamingo-mapreduce-site.xml 파일에 기본값이 정의되어 있다.
    Path temporaryPath = getTimestampTempPath();
    logger.info("Temporary Path : {}", temporaryPath.toString());

    Job lineCountJob =
        prepareJob(
            getInputPath(),
            temporaryPath,
            TextInputFormat.class,
            LineCountMapper.class,
            NullWritable.class,
            Text.class,
            TextOutputFormat.class);

    boolean step1 = lineCountJob.waitForCompletion(true);
    if (!step1) {
      return JOB_FAIL;
    }

    ////////////////////////////////////////////////
    // Calculating a start number per input split
    ////////////////////////////////////////////////

    // Counter에는 각 Mapper 별로 파일별 위치와 총 ROW의 개수를 포함하고 있다.
    Counters counters = lineCountJob.getCounters();
    CounterGroup group = counters.getGroup(LineCountMapper.class.getName());

    // 파일의 위치별로 정렬한다.
    TreeMap<Long, Long> counterMap = new TreeMap<Long, Long>();
    for (Counter counter : group) {
      try {
        counterMap.put(Long.parseLong(counter.getName()), counter.getValue());
      } catch (NumberFormatException ex) {
      }
    }

    ////////////////////////////////////////
    // Generate Sequence Hadoop Job
    ///////////////////////////////////////

    Job generateSequenceJob =
        prepareJob(
            getInputPath(),
            getOutputPath(),
            TextInputFormat.class,
            GenerateSequenceMapper.class,
            NullWritable.class,
            Text.class,
            TextOutputFormat.class);

    generateSequenceJob
        .getConfiguration()
        .set("inputDelimiter", parsedArgs.get("--inputDelimiter"));
    generateSequenceJob
        .getConfiguration()
        .set("outputDelimiter", parsedArgs.get("--outputDelimiter"));
    generateSequenceJob.getConfiguration().set("sequenceIndex", parsedArgs.get("--sequenceIndex"));
    generateSequenceJob.getConfiguration().set("generateType", parsedArgs.get("--generateType"));
    generateSequenceJob.getConfiguration().set("columnSize", parsedArgs.get("--columnSize"));

    if (GenerateType.valueOf(parsedArgs.get("--generateType"))
        .equals(GenerateType.SEQUENCE)) { // SEQUENCE
      generateSequenceJob.getConfiguration().set("startNumber", parsedArgs.get("--startNumber"));
      int index = generateSequenceJob.getConfiguration().getInt("startNumber", 0);

      for (long position : counterMap.keySet()) {
        generateSequenceJob.getConfiguration().set(String.valueOf(position), String.valueOf(index));
        index += counterMap.get(position);
      }
    } else { // TIMESTAMP
      generateSequenceJob.getConfiguration().set("dateFormat", parsedArgs.get("--dateFormat"));
    }

    boolean step2 = generateSequenceJob.waitForCompletion(true);
    if (!step2) {
      return JOB_FAIL;
    }

    try {
      // 임시 경로를 삭제한다.
      FileSystem.get(generateSequenceJob.getConfiguration()).delete(temporaryPath, true);
      logger.info("Now removed {}", temporaryPath.toString());
    } catch (Exception ex) {
      // Exception handling is not need.
    }
    return JOB_SUCCESS;
  }
  @Test
  public void testRedirect() throws Exception {

    Configuration conf = new YarnConfiguration();
    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
    conf.set(YarnConfiguration.RM_ADDRESS, RMADDRESS);
    conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, HSHOSTADDRESS);

    // Start the RM.
    RMService rmService = new RMService("test");
    rmService.init(conf);
    rmService.start();

    // Start the AM.
    AMService amService = new AMService();
    amService.init(conf);
    amService.start(conf);

    // Start the HS.
    HistoryService historyService = new HistoryService();
    historyService.init(conf);
    historyService.start(conf);

    LOG.info("services started");

    Cluster cluster = new Cluster(conf);
    org.apache.hadoop.mapreduce.JobID jobID = new org.apache.hadoop.mapred.JobID("201103121733", 1);
    org.apache.hadoop.mapreduce.Counters counters = cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(amContact);

    LOG.info(
        "Sleeping for 5 seconds before stop for"
            + " the client socket to not get EOF immediately..");
    Thread.sleep(5000);

    // bring down the AM service
    amService.stop();

    LOG.info("Sleeping for 5 seconds after stop for" + " the server to exit cleanly..");
    Thread.sleep(5000);

    amRestarting = true;

    // Same client
    // results are returned from fake (not started job)
    counters = cluster.getJob(jobID).getCounters();
    Assert.assertEquals(0, counters.countCounters());
    Job job = cluster.getJob(jobID);
    org.apache.hadoop.mapreduce.TaskID taskId =
        new org.apache.hadoop.mapreduce.TaskID(jobID, TaskType.MAP, 0);
    TaskAttemptID tId = new TaskAttemptID(taskId, 0);

    // invoke all methods to check that no exception is thrown
    job.killJob();
    job.killTask(tId);
    job.failTask(tId);
    job.getTaskCompletionEvents(0, 100);
    job.getStatus();
    job.getTaskDiagnostics(tId);
    job.getTaskReports(TaskType.MAP);
    job.getTrackingURL();

    amRestarting = false;
    amService = new AMService();
    amService.init(conf);
    amService.start(conf);
    amContact = false; // reset

    counters = cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(amContact);

    // Stop the AM. It is not even restarting. So it should be treated as
    // completed.
    amService.stop();

    // Same client
    counters = cluster.getJob(jobID).getCounters();
    validateCounters(counters);
    Assert.assertTrue(hsContact);

    rmService.stop();
    historyService.stop();
  }
  public static void main(String[] args)
      throws IOException, InterruptedException, ClassNotFoundException {

    Path inputPath = new Path(args[0]);
    Path outputDir = new Path(args[1]);

    // Create configuration
    Configuration conf = new Configuration(true);

    // Create job
    @SuppressWarnings("deprecation")
    Job job = new Job(conf, "CountryIncomeConf");
    job.setJarByClass(CountryIncomeConf.class);

    // Decompressing .gz file Ex. foo.csv.gz to foo.csv

    String uri = args[0];
    FileSystem fs = FileSystem.get(URI.create(uri), conf);

    CompressionCodecFactory factory = new CompressionCodecFactory(conf);
    CompressionCodec codec = factory.getCodec(inputPath);
    if (codec == null) {
      System.err.println("No codec found for " + uri);
      System.exit(1);
    }
    String outputUri = CompressionCodecFactory.removeSuffix(uri, codec.getDefaultExtension());
    InputStream in = null;
    OutputStream out = null;
    try {
      in = codec.createInputStream(fs.open(inputPath));
      out = fs.create(new Path(outputUri));
      IOUtils.copyBytes(in, out, conf);
    } finally {
      IOUtils.closeStream(in);
      IOUtils.closeStream(out);
    }

    // Setup MapReduce
    job.setMapperClass(CountryIncomeMapper.class);
    job.setReducerClass(CountryIncomeReducer.class);
    job.setNumReduceTasks(1);

    // Specify key / value
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    // Input
    // FileInputFormat.addInputPath(job, inputPath);
    FileInputFormat.addInputPaths(job, outputUri);
    job.setInputFormatClass(TextInputFormat.class);

    // Output
    FileOutputFormat.setOutputPath(job, outputDir);
    job.setOutputFormatClass(TextOutputFormat.class);

    // Delete output if exists
    FileSystem hdfs = FileSystem.get(conf);
    if (hdfs.exists(outputDir)) hdfs.delete(outputDir, true);

    // Execute job
    int code = job.waitForCompletion(true) ? 0 : 1;

    // Counter finding and displaying
    Counters counters = job.getCounters();

    // Displaying counters
    System.out.printf(
        "Missing Fields: %d, Error Count: %d\n",
        counters.findCounter(COUNTERS.MISSING_FIELDS_RECORD_COUNT).getValue(),
        counters.findCounter(COUNTERS.NULL_OR_EMPTY).getValue());

    System.exit(code);
  }
Exemple #24
0
 public static Counter getCounter(Enum<?> e) {
   return COUNTERS.findCounter(e);
 }