public void main(String[] args) throws Exception {
    conf = new JobConf(System.getProperty("oozie.action.conf.xml"));
    MorphlinesJob job = new MorphlinesMRDriver().run(conf);

    if (job.isSuccessful()) {
      String countersFile = System.getProperty("oozie.action.output.properties");
      File file = new File(countersFile);

      if (!file.exists()) {
        file.createNewFile();
      }

      FileWriter fw = new FileWriter(file.getAbsoluteFile(), true);
      BufferedWriter bw = new BufferedWriter(fw);
      bw.write("#" + DateTime.now().toString() + "\n");
      CounterGroup counterGroup = job.getCounters().getGroup(MorphlinesMRCounters.COUNTERGROUP);
      String groupname = counterGroup.getDisplayName();
      for (Counter c : counterGroup) {
        bw.write(groupname + "." + c.getDisplayName() + "=" + c.getValue() + "\n");
      }
      bw.close();
    } else {
      throw new RuntimeException("Job is failed. See the log of " + job.getJobID());
    }
  }
 /** {@inheritDoc} */
 @Override
 public synchronized void incrAllCounters(AbstractCounters<Counter, CounterGroup> other) {
   for (CounterGroup group : other) {
     for (Counter counter : group) {
       findCounter(group.getName(), counter.getName()).increment(counter.getValue());
     }
   }
 }
Esempio n. 3
0
 protected Set<Integer> getNeeds(String type, Counters counts) {
   CounterGroup group = counts.getGroup(type);
   HashSet<Integer> result = new HashSet<Integer>(group.size());
   for (Counter counter : group) {
     String name = counter.getName();
     assert name.startsWith("t");
     result.add(Integer.parseInt(name.substring(1)));
   }
   return result;
 }
Esempio n. 4
0
 private Counters getCounters() {
   Counters c = new Counters();
   Map<String, Map<String, Long>> values = counters.value();
   for (Map.Entry<String, Map<String, Long>> e : values.entrySet()) {
     CounterGroup cg = c.getGroup(e.getKey());
     for (Map.Entry<String, Long> f : e.getValue().entrySet()) {
       cg.findCounter(f.getKey()).setValue(f.getValue());
     }
   }
   return c;
 }
 private void validateCounters(org.apache.hadoop.mapreduce.Counters counters) {
   Iterator<org.apache.hadoop.mapreduce.CounterGroup> it = counters.iterator();
   while (it.hasNext()) {
     org.apache.hadoop.mapreduce.CounterGroup group = it.next();
     LOG.info("Group " + group.getDisplayName());
     Iterator<org.apache.hadoop.mapreduce.Counter> itc = group.iterator();
     while (itc.hasNext()) {
       LOG.info("Counter is " + itc.next().getDisplayName());
     }
   }
   Assert.assertEquals(1, counters.countCounters());
 }
Esempio n. 6
0
 static Counters fromAvro(JhCounters counters) {
   Counters result = new Counters();
   for (JhCounterGroup g : counters.groups) {
     CounterGroup group =
         result.addGroup(
             StringInterner.weakIntern(g.name.toString()),
             StringInterner.weakIntern(g.displayName.toString()));
     for (JhCounter c : g.counts) {
       group.addCounter(
           StringInterner.weakIntern(c.name.toString()),
           StringInterner.weakIntern(c.displayName.toString()),
           c.value);
     }
   }
   return result;
 }
 /** {@inheritDoc} */
 @Override
 public synchronized CounterGroup addGroup(CounterGroup grp) {
   return addGroup(grp.getName(), grp.getDisplayName());
 }