Example #1
0
    @Override
    protected void reduce(
        SortedMapWritableComparable key, Iterable<BytesWritable> values, Context context)
        throws IOException, InterruptedException {
      StatisticsProtos.Statistics.Builder statisticsBuilder =
          StatisticsProtos.Statistics.newBuilder();
      StatisticsProtos.KeyValue.Builder kvBuilder = StatisticsProtos.KeyValue.newBuilder();

      for (SortedMapWritableComparable.Entry<WritableComparable, Writable> partEntry :
          key.entrySet()) {
        kvBuilder.clear();
        Text kText = (Text) partEntry.getKey();
        Text vText = (Text) partEntry.getValue();
        kvBuilder.setKey(kText.toString());
        kvBuilder.setValue(vText.toString());
        statisticsBuilder.addPartitions(kvBuilder);
      }
      for (Map.Entry<String, StatisticCalculator> statisticEntry :
          this.statGenConfiguration.getStatisticCalculators().entrySet()) {
        kvBuilder.clear();
        double result = statisticEntry.getValue().calculate(values);
        kvBuilder.setKey(statisticEntry.getKey());
        kvBuilder.setValue(String.valueOf(result));
        statisticsBuilder.addStatistics(kvBuilder);
      }
      statisticsBuilder.setTimestamp(new Date().getTime());
      context.write(
          new Text(genKeyFromMap(key)), new BytesWritable(statisticsBuilder.build().toByteArray()));
    }
Example #2
0
    @Override
    protected void map(Text key, BytesWritable value, Context context)
        throws IOException, InterruptedException {

      StatisticsProtos.InputEntry inputEntry =
          StatisticsProtos.InputEntry.parseFrom(value.copyBytes());

      Set<SortedMapWritableComparable> outputKeyMaps = new HashSet<SortedMapWritableComparable>();
      outputKeyMaps.add(new SortedMapWritableComparable());

      Map<String, Partitioner> partitioners = statGenConfiguration.getPartitioners();

      for (StatisticsProtos.KeyValue field : inputEntry.getFieldList()) {
        String fieldKey = field.getKey();
        if (partitioners.containsKey(fieldKey)) {
          String[] partitions = partitioners.get(fieldKey).partition(field.getValue());
          // the number of output values should be multiplied by number of partitions
          // (partitions.lenght).
          // if 0, then return nothing...
          if (partitions.length == 0) {
            return;
          }
          for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
            outputKeyMap.put(new Text(fieldKey), new Text(partitions[0]));
          }
          // if more than 1, then copy output values
          for (int i = 1; i < partitions.length; i++) {
            Text mapKey = new Text(fieldKey);
            Text mapValue = new Text(partitions[i]);
            Set<SortedMapWritableComparable> newMaps = new HashSet<SortedMapWritableComparable>();
            for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
              SortedMapWritableComparable newMap = new SortedMapWritableComparable(outputKeyMap);
              newMap.put(mapKey, mapValue);
              newMaps.add(newMap);
            }
            outputKeyMaps.addAll(newMaps);
          }
        }
      }

      for (SortedMapWritableComparable outputKeyMap : outputKeyMaps) {
        context.write(outputKeyMap, value);
      }
    }