示例#1
0
 protected void setup(Context context) throws IOException, InterruptedException {
   super.setup(context);
   scriptEngine = scriptEngineManager.getEngineByName("JavaScript");
   try {
     Configuration configuration = context.getConfiguration();
     mapOutputKey = MrUtils.getWritableComparableType("js.map.output.key.type", configuration);
     mapOutputValue = MrUtils.getWritableComparableType("js.map.output.value.type", configuration);
     scriptEngine.eval(
         MrUtils.getScripts(
             context.getConfiguration(), MrUtils.getPathFilter("js.map.filename", configuration)));
     scriptEngine.put("mapOutputKey", mapOutputKey);
     scriptEngine.put("mapOutputValue", mapOutputValue);
     OUTPUT_KEY_CLASS = mapOutputKey.getClass();
     OUTPUT_VALUE_CLASS = mapOutputValue.getClass();
   } catch (ScriptException se) {
     IOException ioe = new IOException(se);
     ioe.setStackTrace(se.getStackTrace());
     throw ioe;
   }
 }
示例#2
0
      @SuppressWarnings("unchecked")
      public void map(
          WritableComparable key,
          Writable value,
          OutputCollector<IntWritable, RecordStatsWritable> output,
          Reporter reporter)
          throws IOException {
        // Set up rawKey and rawValue on the first call to 'map'
        if (recordId == -1) {
          rawKey = createRaw(key.getClass());
          rawValue = createRaw(value.getClass());
        }
        ++recordId;

        if (this.key == sortOutput) {
          // Check if keys are 'sorted' if this
          // record is from sort's output
          if (prevKey == null) {
            prevKey = key;
            keyClass = prevKey.getClass();
          } else {
            // Sanity check
            if (keyClass != key.getClass()) {
              throw new IOException(
                  "Type mismatch in key: expected "
                      + keyClass.getName()
                      + ", recieved "
                      + key.getClass().getName());
            }

            // Check if they were sorted correctly
            if (prevKey.compareTo(key) > 0) {
              throw new IOException(
                  "The 'map-reduce' framework wrongly"
                      + " classifed ("
                      + prevKey
                      + ") > ("
                      + key
                      + ") "
                      + "for record# "
                      + recordId);
            }
            prevKey = key;
          }

          // Check if the sorted output is 'partitioned' right
          int keyPartition = partitioner.getPartition(key, value, noSortReducers);
          if (partition != keyPartition) {
            throw new IOException(
                "Partitions do not match for record# "
                    + recordId
                    + " ! - '"
                    + partition
                    + "' v/s '"
                    + keyPartition
                    + "'");
          }
        }

        // Construct the record-stats and output (this.key, record-stats)
        byte[] keyBytes = rawKey.getRawBytes(key);
        int keyBytesLen = rawKey.getRawBytesLength(key);
        byte[] valueBytes = rawValue.getRawBytes(value);
        int valueBytesLen = rawValue.getRawBytesLength(value);

        int keyValueChecksum =
            (WritableComparator.hashBytes(keyBytes, keyBytesLen)
                ^ WritableComparator.hashBytes(valueBytes, valueBytesLen));

        output.collect(
            this.key, new RecordStatsWritable((keyBytesLen + valueBytesLen), 1, keyValueChecksum));
      }