protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
scriptEngine = scriptEngineManager.getEngineByName("JavaScript");
try {
Configuration configuration = context.getConfiguration();
mapOutputKey = MrUtils.getWritableComparableType("js.map.output.key.type", configuration);
mapOutputValue = MrUtils.getWritableComparableType("js.map.output.value.type", configuration);
scriptEngine.eval(
MrUtils.getScripts(
context.getConfiguration(), MrUtils.getPathFilter("js.map.filename", configuration)));
scriptEngine.put("mapOutputKey", mapOutputKey);
scriptEngine.put("mapOutputValue", mapOutputValue);
OUTPUT_KEY_CLASS = mapOutputKey.getClass();
OUTPUT_VALUE_CLASS = mapOutputValue.getClass();
} catch (ScriptException se) {
IOException ioe = new IOException(se);
ioe.setStackTrace(se.getStackTrace());
throw ioe;
}
}
@SuppressWarnings("unchecked")
public void map(
WritableComparable key,
Writable value,
OutputCollector<IntWritable, RecordStatsWritable> output,
Reporter reporter)
throws IOException {
// Set up rawKey and rawValue on the first call to 'map'
if (recordId == -1) {
rawKey = createRaw(key.getClass());
rawValue = createRaw(value.getClass());
}
++recordId;
if (this.key == sortOutput) {
// Check if keys are 'sorted' if this
// record is from sort's output
if (prevKey == null) {
prevKey = key;
keyClass = prevKey.getClass();
} else {
// Sanity check
if (keyClass != key.getClass()) {
throw new IOException(
"Type mismatch in key: expected "
+ keyClass.getName()
+ ", recieved "
+ key.getClass().getName());
}
// Check if they were sorted correctly
if (prevKey.compareTo(key) > 0) {
throw new IOException(
"The 'map-reduce' framework wrongly"
+ " classifed ("
+ prevKey
+ ") > ("
+ key
+ ") "
+ "for record# "
+ recordId);
}
prevKey = key;
}
// Check if the sorted output is 'partitioned' right
int keyPartition = partitioner.getPartition(key, value, noSortReducers);
if (partition != keyPartition) {
throw new IOException(
"Partitions do not match for record# "
+ recordId
+ " ! - '"
+ partition
+ "' v/s '"
+ keyPartition
+ "'");
}
}
// Construct the record-stats and output (this.key, record-stats)
byte[] keyBytes = rawKey.getRawBytes(key);
int keyBytesLen = rawKey.getRawBytesLength(key);
byte[] valueBytes = rawValue.getRawBytes(value);
int valueBytesLen = rawValue.getRawBytesLength(value);
int keyValueChecksum =
(WritableComparator.hashBytes(keyBytes, keyBytesLen)
^ WritableComparator.hashBytes(valueBytes, valueBytesLen));
output.collect(
this.key, new RecordStatsWritable((keyBytesLen + valueBytesLen), 1, keyValueChecksum));
}
