@Override
public void reduce(
LongWritable key,
Iterator<NullWritable> values,
OutputCollector<NullWritable, Text> output,
Reporter reporter)
throws IOException {
output.collect(NullWritable.get(), new Text(key.toString()));
}
public void reduce(WritablePoint key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
String ids = new String("");
for (LongWritable val : values) {
ids += val.toString() + ", ";
}
Text text = new Text(ids);
int x = part / grid;
int y = part % grid;
out.write(key, text, x + "/" + y);
}
public boolean nextKeyValue() {
value = new Text();
if (key == null) {
key = new LongWritable();
}
if (finish < count) {
key.set(start + finish);
value.set(key.toString());
finish += 1;
return true;
} else {
return false;
}
}
private void splitRead(LongWritable key, String line, Writer writer) throws IOException {
String[] fields = line.split("\t");
if (fields[1].length() != fields[3].length()) {
logger.warn("Warning; mismatching seq and qual lengths in record " + key.toString() + "!");
logger.warn("Seq:");
logger.warn(fields[1]);
logger.warn("Qual:");
logger.warn(fields[3]);
logger.warn("DONE WARNING");
}
writer.write(fields[0] + "\n");
writer.write(fields[1] + "\n");
writer.write(fields[2] + "\n");
writer.write(fields[3] + "\n");
}
@Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String className = key.toString();
String rawRecord = value.toString();
// get the class
Class<? extends USPSRecord> clazz = null;
try {
clazz = uspsUtils.getClassForRawRecord(rawRecord.substring(0, 1), myRecordLength);
if (clazz == null) {
throw new IOException("NO CLASS 4 RAW RECORD:|" + rawRecord + "|");
}
} catch (Exception e) {
throw new IOException("NO CLASS 4 RAW RECORD:|" + rawRecord + "|");
// throw new IOException("Could not determine target USPSRecord class for record of type: "
// + rawRecord.substring(0,1) + " len: " + myRecordLength);
}
// create a new instance to populate
USPSRecord record = null;
try {
record = clazz.newInstance();
} catch (Exception e) {
throw new IOException(
"Could not create target USPSRecord instance for record of type: "
+ clazz.getName()
+ " "
+ e.getMessage());
}
// apply the line to the record
try {
lineParser.applyRawRecord(rawRecord, record);
record.setIdentifier(idGenerator.generateId(record));
} catch (Exception e) {
throw new IOException(
"Could not apply record data to target USPSRecord instance for record of type: "
+ rawRecord
+ " "
+ myRecordLength
+ " "
+ clazz.getName()
+ " "
+ e.getMessage());
}
// get the record ID
String id = record.getIdentifier();
// if a copyright, update this as our latest one...
// if the copyright has changed.., we need to flush all
// previous records that were loaded after the copyrightToApply
// before updating our copyrightToApply which will be applied
// to all subsequent records
if (record instanceof Copyright) {
Copyright tmpC = (Copyright) record;
if (copyrightToApply == null
|| !copyrightToApply.getIdentifier().equals(tmpC.getIdentifier())) {
flushBatch();
copyrightToApply = (Copyright) record;
}
}
// add to batch
batch.add(record);
// commit?
if (batch.size() >= processorBatchSize) {
flushBatch();
}
// or the old version was Text,Text
// context.write(new Text(id), new WritableUSPSRecord(record)); // json output
}
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
Text outKey = new Text(key.toString());
context.write(outKey, value);
}