public void map(
WritableComparable key, Writable value, OutputCollector collector, Reporter reporter)
throws IOException {
// convert on the fly from old formats with UTF8 keys
if (key instanceof UTF8) {
newKey.set(key.toString());
key = newKey;
}
collector.collect(key, new ObjectWritable(value));
}
public int read() throws IOException {
int ret;
if (null == inbuf || -1 == (ret = inbuf.read())) {
if (!r.next(key, val)) {
return -1;
}
byte[] tmp = key.toString().getBytes();
outbuf.write(tmp, 0, tmp.length);
outbuf.write('\t');
tmp = val.toString().getBytes();
outbuf.write(tmp, 0, tmp.length);
outbuf.write('\n');
inbuf.reset(outbuf.getData(), outbuf.getLength());
outbuf.reset();
ret = inbuf.read();
}
return ret;
}
public void map(WritableComparable key, Writable value, OutputCollector output, Reporter reporter)
throws IOException {
// convert on the fly from the old format
if (key instanceof UTF8) {
newKey.set(key.toString());
key = newKey;
}
if (filters != null) {
try {
if (filters.filter(((Text) key).toString()) == null) {
return;
}
} catch (Exception e) {
if (LOG.isWarnEnabled()) {
LOG.warn("Cannot filter key " + key + ": " + e.getMessage());
}
}
}
output.collect(key, value);
}
public void reduce(
WritableComparable key, Iterator values, OutputCollector output, Reporter reporter)
throws IOException {
StringBuffer dump = new StringBuffer();
dump.append("\nRecno:: ").append(recNo++).append("\n");
dump.append("URL:: " + key.toString() + "\n");
while (values.hasNext()) {
Object value = ((ObjectWritable) values.next()).get(); // unwrap
if (value instanceof CrawlDatum) {
dump.append("\nCrawlDatum::\n").append(((CrawlDatum) value).toString());
} else if (value instanceof Content) {
dump.append("\nContent::\n").append(((Content) value).toString());
} else if (value instanceof ParseData) {
dump.append("\nParseData::\n").append(((ParseData) value).toString());
} else if (value instanceof ParseText) {
dump.append("\nParseText::\n").append(((ParseText) value).toString());
} else if (LOG.isWarnEnabled()) {
LOG.warn("Unrecognized type: " + value.getClass());
}
}
output.collect(key, new ObjectWritable(dump.toString()));
}
public int compare(WritableComparable a, WritableComparable b) {
String aStr = a.toString();
String bStr = b.toString();
return compareStr(aStr, bStr);
}