public boolean equals(Object bw) {
if (bw instanceof BigramWritable) {
BigramWritable ow = (BigramWritable) bw;
return leftBigram.equals(ow.leftBigram) && rightBigram.equals(ow.rightBigram);
}
return false;
}
/** Check whether the file list have duplication. */
private static void checkDuplication(FileSystem fs, Path file, Path sorted, Configuration conf)
throws IOException {
SequenceFile.Reader in = null;
try {
SequenceFile.Sorter sorter =
new SequenceFile.Sorter(fs, new Text.Comparator(), Text.class, Text.class, conf);
sorter.sort(file, sorted);
in = new SequenceFile.Reader(fs, sorted, conf);
Text prevdst = null, curdst = new Text();
Text prevsrc = null, cursrc = new Text();
for (; in.next(curdst, cursrc); ) {
if (prevdst != null && curdst.equals(prevdst)) {
throw new DuplicationException(
"Invalid input, there are duplicated files in the sources: "
+ prevsrc
+ ", "
+ cursrc);
}
prevdst = curdst;
curdst = new Text();
prevsrc = cursrc;
cursrc = new Text();
}
} finally {
checkAndClose(in);
}
}
