public IntFieldWriter getIntFieldWriter(String field, boolean blowAway)
throws FileNotFoundException {
if (!blowAway && intFields.contains(field)) {
throw new IllegalArgumentException("already added int field " + field);
}
intFields.add(field);
return SimpleIntFieldWriter.open(outputDirectory, field, maxDocs, writeBTreesOnClose);
}
public static void writeIntBTree(String directory, String intField, File btreeDir)
throws IOException {
final String termsFilename =
Files.buildPath(directory, SimpleIntFieldWriter.getTermsFilename(intField));
if (!new File(termsFilename).exists() || new File(termsFilename).length() == 0L) return;
final CountingInputStream termsList =
new CountingInputStream(new BufferedInputStream(new FileInputStream(termsFilename), 65536));
try {
ImmutableBTreeIndex.Writer.write(
btreeDir,
new AbstractIterator<Generation.Entry<Long, LongPair>>() {
private long lastTerm = 0;
private long lastTermDocOffset = 0L;
private long lastTermFileOffset = 0L;
private long key;
private LongPair value;
@Override
protected Generation.Entry<Long, LongPair> computeNext() {
try {
if (!nextTerm()) return endOfData();
key = lastTerm;
value = new LongPair(lastTermFileOffset, lastTermDocOffset);
for (int i = 0; i < BLOCK_SIZE - 1; ++i) {
if (!nextTerm()) {
break;
}
}
return Generation.Entry.create(key, value);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private boolean nextTerm() throws IOException {
final long termDelta;
// sorry
try {
termDelta = FlamdexUtils.readVLong(termsList);
} catch (EOFException e) {
return false;
}
lastTerm += termDelta;
final long offsetDelta = FlamdexUtils.readVLong(termsList);
lastTermDocOffset += offsetDelta;
lastTermFileOffset = termsList.getCount();
FlamdexUtils.readVLong(termsList); // termDocFreq
return true;
}
},
new LongSerializer(),
new LongPairSerializer(),
65536,
false);
} finally {
termsList.close();
}
}
