@Override
public void close() throws IOException {
IOException ioe = null;
try {
final long dirStart = out.getFilePointer();
final long indexDirStart = indexOut.getFilePointer();
out.writeVInt(fields.size());
for (FieldMetaData field : fields) {
// System.out.println(" field " + field.fieldInfo.name + " " + field.numTerms + " terms");
out.writeVInt(field.fieldInfo.number);
out.writeVLong(field.numTerms);
out.writeVInt(field.rootCode.length);
out.writeBytes(field.rootCode.bytes, field.rootCode.offset, field.rootCode.length);
if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
out.writeVLong(field.sumTotalTermFreq);
}
out.writeVLong(field.sumDocFreq);
out.writeVInt(field.docCount);
indexOut.writeVLong(field.indexStartFP);
}
writeTrailer(out, dirStart);
writeIndexTrailer(indexOut, indexDirStart);
} catch (IOException ioe2) {
ioe = ioe2;
} finally {
IOUtils.closeWhileHandlingException(ioe, out, indexOut, postingsWriter);
}
}
public void close() throws IOException {
try {
final long dirStart = out.getFilePointer();
final int fieldCount = fields.size();
int nonNullFieldCount = 0;
for (int i = 0; i < fieldCount; i++) {
FSTFieldWriter field = fields.get(i);
if (field.fst != null) {
nonNullFieldCount++;
}
}
out.writeVInt(nonNullFieldCount);
for (int i = 0; i < fieldCount; i++) {
FSTFieldWriter field = fields.get(i);
if (field.fst != null) {
out.writeVInt(field.fieldInfo.number);
out.writeVLong(field.indexStart);
}
}
writeTrailer(dirStart);
} finally {
out.close();
}
}
private void writeDoc() throws IOException {
if (isFieldOpen())
throw new IllegalStateException("Field is still open while writing document");
// System.out.println("Writing doc pointer: " + currentDocPointer);
// write document index record
tvx.writeLong(currentDocPointer);
// write document data record
final int size = fields.size();
// write the number of fields
tvd.writeVInt(size);
// write field numbers
for (int i = 0; i < size; i++) {
TVField field = (TVField) fields.elementAt(i);
tvd.writeVInt(field.number);
}
// write field pointers
long lastFieldPointer = 0;
for (int i = 0; i < size; i++) {
TVField field = (TVField) fields.elementAt(i);
tvd.writeVLong(field.tvfPointer - lastFieldPointer);
lastFieldPointer = field.tvfPointer;
}
// System.out.println("After writing doc pointer: " + tvx.getFilePointer());
}
@Override
public void finish() throws IOException {
if (finished) {
throw new IllegalStateException("already finished");
}
finished = true;
CodecUtil.writeFooter(dataOut);
String indexFileName =
IndexFileNames.segmentFileName(
writeState.segmentInfo.name,
writeState.segmentSuffix,
Lucene60PointsFormat.INDEX_EXTENSION);
// Write index file
try (IndexOutput indexOut =
writeState.directory.createOutput(indexFileName, writeState.context)) {
CodecUtil.writeIndexHeader(
indexOut,
Lucene60PointsFormat.META_CODEC_NAME,
Lucene60PointsFormat.INDEX_VERSION_CURRENT,
writeState.segmentInfo.getId(),
writeState.segmentSuffix);
int count = indexFPs.size();
indexOut.writeVInt(count);
for (Map.Entry<String, Long> ent : indexFPs.entrySet()) {
FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(ent.getKey());
if (fieldInfo == null) {
throw new IllegalStateException(
"wrote field=\"" + ent.getKey() + "\" but that field doesn't exist in FieldInfos");
}
indexOut.writeVInt(fieldInfo.number);
indexOut.writeVLong(ent.getValue());
}
CodecUtil.writeFooter(indexOut);
}
}
@Override
protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException {
// To efficiently store payloads/offsets in the posting lists we do not store the length of
// every payload/offset. Instead we omit the length if the previous lengths were the same
//
// However, in order to support skipping, the length at every skip point must be known.
// So we use the same length encoding that we use for the posting lists for the skip data as
// well:
// Case 1: current field does not store payloads/offsets
// SkipDatum --> DocSkip, FreqSkip, ProxSkip
// DocSkip,FreqSkip,ProxSkip --> VInt
// DocSkip records the document number before every SkipInterval th document in
// TermFreqs.
// Document numbers are represented as differences from the previous value in the
// sequence.
// Case 2: current field stores payloads/offsets
// SkipDatum --> DocSkip,
// PayloadLength?,OffsetLength?,FreqSkip,ProxSkip
// DocSkip,FreqSkip,ProxSkip --> VInt
// PayloadLength,OffsetLength--> VInt
// In this case DocSkip/2 is the difference between
// the current and the previous value. If DocSkip
// is odd, then a PayloadLength encoded as VInt follows,
// if DocSkip is even, then it is assumed that the
// current payload/offset lengths equals the lengths at the previous
// skip point
int delta = curDoc - lastSkipDoc[level];
if (curStorePayloads || curStoreOffsets) {
assert curStorePayloads || curPayloadLength == lastSkipPayloadLength[level];
assert curStoreOffsets || curOffsetLength == lastSkipOffsetLength[level];
if (curPayloadLength == lastSkipPayloadLength[level]
&& curOffsetLength == lastSkipOffsetLength[level]) {
// the current payload/offset lengths equals the lengths at the previous skip point,
// so we don't store the lengths again
skipBuffer.writeVInt(delta << 1);
} else {
// the payload and/or offset length is different from the previous one. We shift the
// DocSkip,
// set the lowest bit and store the current payload and/or offset lengths as VInts.
skipBuffer.writeVInt(delta << 1 | 1);
if (curStorePayloads) {
skipBuffer.writeVInt(curPayloadLength);
lastSkipPayloadLength[level] = curPayloadLength;
}
if (curStoreOffsets) {
skipBuffer.writeVInt(curOffsetLength);
lastSkipOffsetLength[level] = curOffsetLength;
}
}
} else {
// current field does not store payloads or offsets
skipBuffer.writeVInt(delta);
}
skipBuffer.writeVLong(curFreqPointer - lastSkipFreqPointer[level]);
skipBuffer.writeVLong(curProxPointer - lastSkipProxPointer[level]);
lastSkipDoc[level] = curDoc;
lastSkipFreqPointer[level] = curFreqPointer;
lastSkipProxPointer[level] = curProxPointer;
}
