@Override
public void write(
Directory directory,
SegmentInfo segmentInfo,
String segmentSuffix,
FieldInfos infos,
IOContext context)
throws IOException {
final String fileName =
IndexFileNames.segmentFileName(
segmentInfo.name, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
try (IndexOutput output = directory.createOutput(fileName, context)) {
CodecUtil.writeHeader(
output, Lucene46FieldInfosFormat.CODEC_NAME, Lucene46FieldInfosFormat.FORMAT_CURRENT);
output.writeVInt(infos.size());
for (FieldInfo fi : infos) {
IndexOptions indexOptions = fi.getIndexOptions();
byte bits = 0x0;
if (fi.hasVectors()) bits |= Lucene46FieldInfosFormat.STORE_TERMVECTOR;
if (fi.omitsNorms()) bits |= Lucene46FieldInfosFormat.OMIT_NORMS;
if (fi.hasPayloads()) bits |= Lucene46FieldInfosFormat.STORE_PAYLOADS;
if (fi.getIndexOptions() != IndexOptions.NONE) {
bits |= Lucene46FieldInfosFormat.IS_INDEXED;
assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0
|| !fi.hasPayloads();
if (indexOptions == IndexOptions.DOCS) {
bits |= Lucene46FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS;
} else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
bits |= Lucene46FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS;
} else if (indexOptions == IndexOptions.DOCS_AND_FREQS) {
bits |= Lucene46FieldInfosFormat.OMIT_POSITIONS;
}
}
output.writeString(fi.name);
output.writeVInt(fi.number);
output.writeByte(bits);
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : DocValuesType.NONE);
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
output.writeLong(fi.getDocValuesGen());
output.writeStringStringMap(fi.attributes());
}
CodecUtil.writeFooter(output);
}
}
SepDocsAndPositionsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs)
throws IOException {
this.liveDocs = liveDocs;
storePayloads = fieldInfo.hasPayloads();
// System.out.println("Sep D&P init");
// TODO: can't we only do this if consumer
// skipped consuming the previous docs?
docIndex.copyFrom(termState.docIndex);
docIndex.seek(docReader);
// System.out.println(" docIndex=" + docIndex);
freqIndex.copyFrom(termState.freqIndex);
freqIndex.seek(freqReader);
// System.out.println(" freqIndex=" + freqIndex);
posIndex.copyFrom(termState.posIndex);
// System.out.println(" posIndex=" + posIndex);
posSeekPending = true;
payloadPending = false;
payloadFP = termState.payloadFP;
skipFP = termState.skipFP;
// System.out.println(" skipFP=" + skipFP);
docFreq = termState.docFreq;
count = 0;
doc = -1;
accum = 0;
pendingPosCount = 0;
pendingPayloadBytes = 0;
skipped = false;
return this;
}
SepDocsEnum init(FieldInfo fieldInfo, SepTermState termState, Bits liveDocs)
throws IOException {
this.liveDocs = liveDocs;
this.indexOptions = fieldInfo.getIndexOptions();
omitTF = indexOptions == IndexOptions.DOCS_ONLY;
storePayloads = fieldInfo.hasPayloads();
// TODO: can't we only do this if consumer
// skipped consuming the previous docs?
docIndex.copyFrom(termState.docIndex);
docIndex.seek(docReader);
if (!omitTF) {
freqIndex.copyFrom(termState.freqIndex);
freqIndex.seek(freqReader);
}
docFreq = termState.docFreq;
// NOTE: unused if docFreq < skipMinimum:
skipFP = termState.skipFP;
count = 0;
doc = -1;
accum = 0;
freq = 1;
skipped = false;
return this;
}
InsaneReader(LeafReader in, String insaneField) {
super(in);
this.insaneField = insaneField;
ArrayList<FieldInfo> filteredInfos = new ArrayList<>();
for (FieldInfo fi : in.getFieldInfos()) {
if (fi.name.equals(insaneField)) {
filteredInfos.add(
new FieldInfo(
fi.name,
fi.number,
fi.hasVectors(),
fi.omitsNorms(),
fi.hasPayloads(),
fi.getIndexOptions(),
DocValuesType.NONE,
-1,
Collections.emptyMap(),
fi.getPointDimensionCount(),
fi.getPointNumBytes()));
} else {
filteredInfos.add(fi);
}
}
fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
}
// Currently, this instance is re-used across fields, so
// our parent calls setField whenever the field changes
@Override
public void setField(FieldInfo fieldInfo) {
this.indexOptions = fieldInfo.getIndexOptions();
if (DEBUG) System.out.println("PW field=" + fieldInfo.name + " indexOptions=" + indexOptions);
storePayloads = fieldInfo.hasPayloads();
wrappedPostingsWriter.setField(fieldInfo);
// DEBUG = BlockTreeTermsWriter.DEBUG;
}
/** Constructs a new FieldInfos from an array of FieldInfo objects */
public FieldInfos(FieldInfo[] infos) {
boolean hasVectors = false;
boolean hasProx = false;
boolean hasPayloads = false;
boolean hasOffsets = false;
boolean hasFreq = false;
boolean hasNorms = false;
boolean hasDocValues = false;
for (FieldInfo info : infos) {
if (info.number < 0) {
throw new IllegalArgumentException(
"illegal field number: " + info.number + " for field " + info.name);
}
FieldInfo previous = byNumber.put(info.number, info);
if (previous != null) {
throw new IllegalArgumentException(
"duplicate field numbers: "
+ previous.name
+ " and "
+ info.name
+ " have: "
+ info.number);
}
previous = byName.put(info.name, info);
if (previous != null) {
throw new IllegalArgumentException(
"duplicate field names: "
+ previous.number
+ " and "
+ info.number
+ " have: "
+ info.name);
}
hasVectors |= info.hasVectors();
hasProx |= info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
hasFreq |= info.getIndexOptions() != IndexOptions.DOCS;
hasOffsets |=
info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
>= 0;
hasNorms |= info.hasNorms();
hasDocValues |= info.getDocValuesType() != DocValuesType.NONE;
hasPayloads |= info.hasPayloads();
}
this.hasVectors = hasVectors;
this.hasProx = hasProx;
this.hasPayloads = hasPayloads;
this.hasOffsets = hasOffsets;
this.hasFreq = hasFreq;
this.hasNorms = hasNorms;
this.hasDocValues = hasDocValues;
this.values = Collections.unmodifiableCollection(byNumber.values());
}
public FieldInfo add(FieldInfo fi) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments
return addOrUpdateInternal(
fi.name,
fi.number,
fi.hasVectors(),
fi.omitsNorms(),
fi.hasPayloads(),
fi.getIndexOptions(),
fi.getDocValuesType());
}
@Override
public void nextTerm(FieldInfo fieldInfo, BlockTermState _termState) throws IOException {
final SepTermState termState = (SepTermState) _termState;
final boolean isFirstTerm = termState.termBlockOrd == 0;
// System.out.println("SEPR.nextTerm termCount=" + termState.termBlockOrd + " isFirstTerm=" +
// isFirstTerm + " bytesReader.pos=" + termState.bytesReader.getPosition());
// System.out.println(" docFreq=" + termState.docFreq);
termState.docIndex.read(termState.bytesReader, isFirstTerm);
// System.out.println(" docIndex=" + termState.docIndex);
if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
termState.freqIndex.read(termState.bytesReader, isFirstTerm);
if (fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
// System.out.println(" freqIndex=" + termState.freqIndex);
termState.posIndex.read(termState.bytesReader, isFirstTerm);
// System.out.println(" posIndex=" + termState.posIndex);
if (fieldInfo.hasPayloads()) {
if (isFirstTerm) {
termState.payloadFP = termState.bytesReader.readVLong();
} else {
termState.payloadFP += termState.bytesReader.readVLong();
}
// System.out.println(" payloadFP=" + termState.payloadFP);
}
}
}
if (termState.docFreq >= skipMinimum) {
// System.out.println(" readSkip @ " + termState.bytesReader.getPosition());
if (isFirstTerm) {
termState.skipFP = termState.bytesReader.readVLong();
} else {
termState.skipFP += termState.bytesReader.readVLong();
}
// System.out.println(" skipFP=" + termState.skipFP);
} else if (isFirstTerm) {
termState.skipFP = 0;
}
}
@Override
public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)
throws IOException {
assert state == PostingsConsumerState.START;
assert positionCount < freq;
positionCount++;
assert position >= lastPosition
|| position == -1; /* we still allow -1 from old 3.x indexes */
lastPosition = position;
if (fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
assert startOffset >= 0;
assert startOffset >= lastStartOffset;
lastStartOffset = startOffset;
assert endOffset >= startOffset;
} else {
assert startOffset == -1;
assert endOffset == -1;
}
if (payload != null) {
assert fieldInfo.hasPayloads();
}
in.addPosition(position, payload, startOffset, endOffset);
}
@Override
public boolean hasPayloads() {
return fieldInfo.hasPayloads();
}