@Override
public TermsConsumer addField(FieldInfo field) throws IOException {
FuzzySet bloomFilter = bloomFilterFactory.getSetForField(state, field);
if (bloomFilter != null) {
assert bloomFilters.containsKey(field) == false;
bloomFilters.put(field, bloomFilter);
return new WrappedTermsConsumer(delegateFieldsConsumer.addField(field), bloomFilter);
} else {
// No, use the unfiltered fieldsConsumer - we are not interested in
// recording any term Bitsets.
return delegateFieldsConsumer.addField(field);
}
}
@Test
public void testNoDocs() throws IOException {
AnalyzingCompletionLookupProvider provider =
new AnalyzingCompletionLookupProvider(true, false, true, true);
RAMDirectory dir = new RAMDirectory();
IndexOutput output = dir.createOutput("foo.txt", IOContext.DEFAULT);
FieldsConsumer consumer = provider.consumer(output);
consumer.write(
new Fields() {
@Override
public Iterator<String> iterator() {
return Arrays.asList("foo").iterator();
}
@Override
public Terms terms(String field) throws IOException {
return null;
}
@Override
public int size() {
return 1;
}
});
consumer.close();
output.close();
IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT);
LookupFactory load = provider.load(input);
PostingsFormat format = new Elasticsearch090PostingsFormat();
NamedAnalyzer analyzer = new NamedAnalyzer("foo", new StandardAnalyzer());
assertNull(
load.getLookup(
new CompletionFieldMapper(
new Names("foo"),
analyzer,
analyzer,
format,
null,
true,
true,
true,
Integer.MAX_VALUE,
indexSettings,
AbstractFieldMapper.MultiFields.empty(),
null,
ContextMapping.EMPTY_MAPPING),
new CompletionSuggestionContext(null)));
dir.close();
}
@Override
public void close() throws IOException {
delegateFieldsConsumer.close();
// Now we are done accumulating values for these fields
List<Entry<FieldInfo, FuzzySet>> nonSaturatedBlooms =
new ArrayList<Map.Entry<FieldInfo, FuzzySet>>();
for (Entry<FieldInfo, FuzzySet> entry : bloomFilters.entrySet()) {
FuzzySet bloomFilter = entry.getValue();
if (!bloomFilterFactory.isSaturated(bloomFilter, entry.getKey())) {
nonSaturatedBlooms.add(entry);
}
}
String bloomFileName =
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION);
IndexOutput bloomOutput = null;
try {
bloomOutput = state.directory.createOutput(bloomFileName, state.context);
CodecUtil.writeHeader(bloomOutput, BLOOM_CODEC_NAME, BLOOM_CODEC_VERSION);
// remember the name of the postings format we will delegate to
bloomOutput.writeString(delegatePostingsFormat.getName());
// First field in the output file is the number of fields+blooms saved
bloomOutput.writeInt(nonSaturatedBlooms.size());
for (Entry<FieldInfo, FuzzySet> entry : nonSaturatedBlooms) {
FieldInfo fieldInfo = entry.getKey();
FuzzySet bloomFilter = entry.getValue();
bloomOutput.writeInt(fieldInfo.number);
saveAppropriatelySizedBloomFilter(bloomOutput, bloomFilter, fieldInfo);
}
} finally {
IOUtils.close(bloomOutput);
}
// We are done with large bitsets so no need to keep them hanging around
bloomFilters.clear();
}
@Override
public void write(Fields fields) throws IOException {
delegatesFieldsConsumer.write(fields);
suggestFieldsConsumer.write(fields);
}
@Override
public void close() throws IOException {
in.close();
}
@Override
public TermsConsumer addField(FieldInfo field) throws IOException {
TermsConsumer consumer = in.addField(field);
assert consumer != null;
return new AssertingTermsConsumer(consumer, field);
}
// TODO ADD more unittests
private void writeData(
Directory dir, Completion090PostingsFormat.CompletionLookupProvider provider)
throws IOException {
IndexOutput output = dir.createOutput("foo.txt", IOContext.DEFAULT);
FieldsConsumer consumer = provider.consumer(output);
final List<TermPosAndPayload> terms = new ArrayList<>();
terms.add(
new TermPosAndPayload(
"foofightersgenerator",
256 - 2,
provider.buildPayload(
new BytesRef("Generator - Foo Fighters"), 9, new BytesRef("id:10"))));
terms.add(
new TermPosAndPayload(
"generator",
256 - 1,
provider.buildPayload(
new BytesRef("Generator - Foo Fighters"), 9, new BytesRef("id:10"))));
Fields fields =
new Fields() {
@Override
public Iterator<String> iterator() {
return Arrays.asList("foo").iterator();
}
@Override
public Terms terms(String field) throws IOException {
if (field.equals("foo")) {
return new Terms() {
@Override
public TermsEnum iterator(TermsEnum reuse) throws IOException {
final Iterator<TermPosAndPayload> iterator = terms.iterator();
return new TermsEnum() {
private TermPosAndPayload current = null;
@Override
public SeekStatus seekCeil(BytesRef text) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public void seekExact(long ord) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public BytesRef term() throws IOException {
return current == null ? null : current.term;
}
@Override
public long ord() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int docFreq() throws IOException {
return current == null ? 0 : 1;
}
@Override
public long totalTermFreq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags)
throws IOException {
final TermPosAndPayload data = current;
return new PostingsEnum() {
boolean done = false;
@Override
public int nextPosition() throws IOException {
return current.pos;
}
@Override
public int startOffset() throws IOException {
return 0;
}
@Override
public int endOffset() throws IOException {
return 0;
}
@Override
public BytesRef getPayload() throws IOException {
return current.payload;
}
@Override
public int freq() throws IOException {
return 1;
}
@Override
public int docID() {
if (done) {
return NO_MORE_DOCS;
}
return 0;
}
@Override
public int nextDoc() throws IOException {
if (done) {
return NO_MORE_DOCS;
}
done = true;
return 0;
}
@Override
public int advance(int target) throws IOException {
if (done) {
return NO_MORE_DOCS;
}
done = true;
return 0;
}
@Override
public long cost() {
return 0;
}
};
}
@Override
public BytesRef next() throws IOException {
if (iterator.hasNext()) {
current = iterator.next();
return current.term;
}
current = null;
return null;
}
};
}
@Override
public long size() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public long getSumTotalTermFreq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public long getSumDocFreq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int getDocCount() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public boolean hasFreqs() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasOffsets() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasPositions() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasPayloads() {
throw new UnsupportedOperationException();
}
};
}
return null;
}
@Override
public int size() {
return 0;
}
};
consumer.write(fields);
consumer.close();
output.close();
}