@Override
public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException {
final long numOrds = globalOrdinals.getValueCount();
final LongBitSet acceptedGlobalOrdinals = new LongBitSet(numOrds);
final TermsEnum termEnum = globalOrdinals.termsEnum();
BytesRef term = termEnum.next();
while (term != null) {
if (Math.floorMod(
StringHelper.murmurhash3_x86_32(term, HASH_PARTITIONING_SEED), incNumPartitions)
== incZeroBasedPartition) {
acceptedGlobalOrdinals.set(termEnum.ord());
}
term = termEnum.next();
}
return acceptedGlobalOrdinals;
}
public synchronized ShapeFieldCache<T> getCache(LeafReader reader) throws IOException {
ShapeFieldCache<T> idx = sidx.get(reader);
if (idx != null) {
return idx;
}
long startTime = System.currentTimeMillis();
log.fine("Building Cache [" + reader.maxDoc() + "]");
idx = new ShapeFieldCache<>(reader.maxDoc(), defaultSize);
int count = 0;
DocsEnum docs = null;
Terms terms = reader.terms(shapeField);
TermsEnum te = null;
if (terms != null) {
te = terms.iterator(te);
BytesRef term = te.next();
while (term != null) {
T shape = readShape(term);
if (shape != null) {
docs = te.docs(null, docs, DocsEnum.FLAG_NONE);
Integer docid = docs.nextDoc();
while (docid != DocIdSetIterator.NO_MORE_DOCS) {
idx.add(docid, shape);
docid = docs.nextDoc();
count++;
}
}
term = te.next();
}
}
sidx.put(reader, idx);
long elapsed = System.currentTimeMillis() - startTime;
log.fine("Cached: [" + count + " in " + elapsed + "ms] " + idx);
return idx;
}
Query createCandidateQuery(IndexReader indexReader) throws IOException {
List<Term> extractedTerms = new ArrayList<>();
// include extractionResultField:failed, because docs with this term have no
// extractedTermsField
// and otherwise we would fail to return these docs. Docs that failed query term extraction
// always need to be verified by MemoryIndex:
extractedTerms.add(new Term(extractionResultField.name(), EXTRACTION_FAILED));
LeafReader reader = indexReader.leaves().get(0).reader();
Fields fields = reader.fields();
for (String field : fields) {
Terms terms = fields.terms(field);
if (terms == null) {
continue;
}
BytesRef fieldBr = new BytesRef(field);
TermsEnum tenum = terms.iterator();
for (BytesRef term = tenum.next(); term != null; term = tenum.next()) {
BytesRefBuilder builder = new BytesRefBuilder();
builder.append(fieldBr);
builder.append(FIELD_VALUE_SEPARATOR);
builder.append(term);
extractedTerms.add(new Term(queryTermsField.name(), builder.toBytesRef()));
}
}
return new TermsQuery(extractedTerms);
}
// tests for reuse only if bits are the same either null or the same instance
public void testReuseDocsEnumSameBitsOrNull() throws IOException {
Directory dir = newDirectory();
Codec cp = TestUtil.alwaysPostingsFormat(new Lucene40RWPostingsFormat());
RandomIndexWriter writer =
new RandomIndexWriter(
random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setCodec(cp));
int numdocs = atLeast(20);
createRandomIndex(numdocs, writer, random());
writer.commit();
DirectoryReader open = DirectoryReader.open(dir);
for (LeafReaderContext ctx : open.leaves()) {
Terms terms = ctx.reader().terms("body");
TermsEnum iterator = terms.iterator();
IdentityHashMap<PostingsEnum, Boolean> enums = new IdentityHashMap<>();
MatchNoBits bits = new Bits.MatchNoBits(open.maxDoc());
PostingsEnum docs = null;
while ((iterator.next()) != null) {
docs =
iterator.postings(
bits, docs, random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(1, enums.size());
enums.clear();
iterator = terms.iterator();
docs = null;
while ((iterator.next()) != null) {
docs =
iterator.postings(
new Bits.MatchNoBits(open.maxDoc()),
docs,
random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(terms.size(), enums.size());
enums.clear();
iterator = terms.iterator();
docs = null;
while ((iterator.next()) != null) {
docs =
iterator.postings(
null, docs, random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(1, enums.size());
}
writer.close();
IOUtils.close(open, dir);
}
public void testChangeGaps() throws Exception {
// LUCENE-5324: check that it is possible to change the wrapper's gaps
final int positionGap = random().nextInt(1000);
final int offsetGap = random().nextInt(1000);
final Analyzer delegate = new MockAnalyzer(random());
final Analyzer a =
new DelegatingAnalyzerWrapper(delegate.getReuseStrategy()) {
@Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
return delegate;
}
@Override
public int getPositionIncrementGap(String fieldName) {
return positionGap;
}
@Override
public int getOffsetGap(String fieldName) {
return offsetGap;
}
};
final RandomIndexWriter writer = new RandomIndexWriter(random(), newDirectory(), a);
final Document doc = new Document();
final FieldType ft = new FieldType();
ft.setIndexOptions(IndexOptions.DOCS);
ft.setTokenized(true);
ft.setStoreTermVectors(true);
ft.setStoreTermVectorPositions(true);
ft.setStoreTermVectorOffsets(true);
doc.add(new Field("f", "a", ft));
doc.add(new Field("f", "a", ft));
writer.addDocument(doc);
final LeafReader reader = getOnlySegmentReader(writer.getReader());
final Fields fields = reader.getTermVectors(0);
final Terms terms = fields.terms("f");
final TermsEnum te = terms.iterator();
assertEquals(new BytesRef("a"), te.next());
final PostingsEnum dpe = te.postings(null, PostingsEnum.ALL);
assertEquals(0, dpe.nextDoc());
assertEquals(2, dpe.freq());
assertEquals(0, dpe.nextPosition());
assertEquals(0, dpe.startOffset());
final int endOffset = dpe.endOffset();
assertEquals(1 + positionGap, dpe.nextPosition());
assertEquals(1 + endOffset + offsetGap, dpe.endOffset());
assertEquals(null, te.next());
reader.close();
writer.close();
writer.w.getDirectory().close();
}
/** Computes which global ordinals are accepted by this IncludeExclude instance. */
@Override
public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals) throws IOException {
LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount());
TermsEnum globalTermsEnum;
Terms globalTerms = new DocValuesTerms(globalOrdinals);
// TODO: specialize based on compiled.type: for ALL and prefixes (sinkState >= 0 ) we can
// avoid i/o and just set bits.
globalTermsEnum = compiled.getTermsEnum(globalTerms);
for (BytesRef term = globalTermsEnum.next(); term != null; term = globalTermsEnum.next()) {
acceptedGlobalOrdinals.set(globalTermsEnum.ord());
}
return acceptedGlobalOrdinals;
}
// make sure we never reuse from another reader even if it is the same field & codec etc
public void testReuseDocsEnumDifferentReader() throws IOException {
Directory dir = newDirectory();
Codec cp = TestUtil.alwaysPostingsFormat(new Lucene40RWPostingsFormat());
MockAnalyzer analyzer = new MockAnalyzer(random());
analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
RandomIndexWriter writer =
new RandomIndexWriter(random(), dir, newIndexWriterConfig(analyzer).setCodec(cp));
int numdocs = atLeast(20);
createRandomIndex(numdocs, writer, random());
writer.commit();
DirectoryReader firstReader = DirectoryReader.open(dir);
DirectoryReader secondReader = DirectoryReader.open(dir);
List<LeafReaderContext> leaves = firstReader.leaves();
List<LeafReaderContext> leaves2 = secondReader.leaves();
for (LeafReaderContext ctx : leaves) {
Terms terms = ctx.reader().terms("body");
TermsEnum iterator = terms.iterator();
IdentityHashMap<PostingsEnum, Boolean> enums = new IdentityHashMap<>();
MatchNoBits bits = new Bits.MatchNoBits(firstReader.maxDoc());
iterator = terms.iterator();
PostingsEnum docs = null;
BytesRef term = null;
while ((term = iterator.next()) != null) {
docs =
iterator.postings(
null,
randomDocsEnum("body", term, leaves2, bits),
random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(terms.size(), enums.size());
iterator = terms.iterator();
enums.clear();
docs = null;
while ((term = iterator.next()) != null) {
docs =
iterator.postings(
bits,
randomDocsEnum("body", term, leaves2, bits),
random().nextBoolean() ? PostingsEnum.FREQS : PostingsEnum.NONE);
enums.put(docs, true);
}
assertEquals(terms.size(), enums.size());
}
writer.close();
IOUtils.close(firstReader, secondReader, dir);
}
/*
* listTermDictionary displays the term dictionary for a field.
*/
static void listTermDictionary(IndexReader reader, String fieldName) throws IOException {
System.out.println("\nTerm Dictionary: field " + fieldName);
/*
Grant says:
MultiFields.getTerms(IndexReader, fieldName)
*/
Terms terms = MultiFields.getTerms(reader, fieldName);
if ((terms == null) || (terms.size() == -1))
System.out.println(" The term dictionary is empty.");
else {
System.out.println(" Vocabulary size: " + terms.size() + " terms");
TermsEnum ithTerm = terms.iterator(null);
/*
* Iterate over the terms in this document.
* Information about a term's occurrences (tf and
* positions) is accessed via the indexing API, which
* returns inverted lists that describe (only) the
* current document.
*/
while (ithTerm.next() != null) {
System.out.format(
" %-30s %d %d\n",
ithTerm.term().utf8ToString(), ithTerm.docFreq(), ithTerm.totalTermFreq());
}
;
}
;
}
/**
* Adds terms and frequencies found in vector into the Map termFreqMap
*
* @param field2termFreqMap a Map of terms and their frequencies per field
* @param vector List of terms and their frequencies for a doc/field
*/
private void addTermFrequencies(
Map<String, Map<String, Int>> field2termFreqMap, Terms vector, String fieldName)
throws IOException {
Map<String, Int> termFreqMap = field2termFreqMap.get(fieldName);
if (termFreqMap == null) {
termFreqMap = new HashMap<>();
field2termFreqMap.put(fieldName, termFreqMap);
}
final TermsEnum termsEnum = vector.iterator();
final CharsRefBuilder spare = new CharsRefBuilder();
BytesRef text;
while ((text = termsEnum.next()) != null) {
spare.copyUTF8Bytes(text);
final String term = spare.toString();
if (isNoiseWord(term)) {
continue;
}
final int freq = (int) termsEnum.totalTermFreq();
// increment frequency
Int cnt = termFreqMap.get(term);
if (cnt == null) {
cnt = new Int();
termFreqMap.put(term, cnt);
cnt.x = freq;
} else {
cnt.x += freq;
}
}
}
/**
* Prepare a document reconstructor.
*
* @param reader IndexReader to read from.
* @param fieldNames if non-null or not empty, data will be collected only from these fields,
* otherwise data will be collected from all fields
* @param numTerms total number of terms in the index, or -1 if unknown (will be calculated)
* @throws Exception
*/
public DocReconstructor(IndexReader reader, String[] fieldNames, int numTerms) throws Exception {
if (reader == null) {
throw new Exception("IndexReader cannot be null.");
}
this.reader = reader;
if (fieldNames == null || fieldNames.length == 0) {
// collect fieldNames
this.fieldNames = (String[]) reader.getFieldNames(FieldOption.ALL).toArray(new String[0]);
} else {
this.fieldNames = fieldNames;
}
if (numTerms == -1) {
Fields fields = MultiFields.getFields(reader);
numTerms = 0;
FieldsEnum fe = fields.iterator();
String fld = null;
while ((fld = fe.next()) != null) {
TermsEnum te = fe.terms();
while (te.next() != null) {
numTerms++;
}
}
this.numTerms = numTerms;
}
deleted = MultiFields.getDeletedDocs(reader);
}
/**
* List all of the files in this index database
*
* @throws IOException If an IO error occurs while reading from the database
*/
public void listFiles() throws IOException {
IndexReader ireader = null;
TermsEnum iter;
Terms terms = null;
try {
ireader = DirectoryReader.open(indexDirectory); // open existing index
int numDocs = ireader.numDocs();
if (numDocs > 0) {
Fields uFields = MultiFields.getFields(ireader); // reader.getTermVectors(0);
terms = uFields.terms(QueryBuilder.U);
}
iter = terms.iterator(null); // init uid iterator
while (iter.term() != null) {
log.fine(Util.uid2url(iter.term().utf8ToString()));
iter.next();
}
} finally {
if (ireader != null) {
try {
ireader.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index reader", e);
}
}
}
}
public void listTokens(int freq) throws IOException {
IndexReader ireader = null;
TermsEnum iter = null;
Terms terms = null;
try {
ireader = DirectoryReader.open(indexDirectory);
int numDocs = ireader.numDocs();
if (numDocs > 0) {
Fields uFields = MultiFields.getFields(ireader); // reader.getTermVectors(0);
terms = uFields.terms(QueryBuilder.DEFS);
}
iter = terms.iterator(null); // init uid iterator
while (iter.term() != null) {
// if (iter.term().field().startsWith("f")) {
if (iter.docFreq() > 16 && iter.term().utf8ToString().length() > freq) {
log.warning(iter.term().utf8ToString());
}
iter.next();
/*} else {
break;
}*/
}
} finally {
if (ireader != null) {
try {
ireader.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing index reader", e);
}
}
}
}
public String[] getTerms() {
IndexReader reader = null;
int maxSize = 100;
Set<String> searchResults = new HashSet<String>();
try {
reader = DirectoryReader.open(dir);
Terms terms = SlowCompositeReaderWrapper.wrap(reader).terms("contents");
TermsEnum termsEnum = terms.iterator(TermsEnum.EMPTY);
BytesRef byteRef = null;
while ((byteRef = termsEnum.next()) != null) {
String term = new String(byteRef.bytes, byteRef.offset, byteRef.length);
searchResults.add(term);
if (searchResults.size() >= maxSize) {
break;
}
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
if (reader != null) {
reader.close();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return searchResults.toArray(new String[searchResults.size()]);
}
/**
* Adds terms and frequencies found in vector into the Map termFreqMap
*
* @param termFreqMap a Map of terms and their frequencies
* @param vector List of terms and their frequencies for a doc/field
* @param fieldName Optional field name of the terms for skip terms
*/
private void addTermFrequencies(
Map<String, Int> termFreqMap, Terms vector, @Nullable String fieldName) throws IOException {
final TermsEnum termsEnum = vector.iterator();
final CharsRefBuilder spare = new CharsRefBuilder();
BytesRef text;
while ((text = termsEnum.next()) != null) {
spare.copyUTF8Bytes(text);
final String term = spare.toString();
if (isNoiseWord(term)) {
continue;
}
if (isSkipTerm(fieldName, term)) {
continue;
}
final PostingsEnum docs = termsEnum.postings(null, null);
int freq = 0;
while (docs != null && docs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
freq += docs.freq();
}
// increment frequency
Int cnt = termFreqMap.get(term);
if (cnt == null) {
cnt = new Int();
termFreqMap.put(term, cnt);
cnt.x = freq;
} else {
cnt.x += freq;
}
}
}
/**
* Add term frequencies for a single document to a frequency map.
*
* @param reader the index
* @param doc doc id
* @param luceneName the index field from which to use the term vector
* @param freq where to add to the token frequencies
*/
public static void getFrequenciesFromTermVector(
IndexReader reader, int doc, String luceneName, Map<String, Integer> freq) {
try {
org.apache.lucene.index.Terms terms = reader.getTermVector(doc, luceneName);
if (terms == null) {
throw new IllegalArgumentException("Field " + luceneName + " has no Terms");
}
TermsEnum termsEnum = terms.iterator();
// Verzamel concordantiewoorden uit term vector
PostingsEnum postingsEnum = null;
while (termsEnum.next() != null) {
postingsEnum = termsEnum.postings(null, postingsEnum, PostingsEnum.FREQS);
String term = termsEnum.term().utf8ToString();
Integer n = freq.get(term);
if (n == null) {
n = 0;
}
while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
n += termsEnum.docFreq();
}
freq.put(term, n);
}
} catch (Exception e) {
throw ExUtil.wrapRuntimeException(e);
}
}
@Override
public void seekExact(long targetOrd) throws IOException {
int delta = (int) (targetOrd - ordBase - ord);
// System.out.println(" seek(ord) targetOrd=" + targetOrd + " delta=" + delta + " ord=" + ord
// + " ii=" + indexInterval);
if (delta < 0 || delta > indexInterval) {
final int idx = (int) (targetOrd >>> indexIntervalBits);
final BytesRef base = indexedTermsArray[idx];
// System.out.println(" do seek term=" + base.utf8ToString());
ord = idx << indexIntervalBits;
delta = (int) (targetOrd - ord);
final TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(base);
assert seekStatus == TermsEnum.SeekStatus.FOUND;
} else {
// System.out.println("seek w/in block");
}
while (--delta >= 0) {
BytesRef br = termsEnum.next();
if (br == null) {
assert false;
return;
}
ord++;
}
setTerm();
assert term != null;
}
@Override
public void hitExecute(SearchContext context, HitContext hitContext) {
if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) {
return;
}
String field = context.getFetchSubPhaseContext(CONTEXT_FACTORY).getField();
if (hitContext.hit().fieldsOrNull() == null) {
hitContext.hit().fields(new HashMap<>());
}
SearchHitField hitField = hitContext.hit().fields().get(NAMES[0]);
if (hitField == null) {
hitField = new InternalSearchHitField(NAMES[0], new ArrayList<>(1));
hitContext.hit().fields().put(NAMES[0], hitField);
}
TermVectorsResponse termVector =
TermVectorsService.getTermVectors(
context.indexShard(),
new TermVectorsRequest(
context.indexShard().shardId().getIndex().getName(),
hitContext.hit().type(),
hitContext.hit().id()));
try {
Map<String, Integer> tv = new HashMap<>();
TermsEnum terms = termVector.getFields().terms(field).iterator();
BytesRef term;
while ((term = terms.next()) != null) {
tv.put(term.utf8ToString(), terms.postings(null, PostingsEnum.ALL).freq());
}
hitField.values().add(tv);
} catch (IOException e) {
ESLoggerFactory.getLogger(FetchSubPhasePluginIT.class.getName())
.info("Swallowed exception", e);
}
}
/**
* Find terms in the index based on a prefix. Useful for autocomplete.
*
* @param index the index
* @param fieldName the field
* @param prefix the prefix we're looking for (null or empty string for all terms)
* @param sensitive match case-sensitively or not?
* @param maxResults max. number of results to return (or -1 for all)
* @return the matching terms
*/
public static List<String> findTermsByPrefix(
LeafReader index, String fieldName, String prefix, boolean sensitive, int maxResults) {
boolean allTerms = prefix == null || prefix.length() == 0;
if (allTerms) {
prefix = "";
sensitive = true; // don't do unnecessary work in this case
}
try {
if (!sensitive) prefix = StringUtil.removeAccents(prefix).toLowerCase();
org.apache.lucene.index.Terms terms = index.terms(fieldName);
List<String> results = new ArrayList<>();
TermsEnum termsEnum = terms.iterator();
BytesRef brPrefix = new BytesRef(prefix.getBytes(LUCENE_DEFAULT_CHARSET));
termsEnum.seekCeil(brPrefix); // find the prefix in the terms list
while (maxResults < 0 || results.size() < maxResults) {
BytesRef term = termsEnum.next();
if (term == null) break;
String termText = term.utf8ToString();
String optDesensitized = termText;
if (!sensitive) optDesensitized = StringUtil.removeAccents(termText).toLowerCase();
if (!allTerms && !optDesensitized.substring(0, prefix.length()).equalsIgnoreCase(prefix)) {
// Doesn't match prefix or different field; no more matches
break;
}
// Match, add term
results.add(termText);
}
return results;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/*
* Utility function to display a term vector.
*/
static void termVectorDisplay(Terms terms) throws IOException {
if ((terms == null) || (terms.size() == -1)) System.out.println(" The field is not stored.");
else {
/*
* The terms for this field are stored.
*/
System.out.println(" Vocabulary size: " + terms.size() + " terms");
TermsEnum ithTerm = terms.iterator(null);
/*
* Iterate over the terms in this document.
* Information about a term's occurrences (tf and
* positions) is accessed via the indexing API, which
* returns inverted lists that describe (only) the
* current document.
*/
while (ithTerm.next() != null) {
System.out.format(
" %10d %-20s %d ",
ithTerm.ord(), ithTerm.term().utf8ToString(), ithTerm.totalTermFreq());
DocsAndPositionsEnum currDoc = ithTerm.docsAndPositions(null, null);
currDoc.nextDoc();
for (int jthPosition = 0; jthPosition < ithTerm.totalTermFreq(); jthPosition++)
System.out.print(currDoc.nextPosition() + " ");
System.out.println();
}
;
}
;
}
private void getPrefixTerms(
ObjectHashSet<Term> terms, final Term prefix, final IndexReader reader) throws IOException {
// SlowCompositeReaderWrapper could be used... but this would merge all terms from each segment
// into one terms
// instance, which is very expensive. Therefore I think it is better to iterate over each leaf
// individually.
List<LeafReaderContext> leaves = reader.leaves();
for (LeafReaderContext leaf : leaves) {
Terms _terms = leaf.reader().terms(field);
if (_terms == null) {
continue;
}
TermsEnum termsEnum = _terms.iterator();
TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(prefix.bytes());
if (TermsEnum.SeekStatus.END == seekStatus) {
continue;
}
for (BytesRef term = termsEnum.term(); term != null; term = termsEnum.next()) {
if (!StringHelper.startsWith(term, prefix.bytes())) {
break;
}
terms.add(new Term(field, BytesRef.deepCopyOf(term)));
if (terms.size() >= maxExpansions) {
return;
}
}
}
}
protected void compareTermVectors(Terms terms, Terms memTerms, String field_name)
throws IOException {
TermsEnum termEnum = terms.iterator();
TermsEnum memTermEnum = memTerms.iterator();
while (termEnum.next() != null) {
assertNotNull(memTermEnum.next());
assertThat(termEnum.totalTermFreq(), equalTo(memTermEnum.totalTermFreq()));
PostingsEnum docsPosEnum = termEnum.postings(null, PostingsEnum.POSITIONS);
PostingsEnum memDocsPosEnum = memTermEnum.postings(null, PostingsEnum.POSITIONS);
String currentTerm = termEnum.term().utf8ToString();
assertThat(
"Token mismatch for field: " + field_name,
currentTerm,
equalTo(memTermEnum.term().utf8ToString()));
docsPosEnum.nextDoc();
memDocsPosEnum.nextDoc();
int freq = docsPosEnum.freq();
assertThat(freq, equalTo(memDocsPosEnum.freq()));
for (int i = 0; i < freq; i++) {
String failDesc = " (field:" + field_name + " term:" + currentTerm + ")";
int memPos = memDocsPosEnum.nextPosition();
int pos = docsPosEnum.nextPosition();
assertThat("Position test failed" + failDesc, memPos, equalTo(pos));
assertThat(
"Start offset test failed" + failDesc,
memDocsPosEnum.startOffset(),
equalTo(docsPosEnum.startOffset()));
assertThat(
"End offset test failed" + failDesc,
memDocsPosEnum.endOffset(),
equalTo(docsPosEnum.endOffset()));
assertThat(
"Missing payload test failed" + failDesc,
docsPosEnum.getPayload(),
equalTo(docsPosEnum.getPayload()));
}
}
assertNull("Still some tokens not processed", memTermEnum.next());
}
public static void fillQueue(TermsEnum termsEnum, TermStatsQueue tiq, String field)
throws Exception {
BytesRef term;
while ((term = termsEnum.next()) != null) {
BytesRef r = new BytesRef();
r.copyBytes(term);
tiq.insertWithOverflow(new TermStats(field, r, termsEnum.docFreq()));
}
}
public void test10kPulsed() throws Exception {
// we always run this test with pulsing codec.
Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing41PostingsFormat(1));
File f = _TestUtil.getTempDir("10kpulsed");
BaseDirectoryWrapper dir = newFSDirectory(f);
dir.setCheckIndexOnClose(false); // we do this ourselves explicitly
RandomIndexWriter iw =
new RandomIndexWriter(
random(),
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
Document document = new Document();
FieldType ft = new FieldType(TextField.TYPE_STORED);
switch (_TestUtil.nextInt(random(), 0, 2)) {
case 0:
ft.setIndexOptions(IndexOptions.DOCS_ONLY);
break;
case 1:
ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
break;
default:
ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
break;
}
Field field = newField("field", "", ft);
document.add(field);
NumberFormat df = new DecimalFormat("00000", new DecimalFormatSymbols(Locale.ROOT));
for (int i = 0; i < 10050; i++) {
field.setStringValue(df.format(i));
iw.addDocument(document);
}
IndexReader ir = iw.getReader();
iw.close();
TermsEnum te = MultiFields.getTerms(ir, "field").iterator(null);
DocsEnum de = null;
for (int i = 0; i < 10050; i++) {
String expected = df.format(i);
assertEquals(expected, te.next().utf8ToString());
de = _TestUtil.docs(random(), te, null, de, 0);
assertTrue(de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(DocIdSetIterator.NO_MORE_DOCS, de.nextDoc());
}
ir.close();
_TestUtil.checkIndex(dir);
dir.close();
}
/**
* We will implement this according to the Lucene specification the formula used: sum ( IDF(qi) *
* (df(qi,D) * (k+1)) / (df(qi,D) + k * (1-b + b*|D| / avgFL)) IDF and avgFL computation are
* described above.
*
* @param doc
* @param terms
* @param context
* @return
*/
@Override
public float extract(Document doc, Terms terms, RerankerContext context) {
Set<String> queryTokens = new HashSet<>(context.getQueryTokens());
TermsEnum termsEnum = null;
try {
termsEnum = terms.iterator();
} catch (IOException e) {
LOG.warn("Error computing BM25, unable to retrieve terms enum");
return 0.0f;
}
IndexReader reader = context.getIndexSearcher().getIndexReader();
long maxDocs = reader.numDocs();
long sumTotalTermFreq = getSumTermFrequency(reader, context.getField());
// Compute by iterating
long docSize = 0L;
// NOTE df cannot be retrieved just from the term vector,
// the term vector here is only a partial term vector that treats this as if we only have 1
// document in the index
Map<String, Integer> docFreqMap = null;
try {
docFreqMap = getDocFreqs(reader, context.getQueryTokens(), context.getField());
} catch (IOException e) {
LOG.warn("Unable to retrieve document frequencies.");
docFreqMap = new HashMap<>();
}
Map<String, Long> termFreqMap = new HashMap<>();
try {
while (termsEnum.next() != null) {
String termString = termsEnum.term().utf8ToString();
docSize += termsEnum.totalTermFreq();
if (queryTokens.contains(termString)) {
termFreqMap.put(termString, termsEnum.totalTermFreq());
}
}
} catch (IOException e) {
LOG.warn("Unable to retrieve termsEnum, treating as 0");
}
float score = 0.0f;
// Iterate over the query tokens
double avgFL = computeAvgFL(sumTotalTermFreq, maxDocs);
for (String token : queryTokens) {
long docFreq = docFreqMap.containsKey(token) ? docFreqMap.get(token) : 0;
double termFreq = termFreqMap.containsKey(token) ? termFreqMap.get(token) : 0;
double numerator = (this.k1 + 1) * termFreq;
double docLengthFactor = this.b * (docSize / avgFL);
double denominator = termFreq + (this.k1) * (1 - this.b + docLengthFactor);
score += computeIDF(docFreq, maxDocs) * numerator / denominator;
}
return score;
}
public void testThreeBlocks() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
List<String> terms = new ArrayList<>();
for (int i = 0; i < 36; i++) {
Document doc = new Document();
String term = "" + (char) (97 + i);
terms.add(term);
if (VERBOSE) {
System.out.println("i=" + i + " term=" + term);
}
doc.add(newTextField("field", term, Field.Store.NO));
w.addDocument(doc);
}
for (int i = 0; i < 36; i++) {
Document doc = new Document();
String term = "m" + (char) (97 + i);
terms.add(term);
if (VERBOSE) {
System.out.println("i=" + i + " term=" + term);
}
doc.add(newTextField("field", term, Field.Store.NO));
w.addDocument(doc);
}
for (int i = 0; i < 36; i++) {
Document doc = new Document();
String term = "mo" + (char) (97 + i);
terms.add(term);
if (VERBOSE) {
System.out.println("i=" + i + " term=" + term);
}
doc.add(newTextField("field", term, Field.Store.NO));
w.addDocument(doc);
}
w.forceMerge(1);
IndexReader r = w.getReader();
TermsEnum te = MultiFields.getTerms(r, "field").iterator(null);
if (VERBOSE) {
while (te.next() != null) {
System.out.println("TERM: " + te.ord() + " " + te.term().utf8ToString());
}
}
assertTrue(te.seekExact(new BytesRef("mo")));
assertEquals(27, te.ord());
te.seekExact(90);
assertEquals(new BytesRef("s"), te.term());
testEnum(te, terms);
r.close();
w.close();
dir.close();
}
@Override
public final boolean incrementToken() throws IOException {
clearAttributes();
BytesRef bytes = termsEnum.next();
if (bytes == null) return false;
charTerm.setEmpty();
charTerm.append(bytes.utf8ToString());
return true;
}
protected void fill(String field, TermsEnum termsEnum) throws IOException {
while (true) {
BytesRef term = termsEnum.next();
if (term != null) {
insertWithOverflow(new TermStats(field, term, termsEnum.docFreq()));
} else {
break;
}
}
}
@Override
public BytesRef next() throws IOException {
if (++ord < 0) {
ord = 0;
}
if (termsEnum.next() == null) {
term = null;
return null;
}
return setTerm(); // this is extra work if we know we are in bounds...
}
public void testBasic() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newTextField("field", "a b c", Field.Store.NO));
w.addDocument(doc);
IndexReader r = w.getReader();
TermsEnum te = MultiFields.getTerms(r, "field").iterator(null);
// Test next()
assertEquals(new BytesRef("a"), te.next());
assertEquals(0L, te.ord());
assertEquals(new BytesRef("b"), te.next());
assertEquals(1L, te.ord());
assertEquals(new BytesRef("c"), te.next());
assertEquals(2L, te.ord());
assertNull(te.next());
// Test seekExact by term
assertTrue(te.seekExact(new BytesRef("b")));
assertEquals(1, te.ord());
assertTrue(te.seekExact(new BytesRef("a")));
assertEquals(0, te.ord());
assertTrue(te.seekExact(new BytesRef("c")));
assertEquals(2, te.ord());
// Test seekExact by ord
te.seekExact(1);
assertEquals(new BytesRef("b"), te.term());
te.seekExact(0);
assertEquals(new BytesRef("a"), te.term());
te.seekExact(2);
assertEquals(new BytesRef("c"), te.term());
r.close();
w.close();
dir.close();
}
private void buildTerm(
XContentBuilder builder, final CharsRefBuilder spare, Terms curTerms, TermsEnum termIter)
throws IOException {
// start term, optimized writing
BytesRef term = termIter.next();
spare.copyUTF8Bytes(term);
builder.startObject(spare.toString());
buildTermStatistics(builder, termIter);
// finally write the term vectors
PostingsEnum posEnum = termIter.postings(null, null, PostingsEnum.ALL);
int termFreq = posEnum.freq();
builder.field(FieldStrings.TERM_FREQ, termFreq);
initMemory(curTerms, termFreq);
initValues(curTerms, posEnum, termFreq);
buildValues(builder, curTerms, termFreq);
builder.endObject();
}