@Override
public void copy(int slot, int doc) {
int v1 = (int) currentReaderValuesField1.get(doc);
field1Values[slot] = v1;
int v2 = (int) currentReaderValuesField2.get(doc);
field2Values[slot] = v2;
}
public void collapse(int ord, int contextDoc, int globalDoc) throws IOException {
float val = Float.intBitsToFloat((int) vals.get(contextDoc));
if (ord > -1) {
if (comp.test(val, ordVals[ord])) {
ords[ord] = globalDoc;
ordVals[ord] = val;
if (needsScores) {
scores[ord] = scorer.score();
}
}
} else if (this.collapsedSet.get(globalDoc)) {
// Elevated doc so do nothing
} else if (this.nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
if (comp.test(val, nullVal)) {
nullVal = val;
nullDoc = globalDoc;
if (needsScores) {
nullScore = scorer.score();
}
}
} else if (this.nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
this.collapsedSet.set(globalDoc);
if (needsScores) {
nullScores.add(scorer.score());
}
}
}
/*
* LUCENE-6006: Test undead norms.
* .....
* C C /
* /< /
* ___ __________/_#__=o
* /(- /(\_\________ \
* \ ) \ )_ \o \
* /|\ /|\ |' |
* | _|
* /o __\
* / ' |
* / / |
* /_/\______|
* ( _( <
* \ \ \
* \ \ |
* \____\____\
* ____\_\__\_\
* /` /` o\
* |___ |_______|
*
*/
public void testReadUndeadNorms() throws Exception {
InputStream resource =
TestLucene42NormsFormat.class.getResourceAsStream("index.42.undeadnorms.zip");
assertNotNull(resource);
Path path = createTempDir("undeadnorms");
TestUtil.unzip(resource, path);
Directory dir = FSDirectory.open(path);
IndexReader r = DirectoryReader.open(dir);
NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
assertNotNull(undeadNorms);
assertEquals(2, r.maxDoc());
assertEquals(0, undeadNorms.get(0));
assertEquals(0, undeadNorms.get(1));
dir.close();
r.close();
}
public void testBasics() throws Exception {
// sanity check of norms writer
// TODO: generalize
LeafReader slow = SlowCompositeReaderWrapper.wrap(reader);
NumericDocValues fooNorms = slow.getNormValues("foo");
NumericDocValues barNorms = slow.getNormValues("bar");
for (int i = 0; i < slow.maxDoc(); i++) {
assertFalse(fooNorms.get(i) == barNorms.get(i));
}
// sanity check of searching
TopDocs foodocs = searcher.search(new TermQuery(new Term("foo", "brown")), 10);
assertTrue(foodocs.totalHits > 0);
TopDocs bardocs = searcher.search(new TermQuery(new Term("bar", "brown")), 10);
assertTrue(bardocs.totalHits > 0);
assertTrue(foodocs.scoreDocs[0].score < bardocs.scoreDocs[0].score);
}
public void testCopyFieldsAndFieldBoostsAndDocBoosts() throws Exception {
SolrCore core = h.getCore();
IndexSchema schema = core.getLatestSchema();
SolrInputDocument doc = new SolrInputDocument();
final float DOC_BOOST = 3.0F;
doc.setDocumentBoost(DOC_BOOST);
doc.addField("id", "42");
SolrInputField inTitle = new SolrInputField("title");
inTitle.addValue("titleA", 2.0F);
inTitle.addValue("titleB", 7.0F);
final float TITLE_BOOST = 2.0F * 7.0F;
assertEquals(TITLE_BOOST, inTitle.getBoost(), 0.0F);
doc.put(inTitle.getName(), inTitle);
SolrInputField inFoo = new SolrInputField("foo_t");
inFoo.addValue("summer time", 1.0F);
inFoo.addValue("in the city", 5.0F);
inFoo.addValue("living is easy", 11.0F);
final float FOO_BOOST = 1.0F * 5.0F * 11.0F;
assertEquals(FOO_BOOST, inFoo.getBoost(), 0.0F);
doc.put(inFoo.getName(), inFoo);
Document out = DocumentBuilder.toDocument(doc, schema);
IndexableField[] outTitle = out.getFields(inTitle.getName());
assertEquals("wrong number of title values", 2, outTitle.length);
IndexableField[] outNoNorms = out.getFields("title_stringNoNorms");
assertEquals("wrong number of nonorms values", 2, outNoNorms.length);
IndexableField[] outFoo = out.getFields(inFoo.getName());
assertEquals("wrong number of foo values", 3, outFoo.length);
IndexableField[] outText = out.getFields("text");
assertEquals("wrong number of text values", 5, outText.length);
// since Lucene no longer has native document boosts, we should find
// the doc boost multiplied into the boost on the first field value
// of each field. All other field values should be 1.0f
// (lucene will multiply all of the field value boosts later)
assertEquals(TITLE_BOOST * DOC_BOOST, outTitle[0].boost(), 0.0F);
assertEquals(1.0F, outTitle[1].boost(), 0.0F);
//
assertEquals(FOO_BOOST * DOC_BOOST, outFoo[0].boost(), 0.0F);
assertEquals(1.0F, outFoo[1].boost(), 0.0F);
assertEquals(1.0F, outFoo[2].boost(), 0.0F);
//
assertEquals(TITLE_BOOST * DOC_BOOST, outText[0].boost(), 0.0F);
assertEquals(1.0F, outText[1].boost(), 0.0F);
assertEquals(FOO_BOOST, outText[2].boost(), 0.0F);
assertEquals(1.0F, outText[3].boost(), 0.0F);
assertEquals(1.0F, outText[4].boost(), 0.0F);
// copyField dest with no norms should not have recieved any boost
assertEquals(1.0F, outNoNorms[0].boost(), 0.0F);
assertEquals(1.0F, outNoNorms[1].boost(), 0.0F);
// now index that SolrInputDocument to check the computed norms
assertU(adoc(doc));
assertU(commit());
SolrQueryRequest req = req("q", "id:42");
try {
// very hack-ish
SolrQueryResponse rsp = new SolrQueryResponse();
core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp);
DocList dl = ((ResultContext) rsp.getValues().get("response")).docs;
assertTrue("can't find the doc we just added", 1 == dl.size());
int docid = dl.iterator().nextDoc();
SolrIndexSearcher searcher = req.getSearcher();
AtomicReader reader =
SlowCompositeReaderWrapper.wrap(searcher.getTopReaderContext().reader());
assertTrue(
"similarity doesn't extend DefaultSimilarity, "
+ "config or defaults have changed since test was written",
searcher.getSimilarity() instanceof DefaultSimilarity);
DefaultSimilarity sim = (DefaultSimilarity) searcher.getSimilarity();
NumericDocValues titleNorms = reader.getNormValues("title");
NumericDocValues fooNorms = reader.getNormValues("foo_t");
NumericDocValues textNorms = reader.getNormValues("text");
assertEquals(expectedNorm(sim, 2, TITLE_BOOST * DOC_BOOST), titleNorms.get(docid));
assertEquals(expectedNorm(sim, 8 - 3, FOO_BOOST * DOC_BOOST), fooNorms.get(docid));
assertEquals(
expectedNorm(sim, 2 + 8 - 3, TITLE_BOOST * FOO_BOOST * DOC_BOOST), textNorms.get(docid));
} finally {
req.close();
}
}
@Override
public int compareBottom(int doc) {
int v = (int) (currentReaderValuesField1.get(doc) + currentReaderValuesField2.get(doc));
return compareValues(bottom, v);
}
public void testDocValuesMemoryIndexVsNormalIndex() throws Exception {
Document doc = new Document();
long randomLong = random().nextLong();
doc.add(new NumericDocValuesField("numeric", randomLong));
if (random().nextBoolean()) {
doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO));
}
int numValues = atLeast(5);
for (int i = 0; i < numValues; i++) {
randomLong = random().nextLong();
doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong));
if (random().nextBoolean()) {
// randomly duplicate field/value
doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong));
}
if (random().nextBoolean()) {
doc.add(new LegacyLongField("numeric", randomLong, Field.Store.NO));
}
}
BytesRef randomTerm = new BytesRef(randomTerm());
doc.add(new BinaryDocValuesField("binary", randomTerm));
if (random().nextBoolean()) {
doc.add(new StringField("binary", randomTerm, Field.Store.NO));
}
randomTerm = new BytesRef(randomTerm());
doc.add(new SortedDocValuesField("sorted", randomTerm));
if (random().nextBoolean()) {
doc.add(new StringField("sorted", randomTerm, Field.Store.NO));
}
numValues = atLeast(5);
for (int i = 0; i < numValues; i++) {
randomTerm = new BytesRef(randomTerm());
doc.add(new SortedSetDocValuesField("sorted_set", randomTerm));
if (random().nextBoolean()) {
// randomly duplicate field/value
doc.add(new SortedSetDocValuesField("sorted_set", randomTerm));
}
if (random().nextBoolean()) {
// randomily just add a normal string field
doc.add(new StringField("sorted_set", randomTerm, Field.Store.NO));
}
}
MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, mockAnalyzer);
IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
LeafReader leafReader = indexReader.leaves().get(0).reader();
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer));
writer.addDocument(doc);
writer.close();
IndexReader controlIndexReader = DirectoryReader.open(dir);
LeafReader controlLeafReader = controlIndexReader.leaves().get(0).reader();
NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric");
NumericDocValues controlNumericDocValues = controlLeafReader.getNumericDocValues("numeric");
assertEquals(controlNumericDocValues.get(0), numericDocValues.get(0));
SortedNumericDocValues sortedNumericDocValues =
leafReader.getSortedNumericDocValues("sorted_numeric");
sortedNumericDocValues.setDocument(0);
SortedNumericDocValues controlSortedNumericDocValues =
controlLeafReader.getSortedNumericDocValues("sorted_numeric");
controlSortedNumericDocValues.setDocument(0);
assertEquals(controlSortedNumericDocValues.count(), sortedNumericDocValues.count());
for (int i = 0; i < controlSortedNumericDocValues.count(); i++) {
assertEquals(controlSortedNumericDocValues.valueAt(i), sortedNumericDocValues.valueAt(i));
}
BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary");
BinaryDocValues controlBinaryDocValues = controlLeafReader.getBinaryDocValues("binary");
assertEquals(controlBinaryDocValues.get(0), binaryDocValues.get(0));
SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted");
SortedDocValues controlSortedDocValues = controlLeafReader.getSortedDocValues("sorted");
assertEquals(controlSortedDocValues.getValueCount(), sortedDocValues.getValueCount());
assertEquals(controlSortedDocValues.get(0), sortedDocValues.get(0));
assertEquals(controlSortedDocValues.getOrd(0), sortedDocValues.getOrd(0));
assertEquals(controlSortedDocValues.lookupOrd(0), sortedDocValues.lookupOrd(0));
SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
sortedSetDocValues.setDocument(0);
SortedSetDocValues controlSortedSetDocValues =
controlLeafReader.getSortedSetDocValues("sorted_set");
controlSortedSetDocValues.setDocument(0);
assertEquals(controlSortedSetDocValues.getValueCount(), sortedSetDocValues.getValueCount());
for (long controlOrd = controlSortedSetDocValues.nextOrd();
controlOrd != SortedSetDocValues.NO_MORE_ORDS;
controlOrd = controlSortedSetDocValues.nextOrd()) {
assertEquals(controlOrd, sortedSetDocValues.nextOrd());
assertEquals(
controlSortedSetDocValues.lookupOrd(controlOrd),
sortedSetDocValues.lookupOrd(controlOrd));
}
assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd());
indexReader.close();
controlIndexReader.close();
dir.close();
}
private void duellReaders(CompositeReader other, LeafReader memIndexReader) throws IOException {
Fields memFields = memIndexReader.fields();
for (String field : MultiFields.getFields(other)) {
Terms memTerms = memFields.terms(field);
Terms iwTerms = memIndexReader.terms(field);
if (iwTerms == null) {
assertNull(memTerms);
} else {
NumericDocValues normValues = MultiDocValues.getNormValues(other, field);
NumericDocValues memNormValues = memIndexReader.getNormValues(field);
if (normValues != null) {
// mem idx always computes norms on the fly
assertNotNull(memNormValues);
assertEquals(normValues.get(0), memNormValues.get(0));
}
assertNotNull(memTerms);
assertEquals(iwTerms.getDocCount(), memTerms.getDocCount());
assertEquals(iwTerms.getSumDocFreq(), memTerms.getSumDocFreq());
assertEquals(iwTerms.getSumTotalTermFreq(), memTerms.getSumTotalTermFreq());
TermsEnum iwTermsIter = iwTerms.iterator();
TermsEnum memTermsIter = memTerms.iterator();
if (iwTerms.hasPositions()) {
final boolean offsets = iwTerms.hasOffsets() && memTerms.hasOffsets();
while (iwTermsIter.next() != null) {
assertNotNull(memTermsIter.next());
assertEquals(iwTermsIter.term(), memTermsIter.term());
PostingsEnum iwDocsAndPos = iwTermsIter.postings(null, PostingsEnum.ALL);
PostingsEnum memDocsAndPos = memTermsIter.postings(null, PostingsEnum.ALL);
while (iwDocsAndPos.nextDoc() != PostingsEnum.NO_MORE_DOCS) {
assertEquals(iwDocsAndPos.docID(), memDocsAndPos.nextDoc());
assertEquals(iwDocsAndPos.freq(), memDocsAndPos.freq());
for (int i = 0; i < iwDocsAndPos.freq(); i++) {
assertEquals(
"term: " + iwTermsIter.term().utf8ToString(),
iwDocsAndPos.nextPosition(),
memDocsAndPos.nextPosition());
if (offsets) {
assertEquals(iwDocsAndPos.startOffset(), memDocsAndPos.startOffset());
assertEquals(iwDocsAndPos.endOffset(), memDocsAndPos.endOffset());
}
if (iwTerms.hasPayloads()) {
assertEquals(iwDocsAndPos.getPayload(), memDocsAndPos.getPayload());
}
}
}
}
} else {
while (iwTermsIter.next() != null) {
assertEquals(iwTermsIter.term(), memTermsIter.term());
PostingsEnum iwDocsAndPos = iwTermsIter.postings(null);
PostingsEnum memDocsAndPos = memTermsIter.postings(null);
while (iwDocsAndPos.nextDoc() != PostingsEnum.NO_MORE_DOCS) {
assertEquals(iwDocsAndPos.docID(), memDocsAndPos.nextDoc());
assertEquals(iwDocsAndPos.freq(), memDocsAndPos.freq());
}
}
}
}
}
}
