private static DocComparatorSource getComparatorSource(Browsable browser, SortField sf) { DocComparatorSource compSource = null; if (SortField.FIELD_DOC.equals(sf)) { compSource = new DocIdDocComparatorSource(); } else if (SortField.FIELD_SCORE.equals(sf) || sf.getType() == SortField.SCORE) { // we want to do reverse sorting regardless for relevance compSource = new ReverseDocComparatorSource(new RelevanceDocComparatorSource()); } else if (sf instanceof BoboCustomSortField) { BoboCustomSortField custField = (BoboCustomSortField) sf; DocComparatorSource src = custField.getCustomComparatorSource(); assert src != null; compSource = src; } else { Set<String> facetNames = browser.getFacetNames(); String sortName = sf.getField(); if (facetNames.contains(sortName)) { FacetHandler<?> handler = browser.getFacetHandler(sortName); assert handler != null; compSource = handler.getDocComparatorSource(); } else { // default lucene field logger.info("doing default lucene sort for: " + sf); compSource = getNonFacetComparatorSource(sf); } } boolean reverse = sf.getReverse(); if (reverse) { compSource = new ReverseDocComparatorSource(compSource); } compSource.setReverse(reverse); return compSource; }
private SortSpec modifySortSpec( SortSpec current, boolean force, ElevationComparatorSource comparator) { boolean modify = false; SortField[] currentSorts = current.getSort().getSort(); List<SchemaField> currentFields = current.getSchemaFields(); ArrayList<SortField> sorts = new ArrayList<SortField>(currentSorts.length + 1); List<SchemaField> fields = new ArrayList<SchemaField>(currentFields.size() + 1); // Perhaps force it to always sort by score if (force && currentSorts[0].getType() != SortField.Type.SCORE) { sorts.add(new SortField("_elevate_", comparator, true)); fields.add(null); modify = true; } for (int i = 0; i < currentSorts.length; i++) { SortField sf = currentSorts[i]; if (sf.getType() == SortField.Type.SCORE) { sorts.add(new SortField("_elevate_", comparator, !sf.getReverse())); fields.add(null); modify = true; } sorts.add(sf); fields.add(currentFields.get(i)); } if (modify) { SortSpec newSpec = new SortSpec(new Sort(sorts.toArray(new SortField[sorts.size()])), fields); newSpec.setOffset(current.getOffset()); newSpec.setCount(current.getCount()); return newSpec; } return null; }
@Override public void parse(XContentParser parser, SearchContext context) throws Exception { XContentParser.Token token = parser.currentToken(); List<SortField> sortFields = Lists.newArrayListWithCapacity(2); if (token == XContentParser.Token.START_ARRAY) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { addCompoundSortField(parser, context, sortFields); } else if (token == XContentParser.Token.VALUE_STRING) { addSortField(context, sortFields, parser.text(), false, false, null, null, null, null); } } } else { addCompoundSortField(parser, context, sortFields); } if (!sortFields.isEmpty()) { // optimize if we just sort on score non reversed, we don't really need sorting boolean sort; if (sortFields.size() > 1) { sort = true; } else { SortField sortField = sortFields.get(0); if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) { sort = false; } else { sort = true; } } if (sort) { context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); } } }
public static boolean includesScore(Sort sort) { if (sort == null) return true; for (SortField sf : sort.getSort()) { if (sf.getType() == SortField.Type.SCORE) return true; } return false; }
@Override protected void sortFieldAssertions( GeoDistanceSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException { assertEquals(SortField.Type.CUSTOM, sortField.getType()); assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse()); assertEquals(builder.fieldName(), sortField.getField()); }
public static SortCollector buildSortCollector( Browsable browser, Query q, SortField[] sort, int offset, int count, boolean forceScoring, boolean fetchStoredFields, Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup, boolean collectDocIdCache) { boolean doScoring = forceScoring; if (sort == null || sort.length == 0) { if (q != null && !(q instanceof MatchAllDocsQuery)) { sort = new SortField[] {SortField.FIELD_SCORE}; } } if (sort == null || sort.length == 0) { sort = new SortField[] {SortField.FIELD_DOC}; } Set<String> facetNames = browser.getFacetNames(); for (SortField sf : sort) { if (sf.getType() == SortField.SCORE) { doScoring = true; break; } } DocComparatorSource compSource; if (sort.length == 1) { SortField sf = convert(browser, sort[0]); compSource = getComparatorSource(browser, sf); } else { DocComparatorSource[] compSources = new DocComparatorSource[sort.length]; for (int i = 0; i < sort.length; ++i) { compSources[i] = getComparatorSource(browser, convert(browser, sort[i])); } compSource = new MultiDocIdComparatorSource(compSources); } return new SortCollectorImpl( compSource, sort, browser, offset, count, doScoring, fetchStoredFields, termVectorsToFetch, groupBy, maxPerGroup, collectDocIdCache); }
private static SortField convert(Browsable browser, SortField sort) { String field = sort.getField(); FacetHandler<?> facetHandler = browser.getFacetHandler(field); if (facetHandler != null) { browser.getFacetHandler(field); BoboCustomSortField sortField = new BoboCustomSortField(field, sort.getReverse(), facetHandler.getDocComparatorSource()); return sortField; } else { return sort; } }
public boolean match(final Sort expected, final Sort acutal) { if (expected == acutal) { return true; } else if ((expected == null) || (acutal == null)) { return false; } else { final SortField[] expectedFields = expected.getSort(); final SortField[] actualFields = acutal.getSort(); if (expectedFields.length != actualFields.length) { return false; } final ArgumentMatcher<String> matcher = ArgumentMatchers.naturalMatcher(); for (int i = 0; i < actualFields.length; i++) { final SortField actualField = actualFields[i]; final SortField expectedField = expectedFields[i]; if (!matcher.match(expectedField.getField(), actualField.getField())) { return false; } if (actualField.getType() != expectedField.getType()) { return false; } if (expectedField.getReverse() != actualField.getReverse()) { return false; } } return true; } }
@Test public void testSortField() { Mapper mapper = new StringMapper("field", null, null, null); Schema schema = mock(Schema.class); when(schema.getAnalyzer()).thenReturn(PreBuiltAnalyzers.DEFAULT.get()); when(schema.getMapper("field")).thenReturn(mapper); SortField sortField = new SortField("field", true); org.apache.lucene.search.SortField luceneSortField = sortField.sortField(schema); assertNotNull(luceneSortField); assertEquals(org.apache.lucene.search.SortField.class, luceneSortField.getClass()); }
@Override protected void sortFieldAssertions( FieldSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException { SortField.Type expectedType; if (builder.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) { expectedType = SortField.Type.DOC; } else { expectedType = SortField.Type.CUSTOM; } assertEquals(expectedType, sortField.getType()); assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse()); if (expectedType == SortField.Type.CUSTOM) { assertEquals(builder.getFieldName(), sortField.getField()); } assertEquals(DocValueFormat.RAW, format); }
/** * 生成Hibernate的FullTextQuery全文搜索对象。 * * @return 返回Hibernate的FullTextQuery全文搜索对象。 */ public FullTextQuery generateQuery() { FullTextQuery fullTextQuery = session.createFullTextQuery(generateLuceneQuery(), clazz); if (!sortFields.isEmpty()) { for (SortField sortField : sortFields) { if (!searchFields.containsKey(sortField.getField())) { throw new HibernateException("全文搜索时指定的排序字段 " + sortField.getField() + " 必须包含在搜索字段中"); } } fullTextQuery.setSort(new Sort(sortFields.toArray(new SortField[] {}))); } if (filter != null) { fullTextQuery.setFilter(filter); } if (criteriaQuery != null) { fullTextQuery.setCriteriaQuery(criteriaQuery); } return fullTextQuery; }
private static DocComparatorSource getNonFacetComparatorSource(SortField sf) { String fieldname = sf.getField(); Locale locale = sf.getLocale(); if (locale != null) { return new DocComparatorSource.StringLocaleComparatorSource(fieldname, locale); } int type = sf.getType(); switch (type) { case SortField.INT: return new DocComparatorSource.IntDocComparatorSource(fieldname); case SortField.FLOAT: return new DocComparatorSource.FloatDocComparatorSource(fieldname); case SortField.LONG: return new DocComparatorSource.LongDocComparatorSource(fieldname); case SortField.DOUBLE: return new DocComparatorSource.LongDocComparatorSource(fieldname); case SortField.BYTE: return new DocComparatorSource.ByteDocComparatorSource(fieldname); case SortField.SHORT: return new DocComparatorSource.ShortDocComparatorSource(fieldname); case SortField.CUSTOM: throw new IllegalArgumentException("lucene custom sort no longer supported: " + fieldname); case SortField.STRING: return new DocComparatorSource.StringOrdComparatorSource(fieldname); case SortField.STRING_VAL: return new DocComparatorSource.StringValComparatorSource(fieldname); default: throw new IllegalStateException("Illegal sort type: " + type + ", for field: " + fieldname); } }
private NamedList unmarshalSortValues( SortSpec sortSpec, NamedList sortFieldValues, IndexSchema schema) { NamedList unmarshalledSortValsPerField = new NamedList(); if (0 == sortFieldValues.size()) return unmarshalledSortValsPerField; List<SchemaField> schemaFields = sortSpec.getSchemaFields(); SortField[] sortFields = sortSpec.getSort().getSort(); int marshalledFieldNum = 0; for (int sortFieldNum = 0; sortFieldNum < sortFields.length; sortFieldNum++) { final SortField sortField = sortFields[sortFieldNum]; final SortField.Type type = sortField.getType(); // :TODO: would be simpler to always serialize every position of SortField[] if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue; final String sortFieldName = sortField.getField(); final String valueFieldName = sortFieldValues.getName(marshalledFieldNum); assert sortFieldName.equals(valueFieldName) : "sortFieldValues name key does not match expected SortField.getField"; List sortVals = (List) sortFieldValues.getVal(marshalledFieldNum); final SchemaField schemaField = schemaFields.get(sortFieldNum); if (null == schemaField) { unmarshalledSortValsPerField.add(sortField.getField(), sortVals); } else { FieldType fieldType = schemaField.getType(); List unmarshalledSortVals = new ArrayList(); for (Object sortVal : sortVals) { unmarshalledSortVals.add(fieldType.unmarshalSortValue(sortVal)); } unmarshalledSortValsPerField.add(sortField.getField(), unmarshalledSortVals); } marshalledFieldNum++; } return unmarshalledSortValsPerField; }
public void testMiddleMaxMissingLast() throws Exception { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); doc.add(newStringField("id", "3", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); doc.add(new SortedSetDocValuesField("value", new BytesRef("a"))); doc.add(new SortedSetDocValuesField("value", new BytesRef("b"))); doc.add(new SortedSetDocValuesField("value", new BytesRef("c"))); doc.add(new SortedSetDocValuesField("value", new BytesRef("d"))); doc.add(newStringField("id", "1", Field.Store.YES)); writer.addDocument(doc); doc = new Document(); doc.add(new SortedSetDocValuesField("value", new BytesRef("b"))); doc.add(newStringField("id", "2", Field.Store.YES)); writer.addDocument(doc); IndexReader ir = writer.getReader(); writer.close(); // slow wrapper does not support random access ordinals (there is no need for that!) IndexSearcher searcher = newSearcher(ir, false); SortField sortField = new SortedSetSortField("value", false, SortedSetSortField.Selector.MIDDLE_MAX); sortField.setMissingValue(SortField.STRING_LAST); Sort sort = new Sort(sortField); TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); assertEquals(3, td.totalHits); // 'b' comes before 'c' assertEquals("2", searcher.doc(td.scoreDocs[0].doc).get("id")); assertEquals("1", searcher.doc(td.scoreDocs[1].doc).get("id")); // null comes last assertEquals("3", searcher.doc(td.scoreDocs[2].doc).get("id")); assertNoFieldCaches(); ir.close(); dir.close(); }
@Override Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException { assert sortField.getType().equals(SortField.Type.STRING); assert finalSortedValues == null && finalOrdMap == null && finalOrds == null; int valueCount = hash.size(); finalSortedValues = hash.sort(); finalOrds = pending.build(); finalOrdMap = new int[valueCount]; for (int ord = 0; ord < valueCount; ord++) { finalOrdMap[finalSortedValues[ord]] = ord; } final SortedDocValues docValues = new BufferedSortedDocValues( hash, valueCount, finalOrds, finalSortedValues, finalOrdMap, docsWithField.iterator()); return Sorter.getDocComparator(maxDoc, sortField, () -> docValues, () -> null); }
public static void writeTopDocs(StreamOutput out, TopDocs topDocs, int from) throws IOException { if (topDocs.scoreDocs.length - from < 0) { out.writeBoolean(false); return; } out.writeBoolean(true); if (topDocs instanceof TopFieldDocs) { out.writeBoolean(true); TopFieldDocs topFieldDocs = (TopFieldDocs) topDocs; out.writeVInt(topDocs.totalHits); out.writeFloat(topDocs.getMaxScore()); out.writeVInt(topFieldDocs.fields.length); for (SortField sortField : topFieldDocs.fields) { if (sortField.getField() == null) { out.writeBoolean(false); } else { out.writeBoolean(true); out.writeString(sortField.getField()); } if (sortField.getComparatorSource() != null) { writeSortType( out, ((IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource()) .reducedType()); } else { writeSortType(out, sortField.getType()); } out.writeBoolean(sortField.getReverse()); } out.writeVInt(topDocs.scoreDocs.length - from); int index = 0; for (ScoreDoc doc : topFieldDocs.scoreDocs) { if (index++ < from) { continue; } FieldDoc fieldDoc = (FieldDoc) doc; out.writeVInt(fieldDoc.fields.length); for (Object field : fieldDoc.fields) { if (field == null) { out.writeByte((byte) 0); } else { Class type = field.getClass(); if (type == String.class) { out.writeByte((byte) 1); out.writeString((String) field); } else if (type == Integer.class) { out.writeByte((byte) 2); out.writeInt((Integer) field); } else if (type == Long.class) { out.writeByte((byte) 3); out.writeLong((Long) field); } else if (type == Float.class) { out.writeByte((byte) 4); out.writeFloat((Float) field); } else if (type == Double.class) { out.writeByte((byte) 5); out.writeDouble((Double) field); } else if (type == Byte.class) { out.writeByte((byte) 6); out.writeByte((Byte) field); } else if (type == Short.class) { out.writeByte((byte) 7); out.writeShort((Short) field); } else if (type == Boolean.class) { out.writeByte((byte) 8); out.writeBoolean((Boolean) field); } else if (type == BytesRef.class) { out.writeByte((byte) 9); out.writeBytesRef((BytesRef) field); } else { throw new IOException("Can't handle sort field value of type [" + type + "]"); } } } out.writeVInt(doc.doc); out.writeFloat(doc.score); } } else { out.writeBoolean(false); out.writeVInt(topDocs.totalHits); out.writeFloat(topDocs.getMaxScore()); out.writeVInt(topDocs.scoreDocs.length - from); int index = 0; for (ScoreDoc doc : topDocs.scoreDocs) { if (index++ < from) { continue; } out.writeVInt(doc.doc); out.writeFloat(doc.score); } } }
private SortField randomIndexSortField() { boolean reversed = random().nextBoolean(); SortField sortField; switch (random().nextInt(10)) { case 0: sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.INT, reversed); if (random().nextBoolean()) { sortField.setMissingValue(random().nextInt()); } break; case 1: sortField = new SortedNumericSortField( TestUtil.randomSimpleString(random()), SortField.Type.INT, reversed); if (random().nextBoolean()) { sortField.setMissingValue(random().nextInt()); } break; case 2: sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.LONG, reversed); if (random().nextBoolean()) { sortField.setMissingValue(random().nextLong()); } break; case 3: sortField = new SortedNumericSortField( TestUtil.randomSimpleString(random()), SortField.Type.LONG, reversed); if (random().nextBoolean()) { sortField.setMissingValue(random().nextLong()); } break; case 4: sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.FLOAT, reversed); if (random().nextBoolean()) { sortField.setMissingValue(random().nextFloat()); } break; case 5: sortField = new SortedNumericSortField( TestUtil.randomSimpleString(random()), SortField.Type.FLOAT, reversed); if (random().nextBoolean()) { sortField.setMissingValue(random().nextFloat()); } break; case 6: sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.DOUBLE, reversed); if (random().nextBoolean()) { sortField.setMissingValue(random().nextDouble()); } break; case 7: sortField = new SortedNumericSortField( TestUtil.randomSimpleString(random()), SortField.Type.DOUBLE, reversed); if (random().nextBoolean()) { sortField.setMissingValue(random().nextDouble()); } break; case 8: sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.STRING, reversed); if (random().nextBoolean()) { sortField.setMissingValue(SortField.STRING_LAST); } break; case 9: sortField = new SortedSetSortField(TestUtil.randomSimpleString(random()), reversed); if (random().nextBoolean()) { sortField.setMissingValue(SortField.STRING_LAST); } break; default: sortField = null; fail(); } return sortField; }
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { Sort sort = rb.getSortSpec().getSort(); SortField[] sortFields = sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort(); NamedList sortVals = new NamedList(); // order is important for the sort fields Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field SolrIndexReader reader = searcher.getReader(); SolrIndexReader[] readers = reader.getLeafReaders(); SolrIndexReader subReader = reader; if (readers.length == 1) { // if there is a single segment, use that subReader and avoid looking up each time subReader = readers[0]; readers = null; } int[] offsets = reader.getLeafOffsets(); for (SortField sortField : sortFields) { int type = sortField.getType(); if (type == SortField.SCORE || type == SortField.DOC) continue; FieldComparator comparator = null; FieldComparator comparators[] = (readers == null) ? null : new FieldComparator[readers.length]; String fieldname = sortField.getField(); FieldType ft = fieldname == null ? null : req.getSchema().getFieldTypeNoEx(fieldname); DocList docList = rb.getResults().docList; ArrayList<Object> vals = new ArrayList<Object>(docList.size()); DocIterator it = rb.getResults().docList.iterator(); int offset = 0; int idx = 0; while (it.hasNext()) { int doc = it.nextDoc(); if (readers != null) { idx = SolrIndexReader.readerIndex(doc, offsets); subReader = readers[idx]; offset = offsets[idx]; comparator = comparators[idx]; } if (comparator == null) { comparator = sortField.getComparator(1, 0); comparator = comparator.setNextReader(subReader, offset); if (comparators != null) comparators[idx] = comparator; } doc -= offset; // adjust for what segment this is in comparator.copy(0, doc); Object val = comparator.value(0); // Sortable float, double, int, long types all just use a string // comparator. For these, we need to put the type into a readable // format. One reason for this is that XML can't represent all // string values (or even all unicode code points). // indexedToReadable() should be a no-op and should // thus be harmless anyway (for all current ways anyway) if (val instanceof String) { field.setValue((String) val); val = ft.toObject(field); } // Must do the same conversion when sorting by a // String field in Lucene, which returns the terms // data as BytesRef: if (val instanceof BytesRef) { field.setValue(((BytesRef) val).utf8ToString()); val = ft.toObject(field); } vals.add(val); } sortVals.add(fieldname, vals); } rsp.add("sort_values", sortVals); } }
public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. // // TODO: See SOLR-5595 boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { NamedList<Object[]> sortVals = new NamedList<>(); // order is important for the sort fields IndexReaderContext topReaderContext = searcher.getTopReaderContext(); List<LeafReaderContext> leaves = topReaderContext.leaves(); LeafReaderContext currentLeaf = null; if (leaves.size() == 1) { // if there is a single segment, use that subReader and avoid looking up each time currentLeaf = leaves.get(0); leaves = null; } DocList docList = rb.getResults().docList; // sort ids from lowest to highest so we can access them in order int nDocs = docList.size(); final long[] sortedIds = new long[nDocs]; final float[] scores = new float[nDocs]; // doc scores, parallel to sortedIds DocList docs = rb.getResults().docList; DocIterator it = docs.iterator(); for (int i = 0; i < nDocs; i++) { sortedIds[i] = (((long) it.nextDoc()) << 32) | i; scores[i] = docs.hasScores() ? it.score() : Float.NaN; } // sort ids and scores together new InPlaceMergeSorter() { @Override protected void swap(int i, int j) { long tmpId = sortedIds[i]; float tmpScore = scores[i]; sortedIds[i] = sortedIds[j]; scores[i] = scores[j]; sortedIds[j] = tmpId; scores[j] = tmpScore; } @Override protected int compare(int i, int j) { return Long.compare(sortedIds[i], sortedIds[j]); } }.sort(0, sortedIds.length); SortSpec sortSpec = rb.getSortSpec(); Sort sort = searcher.weightSort(sortSpec.getSort()); SortField[] sortFields = sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort(); List<SchemaField> schemaFields = sortSpec.getSchemaFields(); for (int fld = 0; fld < schemaFields.size(); fld++) { SchemaField schemaField = schemaFields.get(fld); FieldType ft = null == schemaField ? null : schemaField.getType(); SortField sortField = sortFields[fld]; SortField.Type type = sortField.getType(); // :TODO: would be simpler to always serialize every position of SortField[] if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue; FieldComparator<?> comparator = null; LeafFieldComparator leafComparator = null; Object[] vals = new Object[nDocs]; int lastIdx = -1; int idx = 0; for (int i = 0; i < sortedIds.length; ++i) { long idAndPos = sortedIds[i]; float score = scores[i]; int doc = (int) (idAndPos >>> 32); int position = (int) idAndPos; if (leaves != null) { idx = ReaderUtil.subIndex(doc, leaves); currentLeaf = leaves.get(idx); if (idx != lastIdx) { // we switched segments. invalidate comparator. comparator = null; } } if (comparator == null) { comparator = sortField.getComparator(1, 0); leafComparator = comparator.getLeafComparator(currentLeaf); } doc -= currentLeaf.docBase; // adjust for what segment this is in leafComparator.setScorer(new FakeScorer(doc, score)); leafComparator.copy(0, doc); Object val = comparator.value(0); if (null != ft) val = ft.marshalSortValue(val); vals[position] = val; } sortVals.add(sortField.getField(), vals); } rsp.add("merge_values", sortVals); } }