public boolean match(final Sort expected, final Sort acutal) { if (expected == acutal) { return true; } else if ((expected == null) || (acutal == null)) { return false; } else { final SortField[] expectedFields = expected.getSort(); final SortField[] actualFields = acutal.getSort(); if (expectedFields.length != actualFields.length) { return false; } final ArgumentMatcher<String> matcher = ArgumentMatchers.naturalMatcher(); for (int i = 0; i < actualFields.length; i++) { final SortField actualField = actualFields[i]; final SortField expectedField = expectedFields[i]; if (!matcher.match(expectedField.getField(), actualField.getField())) { return false; } if (actualField.getType() != expectedField.getType()) { return false; } if (expectedField.getReverse() != actualField.getReverse()) { return false; } } return true; } }
private static DocComparatorSource getComparatorSource(Browsable browser, SortField sf) { DocComparatorSource compSource = null; if (SortField.FIELD_DOC.equals(sf)) { compSource = new DocIdDocComparatorSource(); } else if (SortField.FIELD_SCORE.equals(sf) || sf.getType() == SortField.SCORE) { // we want to do reverse sorting regardless for relevance compSource = new ReverseDocComparatorSource(new RelevanceDocComparatorSource()); } else if (sf instanceof BoboCustomSortField) { BoboCustomSortField custField = (BoboCustomSortField) sf; DocComparatorSource src = custField.getCustomComparatorSource(); assert src != null; compSource = src; } else { Set<String> facetNames = browser.getFacetNames(); String sortName = sf.getField(); if (facetNames.contains(sortName)) { FacetHandler<?> handler = browser.getFacetHandler(sortName); assert handler != null; compSource = handler.getDocComparatorSource(); } else { // default lucene field logger.info("doing default lucene sort for: " + sf); compSource = getNonFacetComparatorSource(sf); } } boolean reverse = sf.getReverse(); if (reverse) { compSource = new ReverseDocComparatorSource(compSource); } compSource.setReverse(reverse); return compSource; }
@Override public void parse(XContentParser parser, SearchContext context) throws Exception { XContentParser.Token token = parser.currentToken(); List<SortField> sortFields = Lists.newArrayListWithCapacity(2); if (token == XContentParser.Token.START_ARRAY) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { addCompoundSortField(parser, context, sortFields); } else if (token == XContentParser.Token.VALUE_STRING) { addSortField(context, sortFields, parser.text(), false, false, null, null, null, null); } } } else { addCompoundSortField(parser, context, sortFields); } if (!sortFields.isEmpty()) { // optimize if we just sort on score non reversed, we don't really need sorting boolean sort; if (sortFields.size() > 1) { sort = true; } else { SortField sortField = sortFields.get(0); if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) { sort = false; } else { sort = true; } } if (sort) { context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); } } }
private SortSpec modifySortSpec( SortSpec current, boolean force, ElevationComparatorSource comparator) { boolean modify = false; SortField[] currentSorts = current.getSort().getSort(); List<SchemaField> currentFields = current.getSchemaFields(); ArrayList<SortField> sorts = new ArrayList<SortField>(currentSorts.length + 1); List<SchemaField> fields = new ArrayList<SchemaField>(currentFields.size() + 1); // Perhaps force it to always sort by score if (force && currentSorts[0].getType() != SortField.Type.SCORE) { sorts.add(new SortField("_elevate_", comparator, true)); fields.add(null); modify = true; } for (int i = 0; i < currentSorts.length; i++) { SortField sf = currentSorts[i]; if (sf.getType() == SortField.Type.SCORE) { sorts.add(new SortField("_elevate_", comparator, !sf.getReverse())); fields.add(null); modify = true; } sorts.add(sf); fields.add(currentFields.get(i)); } if (modify) { SortSpec newSpec = new SortSpec(new Sort(sorts.toArray(new SortField[sorts.size()])), fields); newSpec.setOffset(current.getOffset()); newSpec.setCount(current.getCount()); return newSpec; } return null; }
public static boolean includesScore(Sort sort) { if (sort == null) return true; for (SortField sf : sort.getSort()) { if (sf.getType() == SortField.Type.SCORE) return true; } return false; }
@Override protected void sortFieldAssertions( GeoDistanceSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException { assertEquals(SortField.Type.CUSTOM, sortField.getType()); assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse()); assertEquals(builder.fieldName(), sortField.getField()); }
public static SortCollector buildSortCollector( Browsable browser, Query q, SortField[] sort, int offset, int count, boolean forceScoring, boolean fetchStoredFields, Set<String> termVectorsToFetch, String[] groupBy, int maxPerGroup, boolean collectDocIdCache) { boolean doScoring = forceScoring; if (sort == null || sort.length == 0) { if (q != null && !(q instanceof MatchAllDocsQuery)) { sort = new SortField[] {SortField.FIELD_SCORE}; } } if (sort == null || sort.length == 0) { sort = new SortField[] {SortField.FIELD_DOC}; } Set<String> facetNames = browser.getFacetNames(); for (SortField sf : sort) { if (sf.getType() == SortField.SCORE) { doScoring = true; break; } } DocComparatorSource compSource; if (sort.length == 1) { SortField sf = convert(browser, sort[0]); compSource = getComparatorSource(browser, sf); } else { DocComparatorSource[] compSources = new DocComparatorSource[sort.length]; for (int i = 0; i < sort.length; ++i) { compSources[i] = getComparatorSource(browser, convert(browser, sort[i])); } compSource = new MultiDocIdComparatorSource(compSources); } return new SortCollectorImpl( compSource, sort, browser, offset, count, doScoring, fetchStoredFields, termVectorsToFetch, groupBy, maxPerGroup, collectDocIdCache); }
@Override Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException { assert sortField.getType().equals(SortField.Type.STRING); assert finalSortedValues == null && finalOrdMap == null && finalOrds == null; int valueCount = hash.size(); finalSortedValues = hash.sort(); finalOrds = pending.build(); finalOrdMap = new int[valueCount]; for (int ord = 0; ord < valueCount; ord++) { finalOrdMap[finalSortedValues[ord]] = ord; } final SortedDocValues docValues = new BufferedSortedDocValues( hash, valueCount, finalOrds, finalSortedValues, finalOrdMap, docsWithField.iterator()); return Sorter.getDocComparator(maxDoc, sortField, () -> docValues, () -> null); }
@Override protected void sortFieldAssertions( FieldSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException { SortField.Type expectedType; if (builder.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) { expectedType = SortField.Type.DOC; } else { expectedType = SortField.Type.CUSTOM; } assertEquals(expectedType, sortField.getType()); assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse()); if (expectedType == SortField.Type.CUSTOM) { assertEquals(builder.getFieldName(), sortField.getField()); } assertEquals(DocValueFormat.RAW, format); }
private static DocComparatorSource getNonFacetComparatorSource(SortField sf) { String fieldname = sf.getField(); Locale locale = sf.getLocale(); if (locale != null) { return new DocComparatorSource.StringLocaleComparatorSource(fieldname, locale); } int type = sf.getType(); switch (type) { case SortField.INT: return new DocComparatorSource.IntDocComparatorSource(fieldname); case SortField.FLOAT: return new DocComparatorSource.FloatDocComparatorSource(fieldname); case SortField.LONG: return new DocComparatorSource.LongDocComparatorSource(fieldname); case SortField.DOUBLE: return new DocComparatorSource.LongDocComparatorSource(fieldname); case SortField.BYTE: return new DocComparatorSource.ByteDocComparatorSource(fieldname); case SortField.SHORT: return new DocComparatorSource.ShortDocComparatorSource(fieldname); case SortField.CUSTOM: throw new IllegalArgumentException("lucene custom sort no longer supported: " + fieldname); case SortField.STRING: return new DocComparatorSource.StringOrdComparatorSource(fieldname); case SortField.STRING_VAL: return new DocComparatorSource.StringValComparatorSource(fieldname); default: throw new IllegalStateException("Illegal sort type: " + type + ", for field: " + fieldname); } }
private NamedList unmarshalSortValues( SortSpec sortSpec, NamedList sortFieldValues, IndexSchema schema) { NamedList unmarshalledSortValsPerField = new NamedList(); if (0 == sortFieldValues.size()) return unmarshalledSortValsPerField; List<SchemaField> schemaFields = sortSpec.getSchemaFields(); SortField[] sortFields = sortSpec.getSort().getSort(); int marshalledFieldNum = 0; for (int sortFieldNum = 0; sortFieldNum < sortFields.length; sortFieldNum++) { final SortField sortField = sortFields[sortFieldNum]; final SortField.Type type = sortField.getType(); // :TODO: would be simpler to always serialize every position of SortField[] if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue; final String sortFieldName = sortField.getField(); final String valueFieldName = sortFieldValues.getName(marshalledFieldNum); assert sortFieldName.equals(valueFieldName) : "sortFieldValues name key does not match expected SortField.getField"; List sortVals = (List) sortFieldValues.getVal(marshalledFieldNum); final SchemaField schemaField = schemaFields.get(sortFieldNum); if (null == schemaField) { unmarshalledSortValsPerField.add(sortField.getField(), sortVals); } else { FieldType fieldType = schemaField.getType(); List unmarshalledSortVals = new ArrayList(); for (Object sortVal : sortVals) { unmarshalledSortVals.add(fieldType.unmarshalSortValue(sortVal)); } unmarshalledSortValsPerField.add(sortField.getField(), unmarshalledSortVals); } marshalledFieldNum++; } return unmarshalledSortValsPerField; }
protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { Sort sort = rb.getSortSpec().getSort(); SortField[] sortFields = sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort(); NamedList sortVals = new NamedList(); // order is important for the sort fields Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field SolrIndexReader reader = searcher.getReader(); SolrIndexReader[] readers = reader.getLeafReaders(); SolrIndexReader subReader = reader; if (readers.length == 1) { // if there is a single segment, use that subReader and avoid looking up each time subReader = readers[0]; readers = null; } int[] offsets = reader.getLeafOffsets(); for (SortField sortField : sortFields) { int type = sortField.getType(); if (type == SortField.SCORE || type == SortField.DOC) continue; FieldComparator comparator = null; FieldComparator comparators[] = (readers == null) ? null : new FieldComparator[readers.length]; String fieldname = sortField.getField(); FieldType ft = fieldname == null ? null : req.getSchema().getFieldTypeNoEx(fieldname); DocList docList = rb.getResults().docList; ArrayList<Object> vals = new ArrayList<Object>(docList.size()); DocIterator it = rb.getResults().docList.iterator(); int offset = 0; int idx = 0; while (it.hasNext()) { int doc = it.nextDoc(); if (readers != null) { idx = SolrIndexReader.readerIndex(doc, offsets); subReader = readers[idx]; offset = offsets[idx]; comparator = comparators[idx]; } if (comparator == null) { comparator = sortField.getComparator(1, 0); comparator = comparator.setNextReader(subReader, offset); if (comparators != null) comparators[idx] = comparator; } doc -= offset; // adjust for what segment this is in comparator.copy(0, doc); Object val = comparator.value(0); // Sortable float, double, int, long types all just use a string // comparator. For these, we need to put the type into a readable // format. One reason for this is that XML can't represent all // string values (or even all unicode code points). // indexedToReadable() should be a no-op and should // thus be harmless anyway (for all current ways anyway) if (val instanceof String) { field.setValue((String) val); val = ft.toObject(field); } // Must do the same conversion when sorting by a // String field in Lucene, which returns the terms // data as BytesRef: if (val instanceof BytesRef) { field.setValue(((BytesRef) val).utf8ToString()); val = ft.toObject(field); } vals.add(val); } sortVals.add(fieldname, vals); } rsp.add("sort_values", sortVals); } }
public void handleMergeFields(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't // currently have an option to return sort field values. Because of this, we // take the documents given and re-derive the sort values. // // TODO: See SOLR-5595 boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false); if (fsv) { NamedList<Object[]> sortVals = new NamedList<>(); // order is important for the sort fields IndexReaderContext topReaderContext = searcher.getTopReaderContext(); List<LeafReaderContext> leaves = topReaderContext.leaves(); LeafReaderContext currentLeaf = null; if (leaves.size() == 1) { // if there is a single segment, use that subReader and avoid looking up each time currentLeaf = leaves.get(0); leaves = null; } DocList docList = rb.getResults().docList; // sort ids from lowest to highest so we can access them in order int nDocs = docList.size(); final long[] sortedIds = new long[nDocs]; final float[] scores = new float[nDocs]; // doc scores, parallel to sortedIds DocList docs = rb.getResults().docList; DocIterator it = docs.iterator(); for (int i = 0; i < nDocs; i++) { sortedIds[i] = (((long) it.nextDoc()) << 32) | i; scores[i] = docs.hasScores() ? it.score() : Float.NaN; } // sort ids and scores together new InPlaceMergeSorter() { @Override protected void swap(int i, int j) { long tmpId = sortedIds[i]; float tmpScore = scores[i]; sortedIds[i] = sortedIds[j]; scores[i] = scores[j]; sortedIds[j] = tmpId; scores[j] = tmpScore; } @Override protected int compare(int i, int j) { return Long.compare(sortedIds[i], sortedIds[j]); } }.sort(0, sortedIds.length); SortSpec sortSpec = rb.getSortSpec(); Sort sort = searcher.weightSort(sortSpec.getSort()); SortField[] sortFields = sort == null ? new SortField[] {SortField.FIELD_SCORE} : sort.getSort(); List<SchemaField> schemaFields = sortSpec.getSchemaFields(); for (int fld = 0; fld < schemaFields.size(); fld++) { SchemaField schemaField = schemaFields.get(fld); FieldType ft = null == schemaField ? null : schemaField.getType(); SortField sortField = sortFields[fld]; SortField.Type type = sortField.getType(); // :TODO: would be simpler to always serialize every position of SortField[] if (type == SortField.Type.SCORE || type == SortField.Type.DOC) continue; FieldComparator<?> comparator = null; LeafFieldComparator leafComparator = null; Object[] vals = new Object[nDocs]; int lastIdx = -1; int idx = 0; for (int i = 0; i < sortedIds.length; ++i) { long idAndPos = sortedIds[i]; float score = scores[i]; int doc = (int) (idAndPos >>> 32); int position = (int) idAndPos; if (leaves != null) { idx = ReaderUtil.subIndex(doc, leaves); currentLeaf = leaves.get(idx); if (idx != lastIdx) { // we switched segments. invalidate comparator. comparator = null; } } if (comparator == null) { comparator = sortField.getComparator(1, 0); leafComparator = comparator.getLeafComparator(currentLeaf); } doc -= currentLeaf.docBase; // adjust for what segment this is in leafComparator.setScorer(new FakeScorer(doc, score)); leafComparator.copy(0, doc); Object val = comparator.value(0); if (null != ft) val = ft.marshalSortValue(val); vals[position] = val; } sortVals.add(sortField.getField(), vals); } rsp.add("merge_values", sortVals); } }
public static void writeTopDocs(StreamOutput out, TopDocs topDocs, int from) throws IOException { if (topDocs.scoreDocs.length - from < 0) { out.writeBoolean(false); return; } out.writeBoolean(true); if (topDocs instanceof TopFieldDocs) { out.writeBoolean(true); TopFieldDocs topFieldDocs = (TopFieldDocs) topDocs; out.writeVInt(topDocs.totalHits); out.writeFloat(topDocs.getMaxScore()); out.writeVInt(topFieldDocs.fields.length); for (SortField sortField : topFieldDocs.fields) { if (sortField.getField() == null) { out.writeBoolean(false); } else { out.writeBoolean(true); out.writeString(sortField.getField()); } if (sortField.getComparatorSource() != null) { writeSortType( out, ((IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource()) .reducedType()); } else { writeSortType(out, sortField.getType()); } out.writeBoolean(sortField.getReverse()); } out.writeVInt(topDocs.scoreDocs.length - from); int index = 0; for (ScoreDoc doc : topFieldDocs.scoreDocs) { if (index++ < from) { continue; } FieldDoc fieldDoc = (FieldDoc) doc; out.writeVInt(fieldDoc.fields.length); for (Object field : fieldDoc.fields) { if (field == null) { out.writeByte((byte) 0); } else { Class type = field.getClass(); if (type == String.class) { out.writeByte((byte) 1); out.writeString((String) field); } else if (type == Integer.class) { out.writeByte((byte) 2); out.writeInt((Integer) field); } else if (type == Long.class) { out.writeByte((byte) 3); out.writeLong((Long) field); } else if (type == Float.class) { out.writeByte((byte) 4); out.writeFloat((Float) field); } else if (type == Double.class) { out.writeByte((byte) 5); out.writeDouble((Double) field); } else if (type == Byte.class) { out.writeByte((byte) 6); out.writeByte((Byte) field); } else if (type == Short.class) { out.writeByte((byte) 7); out.writeShort((Short) field); } else if (type == Boolean.class) { out.writeByte((byte) 8); out.writeBoolean((Boolean) field); } else if (type == BytesRef.class) { out.writeByte((byte) 9); out.writeBytesRef((BytesRef) field); } else { throw new IOException("Can't handle sort field value of type [" + type + "]"); } } } out.writeVInt(doc.doc); out.writeFloat(doc.score); } } else { out.writeBoolean(false); out.writeVInt(topDocs.totalHits); out.writeFloat(topDocs.getMaxScore()); out.writeVInt(topDocs.scoreDocs.length - from); int index = 0; for (ScoreDoc doc : topDocs.scoreDocs) { if (index++ < from) { continue; } out.writeVInt(doc.doc); out.writeFloat(doc.score); } } }