@Test public void testDateScripts() throws Exception { fillSingleValueAllSet(); IndexNumericFieldData indexFieldData = getForField("value"); AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); ScriptDocValues.Longs scriptValues = (ScriptDocValues.Longs) fieldData.getScriptValues(); scriptValues.setNextDocId(0); assertThat(scriptValues.getValue(), equalTo(2l)); assertThat(scriptValues.getDate().getMillis(), equalTo(2l)); assertThat(scriptValues.getDate().getZone(), equalTo(DateTimeZone.UTC)); }
private void test(List<TLongSet> values) throws Exception { StringField id = new StringField("_id", "", Field.Store.NO); for (int i = 0; i < values.size(); ++i) { Document doc = new Document(); id.setStringValue("" + i); doc.add(id); final TLongSet v = values.get(i); for (TLongIterator it = v.iterator(); it.hasNext(); ) { LongField value = new LongField("value", it.next(), Field.Store.NO); doc.add(value); } writer.addDocument(doc); } writer.forceMerge(1); final IndexNumericFieldData indexFieldData = getForField("value"); final AtomicNumericFieldData atomicFieldData = indexFieldData.load(refreshReader()); final LongValues data = atomicFieldData.getLongValues(); final DoubleValues doubleData = atomicFieldData.getDoubleValues(); final TLongSet set = new TLongHashSet(); final TDoubleSet doubleSet = new TDoubleHashSet(); for (int i = 0; i < values.size(); ++i) { final TLongSet v = values.get(i); assertThat(data.hasValue(i), equalTo(!v.isEmpty())); assertThat(doubleData.hasValue(i), equalTo(!v.isEmpty())); if (v.isEmpty()) { assertThat(data.getValue(i), equalTo(0L)); assertThat(doubleData.getValue(i), equalTo(0d)); } set.clear(); for (LongValues.Iter iter = data.getIter(i); iter.hasNext(); ) { set.add(iter.next()); } assertThat(set, equalTo(v)); final TDoubleSet doubleV = new TDoubleHashSet(); for (TLongIterator it = v.iterator(); it.hasNext(); ) { doubleV.add((double) it.next()); } doubleSet.clear(); for (DoubleValues.Iter iter = doubleData.getIter(i); iter.hasNext(); ) { doubleSet.add(iter.next()); } assertThat(doubleSet, equalTo(doubleV)); } }
@Override public FieldComparator<?> newComparator( String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName()); final float dMissingValue = (Float) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() return new FieldComparator.FloatComparator(numHits, null, null, null) { @Override protected Floats getFloatValues(AtomicReaderContext context, String field) throws IOException { final SortedNumericDoubleValues values = indexFieldData.load(context).getDoubleValues(); final NumericDoubleValues selectedValues; if (nested == null) { selectedValues = sortMode.select(values, dMissingValue); } else { final FixedBitSet rootDocs = nested.rootDocs(context); final FixedBitSet innerDocs = nested.innerDocs(context); selectedValues = sortMode.select( values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc()); } return new Floats() { @Override public float get(int docID) { return (float) selectedValues.get(docID); } }; } }; }
public TermsLongFacetExecutor( IndexNumericFieldData indexFieldData, int size, int shardSize, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context, ImmutableSet<BytesRef> excluded, SearchScript script, CacheRecycler cacheRecycler) { this.indexFieldData = indexFieldData; this.size = size; this.shardSize = shardSize; this.comparatorType = comparatorType; this.script = script; this.excluded = excluded; this.facets = cacheRecycler.longIntMap(-1); if (allTerms) { for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) { int maxDoc = readerContext.reader().maxDoc(); LongValues values = indexFieldData.load(readerContext).getLongValues(); for (int docId = 0; docId < maxDoc; docId++) { final int numValues = values.setDocument(docId); final LongIntOpenHashMap v = facets.v(); for (int i = 0; i < numValues; i++) { v.putIfAbsent(values.nextValue(), 0); } } } } }
@Override protected String getDistanceString(LeafReaderContext ctx, int docId) { StringBuilder values = new StringBuilder(mode.name()); values.append("["); final SortedNumericDoubleValues doubleValues = fieldData.load(ctx).getDoubleValues(); doubleValues.setDocument(docId); final int num = doubleValues.count(); if (num > 0) { for (int i = 0; i < num; i++) { double value = doubleValues.valueAt(i); values.append("Math.max(Math.abs("); values .append(value) .append("(=doc value) - ") .append(origin) .append("(=origin))) - ") .append(offset) .append("(=offset), 0)"); if (i != num - 1) { values.append(", "); } } } else { values.append("0.0"); } values.append("]"); return values.toString(); }
@Test public void testOptimizeTypeLong() throws Exception { Document d = new Document(); d.add(new StringField("_id", "1", Field.Store.NO)); d.add(new LongField("value", Integer.MAX_VALUE + 1l, Field.Store.NO)); writer.addDocument(d); d = new Document(); d.add(new StringField("_id", "2", Field.Store.NO)); d.add(new LongField("value", Integer.MIN_VALUE - 1l, Field.Store.NO)); writer.addDocument(d); IndexNumericFieldData indexFieldData = ifdService.getForField(new FieldMapper.Names("value"), new FieldDataType("long")); AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader()); assertThat(fieldData, instanceOf(PackedArrayAtomicFieldData.class)); assertThat(fieldData.getLongValues().getValue(0), equalTo((long) Integer.MAX_VALUE + 1l)); assertThat(fieldData.getLongValues().getValue(1), equalTo((long) Integer.MIN_VALUE - 1l)); }
@Override protected NumericDoubleValues distance(LeafReaderContext context) { final SortedNumericDoubleValues doubleValues = fieldData.load(context).getDoubleValues(); return mode.select( new MultiValueMode.UnsortedNumericDoubleValues() { @Override public int count() { return doubleValues.count(); } @Override public void setDocument(int docId) { doubleValues.setDocument(docId); } @Override public double valueAt(int index) { return Math.max(0.0d, Math.abs(doubleValues.valueAt(index) - origin) - offset); } }, 0.0); }
@Override protected void doSetNextReader(AtomicReaderContext context) throws IOException { keyValues = keyIndexFieldData.load(context).getLongValues(); valueScript.setNextReader(context); }
@Override protected String getFieldName() { return fieldData.getFieldName(); }
@Override public FacetExecutor parse(String facetName, XContentParser parser, SearchContext context) throws IOException { String field = null; int size = 10; int shardSize = -1; String[] fieldsNames = null; ImmutableSet<BytesRef> excluded = ImmutableSet.of(); String regex = null; String regexFlags = null; TermsFacet.ComparatorType comparatorType = TermsFacet.ComparatorType.COUNT; String scriptLang = null; String script = null; ScriptService.ScriptType scriptType = null; Map<String, Object> params = null; boolean allTerms = false; String executionHint = null; String currentFieldName = null; XContentParser.Token token; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { if ("params".equals(currentFieldName)) { params = parser.map(); } else { throw new ElasticsearchParseException( "unknown parameter [" + currentFieldName + "] while parsing terms facet [" + facetName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("exclude".equals(currentFieldName)) { ImmutableSet.Builder<BytesRef> builder = ImmutableSet.builder(); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { builder.add(parser.bytes()); } excluded = builder.build(); } else if ("fields".equals(currentFieldName)) { List<String> fields = Lists.newArrayListWithCapacity(4); while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { fields.add(parser.text()); } fieldsNames = fields.toArray(new String[fields.size()]); } else { throw new ElasticsearchParseException( "unknown parameter [" + currentFieldName + "] while parsing terms facet [" + facetName + "]"); } } else if (token.isValue()) { if ("field".equals(currentFieldName)) { field = parser.text(); } else if (ScriptService.SCRIPT_INLINE.match(currentFieldName)) { script = parser.text(); scriptType = ScriptService.ScriptType.INLINE; } else if (ScriptService.SCRIPT_ID.match(currentFieldName)) { script = parser.text(); scriptType = ScriptService.ScriptType.INDEXED; } else if (ScriptService.SCRIPT_FILE.match(currentFieldName)) { script = parser.text(); scriptType = ScriptService.ScriptType.FILE; } else if (ScriptService.SCRIPT_LANG.match(currentFieldName)) { scriptLang = parser.text(); } else if ("size".equals(currentFieldName)) { size = parser.intValue(); } else if ("shard_size".equals(currentFieldName) || "shardSize".equals(currentFieldName)) { shardSize = parser.intValue(); } else if ("all_terms".equals(currentFieldName) || "allTerms".equals(currentFieldName)) { allTerms = parser.booleanValue(); } else if ("regex".equals(currentFieldName)) { regex = parser.text(); } else if ("regex_flags".equals(currentFieldName) || "regexFlags".equals(currentFieldName)) { regexFlags = parser.text(); } else if ("order".equals(currentFieldName) || "comparator".equals(currentFieldName)) { comparatorType = TermsFacet.ComparatorType.fromString(parser.text()); } else if ("execution_hint".equals(currentFieldName) || "executionHint".equals(currentFieldName)) { executionHint = parser.textOrNull(); } else { throw new ElasticsearchParseException( "unknown parameter [" + currentFieldName + "] while parsing terms facet [" + facetName + "]"); } } } if (fieldsNames != null && fieldsNames.length == 1) { field = fieldsNames[0]; fieldsNames = null; } Pattern pattern = null; if (regex != null) { pattern = Regex.compile(regex, regexFlags); } SearchScript searchScript = null; if (script != null) { searchScript = context.scriptService().search(context.lookup(), scriptLang, script, scriptType, params); } // shard_size cannot be smaller than size as we need to at least fetch <size> entries from every // shards in order to return <size> if (shardSize < size) { shardSize = size; } if (fieldsNames != null) { // in case of multi files, we only collect the fields that are mapped and facet on them. ArrayList<FieldMapper> mappers = new ArrayList<>(fieldsNames.length); for (int i = 0; i < fieldsNames.length; i++) { FieldMapper mapper = context.smartNameFieldMapper(fieldsNames[i]); if (mapper != null) { mappers.add(mapper); } } if (mappers.isEmpty()) { // non of the fields is mapped return new UnmappedFieldExecutor(size, comparatorType); } return new FieldsTermsStringFacetExecutor( mappers.toArray(new FieldMapper[mappers.size()]), size, shardSize, comparatorType, allTerms, context, excluded, pattern, searchScript); } if (field == null && script != null) { return new ScriptTermsStringFieldFacetExecutor( size, shardSize, comparatorType, context, excluded, pattern, scriptLang, script, scriptType, params, context.cacheRecycler()); } if (field == null) { throw new ElasticsearchParseException( "terms facet [" + facetName + "] must have a field, fields or script parameter"); } FieldMapper fieldMapper = context.smartNameFieldMapper(field); if (fieldMapper == null) { return new UnmappedFieldExecutor(size, comparatorType); } IndexFieldData indexFieldData = context.fieldData().getForField(fieldMapper); if (indexFieldData instanceof IndexNumericFieldData) { IndexNumericFieldData indexNumericFieldData = (IndexNumericFieldData) indexFieldData; if (indexNumericFieldData.getNumericType().isFloatingPoint()) { return new TermsDoubleFacetExecutor( indexNumericFieldData, size, shardSize, comparatorType, allTerms, context, excluded, searchScript, context.cacheRecycler()); } else { return new TermsLongFacetExecutor( indexNumericFieldData, size, shardSize, comparatorType, allTerms, context, excluded, searchScript, context.cacheRecycler()); } } else { if (script != null || "map".equals(executionHint)) { return new TermsStringFacetExecutor( indexFieldData, size, shardSize, comparatorType, allTerms, context, excluded, pattern, searchScript); } else if (indexFieldData instanceof IndexOrdinalsFieldData) { return new TermsStringOrdinalsFacetExecutor( (IndexOrdinalsFieldData) indexFieldData, size, shardSize, comparatorType, allTerms, context, excluded, pattern, ordinalsCacheAbove); } else { return new TermsStringFacetExecutor( indexFieldData, size, shardSize, comparatorType, allTerms, context, excluded, pattern, searchScript); } } }
public FloatValuesComparator( IndexNumericFieldData<?> indexFieldData, float missingValue, int numHits, SortMode sortMode) { super(indexFieldData, missingValue, sortMode); assert indexFieldData.getNumericType().requiredBits() <= 32; this.values = new float[numHits]; }