@Test
public void testDateScripts() throws Exception {
fillSingleValueAllSet();
IndexNumericFieldData indexFieldData = getForField("value");
AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader());
ScriptDocValues.Longs scriptValues = (ScriptDocValues.Longs) fieldData.getScriptValues();
scriptValues.setNextDocId(0);
assertThat(scriptValues.getValue(), equalTo(2l));
assertThat(scriptValues.getDate().getMillis(), equalTo(2l));
assertThat(scriptValues.getDate().getZone(), equalTo(DateTimeZone.UTC));
}
private void test(List<TLongSet> values) throws Exception {
StringField id = new StringField("_id", "", Field.Store.NO);
for (int i = 0; i < values.size(); ++i) {
Document doc = new Document();
id.setStringValue("" + i);
doc.add(id);
final TLongSet v = values.get(i);
for (TLongIterator it = v.iterator(); it.hasNext(); ) {
LongField value = new LongField("value", it.next(), Field.Store.NO);
doc.add(value);
}
writer.addDocument(doc);
}
writer.forceMerge(1);
final IndexNumericFieldData indexFieldData = getForField("value");
final AtomicNumericFieldData atomicFieldData = indexFieldData.load(refreshReader());
final LongValues data = atomicFieldData.getLongValues();
final DoubleValues doubleData = atomicFieldData.getDoubleValues();
final TLongSet set = new TLongHashSet();
final TDoubleSet doubleSet = new TDoubleHashSet();
for (int i = 0; i < values.size(); ++i) {
final TLongSet v = values.get(i);
assertThat(data.hasValue(i), equalTo(!v.isEmpty()));
assertThat(doubleData.hasValue(i), equalTo(!v.isEmpty()));
if (v.isEmpty()) {
assertThat(data.getValue(i), equalTo(0L));
assertThat(doubleData.getValue(i), equalTo(0d));
}
set.clear();
for (LongValues.Iter iter = data.getIter(i); iter.hasNext(); ) {
set.add(iter.next());
}
assertThat(set, equalTo(v));
final TDoubleSet doubleV = new TDoubleHashSet();
for (TLongIterator it = v.iterator(); it.hasNext(); ) {
doubleV.add((double) it.next());
}
doubleSet.clear();
for (DoubleValues.Iter iter = doubleData.getIter(i); iter.hasNext(); ) {
doubleSet.add(iter.next());
}
assertThat(doubleSet, equalTo(doubleV));
}
}
@Override
public FieldComparator<?> newComparator(
String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
assert indexFieldData == null || fieldname.equals(indexFieldData.getFieldNames().indexName());
final float dMissingValue = (Float) missingObject(missingValue, reversed);
// NOTE: it's important to pass null as a missing value in the constructor so that
// the comparator doesn't check docsWithField since we replace missing values in select()
return new FieldComparator.FloatComparator(numHits, null, null, null) {
@Override
protected Floats getFloatValues(AtomicReaderContext context, String field)
throws IOException {
final SortedNumericDoubleValues values = indexFieldData.load(context).getDoubleValues();
final NumericDoubleValues selectedValues;
if (nested == null) {
selectedValues = sortMode.select(values, dMissingValue);
} else {
final FixedBitSet rootDocs = nested.rootDocs(context);
final FixedBitSet innerDocs = nested.innerDocs(context);
selectedValues =
sortMode.select(
values, dMissingValue, rootDocs, innerDocs, context.reader().maxDoc());
}
return new Floats() {
@Override
public float get(int docID) {
return (float) selectedValues.get(docID);
}
};
}
};
}
public TermsLongFacetExecutor(
IndexNumericFieldData indexFieldData,
int size,
int shardSize,
TermsFacet.ComparatorType comparatorType,
boolean allTerms,
SearchContext context,
ImmutableSet<BytesRef> excluded,
SearchScript script,
CacheRecycler cacheRecycler) {
this.indexFieldData = indexFieldData;
this.size = size;
this.shardSize = shardSize;
this.comparatorType = comparatorType;
this.script = script;
this.excluded = excluded;
this.facets = cacheRecycler.longIntMap(-1);
if (allTerms) {
for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
int maxDoc = readerContext.reader().maxDoc();
LongValues values = indexFieldData.load(readerContext).getLongValues();
for (int docId = 0; docId < maxDoc; docId++) {
final int numValues = values.setDocument(docId);
final LongIntOpenHashMap v = facets.v();
for (int i = 0; i < numValues; i++) {
v.putIfAbsent(values.nextValue(), 0);
}
}
}
}
}
@Override
protected String getDistanceString(LeafReaderContext ctx, int docId) {
StringBuilder values = new StringBuilder(mode.name());
values.append("[");
final SortedNumericDoubleValues doubleValues = fieldData.load(ctx).getDoubleValues();
doubleValues.setDocument(docId);
final int num = doubleValues.count();
if (num > 0) {
for (int i = 0; i < num; i++) {
double value = doubleValues.valueAt(i);
values.append("Math.max(Math.abs(");
values
.append(value)
.append("(=doc value) - ")
.append(origin)
.append("(=origin))) - ")
.append(offset)
.append("(=offset), 0)");
if (i != num - 1) {
values.append(", ");
}
}
} else {
values.append("0.0");
}
values.append("]");
return values.toString();
}
@Test
public void testOptimizeTypeLong() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new LongField("value", Integer.MAX_VALUE + 1l, Field.Store.NO));
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
d.add(new LongField("value", Integer.MIN_VALUE - 1l, Field.Store.NO));
writer.addDocument(d);
IndexNumericFieldData indexFieldData =
ifdService.getForField(new FieldMapper.Names("value"), new FieldDataType("long"));
AtomicNumericFieldData fieldData = indexFieldData.load(refreshReader());
assertThat(fieldData, instanceOf(PackedArrayAtomicFieldData.class));
assertThat(fieldData.getLongValues().getValue(0), equalTo((long) Integer.MAX_VALUE + 1l));
assertThat(fieldData.getLongValues().getValue(1), equalTo((long) Integer.MIN_VALUE - 1l));
}
@Override
protected NumericDoubleValues distance(LeafReaderContext context) {
final SortedNumericDoubleValues doubleValues = fieldData.load(context).getDoubleValues();
return mode.select(
new MultiValueMode.UnsortedNumericDoubleValues() {
@Override
public int count() {
return doubleValues.count();
}
@Override
public void setDocument(int docId) {
doubleValues.setDocument(docId);
}
@Override
public double valueAt(int index) {
return Math.max(0.0d, Math.abs(doubleValues.valueAt(index) - origin) - offset);
}
},
0.0);
}
@Override
protected void doSetNextReader(AtomicReaderContext context) throws IOException {
keyValues = keyIndexFieldData.load(context).getLongValues();
valueScript.setNextReader(context);
}
@Override
protected String getFieldName() {
return fieldData.getFieldName();
}
@Override
public FacetExecutor parse(String facetName, XContentParser parser, SearchContext context)
throws IOException {
String field = null;
int size = 10;
int shardSize = -1;
String[] fieldsNames = null;
ImmutableSet<BytesRef> excluded = ImmutableSet.of();
String regex = null;
String regexFlags = null;
TermsFacet.ComparatorType comparatorType = TermsFacet.ComparatorType.COUNT;
String scriptLang = null;
String script = null;
ScriptService.ScriptType scriptType = null;
Map<String, Object> params = null;
boolean allTerms = false;
String executionHint = null;
String currentFieldName = null;
XContentParser.Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("params".equals(currentFieldName)) {
params = parser.map();
} else {
throw new ElasticsearchParseException(
"unknown parameter ["
+ currentFieldName
+ "] while parsing terms facet ["
+ facetName
+ "]");
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("exclude".equals(currentFieldName)) {
ImmutableSet.Builder<BytesRef> builder = ImmutableSet.builder();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
builder.add(parser.bytes());
}
excluded = builder.build();
} else if ("fields".equals(currentFieldName)) {
List<String> fields = Lists.newArrayListWithCapacity(4);
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
fields.add(parser.text());
}
fieldsNames = fields.toArray(new String[fields.size()]);
} else {
throw new ElasticsearchParseException(
"unknown parameter ["
+ currentFieldName
+ "] while parsing terms facet ["
+ facetName
+ "]");
}
} else if (token.isValue()) {
if ("field".equals(currentFieldName)) {
field = parser.text();
} else if (ScriptService.SCRIPT_INLINE.match(currentFieldName)) {
script = parser.text();
scriptType = ScriptService.ScriptType.INLINE;
} else if (ScriptService.SCRIPT_ID.match(currentFieldName)) {
script = parser.text();
scriptType = ScriptService.ScriptType.INDEXED;
} else if (ScriptService.SCRIPT_FILE.match(currentFieldName)) {
script = parser.text();
scriptType = ScriptService.ScriptType.FILE;
} else if (ScriptService.SCRIPT_LANG.match(currentFieldName)) {
scriptLang = parser.text();
} else if ("size".equals(currentFieldName)) {
size = parser.intValue();
} else if ("shard_size".equals(currentFieldName) || "shardSize".equals(currentFieldName)) {
shardSize = parser.intValue();
} else if ("all_terms".equals(currentFieldName) || "allTerms".equals(currentFieldName)) {
allTerms = parser.booleanValue();
} else if ("regex".equals(currentFieldName)) {
regex = parser.text();
} else if ("regex_flags".equals(currentFieldName)
|| "regexFlags".equals(currentFieldName)) {
regexFlags = parser.text();
} else if ("order".equals(currentFieldName) || "comparator".equals(currentFieldName)) {
comparatorType = TermsFacet.ComparatorType.fromString(parser.text());
} else if ("execution_hint".equals(currentFieldName)
|| "executionHint".equals(currentFieldName)) {
executionHint = parser.textOrNull();
} else {
throw new ElasticsearchParseException(
"unknown parameter ["
+ currentFieldName
+ "] while parsing terms facet ["
+ facetName
+ "]");
}
}
}
if (fieldsNames != null && fieldsNames.length == 1) {
field = fieldsNames[0];
fieldsNames = null;
}
Pattern pattern = null;
if (regex != null) {
pattern = Regex.compile(regex, regexFlags);
}
SearchScript searchScript = null;
if (script != null) {
searchScript =
context.scriptService().search(context.lookup(), scriptLang, script, scriptType, params);
}
// shard_size cannot be smaller than size as we need to at least fetch <size> entries from every
// shards in order to return <size>
if (shardSize < size) {
shardSize = size;
}
if (fieldsNames != null) {
// in case of multi files, we only collect the fields that are mapped and facet on them.
ArrayList<FieldMapper> mappers = new ArrayList<>(fieldsNames.length);
for (int i = 0; i < fieldsNames.length; i++) {
FieldMapper mapper = context.smartNameFieldMapper(fieldsNames[i]);
if (mapper != null) {
mappers.add(mapper);
}
}
if (mappers.isEmpty()) {
// non of the fields is mapped
return new UnmappedFieldExecutor(size, comparatorType);
}
return new FieldsTermsStringFacetExecutor(
mappers.toArray(new FieldMapper[mappers.size()]),
size,
shardSize,
comparatorType,
allTerms,
context,
excluded,
pattern,
searchScript);
}
if (field == null && script != null) {
return new ScriptTermsStringFieldFacetExecutor(
size,
shardSize,
comparatorType,
context,
excluded,
pattern,
scriptLang,
script,
scriptType,
params,
context.cacheRecycler());
}
if (field == null) {
throw new ElasticsearchParseException(
"terms facet [" + facetName + "] must have a field, fields or script parameter");
}
FieldMapper fieldMapper = context.smartNameFieldMapper(field);
if (fieldMapper == null) {
return new UnmappedFieldExecutor(size, comparatorType);
}
IndexFieldData indexFieldData = context.fieldData().getForField(fieldMapper);
if (indexFieldData instanceof IndexNumericFieldData) {
IndexNumericFieldData indexNumericFieldData = (IndexNumericFieldData) indexFieldData;
if (indexNumericFieldData.getNumericType().isFloatingPoint()) {
return new TermsDoubleFacetExecutor(
indexNumericFieldData,
size,
shardSize,
comparatorType,
allTerms,
context,
excluded,
searchScript,
context.cacheRecycler());
} else {
return new TermsLongFacetExecutor(
indexNumericFieldData,
size,
shardSize,
comparatorType,
allTerms,
context,
excluded,
searchScript,
context.cacheRecycler());
}
} else {
if (script != null || "map".equals(executionHint)) {
return new TermsStringFacetExecutor(
indexFieldData,
size,
shardSize,
comparatorType,
allTerms,
context,
excluded,
pattern,
searchScript);
} else if (indexFieldData instanceof IndexOrdinalsFieldData) {
return new TermsStringOrdinalsFacetExecutor(
(IndexOrdinalsFieldData) indexFieldData,
size,
shardSize,
comparatorType,
allTerms,
context,
excluded,
pattern,
ordinalsCacheAbove);
} else {
return new TermsStringFacetExecutor(
indexFieldData,
size,
shardSize,
comparatorType,
allTerms,
context,
excluded,
pattern,
searchScript);
}
}
}
public FloatValuesComparator(
IndexNumericFieldData<?> indexFieldData, float missingValue, int numHits, SortMode sortMode) {
super(indexFieldData, missingValue, sortMode);
assert indexFieldData.getNumericType().requiredBits() <= 32;
this.values = new float[numHits];
}