public void testDuelGlobalOrdinals() throws Exception {
Random random = getRandom();
final int numDocs = scaledRandomIntBetween(10, 1000);
final int numValues = scaledRandomIntBetween(10, 500);
final String[] values = new String[numValues];
for (int i = 0; i < numValues; ++i) {
values[i] = new String(RandomStrings.randomAsciiOfLength(random, 10));
}
for (int i = 0; i < numDocs; i++) {
Document d = new Document();
final int numVals = randomInt(3);
for (int j = 0; j < numVals; ++j) {
final String value = RandomPicks.randomFrom(random, Arrays.asList(values));
d.add(new StringField("string", value, Field.Store.NO));
d.add(new SortedSetDocValuesField("bytes", new BytesRef(value)));
}
writer.addDocument(d);
if (randomInt(10) == 0) {
refreshReader();
}
}
refreshReader();
Map<FieldDataType, Type> typeMap = new HashMap<FieldDataType, DuelFieldDataTests.Type>();
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "fst")), Type.Bytes);
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "paged_bytes")),
Type.Bytes);
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "doc_values")),
Type.Bytes);
for (Map.Entry<FieldDataType, Type> entry : typeMap.entrySet()) {
ifdService.clear();
IndexOrdinalsFieldData fieldData =
getForField(entry.getKey(), entry.getValue().name().toLowerCase(Locale.ROOT));
RandomAccessOrds left = fieldData.load(readerContext).getOrdinalsValues();
fieldData.clear();
RandomAccessOrds right =
fieldData
.loadGlobal(topLevelReader)
.load(topLevelReader.leaves().get(0))
.getOrdinalsValues();
assertEquals(left.getValueCount(), right.getValueCount());
for (long ord = 0; ord < left.getValueCount(); ++ord) {
assertEquals(left.lookupOrd(ord), right.lookupOrd(ord));
}
}
}
public void testDuelGeoPoints() throws Exception {
final String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("geopoint")
.field("type", "geo_point")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.string();
final DocumentMapper mapper = mapperService.documentMapperParser().parse(mapping);
Random random = getRandom();
int atLeast = scaledRandomIntBetween(1000, 1500);
int maxValuesPerDoc = randomBoolean() ? 1 : randomIntBetween(2, 40);
// to test deduplication
double defaultLat = randomDouble() * 180 - 90;
double defaultLon = randomDouble() * 360 - 180;
for (int i = 0; i < atLeast; i++) {
final int numValues = randomInt(maxValuesPerDoc);
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().startArray("geopoint");
for (int j = 0; j < numValues; ++j) {
if (randomBoolean()) {
doc.startObject().field("lat", defaultLat).field("lon", defaultLon).endObject();
} else {
doc.startObject()
.field("lat", randomDouble() * 180 - 90)
.field("lon", randomDouble() * 360 - 180)
.endObject();
}
}
doc = doc.endArray().endObject();
final ParsedDocument d = mapper.parse("type", Integer.toString(i), doc.bytes());
writer.addDocument(d.rootDoc());
if (random.nextInt(10) == 0) {
refreshReader();
}
}
AtomicReaderContext context = refreshReader();
Map<FieldDataType, Type> typeMap = new HashMap<>();
final Distance precision = new Distance(1, randomFrom(DistanceUnit.values()));
typeMap.put(
new FieldDataType("geo_point", ImmutableSettings.builder().put("format", "array")),
Type.GeoPoint);
typeMap.put(
new FieldDataType(
"geo_point",
ImmutableSettings.builder().put("format", "compressed").put("precision", precision)),
Type.GeoPoint);
typeMap.put(
new FieldDataType("geo_point", ImmutableSettings.builder().put("format", "doc_values")),
Type.GeoPoint);
ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
while (!list.isEmpty()) {
Entry<FieldDataType, Type> left;
Entry<FieldDataType, Type> right;
if (list.size() > 1) {
left = list.remove(random.nextInt(list.size()));
right = list.remove(random.nextInt(list.size()));
} else {
right = left = list.remove(0);
}
ifdService.clear();
IndexGeoPointFieldData leftFieldData =
getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT));
ifdService.clear();
IndexGeoPointFieldData rightFieldData =
getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT));
duelFieldDataGeoPoint(random, context, leftFieldData, rightFieldData, precision);
duelFieldDataGeoPoint(random, context, rightFieldData, leftFieldData, precision);
DirectoryReader perSegment = DirectoryReader.open(writer, true);
CompositeReaderContext composite = perSegment.getContext();
List<AtomicReaderContext> leaves = composite.leaves();
for (AtomicReaderContext atomicReaderContext : leaves) {
duelFieldDataGeoPoint(
random, atomicReaderContext, leftFieldData, rightFieldData, precision);
}
perSegment.close();
}
}
@Test
public void testDuelAllTypesSingleValue() throws Exception {
final String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("bytes")
.field("type", "string")
.field("index", "not_analyzed")
.startObject("fielddata")
.field("format", LuceneTestCase.defaultCodecSupportsSortedSet() ? "doc_values" : "fst")
.endObject()
.endObject()
.startObject("byte")
.field("type", "byte")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.startObject("short")
.field("type", "short")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.startObject("integer")
.field("type", "integer")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.startObject("long")
.field("type", "long")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.startObject("float")
.field("type", "float")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.startObject("double")
.field("type", "double")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.string();
final DocumentMapper mapper = mapperService.documentMapperParser().parse(mapping);
Random random = getRandom();
int atLeast = scaledRandomIntBetween(1000, 1500);
for (int i = 0; i < atLeast; i++) {
String s = Integer.toString(randomByte());
XContentBuilder doc = XContentFactory.jsonBuilder().startObject();
for (String fieldName :
Arrays.asList("bytes", "byte", "short", "integer", "long", "float", "double")) {
doc = doc.field(fieldName, s);
}
doc = doc.endObject();
final ParsedDocument d = mapper.parse("type", Integer.toString(i), doc.bytes());
writer.addDocument(d.rootDoc());
if (random.nextInt(10) == 0) {
refreshReader();
}
}
AtomicReaderContext context = refreshReader();
Map<FieldDataType, Type> typeMap = new HashMap<>();
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "fst")), Type.Bytes);
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "paged_bytes")),
Type.Bytes);
typeMap.put(
new FieldDataType("byte", ImmutableSettings.builder().put("format", "array")),
Type.Integer);
typeMap.put(
new FieldDataType("short", ImmutableSettings.builder().put("format", "array")),
Type.Integer);
typeMap.put(
new FieldDataType("int", ImmutableSettings.builder().put("format", "array")), Type.Integer);
typeMap.put(
new FieldDataType("long", ImmutableSettings.builder().put("format", "array")), Type.Long);
typeMap.put(
new FieldDataType("double", ImmutableSettings.builder().put("format", "array")),
Type.Double);
typeMap.put(
new FieldDataType("float", ImmutableSettings.builder().put("format", "array")), Type.Float);
typeMap.put(
new FieldDataType("byte", ImmutableSettings.builder().put("format", "doc_values")),
Type.Integer);
typeMap.put(
new FieldDataType("short", ImmutableSettings.builder().put("format", "doc_values")),
Type.Integer);
typeMap.put(
new FieldDataType("int", ImmutableSettings.builder().put("format", "doc_values")),
Type.Integer);
typeMap.put(
new FieldDataType("long", ImmutableSettings.builder().put("format", "doc_values")),
Type.Long);
typeMap.put(
new FieldDataType("double", ImmutableSettings.builder().put("format", "doc_values")),
Type.Double);
typeMap.put(
new FieldDataType("float", ImmutableSettings.builder().put("format", "doc_values")),
Type.Float);
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "doc_values")),
Type.Bytes);
ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
Preprocessor pre = new ToDoublePreprocessor();
while (!list.isEmpty()) {
Entry<FieldDataType, Type> left;
Entry<FieldDataType, Type> right;
if (list.size() > 1) {
left = list.remove(random.nextInt(list.size()));
right = list.remove(random.nextInt(list.size()));
} else {
right = left = list.remove(0);
}
ifdService.clear();
IndexFieldData<?> leftFieldData =
getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT));
ifdService.clear();
IndexFieldData<?> rightFieldData =
getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT));
duelFieldDataBytes(random, context, leftFieldData, rightFieldData, pre);
duelFieldDataBytes(random, context, rightFieldData, leftFieldData, pre);
DirectoryReader perSegment = DirectoryReader.open(writer, true);
CompositeReaderContext composite = perSegment.getContext();
List<AtomicReaderContext> leaves = composite.leaves();
for (AtomicReaderContext atomicReaderContext : leaves) {
duelFieldDataBytes(random, atomicReaderContext, leftFieldData, rightFieldData, pre);
}
}
}
@Test
public void testDuelStrings() throws Exception {
Random random = getRandom();
int atLeast = scaledRandomIntBetween(1000, 1500);
for (int i = 0; i < atLeast; i++) {
Document d = new Document();
d.add(new StringField("_id", "" + i, Field.Store.NO));
if (random.nextInt(15) != 0) {
int[] numbers = getNumbers(random, Integer.MAX_VALUE);
for (int j : numbers) {
final String s = English.longToEnglish(j);
d.add(new StringField("bytes", s, Field.Store.NO));
d.add(new SortedSetDocValuesField("bytes", new BytesRef(s)));
}
if (random.nextInt(10) == 0) {
d.add(new StringField("bytes", "", Field.Store.NO));
d.add(new SortedSetDocValuesField("bytes", new BytesRef()));
}
}
writer.addDocument(d);
if (random.nextInt(10) == 0) {
refreshReader();
}
}
AtomicReaderContext context = refreshReader();
Map<FieldDataType, Type> typeMap = new HashMap<>();
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "fst")), Type.Bytes);
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "paged_bytes")),
Type.Bytes);
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "doc_values")),
Type.Bytes);
// TODO add filters
ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
Preprocessor pre = new Preprocessor();
while (!list.isEmpty()) {
Entry<FieldDataType, Type> left;
Entry<FieldDataType, Type> right;
if (list.size() > 1) {
left = list.remove(random.nextInt(list.size()));
right = list.remove(random.nextInt(list.size()));
} else {
right = left = list.remove(0);
}
ifdService.clear();
IndexFieldData<?> leftFieldData =
getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT));
ifdService.clear();
IndexFieldData<?> rightFieldData =
getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT));
duelFieldDataBytes(random, context, leftFieldData, rightFieldData, pre);
duelFieldDataBytes(random, context, rightFieldData, leftFieldData, pre);
DirectoryReader perSegment = DirectoryReader.open(writer, true);
CompositeReaderContext composite = perSegment.getContext();
List<AtomicReaderContext> leaves = composite.leaves();
for (AtomicReaderContext atomicReaderContext : leaves) {
duelFieldDataBytes(random, atomicReaderContext, leftFieldData, rightFieldData, pre);
}
perSegment.close();
}
}
@Test
public void testDuelDoubles() throws Exception {
final String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("float")
.field("type", "float")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.startObject("double")
.field("type", "double")
.startObject("fielddata")
.field("format", "doc_values")
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.string();
final DocumentMapper mapper = mapperService.documentMapperParser().parse(mapping);
Random random = getRandom();
int atLeast = scaledRandomIntBetween(1000, 1500);
final int maxNumValues = randomBoolean() ? 1 : randomIntBetween(2, 40);
float[] values = new float[maxNumValues];
for (int i = 0; i < atLeast; i++) {
int numValues = randomInt(maxNumValues);
float def = randomBoolean() ? randomFloat() : Float.NaN;
// FD loses values if they are duplicated, so we must deduplicate for this test
Set<Float> vals = new HashSet<Float>();
for (int j = 0; j < numValues; ++j) {
if (randomBoolean()) {
vals.add(def);
} else {
vals.add(randomFloat());
}
}
numValues = vals.size();
int upto = 0;
for (Float f : vals) {
values[upto++] = f.floatValue();
}
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().startArray("float");
for (int j = 0; j < numValues; ++j) {
doc = doc.value(values[j]);
}
doc = doc.endArray().startArray("double");
for (int j = 0; j < numValues; ++j) {
doc = doc.value(values[j]);
}
doc = doc.endArray().endObject();
final ParsedDocument d = mapper.parse("type", Integer.toString(i), doc.bytes());
writer.addDocument(d.rootDoc());
if (random.nextInt(10) == 0) {
refreshReader();
}
}
AtomicReaderContext context = refreshReader();
Map<FieldDataType, Type> typeMap = new HashMap<>();
typeMap.put(
new FieldDataType("double", ImmutableSettings.builder().put("format", "array")),
Type.Double);
typeMap.put(
new FieldDataType("float", ImmutableSettings.builder().put("format", "array")), Type.Float);
typeMap.put(
new FieldDataType("double", ImmutableSettings.builder().put("format", "doc_values")),
Type.Double);
typeMap.put(
new FieldDataType("float", ImmutableSettings.builder().put("format", "doc_values")),
Type.Float);
ArrayList<Entry<FieldDataType, Type>> list = new ArrayList<>(typeMap.entrySet());
while (!list.isEmpty()) {
Entry<FieldDataType, Type> left;
Entry<FieldDataType, Type> right;
if (list.size() > 1) {
left = list.remove(random.nextInt(list.size()));
right = list.remove(random.nextInt(list.size()));
} else {
right = left = list.remove(0);
}
ifdService.clear();
IndexNumericFieldData leftFieldData =
getForField(left.getKey(), left.getValue().name().toLowerCase(Locale.ROOT));
ifdService.clear();
IndexNumericFieldData rightFieldData =
getForField(right.getKey(), right.getValue().name().toLowerCase(Locale.ROOT));
duelFieldDataDouble(random, context, leftFieldData, rightFieldData);
duelFieldDataDouble(random, context, rightFieldData, leftFieldData);
DirectoryReader perSegment = DirectoryReader.open(writer, true);
CompositeReaderContext composite = perSegment.getContext();
List<AtomicReaderContext> leaves = composite.leaves();
for (AtomicReaderContext atomicReaderContext : leaves) {
duelFieldDataDouble(random, atomicReaderContext, leftFieldData, rightFieldData);
}
}
}