/** Randomly adds fields, objects, or arrays to the provided builder. The maximum depth is 5. */
private static void addFields(Random random, XContentBuilder builder, int currentDepth)
throws IOException {
int numFields = RandomNumbers.randomIntBetween(random, 1, 5);
for (int i = 0; i < numFields; i++) {
if (currentDepth < 5 && random.nextBoolean()) {
if (random.nextBoolean()) {
builder.startObject(RandomStrings.randomAsciiOfLengthBetween(random, 3, 10));
addFields(random, builder, currentDepth + 1);
builder.endObject();
} else {
builder.startArray(RandomStrings.randomAsciiOfLengthBetween(random, 3, 10));
int numElements = RandomNumbers.randomIntBetween(random, 1, 5);
boolean object = random.nextBoolean();
int dataType = -1;
if (object == false) {
dataType = randomDataType(random);
}
for (int j = 0; j < numElements; j++) {
if (object) {
builder.startObject();
addFields(random, builder, 5);
builder.endObject();
} else {
builder.value(randomFieldValue(random, dataType));
}
}
builder.endArray();
}
} else {
builder.field(
RandomStrings.randomAsciiOfLengthBetween(random, 3, 10),
randomFieldValue(random, randomDataType(random)));
}
}
}
private static Object randomFieldValue(Random random, int dataType) {
switch (dataType) {
case 0:
return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
case 1:
return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
case 2:
return random.nextLong();
case 3:
return random.nextDouble();
default:
throw new UnsupportedOperationException();
}
}
public void testIsFieldWithinQuery() throws IOException {
KeywordFieldType ft = new KeywordFieldType();
// current impl ignores args and shourd always return INTERSECTS
assertEquals(
Relation.INTERSECTS,
ft.isFieldWithinQuery(
null,
RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
randomBoolean(),
randomBoolean(),
null,
null));
}
public void testDuelGlobalOrdinals() throws Exception {
Random random = getRandom();
final int numDocs = scaledRandomIntBetween(10, 1000);
final int numValues = scaledRandomIntBetween(10, 500);
final String[] values = new String[numValues];
for (int i = 0; i < numValues; ++i) {
values[i] = new String(RandomStrings.randomAsciiOfLength(random, 10));
}
for (int i = 0; i < numDocs; i++) {
Document d = new Document();
final int numVals = randomInt(3);
for (int j = 0; j < numVals; ++j) {
final String value = RandomPicks.randomFrom(random, Arrays.asList(values));
d.add(new StringField("string", value, Field.Store.NO));
d.add(new SortedSetDocValuesField("bytes", new BytesRef(value)));
}
writer.addDocument(d);
if (randomInt(10) == 0) {
refreshReader();
}
}
refreshReader();
Map<FieldDataType, Type> typeMap = new HashMap<FieldDataType, DuelFieldDataTests.Type>();
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "fst")), Type.Bytes);
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "paged_bytes")),
Type.Bytes);
typeMap.put(
new FieldDataType("string", ImmutableSettings.builder().put("format", "doc_values")),
Type.Bytes);
for (Map.Entry<FieldDataType, Type> entry : typeMap.entrySet()) {
ifdService.clear();
IndexOrdinalsFieldData fieldData =
getForField(entry.getKey(), entry.getValue().name().toLowerCase(Locale.ROOT));
RandomAccessOrds left = fieldData.load(readerContext).getOrdinalsValues();
fieldData.clear();
RandomAccessOrds right =
fieldData
.loadGlobal(topLevelReader)
.load(topLevelReader.leaves().get(0))
.getOrdinalsValues();
assertEquals(left.getValueCount(), right.getValueCount());
for (long ord = 0; ord < left.getValueCount(); ++ord) {
assertEquals(left.lookupOrd(ord), right.lookupOrd(ord));
}
}
}
public void testBytes() {
final String[][] values = new String[randomInt(10)][];
for (int i = 0; i < values.length; ++i) {
String[] strings = new String[randomInt(8)];
for (int j = 0; j < strings.length; ++j) {
strings[j] = RandomStrings.randomAsciiOfLength(random(), 5);
}
Arrays.sort(strings);
values[i] = strings;
}
FakeSearchScript script = new FakeSearchScript(values);
ScriptBytesValues scriptValues = new ScriptBytesValues(script);
for (int i = 0; i < values.length; ++i) {
scriptValues.setDocument(i);
assertEquals(values[i].length, scriptValues.count());
for (int j = 0; j < values[i].length; ++j) {
assertEquals(new BytesRef(values[i][j]), scriptValues.valueAt(j));
}
}
}
/**
* Returns a tuple containing random stored field values and their corresponding expected values
* once printed out via {@link
* org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)}
* and parsed back via {@link org.elasticsearch.common.xcontent.XContentParser#objectText()}.
* Generates values based on what can get printed out. Stored fields values are retrieved from
* lucene and converted via {@link
* org.elasticsearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings,
* numbers or booleans.
*
* @param random Random generator
* @param xContentType the content type, used to determine what the expected values are for float
* numbers.
*/
public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(
Random random, XContentType xContentType) {
int numValues = RandomNumbers.randomIntBetween(random, 1, 5);
List<Object> originalValues = new ArrayList<>();
List<Object> expectedParsedValues = new ArrayList<>();
int dataType = RandomNumbers.randomIntBetween(random, 0, 8);
for (int i = 0; i < numValues; i++) {
switch (dataType) {
case 0:
long randomLong = random.nextLong();
originalValues.add(randomLong);
expectedParsedValues.add(randomLong);
break;
case 1:
int randomInt = random.nextInt();
originalValues.add(randomInt);
expectedParsedValues.add(randomInt);
break;
case 2:
Short randomShort = (short) random.nextInt();
originalValues.add(randomShort);
expectedParsedValues.add(randomShort.intValue());
break;
case 3:
Byte randomByte = (byte) random.nextInt();
originalValues.add(randomByte);
expectedParsedValues.add(randomByte.intValue());
break;
case 4:
double randomDouble = random.nextDouble();
originalValues.add(randomDouble);
expectedParsedValues.add(randomDouble);
break;
case 5:
Float randomFloat = random.nextFloat();
originalValues.add(randomFloat);
if (xContentType == XContentType.CBOR) {
// with CBOR we get back a float
expectedParsedValues.add(randomFloat);
} else if (xContentType == XContentType.SMILE) {
// with SMILE we get back a double
expectedParsedValues.add(randomFloat.doubleValue());
} else {
// with JSON AND YAML we get back a double, but with float precision.
expectedParsedValues.add(Double.parseDouble(randomFloat.toString()));
}
break;
case 6:
boolean randomBoolean = random.nextBoolean();
originalValues.add(randomBoolean);
expectedParsedValues.add(randomBoolean);
break;
case 7:
String randomString =
random.nextBoolean()
? RandomStrings.randomAsciiOfLengthBetween(random, 3, 10)
: randomUnicodeOfLengthBetween(random, 3, 10);
originalValues.add(randomString);
expectedParsedValues.add(randomString);
break;
case 8:
byte[] randomBytes =
RandomStrings.randomUnicodeOfLengthBetween(random, 10, 50)
.getBytes(StandardCharsets.UTF_8);
BytesArray randomBytesArray = new BytesArray(randomBytes);
originalValues.add(randomBytesArray);
if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
// JSON and YAML write the base64 format
expectedParsedValues.add(Base64.getEncoder().encodeToString(randomBytes));
} else {
// SMILE and CBOR write the original bytes as they support binary format
expectedParsedValues.add(randomBytesArray);
}
break;
default:
throw new UnsupportedOperationException();
}
}
return Tuple.tuple(originalValues, expectedParsedValues);
}
public static void main(String[] args) {
int NUMBER_OF_KEYS = (int) SizeValue.parseSizeValue("20").singles();
int STRING_SIZE = 5;
long PUT_OPERATIONS = SizeValue.parseSizeValue("5m").singles();
long ITERATIONS = 10;
boolean REUSE = true;
String[] values = new String[NUMBER_OF_KEYS];
for (int i = 0; i < values.length; i++) {
values[i] = RandomStrings.randomAsciiOfLength(ThreadLocalRandom.current(), STRING_SIZE);
}
StopWatch stopWatch;
stopWatch = new StopWatch().start();
ObjectIntOpenHashMap<String> map = new ObjectIntOpenHashMap<>();
for (long iter = 0; iter < ITERATIONS; iter++) {
if (REUSE) {
map.clear();
} else {
map = new ObjectIntOpenHashMap<>();
}
for (long i = 0; i < PUT_OPERATIONS; i++) {
map.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1);
}
}
map.clear();
map = null;
stopWatch.stop();
System.out.println(
"TObjectIntHashMap: "
+ stopWatch.totalTime()
+ ", "
+ stopWatch.totalTime().millisFrac() / ITERATIONS
+ "ms");
stopWatch = new StopWatch().start();
// TObjectIntCustomHashMap<String> iMap = new TObjectIntCustomHashMap<String>(new
// StringIdentityHashingStrategy());
ObjectIntOpenHashMap<String> iMap = new ObjectIntOpenHashMap<>();
for (long iter = 0; iter < ITERATIONS; iter++) {
if (REUSE) {
iMap.clear();
} else {
iMap = new ObjectIntOpenHashMap<>();
}
for (long i = 0; i < PUT_OPERATIONS; i++) {
iMap.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1);
}
}
stopWatch.stop();
System.out.println(
"TObjectIntCustomHashMap(StringIdentity): "
+ stopWatch.totalTime()
+ ", "
+ stopWatch.totalTime().millisFrac() / ITERATIONS
+ "ms");
iMap.clear();
iMap = null;
stopWatch = new StopWatch().start();
iMap = new ObjectIntOpenHashMap<>();
for (long iter = 0; iter < ITERATIONS; iter++) {
if (REUSE) {
iMap.clear();
} else {
iMap = new ObjectIntOpenHashMap<>();
}
for (long i = 0; i < PUT_OPERATIONS; i++) {
iMap.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1);
}
}
stopWatch.stop();
System.out.println(
"TObjectIntCustomHashMap(PureIdentity): "
+ stopWatch.totalTime()
+ ", "
+ stopWatch.totalTime().millisFrac() / ITERATIONS
+ "ms");
iMap.clear();
iMap = null;
// now test with THashMap
stopWatch = new StopWatch().start();
ObjectObjectOpenHashMap<String, StringEntry> tMap = new ObjectObjectOpenHashMap<>();
for (long iter = 0; iter < ITERATIONS; iter++) {
if (REUSE) {
tMap.clear();
} else {
tMap = new ObjectObjectOpenHashMap<>();
}
for (long i = 0; i < PUT_OPERATIONS; i++) {
String key = values[(int) (i % NUMBER_OF_KEYS)];
StringEntry stringEntry = tMap.get(key);
if (stringEntry == null) {
stringEntry = new StringEntry(key, 1);
tMap.put(key, stringEntry);
} else {
stringEntry.counter++;
}
}
}
tMap.clear();
tMap = null;
stopWatch.stop();
System.out.println(
"THashMap: "
+ stopWatch.totalTime()
+ ", "
+ stopWatch.totalTime().millisFrac() / ITERATIONS
+ "ms");
stopWatch = new StopWatch().start();
HashMap<String, StringEntry> hMap = new HashMap<>();
for (long iter = 0; iter < ITERATIONS; iter++) {
if (REUSE) {
hMap.clear();
} else {
hMap = new HashMap<>();
}
for (long i = 0; i < PUT_OPERATIONS; i++) {
String key = values[(int) (i % NUMBER_OF_KEYS)];
StringEntry stringEntry = hMap.get(key);
if (stringEntry == null) {
stringEntry = new StringEntry(key, 1);
hMap.put(key, stringEntry);
} else {
stringEntry.counter++;
}
}
}
hMap.clear();
hMap = null;
stopWatch.stop();
System.out.println(
"HashMap: "
+ stopWatch.totalTime()
+ ", "
+ stopWatch.totalTime().millisFrac() / ITERATIONS
+ "ms");
stopWatch = new StopWatch().start();
IdentityHashMap<String, StringEntry> ihMap = new IdentityHashMap<>();
for (long iter = 0; iter < ITERATIONS; iter++) {
if (REUSE) {
ihMap.clear();
} else {
hMap = new HashMap<>();
}
for (long i = 0; i < PUT_OPERATIONS; i++) {
String key = values[(int) (i % NUMBER_OF_KEYS)];
StringEntry stringEntry = ihMap.get(key);
if (stringEntry == null) {
stringEntry = new StringEntry(key, 1);
ihMap.put(key, stringEntry);
} else {
stringEntry.counter++;
}
}
}
stopWatch.stop();
System.out.println(
"IdentityHashMap: "
+ stopWatch.totalTime()
+ ", "
+ stopWatch.totalTime().millisFrac() / ITERATIONS
+ "ms");
ihMap.clear();
ihMap = null;
int[] iValues = new int[NUMBER_OF_KEYS];
for (int i = 0; i < values.length; i++) {
iValues[i] = ThreadLocalRandom.current().nextInt();
}
stopWatch = new StopWatch().start();
IntIntOpenHashMap intMap = new IntIntOpenHashMap();
for (long iter = 0; iter < ITERATIONS; iter++) {
if (REUSE) {
intMap.clear();
} else {
intMap = new IntIntOpenHashMap();
}
for (long i = 0; i < PUT_OPERATIONS; i++) {
int key = iValues[(int) (i % NUMBER_OF_KEYS)];
intMap.addTo(key, 1);
}
}
stopWatch.stop();
System.out.println(
"TIntIntHashMap: "
+ stopWatch.totalTime()
+ ", "
+ stopWatch.totalTime().millisFrac() / ITERATIONS
+ "ms");
intMap.clear();
intMap = null;
// now test with THashMap
stopWatch = new StopWatch().start();
IntObjectOpenHashMap<IntEntry> tIntMap = new IntObjectOpenHashMap<>();
for (long iter = 0; iter < ITERATIONS; iter++) {
if (REUSE) {
tIntMap.clear();
} else {
tIntMap = new IntObjectOpenHashMap<>();
}
for (long i = 0; i < PUT_OPERATIONS; i++) {
int key = iValues[(int) (i % NUMBER_OF_KEYS)];
IntEntry intEntry = tIntMap.get(key);
if (intEntry == null) {
intEntry = new IntEntry(key, 1);
tIntMap.put(key, intEntry);
} else {
intEntry.counter++;
}
}
}
tIntMap.clear();
tIntMap = null;
stopWatch.stop();
System.out.println(
"TIntObjectHashMap: "
+ stopWatch.totalTime()
+ ", "
+ stopWatch.totalTime().millisFrac() / ITERATIONS
+ "ms");
}
public static void main(String[] args) throws Exception {
Natives.tryMlockall();
Settings settings =
settingsBuilder()
.put("refresh_interval", "-1")
.put(SETTING_NUMBER_OF_SHARDS, 1)
.put(SETTING_NUMBER_OF_REPLICAS, 0)
.build();
String clusterName = TermsAggregationSearchAndIndexingBenchmark.class.getSimpleName();
nodes = new InternalNode[1];
for (int i = 0; i < nodes.length; i++) {
nodes[i] =
(InternalNode)
nodeBuilder()
.settings(settingsBuilder().put(settings).put("name", "node1"))
.clusterName(clusterName)
.node();
}
Client client = nodes[0].client();
client
.admin()
.cluster()
.prepareHealth(indexName)
.setWaitForGreenStatus()
.setTimeout("10s")
.execute()
.actionGet();
try {
client
.admin()
.indices()
.prepareCreate(indexName)
.addMapping(typeName, generateMapping("eager", "lazy"))
.get();
Thread.sleep(5000);
long startTime = System.currentTimeMillis();
ObjectOpenHashSet<String> uniqueTerms = ObjectOpenHashSet.newInstance();
for (int i = 0; i < NUMBER_OF_TERMS; i++) {
boolean added;
do {
added = uniqueTerms.add(RandomStrings.randomAsciiOfLength(random, STRING_TERM_SIZE));
} while (!added);
}
String[] sValues = uniqueTerms.toArray(String.class);
long ITERS = COUNT / BATCH;
long i = 1;
int counter = 0;
for (; i <= ITERS; i++) {
BulkRequestBuilder request = client.prepareBulk();
for (int j = 0; j < BATCH; j++) {
counter++;
XContentBuilder builder = jsonBuilder().startObject();
builder.field("id", Integer.toString(counter));
final String sValue = sValues[counter % sValues.length];
builder.field("s_value", sValue);
builder.field("s_value_dv", sValue);
for (String field : new String[] {"sm_value", "sm_value_dv"}) {
builder.startArray(field);
for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) {
builder.value(sValues[ThreadLocalRandom.current().nextInt(sValues.length)]);
}
builder.endArray();
}
request.add(
Requests.indexRequest(indexName)
.type("type1")
.id(Integer.toString(counter))
.source(builder));
}
BulkResponse response = request.execute().actionGet();
if (response.hasFailures()) {
System.err.println("--> failures...");
}
if (((i * BATCH) % 10000) == 0) {
System.out.println("--> Indexed " + (i * BATCH));
}
}
System.out.println(
"--> Indexing took " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds.");
} catch (IndexAlreadyExistsException e) {
System.out.println("--> Index already exists, ignoring indexing phase, waiting for green");
ClusterHealthResponse clusterHealthResponse =
client
.admin()
.cluster()
.prepareHealth(indexName)
.setWaitForGreenStatus()
.setTimeout("10m")
.execute()
.actionGet();
if (clusterHealthResponse.isTimedOut()) {
System.err.println("--> Timed out waiting for cluster health");
}
}
client
.admin()
.indices()
.preparePutMapping(indexName)
.setType(typeName)
.setSource(generateMapping("lazy", "lazy"))
.get();
client.admin().indices().prepareRefresh().execute().actionGet();
System.out.println(
"--> Number of docs in index: "
+ client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount());
String[] nodeIds = new String[nodes.length];
for (int i = 0; i < nodeIds.length; i++) {
nodeIds[i] = nodes[i].injector().getInstance(Discovery.class).localNode().getId();
}
List<TestRun> testRuns = new ArrayList<>();
testRuns.add(new TestRun("Regular field ordinals", "eager", "lazy", "s_value", "ordinals"));
testRuns.add(
new TestRun("Docvalues field ordinals", "lazy", "eager", "s_value_dv", "ordinals"));
testRuns.add(
new TestRun(
"Regular field global ordinals", "eager_global_ordinals", "lazy", "s_value", null));
testRuns.add(
new TestRun("Docvalues field global", "lazy", "eager_global_ordinals", "s_value_dv", null));
List<TestResult> testResults = new ArrayList<>();
for (TestRun testRun : testRuns) {
client
.admin()
.indices()
.preparePutMapping(indexName)
.setType(typeName)
.setSource(
generateMapping(testRun.indexedFieldEagerLoading, testRun.docValuesEagerLoading))
.get();
client.admin().indices().prepareClearCache(indexName).setFieldDataCache(true).get();
SearchThread searchThread =
new SearchThread(client, testRun.termsAggsField, testRun.termsAggsExecutionHint);
RefreshThread refreshThread = new RefreshThread(client);
System.out.println("--> Running '" + testRun.name + "' round...");
new Thread(refreshThread).start();
new Thread(searchThread).start();
Thread.sleep(2 * 60 * 1000);
refreshThread.stop();
searchThread.stop();
System.out.println("--> Avg refresh time: " + refreshThread.avgRefreshTime + " ms");
System.out.println("--> Avg query time: " + searchThread.avgQueryTime + " ms");
ClusterStatsResponse clusterStateResponse =
client.admin().cluster().prepareClusterStats().setNodesIds(nodeIds).get();
System.out.println(
"--> Heap used: " + clusterStateResponse.getNodesStats().getJvm().getHeapUsed());
ByteSizeValue fieldDataMemoryUsed =
clusterStateResponse.getIndicesStats().getFieldData().getMemorySize();
System.out.println("--> Fielddata memory size: " + fieldDataMemoryUsed);
testResults.add(
new TestResult(
testRun.name,
refreshThread.avgRefreshTime,
searchThread.avgQueryTime,
fieldDataMemoryUsed));
}
System.out.println(
"----------------------------------------- SUMMARY ----------------------------------------------");
System.out.format(
Locale.ENGLISH,
"%30s%18s%15s%15s\n",
"name",
"avg refresh time",
"avg query time",
"fieldata size");
for (TestResult testResult : testResults) {
System.out.format(
Locale.ENGLISH,
"%30s%18s%15s%15s\n",
testResult.name,
testResult.avgRefreshTime,
testResult.avgQueryTime,
testResult.fieldDataSizeInMemory);
}
System.out.println(
"----------------------------------------- SUMMARY ----------------------------------------------");
client.close();
for (InternalNode node : nodes) {
node.close();
}
}
public static String randomString(Random random) {
if (random.nextBoolean()) {
return RandomStrings.randomAsciiOfLengthBetween(random, 1, 10);
}
return RandomStrings.randomUnicodeOfCodepointLengthBetween(random, 1, 10);
}