/** Randomly adds fields, objects, or arrays to the provided builder. The maximum depth is 5. */
 private static void addFields(Random random, XContentBuilder builder, int currentDepth)
     throws IOException {
   int numFields = RandomNumbers.randomIntBetween(random, 1, 5);
   for (int i = 0; i < numFields; i++) {
     if (currentDepth < 5 && random.nextBoolean()) {
       if (random.nextBoolean()) {
         builder.startObject(RandomStrings.randomAsciiOfLengthBetween(random, 3, 10));
         addFields(random, builder, currentDepth + 1);
         builder.endObject();
       } else {
         builder.startArray(RandomStrings.randomAsciiOfLengthBetween(random, 3, 10));
         int numElements = RandomNumbers.randomIntBetween(random, 1, 5);
         boolean object = random.nextBoolean();
         int dataType = -1;
         if (object == false) {
           dataType = randomDataType(random);
         }
         for (int j = 0; j < numElements; j++) {
           if (object) {
             builder.startObject();
             addFields(random, builder, 5);
             builder.endObject();
           } else {
             builder.value(randomFieldValue(random, dataType));
           }
         }
         builder.endArray();
       }
     } else {
       builder.field(
           RandomStrings.randomAsciiOfLengthBetween(random, 3, 10),
           randomFieldValue(random, randomDataType(random)));
     }
   }
 }
 private static Object randomFieldValue(Random random, int dataType) {
   switch (dataType) {
     case 0:
       return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
     case 1:
       return RandomStrings.randomAsciiOfLengthBetween(random, 3, 10);
     case 2:
       return random.nextLong();
     case 3:
       return random.nextDouble();
     default:
       throw new UnsupportedOperationException();
   }
 }
 public void testIsFieldWithinQuery() throws IOException {
   KeywordFieldType ft = new KeywordFieldType();
   // current impl ignores args and shourd always return INTERSECTS
   assertEquals(
       Relation.INTERSECTS,
       ft.isFieldWithinQuery(
           null,
           RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
           RandomStrings.randomAsciiOfLengthBetween(random(), 0, 5),
           randomBoolean(),
           randomBoolean(),
           null,
           null));
 }
  public void testDuelGlobalOrdinals() throws Exception {
    Random random = getRandom();
    final int numDocs = scaledRandomIntBetween(10, 1000);
    final int numValues = scaledRandomIntBetween(10, 500);
    final String[] values = new String[numValues];
    for (int i = 0; i < numValues; ++i) {
      values[i] = new String(RandomStrings.randomAsciiOfLength(random, 10));
    }
    for (int i = 0; i < numDocs; i++) {
      Document d = new Document();
      final int numVals = randomInt(3);
      for (int j = 0; j < numVals; ++j) {
        final String value = RandomPicks.randomFrom(random, Arrays.asList(values));
        d.add(new StringField("string", value, Field.Store.NO));
        d.add(new SortedSetDocValuesField("bytes", new BytesRef(value)));
      }
      writer.addDocument(d);
      if (randomInt(10) == 0) {
        refreshReader();
      }
    }
    refreshReader();

    Map<FieldDataType, Type> typeMap = new HashMap<FieldDataType, DuelFieldDataTests.Type>();
    typeMap.put(
        new FieldDataType("string", ImmutableSettings.builder().put("format", "fst")), Type.Bytes);
    typeMap.put(
        new FieldDataType("string", ImmutableSettings.builder().put("format", "paged_bytes")),
        Type.Bytes);
    typeMap.put(
        new FieldDataType("string", ImmutableSettings.builder().put("format", "doc_values")),
        Type.Bytes);

    for (Map.Entry<FieldDataType, Type> entry : typeMap.entrySet()) {
      ifdService.clear();
      IndexOrdinalsFieldData fieldData =
          getForField(entry.getKey(), entry.getValue().name().toLowerCase(Locale.ROOT));
      RandomAccessOrds left = fieldData.load(readerContext).getOrdinalsValues();
      fieldData.clear();
      RandomAccessOrds right =
          fieldData
              .loadGlobal(topLevelReader)
              .load(topLevelReader.leaves().get(0))
              .getOrdinalsValues();
      assertEquals(left.getValueCount(), right.getValueCount());
      for (long ord = 0; ord < left.getValueCount(); ++ord) {
        assertEquals(left.lookupOrd(ord), right.lookupOrd(ord));
      }
    }
  }
 public void testBytes() {
   final String[][] values = new String[randomInt(10)][];
   for (int i = 0; i < values.length; ++i) {
     String[] strings = new String[randomInt(8)];
     for (int j = 0; j < strings.length; ++j) {
       strings[j] = RandomStrings.randomAsciiOfLength(random(), 5);
     }
     Arrays.sort(strings);
     values[i] = strings;
   }
   FakeSearchScript script = new FakeSearchScript(values);
   ScriptBytesValues scriptValues = new ScriptBytesValues(script);
   for (int i = 0; i < values.length; ++i) {
     scriptValues.setDocument(i);
     assertEquals(values[i].length, scriptValues.count());
     for (int j = 0; j < values[i].length; ++j) {
       assertEquals(new BytesRef(values[i][j]), scriptValues.valueAt(j));
     }
   }
 }
 /**
  * Returns a tuple containing random stored field values and their corresponding expected values
  * once printed out via {@link
  * org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)}
  * and parsed back via {@link org.elasticsearch.common.xcontent.XContentParser#objectText()}.
  * Generates values based on what can get printed out. Stored fields values are retrieved from
  * lucene and converted via {@link
  * org.elasticsearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings,
  * numbers or booleans.
  *
  * @param random Random generator
  * @param xContentType the content type, used to determine what the expected values are for float
  *     numbers.
  */
 public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(
     Random random, XContentType xContentType) {
   int numValues = RandomNumbers.randomIntBetween(random, 1, 5);
   List<Object> originalValues = new ArrayList<>();
   List<Object> expectedParsedValues = new ArrayList<>();
   int dataType = RandomNumbers.randomIntBetween(random, 0, 8);
   for (int i = 0; i < numValues; i++) {
     switch (dataType) {
       case 0:
         long randomLong = random.nextLong();
         originalValues.add(randomLong);
         expectedParsedValues.add(randomLong);
         break;
       case 1:
         int randomInt = random.nextInt();
         originalValues.add(randomInt);
         expectedParsedValues.add(randomInt);
         break;
       case 2:
         Short randomShort = (short) random.nextInt();
         originalValues.add(randomShort);
         expectedParsedValues.add(randomShort.intValue());
         break;
       case 3:
         Byte randomByte = (byte) random.nextInt();
         originalValues.add(randomByte);
         expectedParsedValues.add(randomByte.intValue());
         break;
       case 4:
         double randomDouble = random.nextDouble();
         originalValues.add(randomDouble);
         expectedParsedValues.add(randomDouble);
         break;
       case 5:
         Float randomFloat = random.nextFloat();
         originalValues.add(randomFloat);
         if (xContentType == XContentType.CBOR) {
           // with CBOR we get back a float
           expectedParsedValues.add(randomFloat);
         } else if (xContentType == XContentType.SMILE) {
           // with SMILE we get back a double
           expectedParsedValues.add(randomFloat.doubleValue());
         } else {
           // with JSON AND YAML we get back a double, but with float precision.
           expectedParsedValues.add(Double.parseDouble(randomFloat.toString()));
         }
         break;
       case 6:
         boolean randomBoolean = random.nextBoolean();
         originalValues.add(randomBoolean);
         expectedParsedValues.add(randomBoolean);
         break;
       case 7:
         String randomString =
             random.nextBoolean()
                 ? RandomStrings.randomAsciiOfLengthBetween(random, 3, 10)
                 : randomUnicodeOfLengthBetween(random, 3, 10);
         originalValues.add(randomString);
         expectedParsedValues.add(randomString);
         break;
       case 8:
         byte[] randomBytes =
             RandomStrings.randomUnicodeOfLengthBetween(random, 10, 50)
                 .getBytes(StandardCharsets.UTF_8);
         BytesArray randomBytesArray = new BytesArray(randomBytes);
         originalValues.add(randomBytesArray);
         if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
           // JSON and YAML write the base64 format
           expectedParsedValues.add(Base64.getEncoder().encodeToString(randomBytes));
         } else {
           // SMILE and CBOR write the original bytes as they support binary format
           expectedParsedValues.add(randomBytesArray);
         }
         break;
       default:
         throw new UnsupportedOperationException();
     }
   }
   return Tuple.tuple(originalValues, expectedParsedValues);
 }
  public static void main(String[] args) {

    int NUMBER_OF_KEYS = (int) SizeValue.parseSizeValue("20").singles();
    int STRING_SIZE = 5;
    long PUT_OPERATIONS = SizeValue.parseSizeValue("5m").singles();
    long ITERATIONS = 10;
    boolean REUSE = true;

    String[] values = new String[NUMBER_OF_KEYS];
    for (int i = 0; i < values.length; i++) {
      values[i] = RandomStrings.randomAsciiOfLength(ThreadLocalRandom.current(), STRING_SIZE);
    }

    StopWatch stopWatch;

    stopWatch = new StopWatch().start();
    ObjectIntOpenHashMap<String> map = new ObjectIntOpenHashMap<>();
    for (long iter = 0; iter < ITERATIONS; iter++) {
      if (REUSE) {
        map.clear();
      } else {
        map = new ObjectIntOpenHashMap<>();
      }
      for (long i = 0; i < PUT_OPERATIONS; i++) {
        map.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1);
      }
    }
    map.clear();
    map = null;

    stopWatch.stop();
    System.out.println(
        "TObjectIntHashMap: "
            + stopWatch.totalTime()
            + ", "
            + stopWatch.totalTime().millisFrac() / ITERATIONS
            + "ms");

    stopWatch = new StopWatch().start();
    //        TObjectIntCustomHashMap<String> iMap = new TObjectIntCustomHashMap<String>(new
    // StringIdentityHashingStrategy());
    ObjectIntOpenHashMap<String> iMap = new ObjectIntOpenHashMap<>();
    for (long iter = 0; iter < ITERATIONS; iter++) {
      if (REUSE) {
        iMap.clear();
      } else {
        iMap = new ObjectIntOpenHashMap<>();
      }
      for (long i = 0; i < PUT_OPERATIONS; i++) {
        iMap.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1);
      }
    }
    stopWatch.stop();
    System.out.println(
        "TObjectIntCustomHashMap(StringIdentity): "
            + stopWatch.totalTime()
            + ", "
            + stopWatch.totalTime().millisFrac() / ITERATIONS
            + "ms");
    iMap.clear();
    iMap = null;

    stopWatch = new StopWatch().start();
    iMap = new ObjectIntOpenHashMap<>();
    for (long iter = 0; iter < ITERATIONS; iter++) {
      if (REUSE) {
        iMap.clear();
      } else {
        iMap = new ObjectIntOpenHashMap<>();
      }
      for (long i = 0; i < PUT_OPERATIONS; i++) {
        iMap.addTo(values[(int) (i % NUMBER_OF_KEYS)], 1);
      }
    }
    stopWatch.stop();
    System.out.println(
        "TObjectIntCustomHashMap(PureIdentity): "
            + stopWatch.totalTime()
            + ", "
            + stopWatch.totalTime().millisFrac() / ITERATIONS
            + "ms");
    iMap.clear();
    iMap = null;

    // now test with THashMap
    stopWatch = new StopWatch().start();
    ObjectObjectOpenHashMap<String, StringEntry> tMap = new ObjectObjectOpenHashMap<>();
    for (long iter = 0; iter < ITERATIONS; iter++) {
      if (REUSE) {
        tMap.clear();
      } else {
        tMap = new ObjectObjectOpenHashMap<>();
      }
      for (long i = 0; i < PUT_OPERATIONS; i++) {
        String key = values[(int) (i % NUMBER_OF_KEYS)];
        StringEntry stringEntry = tMap.get(key);
        if (stringEntry == null) {
          stringEntry = new StringEntry(key, 1);
          tMap.put(key, stringEntry);
        } else {
          stringEntry.counter++;
        }
      }
    }

    tMap.clear();
    tMap = null;

    stopWatch.stop();
    System.out.println(
        "THashMap: "
            + stopWatch.totalTime()
            + ", "
            + stopWatch.totalTime().millisFrac() / ITERATIONS
            + "ms");

    stopWatch = new StopWatch().start();
    HashMap<String, StringEntry> hMap = new HashMap<>();
    for (long iter = 0; iter < ITERATIONS; iter++) {
      if (REUSE) {
        hMap.clear();
      } else {
        hMap = new HashMap<>();
      }
      for (long i = 0; i < PUT_OPERATIONS; i++) {
        String key = values[(int) (i % NUMBER_OF_KEYS)];
        StringEntry stringEntry = hMap.get(key);
        if (stringEntry == null) {
          stringEntry = new StringEntry(key, 1);
          hMap.put(key, stringEntry);
        } else {
          stringEntry.counter++;
        }
      }
    }

    hMap.clear();
    hMap = null;

    stopWatch.stop();
    System.out.println(
        "HashMap: "
            + stopWatch.totalTime()
            + ", "
            + stopWatch.totalTime().millisFrac() / ITERATIONS
            + "ms");

    stopWatch = new StopWatch().start();
    IdentityHashMap<String, StringEntry> ihMap = new IdentityHashMap<>();
    for (long iter = 0; iter < ITERATIONS; iter++) {
      if (REUSE) {
        ihMap.clear();
      } else {
        hMap = new HashMap<>();
      }
      for (long i = 0; i < PUT_OPERATIONS; i++) {
        String key = values[(int) (i % NUMBER_OF_KEYS)];
        StringEntry stringEntry = ihMap.get(key);
        if (stringEntry == null) {
          stringEntry = new StringEntry(key, 1);
          ihMap.put(key, stringEntry);
        } else {
          stringEntry.counter++;
        }
      }
    }
    stopWatch.stop();
    System.out.println(
        "IdentityHashMap: "
            + stopWatch.totalTime()
            + ", "
            + stopWatch.totalTime().millisFrac() / ITERATIONS
            + "ms");

    ihMap.clear();
    ihMap = null;

    int[] iValues = new int[NUMBER_OF_KEYS];
    for (int i = 0; i < values.length; i++) {
      iValues[i] = ThreadLocalRandom.current().nextInt();
    }

    stopWatch = new StopWatch().start();
    IntIntOpenHashMap intMap = new IntIntOpenHashMap();
    for (long iter = 0; iter < ITERATIONS; iter++) {
      if (REUSE) {
        intMap.clear();
      } else {
        intMap = new IntIntOpenHashMap();
      }
      for (long i = 0; i < PUT_OPERATIONS; i++) {
        int key = iValues[(int) (i % NUMBER_OF_KEYS)];
        intMap.addTo(key, 1);
      }
    }
    stopWatch.stop();
    System.out.println(
        "TIntIntHashMap: "
            + stopWatch.totalTime()
            + ", "
            + stopWatch.totalTime().millisFrac() / ITERATIONS
            + "ms");

    intMap.clear();
    intMap = null;

    // now test with THashMap
    stopWatch = new StopWatch().start();
    IntObjectOpenHashMap<IntEntry> tIntMap = new IntObjectOpenHashMap<>();
    for (long iter = 0; iter < ITERATIONS; iter++) {
      if (REUSE) {
        tIntMap.clear();
      } else {
        tIntMap = new IntObjectOpenHashMap<>();
      }
      for (long i = 0; i < PUT_OPERATIONS; i++) {
        int key = iValues[(int) (i % NUMBER_OF_KEYS)];
        IntEntry intEntry = tIntMap.get(key);
        if (intEntry == null) {
          intEntry = new IntEntry(key, 1);
          tIntMap.put(key, intEntry);
        } else {
          intEntry.counter++;
        }
      }
    }

    tIntMap.clear();
    tIntMap = null;

    stopWatch.stop();
    System.out.println(
        "TIntObjectHashMap: "
            + stopWatch.totalTime()
            + ", "
            + stopWatch.totalTime().millisFrac() / ITERATIONS
            + "ms");
  }
  public static void main(String[] args) throws Exception {
    Natives.tryMlockall();
    Settings settings =
        settingsBuilder()
            .put("refresh_interval", "-1")
            .put(SETTING_NUMBER_OF_SHARDS, 1)
            .put(SETTING_NUMBER_OF_REPLICAS, 0)
            .build();

    String clusterName = TermsAggregationSearchAndIndexingBenchmark.class.getSimpleName();
    nodes = new InternalNode[1];
    for (int i = 0; i < nodes.length; i++) {
      nodes[i] =
          (InternalNode)
              nodeBuilder()
                  .settings(settingsBuilder().put(settings).put("name", "node1"))
                  .clusterName(clusterName)
                  .node();
    }
    Client client = nodes[0].client();

    client
        .admin()
        .cluster()
        .prepareHealth(indexName)
        .setWaitForGreenStatus()
        .setTimeout("10s")
        .execute()
        .actionGet();
    try {
      client
          .admin()
          .indices()
          .prepareCreate(indexName)
          .addMapping(typeName, generateMapping("eager", "lazy"))
          .get();
      Thread.sleep(5000);

      long startTime = System.currentTimeMillis();
      ObjectOpenHashSet<String> uniqueTerms = ObjectOpenHashSet.newInstance();
      for (int i = 0; i < NUMBER_OF_TERMS; i++) {
        boolean added;
        do {
          added = uniqueTerms.add(RandomStrings.randomAsciiOfLength(random, STRING_TERM_SIZE));
        } while (!added);
      }
      String[] sValues = uniqueTerms.toArray(String.class);
      long ITERS = COUNT / BATCH;
      long i = 1;
      int counter = 0;
      for (; i <= ITERS; i++) {
        BulkRequestBuilder request = client.prepareBulk();
        for (int j = 0; j < BATCH; j++) {
          counter++;

          XContentBuilder builder = jsonBuilder().startObject();
          builder.field("id", Integer.toString(counter));
          final String sValue = sValues[counter % sValues.length];
          builder.field("s_value", sValue);
          builder.field("s_value_dv", sValue);

          for (String field : new String[] {"sm_value", "sm_value_dv"}) {
            builder.startArray(field);
            for (int k = 0; k < NUMBER_OF_MULTI_VALUE_TERMS; k++) {
              builder.value(sValues[ThreadLocalRandom.current().nextInt(sValues.length)]);
            }
            builder.endArray();
          }

          request.add(
              Requests.indexRequest(indexName)
                  .type("type1")
                  .id(Integer.toString(counter))
                  .source(builder));
        }
        BulkResponse response = request.execute().actionGet();
        if (response.hasFailures()) {
          System.err.println("--> failures...");
        }
        if (((i * BATCH) % 10000) == 0) {
          System.out.println("--> Indexed " + (i * BATCH));
        }
      }

      System.out.println(
          "--> Indexing took " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds.");
    } catch (IndexAlreadyExistsException e) {
      System.out.println("--> Index already exists, ignoring indexing phase, waiting for green");
      ClusterHealthResponse clusterHealthResponse =
          client
              .admin()
              .cluster()
              .prepareHealth(indexName)
              .setWaitForGreenStatus()
              .setTimeout("10m")
              .execute()
              .actionGet();
      if (clusterHealthResponse.isTimedOut()) {
        System.err.println("--> Timed out waiting for cluster health");
      }
    }
    client
        .admin()
        .indices()
        .preparePutMapping(indexName)
        .setType(typeName)
        .setSource(generateMapping("lazy", "lazy"))
        .get();
    client.admin().indices().prepareRefresh().execute().actionGet();
    System.out.println(
        "--> Number of docs in index: "
            + client.prepareCount().setQuery(matchAllQuery()).execute().actionGet().getCount());

    String[] nodeIds = new String[nodes.length];
    for (int i = 0; i < nodeIds.length; i++) {
      nodeIds[i] = nodes[i].injector().getInstance(Discovery.class).localNode().getId();
    }

    List<TestRun> testRuns = new ArrayList<>();
    testRuns.add(new TestRun("Regular field ordinals", "eager", "lazy", "s_value", "ordinals"));
    testRuns.add(
        new TestRun("Docvalues field ordinals", "lazy", "eager", "s_value_dv", "ordinals"));
    testRuns.add(
        new TestRun(
            "Regular field global ordinals", "eager_global_ordinals", "lazy", "s_value", null));
    testRuns.add(
        new TestRun("Docvalues field global", "lazy", "eager_global_ordinals", "s_value_dv", null));

    List<TestResult> testResults = new ArrayList<>();
    for (TestRun testRun : testRuns) {
      client
          .admin()
          .indices()
          .preparePutMapping(indexName)
          .setType(typeName)
          .setSource(
              generateMapping(testRun.indexedFieldEagerLoading, testRun.docValuesEagerLoading))
          .get();
      client.admin().indices().prepareClearCache(indexName).setFieldDataCache(true).get();
      SearchThread searchThread =
          new SearchThread(client, testRun.termsAggsField, testRun.termsAggsExecutionHint);
      RefreshThread refreshThread = new RefreshThread(client);
      System.out.println("--> Running '" + testRun.name + "' round...");
      new Thread(refreshThread).start();
      new Thread(searchThread).start();
      Thread.sleep(2 * 60 * 1000);
      refreshThread.stop();
      searchThread.stop();

      System.out.println("--> Avg refresh time: " + refreshThread.avgRefreshTime + " ms");
      System.out.println("--> Avg query time: " + searchThread.avgQueryTime + " ms");

      ClusterStatsResponse clusterStateResponse =
          client.admin().cluster().prepareClusterStats().setNodesIds(nodeIds).get();
      System.out.println(
          "--> Heap used: " + clusterStateResponse.getNodesStats().getJvm().getHeapUsed());
      ByteSizeValue fieldDataMemoryUsed =
          clusterStateResponse.getIndicesStats().getFieldData().getMemorySize();
      System.out.println("--> Fielddata memory size: " + fieldDataMemoryUsed);
      testResults.add(
          new TestResult(
              testRun.name,
              refreshThread.avgRefreshTime,
              searchThread.avgQueryTime,
              fieldDataMemoryUsed));
    }

    System.out.println(
        "----------------------------------------- SUMMARY ----------------------------------------------");
    System.out.format(
        Locale.ENGLISH,
        "%30s%18s%15s%15s\n",
        "name",
        "avg refresh time",
        "avg query time",
        "fieldata size");
    for (TestResult testResult : testResults) {
      System.out.format(
          Locale.ENGLISH,
          "%30s%18s%15s%15s\n",
          testResult.name,
          testResult.avgRefreshTime,
          testResult.avgQueryTime,
          testResult.fieldDataSizeInMemory);
    }
    System.out.println(
        "----------------------------------------- SUMMARY ----------------------------------------------");

    client.close();
    for (InternalNode node : nodes) {
      node.close();
    }
  }
 public static String randomString(Random random) {
   if (random.nextBoolean()) {
     return RandomStrings.randomAsciiOfLengthBetween(random, 1, 10);
   }
   return RandomStrings.randomUnicodeOfCodepointLengthBetween(random, 1, 10);
 }