public void testUpdate() {
    createIndexWithAlias();
    ensureYellow("test");

    UpdateRequestBuilder updateRequestBuilder =
        client()
            .prepareUpdate(indexOrAlias(), "type1", "1")
            .setUpsert("field1", "value1")
            .setDoc("field2", "value2");

    UpdateResponse updateResponse = updateRequestBuilder.get();
    assertThat(updateResponse.getIndex(), equalTo("test"));
    assertThat(updateResponse.getType(), equalTo("type1"));
    assertThat(updateResponse.getId(), equalTo("1"));
    assertThat(updateResponse.isCreated(), equalTo(true));

    GetResponse getResponse = client().prepareGet("test", "type1", "1").get();
    assertThat(getResponse.isExists(), equalTo(true));
    assertThat(getResponse.getSourceAsMap().containsKey("field1"), equalTo(true));
    assertThat(getResponse.getSourceAsMap().containsKey("field2"), equalTo(false));

    updateResponse = updateRequestBuilder.get();
    assertThat(updateResponse.getIndex(), equalTo("test"));
    assertThat(updateResponse.getType(), equalTo("type1"));
    assertThat(updateResponse.getId(), equalTo("1"));
    assertThat(updateResponse.isCreated(), equalTo(false));

    getResponse = client().prepareGet("test", "type1", "1").get();
    assertThat(getResponse.isExists(), equalTo(true));
    assertThat(getResponse.getSourceAsMap().containsKey("field1"), equalTo(true));
    assertThat(getResponse.getSourceAsMap().containsKey("field2"), equalTo(true));
  }
  @Test
  public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throws Exception {
    client.admin().indices().prepareDelete().execute().actionGet();
    String index = "test";
    String type = "type1";

    String mapping =
        jsonBuilder()
            .startObject()
            .startObject("source_excludes")
            .startObject("_source")
            .array("includes", "included")
            .array("exlcudes", "excluded")
            .endObject()
            .endObject()
            .endObject()
            .string();

    client
        .admin()
        .indices()
        .prepareCreate(index)
        .addMapping(type, mapping)
        .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
        .execute()
        .actionGet();

    client
        .prepareIndex(index, type, "1")
        .setSource(
            jsonBuilder()
                .startObject()
                .field("field", "1", "2")
                .field("included", "should be seen")
                .field("excluded", "should not be seen")
                .endObject())
        .execute()
        .actionGet();

    GetResponse responseBeforeFlush =
        client
            .prepareGet(index, type, "1")
            .setFields("_source", "included", "excluded")
            .execute()
            .actionGet();
    client.admin().indices().prepareFlush(index).execute().actionGet();
    GetResponse responseAfterFlush =
        client
            .prepareGet(index, type, "1")
            .setFields("_source", "included", "excluded")
            .execute()
            .actionGet();

    assertThat(responseBeforeFlush.isExists(), is(true));
    assertThat(responseAfterFlush.isExists(), is(true));
    assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("excluded")));
    assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("field")));
    assertThat(responseBeforeFlush.getSourceAsMap(), hasKey("included"));
    assertThat(responseBeforeFlush.getSourceAsString(), is(responseAfterFlush.getSourceAsString()));
  }
  /**
   * Basic test using Index & Realtime Get with internal versioning. This test ensures routing
   * works correctly across versions.
   */
  public void testInternalVersion() throws Exception {
    createIndex("test");
    final boolean routing = randomBoolean();
    int numDocs = randomIntBetween(10, 20);
    for (int i = 0; i < numDocs; i++) {
      String routingKey = routing ? randomRealisticUnicodeOfLength(10) : null;
      String id = Integer.toString(i);
      assertThat(
          id,
          client()
              .prepareIndex("test", "type1", id)
              .setRouting(routingKey)
              .setSource("field1", English.intToEnglish(i))
              .get()
              .isCreated(),
          is(true));
      GetResponse get =
          client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(1).get();
      assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
      assertThat(get.getVersion(), equalTo(1L));
      client()
          .prepareIndex("test", "type1", id)
          .setRouting(routingKey)
          .setSource("field1", English.intToEnglish(i))
          .execute()
          .actionGet();
      get = client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(2).get();
      assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
      assertThat(get.getVersion(), equalTo(2L));
    }

    assertVersionCreated(compatibilityVersion(), "test");
  }
  public void testIndexNameDateMathExpressions() {
    DateTime now = new DateTime(DateTimeZone.UTC);
    String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now);
    String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1));
    String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2));
    createIndex(index1, index2, index3);

    String dateMathExp1 = "<.marvel-{now/d}>";
    String dateMathExp2 = "<.marvel-{now/d-1d}>";
    String dateMathExp3 = "<.marvel-{now/d-2d}>";
    client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
    client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
    client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
    refresh();

    SearchResponse searchResponse =
        client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
    assertHitCount(searchResponse, 3);
    assertSearchHits(searchResponse, "1", "2", "3");

    GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get();
    assertThat(getResponse.isExists(), is(true));
    assertThat(getResponse.getId(), equalTo("1"));

    getResponse = client().prepareGet(dateMathExp2, "type", "2").get();
    assertThat(getResponse.isExists(), is(true));
    assertThat(getResponse.getId(), equalTo("2"));

    getResponse = client().prepareGet(dateMathExp3, "type", "3").get();
    assertThat(getResponse.isExists(), is(true));
    assertThat(getResponse.getId(), equalTo("3"));

    IndicesStatsResponse indicesStatsResponse =
        client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get();
    assertThat(indicesStatsResponse.getIndex(index1), notNullValue());
    assertThat(indicesStatsResponse.getIndex(index2), notNullValue());
    assertThat(indicesStatsResponse.getIndex(index3), notNullValue());

    DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get();
    assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
    assertThat(deleteResponse.getId(), equalTo("1"));

    deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get();
    assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
    assertThat(deleteResponse.getId(), equalTo("2"));

    deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get();
    assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
    assertThat(deleteResponse.getId(), equalTo("3"));
  }
 @Override
 public HashMap<String, String> loadKey(String username) {
   logger.info("loading password for username {}", username);
   HashMap<String, String> ret = new HashMap<>();
   String riverIndexName = getRiverIndexName();
   refreshSearchIndex(riverIndexName);
   GetResponse resp =
       client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet();
   if (resp.isExists()) {
     if (logger.isDebugEnabled()) {
       logger.debug("Password document: {}", resp.getSourceAsString());
     }
     Map<String, Object> newset = resp.getSource();
     Set<String> keys = newset.keySet();
     for (String s : keys) {
       logger.info(
           "Added key {} with a value of {}",
           s,
           XContentMapValues.nodeStringValue(newset.get(s), null));
       ret.put(s, XContentMapValues.nodeStringValue(newset.get(s), null));
     }
   }
   if (ret.isEmpty()) {
     return null;
   }
   return ret;
 }
  public void testDeleteRoutingRequired()
      throws ExecutionException, InterruptedException, IOException {
    createIndexWithAlias();
    assertAcked(
        client()
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType("test")
            .setSource(
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("test")
                    .startObject("_routing")
                    .field("required", true)
                    .endObject()
                    .endObject()
                    .endObject()));
    ensureYellow("test");

    int numDocs = iterations(10, 50);
    IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < numDocs - 2; i++) {
      indexRequestBuilders[i] =
          client()
              .prepareIndex("test", "test", Integer.toString(i))
              .setRouting(randomAsciiOfLength(randomIntBetween(1, 10)))
              .setSource("field", "value");
    }
    String firstDocId = Integer.toString(numDocs - 2);
    indexRequestBuilders[numDocs - 2] =
        client()
            .prepareIndex("test", "test", firstDocId)
            .setRouting("routing")
            .setSource("field", "value");
    String secondDocId = Integer.toString(numDocs - 1);
    String secondRouting = randomAsciiOfLength(randomIntBetween(1, 10));
    indexRequestBuilders[numDocs - 1] =
        client()
            .prepareIndex("test", "test", secondDocId)
            .setRouting(secondRouting)
            .setSource("field", "value");

    indexRandom(true, indexRequestBuilders);

    SearchResponse searchResponse = client().prepareSearch("test").get();
    assertNoFailures(searchResponse);
    assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs));

    DeleteResponse deleteResponse =
        client().prepareDelete("test", "test", firstDocId).setRouting("routing").get();
    assertThat(deleteResponse.isFound(), equalTo(true));
    GetResponse getResponse =
        client().prepareGet("test", "test", firstDocId).setRouting("routing").get();
    assertThat(getResponse.isExists(), equalTo(false));
    refresh();
    searchResponse = client().prepareSearch("test").get();
    assertNoFailures(searchResponse);
    assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1));
  }
 private List<KnapsackState> get(String name) throws IOException {
   ImmutableList.Builder<KnapsackState> builder = ImmutableList.builder();
   try {
     logger.debug("get knapsack states: {}", name);
     final Client client = injector.getInstance(Client.class);
     createIndexIfNotExist(client);
     GetResponse getResponse =
         client.prepareGet(INDEX_NAME, MAPPING_NAME, name).execute().actionGet();
     if (!getResponse.isExists()) {
       return builder.build();
     }
     XContentParser parser = xContent(JSON).createParser(getResponse.getSourceAsBytes());
     while (parser.nextToken() != START_ARRAY) {
       // forward
     }
     while (parser.nextToken() != END_ARRAY) {
       KnapsackState state = new KnapsackState();
       builder.add(state.fromXContent(parser));
     }
     return builder.build();
   } catch (Throwable t) {
     logger.error("get settings failed", t);
     return null;
   }
 }
  @Override
  public Date readDatetimeValue(String spaceKey, String propertyName) throws IOException {
    Date lastDate = null;
    String documentName = prepareValueStoreDocumentName(spaceKey, propertyName);

    if (logger.isDebugEnabled())
      logger.debug(
          "Going to read datetime value from {} property for space {}. Document name is {}.",
          propertyName,
          spaceKey,
          documentName);

    refreshSearchIndex(getRiverIndexName());
    GetResponse lastSeqGetResponse =
        client
            .prepareGet(getRiverIndexName(), riverName.name(), documentName)
            .execute()
            .actionGet();
    if (lastSeqGetResponse.isExists()) {
      Object timestamp = lastSeqGetResponse.getSourceAsMap().get(STORE_FIELD_VALUE);
      if (timestamp != null) {
        lastDate = DateTimeUtils.parseISODateTime(timestamp.toString());
      }
    } else {
      if (logger.isDebugEnabled())
        logger.debug("{} document doesn't exist in remore river persistent store", documentName);
    }
    return lastDate;
  }
 private void test(
     String name, Class<? extends HashFunction> expectedHashFunction, boolean expectedUseType)
     throws Exception {
   Path zippedIndexDir = getDataPath("/org/elasticsearch/cluster/routing/" + name + ".zip");
   Settings baseSettings = prepareBackwardsDataDir(zippedIndexDir);
   internalCluster()
       .startNode(Settings.builder().put(baseSettings).put(Node.HTTP_ENABLED, true).build());
   ensureYellow("test");
   GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().get();
   assertArrayEquals(new String[] {"test"}, getIndexResponse.indices());
   GetSettingsResponse getSettingsResponse =
       client().admin().indices().prepareGetSettings("test").get();
   assertEquals(
       expectedHashFunction.getName(),
       getSettingsResponse.getSetting("test", IndexMetaData.SETTING_LEGACY_ROUTING_HASH_FUNCTION));
   assertEquals(
       Boolean.valueOf(expectedUseType).toString(),
       getSettingsResponse.getSetting("test", IndexMetaData.SETTING_LEGACY_ROUTING_USE_TYPE));
   SearchResponse allDocs = client().prepareSearch("test").get();
   assertSearchResponse(allDocs);
   assertHitCount(allDocs, 4);
   // Make sure routing works
   for (SearchHit hit : allDocs.getHits().hits()) {
     GetResponse get = client().prepareGet(hit.index(), hit.type(), hit.id()).get();
     assertTrue(get.isExists());
   }
 }
  public void testPrimaryRelocation() throws Exception {
    Path dataPath = createTempDir();
    Settings nodeSettings = nodeSettings(dataPath);

    String node1 = internalCluster().startNode(nodeSettings);
    String IDX = "test";

    Settings idxSettings =
        Settings.builder()
            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
            .put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString())
            .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
            .put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true)
            .build();

    prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
    client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get();
    client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();

    GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get();
    GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get();
    assertTrue(gResp1.isExists());
    assertTrue(gResp2.isExists());
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));

    // Node1 has the primary, now node2 has the replica
    String node2 = internalCluster().startNode(nodeSettings);
    ensureGreen(IDX);
    client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
    flushAndRefresh(IDX);

    // now prevent primary from being allocated on node 1 move to node_3
    String node3 = internalCluster().startNode(nodeSettings);
    Settings build =
        Settings.builder().put("index.routing.allocation.exclude._name", node1).build();
    client().admin().indices().prepareUpdateSettings(IDX).setSettings(build).execute().actionGet();

    ensureGreen(IDX);
    logger.info("--> performing query");
    SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
    assertHitCount(resp, 2);

    gResp1 = client().prepareGet(IDX, "doc", "1").get();
    gResp2 = client().prepareGet(IDX, "doc", "2").get();
    assertTrue(gResp1.isExists());
    assertTrue(gResp2.toString(), gResp2.isExists());
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));

    client().prepareIndex(IDX, "doc", "3").setSource("foo", "bar").get();
    client().prepareIndex(IDX, "doc", "4").setSource("foo", "bar").get();
    gResp1 = client().prepareGet(IDX, "doc", "3").setPreference("_primary").get();
    gResp2 = client().prepareGet(IDX, "doc", "4").setPreference("_primary").get();
    assertTrue(gResp1.isExists());
    assertTrue(gResp2.isExists());
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
  }
 protected ESShape getShape(Iterator<MultiGetItemResponse> iterator, int size) {
   ESShapeBuilder shapeBuilder = new ESShapeBuilder(size);
   for (int i = 0; i < size; i++) {
     GetResponse response = iterator.next().getResponse();
     if (!response.isExists()) continue;
     @SuppressWarnings("unchecked")
     Map<String, Object> shape = (Map<String, Object>) response.getField("shape").getValue();
     @SuppressWarnings("unchecked")
     List<Double> coordinates = (List<Double>) shape.get("coordinates");
     shapeBuilder.addLocation(coordinates.get(1), coordinates.get(0));
   }
   return shapeBuilder.build();
 }
  public void testIndexGetAndDelete() throws ExecutionException, InterruptedException {
    createIndexWithAlias();
    ensureYellow("test");

    int numDocs = iterations(10, 50);
    for (int i = 0; i < numDocs; i++) {
      IndexResponse indexResponse =
          client()
              .prepareIndex(indexOrAlias(), "type", Integer.toString(i))
              .setSource("field", "value-" + i)
              .get();
      assertThat(indexResponse.isCreated(), equalTo(true));
      assertThat(indexResponse.getIndex(), equalTo("test"));
      assertThat(indexResponse.getType(), equalTo("type"));
      assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
    }
    refresh();

    String docId = Integer.toString(randomIntBetween(0, numDocs - 1));
    GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
    assertThat(getResponse.isExists(), equalTo(true));
    assertThat(getResponse.getIndex(), equalTo("test"));
    assertThat(getResponse.getType(), equalTo("type"));
    assertThat(getResponse.getId(), equalTo(docId));

    DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get();
    assertThat(deleteResponse.isFound(), equalTo(true));
    assertThat(deleteResponse.getIndex(), equalTo("test"));
    assertThat(deleteResponse.getType(), equalTo("type"));
    assertThat(deleteResponse.getId(), equalTo(docId));

    getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
    assertThat(getResponse.isExists(), equalTo(false));

    refresh();

    SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get();
    assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1));
  }
  public void testReplicaToPrimaryPromotion() throws Exception {
    Path dataPath = createTempDir();
    Settings nodeSettings = nodeSettings(dataPath);

    String node1 = internalCluster().startNode(nodeSettings);
    String IDX = "test";

    Settings idxSettings =
        Settings.builder()
            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
            .put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString())
            .put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
            .put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true)
            .build();

    prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
    client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get();
    client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();

    GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get();
    GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get();
    assertTrue(gResp1.isExists());
    assertTrue(gResp2.isExists());
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));

    // Node1 has the primary, now node2 has the replica
    String node2 = internalCluster().startNode(nodeSettings);
    ensureGreen(IDX);
    client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
    flushAndRefresh(IDX);

    logger.info("--> stopping node1 [{}]", node1);
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1));
    ensureYellow(IDX);

    logger.info("--> performing query");
    SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
    assertHitCount(resp, 2);

    gResp1 = client().prepareGet(IDX, "doc", "1").get();
    gResp2 = client().prepareGet(IDX, "doc", "2").get();
    assertTrue(gResp1.isExists());
    assertTrue(gResp2.toString(), gResp2.isExists());
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));

    client().prepareIndex(IDX, "doc", "1").setSource("foo", "foobar").get();
    client().prepareIndex(IDX, "doc", "2").setSource("foo", "foobar").get();
    gResp1 = client().prepareGet(IDX, "doc", "1").get();
    gResp2 = client().prepareGet(IDX, "doc", "2").get();
    assertTrue(gResp1.isExists());
    assertTrue(gResp2.toString(), gResp2.isExists());
    assertThat(gResp1.getSource().get("foo"), equalTo("foobar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("foobar"));
  }
 /**
  * Basic test using Index &amp; Realtime Get with external versioning. This test ensures routing
  * works correctly across versions.
  */
 public void testExternalVersion() throws Exception {
   createIndex("test");
   final boolean routing = randomBoolean();
   int numDocs = randomIntBetween(10, 20);
   for (int i = 0; i < numDocs; i++) {
     String id = Integer.toString(i);
     String routingKey = routing ? randomRealisticUnicodeOfLength(10) : null;
     final long version = randomIntBetween(0, Integer.MAX_VALUE);
     client()
         .prepareIndex("test", "type1", id)
         .setRouting(routingKey)
         .setVersion(version)
         .setVersionType(VersionType.EXTERNAL)
         .setSource("field1", English.intToEnglish(i))
         .get();
     GetResponse get =
         client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(version).get();
     assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
     assertThat(get.getVersion(), equalTo(version));
     final long nextVersion = version + randomIntBetween(0, Integer.MAX_VALUE);
     client()
         .prepareIndex("test", "type1", id)
         .setRouting(routingKey)
         .setVersion(nextVersion)
         .setVersionType(VersionType.EXTERNAL)
         .setSource("field1", English.intToEnglish(i))
         .get();
     get =
         client()
             .prepareGet("test", "type1", id)
             .setRouting(routingKey)
             .setVersion(nextVersion)
             .get();
     assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
     assertThat(get.getVersion(), equalTo(nextVersion));
   }
 }
 @SuppressWarnings("unchecked")
 protected <T extends ESEntity> T buildEntityFromGetResponse(
     Class<T> entityClass, MultiGetItemResponse item) {
   GetResponse response = item.getResponse();
   if (!response.isExists())
     throw new DaoException(
         String.format(
             "Entity %s does not exist in %s/%s",
             response.getId(), response.getIndex(), response.getType()));
   if (entityClass == null) throw new IllegalArgumentException("Provided Entity class is null");
   else if (entityClass.equals(ESNode.class))
     return (T) ESNode.Builder.buildFromGetReponse(response);
   else if (entityClass.equals(ESWay.class))
     return (T) ESWay.Builder.buildFromGetReponse(response);
   else throw new IllegalArgumentException(entityClass.getSimpleName() + " is not a known Entity");
 }
 protected void createRiver(String jsonDefinition, String river, Object... args) throws Exception {
   logger.info("Create river [{}]", river);
   String setting = getJsonSettings(jsonDefinition, args);
   logger.info("River setting [{}]", setting);
   node.client().prepareIndex("_river", river, "_meta").setSource(setting).execute().actionGet();
   logger.debug("Running Cluster Health");
   ClusterHealthResponse clusterHealth =
       node.client()
           .admin()
           .cluster()
           .health(clusterHealthRequest().waitForGreenStatus())
           .actionGet();
   logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
   GetResponse response =
       getNode().client().prepareGet("_river", river, "_meta").execute().actionGet();
   assertThat(response.isExists(), equalTo(true));
 }
 public RiverState load(Client client) throws IOException {
   if (coordinateIndex == null || coordinateType == null || coordinateId == null) {
     return this;
   }
   GetResponse get = null;
   try {
     get = client.prepareGet(coordinateIndex, coordinateType, coordinateId).execute().actionGet();
   } catch (Exception e) {
     // ignore
   }
   if (get != null && get.isExists()) {
     XContentParser parser =
         XContentFactory.xContent(XContentType.JSON).createParser(get.getSourceAsBytes());
     fromXContent(parser);
   } else {
     counter = 0L;
   }
   return this;
 }
  /** Reconfigure the river. Must be stopped! */
  public synchronized void reconfigure() {
    if (!closed) throw new IllegalStateException("Remote River must be stopped to reconfigure it!");

    logger.info("reconfiguring Remote River");
    String riverIndexName = getRiverIndexName();
    refreshSearchIndex(riverIndexName);
    GetResponse resp =
        client.prepareGet(riverIndexName, riverName().name(), "_meta").execute().actionGet();
    if (resp.isExists()) {
      if (logger.isDebugEnabled()) {
        logger.debug("Configuration document: {}", resp.getSourceAsString());
      }
      Map<String, Object> newset = resp.getSource();
      configure(newset);
    } else {
      throw new IllegalStateException(
          "Configuration document not found to reconfigure remote river " + riverName().name());
    }
  }
Exemple #19
0
  @CheckForNull
  @Override
  public DOMAIN getNullableByKey(KEY key) {
    GetRequestBuilder request =
        client
            .prepareGet()
            .setType(this.getIndexType())
            .setIndex(this.getIndexName())
            .setId(this.getKeyValue(key))
            .setFetchSource(true)
            .setRouting(this.getKeyValue(key));

    GetResponse response = request.get();

    if (response.isExists()) {
      return toDoc(response.getSource());
    }
    return null;
  }
  @Override
  public Article get(long id) {
    GetResponse getResponse =
        transportClient.prepareGet(INDICE, TYPE, String.valueOf(id)).execute().actionGet();
    if (getResponse.isExists()) {
      Map<String, Object> map = getResponse.getSource();
      String title = (String) map.get(TITLE);
      String summary = (String) map.get(SUMMARY);
      String idStr = getResponse.getId();
      logger.debug("get id : {}", idStr);

      Article article = new Article();
      article.setId(Long.parseLong(idStr));
      article.setTitle(title);
      article.setSummary(summary);
      return article;
    } else {
      return null;
    }
  }
 @Override
 public Table load(String key) {
   logger.info("Load called for: " + key);
   GetResponse response =
       elasticsearchConnection
           .getClient()
           .prepareGet()
           .setIndex(TABLE_META_INDEX)
           .setType(TABLE_META_TYPE)
           .setId(key)
           .execute()
           .actionGet();
   if (!response.isExists()) {
     return null;
   }
   try {
     return objectMapper.readValue(response.getSourceAsBytes(), Table.class);
   } catch (Exception e) {
     throw new RuntimeException("Error getting data for table: " + key);
   }
 }
  @Test
  public void realtimeGetWithCompress() throws Exception {
    client.admin().indices().prepareDelete().execute().actionGet();

    client
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
        .addMapping(
            "type",
            jsonBuilder()
                .startObject()
                .startObject("type")
                .startObject("_source")
                .field("compress", true)
                .endObject()
                .endObject()
                .endObject())
        .execute()
        .actionGet();

    ClusterHealthResponse clusterHealth =
        client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));

    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < 10000; i++) {
      sb.append((char) i);
    }
    String fieldValue = sb.toString();
    client.prepareIndex("test", "type", "1").setSource("field", fieldValue).execute().actionGet();

    // realtime get
    GetResponse getResponse = client.prepareGet("test", "type", "1").execute().actionGet();
    assertThat(getResponse.isExists(), equalTo(true));
    assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo(fieldValue));
  }
  /**
   * Get the river definition by its name
   *
   * @param name
   * @return
   */
  public T get(String name) {
    if (logger.isDebugEnabled()) logger.debug("get({})", name);
    T river = null;

    if (name != null) {
      GetRequestBuilder rb = new GetRequestBuilder(client, SMDSearchProperties.ES_META_INDEX);
      rb.setType(SMDSearchProperties.ES_META_RIVERS);
      rb.setId(name);

      try {
        GetResponse response = rb.execute().actionGet();
        if (response.isExists()) {
          river = getHelper().toRiver(buildInstance(), response.getSourceAsMap());
        }
      } catch (IndexMissingException e) {
        // Index does not exists, so RIVER does not exist...
      }
    }

    if (logger.isDebugEnabled()) logger.debug("/get({})={}", name, river);
    return river;
  }
 public final Map<String, GetField> getModelFields(
     final ModelIdT id, final Logger logger, final Marker logMarker, final String... fieldNames)
     throws InvalidModelException, IoExceptionT, NoSuchModelExceptionT {
   logger.debug(logMarker, "getting model {} from index {}", id, indexName);
   try {
     final GetResponse response =
         client
             .prepareGet(indexName, documentType, id.toString())
             .setFields(fieldNames)
             .execute()
             .actionGet();
     if (!response.isExists()) {
       throw exceptionFactory.newNoSuchModelException(id);
     }
     return response.getFields();
   } catch (final IndexNotFoundException e) {
     logger.warn(logMarker, "tried to get model {} from missing index {}", id, indexName);
     throw exceptionFactory.newNoSuchModelException(id);
   } catch (final ElasticsearchException e) {
     throw exceptionFactory.newIoException(
         e, String.format("error getting model %s from index %s", id, indexName));
   }
 }
  @Before
  public void setupElasticAndRedis() throws Exception {

    Settings globalSettings = settingsBuilder().loadFromClasspath("settings.yml").build();
    String json = copyToStringFromClasspath("/simple-redis-river.json");
    Settings riverSettings = settingsBuilder().loadFromSource(json).build();

    index = riverSettings.get("index.name");
    channel = riverSettings.get("redis.channels").split(",")[0];

    String hostname = riverSettings.get("redis.hostname");
    int port = riverSettings.getAsInt("redis.port", 6379);
    logger.debug("Connecting to Redis [hostname={} port={}]...", hostname, port);
    jedis = new Jedis(hostname, port, 0);
    logger.debug("... connected");

    logger.debug("Starting local elastic...");
    node = nodeBuilder().local(true).settings(globalSettings).node();

    logger.info("Create river [{}]", river);
    node.client().prepareIndex("_river", river, "_meta").setSource(json).execute().actionGet();

    logger.debug("Running Cluster Health");
    ClusterHealthResponse clusterHealth =
        node.client()
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForGreenStatus())
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());

    GetResponse response = node.client().prepareGet("_river", river, "_meta").execute().actionGet();
    assertTrue(response.isExists());

    logger.debug("...elasticized ok");
  }
 public final ModelT getModelById(
     final ModelIdT id,
     final Logger logger,
     final Marker logMarker,
     final ModelFactory<ModelEntryT> modelFactory)
     throws InvalidModelException, IoExceptionT, NoSuchModelExceptionT {
   logger.debug(logMarker, "getting model {} from index {}", id, indexName);
   try {
     final GetResponse response =
         client.prepareGet(indexName, documentType, id.toString()).execute().actionGet();
     if (!response.isExists()) {
       throw exceptionFactory.newNoSuchModelException(id);
     }
     return modelFactory
         .createModelEntry(response.getId(), response.getSourceAsBytesRef())
         .getModel();
   } catch (final IndexNotFoundException e) {
     logger.warn(logMarker, "tried to get model {} from missing index {}", id, indexName);
     throw exceptionFactory.newNoSuchModelException(id);
   } catch (final ElasticsearchException e) {
     throw exceptionFactory.newIoException(
         e, String.format("error getting model %s from index %s", id, indexName));
   }
 }
  @Test
  public void simpleGetTests() {
    client.admin().indices().prepareDelete().execute().actionGet();

    client
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
        .execute()
        .actionGet();

    ClusterHealthResponse clusterHealth =
        client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));

    GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(false));

    logger.info("--> index doc 1");
    client
        .prepareIndex("test", "type1", "1")
        .setSource("field1", "value1", "field2", "value2")
        .execute()
        .actionGet();

    logger.info("--> realtime get 1");
    response = client.prepareGet("test", "type1", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
    assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));

    logger.info("--> realtime get 1 (no source)");
    response =
        client
            .prepareGet("test", "type1", "1")
            .setFields(Strings.EMPTY_ARRAY)
            .execute()
            .actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsBytes(), nullValue());

    logger.info("--> realtime get 1 (no type)");
    response = client.prepareGet("test", null, "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
    assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));

    logger.info("--> non realtime get 1");
    response = client.prepareGet("test", "type1", "1").setRealtime(false).execute().actionGet();
    assertThat(response.isExists(), equalTo(false));

    logger.info("--> realtime fetch of field (requires fetching parsing source)");
    response = client.prepareGet("test", "type1", "1").setFields("field1").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsBytes(), nullValue());
    assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1"));
    assertThat(response.getField("field2"), nullValue());

    logger.info("--> flush the index, so we load it from it");
    client.admin().indices().prepareFlush().execute().actionGet();

    logger.info("--> realtime get 1 (loaded from index)");
    response = client.prepareGet("test", "type1", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
    assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));

    logger.info("--> non realtime get 1 (loaded from index)");
    response = client.prepareGet("test", "type1", "1").setRealtime(false).execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
    assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));

    logger.info("--> realtime fetch of field (loaded from index)");
    response = client.prepareGet("test", "type1", "1").setFields("field1").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsBytes(), nullValue());
    assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1"));
    assertThat(response.getField("field2"), nullValue());

    logger.info("--> update doc 1");
    client
        .prepareIndex("test", "type1", "1")
        .setSource("field1", "value1_1", "field2", "value2_1")
        .execute()
        .actionGet();

    logger.info("--> realtime get 1");
    response = client.prepareGet("test", "type1", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1_1"));
    assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2_1"));

    logger.info("--> update doc 1 again");
    client
        .prepareIndex("test", "type1", "1")
        .setSource("field1", "value1_2", "field2", "value2_2")
        .execute()
        .actionGet();

    response = client.prepareGet("test", "type1", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1_2"));
    assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2_2"));

    DeleteResponse deleteResponse =
        client.prepareDelete("test", "type1", "1").execute().actionGet();
    assertThat(deleteResponse.isNotFound(), equalTo(false));

    response = client.prepareGet("test", "type1", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(false));
  }
  @Test
  public void testSimpleTTL() throws Exception {
    assertAcked(
        prepareCreate("test")
            .addMapping(
                "type1",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("type1")
                    .startObject("_timestamp")
                    .field("enabled", true)
                    .field("store", "yes")
                    .endObject()
                    .startObject("_ttl")
                    .field("enabled", true)
                    .endObject()
                    .endObject()
                    .endObject())
            .addMapping(
                "type2",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("type2")
                    .startObject("_timestamp")
                    .field("enabled", true)
                    .field("store", "yes")
                    .endObject()
                    .startObject("_ttl")
                    .field("enabled", true)
                    .field("default", "1d")
                    .endObject()
                    .endObject()
                    .endObject()));
    ensureYellow("test");

    final NumShards test = getNumShards("test");

    long providedTTLValue = 3000;
    logger.info("--> checking ttl");
    // Index one doc without routing, one doc with routing, one doc with not TTL and no default and
    // one doc with default TTL
    long now = System.currentTimeMillis();
    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type1", "1")
            .setSource("field1", "value1")
            .setTimestamp(String.valueOf(now))
            .setTTL(providedTTLValue)
            .setRefresh(true)
            .get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client()
            .prepareIndex("test", "type1", "with_routing")
            .setSource("field1", "value1")
            .setTimestamp(String.valueOf(now))
            .setTTL(providedTTLValue)
            .setRouting("routing")
            .setRefresh(true)
            .get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client().prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").get();
    assertThat(indexResponse.isCreated(), is(true));

    // realtime get check
    long currentTime = System.currentTimeMillis();
    GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
    long ttl0;
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // verify the ttl is still decreasing when going to the replica
    currentTime = System.currentTimeMillis();
    getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // non realtime get (stored)
    currentTime = System.currentTimeMillis();
    getResponse =
        client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // non realtime get going the replica
    currentTime = System.currentTimeMillis();
    getResponse =
        client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }

    // no TTL provided so no TTL fetched
    getResponse =
        client()
            .prepareGet("test", "type1", "no_ttl")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.getField("_ttl"), nullValue());
    // no TTL provided make sure it has default TTL
    getResponse =
        client()
            .prepareGet("test", "type2", "default_ttl")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
    assertThat(ttl0, greaterThan(0L));

    IndicesStatsResponse response =
        client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
    assertThat(
        response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount(),
        equalTo(0L));

    // make sure the purger has done its job for all indexed docs that are expired
    long shouldBeExpiredDate = now + providedTTLValue + PURGE_INTERVAL + 2000;
    currentTime = System.currentTimeMillis();
    if (shouldBeExpiredDate - currentTime > 0) {
      Thread.sleep(shouldBeExpiredDate - currentTime);
    }

    // We can't assume that after waiting for ttl + purgeInterval (waitTime) that the document have
    // actually been deleted.
    // The ttl purging happens in the background in a different thread, and might not have been
    // completed after waiting for waitTime.
    // But we can use index statistics' delete count to be sure that deletes have been executed,
    // that must be incremented before
    // ttl purging has finished.
    logger.info("--> checking purger");
    assertThat(
        awaitBusy(
            new Predicate<Object>() {
              @Override
              public boolean apply(Object input) {
                if (rarely()) {
                  client().admin().indices().prepareFlush("test").get();
                } else if (rarely()) {
                  client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get();
                }
                IndicesStatsResponse response =
                    client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
                // TTL deletes two docs, but it is indexed in the primary shard and replica shard.
                return response
                        .getIndices()
                        .get("test")
                        .getTotal()
                        .getIndexing()
                        .getTotal()
                        .getDeleteCount()
                    == 2L * test.dataCopies;
              }
            },
            5,
            TimeUnit.SECONDS),
        equalTo(true));

    // realtime get check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    // replica realtime get check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));

    // Need to run a refresh, in order for the non realtime get to work.
    client().admin().indices().prepareRefresh("test").execute().actionGet();

    // non realtime get (stored) check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    // non realtime get going the replica check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
  }
  @Test
  public void testGetDocWithMultivaluedFields() throws Exception {
    try {
      client.admin().indices().prepareDelete("test").execute().actionGet();
    } catch (Exception e) {
      // fine
    }
    String mapping1 =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type1")
            .startObject("properties")
            .startObject("field")
            .field("type", "string")
            .field("store", "yes")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .string();
    String mapping2 =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type2")
            .startObject("properties")
            .startObject("field")
            .field("type", "string")
            .field("store", "yes")
            .endObject()
            .endObject()
            .startObject("_source")
            .field("enabled", false)
            .endObject()
            .endObject()
            .endObject()
            .string();
    client
        .admin()
        .indices()
        .prepareCreate("test")
        .addMapping("type1", mapping1)
        .addMapping("type2", mapping2)
        .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
        .execute()
        .actionGet();

    ClusterHealthResponse clusterHealth =
        client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));

    GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(false));
    response = client.prepareGet("test", "type2", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(false));

    client
        .prepareIndex("test", "type1", "1")
        .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
        .execute()
        .actionGet();

    client
        .prepareIndex("test", "type2", "1")
        .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
        .execute()
        .actionGet();

    response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getId(), equalTo("1"));
    assertThat(response.getType(), equalTo("type1"));
    assertThat(response.getFields().size(), equalTo(1));
    assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
    assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
        equalTo("1"));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
        equalTo("2"));

    response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getType(), equalTo("type2"));
    assertThat(response.getId(), equalTo("1"));
    assertThat(response.getFields().size(), equalTo(1));
    assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
    assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
        equalTo("1"));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
        equalTo("2"));

    // Now test values being fetched from stored fields.
    client.admin().indices().prepareRefresh("test").execute().actionGet();
    response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getId(), equalTo("1"));
    assertThat(response.getFields().size(), equalTo(1));
    assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
    assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
        equalTo("1"));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
        equalTo("2"));

    response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getId(), equalTo("1"));
    assertThat(response.getFields().size(), equalTo(1));
    assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
    assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
        equalTo("1"));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
        equalTo("2"));
  }
  @Test
  @Slow
  public void testSnapshotOperations() throws Exception {
    startNode("server1", getClassDefaultSettings());

    // get the environment, so we can clear the work dir when needed
    Environment environment =
        ((InternalNode) node("server1")).injector().getInstance(Environment.class);

    logger.info("Running Cluster Health (waiting for node to startup properly)");
    ClusterHealthResponse clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForGreenStatus())
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));

    // Translog tests

    logger.info("Creating index [{}]", "test");
    client("server1").admin().indices().prepareCreate("test").execute().actionGet();

    // create a mapping
    PutMappingResponse putMappingResponse =
        client("server1")
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType("type1")
            .setSource(mappingSource())
            .execute()
            .actionGet();
    assertThat(putMappingResponse.isAcknowledged(), equalTo(true));

    // verify that mapping is there
    ClusterStateResponse clusterState =
        client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
    assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue());

    // create two and delete the first
    logger.info("Indexing #1");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")))
        .actionGet();
    logger.info("Indexing #2");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("2").source(source("2", "test")))
        .actionGet();

    // perform snapshot to the index
    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Deleting #1");
    client("server1").delete(deleteRequest("test").type("type1").id("1")).actionGet();

    // perform snapshot to the index
    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    // do it again, it should be a no op
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (only translog should be populated)");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));

    // verify that mapping is there
    clusterState = client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
    assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue());

    logger.info("Getting #1, should not exists");
    GetResponse getResponse =
        client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    logger.info("Getting #2");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));

    // Now flush and add some data (so we have index recovery as well)
    logger.info(
        "Flushing, so we have actual content in the index files (#2 should be in the index)");
    client("server1").admin().indices().flush(flushRequest("test")).actionGet();
    logger.info("Indexing #3, so we have something in the translog as well");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("3").source(source("3", "test")))
        .actionGet();

    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (both index and translog) and reuse work dir");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (from the translog)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));

    logger.info("Closing the server");
    closeNode("server1");
    logger.info("Clearing cluster data dir, so there will be a full recovery from the gateway");
    FileSystemUtils.deleteRecursively(environment.dataWithClusterFiles());
    logger.info(
        "Starting the server, should recover from the gateway (both index and translog) without reusing work dir");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (from the translog)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));

    logger.info(
        "Flushing, so we have actual content in the index files (#3 should be in the index now as well)");
    client("server1").admin().indices().flush(flushRequest("test")).actionGet();

    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (just from the index, nothing in the translog)");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));

    logger.info("Deleting the index");
    client("server1").admin().indices().delete(deleteIndexRequest("test")).actionGet();
  }