@Test
  public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throws Exception {
    client.admin().indices().prepareDelete().execute().actionGet();
    String index = "test";
    String type = "type1";

    String mapping =
        jsonBuilder()
            .startObject()
            .startObject("source_excludes")
            .startObject("_source")
            .array("includes", "included")
            .array("exlcudes", "excluded")
            .endObject()
            .endObject()
            .endObject()
            .string();

    client
        .admin()
        .indices()
        .prepareCreate(index)
        .addMapping(type, mapping)
        .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
        .execute()
        .actionGet();

    client
        .prepareIndex(index, type, "1")
        .setSource(
            jsonBuilder()
                .startObject()
                .field("field", "1", "2")
                .field("included", "should be seen")
                .field("excluded", "should not be seen")
                .endObject())
        .execute()
        .actionGet();

    GetResponse responseBeforeFlush =
        client
            .prepareGet(index, type, "1")
            .setFields("_source", "included", "excluded")
            .execute()
            .actionGet();
    client.admin().indices().prepareFlush(index).execute().actionGet();
    GetResponse responseAfterFlush =
        client
            .prepareGet(index, type, "1")
            .setFields("_source", "included", "excluded")
            .execute()
            .actionGet();

    assertThat(responseBeforeFlush.isExists(), is(true));
    assertThat(responseAfterFlush.isExists(), is(true));
    assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("excluded")));
    assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("field")));
    assertThat(responseBeforeFlush.getSourceAsMap(), hasKey("included"));
    assertThat(responseBeforeFlush.getSourceAsString(), is(responseAfterFlush.getSourceAsString()));
  }
 @Override
 public HashMap<String, String> loadKey(String username) {
   logger.info("loading password for username {}", username);
   HashMap<String, String> ret = new HashMap<>();
   String riverIndexName = getRiverIndexName();
   refreshSearchIndex(riverIndexName);
   GetResponse resp =
       client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet();
   if (resp.isExists()) {
     if (logger.isDebugEnabled()) {
       logger.debug("Password document: {}", resp.getSourceAsString());
     }
     Map<String, Object> newset = resp.getSource();
     Set<String> keys = newset.keySet();
     for (String s : keys) {
       logger.info(
           "Added key {} with a value of {}",
           s,
           XContentMapValues.nodeStringValue(newset.get(s), null));
       ret.put(s, XContentMapValues.nodeStringValue(newset.get(s), null));
     }
   }
   if (ret.isEmpty()) {
     return null;
   }
   return ret;
 }
 @Override
 public <T> T mapResult(GetResponse response, Class<T> clazz) {
   T result = mapEntity(response.getSourceAsString(), clazz);
   if (result != null) {
     setPersistentEntityId(result, response.getId(), clazz);
   }
   return result;
 }
 public T get(String id) {
   GetResponse result = client.prepareGet(index, entity, id).execute().actionGet();
   String source = result.getSourceAsString();
   System.out.println("Loaded SLA: " + source);
   try {
     T entity = mapper.readValue(source, clazz);
     entity.setId(result.getId());
     logger.info("Successfully obtained entity with id [{}]", id);
     return entity;
   } catch (IOException e) {
     logger.error("Error retrieving entity with id " + id, e);
     return null;
   }
 }
Пример #5
0
  @Override
  public final String esGet(String index, String type, String id) {
    try {
      final GetResponse gr = client.prepareGet(index, type, id).execute().actionGet();

      if (!gr.exists()) return "Doesn't exist";

      return gr.getSourceAsString();
    } catch (ElasticSearchException e) {
      log.debug("ElasticSearchException {}", e);

      return e.getMessage();
    }
  }
  /** Reconfigure the river. Must be stopped! */
  public synchronized void reconfigure() {
    if (!closed) throw new IllegalStateException("Remote River must be stopped to reconfigure it!");

    logger.info("reconfiguring Remote River");
    String riverIndexName = getRiverIndexName();
    refreshSearchIndex(riverIndexName);
    GetResponse resp =
        client.prepareGet(riverIndexName, riverName().name(), "_meta").execute().actionGet();
    if (resp.isExists()) {
      if (logger.isDebugEnabled()) {
        logger.debug("Configuration document: {}", resp.getSourceAsString());
      }
      Map<String, Object> newset = resp.getSource();
      configure(newset);
    } else {
      throw new IllegalStateException(
          "Configuration document not found to reconfigure remote river " + riverName().name());
    }
  }
  @Test
  @Slow
  public void testSnapshotOperations() throws Exception {
    startNode("server1", getClassDefaultSettings());

    // get the environment, so we can clear the work dir when needed
    Environment environment =
        ((InternalNode) node("server1")).injector().getInstance(Environment.class);

    logger.info("Running Cluster Health (waiting for node to startup properly)");
    ClusterHealthResponse clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForGreenStatus())
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));

    // Translog tests

    logger.info("Creating index [{}]", "test");
    client("server1").admin().indices().prepareCreate("test").execute().actionGet();

    // create a mapping
    PutMappingResponse putMappingResponse =
        client("server1")
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType("type1")
            .setSource(mappingSource())
            .execute()
            .actionGet();
    assertThat(putMappingResponse.isAcknowledged(), equalTo(true));

    // verify that mapping is there
    ClusterStateResponse clusterState =
        client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
    assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue());

    // create two and delete the first
    logger.info("Indexing #1");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")))
        .actionGet();
    logger.info("Indexing #2");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("2").source(source("2", "test")))
        .actionGet();

    // perform snapshot to the index
    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Deleting #1");
    client("server1").delete(deleteRequest("test").type("type1").id("1")).actionGet();

    // perform snapshot to the index
    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    // do it again, it should be a no op
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (only translog should be populated)");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));

    // verify that mapping is there
    clusterState = client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
    assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue());

    logger.info("Getting #1, should not exists");
    GetResponse getResponse =
        client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    logger.info("Getting #2");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));

    // Now flush and add some data (so we have index recovery as well)
    logger.info(
        "Flushing, so we have actual content in the index files (#2 should be in the index)");
    client("server1").admin().indices().flush(flushRequest("test")).actionGet();
    logger.info("Indexing #3, so we have something in the translog as well");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("3").source(source("3", "test")))
        .actionGet();

    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (both index and translog) and reuse work dir");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (from the translog)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));

    logger.info("Closing the server");
    closeNode("server1");
    logger.info("Clearing cluster data dir, so there will be a full recovery from the gateway");
    FileSystemUtils.deleteRecursively(environment.dataWithClusterFiles());
    logger.info(
        "Starting the server, should recover from the gateway (both index and translog) without reusing work dir");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (from the translog)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));

    logger.info(
        "Flushing, so we have actual content in the index files (#3 should be in the index now as well)");
    client("server1").admin().indices().flush(flushRequest("test")).actionGet();

    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (just from the index, nothing in the translog)");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));

    logger.info("Deleting the index");
    client("server1").admin().indices().delete(deleteIndexRequest("test")).actionGet();
  }
  @Test
  @TestLogging(value = "cluster.service:TRACE,action.get:TRACE")
  public void testSimpleRecovery() throws Exception {
    assertAcked(prepareCreate("test", 1).execute().actionGet(5000));

    NumShards numShards = getNumShards("test");

    logger.info("Running Cluster Health");
    ensureYellow();

    client()
        .index(indexRequest("test").type("type1").id("1").source(source("1", "test")))
        .actionGet();
    FlushResponse flushResponse =
        client().admin().indices().flush(flushRequest("test")).actionGet();
    assertThat(flushResponse.getTotalShards(), equalTo(numShards.totalNumShards));
    assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
    assertThat(flushResponse.getFailedShards(), equalTo(0));
    client()
        .index(indexRequest("test").type("type1").id("2").source(source("2", "test")))
        .actionGet();
    RefreshResponse refreshResponse =
        client().admin().indices().refresh(refreshRequest("test")).actionGet();
    assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards));
    assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
    assertThat(refreshResponse.getFailedShards(), equalTo(0));

    allowNodes("test", 2);

    logger.info("Running Cluster Health");
    ensureGreen();

    GetResponse getResult;

    for (int i = 0; i < 5; i++) {
      getResult =
          client()
              .get(getRequest("test").type("type1").id("1").operationThreaded(false))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("1").operationThreaded(false))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
    }

    // now start another one so we move some primaries
    allowNodes("test", 3);
    Thread.sleep(200);
    logger.info("Running Cluster Health");
    ensureGreen();

    for (int i = 0; i < 5; i++) {
      getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
      getResult =
          client()
              .get(getRequest("test").type("type1").id("2").operationThreaded(true))
              .actionGet(1000);
      assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
    }
  }