Beispiel #1
0
  @Override
  public final String esGet(String index, String type, String id) {
    try {
      final GetResponse gr = client.prepareGet(index, type, id).execute().actionGet();

      if (!gr.exists()) return "Doesn't exist";

      return gr.getSourceAsString();
    } catch (ElasticSearchException e) {
      log.debug("ElasticSearchException {}", e);

      return e.getMessage();
    }
  }
Beispiel #2
0
  @Test
  public void testUpdate() throws Exception {
    createIndex();
    ClusterHealthResponse clusterHealth =
        client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
    assertThat(clusterHealth.timedOut(), equalTo(false));
    assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));

    try {
      client
          .prepareUpdate("test", "type1", "1")
          .setScript("ctx._source.field++")
          .execute()
          .actionGet();
      assert false;
    } catch (DocumentMissingException e) {
      // all is well
    }

    client.prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();

    UpdateResponse updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setScript("ctx._source.field += 1")
            .execute()
            .actionGet();
    assertThat(updateResponse.version(), equalTo(2L));

    for (int i = 0; i < 5; i++) {
      GetResponse getResponse = client.prepareGet("test", "type1", "1").execute().actionGet();
      assertThat(getResponse.sourceAsMap().get("field").toString(), equalTo("2"));
    }

    updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setScript("ctx._source.field += count")
            .addScriptParam("count", 3)
            .execute()
            .actionGet();
    assertThat(updateResponse.version(), equalTo(3L));

    for (int i = 0; i < 5; i++) {
      GetResponse getResponse = client.prepareGet("test", "type1", "1").execute().actionGet();
      assertThat(getResponse.sourceAsMap().get("field").toString(), equalTo("5"));
    }

    // check noop
    updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setScript("ctx.op = 'none'")
            .execute()
            .actionGet();
    assertThat(updateResponse.version(), equalTo(3L));

    for (int i = 0; i < 5; i++) {
      GetResponse getResponse = client.prepareGet("test", "type1", "1").execute().actionGet();
      assertThat(getResponse.sourceAsMap().get("field").toString(), equalTo("5"));
    }

    // check delete
    updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setScript("ctx.op = 'delete'")
            .execute()
            .actionGet();
    assertThat(updateResponse.version(), equalTo(4L));

    for (int i = 0; i < 5; i++) {
      GetResponse getResponse = client.prepareGet("test", "type1", "1").execute().actionGet();
      assertThat(getResponse.exists(), equalTo(false));
    }

    // check percolation
    client.prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
    logger.info("--> register a query");
    client
        .prepareIndex("_percolator", "test", "1")
        .setSource(jsonBuilder().startObject().field("query", termQuery("field", 2)).endObject())
        .setRefresh(true)
        .execute()
        .actionGet();
    updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setScript("ctx._source.field += 1")
            .setPercolate("*")
            .execute()
            .actionGet();
    assertThat(updateResponse.matches().size(), equalTo(1));

    // check TTL is kept after an update without TTL
    client
        .prepareIndex("test", "type1", "2")
        .setSource("field", 1)
        .setTTL(86400000L)
        .setRefresh(true)
        .execute()
        .actionGet();
    GetResponse getResponse =
        client.prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
    long ttl = ((Number) getResponse.field("_ttl").value()).longValue();
    assertThat(ttl, greaterThan(0L));
    client
        .prepareUpdate("test", "type1", "2")
        .setScript("ctx._source.field += 1")
        .execute()
        .actionGet();
    getResponse = client.prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
    ttl = ((Number) getResponse.field("_ttl").value()).longValue();
    assertThat(ttl, greaterThan(0L));

    // check TTL update
    client
        .prepareUpdate("test", "type1", "2")
        .setScript("ctx._ttl = 3600000")
        .execute()
        .actionGet();
    getResponse = client.prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
    ttl = ((Number) getResponse.field("_ttl").value()).longValue();
    assertThat(ttl, greaterThan(0L));
    assertThat(ttl, lessThanOrEqualTo(3600000L));

    // check timestamp update
    client
        .prepareIndex("test", "type1", "3")
        .setSource("field", 1)
        .setRefresh(true)
        .execute()
        .actionGet();
    client
        .prepareUpdate("test", "type1", "3")
        .setScript("ctx._timestamp = \"2009-11-15T14:12:12\"")
        .execute()
        .actionGet();
    getResponse =
        client.prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet();
    long timestamp = ((Number) getResponse.field("_timestamp").value()).longValue();
    assertThat(timestamp, equalTo(1258294332000L));

    // check fields parameter
    client.prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
    updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setScript("ctx._source.field += 1")
            .setFields("_source", "field")
            .execute()
            .actionGet();
    assertThat(updateResponse.getResult(), notNullValue());
    assertThat(updateResponse.getResult().sourceRef(), notNullValue());
    assertThat(updateResponse.getResult().field("field").value(), notNullValue());

    // check updates without script
    // add new field
    client.prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
    updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject())
            .execute()
            .actionGet();
    for (int i = 0; i < 5; i++) {
      getResponse = client.prepareGet("test", "type1", "1").execute().actionGet();
      assertThat(getResponse.sourceAsMap().get("field").toString(), equalTo("1"));
      assertThat(getResponse.sourceAsMap().get("field2").toString(), equalTo("2"));
    }

    // change existing field
    updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject())
            .execute()
            .actionGet();
    for (int i = 0; i < 5; i++) {
      getResponse = client.prepareGet("test", "type1", "1").execute().actionGet();
      assertThat(getResponse.sourceAsMap().get("field").toString(), equalTo("3"));
      assertThat(getResponse.sourceAsMap().get("field2").toString(), equalTo("2"));
    }

    // recursive map
    Map<String, Object> testMap = new HashMap<String, Object>();
    Map<String, Object> testMap2 = new HashMap<String, Object>();
    Map<String, Object> testMap3 = new HashMap<String, Object>();
    testMap3.put("commonkey", testMap);
    testMap3.put("map3", 5);
    testMap2.put("map2", 6);
    testMap.put("commonkey", testMap2);
    testMap.put("map1", 8);

    client.prepareIndex("test", "type1", "1").setSource("map", testMap).execute().actionGet();
    updateResponse =
        client
            .prepareUpdate("test", "type1", "1")
            .setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject())
            .execute()
            .actionGet();
    for (int i = 0; i < 5; i++) {
      getResponse = client.prepareGet("test", "type1", "1").execute().actionGet();
      Map map1 = (Map) getResponse.sourceAsMap().get("map");
      assertThat(map1.size(), equalTo(3));
      assertThat(map1.containsKey("map1"), equalTo(true));
      assertThat(map1.containsKey("map3"), equalTo(true));
      assertThat(map1.containsKey("commonkey"), equalTo(true));
      Map map2 = (Map) map1.get("commonkey");
      assertThat(map2.size(), equalTo(3));
      assertThat(map2.containsKey("map1"), equalTo(true));
      assertThat(map2.containsKey("map2"), equalTo(true));
      assertThat(map2.containsKey("commonkey"), equalTo(true));
    }
  }
  @Test
  public void testSnapshotOperations() throws Exception {
    startNode("server1");

    // get the environment, so we can clear the work dir when needed
    Environment environment =
        ((InternalNode) node("server1")).injector().getInstance(Environment.class);

    logger.info("Running Cluster Health (waiting for node to startup properly)");
    ClusterHealthResponse clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForGreenStatus())
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.status());
    assertThat(clusterHealth.timedOut(), equalTo(false));
    assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.GREEN));

    // Translog tests

    logger.info("Creating index [{}]", "test");
    client("server1").admin().indices().prepareCreate("test").execute().actionGet();

    // create a mapping
    PutMappingResponse putMappingResponse =
        client("server1")
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType("type1")
            .setSource(mappingSource())
            .execute()
            .actionGet();
    assertThat(putMappingResponse.acknowledged(), equalTo(true));

    // verify that mapping is there
    ClusterStateResponse clusterState =
        client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
    assertThat(clusterState.state().metaData().index("test").mapping("type1"), notNullValue());

    // create two and delete the first
    logger.info("Indexing #1");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")))
        .actionGet();
    logger.info("Indexing #2");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("2").source(source("2", "test")))
        .actionGet();

    // perform snapshot to the index
    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Deleting #1");
    client("server1").delete(deleteRequest("test").type("type1").id("1")).actionGet();

    // perform snapshot to the index
    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    // do it again, it should be a no op
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (only translog should be populated)");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.status());
    assertThat(clusterHealth.timedOut(), equalTo(false));
    assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.YELLOW));

    // verify that mapping is there
    clusterState = client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
    assertThat(clusterState.state().metaData().index("test").mapping("type1"), notNullValue());

    logger.info("Getting #1, should not exists");
    GetResponse getResponse =
        client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.exists(), equalTo(false));
    logger.info("Getting #2");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.sourceAsString(), equalTo(source("2", "test")));

    // Now flush and add some data (so we have index recovery as well)
    logger.info(
        "Flushing, so we have actual content in the index files (#2 should be in the index)");
    client("server1").admin().indices().flush(flushRequest("test")).actionGet();
    logger.info("Indexing #3, so we have something in the translog as well");
    client("server1")
        .index(Requests.indexRequest("test").type("type1").id("3").source(source("3", "test")))
        .actionGet();

    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (both index and translog) and reuse work dir");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.status());
    assertThat(clusterHealth.timedOut(), equalTo(false));
    assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.exists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.sourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (from the translog)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.sourceAsString(), equalTo(source("3", "test")));

    logger.info("Closing the server");
    closeNode("server1");
    logger.info("Clearing cluster data dir, so there will be a full recovery from the gateway");
    FileSystemUtils.deleteRecursively(environment.dataWithClusterFiles());
    logger.info(
        "Starting the server, should recover from the gateway (both index and translog) without reusing work dir");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.status());
    assertThat(clusterHealth.timedOut(), equalTo(false));
    assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.exists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.sourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (from the translog)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.sourceAsString(), equalTo(source("3", "test")));

    logger.info(
        "Flushing, so we have actual content in the index files (#3 should be in the index now as well)");
    client("server1").admin().indices().flush(flushRequest("test")).actionGet();

    logger.info("Gateway Snapshot");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
    logger.info("Gateway Snapshot (should be a no op)");
    client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();

    logger.info("Closing the server");
    closeNode("server1");
    logger.info(
        "Starting the server, should recover from the gateway (just from the index, nothing in the translog)");
    startNode("server1");

    logger.info("Running Cluster Health (wait for the shards to startup)");
    clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.status());
    assertThat(clusterHealth.timedOut(), equalTo(false));
    assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.YELLOW));

    logger.info("Getting #1, should not exists");
    getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
    assertThat(getResponse.exists(), equalTo(false));
    logger.info("Getting #2 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
    assertThat(getResponse.sourceAsString(), equalTo(source("2", "test")));
    logger.info("Getting #3 (not from the translog, but from the index)");
    getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
    assertThat(getResponse.sourceAsString(), equalTo(source("3", "test")));

    logger.info("Deleting the index");
    client("server1").admin().indices().delete(deleteIndexRequest("test")).actionGet();
  }
  public static void main(String[] args) throws Exception {

    Settings settings = settingsBuilder().put("gateway.type", "none").build();

    Node node1 = nodeBuilder().settings(settings).node();
    Node node2 = nodeBuilder().settings(settings).node();
    final Node client = nodeBuilder().settings(settings).client(true).node();

    final int NUMBER_OF_DOCS = 10000;
    final int NUMBER_OF_THREADS = 10;
    final long NUMBER_OF_ITERATIONS = SizeValue.parseSizeValue("10k").singles();
    final long DELETE_EVERY = 10;

    final CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS);
    Thread[] threads = new Thread[NUMBER_OF_THREADS];
    for (int i = 0; i < threads.length; i++) {
      threads[i] =
          new Thread() {
            @Override
            public void run() {
              try {
                for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) {
                  if ((i % DELETE_EVERY) == 0) {
                    client
                        .client()
                        .prepareDelete(
                            "test",
                            "type1",
                            Integer.toString(ThreadLocalRandom.current().nextInt(NUMBER_OF_DOCS)))
                        .execute()
                        .actionGet();
                  } else {
                    client
                        .client()
                        .prepareIndex(
                            "test",
                            "type1",
                            Integer.toString(ThreadLocalRandom.current().nextInt(NUMBER_OF_DOCS)))
                        .setSource("field1", "value1")
                        .execute()
                        .actionGet();
                  }
                }
              } finally {
                latch.countDown();
              }
            }
          };
    }

    for (Thread thread : threads) {
      thread.start();
    }

    latch.await();
    System.out.println("done indexing, verifying docs");
    client.client().admin().indices().prepareRefresh().execute().actionGet();
    for (int i = 0; i < NUMBER_OF_DOCS; i++) {
      String id = Integer.toString(i);
      for (int j = 0; j < 5; j++) {
        SearchResponse response =
            client
                .client()
                .prepareSearch()
                .setQuery(QueryBuilders.termQuery("_id", id))
                .execute()
                .actionGet();
        if (response.getHits().totalHits() > 1) {
          System.err.println("[" + i + "] FAIL, HITS [" + response.getHits().totalHits() + "]");
        }
      }
      GetResponse getResponse =
          client.client().prepareGet("test", "type1", id).execute().actionGet();
      if (getResponse.exists()) {
        long version = getResponse.version();
        for (int j = 0; j < 5; j++) {
          getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
          if (!getResponse.exists()) {
            System.err.println("[" + i + "] FAIL, EXISTED, and NOT_EXISTED");
            break;
          }
          if (version != getResponse.version()) {
            System.err.println(
                "["
                    + i
                    + "] FAIL, DIFFERENT VERSIONS: ["
                    + version
                    + "], ["
                    + getResponse.version()
                    + "]");
            break;
          }
        }
      } else {
        for (int j = 0; j < 5; j++) {
          getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
          if (getResponse.exists()) {
            System.err.println("[" + i + "] FAIL, EXISTED, and NOT_EXISTED");
            break;
          }
        }
      }
    }
    System.out.println("done.");

    client.close();
    node1.close();
    node2.close();
  }