/**
   * Basic test using Index & Realtime Get with internal versioning. This test ensures routing
   * works correctly across versions.
   */
  public void testInternalVersion() throws Exception {
    createIndex("test");
    final boolean routing = randomBoolean();
    int numDocs = randomIntBetween(10, 20);
    for (int i = 0; i < numDocs; i++) {
      String routingKey = routing ? randomRealisticUnicodeOfLength(10) : null;
      String id = Integer.toString(i);
      assertThat(
          id,
          client()
              .prepareIndex("test", "type1", id)
              .setRouting(routingKey)
              .setSource("field1", English.intToEnglish(i))
              .get()
              .isCreated(),
          is(true));
      GetResponse get =
          client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(1).get();
      assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
      assertThat(get.getVersion(), equalTo(1L));
      client()
          .prepareIndex("test", "type1", id)
          .setRouting(routingKey)
          .setSource("field1", English.intToEnglish(i))
          .execute()
          .actionGet();
      get = client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(2).get();
      assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
      assertThat(get.getVersion(), equalTo(2L));
    }

    assertVersionCreated(compatibilityVersion(), "test");
  }
 /**
  * Basic test using Index &amp; Realtime Get with external versioning. This test ensures routing
  * works correctly across versions.
  */
 public void testExternalVersion() throws Exception {
   createIndex("test");
   final boolean routing = randomBoolean();
   int numDocs = randomIntBetween(10, 20);
   for (int i = 0; i < numDocs; i++) {
     String id = Integer.toString(i);
     String routingKey = routing ? randomRealisticUnicodeOfLength(10) : null;
     final long version = randomIntBetween(0, Integer.MAX_VALUE);
     client()
         .prepareIndex("test", "type1", id)
         .setRouting(routingKey)
         .setVersion(version)
         .setVersionType(VersionType.EXTERNAL)
         .setSource("field1", English.intToEnglish(i))
         .get();
     GetResponse get =
         client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(version).get();
     assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
     assertThat(get.getVersion(), equalTo(version));
     final long nextVersion = version + randomIntBetween(0, Integer.MAX_VALUE);
     client()
         .prepareIndex("test", "type1", id)
         .setRouting(routingKey)
         .setVersion(nextVersion)
         .setVersionType(VersionType.EXTERNAL)
         .setSource("field1", English.intToEnglish(i))
         .get();
     get =
         client()
             .prepareGet("test", "type1", id)
             .setRouting(routingKey)
             .setVersion(nextVersion)
             .get();
     assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
     assertThat(get.getVersion(), equalTo(nextVersion));
   }
 }
Beispiel #3
0
  @Test
  public void testGetSlash() {
    final String id = "get-test/id-2";
    final IndexRequestBuilder indexRequestBuilder =
        restClient().prepareIndex(index, type, id).setSource("field", "value").setRefresh(true);
    indexRequestBuilder.execute().actionGet();

    final GetRequestBuilder getRequestBuilder = restClient().prepareGet(index, type, id);
    final ListenableActionFuture<GetResponse> execute1 = getRequestBuilder.execute();
    final GetResponse get = execute1.actionGet();
    assertEquals(get.getIndex(), index);
    assertEquals(get.getType(), type);
    assertEquals(get.getId(), id);
    assertEquals(get.getVersion(), 1);
    assertTrue(get.getFields().isEmpty());
    assertEquals(get.getSource().get("field"), "value");
  }
  @Test
  @Slow
  public void stressUpdateDeleteConcurrency() throws Exception {
    // We create an index with merging disabled so that deletes don't get merged away
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .addMapping(
            "type1",
            XContentFactory.jsonBuilder()
                .startObject()
                .startObject("type1")
                .startObject("_timestamp")
                .field("enabled", true)
                .field("store", "yes")
                .endObject()
                .startObject("_ttl")
                .field("enabled", true)
                .field("store", "yes")
                .endObject()
                .endObject()
                .endObject())
        .setSettings(
            ImmutableSettings.builder()
                .put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class))
        .execute()
        .actionGet();
    ensureGreen();

    final int numberOfThreads = scaledRandomIntBetween(5, 10);
    final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
    final int numberOfUpdatesPerId = scaledRandomIntBetween(100, 200);
    final int retryOnConflict = randomIntBetween(0, 1);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();

    final class UpdateThread extends Thread {
      final Map<Integer, Integer> failedMap = new HashMap<>();
      final int numberOfIds;
      final int updatesPerId;
      final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
      final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
      private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);
      private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);

      public UpdateThread(int numberOfIds, int updatesPerId) {
        this.numberOfIds = numberOfIds;
        this.updatesPerId = updatesPerId;
      }

      final class UpdateListener implements ActionListener<UpdateResponse> {
        int id;

        public UpdateListener(int id) {
          this.id = id;
        }

        @Override
        public void onResponse(UpdateResponse updateResponse) {
          updateRequestsOutstanding.release(1);
        }

        @Override
        public void onFailure(Throwable e) {
          synchronized (failedMap) {
            incrementMapValue(id, failedMap);
          }
          updateRequestsOutstanding.release(1);
        }
      }

      final class DeleteListener implements ActionListener<DeleteResponse> {
        int id;

        public DeleteListener(int id) {
          this.id = id;
        }

        @Override
        public void onResponse(DeleteResponse deleteResponse) {
          deleteRequestsOutstanding.release(1);
        }

        @Override
        public void onFailure(Throwable e) {
          synchronized (failedMap) {
            incrementMapValue(id, failedMap);
          }
          deleteRequestsOutstanding.release(1);
        }
      }

      @Override
      public void run() {
        try {
          startLatch.await();
          for (int j = 0; j < numberOfIds; j++) {
            for (int k = 0; k < numberOfUpdatesPerId; ++k) {
              updateRequestsOutstanding.acquire();
              UpdateRequest ur =
                  client()
                      .prepareUpdate("test", "type1", Integer.toString(j))
                      .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
                      .setRetryOnConflict(retryOnConflict)
                      .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                      .setListenerThreaded(false)
                      .request();
              client().update(ur, new UpdateListener(j));

              deleteRequestsOutstanding.acquire();
              DeleteRequest dr =
                  client()
                      .prepareDelete("test", "type1", Integer.toString(j))
                      .setListenerThreaded(false)
                      .setOperationThreaded(false)
                      .request();
              client().delete(dr, new DeleteListener(j));
            }
          }
        } catch (Throwable e) {
          logger.error("Something went wrong", e);
          failures.add(e);
        } finally {
          try {
            waitForOutstandingRequests(
                TimeValue.timeValueSeconds(60),
                updateRequestsOutstanding,
                maxUpdateRequests,
                "Update");
            waitForOutstandingRequests(
                TimeValue.timeValueSeconds(60),
                deleteRequestsOutstanding,
                maxDeleteRequests,
                "Delete");
          } catch (ElasticsearchTimeoutException ete) {
            failures.add(ete);
          }
          latch.countDown();
        }
      }

      private void incrementMapValue(int j, Map<Integer, Integer> map) {
        if (!map.containsKey(j)) {
          map.put(j, 0);
        }
        map.put(j, map.get(j) + 1);
      }

      private void waitForOutstandingRequests(
          TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
        long start = System.currentTimeMillis();
        do {
          long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
          logger.info(
              "[{}] going to try and aquire [{}] in [{}]ms [{}] available to aquire right now",
              name,
              maxRequests,
              msRemaining,
              requestsOutstanding.availablePermits());
          try {
            requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
            return;
          } catch (InterruptedException ie) {
            // Just keep swimming
          }
        } while ((System.currentTimeMillis() - start) < timeOut.getMillis());
        throw new ElasticsearchTimeoutException(
            "Requests were still outstanding after the timeout ["
                + timeOut
                + "] for type ["
                + name
                + "]");
      }
    }
    final List<UpdateThread> threads = new ArrayList<>();

    for (int i = 0; i < numberOfThreads; i++) {
      UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId);
      ut.start();
      threads.add(ut);
    }

    startLatch.countDown();
    latch.await();

    for (UpdateThread ut : threads) {
      ut.join(); // Threads should have finished because of the latch.await
    }

    // If are no errors every request recieved a response otherwise the test would have timedout
    // aquiring the request outstanding semaphores.
    for (Throwable throwable : failures) {
      logger.info("Captured failure on concurrent update:", throwable);
    }

    assertThat(failures.size(), equalTo(0));

    // Upsert all the ids one last time to make sure they are available at get time
    // This means that we add 1 to the expected versions and attempts
    // All the previous operations should be complete or failed at this point
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
      UpdateResponse ur =
          client()
              .prepareUpdate("test", "type1", Integer.toString(i))
              .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
              .setRetryOnConflict(Integer.MAX_VALUE)
              .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
              .execute()
              .actionGet();
    }

    refresh();

    for (int i = 0; i < numberOfIdsPerThread; ++i) {
      int totalFailures = 0;
      GetResponse response =
          client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
      if (response.isExists()) {
        assertThat(response.getId(), equalTo(Integer.toString(i)));
        int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
        for (UpdateThread ut : threads) {
          if (ut.failedMap.containsKey(i)) {
            totalFailures += ut.failedMap.get(i);
          }
        }
        expectedVersion -= totalFailures;
        logger.error(
            "Actual version [{}] Expected version [{}] Total failures [{}]",
            response.getVersion(),
            expectedVersion,
            totalFailures);
        assertThat(response.getVersion(), equalTo((long) expectedVersion));
        assertThat(
            response.getVersion() + totalFailures,
            equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
      }
    }
  }
  @Test
  @Slow
  public void testConcurrentUpdateWithRetryOnConflict() throws Exception {
    final boolean useBulkApi = randomBoolean();
    createIndex();
    ensureGreen();

    int numberOfThreads = scaledRandomIntBetween(2, 5);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 10000);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();
    for (int i = 0; i < numberOfThreads; i++) {
      Runnable r =
          new Runnable() {

            @Override
            public void run() {
              try {
                startLatch.await();
                for (int i = 0; i < numberOfUpdatesPerThread; i++) {
                  if (useBulkApi) {
                    UpdateRequestBuilder updateRequestBuilder =
                        client()
                            .prepareUpdate("test", "type1", Integer.toString(i))
                            .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
                            .setRetryOnConflict(Integer.MAX_VALUE)
                            .setUpsert(jsonBuilder().startObject().field("field", 1).endObject());
                    client().prepareBulk().add(updateRequestBuilder).execute().actionGet();
                  } else {
                    client()
                        .prepareUpdate("test", "type1", Integer.toString(i))
                        .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
                        .setRetryOnConflict(Integer.MAX_VALUE)
                        .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                        .execute()
                        .actionGet();
                  }
                }
              } catch (Throwable e) {
                failures.add(e);
              } finally {
                latch.countDown();
              }
            }
          };
      new Thread(r).start();
    }
    startLatch.countDown();
    latch.await();
    for (Throwable throwable : failures) {
      logger.info("Captured failure on concurrent update:", throwable);
    }
    assertThat(failures.size(), equalTo(0));
    for (int i = 0; i < numberOfUpdatesPerThread; i++) {
      GetResponse response =
          client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
      assertThat(response.getId(), equalTo(Integer.toString(i)));
      assertThat(response.isExists(), equalTo(true));
      assertThat(response.getVersion(), equalTo((long) numberOfThreads));
      assertThat((Integer) response.getSource().get("field"), equalTo(numberOfThreads));
    }
  }
  @Test
  public void testVersionedUpdate() throws Exception {
    createIndex("test");
    ensureGreen();

    index("test", "type", "1", "text", "value"); // version is now 1

    assertThrows(
        client()
            .prepareUpdate("test", "type", "1")
            .setScript("ctx._source.text = 'v2'", ScriptService.ScriptType.INLINE)
            .setVersion(2)
            .execute(),
        VersionConflictEngineException.class);

    client()
        .prepareUpdate("test", "type", "1")
        .setScript("ctx._source.text = 'v2'", ScriptService.ScriptType.INLINE)
        .setVersion(1)
        .get();
    assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(2l));

    // and again with a higher version..
    client()
        .prepareUpdate("test", "type", "1")
        .setScript("ctx._source.text = 'v3'", ScriptService.ScriptType.INLINE)
        .setVersion(2)
        .get();

    assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(3l));

    // after delete
    client().prepareDelete("test", "type", "1").get();
    assertThrows(
        client()
            .prepareUpdate("test", "type", "1")
            .setScript("ctx._source.text = 'v2'", ScriptService.ScriptType.INLINE)
            .setVersion(3)
            .execute(),
        DocumentMissingException.class);

    // external versioning
    client()
        .prepareIndex("test", "type", "2")
        .setSource("text", "value")
        .setVersion(10)
        .setVersionType(VersionType.EXTERNAL)
        .get();

    assertThrows(
        client()
            .prepareUpdate("test", "type", "2")
            .setScript("ctx._source.text = 'v2'", ScriptService.ScriptType.INLINE)
            .setVersion(2)
            .setVersionType(VersionType.EXTERNAL)
            .execute(),
        ActionRequestValidationException.class);

    // upserts - the combination with versions is a bit weird. Test are here to ensure we do not
    // change our behavior unintentionally

    // With internal versions, tt means "if object is there with version X, update it or explode. If
    // it is not there, index.
    client()
        .prepareUpdate("test", "type", "3")
        .setScript("ctx._source.text = 'v2'", ScriptService.ScriptType.INLINE)
        .setVersion(10)
        .setUpsert("{ \"text\": \"v0\" }")
        .get();
    GetResponse get = get("test", "type", "3");
    assertThat(get.getVersion(), equalTo(1l));
    assertThat((String) get.getSource().get("text"), equalTo("v0"));

    // With force version
    client()
        .prepareUpdate("test", "type", "4")
        .setScript("ctx._source.text = 'v2'", ScriptService.ScriptType.INLINE)
        .setVersion(10)
        .setVersionType(VersionType.FORCE)
        .setUpsert("{ \"text\": \"v0\" }")
        .get();

    get = get("test", "type", "4");
    assertThat(get.getVersion(), equalTo(10l));
    assertThat((String) get.getSource().get("text"), equalTo("v0"));

    // retry on conflict is rejected:

    assertThrows(
        client().prepareUpdate("test", "type", "1").setVersion(10).setRetryOnConflict(5),
        ActionRequestValidationException.class);
  }