public void testIndexNameDateMathExpressions() {
    DateTime now = new DateTime(DateTimeZone.UTC);
    String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now);
    String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1));
    String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2));
    createIndex(index1, index2, index3);

    String dateMathExp1 = "<.marvel-{now/d}>";
    String dateMathExp2 = "<.marvel-{now/d-1d}>";
    String dateMathExp3 = "<.marvel-{now/d-2d}>";
    client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
    client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
    client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
    refresh();

    SearchResponse searchResponse =
        client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
    assertHitCount(searchResponse, 3);
    assertSearchHits(searchResponse, "1", "2", "3");

    GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get();
    assertThat(getResponse.isExists(), is(true));
    assertThat(getResponse.getId(), equalTo("1"));

    getResponse = client().prepareGet(dateMathExp2, "type", "2").get();
    assertThat(getResponse.isExists(), is(true));
    assertThat(getResponse.getId(), equalTo("2"));

    getResponse = client().prepareGet(dateMathExp3, "type", "3").get();
    assertThat(getResponse.isExists(), is(true));
    assertThat(getResponse.getId(), equalTo("3"));

    IndicesStatsResponse indicesStatsResponse =
        client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get();
    assertThat(indicesStatsResponse.getIndex(index1), notNullValue());
    assertThat(indicesStatsResponse.getIndex(index2), notNullValue());
    assertThat(indicesStatsResponse.getIndex(index3), notNullValue());

    DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get();
    assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
    assertThat(deleteResponse.getId(), equalTo("1"));

    deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get();
    assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
    assertThat(deleteResponse.getId(), equalTo("2"));

    deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get();
    assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
    assertThat(deleteResponse.getId(), equalTo("3"));
  }
 @Override
 public void run() {
   long totalRefreshTime = 0;
   int numExecutedRefreshed = 0;
   while (run) {
     long docIdLimit = COUNT;
     for (long docId = 1; run && docId < docIdLimit; ) {
       try {
         for (int j = 0; j < 8; j++) {
           GetResponse getResponse =
               client.prepareGet(indexName, "type1", String.valueOf(++docId)).get();
           client
               .prepareIndex(indexName, "type1", getResponse.getId())
               .setSource(getResponse.getSource())
               .get();
         }
         long startTime = System.currentTimeMillis();
         client.admin().indices().prepareRefresh(indexName).execute().actionGet();
         totalRefreshTime += System.currentTimeMillis() - startTime;
         numExecutedRefreshed++;
         Thread.sleep(500);
       } catch (Throwable e) {
         e.printStackTrace();
       }
     }
   }
   avgRefreshTime = totalRefreshTime / numExecutedRefreshed;
   stopped = true;
 }
 @Override
 public <T> T mapResult(GetResponse response, Class<T> clazz) {
   T result = mapEntity(response.getSourceAsString(), clazz);
   if (result != null) {
     setPersistentEntityId(result, response.getId(), clazz);
   }
   return result;
 }
  @Test
  public void testGet() {

    GetResponse response =
        client
            .prepareGet("asdf2014", "asdf", "1")
            .setOperationThreaded(false)
            .execute()
            .actionGet();
    if (response != null) System.out.println("Id: " + response.getId());
  }
 public T get(String id) {
   GetResponse result = client.prepareGet(index, entity, id).execute().actionGet();
   String source = result.getSourceAsString();
   System.out.println("Loaded SLA: " + source);
   try {
     T entity = mapper.readValue(source, clazz);
     entity.setId(result.getId());
     logger.info("Successfully obtained entity with id [{}]", id);
     return entity;
   } catch (IOException e) {
     logger.error("Error retrieving entity with id " + id, e);
     return null;
   }
 }
 @SuppressWarnings("unchecked")
 protected <T extends ESEntity> T buildEntityFromGetResponse(
     Class<T> entityClass, MultiGetItemResponse item) {
   GetResponse response = item.getResponse();
   if (!response.isExists())
     throw new DaoException(
         String.format(
             "Entity %s does not exist in %s/%s",
             response.getId(), response.getIndex(), response.getType()));
   if (entityClass == null) throw new IllegalArgumentException("Provided Entity class is null");
   else if (entityClass.equals(ESNode.class))
     return (T) ESNode.Builder.buildFromGetReponse(response);
   else if (entityClass.equals(ESWay.class))
     return (T) ESWay.Builder.buildFromGetReponse(response);
   else throw new IllegalArgumentException(entityClass.getSimpleName() + " is not a known Entity");
 }
示例#7
0
  @Test
  public void testGetSlash() {
    final String id = "get-test/id-2";
    final IndexRequestBuilder indexRequestBuilder =
        restClient().prepareIndex(index, type, id).setSource("field", "value").setRefresh(true);
    indexRequestBuilder.execute().actionGet();

    final GetRequestBuilder getRequestBuilder = restClient().prepareGet(index, type, id);
    final ListenableActionFuture<GetResponse> execute1 = getRequestBuilder.execute();
    final GetResponse get = execute1.actionGet();
    assertEquals(get.getIndex(), index);
    assertEquals(get.getType(), type);
    assertEquals(get.getId(), id);
    assertEquals(get.getVersion(), 1);
    assertTrue(get.getFields().isEmpty());
    assertEquals(get.getSource().get("field"), "value");
  }
  @Override
  public Article get(long id) {
    GetResponse getResponse =
        transportClient.prepareGet(INDICE, TYPE, String.valueOf(id)).execute().actionGet();
    if (getResponse.isExists()) {
      Map<String, Object> map = getResponse.getSource();
      String title = (String) map.get(TITLE);
      String summary = (String) map.get(SUMMARY);
      String idStr = getResponse.getId();
      logger.debug("get id : {}", idStr);

      Article article = new Article();
      article.setId(Long.parseLong(idStr));
      article.setTitle(title);
      article.setSummary(summary);
      return article;
    } else {
      return null;
    }
  }
  public void testIndexGetAndDelete() throws ExecutionException, InterruptedException {
    createIndexWithAlias();
    ensureYellow("test");

    int numDocs = iterations(10, 50);
    for (int i = 0; i < numDocs; i++) {
      IndexResponse indexResponse =
          client()
              .prepareIndex(indexOrAlias(), "type", Integer.toString(i))
              .setSource("field", "value-" + i)
              .get();
      assertThat(indexResponse.isCreated(), equalTo(true));
      assertThat(indexResponse.getIndex(), equalTo("test"));
      assertThat(indexResponse.getType(), equalTo("type"));
      assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
    }
    refresh();

    String docId = Integer.toString(randomIntBetween(0, numDocs - 1));
    GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
    assertThat(getResponse.isExists(), equalTo(true));
    assertThat(getResponse.getIndex(), equalTo("test"));
    assertThat(getResponse.getType(), equalTo("type"));
    assertThat(getResponse.getId(), equalTo(docId));

    DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get();
    assertThat(deleteResponse.isFound(), equalTo(true));
    assertThat(deleteResponse.getIndex(), equalTo("test"));
    assertThat(deleteResponse.getType(), equalTo("type"));
    assertThat(deleteResponse.getId(), equalTo(docId));

    getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
    assertThat(getResponse.isExists(), equalTo(false));

    refresh();

    SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get();
    assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1));
  }
示例#10
0
 public final ModelT getModelById(
     final ModelIdT id,
     final Logger logger,
     final Marker logMarker,
     final ModelFactory<ModelEntryT> modelFactory)
     throws InvalidModelException, IoExceptionT, NoSuchModelExceptionT {
   logger.debug(logMarker, "getting model {} from index {}", id, indexName);
   try {
     final GetResponse response =
         client.prepareGet(indexName, documentType, id.toString()).execute().actionGet();
     if (!response.isExists()) {
       throw exceptionFactory.newNoSuchModelException(id);
     }
     return modelFactory
         .createModelEntry(response.getId(), response.getSourceAsBytesRef())
         .getModel();
   } catch (final IndexNotFoundException e) {
     logger.warn(logMarker, "tried to get model {} from missing index {}", id, indexName);
     throw exceptionFactory.newNoSuchModelException(id);
   } catch (final ElasticsearchException e) {
     throw exceptionFactory.newIoException(
         e, String.format("error getting model %s from index %s", id, indexName));
   }
 }
  @Test
  public void testGetDocWithMultivaluedFields() throws Exception {
    try {
      client.admin().indices().prepareDelete("test").execute().actionGet();
    } catch (Exception e) {
      // fine
    }
    String mapping1 =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type1")
            .startObject("properties")
            .startObject("field")
            .field("type", "string")
            .field("store", "yes")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .string();
    String mapping2 =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type2")
            .startObject("properties")
            .startObject("field")
            .field("type", "string")
            .field("store", "yes")
            .endObject()
            .endObject()
            .startObject("_source")
            .field("enabled", false)
            .endObject()
            .endObject()
            .endObject()
            .string();
    client
        .admin()
        .indices()
        .prepareCreate("test")
        .addMapping("type1", mapping1)
        .addMapping("type2", mapping2)
        .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
        .execute()
        .actionGet();

    ClusterHealthResponse clusterHealth =
        client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));

    GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(false));
    response = client.prepareGet("test", "type2", "1").execute().actionGet();
    assertThat(response.isExists(), equalTo(false));

    client
        .prepareIndex("test", "type1", "1")
        .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
        .execute()
        .actionGet();

    client
        .prepareIndex("test", "type2", "1")
        .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
        .execute()
        .actionGet();

    response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getId(), equalTo("1"));
    assertThat(response.getType(), equalTo("type1"));
    assertThat(response.getFields().size(), equalTo(1));
    assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
    assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
        equalTo("1"));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
        equalTo("2"));

    response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getType(), equalTo("type2"));
    assertThat(response.getId(), equalTo("1"));
    assertThat(response.getFields().size(), equalTo(1));
    assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
    assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
        equalTo("1"));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
        equalTo("2"));

    // Now test values being fetched from stored fields.
    client.admin().indices().prepareRefresh("test").execute().actionGet();
    response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getId(), equalTo("1"));
    assertThat(response.getFields().size(), equalTo(1));
    assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
    assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
        equalTo("1"));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
        equalTo("2"));

    response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet();
    assertThat(response.isExists(), equalTo(true));
    assertThat(response.getId(), equalTo("1"));
    assertThat(response.getFields().size(), equalTo(1));
    assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
    assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
        equalTo("1"));
    assertThat(
        ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
        equalTo("2"));
  }
示例#12
0
  @Test
  @Slow
  public void stressUpdateDeleteConcurrency() throws Exception {
    // We create an index with merging disabled so that deletes don't get merged away
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .addMapping(
            "type1",
            XContentFactory.jsonBuilder()
                .startObject()
                .startObject("type1")
                .startObject("_timestamp")
                .field("enabled", true)
                .field("store", "yes")
                .endObject()
                .startObject("_ttl")
                .field("enabled", true)
                .field("store", "yes")
                .endObject()
                .endObject()
                .endObject())
        .setSettings(
            ImmutableSettings.builder()
                .put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class))
        .execute()
        .actionGet();
    ensureGreen();

    final int numberOfThreads = scaledRandomIntBetween(5, 10);
    final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
    final int numberOfUpdatesPerId = scaledRandomIntBetween(100, 200);
    final int retryOnConflict = randomIntBetween(0, 1);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();

    final class UpdateThread extends Thread {
      final Map<Integer, Integer> failedMap = new HashMap<>();
      final int numberOfIds;
      final int updatesPerId;
      final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
      final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
      private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);
      private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);

      public UpdateThread(int numberOfIds, int updatesPerId) {
        this.numberOfIds = numberOfIds;
        this.updatesPerId = updatesPerId;
      }

      final class UpdateListener implements ActionListener<UpdateResponse> {
        int id;

        public UpdateListener(int id) {
          this.id = id;
        }

        @Override
        public void onResponse(UpdateResponse updateResponse) {
          updateRequestsOutstanding.release(1);
        }

        @Override
        public void onFailure(Throwable e) {
          synchronized (failedMap) {
            incrementMapValue(id, failedMap);
          }
          updateRequestsOutstanding.release(1);
        }
      }

      final class DeleteListener implements ActionListener<DeleteResponse> {
        int id;

        public DeleteListener(int id) {
          this.id = id;
        }

        @Override
        public void onResponse(DeleteResponse deleteResponse) {
          deleteRequestsOutstanding.release(1);
        }

        @Override
        public void onFailure(Throwable e) {
          synchronized (failedMap) {
            incrementMapValue(id, failedMap);
          }
          deleteRequestsOutstanding.release(1);
        }
      }

      @Override
      public void run() {
        try {
          startLatch.await();
          for (int j = 0; j < numberOfIds; j++) {
            for (int k = 0; k < numberOfUpdatesPerId; ++k) {
              updateRequestsOutstanding.acquire();
              UpdateRequest ur =
                  client()
                      .prepareUpdate("test", "type1", Integer.toString(j))
                      .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
                      .setRetryOnConflict(retryOnConflict)
                      .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                      .setListenerThreaded(false)
                      .request();
              client().update(ur, new UpdateListener(j));

              deleteRequestsOutstanding.acquire();
              DeleteRequest dr =
                  client()
                      .prepareDelete("test", "type1", Integer.toString(j))
                      .setListenerThreaded(false)
                      .setOperationThreaded(false)
                      .request();
              client().delete(dr, new DeleteListener(j));
            }
          }
        } catch (Throwable e) {
          logger.error("Something went wrong", e);
          failures.add(e);
        } finally {
          try {
            waitForOutstandingRequests(
                TimeValue.timeValueSeconds(60),
                updateRequestsOutstanding,
                maxUpdateRequests,
                "Update");
            waitForOutstandingRequests(
                TimeValue.timeValueSeconds(60),
                deleteRequestsOutstanding,
                maxDeleteRequests,
                "Delete");
          } catch (ElasticsearchTimeoutException ete) {
            failures.add(ete);
          }
          latch.countDown();
        }
      }

      private void incrementMapValue(int j, Map<Integer, Integer> map) {
        if (!map.containsKey(j)) {
          map.put(j, 0);
        }
        map.put(j, map.get(j) + 1);
      }

      private void waitForOutstandingRequests(
          TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
        long start = System.currentTimeMillis();
        do {
          long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
          logger.info(
              "[{}] going to try and aquire [{}] in [{}]ms [{}] available to aquire right now",
              name,
              maxRequests,
              msRemaining,
              requestsOutstanding.availablePermits());
          try {
            requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
            return;
          } catch (InterruptedException ie) {
            // Just keep swimming
          }
        } while ((System.currentTimeMillis() - start) < timeOut.getMillis());
        throw new ElasticsearchTimeoutException(
            "Requests were still outstanding after the timeout ["
                + timeOut
                + "] for type ["
                + name
                + "]");
      }
    }
    final List<UpdateThread> threads = new ArrayList<>();

    for (int i = 0; i < numberOfThreads; i++) {
      UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId);
      ut.start();
      threads.add(ut);
    }

    startLatch.countDown();
    latch.await();

    for (UpdateThread ut : threads) {
      ut.join(); // Threads should have finished because of the latch.await
    }

    // If are no errors every request recieved a response otherwise the test would have timedout
    // aquiring the request outstanding semaphores.
    for (Throwable throwable : failures) {
      logger.info("Captured failure on concurrent update:", throwable);
    }

    assertThat(failures.size(), equalTo(0));

    // Upsert all the ids one last time to make sure they are available at get time
    // This means that we add 1 to the expected versions and attempts
    // All the previous operations should be complete or failed at this point
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
      UpdateResponse ur =
          client()
              .prepareUpdate("test", "type1", Integer.toString(i))
              .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
              .setRetryOnConflict(Integer.MAX_VALUE)
              .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
              .execute()
              .actionGet();
    }

    refresh();

    for (int i = 0; i < numberOfIdsPerThread; ++i) {
      int totalFailures = 0;
      GetResponse response =
          client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
      if (response.isExists()) {
        assertThat(response.getId(), equalTo(Integer.toString(i)));
        int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
        for (UpdateThread ut : threads) {
          if (ut.failedMap.containsKey(i)) {
            totalFailures += ut.failedMap.get(i);
          }
        }
        expectedVersion -= totalFailures;
        logger.error(
            "Actual version [{}] Expected version [{}] Total failures [{}]",
            response.getVersion(),
            expectedVersion,
            totalFailures);
        assertThat(response.getVersion(), equalTo((long) expectedVersion));
        assertThat(
            response.getVersion() + totalFailures,
            equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
      }
    }
  }
示例#13
0
  @Test
  @Slow
  public void testConcurrentUpdateWithRetryOnConflict() throws Exception {
    final boolean useBulkApi = randomBoolean();
    createIndex();
    ensureGreen();

    int numberOfThreads = scaledRandomIntBetween(2, 5);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 10000);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();
    for (int i = 0; i < numberOfThreads; i++) {
      Runnable r =
          new Runnable() {

            @Override
            public void run() {
              try {
                startLatch.await();
                for (int i = 0; i < numberOfUpdatesPerThread; i++) {
                  if (useBulkApi) {
                    UpdateRequestBuilder updateRequestBuilder =
                        client()
                            .prepareUpdate("test", "type1", Integer.toString(i))
                            .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
                            .setRetryOnConflict(Integer.MAX_VALUE)
                            .setUpsert(jsonBuilder().startObject().field("field", 1).endObject());
                    client().prepareBulk().add(updateRequestBuilder).execute().actionGet();
                  } else {
                    client()
                        .prepareUpdate("test", "type1", Integer.toString(i))
                        .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
                        .setRetryOnConflict(Integer.MAX_VALUE)
                        .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                        .execute()
                        .actionGet();
                  }
                }
              } catch (Throwable e) {
                failures.add(e);
              } finally {
                latch.countDown();
              }
            }
          };
      new Thread(r).start();
    }
    startLatch.countDown();
    latch.await();
    for (Throwable throwable : failures) {
      logger.info("Captured failure on concurrent update:", throwable);
    }
    assertThat(failures.size(), equalTo(0));
    for (int i = 0; i < numberOfUpdatesPerThread; i++) {
      GetResponse response =
          client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
      assertThat(response.getId(), equalTo(Integer.toString(i)));
      assertThat(response.isExists(), equalTo(true));
      assertThat(response.getVersion(), equalTo((long) numberOfThreads));
      assertThat((Integer) response.getSource().get("field"), equalTo(numberOfThreads));
    }
  }