Пример #1
0
  public void testForce() throws Exception {
    createIndex("test");
    ensureGreen(
        "test"); // we are testing force here which doesn't work if we are recovering at the same
    // time - zzzzz...
    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(12)
            .setVersionType(VersionType.FORCE)
            .get();
    assertThat(indexResponse.getVersion(), equalTo(12L));

    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_2")
            .setVersion(12)
            .setVersionType(VersionType.FORCE)
            .get();
    assertThat(indexResponse.getVersion(), equalTo(12L));

    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_2")
            .setVersion(14)
            .setVersionType(VersionType.FORCE)
            .get();
    assertThat(indexResponse.getVersion(), equalTo(14L));

    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(13)
            .setVersionType(VersionType.FORCE)
            .get();
    assertThat(indexResponse.getVersion(), equalTo(13L));

    client().admin().indices().prepareRefresh().execute().actionGet();
    if (randomBoolean()) {
      refresh();
    }
    for (int i = 0; i < 10; i++) {
      assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(13L));
    }

    // deleting with a lower version works.
    long v = randomIntBetween(12, 14);
    DeleteResponse deleteResponse =
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(v)
            .setVersionType(VersionType.FORCE)
            .get();
    assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
    assertThat(deleteResponse.getVersion(), equalTo(v));
  }
Пример #2
0
  @Override
  public String importQuestionnareOds(String filename, String questionnaireName) throws Exception {
    SpreadsheetDocument document = SpreadsheetDocument.loadDocument(new File(filename));
    Table sheet = document.getSheetByIndex(0);

    List<String> questions = new ArrayList<>();

    for (int i = 1; i < sheet.getRowList().size(); i++) {
      Row row = sheet.getRowList().get(i);

      String question = getCellStringValue(row, 0);

      if (StringUtils.isBlank(question)) {
        break;
      }

      String levelString = getCellStringValue(row, 1);
      long level = Long.valueOf(StringUtils.replace(levelString, "D", ""));
      String tagsString = getCellStringValue(row, 2);
      List<String> tags = Collections.emptyList();
      if (StringUtils.isNotBlank(tagsString)) {
        tags = Lists.newArrayList(StringUtils.split(tagsString, ", "));
      }
      String tip = StringUtils.defaultIfBlank(getCellStringValue(row, 3), null);

      XContentBuilder builder =
          jsonBuilder()
              .startObject()
              .field("title", question)
              .field("level", level)
              .field("tags", tags)
              .field("tip", tip)
              .endObject();

      IndexResponse indexResponse =
          client
              .prepareIndex(domainResolver.resolveQuestionIndex(), Types.question)
              .setSource(builder)
              .execute()
              .actionGet();

      questions.add(indexResponse.getId());
    }

    XContentBuilder questionnaireBuilder =
        jsonBuilder()
            .startObject()
            .field("name", questionnaireName)
            .field("questions", questions)
            .endObject();

    IndexResponse indexResponse =
        client
            .prepareIndex(domainResolver.resolveQuestionIndex(), Types.questionnaire)
            .setSource(questionnaireBuilder)
            .execute()
            .actionGet();

    return indexResponse.getId();
  }
  @Override
  public void notify(DelegateExecution execution) throws Exception {

    HistoryService historyService = execution.getProcessEngineServices().getHistoryService();

    HistoricDecisionInstance historicDecisionInstance =
        historyService
            .createHistoricDecisionInstanceQuery()
            .includeInputs()
            .includeOutputs()
            .decisionDefinitionKey((String) execution.getVariableLocal("tableName"))
            .processInstanceId(execution.getProcessInstanceId())
            .singleResult();

    // Fill a new object with stuff...
    FraudScoreTableObject fraudData = new FraudScoreTableObject();

    fraudData.setFraudInstanceID(historicDecisionInstance.getId());

    List<HistoricDecisionInputInstance> inputs = historicDecisionInstance.getInputs();
    for (HistoricDecisionInputInstance historicDecisionInputInstance : inputs) {
      String inputName = historicDecisionInputInstance.getTypeName();
      if (inputName.equals("paymentRejected")) {
        fraudData.setPaymentRejected((Boolean) historicDecisionInputInstance.getValue());
      } else if (inputName.equals("numberOfPayouts")) {
        fraudData.setNumberOfPayouts((Integer) historicDecisionInputInstance.getValue());
      } else if (inputName.equals("historyOfFraud")) {
        fraudData.setHistoryOfFraud((Boolean) historicDecisionInputInstance.getValue());
      } else if (inputName.equals("claimAmount")) {
        fraudData.setCalimAmount((Long) historicDecisionInputInstance.getValue());
      }
    }
    List<HistoricDecisionOutputInstance> outputs = historicDecisionInstance.getOutputs();
    for (HistoricDecisionOutputInstance historicDecisionOutputInstance : outputs) {

      String inputName = historicDecisionOutputInstance.getTypeName();
      if (inputName.equals("frausScore")) {
        fraudData.setFraudScore((Integer) historicDecisionOutputInstance.getValue());
      }
    }

    ObjectMapper mapper = new ObjectMapper();

    String serializedHistoricDecisionInstance =
        mapper.writerWithDefaultPrettyPrinter().writeValueAsString(fraudData);

    Client client =
        TransportClient.builder()
            .build()
            .addTransportAddress(
                new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300));

    IndexResponse response =
        client
            .prepareIndex("camunda", "fraudData", historicDecisionInstance.getId())
            .setSource(serializedHistoricDecisionInstance)
            .get();

    LOGGER.info(response.getId());
  }
  public void index(final Entity entity, final Tenant tenant) throws SearchEngineException {
    try {
      Map<String, Object> source = entityIndexDocumentPurveyor.purveyDocument(entity, tenant);

      if (source.keySet().size() > 0) {
        String entityName =
            StringUtils.substringAfterLast(entity.getClass().getName(), ".").toLowerCase();

        this.logger.debug("Indexing {} with id {}...", entityName, entity.getId());

        IndexRequestBuilder builder =
            this.client
                .prepareIndex("entities", entityName, entity.getId().toString())
                .setSource(source);

        if (tenant != null && !Tenant.class.isAssignableFrom(entity.getClass())) {
          builder.setParent(tenant.getId().toString());
        }

        IndexResponse response = builder.execute().actionGet();
        this.logger.debug("" + response.getType());
      }
    } catch (Exception e) {
      throw new SearchEngineException("Failed to index entity", e);
    }
  }
Пример #5
0
 @Test
 public void testIndexAndDelete() throws Exception {
   prepareIndex(1);
   IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}").get();
   assertShardInfo(indexResponse);
   DeleteResponse deleteResponse =
       client().prepareDelete("idx", "type", indexResponse.getId()).get();
   assertShardInfo(deleteResponse);
 }
 public int indexDocs(final int numOfDoc) throws Exception {
   for (int doc = 0; doc < numOfDoc; doc++) {
     final IndexRequest indexRequest =
         new IndexRequest(index.getName(), "type", Integer.toString(docId.incrementAndGet()))
             .source("{}");
     final IndexResponse response = index(indexRequest);
     assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
   }
   return numOfDoc;
 }
Пример #7
0
  public void testVersioningWithBulk() {
    createIndex("test");
    ensureGreen();

    BulkResponse bulkResponse =
        client()
            .prepareBulk()
            .add(client().prepareIndex("test", "type", "1").setSource("field1", "value1_1"))
            .execute()
            .actionGet();
    assertThat(bulkResponse.hasFailures(), equalTo(false));
    assertThat(bulkResponse.getItems().length, equalTo(1));
    IndexResponse indexResponse = bulkResponse.getItems()[0].getResponse();
    assertThat(indexResponse.getVersion(), equalTo(1L));
  }
 protected void registerRiver(final String typename, final String file)
     throws ElasticsearchException, IOException {
   final IndexResponse res =
       esSetup
           .client()
           .prepareIndex()
           .setIndex("_river")
           .setType(typename)
           .setId("_meta")
           .setSource(loadFile(file))
           .execute()
           .actionGet();
   if (!res.isCreated()) {
     throw new IOException("Unable to register river");
   }
 }
Пример #9
0
  public static String save(
      ElasticSearchContextListener es, String userId, String memeId, String value) {
    try {
      String pendingItemId = getPendingItemId(es, userId, memeId);
      Map<String, Object> data = getRatingsObjectMap(userId, memeId, value);

      if (pendingItemId != null) {
        UpdateResponse response = es.updateRequest(INDEX_NAME, pendingItemId, data).actionGet();
        return response.getId();
      } else {
        IndexResponse response = es.indexRequest(INDEX_NAME, data).actionGet();
        return response.getId();
      }
    } catch (Exception e) {
      return null;
    }
  }
Пример #10
0
  public void testInternalVersioningInitialDelete() throws Exception {
    createIndex("test");
    ensureGreen();

    assertThrows(
        client().prepareDelete("test", "type", "1").setVersion(17).execute(),
        VersionConflictEngineException.class);

    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setCreate(true)
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(1L));
  }
 /**
  * Saves an entity into elastic search.
  *
  * @param object The entity to save.
  */
 public void save(T object) {
   try {
     IndexRequestBuilder indexRequestBuilder =
         object.getId() == null
             ? client.prepareIndex(index, entity)
             : client.prepareIndex(index, entity, object.getId());
     IndexResponse response =
         indexRequestBuilder.setSource(mapper.writeValueAsString(object)).execute().actionGet();
     // Force refresh.
     object.setId(response.getId());
     client.admin().indices().prepareRefresh(index).execute().actionGet();
     if (logger.isDebugEnabled()) {
       BytesStreamOutput out = new BytesStreamOutput(1024 * 1024);
       response.writeTo(out);
       logger.debug("Obtained response: {}", new String(out.bytes().array()));
     }
   } catch (ElasticSearchException | IOException e) {
     logger.error("Error while saving entity", e);
   }
 }
Пример #12
0
  @Override
  public void process(final Exchange exchange) throws Exception {
    final String indexName = exchange.getIn().getHeader(INDEX_NAME, String.class);
    final String indexSchema = exchange.getIn().getHeader(INDEX_SCHEMA, String.class);
    final Indexer indexer = exchange.getIn().getHeader(INDEXER, Indexer.class);
    final Map data = exchange.getIn().getBody(Map.class);
    data.put(
        Constants.CORRELATION_ID,
        exchange.getIn().getHeader(Constants.CORRELATION_ID, String.class));
    final String serValue = mapper.writeValueAsString(data);

    LOG.debug(serValue);
    final IndexResponse response =
        indexer
            .es()
            .prepareIndex(indexName, indexSchema)
            .setSource(serValue)
            .execute()
            .get(1, TimeUnit.MINUTES);
    LOG.debug(response.toString());
  }
  @Test
  public void testVersioningWithBulk() {
    try {
      client.admin().indices().prepareDelete("test").execute().actionGet();
    } catch (IndexMissingException e) {
      // its ok
    }
    client.admin().indices().prepareCreate("test").execute().actionGet();
    client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();

    BulkResponse bulkResponse =
        client
            .prepareBulk()
            .add(client.prepareIndex("test", "type", "1").setSource("field1", "value1_1"))
            .execute()
            .actionGet();
    assertThat(bulkResponse.hasFailures(), equalTo(false));
    assertThat(bulkResponse.getItems().length, equalTo(1));
    IndexResponse indexResponse = bulkResponse.getItems()[0].getResponse();
    assertThat(indexResponse.version(), equalTo(1l));
  }
  public void testIndexGetAndDelete() throws ExecutionException, InterruptedException {
    createIndexWithAlias();
    ensureYellow("test");

    int numDocs = iterations(10, 50);
    for (int i = 0; i < numDocs; i++) {
      IndexResponse indexResponse =
          client()
              .prepareIndex(indexOrAlias(), "type", Integer.toString(i))
              .setSource("field", "value-" + i)
              .get();
      assertThat(indexResponse.isCreated(), equalTo(true));
      assertThat(indexResponse.getIndex(), equalTo("test"));
      assertThat(indexResponse.getType(), equalTo("type"));
      assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
    }
    refresh();

    String docId = Integer.toString(randomIntBetween(0, numDocs - 1));
    GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
    assertThat(getResponse.isExists(), equalTo(true));
    assertThat(getResponse.getIndex(), equalTo("test"));
    assertThat(getResponse.getType(), equalTo("type"));
    assertThat(getResponse.getId(), equalTo(docId));

    DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get();
    assertThat(deleteResponse.isFound(), equalTo(true));
    assertThat(deleteResponse.getIndex(), equalTo("test"));
    assertThat(deleteResponse.getType(), equalTo("type"));
    assertThat(deleteResponse.getId(), equalTo(docId));

    getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
    assertThat(getResponse.isExists(), equalTo(false));

    refresh();

    SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get();
    assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1));
  }
Пример #15
0
  public void testExternalVersioningInitialDelete() throws Exception {
    createIndex("test");
    ensureGreen();

    // Note - external version doesn't throw version conflicts on deletes of non existent records.
    // This is different from internal versioning

    DeleteResponse deleteResponse =
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(17)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertEquals(DocWriteResponse.Operation.NOOP, deleteResponse.getOperation());

    // this should conflict with the delete command transaction which told us that the object was
    // deleted at version 17.
    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(13)
            .setVersionType(VersionType.EXTERNAL)
            .execute(),
        VersionConflictEngineException.class);

    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(18)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(18L));
  }
Пример #16
0
  public static ObjectNode toJSON(ObjectMapper objectMapper, RecomputeResult success)
      throws InterruptedException, ExecutionException {
    final ObjectNode n = objectMapper.createObjectNode();

    // write targetID using index reponse
    final IndexResponse ir = success.indexResponse();
    final ObjectNode targetID = objectMapper.createObjectNode();
    targetID.put("index", ir.index());
    targetID.put("type", ir.type());
    targetID.put("id", ir.id());
    targetID.put("version", ir.version());
    n.put("targetID", targetID);

    // write dictionary of properties and their values
    final ObjectNode properties = objectMapper.createObjectNode();
    for (Map.Entry<String, JsonNode> entry : success.properties().entrySet())
      properties.put(entry.getKey(), entry.getValue());
    n.put("properties", properties);

    // write dictionary of properties and their values
    n.put("sourceDocumentAfterTransform", success.sourceDocument());

    return n;
  }
 public static int Record(long sync_id, String idx) {
   LogLine temp = rec_map.get(sync_id);
   if (null == temp) {
     idx_map.put(sync_id, idx);
   } else {
     temp.dstidx = idx;
     // tell score system about temp;
     rec_map.remove(sync_id);
     // tell elasticsearch;
     IndexResponse response =
         client
             .prepareIndex(DATABASEINDEX, DATABASETYPE)
             .setSource(gson.toJson(temp))
             .execute()
             .actionGet();
     if (!response.isCreated()) {
       System.out.println(PROMPT + "Elasticsearch recording failed!");
       return ERROR;
     }
     // test
     WriteToDisk(temp);
   }
   return SUCCESS;
 }
Пример #18
0
    @Override
    public String toString() {
      StringBuilder sb = new StringBuilder();
      sb.append("id=");
      sb.append(id);
      sb.append(" version=");
      sb.append(version);
      sb.append(" delete?=");
      sb.append(delete);
      sb.append(" threadID=");
      sb.append(threadID);
      sb.append(" indexStartTime=");
      sb.append(indexStartTime);
      sb.append(" indexFinishTime=");
      sb.append(indexFinishTime);
      sb.append(" versionConflict=");
      sb.append(versionConflict);
      sb.append(" alreadyExists?=");
      sb.append(alreadyExists);

      if (response != null) {
        if (response instanceof DeleteResponse) {
          DeleteResponse deleteResponse = (DeleteResponse) response;
          sb.append(" response:");
          sb.append(" index=");
          sb.append(deleteResponse.getIndex());
          sb.append(" id=");
          sb.append(deleteResponse.getId());
          sb.append(" type=");
          sb.append(deleteResponse.getType());
          sb.append(" version=");
          sb.append(deleteResponse.getVersion());
          sb.append(" found=");
          sb.append(deleteResponse.getOperation() == DocWriteResponse.Operation.DELETE);
        } else if (response instanceof IndexResponse) {
          IndexResponse indexResponse = (IndexResponse) response;
          sb.append(" index=");
          sb.append(indexResponse.getIndex());
          sb.append(" id=");
          sb.append(indexResponse.getId());
          sb.append(" type=");
          sb.append(indexResponse.getType());
          sb.append(" version=");
          sb.append(indexResponse.getVersion());
          sb.append(" created=");
          sb.append(indexResponse.getOperation() == DocWriteResponse.Operation.CREATE);
        } else {
          sb.append("  response: " + response);
        }
      } else {
        sb.append("  response: null");
      }

      return sb.toString();
    }
Пример #19
0
  @Test
  public void testIndex() {

    IndexResponse response = null;

    try {
      response =
          client
              .prepareIndex("asdf2014", "asdf", "1")
              .setSource(
                  jsonBuilder().startObject().field("age", 22).field("sex", "male").endObject())
              .execute()
              .actionGet();
    } catch (ElasticsearchException | IOException e) {
      e.printStackTrace();
      throw new RuntimeException(e);
    }

    System.out.println("Index: " + response.getIndex());
    System.out.println("Type: " + response.getType());
    System.out.println("Id: " + response.getId());
    System.err.println("Version: " + response.getVersion());
    System.out.println("IsCreated: " + response.isCreated());
  }
  @Test
  public void testNetworkPartitionDuringReplicaIndexOp() throws Exception {
    final String INDEX = "testidx";

    List<String> nodes = internalCluster().startNodesAsync(2, nodeSettings).get();

    // Create index test with 1 shard, 1 replica and ensure it is green
    createIndex(INDEX);
    ensureGreen(INDEX);

    // Disable allocation so the replica cannot be reallocated when it fails
    Settings s =
        ImmutableSettings.builder().put("cluster.routing.allocation.enable", "none").build();
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(s).get();

    // Determine which node holds the primary shard
    ClusterState state = getNodeClusterState(nodes.get(0));
    IndexShardRoutingTable shard = state.getRoutingTable().index(INDEX).shard(0);
    String primaryNode;
    String replicaNode;
    if (shard.getShards().get(0).primary()) {
      primaryNode = nodes.get(0);
      replicaNode = nodes.get(1);
    } else {
      primaryNode = nodes.get(1);
      replicaNode = nodes.get(0);
    }
    logger.info("--> primary shard is on {}", primaryNode);

    // Index a document to make sure everything works well
    IndexResponse resp =
        internalCluster()
            .client(primaryNode)
            .prepareIndex(INDEX, "doc")
            .setSource("foo", "bar")
            .get();
    assertThat(
        "document exists on primary node",
        internalCluster()
            .client(primaryNode)
            .prepareGet(INDEX, "doc", resp.getId())
            .setPreference("_only_local")
            .get()
            .isExists(),
        equalTo(true));
    assertThat(
        "document exists on replica node",
        internalCluster()
            .client(replicaNode)
            .prepareGet(INDEX, "doc", resp.getId())
            .setPreference("_only_local")
            .get()
            .isExists(),
        equalTo(true));

    // Disrupt the network so indexing requests fail to replicate
    logger.info("--> preventing index/replica operations");
    TransportService mockTransportService =
        internalCluster().getInstance(TransportService.class, primaryNode);
    ((MockTransportService) mockTransportService)
        .addFailToSendNoConnectRule(
            internalCluster().getInstance(Discovery.class, replicaNode).localNode(),
            ImmutableSet.of(IndexAction.NAME + "[r]"));
    mockTransportService = internalCluster().getInstance(TransportService.class, replicaNode);
    ((MockTransportService) mockTransportService)
        .addFailToSendNoConnectRule(
            internalCluster().getInstance(Discovery.class, primaryNode).localNode(),
            ImmutableSet.of(IndexAction.NAME + "[r]"));

    logger.info("--> indexing into primary");
    // the replica shard should now be marked as failed because the replication operation will fail
    resp =
        internalCluster()
            .client(primaryNode)
            .prepareIndex(INDEX, "doc")
            .setSource("foo", "baz")
            .get();
    // wait until the cluster reaches an exact yellow state, meaning replica has failed
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            assertThat(
                client().admin().cluster().prepareHealth().get().getStatus(),
                equalTo(ClusterHealthStatus.YELLOW));
          }
        });
    assertThat(
        "document should still be indexed and available",
        client().prepareGet(INDEX, "doc", resp.getId()).get().isExists(),
        equalTo(true));

    state = getNodeClusterState(randomFrom(nodes.toArray(Strings.EMPTY_ARRAY)));
    RoutingNodes rn = state.routingNodes();
    logger.info(
        "--> counts: total: {}, unassigned: {}, initializing: {}, relocating: {}, started: {}",
        rn.shards(
                new Predicate<MutableShardRouting>() {
                  @Override
                  public boolean apply(
                      org.elasticsearch.cluster.routing.MutableShardRouting input) {
                    return true;
                  }
                })
            .size(),
        rn.shardsWithState(UNASSIGNED).size(),
        rn.shardsWithState(INITIALIZING).size(),
        rn.shardsWithState(RELOCATING).size(),
        rn.shardsWithState(STARTED).size());
    logger.info(
        "--> unassigned: {}, initializing: {}, relocating: {}, started: {}",
        rn.shardsWithState(UNASSIGNED),
        rn.shardsWithState(INITIALIZING),
        rn.shardsWithState(RELOCATING),
        rn.shardsWithState(STARTED));

    assertThat(
        "only a single shard is now active (replica should be failed and not reallocated)",
        rn.shardsWithState(STARTED).size(),
        equalTo(1));
  }
Пример #21
0
  @Test
  public void testSimpleTTL() throws Exception {
    assertAcked(
        prepareCreate("test")
            .addMapping(
                "type1",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("type1")
                    .startObject("_timestamp")
                    .field("enabled", true)
                    .field("store", "yes")
                    .endObject()
                    .startObject("_ttl")
                    .field("enabled", true)
                    .endObject()
                    .endObject()
                    .endObject())
            .addMapping(
                "type2",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("type2")
                    .startObject("_timestamp")
                    .field("enabled", true)
                    .field("store", "yes")
                    .endObject()
                    .startObject("_ttl")
                    .field("enabled", true)
                    .field("default", "1d")
                    .endObject()
                    .endObject()
                    .endObject()));
    ensureYellow("test");

    final NumShards test = getNumShards("test");

    long providedTTLValue = 3000;
    logger.info("--> checking ttl");
    // Index one doc without routing, one doc with routing, one doc with not TTL and no default and
    // one doc with default TTL
    long now = System.currentTimeMillis();
    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type1", "1")
            .setSource("field1", "value1")
            .setTimestamp(String.valueOf(now))
            .setTTL(providedTTLValue)
            .setRefresh(true)
            .get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client()
            .prepareIndex("test", "type1", "with_routing")
            .setSource("field1", "value1")
            .setTimestamp(String.valueOf(now))
            .setTTL(providedTTLValue)
            .setRouting("routing")
            .setRefresh(true)
            .get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client().prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").get();
    assertThat(indexResponse.isCreated(), is(true));

    // realtime get check
    long currentTime = System.currentTimeMillis();
    GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
    long ttl0;
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // verify the ttl is still decreasing when going to the replica
    currentTime = System.currentTimeMillis();
    getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // non realtime get (stored)
    currentTime = System.currentTimeMillis();
    getResponse =
        client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // non realtime get going the replica
    currentTime = System.currentTimeMillis();
    getResponse =
        client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }

    // no TTL provided so no TTL fetched
    getResponse =
        client()
            .prepareGet("test", "type1", "no_ttl")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.getField("_ttl"), nullValue());
    // no TTL provided make sure it has default TTL
    getResponse =
        client()
            .prepareGet("test", "type2", "default_ttl")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
    assertThat(ttl0, greaterThan(0L));

    IndicesStatsResponse response =
        client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
    assertThat(
        response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount(),
        equalTo(0L));

    // make sure the purger has done its job for all indexed docs that are expired
    long shouldBeExpiredDate = now + providedTTLValue + PURGE_INTERVAL + 2000;
    currentTime = System.currentTimeMillis();
    if (shouldBeExpiredDate - currentTime > 0) {
      Thread.sleep(shouldBeExpiredDate - currentTime);
    }

    // We can't assume that after waiting for ttl + purgeInterval (waitTime) that the document have
    // actually been deleted.
    // The ttl purging happens in the background in a different thread, and might not have been
    // completed after waiting for waitTime.
    // But we can use index statistics' delete count to be sure that deletes have been executed,
    // that must be incremented before
    // ttl purging has finished.
    logger.info("--> checking purger");
    assertThat(
        awaitBusy(
            new Predicate<Object>() {
              @Override
              public boolean apply(Object input) {
                if (rarely()) {
                  client().admin().indices().prepareFlush("test").get();
                } else if (rarely()) {
                  client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get();
                }
                IndicesStatsResponse response =
                    client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
                // TTL deletes two docs, but it is indexed in the primary shard and replica shard.
                return response
                        .getIndices()
                        .get("test")
                        .getTotal()
                        .getIndexing()
                        .getTotal()
                        .getDeleteCount()
                    == 2L * test.dataCopies;
              }
            },
            5,
            TimeUnit.SECONDS),
        equalTo(true));

    // realtime get check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    // replica realtime get check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));

    // Need to run a refresh, in order for the non realtime get to work.
    client().admin().indices().prepareRefresh("test").execute().actionGet();

    // non realtime get (stored) check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    // non realtime get going the replica check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
  }
 @Override
 public void setShardInfo(ReplicationResponse.ShardInfo shardInfo) {
   finalResponse.setShardInfo(shardInfo);
 }
Пример #23
0
  @Test
  public void fullTest() throws ExecutionException, InterruptedException, DegraphmalizerException {
    final String target = "test-target";
    final String idx = "test-index";
    final String tp = "test-type";
    final String id = "1234";

    // create target index if it doesn't exist
    if (!ln.es.admin().indices().prepareExists(target).execute().actionGet().exists()) {
      final CreateIndexResponse cir =
          ln.es.admin().indices().prepareCreate(target).execute().actionGet();
      if (!cir.acknowledged()) throw new RuntimeException("failed to create index " + target);
    }

    final IndexResponse ir =
        ln.es.prepareIndex(idx, tp, id).setSource("{\"children\":[1,2,3]}").execute().actionGet();

    ln.log.info(
        "Indexed /{}/{}/{} as version {} into ES", new Object[] {idx, tp, id, ir.version()});

    final IndexResponse ir1 =
        ln.es
            .prepareIndex(idx, tp, "1")
            .setSource("{\"cheese\":\"gorgonzola\"}")
            .execute()
            .actionGet();

    ln.log.info("Indexed /{}/{}/1 as version {} into ES", new Object[] {idx, tp, ir1.version()});

    final IndexResponse ir2 =
        ln.es
            .prepareIndex(idx, tp, "2")
            .setSource("{\"cheese\":\"mozarella\"}")
            .execute()
            .actionGet();

    ln.log.info("Indexed /{}/{}/2 as version {} into ES", new Object[] {idx, tp, ir2.version()});

    // degraphmalize "1" and wait for and print result
    final List<Future<DegraphmalizeResult>> actions = new ArrayList<Future<DegraphmalizeResult>>();

    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.UPDATE,
            DegraphmalizeRequestScope.DOCUMENT,
            new ID(idx, tp, id, ir.version()),
            ln.callback));
    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.UPDATE,
            DegraphmalizeRequestScope.DOCUMENT,
            new ID(idx, tp, "1", ir1.version()),
            ln.callback));
    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.UPDATE,
            DegraphmalizeRequestScope.DOCUMENT,
            new ID(idx, tp, "2", ir2.version()),
            ln.callback));

    for (final Future<DegraphmalizeResult> a : actions) {
      DegraphmalizeResult result = a.get();
      ln.log.info("Degraphmalize complete for : " + result.root());
    }

    ObjectMapper mapper = new ObjectMapper();
    for (final Future<DegraphmalizeResult> a : actions) {
      final DegraphmalizeResult degraphmalizeResult = a.get();
      // Get first node in results
      final ObjectNode result = toJSON(mapper, degraphmalizeResult.results().get(0).get());

      assertThat(result.get("properties").has("nodes-in")).isTrue();
      assertThat(result.get("properties").has("nodes-out")).isTrue();

      if (degraphmalizeResult.root().id().equals("1234")) {
        assertThat(numberOfChildren(result, "nodes-out")).isZero();
        assertThat(numberOfChildren(result, "nodes-in")).isEqualTo(3);
      }

      if (degraphmalizeResult.root().id().equals("1")) {
        assertThat(numberOfChildren(result, "nodes-out")).isEqualTo(1);
        assertThat(numberOfChildren(result, "nodes-in")).isZero();
      }
    }

    GraphUtilities.dumpGraph(new ObjectMapper(), ln.G);
    // Cleanup index
    if (!ln.es.admin().indices().delete(new DeleteIndexRequest(idx)).actionGet().acknowledged()) {
      throw new RuntimeException("failed to delete index " + target);
    }
  }
Пример #24
0
  public void testExternalGTE() throws Exception {
    createIndex("test");

    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(12)
            .setVersionType(VersionType.EXTERNAL_GTE)
            .get();
    assertThat(indexResponse.getVersion(), equalTo(12L));

    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_2")
            .setVersion(12)
            .setVersionType(VersionType.EXTERNAL_GTE)
            .get();
    assertThat(indexResponse.getVersion(), equalTo(12L));

    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_2")
            .setVersion(14)
            .setVersionType(VersionType.EXTERNAL_GTE)
            .get();
    assertThat(indexResponse.getVersion(), equalTo(14L));

    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(13)
            .setVersionType(VersionType.EXTERNAL_GTE),
        VersionConflictEngineException.class);

    client().admin().indices().prepareRefresh().execute().actionGet();
    if (randomBoolean()) {
      refresh();
    }
    for (int i = 0; i < 10; i++) {
      assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(14L));
    }

    // deleting with a lower version fails.
    assertThrows(
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(2)
            .setVersionType(VersionType.EXTERNAL_GTE),
        VersionConflictEngineException.class);

    // Delete with a higher or equal version deletes all versions up to the given one.
    long v = randomIntBetween(14, 17);
    DeleteResponse deleteResponse =
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(v)
            .setVersionType(VersionType.EXTERNAL_GTE)
            .execute()
            .actionGet();
    assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
    assertThat(deleteResponse.getVersion(), equalTo(v));

    // Deleting with a lower version keeps on failing after a delete.
    assertThrows(
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(2)
            .setVersionType(VersionType.EXTERNAL_GTE)
            .execute(),
        VersionConflictEngineException.class);

    // But delete with a higher version is OK.
    deleteResponse =
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(18)
            .setVersionType(VersionType.EXTERNAL_GTE)
            .execute()
            .actionGet();
    assertEquals(DocWriteResponse.Operation.NOOP, deleteResponse.getOperation());
    assertThat(deleteResponse.getVersion(), equalTo(18L));
  }
Пример #25
0
  public void testSimpleVersioningWithFlush() throws Exception {
    createIndex("test");
    ensureGreen();

    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(1L));

    client().admin().indices().prepareFlush().execute().actionGet();

    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_2")
            .setVersion(1)
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(2L));

    client().admin().indices().prepareFlush().execute().actionGet();

    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(1)
            .execute(),
        VersionConflictEngineException.class);

    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(1)
            .execute(),
        VersionConflictEngineException.class);

    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setCreate(true)
            .setSource("field1", "value1_1")
            .execute(),
        VersionConflictEngineException.class);

    assertThrows(
        client().prepareDelete("test", "type", "1").setVersion(1).execute(),
        VersionConflictEngineException.class);
    assertThrows(
        client().prepareDelete("test", "type", "1").setVersion(1).execute(),
        VersionConflictEngineException.class);

    client().admin().indices().prepareRefresh().execute().actionGet();
    for (int i = 0; i < 10; i++) {
      assertThat(
          client().prepareGet("test", "type", "1").execute().actionGet().getVersion(), equalTo(2L));
    }

    for (int i = 0; i < 10; i++) {
      SearchResponse searchResponse =
          client().prepareSearch().setQuery(matchAllQuery()).setVersion(true).execute().actionGet();
      assertThat(searchResponse.getHits().getAt(0).version(), equalTo(2L));
    }
  }
Пример #26
0
  public void testInternalVersioning() throws Exception {
    createIndex("test");
    ensureGreen();

    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(1L));

    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_2")
            .setVersion(1)
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(2L));

    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(1)
            .execute(),
        VersionConflictEngineException.class);

    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(1)
            .execute(),
        VersionConflictEngineException.class);

    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setCreate(true)
            .setSource("field1", "value1_1")
            .execute(),
        VersionConflictEngineException.class);

    assertThrows(
        client().prepareDelete("test", "type", "1").setVersion(1).execute(),
        VersionConflictEngineException.class);
    assertThrows(
        client().prepareDelete("test", "type", "1").setVersion(1).execute(),
        VersionConflictEngineException.class);

    client().admin().indices().prepareRefresh().execute().actionGet();
    for (int i = 0; i < 10; i++) {
      assertThat(
          client().prepareGet("test", "type", "1").execute().actionGet().getVersion(), equalTo(2L));
    }

    // search with versioning
    for (int i = 0; i < 10; i++) {
      SearchResponse searchResponse =
          client().prepareSearch().setQuery(matchAllQuery()).setVersion(true).execute().actionGet();
      assertThat(searchResponse.getHits().getAt(0).version(), equalTo(2L));
    }

    // search without versioning
    for (int i = 0; i < 10; i++) {
      SearchResponse searchResponse =
          client().prepareSearch().setQuery(matchAllQuery()).execute().actionGet();
      assertThat(searchResponse.getHits().getAt(0).version(), equalTo(Versions.NOT_FOUND));
    }

    DeleteResponse deleteResponse =
        client().prepareDelete("test", "type", "1").setVersion(2).execute().actionGet();
    assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
    assertThat(deleteResponse.getVersion(), equalTo(3L));

    assertThrows(
        client().prepareDelete("test", "type", "1").setVersion(2).execute(),
        VersionConflictEngineException.class);

    // This is intricate - the object was deleted but a delete transaction was with the right
    // version. We add another one
    // and thus the transaction is increased.
    deleteResponse =
        client().prepareDelete("test", "type", "1").setVersion(3).execute().actionGet();
    assertEquals(DocWriteResponse.Operation.NOOP, deleteResponse.getOperation());
    assertThat(deleteResponse.getVersion(), equalTo(4L));
  }
Пример #27
0
  public void testExternalVersioning() throws Exception {
    createIndex("test");
    ensureGreen();

    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(12)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(12L));

    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(14)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(14L));

    assertThrows(
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(13)
            .setVersionType(VersionType.EXTERNAL)
            .execute(),
        VersionConflictEngineException.class);

    if (randomBoolean()) {
      refresh();
    }
    for (int i = 0; i < 10; i++) {
      assertThat(
          client().prepareGet("test", "type", "1").execute().actionGet().getVersion(),
          equalTo(14L));
    }

    // deleting with a lower version fails.
    assertThrows(
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(2)
            .setVersionType(VersionType.EXTERNAL)
            .execute(),
        VersionConflictEngineException.class);

    // Delete with a higher version deletes all versions up to the given one.
    DeleteResponse deleteResponse =
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(17)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
    assertThat(deleteResponse.getVersion(), equalTo(17L));

    // Deleting with a lower version keeps on failing after a delete.
    assertThrows(
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(2)
            .setVersionType(VersionType.EXTERNAL)
            .execute(),
        VersionConflictEngineException.class);

    // But delete with a higher version is OK.
    deleteResponse =
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(18)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertEquals(DocWriteResponse.Operation.NOOP, deleteResponse.getOperation());
    assertThat(deleteResponse.getVersion(), equalTo(18L));

    // TODO: This behavior breaks rest api returning http status 201, good news is that it this is
    // only the case until deletes GC kicks in.
    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(19)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(19L));

    deleteResponse =
        client()
            .prepareDelete("test", "type", "1")
            .setVersion(20)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
    assertThat(deleteResponse.getVersion(), equalTo(20L));

    // Make sure that the next delete will be GC. Note we do it on the index settings so it will be
    // cleaned up
    HashMap<String, Object> newSettings = new HashMap<>();
    newSettings.put("index.gc_deletes", -1);
    client()
        .admin()
        .indices()
        .prepareUpdateSettings("test")
        .setSettings(newSettings)
        .execute()
        .actionGet();

    Thread.sleep(300); // gc works based on estimated sampled time. Give it a chance...

    // And now we have previous version return -1
    indexResponse =
        client()
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(20)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(indexResponse.getVersion(), equalTo(20L));
  }
  @Test
  public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
    final int numShards = between(1, 5);
    String mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("properties")
            .startObject("test")
            .field("type", "string")
            .field("index", "not_analyzed")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .string();
    final double exceptionRate;
    final double exceptionOnOpenRate;
    if (frequently()) {
      if (randomBoolean()) {
        if (randomBoolean()) {
          exceptionOnOpenRate = 1.0 / between(5, 100);
          exceptionRate = 0.0d;
        } else {
          exceptionRate = 1.0 / between(5, 100);
          exceptionOnOpenRate = 0.0d;
        }
      } else {
        exceptionOnOpenRate = 1.0 / between(5, 100);
        exceptionRate = 1.0 / between(5, 100);
      }
    } else {
      // rarely no exception
      exceptionRate = 0d;
      exceptionOnOpenRate = 0d;
    }

    Builder settings =
        settingsBuilder()
            .put("index.number_of_shards", numShards)
            .put("index.number_of_replicas", randomIntBetween(0, 1))
            .put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE, exceptionRate)
            .put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE_ON_OPEN, exceptionOnOpenRate)
            .put(MockDirectoryHelper.CHECK_INDEX_ON_CLOSE, true);
    logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(settings)
        .addMapping("type", mapping)
        .execute()
        .actionGet();
    ClusterHealthResponse clusterHealthResponse =
        client()
            .admin()
            .cluster()
            .health(
                Requests.clusterHealthRequest()
                    .waitForYellowStatus()
                    .timeout(TimeValue.timeValueSeconds(5)))
            .get(); // it's OK to timeout here
    final int numDocs;
    final boolean expectAllShardsFailed;
    if (clusterHealthResponse.isTimedOut()) {
      /* some seeds just won't let you create the index at all and we enter a ping-pong mode
       * trying one node after another etc. that is ok but we need to make sure we don't wait
       * forever when indexing documents so we set numDocs = 1 and expecte all shards to fail
       * when we search below.*/
      logger.info("ClusterHealth timed out - only index one doc and expect searches to fail");
      numDocs = 1;
      expectAllShardsFailed = true;
    } else {
      numDocs = between(10, 100);
      expectAllShardsFailed = false;
    }
    long numCreated = 0;
    boolean[] added = new boolean[numDocs];
    for (int i = 0; i < numDocs; i++) {
      try {
        IndexResponse indexResponse =
            client()
                .prepareIndex("test", "type", "" + i)
                .setTimeout(TimeValue.timeValueSeconds(1))
                .setSource("test", English.intToEnglish(i))
                .get();
        if (indexResponse.isCreated()) {
          numCreated++;
          added[i] = true;
        }
      } catch (ElasticSearchException ex) {
      }
    }
    logger.info("Start Refresh");
    RefreshResponse refreshResponse =
        client()
            .admin()
            .indices()
            .prepareRefresh("test")
            .execute()
            .get(); // don't assert on failures here
    final boolean refreshFailed =
        refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
    logger.info(
        "Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ",
        refreshFailed,
        refreshResponse.getFailedShards(),
        refreshResponse.getShardFailures().length,
        refreshResponse.getSuccessfulShards(),
        refreshResponse.getTotalShards());

    final int numSearches = atLeast(10);
    // we don't check anything here really just making sure we don't leave any open files or a
    // broken index behind.
    for (int i = 0; i < numSearches; i++) {
      try {
        int docToQuery = between(0, numDocs - 1);
        long expectedResults = added[docToQuery] ? 1 : 0;
        logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
        SearchResponse searchResponse =
            client()
                .prepareSearch()
                .setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery)))
                .get();
        logger.info(
            "Successful shards: [{}]  numShards: [{}]",
            searchResponse.getSuccessfulShards(),
            numShards);
        if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
          assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(expectedResults));
        }
        // check match all
        searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).get();
        if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
          assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(numCreated));
        }

      } catch (SearchPhaseExecutionException ex) {
        if (!expectAllShardsFailed) {
          throw ex;
        } else {
          logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
        }
      }
    }
  }
Пример #29
0
  @Test
  public void deleteTest() throws ExecutionException, InterruptedException {
    final String target = "test-target";
    final String idx = "test-index";
    final String tp = "test-type";
    final String id = "1234";
    final String othertp = "othertest-type";

    // create target index if it doesn't exist
    if (!ln.es.admin().indices().prepareExists(target).execute().actionGet().exists()) {
      final CreateIndexResponse cir =
          ln.es.admin().indices().prepareCreate(target).execute().actionGet();
      if (!cir.acknowledged()) throw new RuntimeException("failed to create index " + target);
    }

    final IndexResponse ir =
        ln.es.prepareIndex(idx, tp, id).setSource("{\"children\":[1,2,3]}").execute().actionGet();

    ln.log.info(
        "Indexed /{}/{}/{} as version {} into ES", new Object[] {idx, tp, id, ir.version()});

    final IndexResponse ir1 =
        ln.es
            .prepareIndex(idx, tp, "1")
            .setSource("{\"cheese\":\"gorgonzola\"}")
            .execute()
            .actionGet();

    ln.log.info("Indexed /{}/{}/1 as version {} into ES", new Object[] {idx, tp, ir1.version()});

    final IndexResponse ir2 =
        ln.es
            .prepareIndex(idx, tp, "2")
            .setSource("{\"cheese\":\"mozarella\"}")
            .execute()
            .actionGet();

    ln.log.info("Indexed /{}/{}/2 as version {} into ES", new Object[] {idx, tp, ir2.version()});

    final IndexResponse ir3 =
        ln.es
            .prepareIndex(idx, othertp, "3")
            .setSource("{\"cheese\":\"limburger\"}")
            .execute()
            .actionGet();

    ln.log.info(
        "Indexed /{}/{}/3 as version {} into ES", new Object[] {idx, othertp, ir3.version()});

    final List<Future<DegraphmalizeResult>> actions = new ArrayList<Future<DegraphmalizeResult>>();

    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.UPDATE,
            DegraphmalizeRequestScope.DOCUMENT,
            new ID(idx, tp, id, ir.version()),
            ln.callback));
    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.UPDATE,
            DegraphmalizeRequestScope.DOCUMENT,
            new ID(idx, tp, "1", ir1.version()),
            ln.callback));
    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.UPDATE,
            DegraphmalizeRequestScope.DOCUMENT,
            new ID(idx, tp, "2", ir2.version()),
            ln.callback));
    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.UPDATE,
            DegraphmalizeRequestScope.DOCUMENT,
            new ID(idx, othertp, "3", ir3.version()),
            ln.callback));

    for (final Future<DegraphmalizeResult> a : actions) {
      DegraphmalizeResult result = a.get();
      ln.log.info("Degraphmalize complete for : " + result.root());
    }

    GraphUtilities.dumpGraph(new ObjectMapper(), ln.G);

    actions.clear();
    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.DELETE,
            DegraphmalizeRequestScope.DOCUMENT_ANY_VERSION,
            new ID(idx, tp, id, 0),
            ln.callback));
    for (final Future<DegraphmalizeResult> a : actions) {
      DegraphmalizeResult result = a.get();
      ln.log.info("Degraphmalize of {}: {}", result.root(), result);
    }

    actions.clear();
    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.DELETE,
            DegraphmalizeRequestScope.TYPE_IN_INDEX,
            new ID(idx, tp, null, 0),
            ln.callback));
    for (final Future<DegraphmalizeResult> a : actions) {
      DegraphmalizeResult result = a.get();
      ln.log.info("Degraphmalize of {}: {}", result.root(), result);
    }

    // Only the vertex of the othertp type should be present.
    Iterable<Vertex> iterable = GraphUtilities.findVerticesInIndex(ln.G, idx);
    assertThat(Iterables.size(iterable)).isEqualTo(1);

    actions.clear();
    actions.add(
        ln.d.degraphmalize(
            DegraphmalizeRequestType.DELETE,
            DegraphmalizeRequestScope.INDEX,
            new ID(idx, null, null, 0),
            ln.callback));
    for (final Future<DegraphmalizeResult> a : actions) {
      DegraphmalizeResult result = a.get();
      ln.log.info("Degraphmalize of {}: {}", result.root(), result);
    }

    // No more vertices
    Iterable<Vertex> iterable2 = GraphUtilities.findVerticesInIndex(ln.G, idx);
    assertThat(Iterables.size(iterable2)).isEqualTo(0);

    // Cleanup index
    if (!ln.es.admin().indices().delete(new DeleteIndexRequest(idx)).actionGet().acknowledged()) {
      throw new RuntimeException("failed to delete index " + target);
    }
  }
  @Override
  protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(
      ClusterState clusterState, PrimaryOperationRequest shardRequest) {
    final BulkShardRequest request = shardRequest.request;
    final IndexService indexService = indicesService.indexServiceSafe(request.index());
    final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());

    long[] preVersions = new long[request.items().length];
    VersionType[] preVersionTypes = new VersionType[request.items().length];
    Translog.Location location = null;
    for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
      BulkItemRequest item = request.items()[requestIndex];
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        preVersions[requestIndex] = indexRequest.version();
        preVersionTypes[requestIndex] = indexRequest.versionType();
        try {
          WriteResult<IndexResponse> result =
              shardIndexOperation(request, indexRequest, clusterState, indexShard, true);
          location = locationToSync(location, result.location);
          // add the response
          IndexResponse indexResponse = result.response();
          setResponse(
              item,
              new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
        } catch (Throwable e) {
          // rethrow the failure if we are going to retry on primary and let parent failure to
          // handle it
          if (retryPrimaryException(e)) {
            // restore updated versions...
            for (int j = 0; j < requestIndex; j++) {
              applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
            }
            throw (ElasticsearchException) e;
          }
          if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
            logger.trace(
                "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
          } else {
            logger.debug(
                "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
          }
          // if its a conflict failure, and we already executed the request on a primary (and we
          // execute it
          // again, due to primary relocation and only processing up to N bulk items when the shard
          // gets closed)
          // then just use the response we got from the successful execution
          if (item.getPrimaryResponse() != null && isConflictException(e)) {
            setResponse(item, item.getPrimaryResponse());
          } else {
            setResponse(
                item,
                new BulkItemResponse(
                    item.id(),
                    indexRequest.opType().lowercase(),
                    new BulkItemResponse.Failure(
                        request.index(), indexRequest.type(), indexRequest.id(), e)));
          }
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        preVersions[requestIndex] = deleteRequest.version();
        preVersionTypes[requestIndex] = deleteRequest.versionType();

        try {
          // add the response
          final WriteResult<DeleteResponse> writeResult =
              shardDeleteOperation(request, deleteRequest, indexShard);
          DeleteResponse deleteResponse = writeResult.response();
          location = locationToSync(location, writeResult.location);
          setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
        } catch (Throwable e) {
          // rethrow the failure if we are going to retry on primary and let parent failure to
          // handle it
          if (retryPrimaryException(e)) {
            // restore updated versions...
            for (int j = 0; j < requestIndex; j++) {
              applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
            }
            throw (ElasticsearchException) e;
          }
          if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
            logger.trace(
                "{} failed to execute bulk item (delete) {}",
                e,
                shardRequest.shardId,
                deleteRequest);
          } else {
            logger.debug(
                "{} failed to execute bulk item (delete) {}",
                e,
                shardRequest.shardId,
                deleteRequest);
          }
          // if its a conflict failure, and we already executed the request on a primary (and we
          // execute it
          // again, due to primary relocation and only processing up to N bulk items when the shard
          // gets closed)
          // then just use the response we got from the successful execution
          if (item.getPrimaryResponse() != null && isConflictException(e)) {
            setResponse(item, item.getPrimaryResponse());
          } else {
            setResponse(
                item,
                new BulkItemResponse(
                    item.id(),
                    OP_TYPE_DELETE,
                    new BulkItemResponse.Failure(
                        request.index(), deleteRequest.type(), deleteRequest.id(), e)));
          }
        }
      } else if (item.request() instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) item.request();
        preVersions[requestIndex] = updateRequest.version();
        preVersionTypes[requestIndex] = updateRequest.versionType();
        //  We need to do the requested retries plus the initial attempt. We don't do <
        // 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE
        for (int updateAttemptsCount = 0;
            updateAttemptsCount <= updateRequest.retryOnConflict();
            updateAttemptsCount++) {
          UpdateResult updateResult;
          try {
            updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard);
          } catch (Throwable t) {
            updateResult = new UpdateResult(null, null, false, t, null);
          }
          if (updateResult.success()) {
            if (updateResult.writeResult != null) {
              location = locationToSync(location, updateResult.writeResult.location);
            }
            switch (updateResult.result.operation()) {
              case UPSERT:
              case INDEX:
                WriteResult<IndexResponse> result = updateResult.writeResult;
                IndexRequest indexRequest = updateResult.request();
                BytesReference indexSourceAsBytes = indexRequest.source();
                // add the response
                IndexResponse indexResponse = result.response();
                UpdateResponse updateResponse =
                    new UpdateResponse(
                        indexResponse.getShardInfo(),
                        indexResponse.getIndex(),
                        indexResponse.getType(),
                        indexResponse.getId(),
                        indexResponse.getVersion(),
                        indexResponse.isCreated());
                if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
                  Tuple<XContentType, Map<String, Object>> sourceAndContent =
                      XContentHelper.convertToMap(indexSourceAsBytes, true);
                  updateResponse.setGetResult(
                      updateHelper.extractGetResult(
                          updateRequest,
                          shardRequest.request.index(),
                          indexResponse.getVersion(),
                          sourceAndContent.v2(),
                          sourceAndContent.v1(),
                          indexSourceAsBytes));
                }
                item =
                    request.items()[requestIndex] =
                        new BulkItemRequest(request.items()[requestIndex].id(), indexRequest);
                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                break;
              case DELETE:
                WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
                DeleteResponse response = writeResult.response();
                DeleteRequest deleteRequest = updateResult.request();
                updateResponse =
                    new UpdateResponse(
                        response.getShardInfo(),
                        response.getIndex(),
                        response.getType(),
                        response.getId(),
                        response.getVersion(),
                        false);
                updateResponse.setGetResult(
                    updateHelper.extractGetResult(
                        updateRequest,
                        shardRequest.request.index(),
                        response.getVersion(),
                        updateResult.result.updatedSourceAsMap(),
                        updateResult.result.updateSourceContentType(),
                        null));
                // Replace the update request to the translated delete request to execute on the
                // replica.
                item =
                    request.items()[requestIndex] =
                        new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest);
                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                break;
              case NONE:
                setResponse(
                    item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult));
                item.setIgnoreOnReplica(); // no need to go to the replica
                break;
            }
            // NOTE: Breaking out of the retry_on_conflict loop!
            break;
          } else if (updateResult.failure()) {
            Throwable t = updateResult.error;
            if (updateResult.retry) {
              // updateAttemptCount is 0 based and marks current attempt, if it's equal to
              // retryOnConflict we are going out of the iteration
              if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
                setResponse(
                    item,
                    new BulkItemResponse(
                        item.id(),
                        OP_TYPE_UPDATE,
                        new BulkItemResponse.Failure(
                            request.index(), updateRequest.type(), updateRequest.id(), t)));
              }
            } else {
              // rethrow the failure if we are going to retry on primary and let parent failure to
              // handle it
              if (retryPrimaryException(t)) {
                // restore updated versions...
                for (int j = 0; j < requestIndex; j++) {
                  applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
                }
                throw (ElasticsearchException) t;
              }
              // if its a conflict failure, and we already executed the request on a primary (and we
              // execute it
              // again, due to primary relocation and only processing up to N bulk items when the
              // shard gets closed)
              // then just use the response we got from the successful execution
              if (item.getPrimaryResponse() != null && isConflictException(t)) {
                setResponse(item, item.getPrimaryResponse());
              } else if (updateResult.result == null) {
                setResponse(
                    item,
                    new BulkItemResponse(
                        item.id(),
                        OP_TYPE_UPDATE,
                        new BulkItemResponse.Failure(
                            shardRequest.request.index(),
                            updateRequest.type(),
                            updateRequest.id(),
                            t)));
              } else {
                switch (updateResult.result.operation()) {
                  case UPSERT:
                  case INDEX:
                    IndexRequest indexRequest = updateResult.request();
                    if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
                      logger.trace(
                          "{} failed to execute bulk item (index) {}",
                          t,
                          shardRequest.shardId,
                          indexRequest);
                    } else {
                      logger.debug(
                          "{} failed to execute bulk item (index) {}",
                          t,
                          shardRequest.shardId,
                          indexRequest);
                    }
                    setResponse(
                        item,
                        new BulkItemResponse(
                            item.id(),
                            OP_TYPE_UPDATE,
                            new BulkItemResponse.Failure(
                                request.index(), indexRequest.type(), indexRequest.id(), t)));
                    break;
                  case DELETE:
                    DeleteRequest deleteRequest = updateResult.request();
                    if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
                      logger.trace(
                          "{} failed to execute bulk item (delete) {}",
                          t,
                          shardRequest.shardId,
                          deleteRequest);
                    } else {
                      logger.debug(
                          "{} failed to execute bulk item (delete) {}",
                          t,
                          shardRequest.shardId,
                          deleteRequest);
                    }
                    setResponse(
                        item,
                        new BulkItemResponse(
                            item.id(),
                            OP_TYPE_DELETE,
                            new BulkItemResponse.Failure(
                                request.index(), deleteRequest.type(), deleteRequest.id(), t)));
                    break;
                }
              }
              // NOTE: Breaking out of the retry_on_conflict loop!
              break;
            }
          }
        }
      } else {
        throw new IllegalStateException("Unexpected index operation: " + item.request());
      }

      assert item.getPrimaryResponse() != null;
      assert preVersionTypes[requestIndex] != null;
    }

    processAfter(request.refresh(), indexShard, location);
    BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
    BulkItemRequest[] items = request.items();
    for (int i = 0; i < items.length; i++) {
      responses[i] = items[i].getPrimaryResponse();
    }
    return new Tuple<>(
        new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request);
  }