Пример #1
0
  @Override
  public final String esGet(String index, String type, String id) {
    try {
      final GetResponse gr = client.prepareGet(index, type, id).execute().actionGet();

      if (!gr.exists()) return "Doesn't exist";

      return gr.getSourceAsString();
    } catch (ElasticSearchException e) {
      log.debug("ElasticSearchException {}", e);

      return e.getMessage();
    }
  }
Пример #2
0
 void insertIndexes(String str, String filename, int frameno, Client client) {
   // System.err.println(str);
   StringTokenizer token = new StringTokenizer(str, "\t");
   try {
     String color = token.nextToken();
     IndexResponse response =
         client
             .prepareIndex("movie", color, filename + "_" + frameno)
             .setSource(
                 jsonBuilder()
                     .startObject()
                     .field("filename", filename.replace(".", ""))
                     .field("color", color)
                     .field("frameno", String.valueOf(frameno))
                     .field("face", token.nextToken())
                     .field("corner", token.nextToken())
                     .field("apidq1", token.nextToken())
                     .field("apidq2", token.nextToken())
                     .field("apidq3", token.nextToken())
                     .field("apidq4", token.nextToken())
                     .field("apidq5", token.nextToken())
                     .field("apidq6", token.nextToken())
                     .field("apidq7", token.nextToken())
                     .field("apidq8", token.nextToken())
                     .field("hsv1", token.nextToken())
                     .field("hsv2", token.nextToken())
                     .field("hsv3", token.nextToken())
                     .field("hsv4", token.nextToken())
                     .field("edge1", token.nextToken())
                     .field("edge2", token.nextToken())
                     .field("edge3", token.nextToken())
                     .field("edge4", token.nextToken())
                     .field("edge5", token.nextToken())
                     .field("edge6", token.nextToken())
                     .field("edge7", token.nextToken())
                     .field("edge8", token.nextToken())
                     .endObject())
             .execute()
             .actionGet();
   } catch (ElasticSearchException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
   } catch (IOException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
   }
 }
  @Test
  public void testExternalVersioningInitialDelete() throws Exception {
    client.admin().indices().prepareDelete().execute().actionGet();

    client.admin().indices().prepareCreate("test").execute().actionGet();
    client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();

    DeleteResponse deleteResponse =
        client2
            .prepareDelete("test", "type", "1")
            .setVersion(17)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(deleteResponse.isNotFound(), equalTo(true));

    try {
      client
          .prepareIndex("test", "type", "1")
          .setSource("field1", "value1_1")
          .setVersion(13)
          .setVersionType(VersionType.EXTERNAL)
          .execute()
          .actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    client
        .prepareIndex("test", "type", "1")
        .setSource("field1", "value1_1")
        .setVersion(18)
        .setVersionType(VersionType.EXTERNAL)
        .execute()
        .actionGet();
  }
  @Test
  public void testFastVectorHighlighterManyDocs() throws Exception {
    try {
      client.admin().indices().prepareDelete("test").execute().actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(IndexMissingException.class));
    }
    client
        .admin()
        .indices()
        .prepareCreate("test")
        .addMapping("type1", type1TermVectorMapping())
        .execute()
        .actionGet();
    client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();

    int COUNT = 100;
    logger.info("--> indexing docs");
    for (int i = 0; i < COUNT; i++) {
      client
          .prepareIndex("test", "type1", Integer.toString(i))
          .setSource("field1", "test " + i)
          .execute()
          .actionGet();
      if (i % 5 == 0) {
        // flush so we get updated readers and segmented readers
        client.admin().indices().prepareFlush().execute().actionGet();
      }
    }

    client.admin().indices().prepareRefresh().execute().actionGet();

    logger.info("--> searching explicitly on field1 and highlighting on it");
    SearchResponse searchResponse =
        client
            .prepareSearch()
            .setSize(COUNT)
            .setQuery(termQuery("field1", "test"))
            .addHighlightedField("field1", 100, 0)
            .execute()
            .actionGet();
    assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT));
    assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
    for (SearchHit hit : searchResponse.hits()) {
      // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
      assertThat(
          hit.highlightFields().get("field1").fragments()[0],
          equalTo("<em>test</em> " + hit.id() + " "));
    }

    logger.info("--> searching explicitly on field1 and highlighting on it, with DFS");
    searchResponse =
        client
            .prepareSearch()
            .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
            .setSize(COUNT)
            .setQuery(termQuery("field1", "test"))
            .addHighlightedField("field1", 100, 0)
            .execute()
            .actionGet();
    assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT));
    assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
    for (SearchHit hit : searchResponse.hits()) {
      // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
      assertThat(
          hit.highlightFields().get("field1").fragments()[0],
          equalTo("<em>test</em> " + hit.id() + " "));
    }

    logger.info("--> searching explicitly _all and highlighting on _all");
    searchResponse =
        client
            .prepareSearch()
            .setSize(COUNT)
            .setQuery(termQuery("_all", "test"))
            .addHighlightedField("_all", 100, 0)
            .execute()
            .actionGet();
    assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT));
    assertThat(searchResponse.hits().hits().length, equalTo(COUNT));
    for (SearchHit hit : searchResponse.hits()) {
      // LUCENE 3.1 UPGRADE: Caused adding the space at the end...
      assertThat(
          hit.highlightFields().get("_all").fragments()[0],
          equalTo("<em>test</em> " + hit.id() + "  "));
    }
  }
  @Test
  public void testSimpleVersioningWithFlush() throws Exception {
    try {
      client.admin().indices().prepareDelete("test").execute().actionGet();
    } catch (IndexMissingException e) {
      // its ok
    }
    client.admin().indices().prepareCreate("test").execute().actionGet();
    client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();

    IndexResponse indexResponse =
        client
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .execute()
            .actionGet();
    assertThat(indexResponse.version(), equalTo(1l));

    client.admin().indices().prepareFlush().execute().actionGet();

    indexResponse =
        client
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_2")
            .setVersion(1)
            .execute()
            .actionGet();
    assertThat(indexResponse.version(), equalTo(2l));

    client.admin().indices().prepareFlush().execute().actionGet();

    try {
      client
          .prepareIndex("test", "type", "1")
          .setSource("field1", "value1_1")
          .setVersion(1)
          .execute()
          .actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    try {
      client2
          .prepareIndex("test", "type", "1")
          .setSource("field1", "value1_1")
          .setVersion(1)
          .execute()
          .actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    try {
      client
          .prepareIndex("test", "type", "1")
          .setCreate(true)
          .setSource("field1", "value1_1")
          .setVersion(1)
          .execute()
          .actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    try {
      client2
          .prepareIndex("test", "type", "1")
          .setCreate(true)
          .setSource("field1", "value1_1")
          .setVersion(1)
          .execute()
          .actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    try {
      client.prepareDelete("test", "type", "1").setVersion(1).execute().actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    try {
      client2.prepareDelete("test", "type", "1").setVersion(1).execute().actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    client.admin().indices().prepareRefresh().execute().actionGet();
    for (int i = 0; i < 10; i++) {
      assertThat(
          client.prepareGet("test", "type", "1").execute().actionGet().getVersion(), equalTo(2l));
    }

    for (int i = 0; i < 10; i++) {
      SearchResponse searchResponse =
          client.prepareSearch().setQuery(matchAllQuery()).setVersion(true).execute().actionGet();
      assertThat(searchResponse.getHits().getAt(0).version(), equalTo(2l));
    }
  }
  @Test
  public void testExternalVersioning() throws Exception {
    try {
      client.admin().indices().prepareDelete("test").execute().actionGet();
    } catch (IndexMissingException e) {
      // its ok
    }
    client.admin().indices().prepareCreate("test").execute().actionGet();
    client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();

    IndexResponse indexResponse =
        client
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(12)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(indexResponse.version(), equalTo(12l));

    indexResponse =
        client
            .prepareIndex("test", "type", "1")
            .setSource("field1", "value1_1")
            .setVersion(14)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(indexResponse.version(), equalTo(14l));

    try {
      client
          .prepareIndex("test", "type", "1")
          .setSource("field1", "value1_1")
          .setVersion(13)
          .setVersionType(VersionType.EXTERNAL)
          .execute()
          .actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    client.admin().indices().prepareRefresh().execute().actionGet();
    for (int i = 0; i < 10; i++) {
      assertThat(
          client.prepareGet("test", "type", "1").execute().actionGet().getVersion(), equalTo(14l));
    }

    DeleteResponse deleteResponse =
        client2
            .prepareDelete("test", "type", "1")
            .setVersion(17)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(deleteResponse.isNotFound(), equalTo(false));
    assertThat(deleteResponse.getVersion(), equalTo(17l));

    try {
      client2
          .prepareDelete("test", "type", "1")
          .setVersion(2)
          .setVersionType(VersionType.EXTERNAL)
          .execute()
          .actionGet();
    } catch (ElasticSearchException e) {
      assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class));
    }

    deleteResponse =
        client2
            .prepareDelete("test", "type", "1")
            .setVersion(18)
            .setVersionType(VersionType.EXTERNAL)
            .execute()
            .actionGet();
    assertThat(deleteResponse.isNotFound(), equalTo(true));
    assertThat(deleteResponse.getVersion(), equalTo(18l));
  }
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener) {
    ClusterState clusterState = clusterService.state();
    for (ActionRequest request : bulkRequest.requests) {
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        indexRequest.routing(
            clusterState
                .metaData()
                .resolveIndexRouting(indexRequest.routing(), indexRequest.index()));
        indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index()));
        if (allowIdGeneration) {
          if (indexRequest.id() == null) {
            indexRequest.id(UUID.randomBase64UUID());
            // since we generate the id, change it to CREATE
            indexRequest.opType(IndexRequest.OpType.CREATE);
          }
        }
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index()));
      }
    }
    final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()];

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        // handle routing
        MappingMetaData mappingMd =
            clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type());
        if (mappingMd != null) {
          try {
            indexRequest.processRouting(mappingMd);
          } catch (ElasticSearchException e) {
            responses[i] =
                new BulkItemResponse(
                    i,
                    indexRequest.opType().toString().toLowerCase(),
                    new BulkItemResponse.Failure(
                        indexRequest.index(),
                        indexRequest.type(),
                        indexRequest.id(),
                        e.getDetailedMessage()));
            continue;
          }
        }

        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    indexRequest.index(),
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = Lists.newArrayList();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        MappingMetaData mappingMd =
            clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService
                  .operationRouting()
                  .broadcastDeleteShards(clusterState, deleteRequest.index());
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = Lists.newArrayList();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, request));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .deleteShards(
                      clusterState,
                      deleteRequest.index(),
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = Lists.newArrayList();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              shardId.index().name(),
              shardId.id(),
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.replicationType(bulkRequest.replicationType());
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              synchronized (responses) {
                for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) {
                  responses[bulkItemResponse.itemId()] = bulkItemResponse;
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              String message = ExceptionsHelper.detailedMessage(e);
              synchronized (responses) {
                for (BulkItemRequest request : requests) {
                  if (request.request() instanceof IndexRequest) {
                    IndexRequest indexRequest = (IndexRequest) request.request();
                    responses[request.id()] =
                        new BulkItemResponse(
                            request.id(),
                            indexRequest.opType().toString().toLowerCase(),
                            new BulkItemResponse.Failure(
                                indexRequest.index(),
                                indexRequest.type(),
                                indexRequest.id(),
                                message));
                  } else if (request.request() instanceof DeleteRequest) {
                    DeleteRequest deleteRequest = (DeleteRequest) request.request();
                    responses[request.id()] =
                        new BulkItemResponse(
                            request.id(),
                            "delete",
                            new BulkItemResponse.Failure(
                                deleteRequest.index(),
                                deleteRequest.type(),
                                deleteRequest.id(),
                                message));
                  }
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(responses, System.currentTimeMillis() - startTime));
            }
          });
    }
  }