Esempio n. 1
0
 private void createChatsIndex(IndicesAdminClient indices) {
   CreateIndexRequest createBuilder = new CreateIndexRequest("chats");
   try {
     // @formatter:off
     XContentBuilder mappingBuilder =
         XContentFactory.jsonBuilder()
             .startObject()
             .startObject("chat")
             .startObject("properties")
             .startObject("date")
             .field("type", "long")
             .endObject()
             .startObject("type")
             .field("type", "string")
             .field("index", "not_analyzed")
             .endObject()
             .endObject()
             .endObject()
             .endObject();
     createBuilder.mapping("chat", mappingBuilder);
     // @formatter:on
   } catch (IOException e) {
     e.printStackTrace();
   }
   indices.create(createBuilder);
 }
 @Override
 public BaseIngestTransportClient newIndex(String index) {
   if (client == null) {
     logger.warn("no client for create index");
     return this;
   }
   if (index == null) {
     logger.warn("no index name given to create index");
     return this;
   }
   CreateIndexRequest request = new CreateIndexRequest(index);
   if (getSettings() != null) {
     request.settings(getSettings());
   }
   if (getMappings() != null) {
     for (Map.Entry<String, String> me : getMappings().entrySet()) {
       request.mapping(me.getKey(), me.getValue());
     }
   }
   logger.info(
       "creating index {} with settings = {}, mappings = {}",
       index,
       getSettings() != null ? getSettings().getAsMap() : null,
       getMappings());
   try {
     client.admin().indices().create(request).actionGet();
   } catch (Exception e) {
     logger.error(e.getMessage(), e);
   }
   return this;
 }
 CreateIndexRequest buildCreateIndexRequest(
     String indexName, FixtureIndexConfiguration indexConfig) throws IOException {
   CreateIndexRequest request = new CreateIndexRequest(indexName, createSettings());
   if (indexConfig != null) {
     for (String typeName : indexConfig.getTypeNames()) {
       FixtureTypeConfiguration typeConfig = indexConfig.getTypeConfig(typeName);
       if (typeConfig.getMapping() != null) {
         request.mapping(typeName, typeConfig.getMapping().toString());
         log.debug("Add mapping for type [{}] in index [{}]", typeName, indexName);
       }
     }
   }
   return request;
 }
Esempio n. 4
0
 private void createChannelsIndex(IndicesAdminClient indices) {
   CreateIndexRequest createBuilder = new CreateIndexRequest("channels");
   try {
     // @formatter:off
     XContentBuilder mappingBuilder =
         XContentFactory.jsonBuilder()
             .startObject()
             .startObject("channel")
             .startObject("properties")
             .startObject("topic")
             .field("type", "object") // we only have one so don't use type 'nested'
             .endObject()
             .startObject("topic.time")
             .field("type", "long")
             .endObject()
             .startObject("last_activity")
             .field("type", "long")
             .endObject()
             .startObject("added_at")
             .field("type", "long")
             .endObject()
             .startObject("last_activity_valid") // was last_valid_content_at
             .field("type", "long")
             .endObject()
             .startObject("_name_suggest")
             .field("payloads", true)
             .field("index_analyzer", "simple")
             .field("search_analyzer", "simple")
             .field("type", "completion")
             .endObject()
             .endObject()
             .endObject()
             .endObject();
     createBuilder.mapping("channel", mappingBuilder);
     // @formatter:on
   } catch (IOException e) {
     e.printStackTrace();
   }
   indices.create(createBuilder);
 }
  @Override
  protected void doExecute(
      final IndexRequest request, final ActionListener<IndexResponse> listener) {
    // if we don't have a master, we don't have metadata, that's fine, let it find a master using
    // create index API
    ClusterState state = clusterService.state();
    if (autoCreateIndex.shouldAutoCreate(request.index(), state)) {
      CreateIndexRequest createIndexRequest = new CreateIndexRequest(request);
      createIndexRequest.index(request.index());
      createIndexRequest.mapping(request.type());
      createIndexRequest.cause("auto(index api)");
      createIndexRequest.masterNodeTimeout(request.timeout());
      createIndexAction.execute(
          createIndexRequest,
          new ActionListener<CreateIndexResponse>() {
            @Override
            public void onResponse(CreateIndexResponse result) {
              innerExecute(request, listener);
            }

            @Override
            public void onFailure(Throwable e) {
              if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
                // we have the index, do it
                try {
                  innerExecute(request, listener);
                } catch (Throwable e1) {
                  listener.onFailure(e1);
                }
              } else {
                listener.onFailure(e);
              }
            }
          });
    } else {
      innerExecute(request, listener);
    }
  }
Esempio n. 6
0
 /** Sets the settings and mappings as a single source. */
 @SuppressWarnings("unchecked")
 public CreateIndexRequest source(Map<String, ?> source) {
   boolean found = false;
   for (Map.Entry<String, ?> entry : source.entrySet()) {
     String name = entry.getKey();
     if (name.equals("settings")) {
       found = true;
       settings((Map<String, Object>) entry.getValue());
     } else if (name.equals("mappings")) {
       found = true;
       Map<String, Object> mappings = (Map<String, Object>) entry.getValue();
       for (Map.Entry<String, Object> entry1 : mappings.entrySet()) {
         mapping(entry1.getKey(), (Map<String, Object>) entry1.getValue());
       }
     } else if (name.equals("aliases")) {
       found = true;
       aliases((Map<String, Object>) entry.getValue());
     } else {
       // maybe custom?
       IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(name);
       if (proto != null) {
         found = true;
         try {
           customs.put(name, proto.fromMap((Map<String, Object>) entry.getValue()));
         } catch (IOException e) {
           throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", name);
         }
       }
     }
   }
   if (!found) {
     // the top level are settings, use them
     settings(source);
   }
   return this;
 }
  @Override
  protected void doExecute(
      final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
    final long startTime = System.currentTimeMillis();
    final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());

    if (autoCreateIndex.needToCheck()) {
      // Keep track of all unique indices and all unique types per index for the create index
      // requests:
      final Map<String, Set<String>> indicesAndTypes = new HashMap<>();
      for (ActionRequest request : bulkRequest.requests) {
        if (request instanceof DocumentRequest) {
          DocumentRequest req = (DocumentRequest) request;
          Set<String> types = indicesAndTypes.get(req.index());
          if (types == null) {
            indicesAndTypes.put(req.index(), types = new HashSet<>());
          }
          types.add(req.type());
        } else {
          throw new ElasticsearchException(
              "Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
        }
      }
      final AtomicInteger counter = new AtomicInteger(indicesAndTypes.size());
      ClusterState state = clusterService.state();
      for (Map.Entry<String, Set<String>> entry : indicesAndTypes.entrySet()) {
        final String index = entry.getKey();
        if (autoCreateIndex.shouldAutoCreate(index, state)) {
          CreateIndexRequest createIndexRequest = new CreateIndexRequest();
          createIndexRequest.index(index);
          for (String type : entry.getValue()) {
            createIndexRequest.mapping(type);
          }
          createIndexRequest.cause("auto(bulk api)");
          createIndexRequest.masterNodeTimeout(bulkRequest.timeout());
          createIndexAction.execute(
              createIndexRequest,
              new ActionListener<CreateIndexResponse>() {
                @Override
                public void onResponse(CreateIndexResponse result) {
                  if (counter.decrementAndGet() == 0) {
                    try {
                      executeBulk(bulkRequest, startTime, listener, responses);
                    } catch (Throwable t) {
                      listener.onFailure(t);
                    }
                  }
                }

                @Override
                public void onFailure(Throwable e) {
                  if (!(ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException)) {
                    // fail all requests involving this index, if create didnt work
                    for (int i = 0; i < bulkRequest.requests.size(); i++) {
                      ActionRequest request = bulkRequest.requests.get(i);
                      if (request != null
                          && setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
                        bulkRequest.requests.set(i, null);
                      }
                    }
                  }
                  if (counter.decrementAndGet() == 0) {
                    try {
                      executeBulk(bulkRequest, startTime, listener, responses);
                    } catch (Throwable t) {
                      listener.onFailure(t);
                    }
                  }
                }
              });
        } else {
          if (counter.decrementAndGet() == 0) {
            executeBulk(bulkRequest, startTime, listener, responses);
          }
        }
      }
    } else {
      executeBulk(bulkRequest, startTime, listener, responses);
    }
  }
 /**
  * Pull action thread
  *
  * @param request request
  * @param state state
  * @param transportClient bulk client for remote cluster access
  * @param nodeClient bulk client for local cluster access
  */
 final void performPull(
     final KnapsackPullRequest request,
     final KnapsackState state,
     final BulkTransportClient transportClient,
     final BulkNodeClient nodeClient) {
   try {
     logger.info("start of pull: {}", state);
     long count = 0L;
     Map<String, Set<String>> indices = new HashMap<>();
     for (String s : Strings.commaDelimitedListToSet(request.getIndex())) {
       indices.put(s, Strings.commaDelimitedListToSet(request.getType()));
     }
     if (request.withMetadata()) {
       // renaming indices/types
       if (request.getIndexTypeNames() != null) {
         for (Object spec : request.getIndexTypeNames().keySet()) {
           if (spec == null) {
             continue;
           }
           String[] s = spec.toString().split("/");
           String index = s[0];
           String type = s.length > 1 ? s[1] : null;
           if (!"_all".equals(index)) {
             Set<String> types = indices.get(index);
             if (types == null) {
               types = new HashSet<>();
             }
             if (type != null) {
               types.add(type);
             }
             indices.put(index, types);
           }
         }
       }
       // get settings for all indices
       logger.info("getting settings for indices {}", indices.keySet());
       Set<String> settingsIndices = new HashSet<>(indices.keySet());
       settingsIndices.remove("_all");
       Map<String, String> settings =
           getSettings(
               transportClient.client(),
               settingsIndices.toArray(new String[settingsIndices.size()]));
       logger.info("found indices: {}", settings.keySet());
       // we resolved the specs in indices to the real indices in the settings
       // get mapping and alias per index and create index if copy mode is enabled
       for (String index : settings.keySet()) {
         CreateIndexRequest createIndexRequest = createIndexRequest(mapIndex(request, index));
         Set<String> types = indices.get(index);
         createIndexRequest.settings(settings.get(index));
         logger.info("getting mappings for index {} and types {}", index, types);
         Map<String, String> mappings =
             getMapping(
                 transportClient.client(), index, types != null ? new HashSet<>(types) : null);
         logger.info("found mappings: {}", mappings.keySet());
         for (String type : mappings.keySet()) {
           logger.info("adding mapping: {}", mapType(request, index, type));
           createIndexRequest.mapping(mapType(request, index, type), mappings.get(type));
         }
         // create index
         logger.info("creating index: {}", mapIndex(request, index));
         nodeClient.client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
         logger.info("index created: {}", mapIndex(request, index));
         logger.info("getting aliases for index {}", index);
         Map<String, String> aliases = getAliases(client, index);
         logger.info("found {} aliases", aliases.size());
         if (!aliases.isEmpty()) {
           IndicesAliasesRequestBuilder requestBuilder =
               new IndicesAliasesRequestBuilder(
                   nodeClient.client(), IndicesAliasesAction.INSTANCE);
           for (String alias : aliases.keySet()) {
             if (aliases.get(alias).isEmpty()) {
               requestBuilder.addAlias(index, alias);
             } else {
               requestBuilder.addAlias(index, alias, aliases.get(alias)); // with filter
             }
           }
           requestBuilder.execute().actionGet();
           logger.info("aliases created", aliases.size());
         }
       }
     }
     SearchRequest searchRequest = request.getSearchRequest();
     if (searchRequest == null) {
       searchRequest =
           new SearchRequestBuilder(transportClient.client(), SearchAction.INSTANCE)
               .setQuery(QueryBuilders.matchAllQuery())
               .addSort(SortBuilders.fieldSort("_doc"))
               .request();
     }
     long total = 0L;
     for (String index : indices.keySet()) {
       if (!"_all".equals(index)) {
         searchRequest.indices(index);
       }
       Set<String> types = indices.get(index);
       if (types != null) {
         searchRequest.types(types.toArray(new String[types.size()]));
       }
       searchRequest.scroll(request.getTimeout());
       SearchResponse searchResponse =
           transportClient.client().execute(SearchAction.INSTANCE, searchRequest).actionGet();
       do {
         total += searchResponse.getHits().getHits().length;
         logger.debug(
             "total={} hits={} took={}",
             total,
             searchResponse.getHits().getHits().length,
             searchResponse.getTookInMillis());
         for (SearchHit hit : searchResponse.getHits()) {
           indexSearchHit(nodeClient, request, hit);
           count++;
         }
         searchResponse =
             transportClient
                 .client()
                 .execute(
                     SearchScrollAction.INSTANCE,
                     new SearchScrollRequest(searchResponse.getScrollId())
                         .scroll(request.getTimeout()))
                 .actionGet();
       } while (searchResponse.getHits().getHits().length > 0 && !Thread.interrupted());
     }
     nodeClient.flushIngest();
     nodeClient.waitForResponses(TimeValue.timeValueSeconds(60));
     for (String index : indices.keySet()) {
       nodeClient.refreshIndex(index);
     }
     nodeClient.shutdown();
     transportClient.shutdown();
     logger.info("end of pull: {}, docs = {}, count = {}", state, total, count);
   } catch (Throwable e) {
     logger.error(e.getMessage(), e);
   } finally {
     try {
       knapsack.removeImport(state);
     } catch (IOException e) {
       logger.error(e.getMessage(), e);
     }
   }
 }
Esempio n. 9
0
 /**
  * A specialized simplified mapping source method, takes the form of simple properties definition:
  * ("field1", "type=string,store=true").
  */
 public CreateIndexRequest mapping(String type, Object... source) {
   mapping(type, PutMappingRequest.buildFromSimplifiedDef(type, source));
   return this;
 }