private void createChatsIndex(IndicesAdminClient indices) { CreateIndexRequest createBuilder = new CreateIndexRequest("chats"); try { // @formatter:off XContentBuilder mappingBuilder = XContentFactory.jsonBuilder() .startObject() .startObject("chat") .startObject("properties") .startObject("date") .field("type", "long") .endObject() .startObject("type") .field("type", "string") .field("index", "not_analyzed") .endObject() .endObject() .endObject() .endObject(); createBuilder.mapping("chat", mappingBuilder); // @formatter:on } catch (IOException e) { e.printStackTrace(); } indices.create(createBuilder); }
@Override public BaseIngestTransportClient newIndex(String index) { if (client == null) { logger.warn("no client for create index"); return this; } if (index == null) { logger.warn("no index name given to create index"); return this; } CreateIndexRequest request = new CreateIndexRequest(index); if (getSettings() != null) { request.settings(getSettings()); } if (getMappings() != null) { for (Map.Entry<String, String> me : getMappings().entrySet()) { request.mapping(me.getKey(), me.getValue()); } } logger.info( "creating index {} with settings = {}, mappings = {}", index, getSettings() != null ? getSettings().getAsMap() : null, getMappings()); try { client.admin().indices().create(request).actionGet(); } catch (Exception e) { logger.error(e.getMessage(), e); } return this; }
CreateIndexRequest buildCreateIndexRequest( String indexName, FixtureIndexConfiguration indexConfig) throws IOException { CreateIndexRequest request = new CreateIndexRequest(indexName, createSettings()); if (indexConfig != null) { for (String typeName : indexConfig.getTypeNames()) { FixtureTypeConfiguration typeConfig = indexConfig.getTypeConfig(typeName); if (typeConfig.getMapping() != null) { request.mapping(typeName, typeConfig.getMapping().toString()); log.debug("Add mapping for type [{}] in index [{}]", typeName, indexName); } } } return request; }
private void createChannelsIndex(IndicesAdminClient indices) { CreateIndexRequest createBuilder = new CreateIndexRequest("channels"); try { // @formatter:off XContentBuilder mappingBuilder = XContentFactory.jsonBuilder() .startObject() .startObject("channel") .startObject("properties") .startObject("topic") .field("type", "object") // we only have one so don't use type 'nested' .endObject() .startObject("topic.time") .field("type", "long") .endObject() .startObject("last_activity") .field("type", "long") .endObject() .startObject("added_at") .field("type", "long") .endObject() .startObject("last_activity_valid") // was last_valid_content_at .field("type", "long") .endObject() .startObject("_name_suggest") .field("payloads", true) .field("index_analyzer", "simple") .field("search_analyzer", "simple") .field("type", "completion") .endObject() .endObject() .endObject() .endObject(); createBuilder.mapping("channel", mappingBuilder); // @formatter:on } catch (IOException e) { e.printStackTrace(); } indices.create(createBuilder); }
@Override protected void doExecute( final IndexRequest request, final ActionListener<IndexResponse> listener) { // if we don't have a master, we don't have metadata, that's fine, let it find a master using // create index API ClusterState state = clusterService.state(); if (autoCreateIndex.shouldAutoCreate(request.index(), state)) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(request); createIndexRequest.index(request.index()); createIndexRequest.mapping(request.type()); createIndexRequest.cause("auto(index api)"); createIndexRequest.masterNodeTimeout(request.timeout()); createIndexAction.execute( createIndexRequest, new ActionListener<CreateIndexResponse>() { @Override public void onResponse(CreateIndexResponse result) { innerExecute(request, listener); } @Override public void onFailure(Throwable e) { if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) { // we have the index, do it try { innerExecute(request, listener); } catch (Throwable e1) { listener.onFailure(e1); } } else { listener.onFailure(e); } } }); } else { innerExecute(request, listener); } }
@Override protected void masterOperation( final CreateIndexRequest request, final ClusterState state, final ActionListener<CreateIndexResponse> listener) throws ElasticsearchException { String cause = request.cause(); if (cause.length() == 0) { cause = "api"; } CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(cause, request.index()) .ackTimeout(request.timeout()) .masterNodeTimeout(request.masterNodeTimeout()) .settings(request.settings()) .mappings(request.mappings()) .aliases(request.aliases()) .customs(request.customs()); createIndexService.createIndex( updateRequest, new ActionListener<ClusterStateUpdateResponse>() { @Override public void onResponse(ClusterStateUpdateResponse response) { listener.onResponse(new CreateIndexResponse(response.isAcknowledged())); } @Override public void onFailure(Throwable t) { if (t instanceof IndexAlreadyExistsException) { logger.trace("[{}] failed to create", t, request.index()); } else { logger.debug("[{}] failed to create", t, request.index()); } listener.onFailure(t); } }); }
@Override protected ClusterBlockException checkBlock(CreateIndexRequest request, ClusterState state) { return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA, request.index()); }
@Override protected void doExecute( final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) { final long startTime = System.currentTimeMillis(); final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size()); if (autoCreateIndex.needToCheck()) { // Keep track of all unique indices and all unique types per index for the create index // requests: final Map<String, Set<String>> indicesAndTypes = new HashMap<>(); for (ActionRequest request : bulkRequest.requests) { if (request instanceof DocumentRequest) { DocumentRequest req = (DocumentRequest) request; Set<String> types = indicesAndTypes.get(req.index()); if (types == null) { indicesAndTypes.put(req.index(), types = new HashSet<>()); } types.add(req.type()); } else { throw new ElasticsearchException( "Parsed unknown request in bulk actions: " + request.getClass().getSimpleName()); } } final AtomicInteger counter = new AtomicInteger(indicesAndTypes.size()); ClusterState state = clusterService.state(); for (Map.Entry<String, Set<String>> entry : indicesAndTypes.entrySet()) { final String index = entry.getKey(); if (autoCreateIndex.shouldAutoCreate(index, state)) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(); createIndexRequest.index(index); for (String type : entry.getValue()) { createIndexRequest.mapping(type); } createIndexRequest.cause("auto(bulk api)"); createIndexRequest.masterNodeTimeout(bulkRequest.timeout()); createIndexAction.execute( createIndexRequest, new ActionListener<CreateIndexResponse>() { @Override public void onResponse(CreateIndexResponse result) { if (counter.decrementAndGet() == 0) { try { executeBulk(bulkRequest, startTime, listener, responses); } catch (Throwable t) { listener.onFailure(t); } } } @Override public void onFailure(Throwable e) { if (!(ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException)) { // fail all requests involving this index, if create didnt work for (int i = 0; i < bulkRequest.requests.size(); i++) { ActionRequest request = bulkRequest.requests.get(i); if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) { bulkRequest.requests.set(i, null); } } } if (counter.decrementAndGet() == 0) { try { executeBulk(bulkRequest, startTime, listener, responses); } catch (Throwable t) { listener.onFailure(t); } } } }); } else { if (counter.decrementAndGet() == 0) { executeBulk(bulkRequest, startTime, listener, responses); } } } } else { executeBulk(bulkRequest, startTime, listener, responses); } }
/** * Pull action thread * * @param request request * @param state state * @param transportClient bulk client for remote cluster access * @param nodeClient bulk client for local cluster access */ final void performPull( final KnapsackPullRequest request, final KnapsackState state, final BulkTransportClient transportClient, final BulkNodeClient nodeClient) { try { logger.info("start of pull: {}", state); long count = 0L; Map<String, Set<String>> indices = new HashMap<>(); for (String s : Strings.commaDelimitedListToSet(request.getIndex())) { indices.put(s, Strings.commaDelimitedListToSet(request.getType())); } if (request.withMetadata()) { // renaming indices/types if (request.getIndexTypeNames() != null) { for (Object spec : request.getIndexTypeNames().keySet()) { if (spec == null) { continue; } String[] s = spec.toString().split("/"); String index = s[0]; String type = s.length > 1 ? s[1] : null; if (!"_all".equals(index)) { Set<String> types = indices.get(index); if (types == null) { types = new HashSet<>(); } if (type != null) { types.add(type); } indices.put(index, types); } } } // get settings for all indices logger.info("getting settings for indices {}", indices.keySet()); Set<String> settingsIndices = new HashSet<>(indices.keySet()); settingsIndices.remove("_all"); Map<String, String> settings = getSettings( transportClient.client(), settingsIndices.toArray(new String[settingsIndices.size()])); logger.info("found indices: {}", settings.keySet()); // we resolved the specs in indices to the real indices in the settings // get mapping and alias per index and create index if copy mode is enabled for (String index : settings.keySet()) { CreateIndexRequest createIndexRequest = createIndexRequest(mapIndex(request, index)); Set<String> types = indices.get(index); createIndexRequest.settings(settings.get(index)); logger.info("getting mappings for index {} and types {}", index, types); Map<String, String> mappings = getMapping( transportClient.client(), index, types != null ? new HashSet<>(types) : null); logger.info("found mappings: {}", mappings.keySet()); for (String type : mappings.keySet()) { logger.info("adding mapping: {}", mapType(request, index, type)); createIndexRequest.mapping(mapType(request, index, type), mappings.get(type)); } // create index logger.info("creating index: {}", mapIndex(request, index)); nodeClient.client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); logger.info("index created: {}", mapIndex(request, index)); logger.info("getting aliases for index {}", index); Map<String, String> aliases = getAliases(client, index); logger.info("found {} aliases", aliases.size()); if (!aliases.isEmpty()) { IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder( nodeClient.client(), IndicesAliasesAction.INSTANCE); for (String alias : aliases.keySet()) { if (aliases.get(alias).isEmpty()) { requestBuilder.addAlias(index, alias); } else { requestBuilder.addAlias(index, alias, aliases.get(alias)); // with filter } } requestBuilder.execute().actionGet(); logger.info("aliases created", aliases.size()); } } } SearchRequest searchRequest = request.getSearchRequest(); if (searchRequest == null) { searchRequest = new SearchRequestBuilder(transportClient.client(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()) .addSort(SortBuilders.fieldSort("_doc")) .request(); } long total = 0L; for (String index : indices.keySet()) { if (!"_all".equals(index)) { searchRequest.indices(index); } Set<String> types = indices.get(index); if (types != null) { searchRequest.types(types.toArray(new String[types.size()])); } searchRequest.scroll(request.getTimeout()); SearchResponse searchResponse = transportClient.client().execute(SearchAction.INSTANCE, searchRequest).actionGet(); do { total += searchResponse.getHits().getHits().length; logger.debug( "total={} hits={} took={}", total, searchResponse.getHits().getHits().length, searchResponse.getTookInMillis()); for (SearchHit hit : searchResponse.getHits()) { indexSearchHit(nodeClient, request, hit); count++; } searchResponse = transportClient .client() .execute( SearchScrollAction.INSTANCE, new SearchScrollRequest(searchResponse.getScrollId()) .scroll(request.getTimeout())) .actionGet(); } while (searchResponse.getHits().getHits().length > 0 && !Thread.interrupted()); } nodeClient.flushIngest(); nodeClient.waitForResponses(TimeValue.timeValueSeconds(60)); for (String index : indices.keySet()) { nodeClient.refreshIndex(index); } nodeClient.shutdown(); transportClient.shutdown(); logger.info("end of pull: {}, docs = {}, count = {}", state, total, count); } catch (Throwable e) { logger.error(e.getMessage(), e); } finally { try { knapsack.removeImport(state); } catch (IOException e) { logger.error(e.getMessage(), e); } } }