private void indexSearchHit( BulkNodeClient bulkNodeClient, KnapsackPullRequest request, SearchHit hit) throws IOException { IndexRequest indexRequest = new IndexRequest( mapIndex(request, hit.getIndex()), mapType(request, hit.getIndex(), hit.getType()), hit.getId()); for (String f : hit.getFields().keySet()) { switch (f) { case "_parent": indexRequest.parent(hit.getFields().get(f).getValue().toString()); break; case "_routing": indexRequest.routing(hit.getFields().get(f).getValue().toString()); break; case "_timestamp": indexRequest.timestamp(hit.getFields().get(f).getValue().toString()); break; case "_version": indexRequest .versionType(VersionType.EXTERNAL) .version(Long.parseLong(hit.getFields().get(f).getValue().toString())); break; case "_source": indexRequest.source(hit.getSourceAsString()); break; default: indexRequest.source(f, hit.getFields().get(f).getValue().toString()); break; } } if (!hit.getFields().keySet().contains("_source")) { indexRequest.source(hit.getSourceAsString()); } bulkNodeClient.bulkIndex(indexRequest); }
/** * Pull action thread * * @param request request * @param state state * @param transportClient bulk client for remote cluster access * @param nodeClient bulk client for local cluster access */ final void performPull( final KnapsackPullRequest request, final KnapsackState state, final BulkTransportClient transportClient, final BulkNodeClient nodeClient) { try { logger.info("start of pull: {}", state); long count = 0L; Map<String, Set<String>> indices = new HashMap<>(); for (String s : Strings.commaDelimitedListToSet(request.getIndex())) { indices.put(s, Strings.commaDelimitedListToSet(request.getType())); } if (request.withMetadata()) { // renaming indices/types if (request.getIndexTypeNames() != null) { for (Object spec : request.getIndexTypeNames().keySet()) { if (spec == null) { continue; } String[] s = spec.toString().split("/"); String index = s[0]; String type = s.length > 1 ? s[1] : null; if (!"_all".equals(index)) { Set<String> types = indices.get(index); if (types == null) { types = new HashSet<>(); } if (type != null) { types.add(type); } indices.put(index, types); } } } // get settings for all indices logger.info("getting settings for indices {}", indices.keySet()); Set<String> settingsIndices = new HashSet<>(indices.keySet()); settingsIndices.remove("_all"); Map<String, String> settings = getSettings( transportClient.client(), settingsIndices.toArray(new String[settingsIndices.size()])); logger.info("found indices: {}", settings.keySet()); // we resolved the specs in indices to the real indices in the settings // get mapping and alias per index and create index if copy mode is enabled for (String index : settings.keySet()) { CreateIndexRequest createIndexRequest = createIndexRequest(mapIndex(request, index)); Set<String> types = indices.get(index); createIndexRequest.settings(settings.get(index)); logger.info("getting mappings for index {} and types {}", index, types); Map<String, String> mappings = getMapping( transportClient.client(), index, types != null ? new HashSet<>(types) : null); logger.info("found mappings: {}", mappings.keySet()); for (String type : mappings.keySet()) { logger.info("adding mapping: {}", mapType(request, index, type)); createIndexRequest.mapping(mapType(request, index, type), mappings.get(type)); } // create index logger.info("creating index: {}", mapIndex(request, index)); nodeClient.client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); logger.info("index created: {}", mapIndex(request, index)); logger.info("getting aliases for index {}", index); Map<String, String> aliases = getAliases(client, index); logger.info("found {} aliases", aliases.size()); if (!aliases.isEmpty()) { IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder( nodeClient.client(), IndicesAliasesAction.INSTANCE); for (String alias : aliases.keySet()) { if (aliases.get(alias).isEmpty()) { requestBuilder.addAlias(index, alias); } else { requestBuilder.addAlias(index, alias, aliases.get(alias)); // with filter } } requestBuilder.execute().actionGet(); logger.info("aliases created", aliases.size()); } } } SearchRequest searchRequest = request.getSearchRequest(); if (searchRequest == null) { searchRequest = new SearchRequestBuilder(transportClient.client(), SearchAction.INSTANCE) .setQuery(QueryBuilders.matchAllQuery()) .addSort(SortBuilders.fieldSort("_doc")) .request(); } long total = 0L; for (String index : indices.keySet()) { if (!"_all".equals(index)) { searchRequest.indices(index); } Set<String> types = indices.get(index); if (types != null) { searchRequest.types(types.toArray(new String[types.size()])); } searchRequest.scroll(request.getTimeout()); SearchResponse searchResponse = transportClient.client().execute(SearchAction.INSTANCE, searchRequest).actionGet(); do { total += searchResponse.getHits().getHits().length; logger.debug( "total={} hits={} took={}", total, searchResponse.getHits().getHits().length, searchResponse.getTookInMillis()); for (SearchHit hit : searchResponse.getHits()) { indexSearchHit(nodeClient, request, hit); count++; } searchResponse = transportClient .client() .execute( SearchScrollAction.INSTANCE, new SearchScrollRequest(searchResponse.getScrollId()) .scroll(request.getTimeout())) .actionGet(); } while (searchResponse.getHits().getHits().length > 0 && !Thread.interrupted()); } nodeClient.flushIngest(); nodeClient.waitForResponses(TimeValue.timeValueSeconds(60)); for (String index : indices.keySet()) { nodeClient.refreshIndex(index); } nodeClient.shutdown(); transportClient.shutdown(); logger.info("end of pull: {}, docs = {}, count = {}", state, total, count); } catch (Throwable e) { logger.error(e.getMessage(), e); } finally { try { knapsack.removeImport(state); } catch (IOException e) { logger.error(e.getMessage(), e); } } }