private void createChatsIndex(IndicesAdminClient indices) {
CreateIndexRequest createBuilder = new CreateIndexRequest("chats");
try {
// @formatter:off
XContentBuilder mappingBuilder =
XContentFactory.jsonBuilder()
.startObject()
.startObject("chat")
.startObject("properties")
.startObject("date")
.field("type", "long")
.endObject()
.startObject("type")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
.endObject();
createBuilder.mapping("chat", mappingBuilder);
// @formatter:on
} catch (IOException e) {
e.printStackTrace();
}
indices.create(createBuilder);
}
@Override
public BaseIngestTransportClient newIndex(String index) {
if (client == null) {
logger.warn("no client for create index");
return this;
}
if (index == null) {
logger.warn("no index name given to create index");
return this;
}
CreateIndexRequest request = new CreateIndexRequest(index);
if (getSettings() != null) {
request.settings(getSettings());
}
if (getMappings() != null) {
for (Map.Entry<String, String> me : getMappings().entrySet()) {
request.mapping(me.getKey(), me.getValue());
}
}
logger.info(
"creating index {} with settings = {}, mappings = {}",
index,
getSettings() != null ? getSettings().getAsMap() : null,
getMappings());
try {
client.admin().indices().create(request).actionGet();
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
return this;
}
CreateIndexRequest buildCreateIndexRequest(
String indexName, FixtureIndexConfiguration indexConfig) throws IOException {
CreateIndexRequest request = new CreateIndexRequest(indexName, createSettings());
if (indexConfig != null) {
for (String typeName : indexConfig.getTypeNames()) {
FixtureTypeConfiguration typeConfig = indexConfig.getTypeConfig(typeName);
if (typeConfig.getMapping() != null) {
request.mapping(typeName, typeConfig.getMapping().toString());
log.debug("Add mapping for type [{}] in index [{}]", typeName, indexName);
}
}
}
return request;
}
private void createChannelsIndex(IndicesAdminClient indices) {
CreateIndexRequest createBuilder = new CreateIndexRequest("channels");
try {
// @formatter:off
XContentBuilder mappingBuilder =
XContentFactory.jsonBuilder()
.startObject()
.startObject("channel")
.startObject("properties")
.startObject("topic")
.field("type", "object") // we only have one so don't use type 'nested'
.endObject()
.startObject("topic.time")
.field("type", "long")
.endObject()
.startObject("last_activity")
.field("type", "long")
.endObject()
.startObject("added_at")
.field("type", "long")
.endObject()
.startObject("last_activity_valid") // was last_valid_content_at
.field("type", "long")
.endObject()
.startObject("_name_suggest")
.field("payloads", true)
.field("index_analyzer", "simple")
.field("search_analyzer", "simple")
.field("type", "completion")
.endObject()
.endObject()
.endObject()
.endObject();
createBuilder.mapping("channel", mappingBuilder);
// @formatter:on
} catch (IOException e) {
e.printStackTrace();
}
indices.create(createBuilder);
}
@Override
protected void doExecute(
final IndexRequest request, final ActionListener<IndexResponse> listener) {
// if we don't have a master, we don't have metadata, that's fine, let it find a master using
// create index API
ClusterState state = clusterService.state();
if (autoCreateIndex.shouldAutoCreate(request.index(), state)) {
CreateIndexRequest createIndexRequest = new CreateIndexRequest(request);
createIndexRequest.index(request.index());
createIndexRequest.mapping(request.type());
createIndexRequest.cause("auto(index api)");
createIndexRequest.masterNodeTimeout(request.timeout());
createIndexAction.execute(
createIndexRequest,
new ActionListener<CreateIndexResponse>() {
@Override
public void onResponse(CreateIndexResponse result) {
innerExecute(request, listener);
}
@Override
public void onFailure(Throwable e) {
if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
// we have the index, do it
try {
innerExecute(request, listener);
} catch (Throwable e1) {
listener.onFailure(e1);
}
} else {
listener.onFailure(e);
}
}
});
} else {
innerExecute(request, listener);
}
}
/** Sets the settings and mappings as a single source. */
@SuppressWarnings("unchecked")
public CreateIndexRequest source(Map<String, ?> source) {
boolean found = false;
for (Map.Entry<String, ?> entry : source.entrySet()) {
String name = entry.getKey();
if (name.equals("settings")) {
found = true;
settings((Map<String, Object>) entry.getValue());
} else if (name.equals("mappings")) {
found = true;
Map<String, Object> mappings = (Map<String, Object>) entry.getValue();
for (Map.Entry<String, Object> entry1 : mappings.entrySet()) {
mapping(entry1.getKey(), (Map<String, Object>) entry1.getValue());
}
} else if (name.equals("aliases")) {
found = true;
aliases((Map<String, Object>) entry.getValue());
} else {
// maybe custom?
IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(name);
if (proto != null) {
found = true;
try {
customs.put(name, proto.fromMap((Map<String, Object>) entry.getValue()));
} catch (IOException e) {
throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", name);
}
}
}
}
if (!found) {
// the top level are settings, use them
settings(source);
}
return this;
}
@Override
protected void doExecute(
final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
final long startTime = System.currentTimeMillis();
final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
if (autoCreateIndex.needToCheck()) {
// Keep track of all unique indices and all unique types per index for the create index
// requests:
final Map<String, Set<String>> indicesAndTypes = new HashMap<>();
for (ActionRequest request : bulkRequest.requests) {
if (request instanceof DocumentRequest) {
DocumentRequest req = (DocumentRequest) request;
Set<String> types = indicesAndTypes.get(req.index());
if (types == null) {
indicesAndTypes.put(req.index(), types = new HashSet<>());
}
types.add(req.type());
} else {
throw new ElasticsearchException(
"Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
}
}
final AtomicInteger counter = new AtomicInteger(indicesAndTypes.size());
ClusterState state = clusterService.state();
for (Map.Entry<String, Set<String>> entry : indicesAndTypes.entrySet()) {
final String index = entry.getKey();
if (autoCreateIndex.shouldAutoCreate(index, state)) {
CreateIndexRequest createIndexRequest = new CreateIndexRequest();
createIndexRequest.index(index);
for (String type : entry.getValue()) {
createIndexRequest.mapping(type);
}
createIndexRequest.cause("auto(bulk api)");
createIndexRequest.masterNodeTimeout(bulkRequest.timeout());
createIndexAction.execute(
createIndexRequest,
new ActionListener<CreateIndexResponse>() {
@Override
public void onResponse(CreateIndexResponse result) {
if (counter.decrementAndGet() == 0) {
try {
executeBulk(bulkRequest, startTime, listener, responses);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
@Override
public void onFailure(Throwable e) {
if (!(ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException)) {
// fail all requests involving this index, if create didnt work
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request != null
&& setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
bulkRequest.requests.set(i, null);
}
}
}
if (counter.decrementAndGet() == 0) {
try {
executeBulk(bulkRequest, startTime, listener, responses);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
});
} else {
if (counter.decrementAndGet() == 0) {
executeBulk(bulkRequest, startTime, listener, responses);
}
}
}
} else {
executeBulk(bulkRequest, startTime, listener, responses);
}
}
/**
* Pull action thread
*
* @param request request
* @param state state
* @param transportClient bulk client for remote cluster access
* @param nodeClient bulk client for local cluster access
*/
final void performPull(
final KnapsackPullRequest request,
final KnapsackState state,
final BulkTransportClient transportClient,
final BulkNodeClient nodeClient) {
try {
logger.info("start of pull: {}", state);
long count = 0L;
Map<String, Set<String>> indices = new HashMap<>();
for (String s : Strings.commaDelimitedListToSet(request.getIndex())) {
indices.put(s, Strings.commaDelimitedListToSet(request.getType()));
}
if (request.withMetadata()) {
// renaming indices/types
if (request.getIndexTypeNames() != null) {
for (Object spec : request.getIndexTypeNames().keySet()) {
if (spec == null) {
continue;
}
String[] s = spec.toString().split("/");
String index = s[0];
String type = s.length > 1 ? s[1] : null;
if (!"_all".equals(index)) {
Set<String> types = indices.get(index);
if (types == null) {
types = new HashSet<>();
}
if (type != null) {
types.add(type);
}
indices.put(index, types);
}
}
}
// get settings for all indices
logger.info("getting settings for indices {}", indices.keySet());
Set<String> settingsIndices = new HashSet<>(indices.keySet());
settingsIndices.remove("_all");
Map<String, String> settings =
getSettings(
transportClient.client(),
settingsIndices.toArray(new String[settingsIndices.size()]));
logger.info("found indices: {}", settings.keySet());
// we resolved the specs in indices to the real indices in the settings
// get mapping and alias per index and create index if copy mode is enabled
for (String index : settings.keySet()) {
CreateIndexRequest createIndexRequest = createIndexRequest(mapIndex(request, index));
Set<String> types = indices.get(index);
createIndexRequest.settings(settings.get(index));
logger.info("getting mappings for index {} and types {}", index, types);
Map<String, String> mappings =
getMapping(
transportClient.client(), index, types != null ? new HashSet<>(types) : null);
logger.info("found mappings: {}", mappings.keySet());
for (String type : mappings.keySet()) {
logger.info("adding mapping: {}", mapType(request, index, type));
createIndexRequest.mapping(mapType(request, index, type), mappings.get(type));
}
// create index
logger.info("creating index: {}", mapIndex(request, index));
nodeClient.client().execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
logger.info("index created: {}", mapIndex(request, index));
logger.info("getting aliases for index {}", index);
Map<String, String> aliases = getAliases(client, index);
logger.info("found {} aliases", aliases.size());
if (!aliases.isEmpty()) {
IndicesAliasesRequestBuilder requestBuilder =
new IndicesAliasesRequestBuilder(
nodeClient.client(), IndicesAliasesAction.INSTANCE);
for (String alias : aliases.keySet()) {
if (aliases.get(alias).isEmpty()) {
requestBuilder.addAlias(index, alias);
} else {
requestBuilder.addAlias(index, alias, aliases.get(alias)); // with filter
}
}
requestBuilder.execute().actionGet();
logger.info("aliases created", aliases.size());
}
}
}
SearchRequest searchRequest = request.getSearchRequest();
if (searchRequest == null) {
searchRequest =
new SearchRequestBuilder(transportClient.client(), SearchAction.INSTANCE)
.setQuery(QueryBuilders.matchAllQuery())
.addSort(SortBuilders.fieldSort("_doc"))
.request();
}
long total = 0L;
for (String index : indices.keySet()) {
if (!"_all".equals(index)) {
searchRequest.indices(index);
}
Set<String> types = indices.get(index);
if (types != null) {
searchRequest.types(types.toArray(new String[types.size()]));
}
searchRequest.scroll(request.getTimeout());
SearchResponse searchResponse =
transportClient.client().execute(SearchAction.INSTANCE, searchRequest).actionGet();
do {
total += searchResponse.getHits().getHits().length;
logger.debug(
"total={} hits={} took={}",
total,
searchResponse.getHits().getHits().length,
searchResponse.getTookInMillis());
for (SearchHit hit : searchResponse.getHits()) {
indexSearchHit(nodeClient, request, hit);
count++;
}
searchResponse =
transportClient
.client()
.execute(
SearchScrollAction.INSTANCE,
new SearchScrollRequest(searchResponse.getScrollId())
.scroll(request.getTimeout()))
.actionGet();
} while (searchResponse.getHits().getHits().length > 0 && !Thread.interrupted());
}
nodeClient.flushIngest();
nodeClient.waitForResponses(TimeValue.timeValueSeconds(60));
for (String index : indices.keySet()) {
nodeClient.refreshIndex(index);
}
nodeClient.shutdown();
transportClient.shutdown();
logger.info("end of pull: {}, docs = {}, count = {}", state, total, count);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
} finally {
try {
knapsack.removeImport(state);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}
/**
* A specialized simplified mapping source method, takes the form of simple properties definition:
* ("field1", "type=string,store=true").
*/
public CreateIndexRequest mapping(String type, Object... source) {
mapping(type, PutMappingRequest.buildFromSimplifiedDef(type, source));
return this;
}