@Override
protected void masterOperation(
final SnapshotsStatusRequest request,
final ClusterState state,
final ActionListener<SnapshotsStatusResponse> listener)
throws Exception {
ImmutableList<SnapshotMetaData.Entry> currentSnapshots =
snapshotsService.currentSnapshots(request.repository(), request.snapshots());
if (currentSnapshots.isEmpty()) {
listener.onResponse(buildResponse(request, currentSnapshots, null));
return;
}
Set<String> nodesIds = newHashSet();
for (SnapshotMetaData.Entry entry : currentSnapshots) {
for (SnapshotMetaData.ShardSnapshotStatus status : entry.shards().values()) {
if (status.nodeId() != null) {
nodesIds.add(status.nodeId());
}
}
}
if (!nodesIds.isEmpty()) {
// There are still some snapshots running - check their progress
SnapshotId[] snapshotIds = new SnapshotId[currentSnapshots.size()];
for (int i = 0; i < currentSnapshots.size(); i++) {
snapshotIds[i] = currentSnapshots.get(i).snapshotId();
}
transportNodesSnapshotsStatus.status(
nodesIds.toArray(new String[nodesIds.size()]),
snapshotIds,
request.masterNodeTimeout(),
new ActionListener<TransportNodesSnapshotsStatus.NodesSnapshotStatus>() {
@Override
public void onResponse(
TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses) {
try {
ImmutableList<SnapshotMetaData.Entry> currentSnapshots =
snapshotsService.currentSnapshots(request.repository(), request.snapshots());
listener.onResponse(buildResponse(request, currentSnapshots, nodeSnapshotStatuses));
} catch (Throwable e) {
listener.onFailure(e);
}
}
@Override
public void onFailure(Throwable e) {
listener.onFailure(e);
}
});
} else {
// We don't have any in-progress shards, just return current stats
listener.onResponse(buildResponse(request, currentSnapshots, null));
}
}
public void start() {
if (shardsIts.size() == 0) {
// no shards
try {
listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
} catch (Throwable e) {
listener.onFailure(e);
}
return;
}
request.beforeStart();
// count the local operations, and perform the non local ones
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.nextOrNull();
if (shard != null) {
performOperation(shardIt, shard, shardIndex);
} else {
// really, no shards active in this group
onOperation(
null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
}
}
}
public void start() {
if (expectedSuccessfulOps == 0) {
// no search shards to search on, bail with empty response
// (it happens with search across _all with no indices around and consistent with broadcast
// operations)
listener.onResponse(
new SearchResponse(
InternalSearchResponse.empty(),
null,
0,
0,
buildTookInMillis(),
ShardSearchFailure.EMPTY_ARRAY));
return;
}
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.nextOrNull();
if (shard != null) {
performFirstPhase(shardIndex, shardIt, shard);
} else {
// really, no shards active in this group
onFirstPhaseResult(
shardIndex,
null,
null,
shardIt,
new NoShardAvailableActionException(shardIt.shardId()));
}
}
}
@Override
public CountDownLatch updateNodeStats(final ActionListener<NodesStatsResponse> listener) {
NodesStatsResponse response =
new NodesStatsResponse(clusterName, Arrays.asList(stats), Collections.emptyList());
listener.onResponse(response);
return new CountDownLatch(0);
}
@Override
protected void doExecute(
final KnapsackPullRequest request, ActionListener<KnapsackPullResponse> listener) {
final KnapsackState state =
new KnapsackState().setMode("pull").setNodeName(nodeService.nodeName());
final KnapsackPullResponse response = new KnapsackPullResponse().setState(state);
try {
final BulkTransportClient transportClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.put(clientSettings(client, request))
.toBulkTransportClient();
final BulkNodeClient nodeClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.toBulkNodeClient(client);
state.setTimestamp(new DateTime());
response.setRunning(true);
knapsack.submit(
new Thread() {
public void run() {
performPull(request, state, transportClient, nodeClient);
}
});
listener.onResponse(response);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
listener.onFailure(e);
}
}
@Override
public void login(final RestRequest request, final ActionListener<String[]> listener) {
String username = request.param(usernameKey);
String password = request.param(passwordKey);
final BytesReference content = request.content();
final XContentType xContentType = XContentFactory.xContentType(content);
XContentParser parser = null;
try {
parser = XContentFactory.xContent(xContentType).createParser(content);
final XContentParser.Token t = parser.nextToken();
if (t != null) {
final Map<String, Object> contentMap = parser.map();
username = MapUtil.getAsString(contentMap, usernameKey, username);
password = MapUtil.getAsString(contentMap, passwordKey, password);
}
} catch (final Exception e) {
listener.onFailure(e);
return;
} finally {
if (parser != null) {
parser.close();
}
}
if (username == null) {
listener.onResponse(new String[0]);
return;
}
processLogin(username, password, listener);
}
private void doFinish() {
if (finished.compareAndSet(false, true)) {
Releasables.close(indexShardReference);
final ShardId shardId = shardIt.shardId();
final ActionWriteResponse.ShardInfo.Failure[] failuresArray;
if (!shardReplicaFailures.isEmpty()) {
int slot = 0;
failuresArray = new ActionWriteResponse.ShardInfo.Failure[shardReplicaFailures.size()];
for (Map.Entry<String, Throwable> entry : shardReplicaFailures.entrySet()) {
RestStatus restStatus = ExceptionsHelper.status(entry.getValue());
failuresArray[slot++] =
new ActionWriteResponse.ShardInfo.Failure(
shardId.getIndex(),
shardId.getId(),
entry.getKey(),
entry.getValue(),
restStatus,
false);
}
} else {
failuresArray = ActionWriteResponse.EMPTY;
}
finalResponse.setShardInfo(
new ActionWriteResponse.ShardInfo(totalShards, success.get(), failuresArray));
listener.onResponse(finalResponse);
}
}
void finishHim() {
try {
listener.onResponse(newResponse(request, shardsResponses, clusterState));
} catch (Throwable e) {
listener.onFailure(e);
}
}
void finishOnRemoteSuccess(Response response) {
if (finished.compareAndSet(false, true)) {
logger.trace("operation succeeded");
listener.onResponse(response);
} else {
assert false : "finishOnRemoteSuccess called but operation is already finished";
}
}
public void finishVerification(
ActionListener<VerifyResponse> listener,
List<DiscoveryNode> nodes,
CopyOnWriteArrayList<VerificationFailure> errors) {
listener.onResponse(
new RepositoriesService.VerifyResponse(
nodes.toArray(new DiscoveryNode[nodes.size()]),
errors.toArray(new VerificationFailure[errors.size()])));
}
@Override
protected void doMasterOperation(
final GetWarmersRequest request,
final ClusterState state,
final ActionListener<GetWarmersResponse> listener)
throws ElasticSearchException {
ImmutableOpenMap<String, ImmutableList<IndexWarmersMetaData.Entry>> result =
state.metaData().findWarmers(request.indices(), request.types(), request.warmers());
listener.onResponse(new GetWarmersResponse(result));
}
@Override
protected void masterOperation(
final ClusterStateRequest request,
final ClusterState state,
ActionListener<ClusterStateResponse> listener)
throws ElasticsearchException {
ClusterState currentState = clusterService.state();
logger.trace("Serving cluster state request using version {}", currentState.version());
ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName());
builder.version(currentState.version());
if (request.nodes()) {
builder.nodes(currentState.nodes());
}
if (request.routingTable()) {
if (request.indices().length > 0) {
RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
for (String filteredIndex : request.indices()) {
if (currentState.routingTable().getIndicesRouting().containsKey(filteredIndex)) {
routingTableBuilder.add(
currentState.routingTable().getIndicesRouting().get(filteredIndex));
}
}
builder.routingTable(routingTableBuilder);
} else {
builder.routingTable(currentState.routingTable());
}
}
if (request.blocks()) {
builder.blocks(currentState.blocks());
}
if (request.metaData()) {
MetaData.Builder mdBuilder;
if (request.indices().length == 0) {
mdBuilder = MetaData.builder(currentState.metaData());
} else {
mdBuilder = MetaData.builder();
}
if (request.indices().length > 0) {
String[] indices =
currentState
.metaData()
.concreteIndices(IndicesOptions.lenientExpandOpen(), request.indices());
for (String filteredIndex : indices) {
IndexMetaData indexMetaData = currentState.metaData().index(filteredIndex);
if (indexMetaData != null) {
mdBuilder.put(indexMetaData, false);
}
}
}
builder.metaData(mdBuilder);
}
listener.onResponse(new ClusterStateResponse(clusterName, builder.build()));
}
@Override
protected void masterOperation(
GetAliasesRequest request, ClusterState state, ActionListener<GetAliasesResponse> listener)
throws ElasticsearchException {
String[] concreteIndices =
state.metaData().concreteIndices(request.indicesOptions(), request.indices());
@SuppressWarnings("unchecked") // ImmutableList to List results incompatible type
ImmutableOpenMap<String, List<AliasMetaData>> result =
(ImmutableOpenMap) state.metaData().findAliases(request.aliases(), concreteIndices);
listener.onResponse(new GetAliasesResponse(result));
}
@Override
public <
Request extends ActionRequest,
Response extends ActionResponse,
RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder, Client>>
void execute(
Action<Request, Response, RequestBuilder, Client> action,
Request request,
ActionListener<Response> listener) {
listener.onResponse(null);
}
@Override
protected void masterOperation(
final TypesExistsRequest request,
final ClusterState state,
final ActionListener<TypesExistsResponse> listener) {
String[] concreteIndices =
indexNameExpressionResolver.concreteIndexNames(
state, request.indicesOptions(), request.indices());
if (concreteIndices.length == 0) {
listener.onResponse(new TypesExistsResponse(false));
return;
}
for (String concreteIndex : concreteIndices) {
if (!state.metaData().hasConcreteIndex(concreteIndex)) {
listener.onResponse(new TypesExistsResponse(false));
return;
}
ImmutableOpenMap<String, MappingMetaData> mappings =
state.metaData().getIndices().get(concreteIndex).getMappings();
if (mappings.isEmpty()) {
listener.onResponse(new TypesExistsResponse(false));
return;
}
for (String type : request.types()) {
if (!mappings.containsKey(type)) {
listener.onResponse(new TypesExistsResponse(false));
return;
}
}
}
listener.onResponse(new TypesExistsResponse(true));
}
private void finish() {
if (finished.compareAndSet(false, true)) {
final ReplicationResponse.ShardInfo.Failure[] failuresArray;
if (shardReplicaFailures.isEmpty()) {
failuresArray = ReplicationResponse.EMPTY;
} else {
failuresArray = new ReplicationResponse.ShardInfo.Failure[shardReplicaFailures.size()];
shardReplicaFailures.toArray(failuresArray);
}
primaryResult.setShardInfo(
new ReplicationResponse.ShardInfo(
totalShards.get(), successfulShards.get(), failuresArray));
resultListener.onResponse(primaryResult);
}
}
@Override
protected void masterOperation(
final TypesExistsRequest request,
final ClusterState state,
final ActionListener<TypesExistsResponse> listener)
throws ElasticSearchException {
String[] concreteIndices =
state.metaData().concreteIndices(request.indices(), request.ignoreIndices(), false);
if (concreteIndices.length == 0) {
listener.onResponse(new TypesExistsResponse(false));
return;
}
for (String concreteIndex : concreteIndices) {
if (!state.metaData().hasConcreteIndex(concreteIndex)) {
listener.onResponse(new TypesExistsResponse(false));
return;
}
ImmutableMap<String, MappingMetaData> mappings =
state.metaData().getIndices().get(concreteIndex).mappings();
if (mappings.isEmpty()) {
listener.onResponse(new TypesExistsResponse(false));
return;
}
for (String type : request.types()) {
if (!mappings.containsKey(type)) {
listener.onResponse(new TypesExistsResponse(false));
return;
}
}
}
listener.onResponse(new TypesExistsResponse(true));
}
private void innerFinishHim() {
InternalSearchResponse internalResponse =
searchPhaseController.merge(sortedShardList, queryResults, fetchResults);
String scrollId = null;
if (request.scroll() != null) {
scrollId = request.scrollId();
}
listener.onResponse(
new SearchResponse(
internalResponse,
scrollId,
this.scrollId.getContext().length,
successfulOps.get(),
System.currentTimeMillis() - startTime,
buildShardFailures()));
}
protected void onCompletion() {
Response response = null;
try {
response =
newResponse(request, responses, unavailableShardExceptions, nodeIds, clusterState);
} catch (Throwable t) {
logger.debug("failed to combine responses from nodes", t);
listener.onFailure(t);
}
if (response != null) {
try {
listener.onResponse(response);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
void finish() {
MultiPercolateResponse.Item[] finalResponse =
new MultiPercolateResponse.Item[reducedResponses.length()];
for (int slot = 0; slot < reducedResponses.length(); slot++) {
Object element = reducedResponses.get(slot);
assert element != null : "Element[" + slot + "] shouldn't be null";
if (element instanceof PercolateResponse) {
finalResponse[slot] = new MultiPercolateResponse.Item((PercolateResponse) element);
} else if (element instanceof Throwable) {
finalResponse[slot] = new MultiPercolateResponse.Item((Throwable) element);
} else if (element instanceof MultiGetResponse.Failure) {
finalResponse[slot] =
new MultiPercolateResponse.Item(((MultiGetResponse.Failure) element).getFailure());
}
}
finalListener.onResponse(new MultiPercolateResponse(finalResponse));
}
private void innerFinishHim() throws Exception {
ScoreDoc[] sortedShardList = searchPhaseController.sortDocs(true, queryFetchResults);
final InternalSearchResponse internalResponse =
searchPhaseController.merge(
sortedShardList, queryFetchResults, queryFetchResults, request);
String scrollId = null;
if (request.scroll() != null) {
scrollId = request.scrollId();
}
listener.onResponse(
new SearchResponse(
internalResponse,
scrollId,
this.scrollId.getContext().length,
successfulOps.get(),
buildTookInMillis(),
buildShardFailures()));
}
@Override
protected void doExecute(
GetFieldMappingsRequest request, final ActionListener<GetFieldMappingsResponse> listener) {
ClusterState clusterState = clusterService.state();
String[] concreteIndices =
indexNameExpressionResolver.concreteIndexNames(clusterState, request);
final AtomicInteger indexCounter = new AtomicInteger();
final AtomicInteger completionCounter = new AtomicInteger(concreteIndices.length);
final AtomicReferenceArray<Object> indexResponses =
new AtomicReferenceArray<>(concreteIndices.length);
if (concreteIndices.length == 0) {
listener.onResponse(new GetFieldMappingsResponse());
} else {
boolean probablySingleFieldRequest =
concreteIndices.length == 1
&& request.types().length == 1
&& request.fields().length == 1;
for (final String index : concreteIndices) {
GetFieldMappingsIndexRequest shardRequest =
new GetFieldMappingsIndexRequest(request, index, probablySingleFieldRequest);
shardAction.execute(
shardRequest,
new ActionListener<GetFieldMappingsResponse>() {
@Override
public void onResponse(GetFieldMappingsResponse result) {
indexResponses.set(indexCounter.getAndIncrement(), result);
if (completionCounter.decrementAndGet() == 0) {
listener.onResponse(merge(indexResponses));
}
}
@Override
public void onFailure(Exception e) {
int index = indexCounter.getAndIncrement();
indexResponses.set(index, e);
if (completionCounter.decrementAndGet() == 0) {
listener.onResponse(merge(indexResponses));
}
}
});
}
}
}
@Override
protected void doExecute(
final KnapsackStateRequest request, ActionListener<KnapsackStateResponse> listener) {
final KnapsackStateResponse response = new KnapsackStateResponse();
try {
if (knapsack != null) {
for (KnapsackState state : knapsack.getExports()) {
response.addState(state);
}
for (KnapsackState state : knapsack.getImports()) {
response.addState(state);
}
}
listener.onResponse(response);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
listener.onFailure(e);
}
}
@Override
public void performOn(
ShardRouting replicaRouting,
IndexRequest request,
ActionListener<TransportResponse.Empty> listener) {
try {
IndexShard replica =
replicationGroup
.replicas
.stream()
.filter(s -> replicaRouting.isSameAllocation(s.routingEntry()))
.findFirst()
.get();
TransportIndexAction.executeIndexRequestOnReplica(request, replica);
listener.onResponse(TransportResponse.Empty.INSTANCE);
} catch (Exception t) {
listener.onFailure(t);
}
}
/** First get should try and use a shard that exists on a local node for better performance */
private void performFirst() {
if (shardsIt == null) {
// just execute it on the local node
if (internalRequest.request().operationThreaded()) {
internalRequest.request().beforeLocalFork();
threadPool
.executor(executor())
.execute(
new Runnable() {
@Override
public void run() {
try {
Response response = shardOperation(internalRequest.request(), null);
listener.onResponse(response);
} catch (Throwable e) {
onFailure(null, e);
}
}
});
return;
} else {
try {
final Response response = shardOperation(internalRequest.request(), null);
listener.onResponse(response);
return;
} catch (Throwable e) {
onFailure(null, e);
}
}
return;
}
if (internalRequest.request().preferLocalShard()) {
boolean foundLocal = false;
ShardRouting shardX;
while ((shardX = shardsIt.nextOrNull()) != null) {
final ShardRouting shard = shardX;
if (shard.currentNodeId().equals(nodes.localNodeId())) {
foundLocal = true;
if (internalRequest.request().operationThreaded()) {
internalRequest.request().beforeLocalFork();
threadPool
.executor(executor())
.execute(
new Runnable() {
@Override
public void run() {
try {
Response response =
shardOperation(internalRequest.request(), shard.shardId());
listener.onResponse(response);
} catch (Throwable e) {
shardsIt.reset();
onFailure(shard, e);
}
}
});
return;
} else {
try {
final Response response =
shardOperation(internalRequest.request(), shard.shardId());
listener.onResponse(response);
return;
} catch (Throwable e) {
shardsIt.reset();
onFailure(shard, e);
}
}
}
}
if (!foundLocal) {
// no local node get, go remote
shardsIt.reset();
perform(null);
}
} else {
perform(null);
}
}
public void start() {
if (shardsIts.size() == 0) {
// no shards
try {
listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
} catch (Throwable e) {
listener.onFailure(e);
}
}
request.beforeStart();
// count the local operations, and perform the non local ones
int localOperations = 0;
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
localOperations++;
} else {
// do the remote operation here, the localAsync flag is not relevant
performOperation(shardIt, shardIndex, true);
}
} else {
// really, no shards active in this group
onOperation(
null, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
}
}
// we have local operations, perform them now
if (localOperations > 0) {
if (request.operationThreading() == BroadcastOperationThreading.SINGLE_THREAD) {
request.beforeLocalFork();
threadPool
.executor(executor)
.execute(
new Runnable() {
@Override
public void run() {
int shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
performOperation(shardIt, shardIndex, false);
}
}
}
}
});
} else {
boolean localAsync =
request.operationThreading() == BroadcastOperationThreading.THREAD_PER_SHARD;
if (localAsync) {
request.beforeLocalFork();
}
shardIndex = -1;
for (final ShardIterator shardIt : shardsIts) {
shardIndex++;
final ShardRouting shard = shardIt.firstOrNull();
if (shard != null) {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
performOperation(shardIt, shardIndex, localAsync);
}
}
}
}
}
}
@Override
public void onResponse(final SearchResponse response) {
if (!isConnected()) {
onFailure(new DfContentException("Disconnected."));
return;
}
final String scrollId = response.getScrollId();
if (isFirstScan()) {
client
.prepareSearchScroll(scrollId)
.setScroll(RequestUtil.getScroll(request))
.execute(this);
return;
}
final SearchHits hits = response.getHits();
final int size = hits.getHits().length;
currentCount += size;
logger.info(
"scrollId: "
+ scrollId
+ ", totalHits: "
+ hits.totalHits()
+ ", hits: "
+ size
+ ", current: "
+ currentCount);
try {
for (final SearchHit hit : hits) {
final String index = bulkIndex == null ? hit.index() : bulkIndex;
final String type = bulkType == null ? hit.type() : bulkType;
final String operation =
"{\"index\":{\"_index\":\""
+ index
+ "\",\"_type\":\""
+ type
+ "\",\"_id\":\""
+ hit.id()
+ "\"}}";
final String source = hit.sourceAsString();
writer.append(operation).append('\n');
writer.append(source).append('\n');
}
if (size == 0 || scrollId == null) {
// end
writer.flush();
close();
listener.onResponse(null);
} else {
client
.prepareSearchScroll(scrollId)
.setScroll(RequestUtil.getScroll(request))
.execute(this);
}
} catch (final Exception e) {
onFailure(e);
}
}
void finishHim() {
listener.onResponse(newResponse(request, shardsResponses, clusterState));
}
public void respond(ActionListener<IndexResponse> listener) {
listener.onResponse(finalResponse);
}
private void perform(final Throwable lastException) {
final ShardRouting shard = shardsIt == null ? null : shardsIt.nextOrNull();
if (shard == null) {
Throwable failure = lastException;
if (failure == null) {
failure =
new NoShardAvailableActionException(
null, "No shard available for [" + internalRequest.request() + "]");
} else {
if (logger.isDebugEnabled()) {
logger.debug("failed to execute [" + internalRequest.request() + "]", failure);
}
}
listener.onFailure(failure);
} else {
if (shard.currentNodeId().equals(nodes.localNodeId())) {
// we don't prefer local shard, so try and do it here
if (!internalRequest.request().preferLocalShard()) {
try {
if (internalRequest.request().operationThreaded()) {
internalRequest.request().beforeLocalFork();
threadPool
.executor(executor)
.execute(
new Runnable() {
@Override
public void run() {
try {
Response response =
shardOperation(internalRequest.request(), shard.shardId());
listener.onResponse(response);
} catch (Throwable e) {
onFailure(shard, e);
}
}
});
} else {
final Response response =
shardOperation(internalRequest.request(), shard.shardId());
listener.onResponse(response);
}
} catch (Throwable e) {
onFailure(shard, e);
}
} else {
perform(lastException);
}
} else {
DiscoveryNode node = nodes.get(shard.currentNodeId());
transportService.sendRequest(
node,
transportShardAction,
new ShardSingleOperationRequest(internalRequest.request(), shard.shardId()),
new BaseTransportResponseHandler<Response>() {
@Override
public Response newInstance() {
return newResponse();
}
@Override
public String executor() {
return ThreadPool.Names.SAME;
}
@Override
public void handleResponse(final Response response) {
listener.onResponse(response);
}
@Override
public void handleException(TransportException exp) {
onFailure(shard, exp);
}
});
}
}
}
