private ClusterHealthResponse clusterHealth(
ClusterHealthRequest request, ClusterState clusterState) {
if (logger.isTraceEnabled()) {
logger.trace("Calculating health based on state version [{}]", clusterState.version());
}
String[] concreteIndices;
try {
concreteIndices =
clusterState.metaData().concreteIndices(request.indicesOptions(), request.indices());
} catch (IndexMissingException e) {
// one of the specified indices is not there - treat it as RED.
ClusterHealthResponse response =
new ClusterHealthResponse(clusterName.value(), Strings.EMPTY_ARRAY, clusterState);
response.status = ClusterHealthStatus.RED;
return response;
}
return new ClusterHealthResponse(clusterName.value(), concreteIndices, clusterState);
}
@Override
protected void masterOperation(
final ClusterHealthRequest request,
final ClusterState unusedState,
final ActionListener<ClusterHealthResponse> listener)
throws ElasticsearchException {
long endTime = System.currentTimeMillis() + request.timeout().millis();
if (request.waitForEvents() != null) {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<ElasticsearchException> failure = new AtomicReference<>();
clusterService.submitStateUpdateTask(
"cluster_health (wait_for_events [" + request.waitForEvents() + "])",
request.waitForEvents(),
new ProcessedClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) {
return currentState;
}
@Override
public void clusterStateProcessed(
String source, ClusterState oldState, ClusterState newState) {
latch.countDown();
}
@Override
public void onFailure(String source, Throwable t) {
logger.error("unexpected failure during [{}]", t, source);
failure.set(new ElasticsearchException("Error while waiting for events", t));
latch.countDown();
}
@Override
public boolean runOnlyOnMaster() {
return !request.local();
}
});
try {
latch.await(request.timeout().millis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
// ignore
}
if (failure.get() != null) {
throw failure.get();
}
}
int waitFor = 5;
if (request.waitForStatus() == null) {
waitFor--;
}
if (request.waitForRelocatingShards() == -1) {
waitFor--;
}
if (request.waitForActiveShards() == -1) {
waitFor--;
}
if (request.waitForNodes().isEmpty()) {
waitFor--;
}
if (request.indices().length == 0) { // check that they actually exists in the meta data
waitFor--;
}
if (waitFor == 0) {
// no need to wait for anything
ClusterState clusterState = clusterService.state();
listener.onResponse(clusterHealth(request, clusterState));
return;
}
while (true) {
int waitForCounter = 0;
ClusterState clusterState = clusterService.state();
ClusterHealthResponse response = clusterHealth(request, clusterState);
if (request.waitForStatus() != null
&& response.getStatus().value() <= request.waitForStatus().value()) {
waitForCounter++;
}
if (request.waitForRelocatingShards() != -1
&& response.getRelocatingShards() <= request.waitForRelocatingShards()) {
waitForCounter++;
}
if (request.waitForActiveShards() != -1
&& response.getActiveShards() >= request.waitForActiveShards()) {
waitForCounter++;
}
if (request.indices().length > 0) {
try {
clusterState.metaData().concreteIndices(IndicesOptions.strictExpand(), request.indices());
waitForCounter++;
} catch (IndexMissingException e) {
response.status = ClusterHealthStatus.RED; // no indices, make sure its RED
// missing indices, wait a bit more...
}
}
if (!request.waitForNodes().isEmpty()) {
if (request.waitForNodes().startsWith(">=")) {
int expected = Integer.parseInt(request.waitForNodes().substring(2));
if (response.getNumberOfNodes() >= expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("ge(")) {
int expected =
Integer.parseInt(
request.waitForNodes().substring(3, request.waitForNodes().length() - 1));
if (response.getNumberOfNodes() >= expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("<=")) {
int expected = Integer.parseInt(request.waitForNodes().substring(2));
if (response.getNumberOfNodes() <= expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("le(")) {
int expected =
Integer.parseInt(
request.waitForNodes().substring(3, request.waitForNodes().length() - 1));
if (response.getNumberOfNodes() <= expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith(">")) {
int expected = Integer.parseInt(request.waitForNodes().substring(1));
if (response.getNumberOfNodes() > expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("gt(")) {
int expected =
Integer.parseInt(
request.waitForNodes().substring(3, request.waitForNodes().length() - 1));
if (response.getNumberOfNodes() > expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("<")) {
int expected = Integer.parseInt(request.waitForNodes().substring(1));
if (response.getNumberOfNodes() < expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("lt(")) {
int expected =
Integer.parseInt(
request.waitForNodes().substring(3, request.waitForNodes().length() - 1));
if (response.getNumberOfNodes() < expected) {
waitForCounter++;
}
} else {
int expected = Integer.parseInt(request.waitForNodes());
if (response.getNumberOfNodes() == expected) {
waitForCounter++;
}
}
}
if (waitForCounter == waitFor) {
listener.onResponse(response);
return;
}
if (System.currentTimeMillis() > endTime) {
response.timedOut = true;
listener.onResponse(response);
return;
}
try {
Thread.sleep(200);
} catch (InterruptedException e) {
response.timedOut = true;
listener.onResponse(response);
return;
}
}
}
@Override
protected ClusterHealthResponse masterOperation(ClusterHealthRequest request, ClusterState state)
throws ElasticSearchException {
int waitFor = 4;
if (request.waitForStatus() == null) {
waitFor--;
}
if (request.waitForRelocatingShards() == -1) {
waitFor--;
}
if (request.waitForActiveShards() == -1) {
waitFor--;
}
if (request.waitForNodes().isEmpty()) {
waitFor--;
}
if (waitFor == 0) {
// no need to wait for anything
return clusterHealth(request);
}
long endTime = System.currentTimeMillis() + request.timeout().millis();
while (true) {
int waitForCounter = 0;
ClusterHealthResponse response = clusterHealth(request);
if (request.waitForStatus() != null
&& response.status().value() <= request.waitForStatus().value()) {
waitForCounter++;
}
if (request.waitForRelocatingShards() != -1
&& response.relocatingShards() <= request.waitForRelocatingShards()) {
waitForCounter++;
}
if (request.waitForActiveShards() != -1
&& response.activeShards() >= request.waitForActiveShards()) {
waitForCounter++;
}
if (!request.waitForNodes().isEmpty()) {
if (request.waitForNodes().startsWith(">=")) {
int expected = Integer.parseInt(request.waitForNodes().substring(2));
if (response.numberOfNodes() >= expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("ge(")) {
int expected =
Integer.parseInt(
request.waitForNodes().substring(3, request.waitForNodes().length() - 1));
if (response.numberOfNodes() >= expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("<=")) {
int expected = Integer.parseInt(request.waitForNodes().substring(2));
if (response.numberOfNodes() <= expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("le(")) {
int expected =
Integer.parseInt(
request.waitForNodes().substring(3, request.waitForNodes().length() - 1));
if (response.numberOfNodes() <= expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith(">")) {
int expected = Integer.parseInt(request.waitForNodes().substring(1));
if (response.numberOfNodes() > expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("gt(")) {
int expected =
Integer.parseInt(
request.waitForNodes().substring(3, request.waitForNodes().length() - 1));
if (response.numberOfNodes() > expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("<")) {
int expected = Integer.parseInt(request.waitForNodes().substring(1));
if (response.numberOfNodes() < expected) {
waitForCounter++;
}
} else if (request.waitForNodes().startsWith("lt(")) {
int expected =
Integer.parseInt(
request.waitForNodes().substring(3, request.waitForNodes().length() - 1));
if (response.numberOfNodes() < expected) {
waitForCounter++;
}
} else {
int expected = Integer.parseInt(request.waitForNodes());
if (response.numberOfNodes() == expected) {
waitForCounter++;
}
}
}
if (waitForCounter == waitFor) {
return response;
}
if (timerService.estimatedTimeInMillis() > endTime) {
response.timedOut = true;
return response;
}
try {
Thread.sleep(200);
} catch (InterruptedException e) {
response.timedOut = true;
// we got interrupted, bail
return response;
}
}
}
private ClusterHealthResponse clusterHealth(ClusterHealthRequest request) {
ClusterState clusterState = clusterService.state();
RoutingTableValidation validation =
clusterState.routingTable().validate(clusterState.metaData());
ClusterHealthResponse response =
new ClusterHealthResponse(clusterName.value(), validation.failures());
response.numberOfNodes = clusterState.nodes().size();
response.numberOfDataNodes = clusterState.nodes().dataNodes().size();
for (String index : clusterState.metaData().concreteIndices(request.indices())) {
IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(index);
IndexMetaData indexMetaData = clusterState.metaData().index(index);
if (indexRoutingTable == null) {
continue;
}
ClusterIndexHealth indexHealth =
new ClusterIndexHealth(
index,
indexMetaData.numberOfShards(),
indexMetaData.numberOfReplicas(),
validation.indexFailures(indexMetaData.index()));
for (IndexShardRoutingTable shardRoutingTable : indexRoutingTable) {
ClusterShardHealth shardHealth = new ClusterShardHealth(shardRoutingTable.shardId().id());
for (ShardRouting shardRouting : shardRoutingTable) {
if (shardRouting.active()) {
shardHealth.activeShards++;
if (shardRouting.relocating()) {
// the shard is relocating, the one he is relocating to will be in initializing state,
// so we don't count it
shardHealth.relocatingShards++;
}
if (shardRouting.primary()) {
shardHealth.primaryActive = true;
}
} else if (shardRouting.initializing()) {
shardHealth.initializingShards++;
} else if (shardRouting.unassigned()) {
shardHealth.unassignedShards++;
}
}
if (shardHealth.primaryActive) {
if (shardHealth.activeShards == shardRoutingTable.size()) {
shardHealth.status = ClusterHealthStatus.GREEN;
} else {
shardHealth.status = ClusterHealthStatus.YELLOW;
}
} else {
shardHealth.status = ClusterHealthStatus.RED;
}
indexHealth.shards.put(shardHealth.id(), shardHealth);
}
for (ClusterShardHealth shardHealth : indexHealth) {
if (shardHealth.primaryActive()) {
indexHealth.activePrimaryShards++;
}
indexHealth.activeShards += shardHealth.activeShards;
indexHealth.relocatingShards += shardHealth.relocatingShards;
indexHealth.initializingShards += shardHealth.initializingShards;
indexHealth.unassignedShards += shardHealth.unassignedShards;
}
// update the index status
indexHealth.status = ClusterHealthStatus.GREEN;
if (!indexHealth.validationFailures().isEmpty()) {
indexHealth.status = ClusterHealthStatus.RED;
} else if (indexHealth
.shards()
.isEmpty()) { // might be since none has been created yet (two phase index creation)
indexHealth.status = ClusterHealthStatus.RED;
} else {
for (ClusterShardHealth shardHealth : indexHealth) {
if (shardHealth.status() == ClusterHealthStatus.RED) {
indexHealth.status = ClusterHealthStatus.RED;
break;
}
if (shardHealth.status() == ClusterHealthStatus.YELLOW) {
indexHealth.status = ClusterHealthStatus.YELLOW;
}
}
}
response.indices.put(indexHealth.index(), indexHealth);
}
for (ClusterIndexHealth indexHealth : response) {
response.activePrimaryShards += indexHealth.activePrimaryShards;
response.activeShards += indexHealth.activeShards;
response.relocatingShards += indexHealth.relocatingShards;
response.initializingShards += indexHealth.initializingShards;
response.unassignedShards += indexHealth.unassignedShards;
}
response.status = ClusterHealthStatus.GREEN;
if (!response.validationFailures().isEmpty()) {
response.status = ClusterHealthStatus.RED;
} else if (clusterState
.blocks()
.hasGlobalBlock(GatewayService.NOT_RECOVERED_FROM_GATEWAY_BLOCK)) {
response.status = ClusterHealthStatus.RED;
} else {
for (ClusterIndexHealth indexHealth : response) {
if (indexHealth.status() == ClusterHealthStatus.RED) {
response.status = ClusterHealthStatus.RED;
break;
}
if (indexHealth.status() == ClusterHealthStatus.YELLOW) {
response.status = ClusterHealthStatus.YELLOW;
}
}
}
return response;
}
