@Override
public void onFailure(Exception e) {
if (ExceptionsHelper.unwrap(e, EsRejectedExecutionException.class) != null) {
if (retries.hasNext()) {
retryCount += 1;
TimeValue delay = retries.next();
logger.trace("retrying rejected search after [{}]", e, delay);
countSearchRetry.run();
threadPool.schedule(delay, ThreadPool.Names.SAME, this);
} else {
logger.warn(
"giving up on search because we retried [{}] times without success", e, retryCount);
fail.accept(e);
}
} else {
logger.warn("giving up on search because it failed with a non-retryable exception", e);
fail.accept(e);
}
}
@Override
public void messageReceived(
final RecoveryTranslogOperationsRequest request, final TransportChannel channel)
throws IOException {
try (RecoveriesCollection.RecoveryRef recoveryRef =
onGoingRecoveries.getRecoverySafe(request.recoveryId(), request.shardId())) {
final ClusterStateObserver observer =
new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext());
final RecoveryTarget recoveryTarget = recoveryRef.status();
try {
recoveryTarget.indexTranslogOperations(request.operations(), request.totalTranslogOps());
channel.sendResponse(TransportResponse.Empty.INSTANCE);
} catch (TranslogRecoveryPerformer.BatchOperationException exception) {
MapperException mapperException =
(MapperException) ExceptionsHelper.unwrap(exception, MapperException.class);
if (mapperException == null) {
throw exception;
}
// in very rare cases a translog replay from primary is processed before a mapping update
// on this node
// which causes local mapping changes since the mapping (clusterstate) might not have
// arrived on this node.
// we want to wait until these mappings are processed but also need to do some maintenance
// and roll back the
// number of processed (completed) operations in this batch to ensure accounting is
// correct.
logger.trace(
(Supplier<?>)
() ->
new ParameterizedMessage(
"delaying recovery due to missing mapping changes (rolling back stats for [{}] ops)",
exception.completedOperations()),
exception);
final RecoveryState.Translog translog = recoveryTarget.state().getTranslog();
translog.decrementRecoveredOperations(
exception.completedOperations()); // do the maintainance and rollback competed ops
// we do not need to use a timeout here since the entire recovery mechanism has an
// inactivity protection (it will be
// canceled)
observer.waitForNextChange(
new ClusterStateObserver.Listener() {
@Override
public void onNewClusterState(ClusterState state) {
try {
messageReceived(request, channel);
} catch (Exception e) {
onFailure(e);
}
}
protected void onFailure(Exception e) {
try {
channel.sendResponse(e);
} catch (IOException e1) {
logger.warn("failed to send error back to recovery source", e1);
}
}
@Override
public void onClusterServiceClose() {
onFailure(
new ElasticsearchException(
"cluster service was closed while waiting for mapping updates"));
}
@Override
public void onTimeout(TimeValue timeout) {
// note that we do not use a timeout (see comment above)
onFailure(
new ElasticsearchTimeoutException(
"timed out waiting for mapping updates (timeout [" + timeout + "])"));
}
});
}
}
}
@Override
public void messageReceived(
final RecoveryTranslogOperationsRequest request, final TransportChannel channel)
throws Exception {
try (RecoveriesCollection.StatusRef statusRef =
onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) {
final ClusterStateObserver observer =
new ClusterStateObserver(clusterService, null, logger);
final RecoveryStatus recoveryStatus = statusRef.status();
final RecoveryState.Translog translog = recoveryStatus.state().getTranslog();
translog.totalOperations(request.totalTranslogOps());
assert recoveryStatus.indexShard().recoveryState() == recoveryStatus.state();
try {
recoveryStatus.indexShard().performBatchRecovery(request.operations());
channel.sendResponse(TransportResponse.Empty.INSTANCE);
} catch (TranslogRecoveryPerformer.BatchOperationException exception) {
MapperException mapperException =
(MapperException) ExceptionsHelper.unwrap(exception, MapperException.class);
if (mapperException == null) {
throw exception;
}
// in very rare cases a translog replay from primary is processed before a mapping update
// on this node
// which causes local mapping changes. we want to wait until these mappings are processed.
logger.trace(
"delaying recovery due to missing mapping changes (rolling back stats for [{}] ops)",
exception,
exception.completedOperations());
translog.decrementRecoveredOperations(exception.completedOperations());
// we do not need to use a timeout here since the entire recovery mechanism has an
// inactivity protection (it will be
// canceled)
observer.waitForNextChange(
new ClusterStateObserver.Listener() {
@Override
public void onNewClusterState(ClusterState state) {
try {
messageReceived(request, channel);
} catch (Exception e) {
onFailure(e);
}
}
protected void onFailure(Exception e) {
try {
channel.sendResponse(e);
} catch (IOException e1) {
logger.warn("failed to send error back to recovery source", e1);
}
}
@Override
public void onClusterServiceClose() {
onFailure(
new ElasticsearchException(
"cluster service was closed while waiting for mapping updates"));
}
@Override
public void onTimeout(TimeValue timeout) {
// note that we do not use a timeout (see comment above)
onFailure(
new ElasticsearchTimeoutException(
"timed out waiting for mapping updates (timeout [" + timeout + "])"));
}
});
}
}
}
/**
* Test that a breaker correctly redistributes to a different breaker, in this case, the fielddata
* breaker borrows space from the request breaker
*/
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18325")
public void testParentChecking() throws Exception {
if (noopBreakerUsed()) {
logger.info("--> noop breakers used, skipping test");
return;
}
assertAcked(
prepareCreate(
"cb-test", 1, Settings.builder().put(SETTING_NUMBER_OF_REPLICAS, between(0, 1)))
.addMapping("type", "test", "type=text,fielddata=true"));
Client client = client();
// index some different terms so we have some field data for loading
int docCount = scaledRandomIntBetween(300, 1000);
List<IndexRequestBuilder> reqs = new ArrayList<>();
for (long id = 0; id < docCount; id++) {
reqs.add(
client
.prepareIndex("cb-test", "type", Long.toString(id))
.setSource("test", "value" + id));
}
indexRandom(true, reqs);
Settings resetSettings =
Settings.builder()
.put(
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
"10b")
.put(
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(),
1.0)
.build();
assertAcked(
client.admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings));
// Perform a search to load field data for the "test" field
try {
client
.prepareSearch("cb-test")
.setQuery(matchAllQuery())
.addSort("test", SortOrder.DESC)
.get();
fail("should have thrown an exception");
} catch (Exception e) {
String errMsg =
"[fielddata] Data too large, data for [test] would be larger than limit of [10/10b]";
assertThat(
"Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
e.toString(),
containsString(errMsg));
}
assertFailures(
client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC),
RestStatus.INTERNAL_SERVER_ERROR,
containsString("Data too large, data for [test] would be larger than limit of [10/10b]"));
reset();
// Adjust settings so the parent breaker will fail, but neither the fielddata breaker nor the
// node request breaker will fail
resetSettings =
Settings.builder()
.put(
HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "500b")
.put(
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
"90%")
.put(
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(),
1.0)
.build();
client
.admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(resetSettings)
.execute()
.actionGet();
// Perform a search to load field data for the "test" field
try {
client
.prepareSearch("cb-test")
.setQuery(matchAllQuery())
.addSort("test", SortOrder.DESC)
.get();
fail("should have thrown an exception");
} catch (Exception e) {
final Throwable cause = ExceptionsHelper.unwrap(e, CircuitBreakingException.class);
assertNotNull("CircuitBreakingException is not the cause of " + e, cause);
String errMsg = "would be larger than limit of [500/500b]]";
assertThat(
"Exception: [" + cause.toString() + "] should contain a CircuitBreakingException",
cause.toString(),
startsWith("CircuitBreakingException[[parent] Data too large"));
assertThat(
"Exception: [" + cause.toString() + "] should contain a CircuitBreakingException",
cause.toString(),
endsWith(errMsg));
} finally {
// reset before teardown as it requires properly set up breakers
reset();
}
}