@Override
protected void doExecute(
final KnapsackPullRequest request, ActionListener<KnapsackPullResponse> listener) {
final KnapsackState state =
new KnapsackState().setMode("pull").setNodeName(nodeService.nodeName());
final KnapsackPullResponse response = new KnapsackPullResponse().setState(state);
try {
final BulkTransportClient transportClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.put(clientSettings(client, request))
.toBulkTransportClient();
final BulkNodeClient nodeClient =
ClientBuilder.builder()
.put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
.put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
.put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
.toBulkNodeClient(client);
state.setTimestamp(new DateTime());
response.setRunning(true);
knapsack.submit(
new Thread() {
public void run() {
performPull(request, state, transportClient, nodeClient);
}
});
listener.onResponse(response);
} catch (Throwable e) {
logger.error(e.getMessage(), e);
listener.onFailure(e);
}
}
@Override
public void run() {
try {
startLatch.await();
for (int j = 0; j < numberOfIds; j++) {
for (int k = 0; k < numberOfUpdatesPerId; ++k) {
updateRequestsOutstanding.acquire();
UpdateRequest ur =
client()
.prepareUpdate("test", "type1", Integer.toString(j))
.setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
.setRetryOnConflict(retryOnConflict)
.setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
.setListenerThreaded(false)
.request();
client().update(ur, new UpdateListener(j));
deleteRequestsOutstanding.acquire();
DeleteRequest dr =
client()
.prepareDelete("test", "type1", Integer.toString(j))
.setListenerThreaded(false)
.setOperationThreaded(false)
.request();
client().delete(dr, new DeleteListener(j));
}
}
} catch (Throwable e) {
logger.error("Something went wrong", e);
failures.add(e);
} finally {
try {
waitForOutstandingRequests(
TimeValue.timeValueSeconds(60),
updateRequestsOutstanding,
maxUpdateRequests,
"Update");
waitForOutstandingRequests(
TimeValue.timeValueSeconds(60),
deleteRequestsOutstanding,
maxDeleteRequests,
"Delete");
} catch (ElasticsearchTimeoutException ete) {
failures.add(ete);
}
latch.countDown();
}
}
@Inject
public DiskThresholdDecider(
Settings settings,
NodeSettingsService nodeSettingsService,
ClusterInfoService infoService,
Client client) {
super(settings);
String lowWatermark = settings.get(CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, "85%");
String highWatermark = settings.get(CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, "90%");
if (!validWatermarkSetting(lowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK)) {
throw new ElasticsearchParseException("unable to parse low watermark [{}]", lowWatermark);
}
if (!validWatermarkSetting(highWatermark, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK)) {
throw new ElasticsearchParseException("unable to parse high watermark [{}]", highWatermark);
}
// Watermark is expressed in terms of used data, but we need "free" data watermark
this.freeDiskThresholdLow = 100.0 - thresholdPercentageFromWatermark(lowWatermark);
this.freeDiskThresholdHigh = 100.0 - thresholdPercentageFromWatermark(highWatermark);
this.freeBytesThresholdLow =
thresholdBytesFromWatermark(lowWatermark, CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK);
this.freeBytesThresholdHigh =
thresholdBytesFromWatermark(highWatermark, CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK);
this.includeRelocations =
settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS, true);
this.rerouteInterval =
settings.getAsTime(
CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL, TimeValue.timeValueSeconds(60));
this.enabled = settings.getAsBoolean(CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED, true);
nodeSettingsService.addListener(new ApplySettings());
infoService.addListener(new DiskListener(client));
}
/**
* Creates a new TranslogConfig instance
*
* @param shardId the shard ID this translog belongs to
* @param translogPath the path to use for the transaction log files
* @param indexSettings the index settings used to set internal variables
* @param durabilty the default durability setting for the translog
* @param bigArrays a bigArrays instance used for temporarily allocating write operations
* @param threadPool a {@link ThreadPool} to schedule async sync durability
*/
public TranslogConfig(
ShardId shardId,
Path translogPath,
Settings indexSettings,
Translog.Durabilty durabilty,
BigArrays bigArrays,
@Nullable ThreadPool threadPool) {
this.indexSettings = indexSettings;
this.shardId = shardId;
this.translogPath = translogPath;
this.durabilty = durabilty;
this.threadPool = threadPool;
this.bigArrays = bigArrays;
this.type =
TranslogWriter.Type.fromString(
indexSettings.get(INDEX_TRANSLOG_FS_TYPE, TranslogWriter.Type.BUFFERED.name()));
this.bufferSize =
(int)
indexSettings
.getAsBytesSize(
INDEX_TRANSLOG_BUFFER_SIZE,
IndexingMemoryController.INACTIVE_SHARD_TRANSLOG_BUFFER)
.bytes(); // Not really interesting, updated by IndexingMemoryController...
syncInterval =
indexSettings.getAsTime(INDEX_TRANSLOG_SYNC_INTERVAL, TimeValue.timeValueSeconds(5));
if (syncInterval.millis() > 0 && threadPool != null) {
syncOnEachOperation = false;
} else if (syncInterval.millis() == 0) {
syncOnEachOperation = true;
} else {
syncOnEachOperation = false;
}
}
@Before
public void startNodes() {
try {
setClusterName();
startNode("1");
startNode("2"); // we need 2 nodes for knapsack
findNodeAddress();
try {
ClusterHealthResponse healthResponse =
client("1")
.execute(
ClusterHealthAction.INSTANCE,
new ClusterHealthRequest()
.waitForYellowStatus()
.timeout(TimeValue.timeValueSeconds(30)))
.actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException(
"cluster state is "
+ healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException(
"timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
}
logger.info("ready");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
}
public static class PutRequest {
final String[] indices;
final String mappingType;
final String mappingSource;
boolean ignoreConflicts = false;
TimeValue timeout = TimeValue.timeValueSeconds(10);
public PutRequest(String[] indices, String mappingType, String mappingSource) {
this.indices = indices;
this.mappingType = mappingType;
this.mappingSource = mappingSource;
}
public PutRequest ignoreConflicts(boolean ignoreConflicts) {
this.ignoreConflicts = ignoreConflicts;
return this;
}
public PutRequest timeout(TimeValue timeout) {
this.timeout = timeout;
return this;
}
}
@Test
public void testRandomDocsNodeClient() throws Exception {
final NodeClient es =
new NodeClient()
.maxActionsPerBulkRequest(1000)
.flushInterval(TimeValue.timeValueSeconds(10))
.newClient(client("1"))
.newIndex("test");
try {
for (int i = 0; i < 12345; i++) {
es.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
}
es.flush();
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
es.shutdown();
assertEquals(13, es.getState().getTotalIngest().count());
if (es.hasThrowable()) {
logger.error("error", es.getThrowable());
}
assertFalse(es.hasThrowable());
}
}
@Inject
public DiscoveryService(Settings settings, Discovery discovery) {
super(settings);
this.discovery = discovery;
this.initialStateTimeout =
componentSettings.getAsTime("initial_state_timeout", TimeValue.timeValueSeconds(30));
}
public final class ProcessService extends AbstractComponent {
private final ProcessProbe probe;
private final ProcessInfo info;
private final SingleObjectCache<ProcessStats> processStatsCache;
public static final Setting<TimeValue> REFRESH_INTERVAL_SETTING =
Setting.timeSetting(
"monitor.process.refresh_interval",
TimeValue.timeValueSeconds(1),
TimeValue.timeValueSeconds(1),
Property.NodeScope);
public ProcessService(Settings settings) {
super(settings);
this.probe = ProcessProbe.getInstance();
final TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
processStatsCache = new ProcessStatsCache(refreshInterval, probe.processStats());
this.info = probe.processInfo();
this.info.refreshInterval = refreshInterval.millis();
logger.debug("using refresh_interval [{}]", refreshInterval);
}
public ProcessInfo info() {
return this.info;
}
public ProcessStats stats() {
return processStatsCache.getOrRefresh();
}
private class ProcessStatsCache extends SingleObjectCache<ProcessStats> {
public ProcessStatsCache(TimeValue interval, ProcessStats initValue) {
super(interval, initValue);
}
@Override
protected ProcessStats refresh() {
return probe.processStats();
}
}
}
/**
* We build a plugin manager instance which wait only for 30 seconds before raising an
* ElasticsearchTimeoutException
*/
private static PluginManager pluginManager(
String pluginUrl, Tuple<Settings, Environment> initialSettings) throws IOException {
if (!Files.exists(initialSettings.v2().pluginsFile())) {
Files.createDirectories(initialSettings.v2().pluginsFile());
}
return new PluginManager(
initialSettings.v2(),
pluginUrl,
PluginManager.OutputMode.SILENT,
TimeValue.timeValueSeconds(30));
}
@Test
public void testNewIndexNodeClient() throws Exception {
final NodeClient es =
new NodeClient()
.flushInterval(TimeValue.timeValueSeconds(5))
.newClient(client("1"))
.newIndex("test");
es.shutdown();
if (es.hasThrowable()) {
logger.error("error", es.getThrowable());
}
assertFalse(es.hasThrowable());
}
@Test
public void testBulkProcess() {
BulkProcessor bulkProcessor =
BulkProcessor.builder(
client,
new BulkProcessor.Listener() {
@Override
public void beforeBulk(long executionId, BulkRequest request) {
/**
* Why did those methods not be run?<br>
* Because we did not close the bulkProcessor to make it flush cache?<br>
* No, the reason is we do not wait those little seconds to close the
* bulk-processor when it was set FlushInterval after five second.
*/
System.out.println("BulkProcessor's beforeBulk.");
}
@Override
public void afterBulk(
long executionId, BulkRequest request, BulkResponse response) {
// TODO:
System.out.println("BulkProcessor's afterBulk.");
}
@Override
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
// TODO:
System.out.println("BulkProcessor's afterBulk when it failed.");
}
})
.setBulkActions(10000)
.setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))
.setFlushInterval(TimeValue.timeValueSeconds(5))
.setConcurrentRequests(1)
.build();
/** " ' " --> " \" " */
bulkProcessor.add(new IndexRequest("asdf2014", "asdf", "1").source("{\"name\":\"asdf\"}"));
bulkProcessor.add(new DeleteRequest("asdf2014", "asdf", "2"));
try {
bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
} catch (InterruptedException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
bulkProcessor.close();
}
public static class Request {
final String index;
TimeValue timeout = TimeValue.timeValueSeconds(10);
public Request(String index) {
this.index = index;
}
public Request timeout(TimeValue timeout) {
this.timeout = timeout;
return this;
}
}
@Test
public void testThreadedRandomDocsNodeClient() throws Exception {
int max = Runtime.getRuntime().availableProcessors();
int maxactions = 1000;
final int maxloop = 12345;
final NodeClient client =
new NodeClient()
.maxActionsPerBulkRequest(maxactions)
.flushInterval(TimeValue.timeValueSeconds(600)) // disable auto flush for this test
.newClient(client("1"))
.newIndex("test")
.startBulk("test");
try {
ThreadPoolExecutor pool =
EsExecutors.newFixed(max, 30, EsExecutors.daemonThreadFactory("nodeclient-test"));
final CountDownLatch latch = new CountDownLatch(max);
for (int i = 0; i < max; i++) {
pool.execute(
new Runnable() {
public void run() {
for (int i = 0; i < maxloop; i++) {
client.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
}
latch.countDown();
}
});
}
logger.info("waiting for max 60 seconds...");
latch.await(60, TimeUnit.SECONDS);
logger.info("flush...");
client.flush();
logger.info("waiting for pool shutdown...");
pool.shutdown();
logger.info("pool is shut down");
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.stopBulk("test").shutdown();
logger.info("total bulk requests = {}", client.getState().getTotalIngest().count());
assertEquals(max * maxloop / maxactions + 1, client.getState().getTotalIngest().count());
if (client.hasThrowable()) {
logger.error("error", client.getThrowable());
}
assertFalse(client.hasThrowable());
}
}
@Inject
public InternalClusterService(
Settings settings,
DiscoveryService discoveryService,
OperationRouting operationRouting,
TransportService transportService,
ThreadPool threadPool,
TimerService timerService) {
super(settings);
this.operationRouting = operationRouting;
this.transportService = transportService;
this.discoveryService = discoveryService;
this.threadPool = threadPool;
this.timerService = timerService;
this.reconnectInterval =
componentSettings.getAsTime("reconnect_interval", TimeValue.timeValueSeconds(10));
}
@Test
public void testMappingNodeClient() throws Exception {
final NodeClient es =
new NodeClient().flushInterval(TimeValue.timeValueSeconds(5)).newClient(client("1"));
es.addMapping("test", "{\"test\":{\"properties\":{\"location\":{\"type\":\"geo_point\"}}}}");
es.newIndex("test");
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
GetMappingsResponse getMappingsResponse =
es.client().admin().indices().getMappings(getMappingsRequest).actionGet();
logger.info("mappings={}", getMappingsResponse.getMappings());
es.shutdown();
if (es.hasThrowable()) {
logger.error("error", es.getThrowable());
}
assertFalse(es.hasThrowable());
}
@Override
protected void doStop() throws ElasticsearchException {
pingService.stop();
masterFD.stop("zen disco stop");
nodesFD.stop();
initialStateSent.set(false);
if (sendLeaveRequest) {
if (!master && latestDiscoNodes.masterNode() != null) {
try {
membership.sendLeaveRequestBlocking(
latestDiscoNodes.masterNode(), localNode, TimeValue.timeValueSeconds(1));
} catch (Exception e) {
logger.debug(
"failed to send leave request to master [{}]", e, latestDiscoNodes.masterNode());
}
} else {
DiscoveryNode[] possibleMasters =
electMaster.nextPossibleMasters(latestDiscoNodes.nodes().values(), 5);
for (DiscoveryNode possibleMaster : possibleMasters) {
if (localNode.equals(possibleMaster)) {
continue;
}
try {
membership.sendLeaveRequest(latestDiscoNodes.masterNode(), possibleMaster);
} catch (Exception e) {
logger.debug(
"failed to send leave request from master [{}] to possible master [{}]",
e,
latestDiscoNodes.masterNode(),
possibleMaster);
}
}
}
}
master = false;
if (currentJoinThread != null) {
try {
currentJoinThread.interrupt();
} catch (Exception e) {
// ignore
}
}
}
@Inject
public GatewayAllocator(
Settings settings,
TransportNodesListGatewayStartedShards listGatewayStartedShards,
TransportNodesListShardStoreMetaData listShardStoreMetaData) {
super(settings);
this.listGatewayStartedShards = listGatewayStartedShards;
this.listShardStoreMetaData = listShardStoreMetaData;
this.listTimeout =
componentSettings.getAsTime(
"list_timeout",
settings.getAsTime("gateway.local.list_timeout", TimeValue.timeValueSeconds(30)));
this.initialShards =
componentSettings.get(
"initial_shards", settings.get("gateway.local.initial_shards", "quorum"));
logger.debug("using initial_shards [{}], list_timeout [{}]", initialShards, listTimeout);
}
public static long parseHumanDTToMills(String humanDateTime) {
long datetime = 0;
if (humanDateTime.indexOf("ms") != -1) {
datetime =
TimeValue.timeValueMillis(NumberUtils.toLong(humanDateTime.replaceAll("ms", "")))
.getMillis();
} else if (humanDateTime.indexOf("h") != -1) {
datetime =
TimeValue.timeValueHours(NumberUtils.toLong(humanDateTime.replaceAll("h", "")))
.getMillis();
} else if (humanDateTime.indexOf("m") != -1) {
datetime =
TimeValue.timeValueMinutes(NumberUtils.toLong(humanDateTime.replaceAll("m", "")))
.getMillis();
} else if (humanDateTime.indexOf("s") != -1) {
datetime =
TimeValue.timeValueSeconds(NumberUtils.toLong(humanDateTime.replaceAll("s", "")))
.getMillis();
}
return datetime;
}
@Inject
public IndicesRequestCache(
Settings settings, ClusterService clusterService, ThreadPool threadPool) {
super(settings);
this.clusterService = clusterService;
this.threadPool = threadPool;
this.cleanInterval =
settings.getAsTime(INDICES_CACHE_REQUEST_CLEAN_INTERVAL, TimeValue.timeValueSeconds(60));
String size = settings.get(INDICES_CACHE_QUERY_SIZE);
if (size == null) {
size = settings.get(DEPRECATED_INDICES_CACHE_QUERY_SIZE);
if (size != null) {
deprecationLogger.deprecated(
"The ["
+ DEPRECATED_INDICES_CACHE_QUERY_SIZE
+ "] settings is now deprecated, use ["
+ INDICES_CACHE_QUERY_SIZE
+ "] instead");
}
}
if (size == null) {
// this cache can be very small yet still be very effective
size = "1%";
}
this.size = size;
this.expire = settings.getAsTime(INDICES_CACHE_QUERY_EXPIRE, null);
// defaults to 4, but this is a busy map for all indices, increase it a bit by default
this.concurrencyLevel = settings.getAsInt(INDICES_CACHE_QUERY_CONCURRENCY_LEVEL, 16);
if (concurrencyLevel <= 0) {
throw new IllegalArgumentException(
"concurrency_level must be > 0 but was: " + concurrencyLevel);
}
buildCache();
this.reaper = new Reaper();
threadPool.schedule(cleanInterval, ThreadPool.Names.SAME, reaper);
}
@Inject
public IndexShardGatewayService(
ShardId shardId,
@IndexSettings Settings indexSettings,
IndexSettingsService indexSettingsService,
ThreadPool threadPool,
IndexShard indexShard,
IndexShardGateway shardGateway,
IndexShardSnapshotAndRestoreService snapshotService,
RepositoriesService repositoriesService) {
super(shardId, indexSettings);
this.threadPool = threadPool;
this.indexSettingsService = indexSettingsService;
this.indexShard = (InternalIndexShard) indexShard;
this.shardGateway = shardGateway;
this.snapshotService = snapshotService;
this.snapshotOnClose = componentSettings.getAsBoolean("snapshot_on_close", true);
this.snapshotInterval =
componentSettings.getAsTime("snapshot_interval", TimeValue.timeValueSeconds(10));
indexSettingsService.addListener(applySettings);
}
public class ClusterService extends AbstractLifecycleComponent {
public static final Setting<TimeValue> CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING =
Setting.positiveTimeSetting(
"cluster.service.slow_task_logging_threshold",
TimeValue.timeValueSeconds(30),
Property.Dynamic,
Property.NodeScope);
public static final String UPDATE_THREAD_NAME = "clusterService#updateTask";
private final ThreadPool threadPool;
private final ClusterName clusterName;
private BiConsumer<ClusterChangedEvent, Discovery.AckListener> clusterStatePublisher;
private final OperationRouting operationRouting;
private final ClusterSettings clusterSettings;
private TimeValue slowTaskLoggingThreshold;
private volatile PrioritizedEsThreadPoolExecutor updateTasksExecutor;
/** Those 3 state listeners are changing infrequently - CopyOnWriteArrayList is just fine */
private final Collection<ClusterStateListener> priorityClusterStateListeners =
new CopyOnWriteArrayList<>();
private final Collection<ClusterStateListener> clusterStateListeners =
new CopyOnWriteArrayList<>();
private final Collection<ClusterStateListener> lastClusterStateListeners =
new CopyOnWriteArrayList<>();
private final Map<ClusterStateTaskExecutor, List<UpdateTask>> updateTasksPerExecutor =
new HashMap<>();
// TODO this is rather frequently changing I guess a Synced Set would be better here and a
// dedicated remove API
private final Collection<ClusterStateListener> postAppliedListeners =
new CopyOnWriteArrayList<>();
private final Iterable<ClusterStateListener> preAppliedListeners =
Iterables.concat(
priorityClusterStateListeners, clusterStateListeners, lastClusterStateListeners);
private final LocalNodeMasterListeners localNodeMasterListeners;
private final Queue<NotifyTimeout> onGoingTimeouts = ConcurrentCollections.newQueue();
private volatile ClusterState clusterState;
private final ClusterBlocks.Builder initialBlocks;
private NodeConnectionsService nodeConnectionsService;
public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
super(settings);
this.operationRouting = new OperationRouting(settings, clusterSettings);
this.threadPool = threadPool;
this.clusterSettings = clusterSettings;
this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
// will be replaced on doStart.
this.clusterState = ClusterState.builder(clusterName).build();
this.clusterSettings.addSettingsUpdateConsumer(
CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING, this::setSlowTaskLoggingThreshold);
this.slowTaskLoggingThreshold =
CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(settings);
localNodeMasterListeners = new LocalNodeMasterListeners(threadPool);
initialBlocks = ClusterBlocks.builder();
}
private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
this.slowTaskLoggingThreshold = slowTaskLoggingThreshold;
}
public synchronized void setClusterStatePublisher(
BiConsumer<ClusterChangedEvent, Discovery.AckListener> publisher) {
clusterStatePublisher = publisher;
}
public synchronized void setLocalNode(DiscoveryNode localNode) {
assert clusterState.nodes().getLocalNodeId() == null : "local node is already set";
DiscoveryNodes.Builder nodeBuilder =
DiscoveryNodes.builder(clusterState.nodes()).add(localNode).localNodeId(localNode.getId());
this.clusterState = ClusterState.builder(clusterState).nodes(nodeBuilder).build();
}
public synchronized void setNodeConnectionsService(
NodeConnectionsService nodeConnectionsService) {
assert this.nodeConnectionsService == null : "nodeConnectionsService is already set";
this.nodeConnectionsService = nodeConnectionsService;
}
/** Adds an initial block to be set on the first cluster state created. */
public synchronized void addInitialStateBlock(ClusterBlock block) throws IllegalStateException {
if (lifecycle.started()) {
throw new IllegalStateException("can't set initial block when started");
}
initialBlocks.addGlobalBlock(block);
}
/** Remove an initial block to be set on the first cluster state created. */
public synchronized void removeInitialStateBlock(ClusterBlock block)
throws IllegalStateException {
removeInitialStateBlock(block.id());
}
/** Remove an initial block to be set on the first cluster state created. */
public synchronized void removeInitialStateBlock(int blockId) throws IllegalStateException {
if (lifecycle.started()) {
throw new IllegalStateException("can't set initial block when started");
}
initialBlocks.removeGlobalBlock(blockId);
}
@Override
protected synchronized void doStart() {
Objects.requireNonNull(
clusterStatePublisher, "please set a cluster state publisher before starting");
Objects.requireNonNull(
clusterState.nodes().getLocalNode(), "please set the local node before starting");
Objects.requireNonNull(
nodeConnectionsService, "please set the node connection service before starting");
add(localNodeMasterListeners);
this.clusterState = ClusterState.builder(clusterState).blocks(initialBlocks).build();
this.updateTasksExecutor =
EsExecutors.newSinglePrioritizing(
UPDATE_THREAD_NAME,
daemonThreadFactory(settings, UPDATE_THREAD_NAME),
threadPool.getThreadContext());
this.clusterState = ClusterState.builder(clusterState).blocks(initialBlocks).build();
}
@Override
protected synchronized void doStop() {
for (NotifyTimeout onGoingTimeout : onGoingTimeouts) {
onGoingTimeout.cancel();
try {
onGoingTimeout.cancel();
onGoingTimeout.listener.onClose();
} catch (Exception ex) {
logger.debug("failed to notify listeners on shutdown", ex);
}
}
ThreadPool.terminate(updateTasksExecutor, 10, TimeUnit.SECONDS);
// close timeout listeners that did not have an ongoing timeout
postAppliedListeners
.stream()
.filter(listener -> listener instanceof TimeoutClusterStateListener)
.map(listener -> (TimeoutClusterStateListener) listener)
.forEach(TimeoutClusterStateListener::onClose);
remove(localNodeMasterListeners);
}
@Override
protected synchronized void doClose() {}
/** The local node. */
public DiscoveryNode localNode() {
DiscoveryNode localNode = clusterState.getNodes().getLocalNode();
if (localNode == null) {
throw new IllegalStateException("No local node found. Is the node started?");
}
return localNode;
}
public OperationRouting operationRouting() {
return operationRouting;
}
/** The current state. */
public ClusterState state() {
return this.clusterState;
}
/** Adds a priority listener for updated cluster states. */
public void addFirst(ClusterStateListener listener) {
priorityClusterStateListeners.add(listener);
}
/** Adds last listener. */
public void addLast(ClusterStateListener listener) {
lastClusterStateListeners.add(listener);
}
/** Adds a listener for updated cluster states. */
public void add(ClusterStateListener listener) {
clusterStateListeners.add(listener);
}
/** Removes a listener for updated cluster states. */
public void remove(ClusterStateListener listener) {
clusterStateListeners.remove(listener);
priorityClusterStateListeners.remove(listener);
lastClusterStateListeners.remove(listener);
postAppliedListeners.remove(listener);
for (Iterator<NotifyTimeout> it = onGoingTimeouts.iterator(); it.hasNext(); ) {
NotifyTimeout timeout = it.next();
if (timeout.listener.equals(listener)) {
timeout.cancel();
it.remove();
}
}
}
/** Add a listener for on/off local node master events */
public void add(LocalNodeMasterListener listener) {
localNodeMasterListeners.add(listener);
}
/** Remove the given listener for on/off local master events */
public void remove(LocalNodeMasterListener listener) {
localNodeMasterListeners.remove(listener);
}
/**
* Adds a cluster state listener that will timeout after the provided timeout, and is executed
* after the clusterstate has been successfully applied ie. is in state {@link
* org.elasticsearch.cluster.ClusterState.ClusterStateStatus#APPLIED} NOTE: a {@code null} timeout
* means that the listener will never be removed automatically
*/
public void add(@Nullable final TimeValue timeout, final TimeoutClusterStateListener listener) {
if (lifecycle.stoppedOrClosed()) {
listener.onClose();
return;
}
// call the post added notification on the same event thread
try {
updateTasksExecutor.execute(
new SourcePrioritizedRunnable(Priority.HIGH, "_add_listener_") {
@Override
public void run() {
if (timeout != null) {
NotifyTimeout notifyTimeout = new NotifyTimeout(listener, timeout);
notifyTimeout.future =
threadPool.schedule(timeout, ThreadPool.Names.GENERIC, notifyTimeout);
onGoingTimeouts.add(notifyTimeout);
}
postAppliedListeners.add(listener);
listener.postAdded();
}
});
} catch (EsRejectedExecutionException e) {
if (lifecycle.stoppedOrClosed()) {
listener.onClose();
} else {
throw e;
}
}
}
/**
* Submits a cluster state update task; unlike {@link #submitStateUpdateTask(String, Object,
* ClusterStateTaskConfig, ClusterStateTaskExecutor, ClusterStateTaskListener)}, submitted updates
* will not be batched.
*
* @param source the source of the cluster state update task
* @param updateTask the full context for the cluster state update task
*/
public void submitStateUpdateTask(final String source, final ClusterStateUpdateTask updateTask) {
submitStateUpdateTask(source, updateTask, updateTask, updateTask, updateTask);
}
/**
* Submits a cluster state update task; submitted updates will be batched across the same instance
* of executor. The exact batching semantics depend on the underlying implementation but a rough
* guideline is that if the update task is submitted while there are pending update tasks for the
* same executor, these update tasks will all be executed on the executor in a single batch
*
* @param source the source of the cluster state update task
* @param task the state needed for the cluster state update task
* @param config the cluster state update task configuration
* @param executor the cluster state update task executor; tasks that share the same executor will
* be executed batches on this executor
* @param listener callback after the cluster state update task completes
* @param <T> the type of the cluster state update task state
*/
public <T> void submitStateUpdateTask(
final String source,
final T task,
final ClusterStateTaskConfig config,
final ClusterStateTaskExecutor<T> executor,
final ClusterStateTaskListener listener) {
submitStateUpdateTasks(source, Collections.singletonMap(task, listener), config, executor);
}
/**
* Submits a batch of cluster state update tasks; submitted updates are guaranteed to be processed
* together, potentially with more tasks of the same executor.
*
* @param source the source of the cluster state update task
* @param tasks a map of update tasks and their corresponding listeners
* @param config the cluster state update task configuration
* @param executor the cluster state update task executor; tasks that share the same executor will
* be executed batches on this executor
* @param <T> the type of the cluster state update task state
*/
public <T> void submitStateUpdateTasks(
final String source,
final Map<T, ClusterStateTaskListener> tasks,
final ClusterStateTaskConfig config,
final ClusterStateTaskExecutor<T> executor) {
if (!lifecycle.started()) {
return;
}
if (tasks.isEmpty()) {
return;
}
try {
// convert to an identity map to check for dups based on update tasks semantics of using
// identity instead of equal
final IdentityHashMap<T, ClusterStateTaskListener> tasksIdentity =
new IdentityHashMap<>(tasks);
final List<UpdateTask<T>> updateTasks =
tasksIdentity
.entrySet()
.stream()
.map(
entry ->
new UpdateTask<>(
source, entry.getKey(), config, executor, safe(entry.getValue(), logger)))
.collect(Collectors.toList());
synchronized (updateTasksPerExecutor) {
List<UpdateTask> existingTasks =
updateTasksPerExecutor.computeIfAbsent(executor, k -> new ArrayList<>());
for (@SuppressWarnings("unchecked") UpdateTask<T> existing : existingTasks) {
if (tasksIdentity.containsKey(existing.task)) {
throw new IllegalStateException(
"task ["
+ executor.describeTasks(Collections.singletonList(existing.task))
+ "] with source ["
+ source
+ "] is already queued");
}
}
existingTasks.addAll(updateTasks);
}
final UpdateTask<T> firstTask = updateTasks.get(0);
if (config.timeout() != null) {
updateTasksExecutor.execute(
firstTask,
threadPool.scheduler(),
config.timeout(),
() ->
threadPool
.generic()
.execute(
() -> {
for (UpdateTask<T> task : updateTasks) {
if (task.processed.getAndSet(true) == false) {
logger.debug(
"cluster state update task [{}] timed out after [{}]",
source,
config.timeout());
task.listener.onFailure(
source,
new ProcessClusterEventTimeoutException(
config.timeout(), source));
}
}
}));
} else {
updateTasksExecutor.execute(firstTask);
}
} catch (EsRejectedExecutionException e) {
// ignore cases where we are shutting down..., there is really nothing interesting
// to be done here...
if (!lifecycle.stoppedOrClosed()) {
throw e;
}
}
}
/** Returns the tasks that are pending. */
public List<PendingClusterTask> pendingTasks() {
PrioritizedEsThreadPoolExecutor.Pending[] pendings = updateTasksExecutor.getPending();
List<PendingClusterTask> pendingClusterTasks = new ArrayList<>(pendings.length);
for (PrioritizedEsThreadPoolExecutor.Pending pending : pendings) {
final String source;
final long timeInQueue;
// we have to capture the task as it will be nulled after execution and we don't want to
// change while we check things here.
final Object task = pending.task;
if (task == null) {
continue;
} else if (task instanceof SourcePrioritizedRunnable) {
SourcePrioritizedRunnable runnable = (SourcePrioritizedRunnable) task;
source = runnable.source();
timeInQueue = runnable.getAgeInMillis();
} else {
assert false : "expected SourcePrioritizedRunnable got " + task.getClass();
source = "unknown [" + task.getClass() + "]";
timeInQueue = 0;
}
pendingClusterTasks.add(
new PendingClusterTask(
pending.insertionOrder,
pending.priority,
new Text(source),
timeInQueue,
pending.executing));
}
return pendingClusterTasks;
}
/** Returns the number of currently pending tasks. */
public int numberOfPendingTasks() {
return updateTasksExecutor.getNumberOfPendingTasks();
}
/**
* Returns the maximum wait time for tasks in the queue
*
* @return A zero time value if the queue is empty, otherwise the time value oldest task waiting
* in the queue
*/
public TimeValue getMaxTaskWaitTime() {
return updateTasksExecutor.getMaxTaskWaitTime();
}
/** asserts that the current thread is the cluster state update thread */
public static boolean assertClusterStateThread() {
assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME)
: "not called from the cluster state update thread";
return true;
}
public ClusterName getClusterName() {
return clusterName;
}
abstract static class SourcePrioritizedRunnable extends PrioritizedRunnable {
protected final String source;
public SourcePrioritizedRunnable(Priority priority, String source) {
super(priority);
this.source = source;
}
public String source() {
return source;
}
}
<T> void runTasksForExecutor(ClusterStateTaskExecutor<T> executor) {
final ArrayList<UpdateTask<T>> toExecute = new ArrayList<>();
final Map<String, ArrayList<T>> processTasksBySource = new HashMap<>();
synchronized (updateTasksPerExecutor) {
List<UpdateTask> pending = updateTasksPerExecutor.remove(executor);
if (pending != null) {
for (UpdateTask<T> task : pending) {
if (task.processed.getAndSet(true) == false) {
logger.trace("will process {}", task.toString(executor));
toExecute.add(task);
processTasksBySource
.computeIfAbsent(task.source, s -> new ArrayList<>())
.add(task.task);
} else {
logger.trace("skipping {}, already processed", task.toString(executor));
}
}
}
}
if (toExecute.isEmpty()) {
return;
}
final String tasksSummary =
processTasksBySource
.entrySet()
.stream()
.map(
entry -> {
String tasks = executor.describeTasks(entry.getValue());
return tasks.isEmpty() ? entry.getKey() : entry.getKey() + "[" + tasks + "]";
})
.reduce((s1, s2) -> s1 + ", " + s2)
.orElse("");
if (!lifecycle.started()) {
logger.debug("processing [{}]: ignoring, cluster_service not started", tasksSummary);
return;
}
logger.debug("processing [{}]: execute", tasksSummary);
ClusterState previousClusterState = clusterState;
if (!previousClusterState.nodes().isLocalNodeElectedMaster() && executor.runOnlyOnMaster()) {
logger.debug("failing [{}]: local node is no longer master", tasksSummary);
toExecute.stream().forEach(task -> task.listener.onNoLongerMaster(task.source));
return;
}
ClusterStateTaskExecutor.BatchResult<T> batchResult;
long startTimeNS = currentTimeInNanos();
try {
List<T> inputs =
toExecute.stream().map(tUpdateTask -> tUpdateTask.task).collect(Collectors.toList());
batchResult = executor.execute(previousClusterState, inputs);
} catch (Exception e) {
TimeValue executionTime =
TimeValue.timeValueMillis(
Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
if (logger.isTraceEnabled()) {
logger.trace(
(Supplier<?>)
() ->
new ParameterizedMessage(
"failed to execute cluster state update in [{}], state:\nversion [{}], source [{}]\n{}{}{}",
executionTime,
previousClusterState.version(),
tasksSummary,
previousClusterState.nodes().prettyPrint(),
previousClusterState.routingTable().prettyPrint(),
previousClusterState.getRoutingNodes().prettyPrint()),
e);
}
warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
batchResult =
ClusterStateTaskExecutor.BatchResult.<T>builder()
.failures(toExecute.stream().map(updateTask -> updateTask.task)::iterator, e)
.build(previousClusterState);
}
assert batchResult.executionResults != null;
assert batchResult.executionResults.size() == toExecute.size()
: String.format(
Locale.ROOT,
"expected [%d] task result%s but was [%d]",
toExecute.size(),
toExecute.size() == 1 ? "" : "s",
batchResult.executionResults.size());
boolean assertsEnabled = false;
assert (assertsEnabled = true);
if (assertsEnabled) {
for (UpdateTask<T> updateTask : toExecute) {
assert batchResult.executionResults.containsKey(updateTask.task)
: "missing task result for " + updateTask.toString(executor);
}
}
ClusterState newClusterState = batchResult.resultingState;
final ArrayList<UpdateTask<T>> proccessedListeners = new ArrayList<>();
// fail all tasks that have failed and extract those that are waiting for results
for (UpdateTask<T> updateTask : toExecute) {
assert batchResult.executionResults.containsKey(updateTask.task)
: "missing " + updateTask.toString(executor);
final ClusterStateTaskExecutor.TaskResult executionResult =
batchResult.executionResults.get(updateTask.task);
executionResult.handle(
() -> proccessedListeners.add(updateTask),
ex -> {
logger.debug(
(Supplier<?>)
() ->
new ParameterizedMessage(
"cluster state update task {} failed", updateTask.toString(executor)),
ex);
updateTask.listener.onFailure(updateTask.source, ex);
});
}
if (previousClusterState == newClusterState) {
for (UpdateTask<T> task : proccessedListeners) {
if (task.listener instanceof AckedClusterStateTaskListener) {
// no need to wait for ack if nothing changed, the update can be counted as acknowledged
((AckedClusterStateTaskListener) task.listener).onAllNodesAcked(null);
}
task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
}
TimeValue executionTime =
TimeValue.timeValueMillis(
Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
logger.debug(
"processing [{}]: took [{}] no change in cluster_state", tasksSummary, executionTime);
warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
return;
}
try {
ArrayList<Discovery.AckListener> ackListeners = new ArrayList<>();
if (newClusterState.nodes().isLocalNodeElectedMaster()) {
// only the master controls the version numbers
Builder builder = ClusterState.builder(newClusterState).incrementVersion();
if (previousClusterState.routingTable() != newClusterState.routingTable()) {
builder.routingTable(
RoutingTable.builder(newClusterState.routingTable())
.version(newClusterState.routingTable().version() + 1)
.build());
}
if (previousClusterState.metaData() != newClusterState.metaData()) {
builder.metaData(
MetaData.builder(newClusterState.metaData())
.version(newClusterState.metaData().version() + 1));
}
newClusterState = builder.build();
for (UpdateTask<T> task : proccessedListeners) {
if (task.listener instanceof AckedClusterStateTaskListener) {
final AckedClusterStateTaskListener ackedListener =
(AckedClusterStateTaskListener) task.listener;
if (ackedListener.ackTimeout() == null || ackedListener.ackTimeout().millis() == 0) {
ackedListener.onAckTimeout();
} else {
try {
ackListeners.add(
new AckCountDownListener(
ackedListener,
newClusterState.version(),
newClusterState.nodes(),
threadPool));
} catch (EsRejectedExecutionException ex) {
if (logger.isDebugEnabled()) {
logger.debug(
"Couldn't schedule timeout thread - node might be shutting down", ex);
}
// timeout straightaway, otherwise we could wait forever as the timeout thread has
// not started
ackedListener.onAckTimeout();
}
}
}
}
}
final Discovery.AckListener ackListener = new DelegetingAckListener(ackListeners);
newClusterState.status(ClusterState.ClusterStateStatus.BEING_APPLIED);
if (logger.isTraceEnabled()) {
logger.trace(
"cluster state updated, source [{}]\n{}", tasksSummary, newClusterState.prettyPrint());
} else if (logger.isDebugEnabled()) {
logger.debug(
"cluster state updated, version [{}], source [{}]",
newClusterState.version(),
tasksSummary);
}
ClusterChangedEvent clusterChangedEvent =
new ClusterChangedEvent(tasksSummary, newClusterState, previousClusterState);
// new cluster state, notify all listeners
final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta();
if (nodesDelta.hasChanges() && logger.isInfoEnabled()) {
String summary = nodesDelta.shortSummary();
if (summary.length() > 0) {
logger.info("{}, reason: {}", summary, tasksSummary);
}
}
nodeConnectionsService.connectToAddedNodes(clusterChangedEvent);
// if we are the master, publish the new state to all nodes
// we publish here before we send a notification to all the listeners, since if it fails
// we don't want to notify
if (newClusterState.nodes().isLocalNodeElectedMaster()) {
logger.debug("publishing cluster state version [{}]", newClusterState.version());
try {
clusterStatePublisher.accept(clusterChangedEvent, ackListener);
} catch (Discovery.FailedToCommitClusterStateException t) {
final long version = newClusterState.version();
logger.warn(
(Supplier<?>)
() ->
new ParameterizedMessage(
"failing [{}]: failed to commit cluster state version [{}]",
tasksSummary,
version),
t);
proccessedListeners.forEach(task -> task.listener.onFailure(task.source, t));
return;
}
}
// update the current cluster state
clusterState = newClusterState;
logger.debug("set local cluster state to version {}", newClusterState.version());
try {
// nothing to do until we actually recover from the gateway or any other block indicates we
// need to disable persistency
if (clusterChangedEvent.state().blocks().disableStatePersistence() == false
&& clusterChangedEvent.metaDataChanged()) {
final Settings incomingSettings = clusterChangedEvent.state().metaData().settings();
clusterSettings.applySettings(incomingSettings);
}
} catch (Exception ex) {
logger.warn("failed to apply cluster settings", ex);
}
for (ClusterStateListener listener : preAppliedListeners) {
try {
listener.clusterChanged(clusterChangedEvent);
} catch (Exception ex) {
logger.warn("failed to notify ClusterStateListener", ex);
}
}
nodeConnectionsService.disconnectFromRemovedNodes(clusterChangedEvent);
newClusterState.status(ClusterState.ClusterStateStatus.APPLIED);
for (ClusterStateListener listener : postAppliedListeners) {
try {
listener.clusterChanged(clusterChangedEvent);
} catch (Exception ex) {
logger.warn("failed to notify ClusterStateListener", ex);
}
}
// manual ack only from the master at the end of the publish
if (newClusterState.nodes().isLocalNodeElectedMaster()) {
try {
ackListener.onNodeAck(newClusterState.nodes().getLocalNode(), null);
} catch (Exception e) {
final DiscoveryNode localNode = newClusterState.nodes().getLocalNode();
logger.debug(
(Supplier<?>)
() ->
new ParameterizedMessage(
"error while processing ack for master node [{}]", localNode),
e);
}
}
for (UpdateTask<T> task : proccessedListeners) {
task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
}
try {
executor.clusterStatePublished(clusterChangedEvent);
} catch (Exception e) {
logger.error(
(Supplier<?>)
() ->
new ParameterizedMessage(
"exception thrown while notifying executor of new cluster state publication [{}]",
tasksSummary),
e);
}
TimeValue executionTime =
TimeValue.timeValueMillis(
Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
logger.debug(
"processing [{}]: took [{}] done applying updated cluster_state (version: {}, uuid: {})",
tasksSummary,
executionTime,
newClusterState.version(),
newClusterState.stateUUID());
warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
} catch (Exception e) {
TimeValue executionTime =
TimeValue.timeValueMillis(
Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
final long version = newClusterState.version();
final String stateUUID = newClusterState.stateUUID();
final String prettyPrint = newClusterState.prettyPrint();
logger.warn(
(Supplier<?>)
() ->
new ParameterizedMessage(
"failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}",
executionTime,
version,
stateUUID,
tasksSummary,
prettyPrint),
e);
// TODO: do we want to call updateTask.onFailure here?
}
}
// this one is overridden in tests so we can control time
protected long currentTimeInNanos() {
return System.nanoTime();
}
private static SafeClusterStateTaskListener safe(
ClusterStateTaskListener listener, Logger logger) {
if (listener instanceof AckedClusterStateTaskListener) {
return new SafeAckedClusterStateTaskListener(
(AckedClusterStateTaskListener) listener, logger);
} else {
return new SafeClusterStateTaskListener(listener, logger);
}
}
private static class SafeClusterStateTaskListener implements ClusterStateTaskListener {
private final ClusterStateTaskListener listener;
private final Logger logger;
public SafeClusterStateTaskListener(ClusterStateTaskListener listener, Logger logger) {
this.listener = listener;
this.logger = logger;
}
@Override
public void onFailure(String source, Exception e) {
try {
listener.onFailure(source, e);
} catch (Exception inner) {
inner.addSuppressed(e);
logger.error(
(Supplier<?>)
() ->
new ParameterizedMessage(
"exception thrown by listener notifying of failure from [{}]", source),
inner);
}
}
@Override
public void onNoLongerMaster(String source) {
try {
listener.onNoLongerMaster(source);
} catch (Exception e) {
logger.error(
(Supplier<?>)
() ->
new ParameterizedMessage(
"exception thrown by listener while notifying no longer master from [{}]",
source),
e);
}
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
try {
listener.clusterStateProcessed(source, oldState, newState);
} catch (Exception e) {
logger.error(
(Supplier<?>)
() ->
new ParameterizedMessage(
"exception thrown by listener while notifying of cluster state processed from [{}], old cluster state:\n"
+ "{}\nnew cluster state:\n{}",
source,
oldState.prettyPrint(),
newState.prettyPrint()),
e);
}
}
}
private static class SafeAckedClusterStateTaskListener extends SafeClusterStateTaskListener
implements AckedClusterStateTaskListener {
private final AckedClusterStateTaskListener listener;
private final Logger logger;
public SafeAckedClusterStateTaskListener(
AckedClusterStateTaskListener listener, Logger logger) {
super(listener, logger);
this.listener = listener;
this.logger = logger;
}
@Override
public boolean mustAck(DiscoveryNode discoveryNode) {
return listener.mustAck(discoveryNode);
}
@Override
public void onAllNodesAcked(@Nullable Exception e) {
try {
listener.onAllNodesAcked(e);
} catch (Exception inner) {
inner.addSuppressed(e);
logger.error("exception thrown by listener while notifying on all nodes acked", inner);
}
}
@Override
public void onAckTimeout() {
try {
listener.onAckTimeout();
} catch (Exception e) {
logger.error("exception thrown by listener while notifying on ack timeout", e);
}
}
@Override
public TimeValue ackTimeout() {
return listener.ackTimeout();
}
}
class UpdateTask<T> extends SourcePrioritizedRunnable {
public final T task;
public final ClusterStateTaskConfig config;
public final ClusterStateTaskExecutor<T> executor;
public final ClusterStateTaskListener listener;
public final AtomicBoolean processed = new AtomicBoolean();
UpdateTask(
String source,
T task,
ClusterStateTaskConfig config,
ClusterStateTaskExecutor<T> executor,
ClusterStateTaskListener listener) {
super(config.priority(), source);
this.task = task;
this.config = config;
this.executor = executor;
this.listener = listener;
}
@Override
public void run() {
// if this task is already processed, the executor shouldn't execute other tasks (that arrived
// later),
// to give other executors a chance to execute their tasks.
if (processed.get() == false) {
runTasksForExecutor(executor);
}
}
public String toString(ClusterStateTaskExecutor<T> executor) {
String taskDescription = executor.describeTasks(Collections.singletonList(task));
if (taskDescription.isEmpty()) {
return "[" + source + "]";
} else {
return "[" + source + "[" + taskDescription + "]]";
}
}
}
private void warnAboutSlowTaskIfNeeded(TimeValue executionTime, String source) {
if (executionTime.getMillis() > slowTaskLoggingThreshold.getMillis()) {
logger.warn(
"cluster state update task [{}] took [{}] above the warn threshold of {}",
source,
executionTime,
slowTaskLoggingThreshold);
}
}
class NotifyTimeout implements Runnable {
final TimeoutClusterStateListener listener;
final TimeValue timeout;
volatile ScheduledFuture future;
NotifyTimeout(TimeoutClusterStateListener listener, TimeValue timeout) {
this.listener = listener;
this.timeout = timeout;
}
public void cancel() {
FutureUtils.cancel(future);
}
@Override
public void run() {
if (future != null && future.isCancelled()) {
return;
}
if (lifecycle.stoppedOrClosed()) {
listener.onClose();
} else {
listener.onTimeout(this.timeout);
}
// note, we rely on the listener to remove itself in case of timeout if needed
}
}
private static class LocalNodeMasterListeners implements ClusterStateListener {
private final List<LocalNodeMasterListener> listeners = new CopyOnWriteArrayList<>();
private final ThreadPool threadPool;
private volatile boolean master = false;
private LocalNodeMasterListeners(ThreadPool threadPool) {
this.threadPool = threadPool;
}
@Override
public void clusterChanged(ClusterChangedEvent event) {
if (!master && event.localNodeMaster()) {
master = true;
for (LocalNodeMasterListener listener : listeners) {
Executor executor = threadPool.executor(listener.executorName());
executor.execute(new OnMasterRunnable(listener));
}
return;
}
if (master && !event.localNodeMaster()) {
master = false;
for (LocalNodeMasterListener listener : listeners) {
Executor executor = threadPool.executor(listener.executorName());
executor.execute(new OffMasterRunnable(listener));
}
}
}
private void add(LocalNodeMasterListener listener) {
listeners.add(listener);
}
private void remove(LocalNodeMasterListener listener) {
listeners.remove(listener);
}
private void clear() {
listeners.clear();
}
}
private static class OnMasterRunnable implements Runnable {
private final LocalNodeMasterListener listener;
private OnMasterRunnable(LocalNodeMasterListener listener) {
this.listener = listener;
}
@Override
public void run() {
listener.onMaster();
}
}
private static class OffMasterRunnable implements Runnable {
private final LocalNodeMasterListener listener;
private OffMasterRunnable(LocalNodeMasterListener listener) {
this.listener = listener;
}
@Override
public void run() {
listener.offMaster();
}
}
private static class DelegetingAckListener implements Discovery.AckListener {
private final List<Discovery.AckListener> listeners;
private DelegetingAckListener(List<Discovery.AckListener> listeners) {
this.listeners = listeners;
}
@Override
public void onNodeAck(DiscoveryNode node, @Nullable Exception e) {
for (Discovery.AckListener listener : listeners) {
listener.onNodeAck(node, e);
}
}
@Override
public void onTimeout() {
throw new UnsupportedOperationException("no timeout delegation");
}
}
private static class AckCountDownListener implements Discovery.AckListener {
private static final Logger logger = Loggers.getLogger(AckCountDownListener.class);
private final AckedClusterStateTaskListener ackedTaskListener;
private final CountDown countDown;
private final DiscoveryNodes nodes;
private final long clusterStateVersion;
private final Future<?> ackTimeoutCallback;
private Exception lastFailure;
AckCountDownListener(
AckedClusterStateTaskListener ackedTaskListener,
long clusterStateVersion,
DiscoveryNodes nodes,
ThreadPool threadPool) {
this.ackedTaskListener = ackedTaskListener;
this.clusterStateVersion = clusterStateVersion;
this.nodes = nodes;
int countDown = 0;
for (DiscoveryNode node : nodes) {
if (ackedTaskListener.mustAck(node)) {
countDown++;
}
}
// we always wait for at least 1 node (the master)
countDown = Math.max(1, countDown);
logger.trace(
"expecting {} acknowledgements for cluster_state update (version: {})",
countDown,
clusterStateVersion);
this.countDown = new CountDown(countDown);
this.ackTimeoutCallback =
threadPool.schedule(
ackedTaskListener.ackTimeout(),
ThreadPool.Names.GENERIC,
new Runnable() {
@Override
public void run() {
onTimeout();
}
});
}
@Override
public void onNodeAck(DiscoveryNode node, @Nullable Exception e) {
if (!ackedTaskListener.mustAck(node)) {
// we always wait for the master ack anyway
if (!node.equals(nodes.getMasterNode())) {
return;
}
}
if (e == null) {
logger.trace(
"ack received from node [{}], cluster_state update (version: {})",
node,
clusterStateVersion);
} else {
this.lastFailure = e;
logger.debug(
(Supplier<?>)
() ->
new ParameterizedMessage(
"ack received from node [{}], cluster_state update (version: {})",
node,
clusterStateVersion),
e);
}
if (countDown.countDown()) {
logger.trace(
"all expected nodes acknowledged cluster_state update (version: {})",
clusterStateVersion);
FutureUtils.cancel(ackTimeoutCallback);
ackedTaskListener.onAllNodesAcked(lastFailure);
}
}
@Override
public void onTimeout() {
if (countDown.fastForward()) {
logger.trace(
"timeout waiting for acknowledgement for cluster_state update (version: {})",
clusterStateVersion);
ackedTaskListener.onAckTimeout();
}
}
}
public ClusterSettings getClusterSettings() {
return clusterSettings;
}
public Settings getSettings() {
return settings;
}
}
@Test
public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
final int numShards = between(1, 5);
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("test")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
final double exceptionRate;
final double exceptionOnOpenRate;
if (frequently()) {
if (randomBoolean()) {
if (randomBoolean()) {
exceptionOnOpenRate = 1.0 / between(5, 100);
exceptionRate = 0.0d;
} else {
exceptionRate = 1.0 / between(5, 100);
exceptionOnOpenRate = 0.0d;
}
} else {
exceptionOnOpenRate = 1.0 / between(5, 100);
exceptionRate = 1.0 / between(5, 100);
}
} else {
// rarely no exception
exceptionRate = 0d;
exceptionOnOpenRate = 0d;
}
Builder settings =
settingsBuilder()
.put("index.number_of_shards", numShards)
.put("index.number_of_replicas", randomIntBetween(0, 1))
.put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE, exceptionRate)
.put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE_ON_OPEN, exceptionOnOpenRate)
.put(MockDirectoryHelper.CHECK_INDEX_ON_CLOSE, true);
logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(settings)
.addMapping("type", mapping)
.execute()
.actionGet();
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.health(
Requests.clusterHealthRequest()
.waitForYellowStatus()
.timeout(TimeValue.timeValueSeconds(5)))
.get(); // it's OK to timeout here
final int numDocs;
final boolean expectAllShardsFailed;
if (clusterHealthResponse.isTimedOut()) {
/* some seeds just won't let you create the index at all and we enter a ping-pong mode
* trying one node after another etc. that is ok but we need to make sure we don't wait
* forever when indexing documents so we set numDocs = 1 and expecte all shards to fail
* when we search below.*/
logger.info("ClusterHealth timed out - only index one doc and expect searches to fail");
numDocs = 1;
expectAllShardsFailed = true;
} else {
numDocs = between(10, 100);
expectAllShardsFailed = false;
}
long numCreated = 0;
boolean[] added = new boolean[numDocs];
for (int i = 0; i < numDocs; i++) {
try {
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "" + i)
.setTimeout(TimeValue.timeValueSeconds(1))
.setSource("test", English.intToEnglish(i))
.get();
if (indexResponse.isCreated()) {
numCreated++;
added[i] = true;
}
} catch (ElasticSearchException ex) {
}
}
logger.info("Start Refresh");
RefreshResponse refreshResponse =
client()
.admin()
.indices()
.prepareRefresh("test")
.execute()
.get(); // don't assert on failures here
final boolean refreshFailed =
refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
logger.info(
"Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ",
refreshFailed,
refreshResponse.getFailedShards(),
refreshResponse.getShardFailures().length,
refreshResponse.getSuccessfulShards(),
refreshResponse.getTotalShards());
final int numSearches = atLeast(10);
// we don't check anything here really just making sure we don't leave any open files or a
// broken index behind.
for (int i = 0; i < numSearches; i++) {
try {
int docToQuery = between(0, numDocs - 1);
long expectedResults = added[docToQuery] ? 1 : 0;
logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery)))
.get();
logger.info(
"Successful shards: [{}] numShards: [{}]",
searchResponse.getSuccessfulShards(),
numShards);
if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(expectedResults));
}
// check match all
searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).get();
if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(numCreated));
}
} catch (SearchPhaseExecutionException ex) {
if (!expectAllShardsFailed) {
throw ex;
} else {
logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
}
}
}
}
@Override
public TimeValue defaultValue() {
return TimeValue.timeValueSeconds(5);
}
/** A node level service that delete expired docs on node primary shards. */
public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLService> {
public static final Setting<TimeValue> INDICES_TTL_INTERVAL_SETTING =
Setting.positiveTimeSetting(
"indices.ttl.interval",
TimeValue.timeValueSeconds(60),
Property.Dynamic,
Property.NodeScope);
private final ClusterService clusterService;
private final IndicesService indicesService;
private final TransportBulkAction bulkAction;
private final int bulkSize;
private PurgerThread purgerThread;
@Inject
public IndicesTTLService(
Settings settings,
ClusterService clusterService,
IndicesService indicesService,
ClusterSettings clusterSettings,
TransportBulkAction bulkAction) {
super(settings);
this.clusterService = clusterService;
this.indicesService = indicesService;
TimeValue interval = INDICES_TTL_INTERVAL_SETTING.get(settings);
this.bulkAction = bulkAction;
this.bulkSize = this.settings.getAsInt("indices.ttl.bulk_size", 10000);
this.purgerThread =
new PurgerThread(EsExecutors.threadName(settings, "[ttl_expire]"), interval);
clusterSettings.addSettingsUpdateConsumer(
INDICES_TTL_INTERVAL_SETTING, this.purgerThread::resetInterval);
}
@Override
protected void doStart() {
this.purgerThread.start();
}
@Override
protected void doStop() {
try {
this.purgerThread.shutdown();
} catch (InterruptedException e) {
// we intentionally do not want to restore the interruption flag, we're about to shutdown
// anyway
}
}
@Override
protected void doClose() {}
private class PurgerThread extends Thread {
private final AtomicBoolean running = new AtomicBoolean(true);
private final Notifier notifier;
private final CountDownLatch shutdownLatch = new CountDownLatch(1);
public PurgerThread(String name, TimeValue interval) {
super(name);
setDaemon(true);
this.notifier = new Notifier(interval);
}
public void shutdown() throws InterruptedException {
if (running.compareAndSet(true, false)) {
notifier.doNotify();
shutdownLatch.await();
}
}
public void resetInterval(TimeValue interval) {
notifier.setTimeout(interval);
}
@Override
public void run() {
try {
while (running.get()) {
try {
List<IndexShard> shardsToPurge = getShardsToPurge();
purgeShards(shardsToPurge);
} catch (Throwable e) {
if (running.get()) {
logger.warn("failed to execute ttl purge", e);
}
}
if (running.get()) {
notifier.await();
}
}
} finally {
shutdownLatch.countDown();
}
}
/**
* Returns the shards to purge, i.e. the local started primary shards that have ttl enabled and
* disable_purge to false
*/
private List<IndexShard> getShardsToPurge() {
List<IndexShard> shardsToPurge = new ArrayList<>();
MetaData metaData = clusterService.state().metaData();
for (IndexService indexService : indicesService) {
// check the value of disable_purge for this index
IndexMetaData indexMetaData = metaData.index(indexService.index());
if (indexMetaData == null) {
continue;
}
if (indexService.getIndexSettings().isTTLPurgeDisabled()) {
continue;
}
// check if ttl is enabled for at least one type of this index
boolean hasTTLEnabled = false;
for (String type : indexService.mapperService().types()) {
DocumentMapper documentType = indexService.mapperService().documentMapper(type);
if (documentType.TTLFieldMapper().enabled()) {
hasTTLEnabled = true;
break;
}
}
if (hasTTLEnabled) {
for (IndexShard indexShard : indexService) {
if (indexShard.state() == IndexShardState.STARTED
&& indexShard.routingEntry().primary()
&& indexShard.routingEntry().started()) {
shardsToPurge.add(indexShard);
}
}
}
}
return shardsToPurge;
}
public TimeValue getInterval() {
return notifier.getTimeout();
}
}
private void purgeShards(List<IndexShard> shardsToPurge) {
for (IndexShard shardToPurge : shardsToPurge) {
Query query =
shardToPurge
.mapperService()
.fullName(TTLFieldMapper.NAME)
.rangeQuery(null, System.currentTimeMillis(), false, true);
Engine.Searcher searcher = shardToPurge.acquireSearcher("indices_ttl");
try {
logger.debug(
"[{}][{}] purging shard",
shardToPurge.routingEntry().index(),
shardToPurge.routingEntry().id());
ExpiredDocsCollector expiredDocsCollector = new ExpiredDocsCollector();
searcher.searcher().search(query, expiredDocsCollector);
List<DocToPurge> docsToPurge = expiredDocsCollector.getDocsToPurge();
BulkRequest bulkRequest = new BulkRequest();
for (DocToPurge docToPurge : docsToPurge) {
bulkRequest.add(
new DeleteRequest()
.index(shardToPurge.routingEntry().getIndexName())
.type(docToPurge.type)
.id(docToPurge.id)
.version(docToPurge.version)
.routing(docToPurge.routing));
bulkRequest = processBulkIfNeeded(bulkRequest, false);
}
processBulkIfNeeded(bulkRequest, true);
} catch (Exception e) {
logger.warn("failed to purge", e);
} finally {
searcher.close();
}
}
}
private static class DocToPurge {
public final String type;
public final String id;
public final long version;
public final String routing;
public DocToPurge(String type, String id, long version, String routing) {
this.type = type;
this.id = id;
this.version = version;
this.routing = routing;
}
}
private class ExpiredDocsCollector extends SimpleCollector {
private LeafReaderContext context;
private List<DocToPurge> docsToPurge = new ArrayList<>();
public ExpiredDocsCollector() {}
@Override
public void setScorer(Scorer scorer) {}
@Override
public boolean needsScores() {
return false;
}
@Override
public void collect(int doc) {
try {
FieldsVisitor fieldsVisitor = new FieldsVisitor(false);
context.reader().document(doc, fieldsVisitor);
Uid uid = fieldsVisitor.uid();
final long version =
Versions.loadVersion(context.reader(), new Term(UidFieldMapper.NAME, uid.toBytesRef()));
docsToPurge.add(new DocToPurge(uid.type(), uid.id(), version, fieldsVisitor.routing()));
} catch (Exception e) {
logger.trace("failed to collect doc", e);
}
}
@Override
public void doSetNextReader(LeafReaderContext context) throws IOException {
this.context = context;
}
public List<DocToPurge> getDocsToPurge() {
return this.docsToPurge;
}
}
private BulkRequest processBulkIfNeeded(BulkRequest bulkRequest, boolean force) {
if ((force && bulkRequest.numberOfActions() > 0) || bulkRequest.numberOfActions() >= bulkSize) {
try {
bulkAction.executeBulk(
bulkRequest,
new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse bulkResponse) {
if (bulkResponse.hasFailures()) {
int failedItems = 0;
for (BulkItemResponse response : bulkResponse) {
if (response.isFailed()) failedItems++;
}
if (logger.isTraceEnabled()) {
logger.trace(
"bulk deletion failures for [{}]/[{}] items, failure message: [{}]",
failedItems,
bulkResponse.getItems().length,
bulkResponse.buildFailureMessage());
} else {
logger.error(
"bulk deletion failures for [{}]/[{}] items",
failedItems,
bulkResponse.getItems().length);
}
} else {
logger.trace("bulk deletion took {}ms", bulkResponse.getTookInMillis());
}
}
@Override
public void onFailure(Throwable e) {
if (logger.isTraceEnabled()) {
logger.trace("failed to execute bulk", e);
} else {
logger.warn("failed to execute bulk: ", e);
}
}
});
} catch (Exception e) {
logger.warn("failed to process bulk", e);
}
bulkRequest = new BulkRequest();
}
return bulkRequest;
}
private static final class Notifier {
private final ReentrantLock lock = new ReentrantLock();
private final Condition condition = lock.newCondition();
private volatile TimeValue timeout;
public Notifier(TimeValue timeout) {
assert timeout != null;
this.timeout = timeout;
}
public void await() {
lock.lock();
try {
condition.await(timeout.millis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
// we intentionally do not want to restore the interruption flag, we're about to shutdown
// anyway
} finally {
lock.unlock();
}
}
public void setTimeout(TimeValue timeout) {
assert timeout != null;
this.timeout = timeout;
doNotify();
}
public TimeValue getTimeout() {
return timeout;
}
public void doNotify() {
lock.lock();
try {
condition.signalAll();
} finally {
lock.unlock();
}
}
}
}
/*
* Holds all the configuration that is used to create an {@link Engine}.
* Once {@link Engine} has been created with this object, changes to this
* object will affect the {@link Engine} instance.
*/
public final class EngineConfig {
private final ShardId shardId;
private volatile boolean failOnMergeFailure = true;
private volatile boolean failEngineOnCorruption = true;
private volatile ByteSizeValue indexingBufferSize;
private volatile int indexConcurrency = IndexWriterConfig.DEFAULT_MAX_THREAD_STATES;
private volatile boolean compoundOnFlush = true;
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
private volatile boolean enableGcDeletes = true;
private volatile String codecName = DEFAULT_CODEC_NAME;
private final boolean optimizeAutoGenerateId;
private final ThreadPool threadPool;
private final ShardIndexingService indexingService;
private final IndexSettingsService indexSettingsService;
@Nullable private final IndicesWarmer warmer;
private final Store store;
private final SnapshotDeletionPolicy deletionPolicy;
private final Translog translog;
private final MergePolicyProvider mergePolicyProvider;
private final MergeSchedulerProvider mergeScheduler;
private final Analyzer analyzer;
private final Similarity similarity;
private final CodecService codecService;
private final Engine.FailedEngineListener failedEngineListener;
/**
* Index setting for index concurrency / number of threadstates in the indexwriter. The default is
* depending on the number of CPUs in the system. We use a 0.65 the number of CPUs or at least
* {@value org.apache.lucene.index.IndexWriterConfig#DEFAULT_MAX_THREAD_STATES} This setting is
* realtime updateable
*/
public static final String INDEX_CONCURRENCY_SETTING = "index.index_concurrency";
/** Index setting for compound file on flush. This setting is realtime updateable. */
public static final String INDEX_COMPOUND_ON_FLUSH = "index.compound_on_flush";
/**
* Setting to control auto generated ID optimizations. Default is <code>true</code> if not
* present. This setting is <b>not</b> realtime updateable.
*/
public static final String INDEX_OPTIMIZE_AUTOGENERATED_ID_SETTING =
"index.optimize_auto_generated_id";
/**
* Index setting to enable / disable deletes garbage collection. This setting is realtime
* updateable
*/
public static final String INDEX_GC_DELETES_SETTING = "index.gc_deletes";
/**
* Index setting to enable / disable engine failures on merge exceptions. Default is <code>true
* </code> / <tt>enabled</tt>. This setting is realtime updateable.
*/
public static final String INDEX_FAIL_ON_MERGE_FAILURE_SETTING = "index.fail_on_merge_failure";
/**
* Index setting to enable / disable engine failures on detected index corruptions. Default is
* <code>true</code> / <tt>enabled</tt>. This setting is realtime updateable.
*/
public static final String INDEX_FAIL_ON_CORRUPTION_SETTING = "index.fail_on_corruption";
/**
* Index setting to control the initial index buffer size. This setting is <b>not</b> realtime
* updateable.
*/
public static final String INDEX_BUFFER_SIZE_SETTING = "index.buffer_size";
/**
* Index setting to change the low level lucene codec used for writing new segments. This setting
* is realtime updateable.
*/
public static final String INDEX_CODEC_SETTING = "index.codec";
public static final TimeValue DEFAULT_REFRESH_INTERVAL = new TimeValue(1, TimeUnit.SECONDS);
public static final TimeValue DEFAULT_GC_DELETES = TimeValue.timeValueSeconds(60);
public static final ByteSizeValue DEFAUTL_INDEX_BUFFER_SIZE =
new ByteSizeValue(64, ByteSizeUnit.MB);
public static final ByteSizeValue INACTIVE_SHARD_INDEXING_BUFFER =
ByteSizeValue.parseBytesSizeValue("500kb");
private static final String DEFAULT_CODEC_NAME = "default";
/** Creates a new {@link org.elasticsearch.index.engine.EngineConfig} */
public EngineConfig(
ShardId shardId,
boolean optimizeAutoGenerateId,
ThreadPool threadPool,
ShardIndexingService indexingService,
IndexSettingsService indexSettingsService,
IndicesWarmer warmer,
Store store,
SnapshotDeletionPolicy deletionPolicy,
Translog translog,
MergePolicyProvider mergePolicyProvider,
MergeSchedulerProvider mergeScheduler,
Analyzer analyzer,
Similarity similarity,
CodecService codecService,
Engine.FailedEngineListener failedEngineListener) {
this.shardId = shardId;
this.optimizeAutoGenerateId = optimizeAutoGenerateId;
this.threadPool = threadPool;
this.indexingService = indexingService;
this.indexSettingsService = indexSettingsService;
this.warmer = warmer;
this.store = store;
this.deletionPolicy = deletionPolicy;
this.translog = translog;
this.mergePolicyProvider = mergePolicyProvider;
this.mergeScheduler = mergeScheduler;
this.analyzer = analyzer;
this.similarity = similarity;
this.codecService = codecService;
this.failedEngineListener = failedEngineListener;
Settings indexSettings = indexSettingsService.getSettings();
this.compoundOnFlush =
indexSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush);
this.indexConcurrency =
indexSettings.getAsInt(
EngineConfig.INDEX_CONCURRENCY_SETTING,
Math.max(
IndexWriterConfig.DEFAULT_MAX_THREAD_STATES,
(int) (EsExecutors.boundedNumberOfProcessors(indexSettings) * 0.65)));
codecName =
indexSettings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME);
indexingBufferSize =
indexSettings.getAsBytesSize(INDEX_BUFFER_SIZE_SETTING, DEFAUTL_INDEX_BUFFER_SIZE);
failEngineOnCorruption = indexSettings.getAsBoolean(INDEX_FAIL_ON_CORRUPTION_SETTING, true);
failOnMergeFailure = indexSettings.getAsBoolean(INDEX_FAIL_ON_MERGE_FAILURE_SETTING, true);
gcDeletesInMillis =
indexSettings.getAsTime(INDEX_GC_DELETES_SETTING, EngineConfig.DEFAULT_GC_DELETES).millis();
}
/** Sets the indexing buffer */
public void setIndexingBufferSize(ByteSizeValue indexingBufferSize) {
this.indexingBufferSize = indexingBufferSize;
}
/**
* Sets the index concurrency
*
* @see #getIndexConcurrency()
*/
public void setIndexConcurrency(int indexConcurrency) {
this.indexConcurrency = indexConcurrency;
}
/**
* Enables / disables gc deletes
*
* @see #isEnableGcDeletes()
*/
public void setEnableGcDeletes(boolean enableGcDeletes) {
this.enableGcDeletes = enableGcDeletes;
}
/**
* Returns <code>true</code> iff the engine should be failed if a merge error is hit. Defaults to
* <code>true</code>
*/
public boolean isFailOnMergeFailure() {
return failOnMergeFailure;
}
/**
* Returns <code>true</code> if the engine should be failed in the case of a corrupted index.
* Defaults to <code>true</code>
*/
public boolean isFailEngineOnCorruption() {
return failEngineOnCorruption;
}
/**
* Returns the initial index buffer size. This setting is only read on startup and otherwise
* controlled by {@link org.elasticsearch.indices.memory.IndexingMemoryController}
*/
public ByteSizeValue getIndexingBufferSize() {
return indexingBufferSize;
}
/**
* Returns the index concurrency that directly translates into the number of thread states used in
* the engines {@code IndexWriter}.
*
* @see org.apache.lucene.index.IndexWriterConfig#getMaxThreadStates()
*/
public int getIndexConcurrency() {
return indexConcurrency;
}
/**
* Returns <code>true</code> iff flushed segments should be written as compound file system.
* Defaults to <code>true</code>
*/
public boolean isCompoundOnFlush() {
return compoundOnFlush;
}
/** Returns the GC deletes cycle in milliseconds. */
public long getGcDeletesInMillis() {
return gcDeletesInMillis;
}
/**
* Returns <code>true</code> iff delete garbage collection in the engine should be enabled. This
* setting is updateable in realtime and forces a volatile read. Consumers can safely read this
* value directly go fetch it's latest value. The default is <code>true</code>
*
* <p>Engine GC deletion if enabled collects deleted documents from in-memory realtime data
* structures after a certain amount of time ({@link #getGcDeletesInMillis()} if enabled. Before
* deletes are GCed they will cause re-adding the document that was deleted to fail.
*/
public boolean isEnableGcDeletes() {
return enableGcDeletes;
}
/**
* Returns the {@link Codec} used in the engines {@link org.apache.lucene.index.IndexWriter}
*
* <p>Note: this settings is only read on startup and if a new writer is created. This happens
* either due to a settings change in the {@link
* org.elasticsearch.index.engine.EngineConfig.EngineSettingsListener} or if {@link
* Engine#flush(org.elasticsearch.index.engine.Engine.FlushType, boolean, boolean)} with {@link
* org.elasticsearch.index.engine.Engine.FlushType#NEW_WRITER} is executed.
*/
public Codec getCodec() {
return codecService.codec(codecName);
}
/**
* Returns <code>true</code> iff documents with auto-generated IDs are optimized if possible. This
* mainly means that they are simply appended to the index if no update call is necessary.
*/
public boolean isOptimizeAutoGenerateId() {
return optimizeAutoGenerateId;
}
/**
* Returns a thread-pool mainly used to get estimated time stamps from {@link
* org.elasticsearch.threadpool.ThreadPool#estimatedTimeInMillis()} and to schedule async force
* merge calls on the {@link org.elasticsearch.threadpool.ThreadPool.Names#OPTIMIZE} thread-pool
*/
public ThreadPool getThreadPool() {
return threadPool;
}
/**
* Returns a {@link org.elasticsearch.index.indexing.ShardIndexingService} used inside the engine
* to inform about pre and post index and create operations. The operations are used for statistic
* purposes etc.
*
* @see
* org.elasticsearch.index.indexing.ShardIndexingService#postCreate(org.elasticsearch.index.engine.Engine.Create)
* @see
* org.elasticsearch.index.indexing.ShardIndexingService#preCreate(org.elasticsearch.index.engine.Engine.Create)
*/
public ShardIndexingService getIndexingService() {
return indexingService;
}
/**
* Returns an {@link org.elasticsearch.index.settings.IndexSettingsService} used to register a
* {@link org.elasticsearch.index.engine.EngineConfig.EngineSettingsListener} instance in order to
* get notification for realtime changeable settings exposed in this {@link
* org.elasticsearch.index.engine.EngineConfig}.
*/
public IndexSettingsService getIndexSettingsService() {
return indexSettingsService;
}
/**
* Returns an {@link org.elasticsearch.indices.IndicesWarmer} used to warm new searchers before
* they are used for searching. Note: This method might retrun <code>null</code>
*/
@Nullable
public IndicesWarmer getWarmer() {
return warmer;
}
/**
* Returns the {@link org.elasticsearch.index.store.Store} instance that provides access to the
* {@link org.apache.lucene.store.Directory} used for the engines {@link
* org.apache.lucene.index.IndexWriter} to write it's index files to.
*
* <p>Note: In order to use this instance the consumer needs to increment the stores reference
* before it's used the first time and hold it's reference until it's not needed anymore.
*/
public Store getStore() {
return store;
}
/**
* Returns a {@link org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy} used in the
* engines {@link org.apache.lucene.index.IndexWriter}.
*/
public SnapshotDeletionPolicy getDeletionPolicy() {
return deletionPolicy;
}
/** Returns a {@link Translog instance} */
public Translog getTranslog() {
return translog;
}
/**
* Returns the {@link org.elasticsearch.index.merge.policy.MergePolicyProvider} used to obtain a
* {@link org.apache.lucene.index.MergePolicy} for the engines {@link
* org.apache.lucene.index.IndexWriter}
*/
public MergePolicyProvider getMergePolicyProvider() {
return mergePolicyProvider;
}
/**
* Returns the {@link org.elasticsearch.index.merge.scheduler.MergeSchedulerProvider} used to
* obtain a {@link org.apache.lucene.index.MergeScheduler} for the engines {@link
* org.apache.lucene.index.IndexWriter}
*/
public MergeSchedulerProvider getMergeScheduler() {
return mergeScheduler;
}
/** Returns a listener that should be called on engine failure */
public Engine.FailedEngineListener getFailedEngineListener() {
return failedEngineListener;
}
/** Returns the latest index settings directly from the index settings service. */
public Settings getIndexSettings() {
return indexSettingsService.getSettings();
}
/** Returns the engines shard ID */
public ShardId getShardId() {
return shardId;
}
/**
* Returns the analyzer as the default analyzer in the engines {@link
* org.apache.lucene.index.IndexWriter}
*/
public Analyzer getAnalyzer() {
return analyzer;
}
/**
* Returns the {@link org.apache.lucene.search.similarities.Similarity} used for indexing and
* searching.
*/
public Similarity getSimilarity() {
return similarity;
}
/**
* Basic realtime updateable settings listener that can be used ot receive notification if an
* index setting changed.
*/
public abstract static class EngineSettingsListener implements IndexSettingsService.Listener {
private final ESLogger logger;
private final EngineConfig config;
public EngineSettingsListener(ESLogger logger, EngineConfig config) {
this.logger = logger;
this.config = config;
}
@Override
public final void onRefreshSettings(Settings settings) {
boolean change = false;
long gcDeletesInMillis =
settings
.getAsTime(
EngineConfig.INDEX_GC_DELETES_SETTING,
TimeValue.timeValueMillis(config.getGcDeletesInMillis()))
.millis();
if (gcDeletesInMillis != config.getGcDeletesInMillis()) {
logger.info(
"updating {} from [{}] to [{}]",
EngineConfig.INDEX_GC_DELETES_SETTING,
TimeValue.timeValueMillis(config.getGcDeletesInMillis()),
TimeValue.timeValueMillis(gcDeletesInMillis));
config.gcDeletesInMillis = gcDeletesInMillis;
change = true;
}
final boolean compoundOnFlush =
settings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, config.isCompoundOnFlush());
if (compoundOnFlush != config.isCompoundOnFlush()) {
logger.info(
"updating {} from [{}] to [{}]",
EngineConfig.INDEX_COMPOUND_ON_FLUSH,
config.isCompoundOnFlush(),
compoundOnFlush);
config.compoundOnFlush = compoundOnFlush;
change = true;
}
final boolean failEngineOnCorruption =
settings.getAsBoolean(
EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING, config.isFailEngineOnCorruption());
if (failEngineOnCorruption != config.isFailEngineOnCorruption()) {
logger.info(
"updating {} from [{}] to [{}]",
EngineConfig.INDEX_FAIL_ON_CORRUPTION_SETTING,
config.isFailEngineOnCorruption(),
failEngineOnCorruption);
config.failEngineOnCorruption = failEngineOnCorruption;
change = true;
}
int indexConcurrency =
settings.getAsInt(EngineConfig.INDEX_CONCURRENCY_SETTING, config.getIndexConcurrency());
if (indexConcurrency != config.getIndexConcurrency()) {
logger.info(
"updating index.index_concurrency from [{}] to [{}]",
config.getIndexConcurrency(),
indexConcurrency);
config.setIndexConcurrency(indexConcurrency);
// we have to flush in this case, since it only applies on a new index writer
change = true;
}
final String codecName = settings.get(EngineConfig.INDEX_CODEC_SETTING, config.codecName);
if (!codecName.equals(config.codecName)) {
logger.info(
"updating {} from [{}] to [{}]",
EngineConfig.INDEX_CODEC_SETTING,
config.codecName,
codecName);
config.codecName = codecName;
// we want to flush in this case, so the new codec will be reflected right away...
change = true;
}
final boolean failOnMergeFailure =
settings.getAsBoolean(
EngineConfig.INDEX_FAIL_ON_MERGE_FAILURE_SETTING, config.isFailOnMergeFailure());
if (failOnMergeFailure != config.isFailOnMergeFailure()) {
logger.info(
"updating {} from [{}] to [{}]",
EngineConfig.INDEX_FAIL_ON_MERGE_FAILURE_SETTING,
config.isFailOnMergeFailure(),
failOnMergeFailure);
config.failOnMergeFailure = failOnMergeFailure;
change = true;
}
if (change) {
onChange();
}
}
/**
* This method is called if any of the settings that are exposed as realtime updateble settings
* has changed. This method should be overwritten by subclasses to react on settings changes.
*/
protected abstract void onChange();
}
}
@Override
protected void doStart() throws ElasticsearchException {
threadPool.scheduleWithFixedDelay(new Monitor(), TimeValue.timeValueSeconds(3));
}
public static void main(String[] args) {
int exitcode = 0;
try {
OptionParser parser =
new OptionParser() {
{
accepts("elasticsearch").withRequiredArg().ofType(String.class).required();
accepts("index").withRequiredArg().ofType(String.class).required();
accepts("type").withRequiredArg().ofType(String.class).required();
accepts("maxbulkactions").withRequiredArg().ofType(Integer.class).defaultsTo(1000);
accepts("maxconcurrentbulkrequests")
.withRequiredArg()
.ofType(Integer.class)
.defaultsTo(4 * Runtime.getRuntime().availableProcessors());
accepts("mock").withOptionalArg().ofType(Boolean.class).defaultsTo(Boolean.FALSE);
accepts("path").withRequiredArg().ofType(String.class).required();
accepts("pattern")
.withRequiredArg()
.ofType(String.class)
.required()
.defaultsTo("*.txt");
accepts("threads").withRequiredArg().ofType(Integer.class).defaultsTo(1);
accepts("help");
}
};
final OptionSet options = parser.parse(args);
if (options.hasArgument("help")) {
System.err.println(
"Help for "
+ Medline.class.getCanonicalName()
+ lf
+ " --help print this help message"
+ lf
+ " --elasticsearch <uri> Elasticesearch URI"
+ lf
+ " --index <index> Elasticsearch index name"
+ lf
+ " --type <type> Elasticsearch type name"
+ lf
+ " --maxbulkactions <n> the number of bulk actions per request (optional, default: 1000)"
+ " --maxconcurrentbulkrequests <n>the number of concurrent bulk requests (optional, default: 4 * cpu cores)"
+ " --path <path> a file path from where the input files are recursively collected (required)"
+ lf
+ " --pattern <pattern> a regex for selecting matching file names for input (default: *.txt)"
+ lf
+ " --threads <n> the number of threads (optional, default: <num-of=cpus)");
System.exit(1);
}
input =
new Finder((String) options.valueOf("pattern"))
.find((String) options.valueOf("path"))
.getURIs();
final Integer threads = (Integer) options.valueOf("threads");
logger.info("found {} input files", input.size());
URI esURI = URI.create((String) options.valueOf("elasticsearch"));
index = (String) options.valueOf("index");
type = (String) options.valueOf("type");
int maxbulkactions = (Integer) options.valueOf("maxbulkactions");
int maxconcurrentbulkrequests = (Integer) options.valueOf("maxconcurrentbulkrequests");
boolean mock = (Boolean) options.valueOf("mock");
final IngestClient es = mock ? new MockIngestClient() : new IngestClient();
es.maxBulkActions(maxbulkactions)
.maxConcurrentBulkRequests(maxconcurrentbulkrequests)
.newClient(esURI)
.waitForCluster(ClusterHealthStatus.YELLOW, TimeValue.timeValueSeconds(30));
logger.info("creating new index ...");
es.setIndex(index).setType(type).newIndex();
logger.info("... new index created");
final ResourceSink sink = new ResourceSink(es);
ImportService service =
new ImportService()
.threads(threads)
.factory(
new ImporterFactory() {
@Override
public Importer newImporter() {
return new SpringerCitations(sink);
}
})
.execute();
logger.info(
"finished, number of files = {}, resources indexed = {}", fileCounter, sink.getCounter());
service.shutdown();
logger.info("service shutdown");
es.shutdown();
logger.info("elasticsearch client shutdown");
} catch (IOException | InterruptedException | ExecutionException e) {
logger.error(e.getMessage(), e);
exitcode = 1;
}
System.exit(exitcode);
}
@Test
public void testSimplePings() {
ThreadPool threadPool = new ThreadPool();
ClusterName clusterName = new ClusterName("test");
NettyTransport transportA = new NettyTransport(threadPool);
final TransportService transportServiceA = new TransportService(transportA, threadPool).start();
final DiscoveryNode nodeA =
new DiscoveryNode("A", transportServiceA.boundAddress().publishAddress());
InetSocketTransportAddress addressA =
(InetSocketTransportAddress) transportA.boundAddress().publishAddress();
NettyTransport transportB = new NettyTransport(threadPool);
final TransportService transportServiceB = new TransportService(transportB, threadPool).start();
final DiscoveryNode nodeB =
new DiscoveryNode("B", transportServiceA.boundAddress().publishAddress());
InetSocketTransportAddress addressB =
(InetSocketTransportAddress) transportB.boundAddress().publishAddress();
Settings hostsSettings =
ImmutableSettings.settingsBuilder()
.putArray(
"discovery.zen.ping.unicast.hosts",
addressA.address().getAddress().getHostAddress()
+ ":"
+ addressA.address().getPort(),
addressB.address().getAddress().getHostAddress()
+ ":"
+ addressB.address().getPort())
.build();
UnicastZenPing zenPingA =
new UnicastZenPing(hostsSettings, threadPool, transportServiceA, clusterName, null);
zenPingA.setNodesProvider(
new DiscoveryNodesProvider() {
@Override
public DiscoveryNodes nodes() {
return DiscoveryNodes.newNodesBuilder().put(nodeA).localNodeId("A").build();
}
@Override
public NodeService nodeService() {
return null;
}
});
zenPingA.start();
UnicastZenPing zenPingB =
new UnicastZenPing(hostsSettings, threadPool, transportServiceB, clusterName, null);
zenPingB.setNodesProvider(
new DiscoveryNodesProvider() {
@Override
public DiscoveryNodes nodes() {
return DiscoveryNodes.newNodesBuilder().put(nodeB).localNodeId("B").build();
}
@Override
public NodeService nodeService() {
return null;
}
});
zenPingB.start();
try {
ZenPing.PingResponse[] pingResponses = zenPingA.pingAndWait(TimeValue.timeValueSeconds(1));
assertThat(pingResponses.length, equalTo(1));
assertThat(pingResponses[0].target().id(), equalTo("B"));
} finally {
zenPingA.close();
zenPingB.close();
transportServiceA.close();
transportServiceB.close();
threadPool.shutdown();
}
}
public class RecoverySettings extends AbstractComponent {
public static final Setting<ByteSizeValue> INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING =
Setting.byteSizeSetting(
"indices.recovery.max_bytes_per_sec",
new ByteSizeValue(40, ByteSizeUnit.MB),
Property.Dynamic,
Property.NodeScope);
/**
* how long to wait before retrying after issues cause by cluster state syncing between nodes
* i.e., local node is not yet known on remote node, remote shard not yet started etc.
*/
public static final Setting<TimeValue> INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING =
Setting.positiveTimeSetting(
"indices.recovery.retry_delay_state_sync",
TimeValue.timeValueMillis(500),
Property.Dynamic,
Property.NodeScope);
/** how long to wait before retrying after network related issues */
public static final Setting<TimeValue> INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING =
Setting.positiveTimeSetting(
"indices.recovery.retry_delay_network",
TimeValue.timeValueSeconds(5),
Property.Dynamic,
Property.NodeScope);
/** timeout value to use for requests made as part of the recovery process */
public static final Setting<TimeValue> INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING =
Setting.positiveTimeSetting(
"indices.recovery.internal_action_timeout",
TimeValue.timeValueMinutes(15),
Property.Dynamic,
Property.NodeScope);
/**
* timeout value to use for requests made as part of the recovery process that are expected to
* take long time. defaults to twice `indices.recovery.internal_action_timeout`.
*/
public static final Setting<TimeValue> INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING =
Setting.timeSetting(
"indices.recovery.internal_action_long_timeout",
(s) ->
TimeValue.timeValueMillis(
INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.get(s).millis() * 2),
TimeValue.timeValueSeconds(0),
Property.Dynamic,
Property.NodeScope);
/**
* recoveries that don't show any activity for more then this interval will be failed. defaults to
* `indices.recovery.internal_action_long_timeout`
*/
public static final Setting<TimeValue> INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING =
Setting.timeSetting(
"indices.recovery.recovery_activity_timeout",
INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING::get,
TimeValue.timeValueSeconds(0),
Property.Dynamic,
Property.NodeScope);
public static final ByteSizeValue DEFAULT_CHUNK_SIZE = new ByteSizeValue(512, ByteSizeUnit.KB);
private volatile ByteSizeValue maxBytesPerSec;
private volatile SimpleRateLimiter rateLimiter;
private volatile TimeValue retryDelayStateSync;
private volatile TimeValue retryDelayNetwork;
private volatile TimeValue activityTimeout;
private volatile TimeValue internalActionTimeout;
private volatile TimeValue internalActionLongTimeout;
private volatile ByteSizeValue chunkSize = DEFAULT_CHUNK_SIZE;
@Inject
public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
super(settings);
this.retryDelayStateSync = INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING.get(settings);
// doesn't have to be fast as nodes are reconnected every 10s by default (see
// InternalClusterService.ReconnectToNodes)
// and we want to give the master time to remove a faulty node
this.retryDelayNetwork = INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.get(settings);
this.internalActionTimeout = INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.get(settings);
this.internalActionLongTimeout =
INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.get(settings);
this.activityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings);
this.maxBytesPerSec = INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.get(settings);
if (maxBytesPerSec.getBytes() <= 0) {
rateLimiter = null;
} else {
rateLimiter = new SimpleRateLimiter(maxBytesPerSec.getMbFrac());
}
logger.debug("using max_bytes_per_sec[{}]", maxBytesPerSec);
clusterSettings.addSettingsUpdateConsumer(
INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING, this::setMaxBytesPerSec);
clusterSettings.addSettingsUpdateConsumer(
INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING, this::setRetryDelayStateSync);
clusterSettings.addSettingsUpdateConsumer(
INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING, this::setRetryDelayNetwork);
clusterSettings.addSettingsUpdateConsumer(
INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING, this::setInternalActionTimeout);
clusterSettings.addSettingsUpdateConsumer(
INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING, this::setInternalActionLongTimeout);
clusterSettings.addSettingsUpdateConsumer(
INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setActivityTimeout);
}
public RateLimiter rateLimiter() {
return rateLimiter;
}
public TimeValue retryDelayNetwork() {
return retryDelayNetwork;
}
public TimeValue retryDelayStateSync() {
return retryDelayStateSync;
}
public TimeValue activityTimeout() {
return activityTimeout;
}
public TimeValue internalActionTimeout() {
return internalActionTimeout;
}
public TimeValue internalActionLongTimeout() {
return internalActionLongTimeout;
}
public ByteSizeValue getChunkSize() {
return chunkSize;
}
void setChunkSize(ByteSizeValue chunkSize) { // only settable for tests
if (chunkSize.bytesAsInt() <= 0) {
throw new IllegalArgumentException("chunkSize must be > 0");
}
this.chunkSize = chunkSize;
}
public void setRetryDelayStateSync(TimeValue retryDelayStateSync) {
this.retryDelayStateSync = retryDelayStateSync;
}
public void setRetryDelayNetwork(TimeValue retryDelayNetwork) {
this.retryDelayNetwork = retryDelayNetwork;
}
public void setActivityTimeout(TimeValue activityTimeout) {
this.activityTimeout = activityTimeout;
}
public void setInternalActionTimeout(TimeValue internalActionTimeout) {
this.internalActionTimeout = internalActionTimeout;
}
public void setInternalActionLongTimeout(TimeValue internalActionLongTimeout) {
this.internalActionLongTimeout = internalActionLongTimeout;
}
private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) {
this.maxBytesPerSec = maxBytesPerSec;
if (maxBytesPerSec.getBytes() <= 0) {
rateLimiter = null;
} else if (rateLimiter != null) {
rateLimiter.setMBPerSec(maxBytesPerSec.getMbFrac());
} else {
rateLimiter = new SimpleRateLimiter(maxBytesPerSec.getMbFrac());
}
}
}