@SuppressWarnings("unchecked")
void onShardResponse(ShardId shardId, TransportShardMultiPercolateAction.Response response) {
logger.trace("{} Percolate shard response", shardId);
try {
for (TransportShardMultiPercolateAction.Response.Item item : response.items()) {
AtomicReferenceArray shardResults = responsesByItemAndShard.get(item.slot());
if (shardResults == null) {
assert false : "shardResults can't be null";
continue;
}
if (item.failed()) {
shardResults.set(
shardId.id(),
new BroadcastShardOperationFailedException(shardId, item.error().string()));
} else {
shardResults.set(shardId.id(), item.response());
}
assert expectedOperationsPerItem.get(item.slot()).get() >= 1
: "slot[" + item.slot() + "] can't be lower than one";
if (expectedOperationsPerItem.get(item.slot()).decrementAndGet() == 0) {
// Failure won't bubble up, since we fail the whole request now via the catch clause
// below,
// so expectedOperationsPerItem will not be decremented twice.
reduce(item.slot());
}
}
} catch (Throwable e) {
logger.error("{} Percolate original reduce error", e, shardId);
finalListener.onFailure(e);
}
}
/**
* Execute the given {@link IndexRequest} on a primary shard, throwing a {@link
* RetryOnPrimaryException} if the operation needs to be re-tried.
*/
protected final WriteResult<IndexResponse> executeIndexRequestOnPrimary(
BulkShardRequest shardRequest, IndexRequest request, IndexShard indexShard) throws Throwable {
Engine.Index operation = prepareIndexOperationOnPrimary(shardRequest, request, indexShard);
Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
final ShardId shardId = indexShard.shardId();
if (update != null) {
final String indexName = shardId.getIndex();
mappingUpdatedAction.updateMappingOnMasterSynchronously(indexName, request.type(), update);
operation = prepareIndexOperationOnPrimary(shardRequest, request, indexShard);
update = operation.parsedDoc().dynamicMappingsUpdate();
if (update != null) {
throw new RetryOnPrimaryException(
shardId, "Dynamics mappings are not available on the node that holds the primary yet");
}
}
final boolean created = indexShard.index(operation);
// update the version on request so it will happen on the replicas
final long version = operation.version();
request.version(version);
request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
assert request.versionType().validateVersionForWrites(request.version());
return new WriteResult(
new IndexResponse(
shardId.getIndex(), request.type(), request.id(), request.version(), created),
operation.getTranslogLocation());
}
@Override
protected GetFieldMappingsResponse shardOperation(
final GetFieldMappingsIndexRequest request, ShardId shardId) {
assert shardId != null;
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
Collection<String> typeIntersection;
if (request.types().length == 0) {
typeIntersection = indexService.mapperService().types();
} else {
typeIntersection =
indexService
.mapperService()
.types()
.stream()
.filter(type -> Regex.simpleMatch(request.types(), type))
.collect(Collectors.toCollection(ArrayList::new));
if (typeIntersection.isEmpty()) {
throw new TypeMissingException(shardId.getIndex(), request.types());
}
}
MapBuilder<String, Map<String, FieldMappingMetaData>> typeMappings = new MapBuilder<>();
for (String type : typeIntersection) {
DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
Map<String, FieldMappingMetaData> fieldMapping =
findFieldMappingsByType(documentMapper, request);
if (!fieldMapping.isEmpty()) {
typeMappings.put(type, fieldMapping);
}
}
return new GetFieldMappingsResponse(
singletonMap(shardId.getIndexName(), typeMappings.immutableMap()));
}
private void doFinish() {
if (finished.compareAndSet(false, true)) {
Releasables.close(indexShardReference);
final ShardId shardId = shardIt.shardId();
final ActionWriteResponse.ShardInfo.Failure[] failuresArray;
if (!shardReplicaFailures.isEmpty()) {
int slot = 0;
failuresArray = new ActionWriteResponse.ShardInfo.Failure[shardReplicaFailures.size()];
for (Map.Entry<String, Throwable> entry : shardReplicaFailures.entrySet()) {
RestStatus restStatus = ExceptionsHelper.status(entry.getValue());
failuresArray[slot++] =
new ActionWriteResponse.ShardInfo.Failure(
shardId.getIndex(),
shardId.getId(),
entry.getKey(),
entry.getValue(),
restStatus,
false);
}
} else {
failuresArray = ActionWriteResponse.EMPTY;
}
finalResponse.setShardInfo(
new ActionWriteResponse.ShardInfo(totalShards, success.get(), failuresArray));
listener.onResponse(finalResponse);
}
}
@Override
public synchronized IndexShard createShard(int sShardId) throws ElasticSearchException {
ShardId shardId = new ShardId(index, sShardId);
if (shardsInjectors.containsKey(shardId.id())) {
throw new IndexShardAlreadyExistsException(shardId + " already exists");
}
indicesLifecycle.beforeIndexShardCreated(shardId);
logger.debug("creating shard_id [{}]", shardId.id());
ModulesBuilder modules = new ModulesBuilder();
modules.add(new ShardsPluginsModule(indexSettings, pluginsService));
modules.add(new IndexShardModule(shardId));
modules.add(new StoreModule(indexSettings, injector.getInstance(IndexStore.class)));
modules.add(new DeletionPolicyModule(indexSettings));
modules.add(new MergePolicyModule(indexSettings));
modules.add(new MergeSchedulerModule(indexSettings));
modules.add(new TranslogModule(indexSettings));
modules.add(new EngineModule(indexSettings));
modules.add(new IndexShardGatewayModule(injector.getInstance(IndexGateway.class)));
Injector shardInjector = modules.createChildInjector(injector);
shardsInjectors =
newMapBuilder(shardsInjectors).put(shardId.id(), shardInjector).immutableMap();
IndexShard indexShard = shardInjector.getInstance(IndexShard.class);
indicesLifecycle.afterIndexShardCreated(indexShard);
shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
return indexShard;
}
/**
* Just like {@link #getLogger(Class,
* org.elasticsearch.common.settings.Settings,ShardId,String...)} but String loggerName instead of
* Class.
*/
public static ESLogger getLogger(
String loggerName, Settings settings, ShardId shardId, String... prefixes) {
return getLogger(
loggerName,
settings,
asArrayList(shardId.getIndexName(), Integer.toString(shardId.id()), prefixes)
.toArray(new String[0]));
}
public static ESLogger getLogger(
Class clazz, Settings settings, ShardId shardId, String... prefixes) {
return getLogger(
clazz,
settings,
shardId.getIndex(),
asArrayList(Integer.toString(shardId.id()), prefixes).toArray(new String[0]));
}
/** Clears the post allocation flag for the specified shard */
public Builder clearPostAllocationFlag(ShardId shardId) {
assert this.index.equals(shardId.index().name());
IndexShardRoutingTable indexShard = shards.get(shardId.id());
shards.put(
indexShard.shardId().id(),
new IndexShardRoutingTable(indexShard.shardId(), indexShard.shards(), false));
return this;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
if (out.getVersion().before(Version.V_1_4_0)) {
// older nodes expect the concrete index as part of the request
request.index(shardId.getIndex());
}
request.writeTo(out);
if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
shardId.writeTo(out);
} else {
out.writeVInt(shardId.id());
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
shardId = ShardId.readShardId(in);
int size = in.readVInt();
phase1FileNames = new ArrayList<String>(size);
for (int i = 0; i < size; i++) {
phase1FileNames.add(in.readUTF());
}
size = in.readVInt();
phase1FileSizes = new ArrayList<Long>(size);
for (int i = 0; i < size; i++) {
phase1FileSizes.add(in.readVLong());
}
size = in.readVInt();
phase1ExistingFileNames = new ArrayList<String>(size);
for (int i = 0; i < size; i++) {
phase1ExistingFileNames.add(in.readUTF());
}
size = in.readVInt();
phase1ExistingFileSizes = new ArrayList<Long>(size);
for (int i = 0; i < size; i++) {
phase1ExistingFileSizes.add(in.readVLong());
}
phase1TotalSize = in.readVLong();
phase1ExistingTotalSize = in.readVLong();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
shardId.writeTo(out);
out.writeVInt(phase1FileNames.size());
for (String phase1FileName : phase1FileNames) {
out.writeUTF(phase1FileName);
}
out.writeVInt(phase1FileSizes.size());
for (Long phase1FileSize : phase1FileSizes) {
out.writeVLong(phase1FileSize);
}
out.writeVInt(phase1ExistingFileNames.size());
for (String phase1ExistingFileName : phase1ExistingFileNames) {
out.writeUTF(phase1ExistingFileName);
}
out.writeVInt(phase1ExistingFileSizes.size());
for (Long phase1ExistingFileSize : phase1ExistingFileSizes) {
out.writeVLong(phase1ExistingFileSize);
}
out.writeVLong(phase1TotalSize);
out.writeVLong(phase1ExistingTotalSize);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeLong(tookInMillis);
out.writeLong(ingestId);
shardId.writeTo(out);
out.writeVInt(successCount);
out.writeVInt(quorumShards);
out.writeVInt(actionRequests.size());
for (ActionRequest actionRequest : actionRequests) {
if (actionRequest == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
if (actionRequest instanceof IndexRequest) {
out.writeBoolean(true);
} else if (actionRequest instanceof DeleteRequest) {
out.writeBoolean(false);
} else {
throw new ElasticsearchIllegalStateException(
"action request not supported: " + actionRequest.getClass().getName());
}
actionRequest.writeTo(out);
}
}
out.writeVInt(failures.size());
for (IngestActionFailure f : failures) {
f.writeTo(out);
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
tookInMillis = in.readLong();
ingestId = in.readLong();
shardId = ShardId.readShardId(in);
successCount = in.readVInt();
quorumShards = in.readVInt();
actionRequests = newLinkedList();
int size = in.readVInt();
for (int i = 0; i < size; i++) {
boolean exists = in.readBoolean();
if (exists) {
boolean b = in.readBoolean();
if (b) {
IndexRequest indexRequest = new IndexRequest();
indexRequest.readFrom(in);
actionRequests.add(indexRequest);
} else {
DeleteRequest deleteRequest = new DeleteRequest();
deleteRequest.readFrom(in);
actionRequests.add(deleteRequest);
}
} else {
actionRequests.add(null);
}
}
failures = newLinkedList();
size = in.readVInt();
for (int i = 0; i < size; i++) {
failures.add(IngestActionFailure.from(in));
}
}
private void closeShard(
String reason, ShardId sId, IndexShard indexShard, Store store, IndexEventListener listener) {
final int shardId = sId.id();
final Settings indexSettings = this.getIndexSettings().getSettings();
try {
try {
listener.beforeIndexShardClosed(sId, indexShard, indexSettings);
} finally {
// this logic is tricky, we want to close the engine so we rollback the changes done to it
// and close the shard so no operations are allowed to it
if (indexShard != null) {
try {
// only flush we are we closed (closed index or shutdown) and if we are not deleted
final boolean flushEngine = deleted.get() == false && closed.get();
indexShard.close(reason, flushEngine);
} catch (Exception e) {
logger.debug("[{}] failed to close index shard", e, shardId);
// ignore
}
}
// call this before we close the store, so we can release resources for it
listener.afterIndexShardClosed(sId, indexShard, indexSettings);
}
} finally {
try {
store.close();
} catch (Exception e) {
logger.warn(
"[{}] failed to close store on shard removal (reason: [{}])", e, shardId, reason);
}
}
}
@Override
public int hashCode() {
int result = shardId.hashCode();
result = 31 * result + shards.hashCode();
result = 31 * result + (primaryAllocatedPostApi ? 1 : 0);
return result;
}
@Override
public String toString() {
if (shardId != null) {
return shardId.toString();
} else {
return index;
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
shardId.writeTo(out);
out.writeVInt(shards.length);
for (ShardStats stats : shards) {
stats.writeTo(out);
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
shardId = ShardId.readShardId(in);
int shardSize = in.readVInt();
shards = new ShardStats[shardSize];
for (int i = 0; i < shardSize; i++) {
shards[i] = ShardStats.readShardStats(in);
}
}
@Override
public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
if (shardId != null) {
final IndexShard shard = indexService.getShardOrNull(shardId.id());
if (shard != null) {
shard.fieldData().onRemoval(shardId, fieldName, wasEvicted, sizeInBytes);
}
}
}
@Override
public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
if (shardId != null) {
final IndexShard shard = indexService.getShardOrNull(shardId.id());
if (shard != null) {
shard.fieldData().onCache(shardId, fieldName, ramUsage);
}
}
}
private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException {
logger.trace("listing store meta data for {}", shardId);
long startTimeNS = System.nanoTime();
boolean exists = false;
try {
IndexService indexService = indicesService.indexService(shardId.index().name());
if (indexService != null) {
IndexShard indexShard = indexService.shard(shardId.id());
if (indexShard != null) {
final Store store = indexShard.store();
store.incRef();
try {
exists = true;
return new StoreFilesMetaData(true, shardId, store.getMetadataOrEmpty());
} finally {
store.decRef();
}
}
}
// try and see if we an list unallocated
IndexMetaData metaData = clusterService.state().metaData().index(shardId.index().name());
if (metaData == null) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
String storeType = metaData.getSettings().get(IndexStoreModule.STORE_TYPE, "fs");
if (!storeType.contains("fs")) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
final ShardPath shardPath =
ShardPath.loadShardPath(logger, nodeEnv, shardId, metaData.getSettings());
if (shardPath == null) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
return new StoreFilesMetaData(
false, shardId, Store.readMetadataSnapshot(shardPath.resolveIndex(), logger));
} finally {
TimeValue took = new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS);
if (exists) {
logger.debug("{} loaded store meta data (took [{}])", shardId, took);
} else {
logger.trace("{} didn't find any store meta data to load (took [{}])", shardId, took);
}
}
}
public static void main(String[] args) throws Exception {
ShardId shardId = new ShardId(new Index("index"), 1);
Settings settings = EMPTY_SETTINGS;
// Store store = new RamStore(shardId, settings);
Store store = new ByteBufferStore(shardId, settings, null, new ByteBufferCache(settings));
// Store store = new NioFsStore(shardId, settings);
store.deleteContent();
ThreadPool threadPool = new ScalingThreadPool();
SnapshotDeletionPolicy deletionPolicy =
new SnapshotDeletionPolicy(new KeepOnlyLastDeletionPolicy(shardId, settings));
Engine engine =
new RobinEngine(
shardId,
settings,
store,
deletionPolicy,
new MemoryTranslog(shardId, settings),
new LogByteSizeMergePolicyProvider(store),
new ConcurrentMergeSchedulerProvider(shardId, settings),
new AnalysisService(shardId.index()),
new SimilarityService(shardId.index()));
engine.start();
SimpleEngineBenchmark benchmark =
new SimpleEngineBenchmark(store, engine)
.numberOfContentItems(1000)
.searcherThreads(50)
.searcherIterations(10000)
.writerThreads(10)
.writerIterations(10000)
.refreshSchedule(new TimeValue(1, TimeUnit.SECONDS))
.flushSchedule(new TimeValue(1, TimeUnit.MINUTES))
.create(false)
.build();
benchmark.run();
engine.close();
store.close();
threadPool.shutdown();
}
/**
* Checks whether we can perform a write based on the required active shard count setting. Returns
* **null* if OK to proceed, or a string describing the reason to stop
*/
protected String checkActiveShardCount() {
final ShardId shardId = primary.routingEntry().shardId();
final String indexName = shardId.getIndexName();
final ClusterState state = clusterStateSupplier.get();
assert state != null : "replication operation must have access to the cluster state";
final ActiveShardCount waitForActiveShards = request.waitForActiveShards();
if (waitForActiveShards == ActiveShardCount.NONE) {
return null; // not waiting for any shards
}
IndexRoutingTable indexRoutingTable = state.getRoutingTable().index(indexName);
if (indexRoutingTable == null) {
logger.trace("[{}] index not found in the routing table", shardId);
return "Index " + indexName + " not found in the routing table";
}
IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(shardId.getId());
if (shardRoutingTable == null) {
logger.trace("[{}] shard not found in the routing table", shardId);
return "Shard " + shardId + " not found in the routing table";
}
if (waitForActiveShards.enoughShardsActive(shardRoutingTable)) {
return null;
} else {
final String resolvedShards =
waitForActiveShards == ActiveShardCount.ALL
? Integer.toString(shardRoutingTable.shards().size())
: waitForActiveShards.toString();
logger.trace(
"[{}] not enough active copies to meet shard count of [{}] (have {}, needed {}), scheduling a retry. op [{}], "
+ "request [{}]",
shardId,
waitForActiveShards,
shardRoutingTable.activeShards().size(),
resolvedShards,
opType,
request);
return "Not enough active copies to meet shard count of ["
+ waitForActiveShards
+ "] (have "
+ shardRoutingTable.activeShards().size()
+ ", needed "
+ resolvedShards
+ ").";
}
}
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {
if (shardId != null) {
final IndexShard shard = indexService.getShardOrNull(shardId.id());
if (shard != null) {
long ramBytesUsed = accountable != null ? accountable.ramBytesUsed() : 0L;
shard.shardBitsetFilterCache().onRemoval(ramBytesUsed);
}
}
}
@Before
public void setUp() throws Exception {
super.setUp();
defaultSettings =
ImmutableSettings.builder()
.put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, getRandom().nextBoolean())
.build(); // TODO randomize more settings
threadPool = new ThreadPool();
store = createStore();
store.deleteContent();
storeReplica = createStoreReplica();
storeReplica.deleteContent();
engineSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
engine = createEngine(engineSettingsService, store, createTranslog());
engine.start();
replicaSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
replicaEngine = createEngine(replicaSettingsService, storeReplica, createTranslogReplica());
replicaEngine.start();
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
if (in.readBoolean()) {
shardId = ShardId.readShardId(in);
} else {
shardId = null;
}
consistencyLevel = WriteConsistencyLevel.fromId(in.readByte());
timeout = TimeValue.readTimeValue(in);
index = in.readString();
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
request = newRequest();
request.readFrom(in);
if (in.getVersion().onOrAfter(Version.V_1_4_0)) {
shardId = ShardId.readShardId(in);
} else {
// older nodes will send the concrete index as part of the request
shardId = new ShardId(request.index(), in.readVInt());
}
}
protected Engine createEngine(
IndexSettingsService indexSettingsService, Store store, Translog translog) {
return new RobinEngine(
shardId,
defaultSettings,
threadPool,
indexSettingsService,
new ShardIndexingService(
shardId,
EMPTY_SETTINGS,
new ShardSlowLogIndexingService(shardId, EMPTY_SETTINGS, indexSettingsService)),
null,
store,
createSnapshotDeletionPolicy(),
translog,
createMergePolicy(),
createMergeScheduler(),
new AnalysisService(shardId.index()),
new SimilarityService(shardId.index()),
new CodecService(shardId.index()));
}
@Override
protected GetFieldMappingsResponse shardOperation(
final GetFieldMappingsIndexRequest request, ShardId shardId) throws ElasticsearchException {
assert shardId != null;
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
Collection<String> typeIntersection;
if (request.types().length == 0) {
typeIntersection = indexService.mapperService().types();
} else {
typeIntersection =
Collections2.filter(
indexService.mapperService().types(),
new Predicate<String>() {
@Override
public boolean apply(String type) {
return Regex.simpleMatch(request.types(), type);
}
});
if (typeIntersection.isEmpty()) {
throw new TypeMissingException(shardId.index(), request.types());
}
}
MapBuilder<String, ImmutableMap<String, FieldMappingMetaData>> typeMappings =
new MapBuilder<>();
for (String type : typeIntersection) {
DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
ImmutableMap<String, FieldMappingMetaData> fieldMapping =
findFieldMappingsByType(documentMapper, request);
if (!fieldMapping.isEmpty()) {
typeMappings.put(type, fieldMapping);
}
}
return new GetFieldMappingsResponse(
ImmutableMap.of(shardId.getIndex(), typeMappings.immutableMap()));
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
IndexShardRoutingTable that = (IndexShardRoutingTable) o;
if (primaryAllocatedPostApi != that.primaryAllocatedPostApi) return false;
if (!shardId.equals(that.shardId)) return false;
if (!shards.equals(that.shards)) return false;
return true;
}