/** Clears the post allocation flag for the specified shard */
public Builder clearPostAllocationFlag(ShardId shardId) {
assert this.index.equals(shardId.index().name());
IndexShardRoutingTable indexShard = shards.get(shardId.id());
shards.put(
indexShard.shardId().id(),
new IndexShardRoutingTable(indexShard.shardId(), indexShard.shards(), false));
return this;
}
public static void main(String[] args) throws Exception {
ShardId shardId = new ShardId(new Index("index"), 1);
Settings settings = EMPTY_SETTINGS;
// Store store = new RamStore(shardId, settings);
Store store = new ByteBufferStore(shardId, settings, null, new ByteBufferCache(settings));
// Store store = new NioFsStore(shardId, settings);
store.deleteContent();
ThreadPool threadPool = new ScalingThreadPool();
SnapshotDeletionPolicy deletionPolicy =
new SnapshotDeletionPolicy(new KeepOnlyLastDeletionPolicy(shardId, settings));
Engine engine =
new RobinEngine(
shardId,
settings,
store,
deletionPolicy,
new MemoryTranslog(shardId, settings),
new LogByteSizeMergePolicyProvider(store),
new ConcurrentMergeSchedulerProvider(shardId, settings),
new AnalysisService(shardId.index()),
new SimilarityService(shardId.index()));
engine.start();
SimpleEngineBenchmark benchmark =
new SimpleEngineBenchmark(store, engine)
.numberOfContentItems(1000)
.searcherThreads(50)
.searcherIterations(10000)
.writerThreads(10)
.writerIterations(10000)
.refreshSchedule(new TimeValue(1, TimeUnit.SECONDS))
.flushSchedule(new TimeValue(1, TimeUnit.MINUTES))
.create(false)
.build();
benchmark.run();
engine.close();
store.close();
threadPool.shutdown();
}
private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException {
logger.trace("listing store meta data for {}", shardId);
long startTimeNS = System.nanoTime();
boolean exists = false;
try {
IndexService indexService = indicesService.indexService(shardId.index().name());
if (indexService != null) {
IndexShard indexShard = indexService.shard(shardId.id());
if (indexShard != null) {
final Store store = indexShard.store();
store.incRef();
try {
exists = true;
return new StoreFilesMetaData(true, shardId, store.getMetadataOrEmpty());
} finally {
store.decRef();
}
}
}
// try and see if we an list unallocated
IndexMetaData metaData = clusterService.state().metaData().index(shardId.index().name());
if (metaData == null) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
String storeType = metaData.getSettings().get(IndexStoreModule.STORE_TYPE, "fs");
if (!storeType.contains("fs")) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
final ShardPath shardPath =
ShardPath.loadShardPath(logger, nodeEnv, shardId, metaData.getSettings());
if (shardPath == null) {
return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
}
return new StoreFilesMetaData(
false, shardId, Store.readMetadataSnapshot(shardPath.resolveIndex(), logger));
} finally {
TimeValue took = new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS);
if (exists) {
logger.debug("{} loaded store meta data (took [{}])", shardId, took);
} else {
logger.trace("{} didn't find any store meta data to load (took [{}])", shardId, took);
}
}
}
@Before
public void setUp() throws Exception {
super.setUp();
defaultSettings =
ImmutableSettings.builder()
.put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, getRandom().nextBoolean())
.build(); // TODO randomize more settings
threadPool = new ThreadPool();
store = createStore();
store.deleteContent();
storeReplica = createStoreReplica();
storeReplica.deleteContent();
engineSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
engine = createEngine(engineSettingsService, store, createTranslog());
engine.start();
replicaSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
replicaEngine = createEngine(replicaSettingsService, storeReplica, createTranslogReplica());
replicaEngine.start();
}
protected Engine createEngine(
IndexSettingsService indexSettingsService, Store store, Translog translog) {
return new RobinEngine(
shardId,
defaultSettings,
threadPool,
indexSettingsService,
new ShardIndexingService(
shardId,
EMPTY_SETTINGS,
new ShardSlowLogIndexingService(shardId, EMPTY_SETTINGS, indexSettingsService)),
null,
store,
createSnapshotDeletionPolicy(),
translog,
createMergePolicy(),
createMergeScheduler(),
new AnalysisService(shardId.index()),
new SimilarityService(shardId.index()),
new CodecService(shardId.index()));
}
@Override
public void onFailedEngine(
final ShardId shardId, final String reason, final @Nullable Throwable failure) {
ShardRouting shardRouting = null;
final IndexService indexService = indicesService.indexService(shardId.index().name());
if (indexService != null) {
IndexShard indexShard = indexService.shard(shardId.id());
if (indexShard != null) {
shardRouting = indexShard.routingEntry();
}
}
if (shardRouting == null) {
logger.warn(
"[{}][{}] engine failed, but can't find index shard. failure reason: [{}]",
failure,
shardId.index().name(),
shardId.id(),
reason);
return;
}
final ShardRouting fShardRouting = shardRouting;
threadPool
.generic()
.execute(
new Runnable() {
@Override
public void run() {
synchronized (mutex) {
failAndRemoveShard(
fShardRouting,
indexService,
true,
"engine failure, reason [" + reason + "]",
failure);
}
}
});
}
@Override
protected GetFieldMappingsResponse shardOperation(
final GetFieldMappingsIndexRequest request, ShardId shardId) throws ElasticsearchException {
assert shardId != null;
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
Collection<String> typeIntersection;
if (request.types().length == 0) {
typeIntersection = indexService.mapperService().types();
} else {
typeIntersection =
Collections2.filter(
indexService.mapperService().types(),
new Predicate<String>() {
@Override
public boolean apply(String type) {
return Regex.simpleMatch(request.types(), type);
}
});
if (typeIntersection.isEmpty()) {
throw new TypeMissingException(shardId.index(), request.types());
}
}
MapBuilder<String, ImmutableMap<String, FieldMappingMetaData>> typeMappings =
new MapBuilder<>();
for (String type : typeIntersection) {
DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
ImmutableMap<String, FieldMappingMetaData> fieldMapping =
findFieldMappingsByType(documentMapper, request);
if (!fieldMapping.isEmpty()) {
typeMappings.put(type, fieldMapping);
}
}
return new GetFieldMappingsResponse(
ImmutableMap.of(shardId.getIndex(), typeMappings.immutableMap()));
}
/** Resolves the given shards directory against this NodePath */
public Path resolve(ShardId shardId) {
return resolve(shardId.index()).resolve(Integer.toString(shardId.id()));
}
/**
* Resolve the custom path for a index's shard. Uses the {@code IndexMetaData.SETTING_DATA_PATH}
* setting to determine the root path for the index.
*
* @param indexSettings settings for the index
* @param shardId shard to resolve the path to
*/
public Path resolveCustomLocation(@IndexSettings Settings indexSettings, final ShardId shardId) {
return resolveCustomLocation(indexSettings, shardId.index().name())
.resolve(Integer.toString(shardId.id()));
}
@Override
protected void doExecute(
final MultiGetRequest request, final ActionListener<MultiGetResponse> listener) {
ClusterState clusterState = clusterService.state();
Map<ShardId, MultiGetShardRequest> shardRequests = new HashMap<ShardId, MultiGetShardRequest>();
for (int i = 0; i < request.items.size(); i++) {
MultiGetRequest.Item item = request.items.get(i);
ShardId shardId =
clusterService
.operationRouting()
.getShards(clusterState, item.index(), item.type(), item.id(), item.routing(), null)
.shardId();
MultiGetShardRequest shardRequest = shardRequests.get(shardId);
if (shardRequest == null) {
shardRequest = new MultiGetShardRequest(shardId.index().name(), shardId.id());
shardRequest.preference(request.preference);
shardRequest.realtime(request.realtime);
shardRequest.refresh(request.refresh);
shardRequests.put(shardId, shardRequest);
}
shardRequest.add(i, item.type(), item.id(), item.fields());
}
final MultiGetItemResponse[] responses = new MultiGetItemResponse[request.items.size()];
final AtomicInteger counter = new AtomicInteger(shardRequests.size());
for (final MultiGetShardRequest shardRequest : shardRequests.values()) {
shardAction.execute(
shardRequest,
new ActionListener<MultiGetShardResponse>() {
@Override
public void onResponse(MultiGetShardResponse response) {
synchronized (responses) {
for (int i = 0; i < response.locations.size(); i++) {
responses[response.locations.get(i)] =
new MultiGetItemResponse(response.responses.get(i), response.failures.get(i));
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
@Override
public void onFailure(Throwable e) {
// create failures for all relevant requests
String message = ExceptionsHelper.detailedMessage(e);
synchronized (responses) {
for (int i = 0; i < shardRequest.locations.size(); i++) {
responses[shardRequest.locations.get(i)] =
new MultiGetItemResponse(
null,
new MultiGetResponse.Failure(
shardRequest.index(),
shardRequest.types.get(i),
shardRequest.ids.get(i),
message));
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
private void finishHim() {
listener.onResponse(new MultiGetResponse(responses));
}
});
}
}
protected Releasable getIndexShardOperationsCounter(ShardId shardId) {
IndexService indexService = indicesService.indexServiceSafe(shardId.index().getName());
IndexShard indexShard = indexService.getShard(shardId.id());
return new IndexShardReference(indexShard);
}
private void executeBulk(
final BulkRequest bulkRequest,
final long startTime,
final ActionListener<BulkResponse> listener) {
ClusterState clusterState = clusterService.state();
for (ActionRequest request : bulkRequest.requests) {
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
indexRequest.routing(
clusterState
.metaData()
.resolveIndexRouting(indexRequest.routing(), indexRequest.index()));
indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index()));
if (allowIdGeneration) {
if (indexRequest.id() == null) {
indexRequest.id(UUID.randomBase64UUID());
// since we generate the id, change it to CREATE
indexRequest.opType(IndexRequest.OpType.CREATE);
}
}
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index()));
}
}
final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()];
// first, go over all the requests and create a ShardId -> Operations mapping
Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
// handle routing
MappingMetaData mappingMd =
clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type());
if (mappingMd != null) {
try {
indexRequest.processRouting(mappingMd);
} catch (ElasticSearchException e) {
responses[i] =
new BulkItemResponse(
i,
indexRequest.opType().toString().toLowerCase(),
new BulkItemResponse.Failure(
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
e.getDetailedMessage()));
continue;
}
}
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
indexRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
MappingMetaData mappingMd =
clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& deleteRequest.routing() == null) {
// if routing is required, and no routing on the delete request, we need to broadcast
// it....
GroupShardsIterator groupShards =
clusterService
.operationRouting()
.broadcastDeleteShards(clusterState, deleteRequest.index());
for (ShardIterator shardIt : groupShards) {
List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardIt.shardId(), list);
}
list.add(new BulkItemRequest(i, request));
}
} else {
ShardId shardId =
clusterService
.operationRouting()
.deleteShards(
clusterState,
deleteRequest.index(),
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
}
}
if (requestsByShard.isEmpty()) {
listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime));
return;
}
final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
final ShardId shardId = entry.getKey();
final List<BulkItemRequest> requests = entry.getValue();
BulkShardRequest bulkShardRequest =
new BulkShardRequest(
shardId.index().name(),
shardId.id(),
bulkRequest.refresh(),
requests.toArray(new BulkItemRequest[requests.size()]));
bulkShardRequest.replicationType(bulkRequest.replicationType());
bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
shardBulkAction.execute(
bulkShardRequest,
new ActionListener<BulkShardResponse>() {
@Override
public void onResponse(BulkShardResponse bulkShardResponse) {
synchronized (responses) {
for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) {
responses[bulkItemResponse.itemId()] = bulkItemResponse;
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
@Override
public void onFailure(Throwable e) {
// create failures for all relevant requests
String message = ExceptionsHelper.detailedMessage(e);
synchronized (responses) {
for (BulkItemRequest request : requests) {
if (request.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request.request();
responses[request.id()] =
new BulkItemResponse(
request.id(),
indexRequest.opType().toString().toLowerCase(),
new BulkItemResponse.Failure(
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
message));
} else if (request.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request.request();
responses[request.id()] =
new BulkItemResponse(
request.id(),
"delete",
new BulkItemResponse.Failure(
deleteRequest.index(),
deleteRequest.type(),
deleteRequest.id(),
message));
}
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
private void finishHim() {
listener.onResponse(
new BulkResponse(responses, System.currentTimeMillis() - startTime));
}
});
}
}