@SuppressWarnings("unchecked") void onShardResponse(ShardId shardId, TransportShardMultiPercolateAction.Response response) { logger.trace("{} Percolate shard response", shardId); try { for (TransportShardMultiPercolateAction.Response.Item item : response.items()) { AtomicReferenceArray shardResults = responsesByItemAndShard.get(item.slot()); if (shardResults == null) { assert false : "shardResults can't be null"; continue; } if (item.failed()) { shardResults.set( shardId.id(), new BroadcastShardOperationFailedException(shardId, item.error().string())); } else { shardResults.set(shardId.id(), item.response()); } assert expectedOperationsPerItem.get(item.slot()).get() >= 1 : "slot[" + item.slot() + "] can't be lower than one"; if (expectedOperationsPerItem.get(item.slot()).decrementAndGet() == 0) { // Failure won't bubble up, since we fail the whole request now via the catch clause // below, // so expectedOperationsPerItem will not be decremented twice. reduce(item.slot()); } } } catch (Throwable e) { logger.error("{} Percolate original reduce error", e, shardId); finalListener.onFailure(e); } }
@Override public synchronized IndexShard createShard(int sShardId) throws ElasticSearchException { ShardId shardId = new ShardId(index, sShardId); if (shardsInjectors.containsKey(shardId.id())) { throw new IndexShardAlreadyExistsException(shardId + " already exists"); } indicesLifecycle.beforeIndexShardCreated(shardId); logger.debug("creating shard_id [{}]", shardId.id()); ModulesBuilder modules = new ModulesBuilder(); modules.add(new ShardsPluginsModule(indexSettings, pluginsService)); modules.add(new IndexShardModule(shardId)); modules.add(new StoreModule(indexSettings, injector.getInstance(IndexStore.class))); modules.add(new DeletionPolicyModule(indexSettings)); modules.add(new MergePolicyModule(indexSettings)); modules.add(new MergeSchedulerModule(indexSettings)); modules.add(new TranslogModule(indexSettings)); modules.add(new EngineModule(indexSettings)); modules.add(new IndexShardGatewayModule(injector.getInstance(IndexGateway.class))); Injector shardInjector = modules.createChildInjector(injector); shardsInjectors = newMapBuilder(shardsInjectors).put(shardId.id(), shardInjector).immutableMap(); IndexShard indexShard = shardInjector.getInstance(IndexShard.class); indicesLifecycle.afterIndexShardCreated(indexShard); shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap(); return indexShard; }
private void closeShard( String reason, ShardId sId, IndexShard indexShard, Store store, IndexEventListener listener) { final int shardId = sId.id(); final Settings indexSettings = this.getIndexSettings().getSettings(); try { try { listener.beforeIndexShardClosed(sId, indexShard, indexSettings); } finally { // this logic is tricky, we want to close the engine so we rollback the changes done to it // and close the shard so no operations are allowed to it if (indexShard != null) { try { // only flush we are we closed (closed index or shutdown) and if we are not deleted final boolean flushEngine = deleted.get() == false && closed.get(); indexShard.close(reason, flushEngine); } catch (Exception e) { logger.debug("[{}] failed to close index shard", e, shardId); // ignore } } // call this before we close the store, so we can release resources for it listener.afterIndexShardClosed(sId, indexShard, indexSettings); } } finally { try { store.close(); } catch (Exception e) { logger.warn( "[{}] failed to close store on shard removal (reason: [{}])", e, shardId, reason); } } }
public static ESLogger getLogger( Class clazz, Settings settings, ShardId shardId, String... prefixes) { return getLogger( clazz, settings, shardId.getIndex(), asArrayList(Integer.toString(shardId.id()), prefixes).toArray(new String[0])); }
/** * Just like {@link #getLogger(Class, * org.elasticsearch.common.settings.Settings,ShardId,String...)} but String loggerName instead of * Class. */ public static ESLogger getLogger( String loggerName, Settings settings, ShardId shardId, String... prefixes) { return getLogger( loggerName, settings, asArrayList(shardId.getIndexName(), Integer.toString(shardId.id()), prefixes) .toArray(new String[0])); }
/** Clears the post allocation flag for the specified shard */ public Builder clearPostAllocationFlag(ShardId shardId) { assert this.index.equals(shardId.index().name()); IndexShardRoutingTable indexShard = shards.get(shardId.id()); shards.put( indexShard.shardId().id(), new IndexShardRoutingTable(indexShard.shardId(), indexShard.shards(), false)); return this; }
@Override public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) { if (shardId != null) { final IndexShard shard = indexService.getShardOrNull(shardId.id()); if (shard != null) { shard.fieldData().onRemoval(shardId, fieldName, wasEvicted, sizeInBytes); } } }
@Override public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { if (shardId != null) { final IndexShard shard = indexService.getShardOrNull(shardId.id()); if (shard != null) { shard.fieldData().onCache(shardId, fieldName, ramUsage); } } }
@Override public void onRemoval(ShardId shardId, Accountable accountable) { if (shardId != null) { final IndexShard shard = indexService.getShardOrNull(shardId.id()); if (shard != null) { long ramBytesUsed = accountable != null ? accountable.ramBytesUsed() : 0L; shard.shardBitsetFilterCache().onRemoval(ramBytesUsed); } } }
@Override public void onFailedEngine( final ShardId shardId, final String reason, final @Nullable Throwable failure) { ShardRouting shardRouting = null; final IndexService indexService = indicesService.indexService(shardId.index().name()); if (indexService != null) { IndexShard indexShard = indexService.shard(shardId.id()); if (indexShard != null) { shardRouting = indexShard.routingEntry(); } } if (shardRouting == null) { logger.warn( "[{}][{}] engine failed, but can't find index shard. failure reason: [{}]", failure, shardId.index().name(), shardId.id(), reason); return; } final ShardRouting fShardRouting = shardRouting; threadPool .generic() .execute( new Runnable() { @Override public void run() { synchronized (mutex) { failAndRemoveShard( fShardRouting, indexService, true, "engine failure, reason [" + reason + "]", failure); } } }); }
@Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); if (out.getVersion().before(Version.V_1_4_0)) { // older nodes expect the concrete index as part of the request request.index(shardId.getIndex()); } request.writeTo(out); if (out.getVersion().onOrAfter(Version.V_1_4_0)) { shardId.writeTo(out); } else { out.writeVInt(shardId.id()); } }
private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException { logger.trace("listing store meta data for {}", shardId); long startTimeNS = System.nanoTime(); boolean exists = false; try { IndexService indexService = indicesService.indexService(shardId.index().name()); if (indexService != null) { IndexShard indexShard = indexService.shard(shardId.id()); if (indexShard != null) { final Store store = indexShard.store(); store.incRef(); try { exists = true; return new StoreFilesMetaData(true, shardId, store.getMetadataOrEmpty()); } finally { store.decRef(); } } } // try and see if we an list unallocated IndexMetaData metaData = clusterService.state().metaData().index(shardId.index().name()); if (metaData == null) { return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY); } String storeType = metaData.getSettings().get(IndexStoreModule.STORE_TYPE, "fs"); if (!storeType.contains("fs")) { return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY); } final ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnv, shardId, metaData.getSettings()); if (shardPath == null) { return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY); } return new StoreFilesMetaData( false, shardId, Store.readMetadataSnapshot(shardPath.resolveIndex(), logger)); } finally { TimeValue took = new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS); if (exists) { logger.debug("{} loaded store meta data (took [{}])", shardId, took); } else { logger.trace("{} didn't find any store meta data to load (took [{}])", shardId, took); } } }
@Override protected void shardOperationOnReplica(ShardId shardId, IndexRequest request) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.shardSafe(shardId.id()); SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, request.source()) .index(shardId.getIndex()) .type(request.type()) .id(request.id()) .routing(request.routing()) .parent(request.parent()) .timestamp(request.timestamp()) .ttl(request.ttl()); final Engine.IndexingOperation operation; if (request.opType() == IndexRequest.OpType.INDEX) { operation = indexShard.prepareIndex( sourceToParse, request.version(), request.versionType(), Engine.Operation.Origin.REPLICA); } else { assert request.opType() == IndexRequest.OpType.CREATE : request.opType(); operation = indexShard.prepareCreate( sourceToParse, request.version(), request.versionType(), Engine.Operation.Origin.REPLICA); } Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { throw new RetryOnReplicaException( shardId, "Mappings are not available on the replica yet, triggered update: " + update); } operation.execute(indexShard); processAfter(request.refresh(), indexShard, operation.getTranslogLocation()); }
protected BlobStoreIndexShardGateway( ShardId shardId, @IndexSettings Settings indexSettings, ThreadPool threadPool, IndexGateway indexGateway, IndexShard indexShard, Store store) { super(shardId, indexSettings); this.threadPool = threadPool; this.indexShard = (InternalIndexShard) indexShard; this.store = store; BlobStoreIndexGateway blobStoreIndexGateway = (BlobStoreIndexGateway) indexGateway; this.chunkSize = blobStoreIndexGateway.chunkSize(); // can be null -> no chunking this.blobStore = blobStoreIndexGateway.blobStore(); this.shardPath = blobStoreIndexGateway.shardPath(shardId.id()); this.blobContainer = blobStore.immutableBlobContainer(shardPath); this.recoveryStatus = new RecoveryStatus(); }
@SuppressWarnings("unchecked") void onShardFailure(ShardId shardId, Throwable e) { logger.debug("{} Shard multi percolate failure", e, shardId); try { IntArrayList slots = shardToSlots.get(shardId); for (int i = 0; i < slots.size(); i++) { int slot = slots.get(i); AtomicReferenceArray shardResults = responsesByItemAndShard.get(slot); if (shardResults == null) { continue; } shardResults.set(shardId.id(), new BroadcastShardOperationFailedException(shardId, e)); assert expectedOperationsPerItem.get(slot).get() >= 1 : "slot[" + slot + "] can't be lower than one. Caused by: " + e.getMessage(); if (expectedOperationsPerItem.get(slot).decrementAndGet() == 0) { reduce(slot); } } } catch (Throwable t) { logger.error("{} Percolate original reduce error, original error {}", t, shardId, e); finalListener.onFailure(t); } }
protected Releasable getIndexShardOperationsCounter(ShardId shardId) { IndexService indexService = indicesService.indexServiceSafe(shardId.index().getName()); IndexShard indexShard = indexService.getShard(shardId.id()); return new IndexShardReference(indexShard); }
public void setShard(ShardId shardId) { if (shardId != null) { setIndex(shardId.getIndex()); addHeader(SHARD_HEADER_KEY, Integer.toString(shardId.id())); } }
public synchronized IndexShard createShard(ShardRouting routing) throws IOException { final boolean primary = routing.primary(); /* * TODO: we execute this in parallel but it's a synced method. Yet, we might * be able to serialize the execution via the cluster state in the future. for now we just * keep it synced. */ if (closed.get()) { throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed"); } final Settings indexSettings = this.indexSettings.getSettings(); final ShardId shardId = routing.shardId(); boolean success = false; Store store = null; IndexShard indexShard = null; ShardLock lock = null; try { lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5)); eventListener.beforeIndexShardCreated(shardId, indexSettings); ShardPath path; try { path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings); } catch (IllegalStateException ex) { logger.warn("{} failed to load shard path, trying to remove leftover", shardId); try { ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings); path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings); } catch (Exception inner) { ex.addSuppressed(inner); throw ex; } } if (path == null) { // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard // will be, e.g. for a shard // that's being relocated/replicated we know how large it will become once it's done // copying: // Count up how many shards are currently on each data path: Map<Path, Integer> dataPathToShardCount = new HashMap<>(); for (IndexShard shard : this) { Path dataPath = shard.shardPath().getRootStatePath(); Integer curCount = dataPathToShardCount.get(dataPath); if (curCount == null) { curCount = 0; } dataPathToShardCount.put(dataPath, curCount + 1); } path = ShardPath.selectNewPathForShard( nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount); logger.debug("{} creating using a new path [{}]", shardId, path); } else { logger.debug("{} creating using an existing path [{}]", shardId, path); } if (shards.containsKey(shardId.id())) { throw new IndexShardAlreadyExistsException(shardId + " already exists"); } logger.debug("creating shard_id {}", shardId); // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the // primary. final boolean canDeleteShardContent = IndexMetaData.isOnSharedFilesystem(indexSettings) == false || (primary && IndexMetaData.isOnSharedFilesystem(indexSettings)); final Engine.Warmer engineWarmer = (searcher) -> { IndexShard shard = getShardOrNull(shardId.getId()); if (shard != null) { warmer.warm(searcher, shard, IndexService.this.indexSettings); } }; store = new Store( shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener( shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId))); if (useShadowEngine(primary, indexSettings)) { indexShard = new ShadowIndexShard( routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners); // no indexing listeners - shadow engines don't index } else { indexShard = new IndexShard( routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners, indexingOperationListeners); } eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); eventListener.afterIndexShardCreated(indexShard); shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap(); success = true; return indexShard; } catch (ShardLockObtainFailedException e) { throw new IOException("failed to obtain in-memory shard lock", e); } finally { if (success == false) { if (lock != null) { IOUtils.closeWhileHandlingException(lock); } closeShard("initialization failed", shardId, indexShard, store, eventListener); } } }
/** * Creates cluster state with and index that has one shard and #(replicaStates) replicas * * @param index name of the index * @param activePrimaryLocal if active primary should coincide with the local node in the cluster * state * @param primaryState state of primary * @param replicaStates states of the replicas. length of this array determines also the number of * replicas */ public static ClusterState state( String index, boolean activePrimaryLocal, ShardRoutingState primaryState, ShardRoutingState... replicaStates) { final int numberOfReplicas = replicaStates.length; int numberOfNodes = numberOfReplicas + 1; if (primaryState == ShardRoutingState.RELOCATING) { numberOfNodes++; } for (ShardRoutingState state : replicaStates) { if (state == ShardRoutingState.RELOCATING) { numberOfNodes++; } } numberOfNodes = Math.max(2, numberOfNodes); // we need a non-local master to test shard failures final ShardId shardId = new ShardId(index, "_na_", 0); DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); Set<String> unassignedNodes = new HashSet<>(); for (int i = 0; i < numberOfNodes + 1; i++) { final DiscoveryNode node = newNode(i); discoBuilder = discoBuilder.put(node); unassignedNodes.add(node.getId()); } discoBuilder.localNodeId(newNode(0).getId()); discoBuilder.masterNodeId( newNode(1).getId()); // we need a non-local master to test shard failures final int primaryTerm = 1 + randomInt(200); IndexMetaData indexMetaData = IndexMetaData.builder(index) .settings( Settings.builder() .put(SETTING_VERSION_CREATED, Version.CURRENT) .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) .put(SETTING_CREATION_DATE, System.currentTimeMillis())) .primaryTerm(0, primaryTerm) .build(); RoutingTable.Builder routing = new RoutingTable.Builder(); routing.addAsNew(indexMetaData); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); String primaryNode = null; String relocatingNode = null; UnassignedInfo unassignedInfo = null; if (primaryState != ShardRoutingState.UNASSIGNED) { if (activePrimaryLocal) { primaryNode = newNode(0).getId(); unassignedNodes.remove(primaryNode); } else { Set<String> unassignedNodesExecludingPrimary = new HashSet<>(unassignedNodes); unassignedNodesExecludingPrimary.remove(newNode(0).getId()); primaryNode = selectAndRemove(unassignedNodesExecludingPrimary); } if (primaryState == ShardRoutingState.RELOCATING) { relocatingNode = selectAndRemove(unassignedNodes); } } else { unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null); } indexShardRoutingBuilder.addShard( TestShardRouting.newShardRouting( index, 0, primaryNode, relocatingNode, null, true, primaryState, unassignedInfo)); for (ShardRoutingState replicaState : replicaStates) { String replicaNode = null; relocatingNode = null; unassignedInfo = null; if (replicaState != ShardRoutingState.UNASSIGNED) { assert primaryNode != null : "a replica is assigned but the primary isn't"; replicaNode = selectAndRemove(unassignedNodes); if (replicaState == ShardRoutingState.RELOCATING) { relocatingNode = selectAndRemove(unassignedNodes); } } else { unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null); } indexShardRoutingBuilder.addShard( TestShardRouting.newShardRouting( index, shardId.id(), replicaNode, relocatingNode, null, false, replicaState, unassignedInfo)); } ClusterState.Builder state = ClusterState.builder(new ClusterName("test")); state.nodes(discoBuilder); state.metaData(MetaData.builder().put(indexMetaData, false).generateClusterUuidIfNeeded()); state.routingTable( RoutingTable.builder() .add( IndexRoutingTable.builder(indexMetaData.getIndex()) .addIndexShard(indexShardRoutingBuilder.build())) .build()); return state.build(); }
@Override public Integer value() { return shardId.id(); }
@Override protected void doExecute( final MultiGetRequest request, final ActionListener<MultiGetResponse> listener) { ClusterState clusterState = clusterService.state(); Map<ShardId, MultiGetShardRequest> shardRequests = new HashMap<ShardId, MultiGetShardRequest>(); for (int i = 0; i < request.items.size(); i++) { MultiGetRequest.Item item = request.items.get(i); ShardId shardId = clusterService .operationRouting() .getShards(clusterState, item.index(), item.type(), item.id(), item.routing(), null) .shardId(); MultiGetShardRequest shardRequest = shardRequests.get(shardId); if (shardRequest == null) { shardRequest = new MultiGetShardRequest(shardId.index().name(), shardId.id()); shardRequest.preference(request.preference); shardRequest.realtime(request.realtime); shardRequest.refresh(request.refresh); shardRequests.put(shardId, shardRequest); } shardRequest.add(i, item.type(), item.id(), item.fields()); } final MultiGetItemResponse[] responses = new MultiGetItemResponse[request.items.size()]; final AtomicInteger counter = new AtomicInteger(shardRequests.size()); for (final MultiGetShardRequest shardRequest : shardRequests.values()) { shardAction.execute( shardRequest, new ActionListener<MultiGetShardResponse>() { @Override public void onResponse(MultiGetShardResponse response) { synchronized (responses) { for (int i = 0; i < response.locations.size(); i++) { responses[response.locations.get(i)] = new MultiGetItemResponse(response.responses.get(i), response.failures.get(i)); } } if (counter.decrementAndGet() == 0) { finishHim(); } } @Override public void onFailure(Throwable e) { // create failures for all relevant requests String message = ExceptionsHelper.detailedMessage(e); synchronized (responses) { for (int i = 0; i < shardRequest.locations.size(); i++) { responses[shardRequest.locations.get(i)] = new MultiGetItemResponse( null, new MultiGetResponse.Failure( shardRequest.index(), shardRequest.types.get(i), shardRequest.ids.get(i), message)); } } if (counter.decrementAndGet() == 0) { finishHim(); } } private void finishHim() { listener.onResponse(new MultiGetResponse(responses)); } }); } }
/** * Prepares an update request by converting it into an index request. * * <p>TODO: detect a NOOP and return an update response if true */ @SuppressWarnings("unchecked") public IndexRequest prepareUpdate( DocTableInfo tableInfo, ShardUpsertRequest request, ShardUpsertRequest.Item item, ShardId shardId) throws ElasticsearchException { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.shardSafe(shardId.id()); final GetResult getResult = indexShard .getService() .get( request.type(), item.id(), new String[] {RoutingFieldMapper.NAME, ParentFieldMapper.NAME, TTLFieldMapper.NAME}, true, item.version(), VersionType.INTERNAL, FetchSourceContext.FETCH_SOURCE, false); if (!getResult.isExists()) { throw new DocumentMissingException( new ShardId(request.index(), request.shardId()), request.type(), item.id()); } if (getResult.internalSourceRef() == null) { // no source, we can't do nothing, through a failure... throw new DocumentSourceMissingException( new ShardId(request.index(), request.shardId()), request.type(), item.id()); } Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); final Map<String, Object> updatedSourceAsMap; final XContentType updateSourceContentType = sourceAndContent.v1(); String routing = getResult.getFields().containsKey(RoutingFieldMapper.NAME) ? getResult.field(RoutingFieldMapper.NAME).getValue().toString() : null; String parent = getResult.getFields().containsKey(ParentFieldMapper.NAME) ? getResult.field(ParentFieldMapper.NAME).getValue().toString() : null; updatedSourceAsMap = sourceAndContent.v2(); SymbolToFieldExtractorContext ctx = new SymbolToFieldExtractorContext(functions, item.insertValues()); Map<String, Object> pathsToUpdate = new LinkedHashMap<>(); Map<String, Object> updatedGeneratedColumns = new LinkedHashMap<>(); for (int i = 0; i < request.updateColumns().length; i++) { /** * NOTE: mapping isn't applied. So if an Insert was done using the ES Rest Endpoint the data * might be returned in the wrong format (date as string instead of long) */ String columnPath = request.updateColumns()[i]; Object value = SYMBOL_TO_FIELD_EXTRACTOR.convert(item.updateAssignments()[i], ctx).extract(getResult); ReferenceInfo referenceInfo = tableInfo.getReferenceInfo(ColumnIdent.fromPath(columnPath)); if (referenceInfo instanceof GeneratedReferenceInfo) { updatedGeneratedColumns.put(columnPath, value); } else { pathsToUpdate.put(columnPath, value); } } processGeneratedColumns( tableInfo, pathsToUpdate, updatedGeneratedColumns, request.validateGeneratedColumns(), getResult); updateSourceByPaths(updatedSourceAsMap, pathsToUpdate); final IndexRequest indexRequest = Requests.indexRequest(request.index()) .type(request.type()) .id(item.id()) .routing(routing) .parent(parent) .source(updatedSourceAsMap, updateSourceContentType) .version(getResult.getVersion()); indexRequest.operationThreaded(false); return indexRequest; }
@Override protected void shardOperationOnReplica(ShardId shardId, BulkShardRequest request) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.shardSafe(shardId.id()); Translog.Location location = null; for (int i = 0; i < request.items().length; i++) { BulkItemRequest item = request.items()[i]; if (item == null || item.isIgnoreOnReplica()) { continue; } if (item.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) item.request(); try { SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source()) .index(shardId.getIndex()) .type(indexRequest.type()) .id(indexRequest.id()) .routing(indexRequest.routing()) .parent(indexRequest.parent()) .timestamp(indexRequest.timestamp()) .ttl(indexRequest.ttl()); final Engine.IndexingOperation operation; if (indexRequest.opType() == IndexRequest.OpType.INDEX) { operation = indexShard.prepareIndex( sourceToParse, indexRequest.version(), indexRequest.versionType(), Engine.Operation.Origin.REPLICA, request.canHaveDuplicates() || indexRequest.canHaveDuplicates()); } else { assert indexRequest.opType() == IndexRequest.OpType.CREATE : indexRequest.opType(); operation = indexShard.prepareCreate( sourceToParse, indexRequest.version(), indexRequest.versionType(), Engine.Operation.Origin.REPLICA, request.canHaveDuplicates() || indexRequest.canHaveDuplicates(), indexRequest.autoGeneratedId()); } Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { throw new RetryOnReplicaException( shardId, "Mappings are not available on the replica yet, triggered update: " + update); } operation.execute(indexShard); location = locationToSync(location, operation.getTranslogLocation()); } catch (Throwable e) { // if its not an ignore replica failure, we need to make sure to bubble up the failure // so we will fail the shard if (!ignoreReplicaException(e)) { throw e; } } } else if (item.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) item.request(); try { Engine.Delete delete = indexShard.prepareDelete( deleteRequest.type(), deleteRequest.id(), deleteRequest.version(), deleteRequest.versionType(), Engine.Operation.Origin.REPLICA); indexShard.delete(delete); location = locationToSync(location, delete.getTranslogLocation()); } catch (Throwable e) { // if its not an ignore replica failure, we need to make sure to bubble up the failure // so we will fail the shard if (!ignoreReplicaException(e)) { throw e; } } } else { throw new IllegalStateException("Unexpected index operation: " + item.request()); } } processAfter(request.refresh(), indexShard, location); }
/** Resolves the given shards directory against this NodePath */ public Path resolve(ShardId shardId) { return resolve(shardId.index()).resolve(Integer.toString(shardId.id())); }
/** * Resolve the custom path for a index's shard. Uses the {@code IndexMetaData.SETTING_DATA_PATH} * setting to determine the root path for the index. * * @param indexSettings settings for the index * @param shardId shard to resolve the path to */ public Path resolveCustomLocation(@IndexSettings Settings indexSettings, final ShardId shardId) { return resolveCustomLocation(indexSettings, shardId.index().name()) .resolve(Integer.toString(shardId.id())); }
private void executeBulk( final BulkRequest bulkRequest, final long startTime, final ActionListener<BulkResponse> listener) { ClusterState clusterState = clusterService.state(); for (ActionRequest request : bulkRequest.requests) { if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; indexRequest.routing( clusterState .metaData() .resolveIndexRouting(indexRequest.routing(), indexRequest.index())); indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index())); if (allowIdGeneration) { if (indexRequest.id() == null) { indexRequest.id(UUID.randomBase64UUID()); // since we generate the id, change it to CREATE indexRequest.opType(IndexRequest.OpType.CREATE); } } } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index())); } } final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()]; // first, go over all the requests and create a ShardId -> Operations mapping Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap(); for (int i = 0; i < bulkRequest.requests.size(); i++) { ActionRequest request = bulkRequest.requests.get(i); if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; // handle routing MappingMetaData mappingMd = clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type()); if (mappingMd != null) { try { indexRequest.processRouting(mappingMd); } catch (ElasticSearchException e) { responses[i] = new BulkItemResponse( i, indexRequest.opType().toString().toLowerCase(), new BulkItemResponse.Failure( indexRequest.index(), indexRequest.type(), indexRequest.id(), e.getDetailedMessage())); continue; } } ShardId shardId = clusterService .operationRouting() .indexShards( clusterState, indexRequest.index(), indexRequest.type(), indexRequest.id(), indexRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; MappingMetaData mappingMd = clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type()); if (mappingMd != null && mappingMd.routing().required() && deleteRequest.routing() == null) { // if routing is required, and no routing on the delete request, we need to broadcast // it.... GroupShardsIterator groupShards = clusterService .operationRouting() .broadcastDeleteShards(clusterState, deleteRequest.index()); for (ShardIterator shardIt : groupShards) { List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId()); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardIt.shardId(), list); } list.add(new BulkItemRequest(i, request)); } } else { ShardId shardId = clusterService .operationRouting() .deleteShards( clusterState, deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), deleteRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } } } if (requestsByShard.isEmpty()) { listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime)); return; } final AtomicInteger counter = new AtomicInteger(requestsByShard.size()); for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) { final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest( shardId.index().name(), shardId.id(), bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.replicationType(bulkRequest.replicationType()); bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel()); shardBulkAction.execute( bulkShardRequest, new ActionListener<BulkShardResponse>() { @Override public void onResponse(BulkShardResponse bulkShardResponse) { synchronized (responses) { for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) { responses[bulkItemResponse.itemId()] = bulkItemResponse; } } if (counter.decrementAndGet() == 0) { finishHim(); } } @Override public void onFailure(Throwable e) { // create failures for all relevant requests String message = ExceptionsHelper.detailedMessage(e); synchronized (responses) { for (BulkItemRequest request : requests) { if (request.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request.request(); responses[request.id()] = new BulkItemResponse( request.id(), indexRequest.opType().toString().toLowerCase(), new BulkItemResponse.Failure( indexRequest.index(), indexRequest.type(), indexRequest.id(), message)); } else if (request.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request.request(); responses[request.id()] = new BulkItemResponse( request.id(), "delete", new BulkItemResponse.Failure( deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), message)); } } } if (counter.decrementAndGet() == 0) { finishHim(); } } private void finishHim() { listener.onResponse( new BulkResponse(responses, System.currentTimeMillis() - startTime)); } }); } }
@Override protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) throws ElasticsearchException { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.shardSafe(shardId.id()); Term uidTerm = new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(request.type(), request.id())); Engine.GetResult result = indexShard.get(new Engine.Get(false, uidTerm)); if (!result.exists()) { return new ExplainResponse(shardId.getIndex(), request.type(), request.id(), false); } SearchContext context = new DefaultSearchContext( 0, new ShardSearchRequest(request) .types(new String[] {request.type()}) .filteringAliases(request.filteringAlias()) .nowInMillis(request.nowInMillis), null, result.searcher(), indexService, indexShard, scriptService, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter()); SearchContext.setCurrent(context); try { context.parsedQuery(indexService.queryParserService().parseQuery(request.source())); context.preProcess(); int topLevelDocId = result.docIdAndVersion().docId + result.docIdAndVersion().context.docBase; Explanation explanation = context.searcher().explain(context.query(), topLevelDocId); for (RescoreSearchContext ctx : context.rescore()) { Rescorer rescorer = ctx.rescorer(); explanation = rescorer.explain(topLevelDocId, context, ctx, explanation); } if (request.fields() != null || (request.fetchSourceContext() != null && request.fetchSourceContext().fetchSource())) { // Advantage is that we're not opening a second searcher to retrieve the _source. Also // because we are working in the same searcher in engineGetResult we can be sure that a // doc isn't deleted between the initial get and this call. GetResult getResult = indexShard .getService() .get( result, request.id(), request.type(), request.fields(), request.fetchSourceContext(), false); return new ExplainResponse( shardId.getIndex(), request.type(), request.id(), true, explanation, getResult); } else { return new ExplainResponse( shardId.getIndex(), request.type(), request.id(), true, explanation); } } catch (IOException e) { throw new ElasticsearchException("Could not explain", e); } finally { context.close(); SearchContext.removeCurrent(); } }