/** * Send the given snapshot's operations to this handler's target node. * * <p>Operations are bulked into a single request depending on an operation count limit or * size-in-bytes limit * * @return the total number of translog operations that were sent */ protected int sendSnapshot(Translog.Snapshot snapshot) throws ElasticsearchException { int ops = 0; long size = 0; int totalOperations = 0; final List<Translog.Operation> operations = Lists.newArrayList(); Translog.Operation operation = snapshot.next(); final TransportRequestOptions recoveryOptions = TransportRequestOptions.options() .withCompress(recoverySettings.compress()) .withType(TransportRequestOptions.Type.RECOVERY) .withTimeout(recoverySettings.internalActionLongTimeout()); if (operation == null) { logger.trace( "[{}][{}] no translog operations (id: [{}]) to send to {}", indexName, shardId, snapshot.translogId(), request.targetNode()); } while (operation != null) { if (shard.state() == IndexShardState.CLOSED) { throw new IndexShardClosedException(request.shardId()); } cancellableThreads.checkForCancel(); operations.add(operation); ops += 1; size += operation.estimateSize(); totalOperations++; // Check if this request is past the size or bytes threshold, and // if so, send it off if (ops >= recoverySettings.translogOps() || size >= recoverySettings.translogSize().bytes()) { // don't throttle translog, since we lock for phase3 indexing, // so we need to move it as fast as possible. Note, since we // index docs to replicas while the index files are recovered // the lock can potentially be removed, in which case, it might // make sense to re-enable throttling in this phase // if (recoverySettings.rateLimiter() != null) { // recoverySettings.rateLimiter().pause(size); // } if (logger.isTraceEnabled()) { logger.trace( "[{}][{}] sending batch of [{}][{}] (total: [{}], id: [{}]) translog operations to {}", indexName, shardId, ops, new ByteSizeValue(size), shard.translog().estimatedNumberOfOperations(), snapshot.translogId(), request.targetNode()); } cancellableThreads.execute( new Interruptable() { @Override public void run() throws InterruptedException { final RecoveryTranslogOperationsRequest translogOperationsRequest = new RecoveryTranslogOperationsRequest( request.recoveryId(), request.shardId(), operations, shard.translog().estimatedNumberOfOperations()); transportService .submitRequest( request.targetNode(), RecoveryTarget.Actions.TRANSLOG_OPS, translogOperationsRequest, recoveryOptions, EmptyTransportResponseHandler.INSTANCE_SAME) .txGet(); } }); ops = 0; size = 0; operations.clear(); } operation = snapshot.next(); } // send the leftover if (logger.isTraceEnabled()) { logger.trace( "[{}][{}] sending final batch of [{}][{}] (total: [{}], id: [{}]) translog operations to {}", indexName, shardId, ops, new ByteSizeValue(size), shard.translog().estimatedNumberOfOperations(), snapshot.translogId(), request.targetNode()); } if (!operations.isEmpty()) { cancellableThreads.execute( new Interruptable() { @Override public void run() throws InterruptedException { RecoveryTranslogOperationsRequest translogOperationsRequest = new RecoveryTranslogOperationsRequest( request.recoveryId(), request.shardId(), operations, shard.translog().estimatedNumberOfOperations()); transportService .submitRequest( request.targetNode(), RecoveryTarget.Actions.TRANSLOG_OPS, translogOperationsRequest, recoveryOptions, EmptyTransportResponseHandler.INSTANCE_SAME) .txGet(); } }); } return totalOperations; }