private void recoverFile( final CommitPoint.FileInfo fileInfo, final ImmutableMap<String, BlobMetaData> blobs, final CountDownLatch latch, final List<Throwable> failures) { final IndexOutput indexOutput; try { // we create an output with no checksum, this is because the pure binary data of the file is // not // the checksum (because of seek). We will create the checksum file once copying is done indexOutput = store.createOutputWithNoChecksum(fileInfo.physicalName()); } catch (IOException e) { failures.add(e); latch.countDown(); return; } String firstFileToRecover = fileInfo.name(); if (!blobs.containsKey(fileInfo.name())) { // chunking, append part0 to it firstFileToRecover = fileInfo.name() + ".part0"; } if (!blobs.containsKey(firstFileToRecover)) { // no file, what to do, what to do? logger.warn( "no file [{}]/[{}] to recover, ignoring it", fileInfo.name(), fileInfo.physicalName()); latch.countDown(); return; } final AtomicInteger partIndex = new AtomicInteger(); blobContainer.readBlob( firstFileToRecover, new BlobContainer.ReadBlobListener() { @Override public synchronized void onPartial(byte[] data, int offset, int size) throws IOException { recoveryStatus.index().addCurrentFilesSize(size); indexOutput.writeBytes(data, offset, size); } @Override public synchronized void onCompleted() { int part = partIndex.incrementAndGet(); String partName = fileInfo.name() + ".part" + part; if (blobs.containsKey(partName)) { // continue with the new part blobContainer.readBlob(partName, this); return; } else { // we are done... try { indexOutput.close(); // write the checksum if (fileInfo.checksum() != null) { store.writeChecksum(fileInfo.physicalName(), fileInfo.checksum()); } store.directory().sync(Collections.singleton(fileInfo.physicalName())); } catch (IOException e) { onFailure(e); return; } } latch.countDown(); } @Override public void onFailure(Throwable t) { failures.add(t); latch.countDown(); } }); }
private void recoverTranslog(CommitPoint commitPoint, ImmutableMap<String, BlobMetaData> blobs) throws IndexShardGatewayRecoveryException { if (commitPoint.translogFiles().isEmpty()) { // no translog files, bail indexShard.start("post recovery from gateway, no translog"); return; } try { indexShard.performRecoveryPrepareForTranslog(); final AtomicReference<Throwable> failure = new AtomicReference<Throwable>(); final CountDownLatch latch = new CountDownLatch(1); final Iterator<CommitPoint.FileInfo> transIt = commitPoint.translogFiles().iterator(); blobContainer.readBlob( transIt.next().name(), new BlobContainer.ReadBlobListener() { FastByteArrayOutputStream bos = new FastByteArrayOutputStream(); boolean ignore = false; @Override public synchronized void onPartial(byte[] data, int offset, int size) throws IOException { if (ignore) { return; } bos.write(data, offset, size); // if we don't have enough to read the header size of the first translog, bail and // wait for the next one if (bos.size() < 4) { return; } BytesStreamInput si = new BytesStreamInput(bos.unsafeByteArray(), 0, bos.size()); int position; while (true) { try { position = si.position(); if (position + 4 > bos.size()) { break; } int opSize = si.readInt(); int curPos = si.position(); if ((si.position() + opSize) > bos.size()) { break; } Translog.Operation operation = TranslogStreams.readTranslogOperation(si); if ((si.position() - curPos) != opSize) { logger.warn( "mismatch in size, expected [{}], got [{}]", opSize, si.position() - curPos); } recoveryStatus.translog().addTranslogOperations(1); indexShard.performRecoveryOperation(operation); if (si.position() >= bos.size()) { position = si.position(); break; } } catch (Exception e) { logger.warn( "failed to retrieve translog after [{}] operations, ignoring the rest, considered corrupted", e, recoveryStatus.translog().currentTranslogOperations()); ignore = true; latch.countDown(); return; } } FastByteArrayOutputStream newBos = new FastByteArrayOutputStream(); int leftOver = bos.size() - position; if (leftOver > 0) { newBos.write(bos.unsafeByteArray(), position, leftOver); } bos = newBos; } @Override public synchronized void onCompleted() { if (ignore) { return; } if (!transIt.hasNext()) { latch.countDown(); return; } blobContainer.readBlob(transIt.next().name(), this); } @Override public void onFailure(Throwable t) { failure.set(t); latch.countDown(); } }); latch.await(); if (failure.get() != null) { throw failure.get(); } indexShard.performRecoveryFinalization(true); } catch (Throwable e) { throw new IndexShardGatewayRecoveryException(shardId, "Failed to recover translog", e); } }