private void removeFile(PnfsId pnfsId) {
try {
repository.setState(pnfsId, ReplicaState.REMOVED);
} catch (IllegalTransitionException f) {
LOGGER.warn(
"File not found in name space, but failed to remove {}: {}", pnfsId, f.getMessage());
} catch (CacheException f) {
LOGGER.error(
"File not found in name space, but failed to remove {}: {}", pnfsId, f.getMessage());
} catch (InterruptedException f) {
LOGGER.warn("File not found in name space, but failed to remove {}: {}", pnfsId, f);
}
}
private void done(Throwable cause) {
PnfsId pnfsId = getFileAttributes().getPnfsId();
if (cause != null) {
if (cause instanceof InterruptedException || cause instanceof CancellationException) {
cause = new TimeoutCacheException("Flush was cancelled.", cause);
}
if (cause instanceof CacheException) {
infoMsg.setResult(((CacheException) cause).getRc(), cause.getMessage());
} else {
infoMsg.setResult(CacheException.DEFAULT_ERROR_CODE, cause.getMessage());
}
}
infoMsg.setTransferTime(System.currentTimeMillis() - activatedAt);
infoMsg.setFileSize(getFileAttributes().getSize());
infoMsg.setTimeQueued(activatedAt - createdAt);
if (!suppressedStoreErrors.contains(infoMsg.getResultCode())) {
if (infoMsg.getResultCode() != 0) {
LOGGER.warn("Flush of {} failed with: {}.", pnfsId, cause.toString());
}
billingStub.notify(infoMsg);
}
flushRequests.removeAndCallback(pnfsId, cause);
}
@Override
public void completed(Set<URI> uris) {
try {
descriptor.close();
FileAttributes fileAttributesForNotification = getFileAttributesForNotification(uris);
infoMsg.setStorageInfo(fileAttributesForNotification.getStorageInfo());
PnfsId pnfsId = getFileAttributes().getPnfsId();
notifyNamespace(pnfsId, fileAttributesForNotification);
try {
repository.setState(pnfsId, ReplicaState.CACHED);
} catch (IllegalTransitionException ignored) {
/* Apparently the file is no longer precious. Most
* likely it got deleted, which is fine, since the
* flush already succeeded.
*/
}
done(null);
LOGGER.info("Flushed {} to nearline storage: {}", pnfsId, Joiner.on(' ').join(uris));
} catch (Exception e) {
done(e);
}
}
public void failed(Exception cause) {
if (cause instanceof InterruptedException || cause instanceof CancellationException) {
cause = new TimeoutCacheException("Stage was cancelled.", cause);
}
LOGGER.warn("Remove of {} failed with {}.", uri, cause);
removeRequests.removeAndCallback(uri, cause);
}
@Override
public ListenableFuture<Void> allocate() {
LOGGER.debug("Allocating space for stage of {}.", getFileAttributes().getPnfsId());
return register(
executor.submit(
() -> {
descriptor.allocate(descriptor.getFileAttributes().getSize());
return null;
}));
}
public FlushRequestImpl(NearlineStorage nearlineStorage, PnfsId pnfsId)
throws CacheException, InterruptedException {
super(nearlineStorage);
infoMsg = new StorageInfoMessage(cellAddress.toString(), pnfsId, false);
descriptor = repository.openEntry(pnfsId, NO_FLAGS);
String path = descriptor.getFileAttributes().getStorageInfo().getKey("path");
if (path != null) {
infoMsg.setBillingPath(path);
}
LOGGER.debug("Flush request created for {}.", pnfsId);
}
@Override
public void completed(Set<Checksum> checksums) {
Throwable error = null;
try {
if (checksumModule.hasPolicy(ChecksumModule.PolicyFlag.GET_CRC_FROM_HSM)) {
LOGGER.info(
"Obtained checksums {} for {} from HSM", checksums, getFileAttributes().getPnfsId());
descriptor.addChecksums(checksums);
}
checksumModule.enforcePostRestorePolicy(descriptor);
descriptor.commit();
LOGGER.info("Staged {} from nearline storage.", getFileAttributes().getPnfsId());
} catch (InterruptedException | CacheException | RuntimeException | Error e) {
error = e;
} catch (NoSuchAlgorithmException e) {
error = new CacheException(1010, "Checksum calculation failed: " + e.getMessage(), e);
} catch (IOException e) {
error =
new DiskErrorCacheException(
"Checksum calculation failed due to I/O error: " + e.getMessage(), e);
} finally {
done(error);
}
}
public StageRequestImpl(NearlineStorage storage, FileAttributes fileAttributes)
throws CacheException {
super(storage);
PnfsId pnfsId = fileAttributes.getPnfsId();
infoMsg = new StorageInfoMessage(cellAddress.toString(), pnfsId, true);
infoMsg.setStorageInfo(fileAttributes.getStorageInfo());
infoMsg.setFileSize(fileAttributes.getSize());
descriptor =
repository.createEntry(
fileAttributes,
ReplicaState.FROM_STORE,
ReplicaState.CACHED,
Collections.emptyList(),
EnumSet.noneOf(Repository.OpenFlags.class));
LOGGER.debug("Stage request created for {}.", pnfsId);
}
@Override
public ListenableFuture<Void> apply(Void ignored)
throws CacheException, InterruptedException, NoSuchAlgorithmException, IOException {
final PnfsId pnfsId = descriptor.getFileAttributes().getPnfsId();
LOGGER.debug("Checking if {} still exists.", pnfsId);
try {
pnfs.getFileAttributes(pnfsId, EnumSet.noneOf(FileAttribute.class));
} catch (FileNotFoundCacheException e) {
// Remove file asynchronously to prevent request cancellation from
// interrupting the state update.
executor.execute(() -> removeFile(pnfsId));
throw new FileNotFoundCacheException(
"File not found in name space during pre-flush check.", e);
}
checksumModule.enforcePreFlushPolicy(descriptor);
return Futures.immediateFuture(null);
}
private void done(Throwable cause) {
PnfsId pnfsId = getFileAttributes().getPnfsId();
if (cause != null) {
if (cause instanceof InterruptedException || cause instanceof CancellationException) {
cause = new TimeoutCacheException("Stage was cancelled.", cause);
}
LOGGER.warn("Stage of {} failed with {}.", pnfsId, cause);
}
descriptor.close();
if (cause instanceof CacheException) {
infoMsg.setResult(((CacheException) cause).getRc(), cause.getMessage());
} else if (cause != null) {
infoMsg.setResult(CacheException.DEFAULT_ERROR_CODE, cause.toString());
}
infoMsg.setTransferTime(System.currentTimeMillis() - activatedAt);
billingStub.notify(infoMsg);
stageRequests.removeAndCallback(pnfsId, cause);
}
private void notifyNamespace(PnfsId pnfsid, FileAttributes fileAttributes)
throws InterruptedException {
while (true) {
try {
pnfs.fileFlushed(pnfsid, fileAttributes);
break;
} catch (CacheException e) {
if (e.getRc() == CacheException.FILE_NOT_FOUND
|| e.getRc() == CacheException.NOT_IN_TRASH) {
/* In case the file was deleted, we are presented
* with the problem that the file is now on tape,
* however the location has not been registered
* centrally. Hence the copy on tape will not be
* removed by the HSM cleaner. The sensible thing
* seems to be to remove the file from tape here.
* For now we ignore this issue (REVISIT).
*/
break;
}
/* The message to the PnfsManager failed. There are several
* possible reasons for this; we may have lost the
* connection to the PnfsManager; the PnfsManager may have
* lost its connection to the namespace or otherwise be in
* trouble; bugs; etc.
*
* We keep retrying until we succeed. This will effectively
* block this thread from flushing any other files, which
* seems sensible when we have trouble talking to the
* PnfsManager. If the pool crashes or gets restarted while
* waiting here, we will end up flushing the file again. We
* assume that the nearline storage is able to eliminate the
* duplicate; or at least tolerate the duplicate (given that
* this situation should be rare, we can live with a little
* bit of wasted tape).
*/
LOGGER.error(
"Error notifying pnfsmanager about a flushed file: {} ({})",
e.getMessage(),
e.getRc());
}
TimeUnit.MINUTES.sleep(2);
}
}
@Override
public synchronized ListenableFuture<Void> activate() {
LOGGER.debug("Activating stage of {}.", getFileAttributes().getPnfsId());
return super.activate();
}
@Override
public ListenableFuture<String> activateWithPath() {
LOGGER.debug("Activating flush of {}.", getFileAttributes().getPnfsId());
return register(transformAsync(super.activate(), new PreFlushWithPathFunction(), executor));
}
public void completed(Void result) {
LOGGER.info("Removed {} from nearline storage.", uri);
removeRequests.removeAndCallback(uri, null);
}
@Override
public synchronized ListenableFuture<Void> activate() {
LOGGER.debug("Activating remove of {}.", uri);
return super.activate();
}
RemoveRequestImpl(NearlineStorage storage, URI uri) {
super(storage);
this.uri = uri;
LOGGER.debug("Remove request created for {}.", uri);
}