public void close(Reporter reporter) throws IOException {
for (Integer shard : lps.keySet()) {
String lpDir = localManager.localTmpDir("" + shard);
LOG.info("Closing LP for shard " + shard + " at " + lpDir);
lps.get(shard).close();
LOG.info("Closed LP for shard " + shard + " at " + lpDir);
progress();
String remoteDir = args.outputDirHdfs + "/" + shard;
// Do all this stuff to ensure that S3 actually does delete
int deleteAttempt = 4;
while (fileSystem.exists(new Path(remoteDir)) && deleteAttempt > 0) {
LOG.info("Deleting existing shard " + shard + " at " + remoteDir);
fileSystem.delete(new Path(remoteDir), true);
--deleteAttempt;
}
if (fileSystem.exists(new Path(remoteDir)) && deleteAttempt == 0) {
throw new IOException(
"Failed to delete shard "
+ shard
+ " at "
+ remoteDir
+ " after "
+ deleteAttempt
+ " attempts!");
} else {
LOG.info("Deleted existing shard " + shard + " at " + remoteDir);
}
LOG.info("Copying " + lpDir + " to " + remoteDir);
fileSystem.copyFromLocalFile(new Path(lpDir), new Path(remoteDir));
LOG.info("Copied " + lpDir + " to " + remoteDir);
progress();
}
localManager.cleanup();
}
