public void download(S3Artifact s3Artifact, Path downloadTo) {
final long start = System.currentTimeMillis();
boolean success = false;
try {
downloadThrows(s3Artifact, downloadTo);
success = true;
} catch (Throwable t) {
throw Throwables.propagate(t);
} finally {
log.info(
"S3 Download {}/{} finished {} after {}",
s3Artifact.getS3Bucket(),
s3Artifact.getS3ObjectKey(),
success ? "successfully" : "with error",
JavaUtils.duration(start));
}
}
private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception {
log.info("Downloading {}", s3Artifact);
Jets3tProperties jets3tProperties =
Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME);
jets3tProperties.setProperty(
"httpclient.socket-timeout-ms",
Long.toString(configuration.getS3ChunkDownloadTimeoutMillis()));
final S3Service s3 =
new RestS3Service(
getCredentialsForBucket(s3Artifact.getS3Bucket()), null, null, jets3tProperties);
long length = 0;
if (s3Artifact.getFilesize().isPresent()) {
length = s3Artifact.getFilesize().get();
} else {
StorageObject details =
s3.getObjectDetails(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
Preconditions.checkNotNull(
details,
"Couldn't find object at %s/%s",
s3Artifact.getS3Bucket(),
s3Artifact.getS3ObjectKey());
length = details.getContentLength();
}
int numChunks = (int) (length / configuration.getS3ChunkSize());
if (length % configuration.getS3ChunkSize() > 0) {
numChunks++;
}
final long chunkSize = length / numChunks + (length % numChunks);
log.info(
"Downloading {}/{} in {} chunks of {} bytes to {}",
s3Artifact.getS3Bucket(),
s3Artifact.getS3ObjectKey(),
numChunks,
chunkSize,
downloadTo);
final ExecutorService chunkExecutorService =
Executors.newFixedThreadPool(
numChunks,
new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat("S3ArtifactDownloaderChunkThread-%d")
.build());
final List<Future<Path>> futures = Lists.newArrayListWithCapacity(numChunks);
for (int chunk = 0; chunk < numChunks; chunk++) {
futures.add(
chunkExecutorService.submit(
new S3ArtifactChunkDownloader(
configuration,
log,
s3,
s3Artifact,
downloadTo,
chunk,
chunkSize,
length,
exceptionNotifier)));
}
long remainingMillis = configuration.getS3DownloadTimeoutMillis();
boolean failed = false;
for (int chunk = 0; chunk < numChunks; chunk++) {
final Future<Path> future = futures.get(chunk);
if (failed) {
future.cancel(true);
continue;
}
final long start = System.currentTimeMillis();
if (!handleChunk(s3Artifact, future, downloadTo, chunk, start, remainingMillis)) {
failed = true;
}
remainingMillis -= (System.currentTimeMillis() - start);
}
chunkExecutorService.shutdownNow();
Preconditions.checkState(
!failed, "Downloading %s/%s failed", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
}