public void download(S3Artifact s3Artifact, Path downloadTo) {
final long start = System.currentTimeMillis();
boolean success = false;
try {
downloadThrows(s3Artifact, downloadTo);
success = true;
} catch (Throwable t) {
throw Throwables.propagate(t);
} finally {
log.info(
"S3 Download {}/{} finished {} after {}",
s3Artifact.getS3Bucket(),
s3Artifact.getS3ObjectKey(),
success ? "successfully" : "with error",
JavaUtils.duration(start));
}
}
private boolean handleChunk(
S3Artifact s3Artifact,
Future<Path> future,
Path downloadTo,
int chunk,
long start,
long remainingMillis) {
if (remainingMillis <= 0) {
remainingMillis = 1;
}
try {
Path path = future.get(remainingMillis, TimeUnit.MILLISECONDS);
if (chunk > 0) {
combineChunk(downloadTo, path);
}
return true;
} catch (TimeoutException te) {
log.error(
"Chunk {} for {} timed out after {} - had {} remaining",
chunk,
s3Artifact.getFilename(),
JavaUtils.duration(start),
JavaUtils.durationFromMillis(remainingMillis));
future.cancel(true);
exceptionNotifier.notify(
te,
ImmutableMap.of("filename", s3Artifact.getFilename(), "chunk", Integer.toString(chunk)));
} catch (Throwable t) {
log.error("Error while handling chunk {} for {}", chunk, s3Artifact.getFilename(), t);
exceptionNotifier.notify(
t,
ImmutableMap.of("filename", s3Artifact.getFilename(), "chunk", Integer.toString(chunk)));
}
return false;
}
private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception {
log.info("Downloading {}", s3Artifact);
Jets3tProperties jets3tProperties =
Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME);
jets3tProperties.setProperty(
"httpclient.socket-timeout-ms",
Long.toString(configuration.getS3ChunkDownloadTimeoutMillis()));
final S3Service s3 =
new RestS3Service(
getCredentialsForBucket(s3Artifact.getS3Bucket()), null, null, jets3tProperties);
long length = 0;
if (s3Artifact.getFilesize().isPresent()) {
length = s3Artifact.getFilesize().get();
} else {
StorageObject details =
s3.getObjectDetails(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
Preconditions.checkNotNull(
details,
"Couldn't find object at %s/%s",
s3Artifact.getS3Bucket(),
s3Artifact.getS3ObjectKey());
length = details.getContentLength();
}
int numChunks = (int) (length / configuration.getS3ChunkSize());
if (length % configuration.getS3ChunkSize() > 0) {
numChunks++;
}
final long chunkSize = length / numChunks + (length % numChunks);
log.info(
"Downloading {}/{} in {} chunks of {} bytes to {}",
s3Artifact.getS3Bucket(),
s3Artifact.getS3ObjectKey(),
numChunks,
chunkSize,
downloadTo);
final ExecutorService chunkExecutorService =
Executors.newFixedThreadPool(
numChunks,
new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat("S3ArtifactDownloaderChunkThread-%d")
.build());
final List<Future<Path>> futures = Lists.newArrayListWithCapacity(numChunks);
for (int chunk = 0; chunk < numChunks; chunk++) {
futures.add(
chunkExecutorService.submit(
new S3ArtifactChunkDownloader(
configuration,
log,
s3,
s3Artifact,
downloadTo,
chunk,
chunkSize,
length,
exceptionNotifier)));
}
long remainingMillis = configuration.getS3DownloadTimeoutMillis();
boolean failed = false;
for (int chunk = 0; chunk < numChunks; chunk++) {
final Future<Path> future = futures.get(chunk);
if (failed) {
future.cancel(true);
continue;
}
final long start = System.currentTimeMillis();
if (!handleChunk(s3Artifact, future, downloadTo, chunk, start, remainingMillis)) {
failed = true;
}
remainingMillis -= (System.currentTimeMillis() - start);
}
chunkExecutorService.shutdownNow();
Preconditions.checkState(
!failed, "Downloading %s/%s failed", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
}