Example #1
0
 @Override
 public long getFileSize(String path) throws IOException {
   StorageObject details = getObjectDetails(path);
   if (details != null) {
     return details.getContentLength();
   } else {
     throw new FileNotFoundException(path);
   }
 }
Example #2
0
 @Override
 public long getModificationTimeMs(String path) throws IOException {
   StorageObject details = getObjectDetails(path);
   if (details != null) {
     return details.getLastModifiedDate().getTime();
   } else {
     throw new FileNotFoundException(path);
   }
 }
  private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception {
    log.info("Downloading {}", s3Artifact);

    Jets3tProperties jets3tProperties =
        Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME);
    jets3tProperties.setProperty(
        "httpclient.socket-timeout-ms",
        Long.toString(configuration.getS3ChunkDownloadTimeoutMillis()));

    final S3Service s3 =
        new RestS3Service(
            getCredentialsForBucket(s3Artifact.getS3Bucket()), null, null, jets3tProperties);

    long length = 0;

    if (s3Artifact.getFilesize().isPresent()) {
      length = s3Artifact.getFilesize().get();
    } else {
      StorageObject details =
          s3.getObjectDetails(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());

      Preconditions.checkNotNull(
          details,
          "Couldn't find object at %s/%s",
          s3Artifact.getS3Bucket(),
          s3Artifact.getS3ObjectKey());

      length = details.getContentLength();
    }

    int numChunks = (int) (length / configuration.getS3ChunkSize());

    if (length % configuration.getS3ChunkSize() > 0) {
      numChunks++;
    }

    final long chunkSize = length / numChunks + (length % numChunks);

    log.info(
        "Downloading {}/{} in {} chunks of {} bytes to {}",
        s3Artifact.getS3Bucket(),
        s3Artifact.getS3ObjectKey(),
        numChunks,
        chunkSize,
        downloadTo);

    final ExecutorService chunkExecutorService =
        Executors.newFixedThreadPool(
            numChunks,
            new ThreadFactoryBuilder()
                .setDaemon(true)
                .setNameFormat("S3ArtifactDownloaderChunkThread-%d")
                .build());
    final List<Future<Path>> futures = Lists.newArrayListWithCapacity(numChunks);

    for (int chunk = 0; chunk < numChunks; chunk++) {
      futures.add(
          chunkExecutorService.submit(
              new S3ArtifactChunkDownloader(
                  configuration,
                  log,
                  s3,
                  s3Artifact,
                  downloadTo,
                  chunk,
                  chunkSize,
                  length,
                  exceptionNotifier)));
    }

    long remainingMillis = configuration.getS3DownloadTimeoutMillis();
    boolean failed = false;

    for (int chunk = 0; chunk < numChunks; chunk++) {
      final Future<Path> future = futures.get(chunk);

      if (failed) {
        future.cancel(true);
        continue;
      }

      final long start = System.currentTimeMillis();

      if (!handleChunk(s3Artifact, future, downloadTo, chunk, start, remainingMillis)) {
        failed = true;
      }

      remainingMillis -= (System.currentTimeMillis() - start);
    }

    chunkExecutorService.shutdownNow();

    Preconditions.checkState(
        !failed, "Downloading %s/%s failed", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
  }