Ejemplo n.º 1
0
  @Override
  public F.Promise<Void> store(Path path, String key) {
    Promise<Void> promise = Futures.promise();

    TransferManager transferManager = new TransferManager(credentials);
    Upload upload = transferManager.upload(bucketName, key, path.toFile());

    upload.addProgressListener(
        (ProgressListener)
            progressEvent -> {
              if (progressEvent.getEventType().isTransferEvent()) {
                if (progressEvent
                    .getEventType()
                    .equals(ProgressEventType.TRANSFER_COMPLETED_EVENT)) {
                  transferManager.shutdownNow();
                  promise.success(null);
                } else if (progressEvent
                    .getEventType()
                    .equals(ProgressEventType.TRANSFER_FAILED_EVENT)) {
                  transferManager.shutdownNow();
                  promise.failure(new Exception("Upload failed"));
                }
              }
            });

    return F.Promise.wrap(promise.future());
  }
Ejemplo n.º 2
0
 @Override
 protected UploadResult doInBackground(File... files) {
   try {
     for (File file : files) {
       TransferManager transferManager = new TransferManager(amazonS3);
       Upload upload = transferManager.upload(BUCKET, file.getName(), file);
       return upload.waitForUploadResult();
     }
   } catch (InterruptedException e) {
     Log.wtf("UPLOAD", e);
   }
   return null;
 }
 private void createAmazonS3Bucket() {
   try {
     if (tx.getAmazonS3Client().doesBucketExist(bucketName) == false) {
       tx.getAmazonS3Client().createBucket(bucketName);
     }
   } catch (AmazonClientException ace) {
     JOptionPane.showMessageDialog(
         frame,
         "Unable to create a new Amazon S3 bucket: " + ace.getMessage(),
         "Error Creating Bucket",
         JOptionPane.ERROR_MESSAGE);
   }
 }
Ejemplo n.º 4
0
  @Override
  public PutObjectResult putObject(PutObjectRequest req)
      throws AmazonClientException, AmazonServiceException {
    if (!multipartUpload) return super.putObject(req);

    final long contentLen = TransferManagerUtils.getContentLength(req);

    String tempFilename = req.getKey() + ".tmp";
    String origFilename = req.getKey();

    req.setKey(tempFilename);

    XProgressListener progressListener = new XProgressListener();

    req.setProgressListener(new ProgressListenerChain(progressListener));

    progressListener.setContentLen(contentLen);
    progressListener.setUpload(transferManager.upload(req));

    try {
      progressListener.getUpload().waitForCompletion();
    } catch (InterruptedException e) {
      throw new AmazonClientException(e.getMessage(), e);
    }

    CopyObjectRequest copyReq =
        new CopyObjectRequest(req.getBucketName(), tempFilename, req.getBucketName(), origFilename);

    copyObject(copyReq);

    deleteObject(new DeleteObjectRequest(req.getBucketName(), tempFilename));

    return null;
  }
Ejemplo n.º 5
0
  private static void moveToS3(String outputFilePath, String existingBucketName)
      throws InterruptedException {
    System.setProperty("aws.accessKeyId", AWS_ACCESS_KEY_ID);
    System.setProperty("aws.secretKey", AWS_ACCESS_SECRET_KEY);
    File outputFile = new File(outputFilePath);
    String keyName = outputFile.getName();

    TransferManager tm = new TransferManager(new SystemPropertiesCredentialsProvider());
    // TransferManager processes all transfers asynchronously,
    // so this call will return immediately.
    Upload upload = tm.upload(existingBucketName, keyName, new File(outputFilePath));

    try {
      // Or you can block and wait for the upload to finish
      upload.waitForCompletion();
      System.out.println("Upload complete.");
    } catch (AmazonClientException amazonClientException) {
      System.out.println("Unable to upload file, upload was aborted.");
      amazonClientException.printStackTrace();
    }
  }
Ejemplo n.º 6
0
    public PrestoS3OutputStream(
        AmazonS3 s3, TransferManagerConfiguration config, String host, String key, File tempFile)
        throws IOException {
      super(
          new BufferedOutputStream(
              new FileOutputStream(checkNotNull(tempFile, "tempFile is null"))));

      transferManager = new TransferManager(checkNotNull(s3, "s3 is null"));
      transferManager.setConfiguration(checkNotNull(config, "config is null"));

      this.host = checkNotNull(host, "host is null");
      this.key = checkNotNull(key, "key is null");
      this.tempFile = tempFile;

      log.debug("OutputStream for key '%s' using file: %s", key, tempFile);
    }
Ejemplo n.º 7
0
    @Override
    public void close() throws IOException {
      if (closed) {
        return;
      }
      closed = true;

      try {
        super.close();
        uploadObject();
      } finally {
        if (!tempFile.delete()) {
          log.warn("Could not delete temporary file: %s", tempFile);
        }
        // close transfer manager but keep underlying S3 client open
        transferManager.shutdownNow(false);
      }
    }
Ejemplo n.º 8
0
    private void uploadObject() throws IOException {
      try {
        log.debug(
            "Starting upload for host: %s, key: %s, file: %s, size: %s",
            host, key, tempFile, tempFile.length());
        Upload upload = transferManager.upload(host, key, tempFile);

        if (log.isDebugEnabled()) {
          upload.addProgressListener(createProgressListener(upload));
        }

        upload.waitForCompletion();
        log.debug("Completed upload for host: %s, key: %s", host, key);
      } catch (AmazonClientException e) {
        throw new IOException(e);
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new InterruptedIOException();
      }
    }
 @Override
 public ListenableFuture<Void> upload(
     String bucketName, String name, InputStream input, ObjectMetadata meta) throws IOException {
   SettableFuture<Void> future = SettableFuture.create();
   LOG.info("Starting upload, bucket={}, key={}", bucketName, name);
   final Upload up = tx.upload(bucketName, name, input, meta);
   up.addProgressListener(
       (ProgressEvent progressEvent) -> {
         switch (progressEvent.getEventType()) {
           case CLIENT_REQUEST_FAILED_EVENT:
             future.setException(
                 new IOException(
                     "Failed to complete upload after "
                         + progressEvent.getBytesTransferred()
                         + " bytes"));
           case CLIENT_REQUEST_SUCCESS_EVENT:
             LOG.info("Upload completed, bucket={}, key={}", bucketName, name);
             future.set(null);
         }
       });
   return future;
 }
Ejemplo n.º 10
0
 public void deleteMultiparts(String bucketName, Date since) {
   transferManager.abortMultipartUploads(bucketName, since);
 }
Ejemplo n.º 11
0
 protected void init() {
   transferManager = new TransferManager(this);
   TransferManagerConfiguration configuration = new TransferManagerConfiguration();
   configuration.setMultipartUploadThreshold(100 * Constants.KB);
   transferManager.setConfiguration(configuration);
 }
  public static List<ExternalConfigurationModel> buildFromConfig(String configFilePath)
      throws Exception {
    List<ExternalConfigurationModel> response = new ArrayList<>();

    // reference the config file as a full path
    File configFile = new File(configFilePath);
    if (!configFile.exists()) {

      // try to load the file from the classpath
      InputStream classpathConfig =
          ExternalConfigurationModel.class.getClassLoader().getResourceAsStream(configFilePath);
      if (classpathConfig != null && classpathConfig.available() > 0) {
        configFile =
            new File(
                ExternalConfigurationModel.class
                    .getResource((configFilePath.startsWith("/") ? "" : "/") + configFilePath)
                    .toURI());

        LOG.info(String.format("Loaded Configuration %s from Classpath", configFilePath));
      } else {
        if (configFilePath.startsWith("s3://")) {
          AmazonS3 s3Client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain());
          TransferManager tm = new TransferManager(s3Client);

          // parse the config path to get the bucket name and prefix
          final String s3ProtoRegex = "s3:\\/\\/";
          String bucket = configFilePath.replaceAll(s3ProtoRegex, "").split("/")[0];
          String prefix =
              configFilePath.replaceAll(String.format("%s%s\\/", s3ProtoRegex, bucket), "");

          // download the file using TransferManager
          configFile = File.createTempFile(configFilePath, null);
          Download download = tm.download(bucket, prefix, configFile);
          download.waitForCompletion();

          // shut down the transfer manager
          tm.shutdownNow();

          LOG.info(
              String.format(
                  "Loaded Configuration from Amazon S3 %s/%s to %s",
                  bucket, prefix, configFile.getAbsolutePath()));
        } else {
          // load the file from external URL
          try {
            configFile = File.createTempFile(configFilePath, null);
            FileUtils.copyURLToFile(new URL(configFilePath), configFile, 1000, 1000);
            LOG.info(
                String.format(
                    "Loaded Configuration from %s to %s",
                    configFilePath, configFile.getAbsolutePath()));
          } catch (IOException e) {
            // handle the timeouts and so on with a generalised
            // config
            // file not found handler later
          }
        }
      }
    } else {
      LOG.info(String.format("Loaded Configuration from Filesystem %s", configFilePath));
    }

    // if we haven't been able to load a config file, then bail
    if (configFile == null || !configFile.exists()) {
      throw new InvalidConfigurationException(
          String.format("Unable to Load Config File from %s", configFilePath));
    }

    JsonNode document = StreamAggregatorUtils.asJsonNode(configFile);

    ExternalConfigurationModel config = null;

    Iterator<JsonNode> i = document.elements();
    while (i.hasNext()) {
      config = new ExternalConfigurationModel();

      JsonNode section = i.next();

      // set generic properties
      config.setNamespace(StreamAggregatorUtils.readValueAsString(section, "namespace"));
      config.setDateFormat(StreamAggregatorUtils.readValueAsString(section, "dateFormat"));
      addTimeHorizons(section, config);
      setAggregatorType(section, config);

      // set the label items
      JsonNode labelItems = StreamAggregatorUtils.readJsonValue(section, "labelItems");
      if (labelItems != null && labelItems.size() > 0) {
        Iterator<JsonNode> iterator = labelItems.elements();
        while (iterator.hasNext()) {
          JsonNode n = iterator.next();
          config.addLabelItems(n.asText());
        }
      }
      config.setLabelAttributeAlias(
          StreamAggregatorUtils.readValueAsString(section, "labelAttributeAlias"));

      config.setDateItem(StreamAggregatorUtils.readValueAsString(section, "dateItem"));
      config.setDateAttributeAlias(
          StreamAggregatorUtils.readValueAsString(section, "dateAttributeAlias"));
      JsonNode summaryItems = StreamAggregatorUtils.readJsonValue(section, "summaryItems");
      if (summaryItems != null && summaryItems.size() > 0) {
        Iterator<JsonNode> iterator = summaryItems.elements();
        while (iterator.hasNext()) {
          JsonNode n = iterator.next();
          config.addSummaryItem(n.asText());
        }
      }

      config.setTableName(StreamAggregatorUtils.readValueAsString(section, "tableName"));

      String readIO = StreamAggregatorUtils.readValueAsString(section, "readIOPS");
      if (readIO != null) config.setReadIOPs(Long.parseLong(readIO));
      String writeIO = StreamAggregatorUtils.readValueAsString(section, "writeIOPS");
      if (writeIO != null) config.setWriteIOPs(Long.parseLong(writeIO));

      // configure tolerance of data extraction problems
      String failOnDataExtraction =
          StreamAggregatorUtils.readValueAsString(section, "failOnDataExtraction");
      if (failOnDataExtraction != null)
        config.setFailOnDataExtraction(Boolean.parseBoolean(failOnDataExtraction));

      // configure whether metrics should be emitted
      String emitMetrics = StreamAggregatorUtils.readValueAsString(section, "emitMetrics");
      String metricsEmitterClassname =
          StreamAggregatorUtils.readValueAsString(section, "metricsEmitterClass");
      if (emitMetrics != null || metricsEmitterClassname != null) {
        if (metricsEmitterClassname != null) {
          config.setMetricsEmitter(
              (Class<IMetricsEmitter>)
                  ClassLoader.getSystemClassLoader().loadClass(metricsEmitterClassname));
        } else {
          config.setEmitMetrics(Boolean.parseBoolean(emitMetrics));
        }
      }

      // configure the data store class
      String dataStoreClass = StreamAggregatorUtils.readValueAsString(section, "IDataStore");
      if (dataStoreClass != null) {
        Class<IDataStore> dataStore =
            (Class<IDataStore>) ClassLoader.getSystemClassLoader().loadClass(dataStoreClass);
        config.setDataStore(dataStore);
      }

      // get the data extractor configuration, so we know what other json
      // elements to retrieve from the configuration document
      String useExtractor = null;
      try {
        useExtractor = StreamAggregatorUtils.readValueAsString(section, "dataExtractor");
        config.setDataExtractor(DataExtractor.valueOf(useExtractor));
      } catch (Exception e) {
        throw new Exception(
            String.format("Unable to configure aggregator with Data Extractor %s", useExtractor));
      }

      switch (config.getDataExtractor()) {
        case CSV:
          configureStringCommon(section, config);
          configureCsv(section, config);
          break;
        case JSON:
          configureStringCommon(section, config);
          break;
        case OBJECT:
          configureObject(section, config);
          break;
        case REGEX:
          configureRegex(section, config);
      }

      response.add(config);
    }
    return response;
  }