/**
   * Reads next key from S3
   *
   * @return {@link S3ObjectSummary} of the key or <code>null</code> if there are no more keys to
   *     read
   */
  public S3ObjectSummary getNextKey() {
    if (objectListing == null) {
      LOG.debug("Listing objects");
      objectListing = s3Client.listObjects(listObjectsRequest);
    }

    // we have more elements in list
    if (currentPosition < objectListing.getObjectSummaries().size()) {
      currentObject = objectListing.getObjectSummaries().get(currentPosition++);
    } else if (objectListing.isTruncated()) {
      LOG.debug("Current batch reached its end. Fetching next page.");
      // when we're at the end, check if there's more data to read
      listObjectsRequest.setMarker(objectListing.getNextMarker());
      LOG.debug("New marker is set to {}", listObjectsRequest.getMarker());

      objectListing = s3Client.listObjects(listObjectsRequest);
      currentPosition = 0;

      if (currentPosition < objectListing.getObjectSummaries().size()) {
        currentObject = objectListing.getObjectSummaries().get(currentPosition++);
      } else {
        LOG.debug("No more objects to read");
        currentObject = null;
      }
    } else {
      currentObject = null;
    }

    return currentObject;
  }
Exemple #2
0
  /**
   * get the maps
   *
   * @return List<String> list of name of maps
   */
  public List<String> getMapsList(String userName) {
    List<String> maps = new ArrayList<String>();
    ListObjectsRequest listObjectsRequest =
        new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix("maps/public/")
            .withDelimiter("/");
    ObjectListing objects = s3.listObjects(listObjectsRequest);

    for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
      if (!objectSummary.getKey().equals("maps/public")) {
        System.out.println("get map: " + objectSummary.getKey());
        maps.add(objectSummary.getKey());
      }
    }

    ListObjectsRequest listObjectsRequest2 =
        new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix("maps/" + userName + "/")
            .withDelimiter("/");
    ObjectListing objects2 = s3.listObjects(listObjectsRequest);

    for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
      if (!objectSummary.getKey().equals("maps/" + userName)) {
        System.out.println("get map: " + objectSummary.getKey());
        maps.add(objectSummary.getKey());
      }
    }

    return maps;
  }
 @Override
 public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
   List<NoteInfo> infos = new LinkedList<>();
   NoteInfo info;
   try {
     ListObjectsRequest listObjectsRequest =
         new ListObjectsRequest().withBucketName(bucketName).withPrefix(user + "/" + "notebook");
     ObjectListing objectListing;
     do {
       objectListing = s3client.listObjects(listObjectsRequest);
       for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
         if (objectSummary.getKey().endsWith("note.json")) {
           info = getNoteInfo(objectSummary.getKey());
           if (info != null) {
             infos.add(info);
           }
         }
       }
       listObjectsRequest.setMarker(objectListing.getNextMarker());
     } while (objectListing.isTruncated());
   } catch (AmazonClientException ace) {
     throw new IOException("Unable to list objects in S3: " + ace, ace);
   }
   return infos;
 }
 @Override
 public boolean exists(String key, boolean isFile) throws IOException {
   if (isFile) {
     Preconditions.checkArgument(
         key.charAt(key.length() - 1) != '/', "File should not end in %s: %s", '/', key);
   } else {
     Preconditions.checkArgument(
         key.charAt(key.length() - 1) == '/', "Dir should end in %s: %s", '/', key);
   }
   ObjectListing listing;
   try {
     listing = s3Client.listObjects(Namespaces.get().getBucket(), key);
   } catch (AmazonClientException ace) {
     throw new IOException(ace);
   }
   if (listing != null) {
     for (S3ObjectSummary s3os : listing.getObjectSummaries()) {
       String object = s3os.getKey();
       if (object.equals(key) || (!isFile && object.startsWith(key))) {
         // Counts if the keys match exactly, or, the match continues with a subdir path in which
         // case this must be a directory
         return true;
       }
     }
   }
   return false;
 }
  public BucketIterator(String bucket, S3Downloader s3) {
    this.bucket = bucket;

    if (s3 == null) s3 = new S3Downloader();
    currList = s3.listObjects(bucket);
    currObjects = currList.getObjectSummaries();
  }
  public LinkedList<String> listNames(String prefix, String bucketName, String[] canonicalIDs)
      throws StorageCloudException {
    LinkedList<String> find = new LinkedList<String>();
    try {
      ObjectListing objectListing = null;
      if (bucketName == null)
        objectListing =
            conn.listObjects(
                new ListObjectsRequest().withBucketName(defaultBucketName).withPrefix(prefix));
      else {
        bucketName = bucketName.concat(location);
        objectListing =
            conn.listObjects(
                new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));
      }
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        find.add(objectSummary.getKey());
      }
    } catch (AmazonServiceException e1) {
      throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
    } catch (AmazonClientException e2) {
      throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
    }

    return find;
  }
  public String[] setAcl(String bucketNameToShare, String[] canonicalId, String permission)
      throws StorageCloudException {
    boolean withRead = false;
    if (bucketNameToShare != null) {
      bucketNameToShare = bucketNameToShare.concat(location);
      if (!conn.doesBucketExist(bucketNameToShare)) {
        conn.createBucket(bucketNameToShare, region);
      }
    } else {
      return null;
    }

    // set acl
    AccessControlList acl = conn.getBucketAcl(bucketNameToShare);
    for (int i = 0; i < canonicalId.length; i++) {
      if (permission.equals("rw")) {
        CanonicalGrantee grantee = new CanonicalGrantee(canonicalId[i]);
        acl.grantPermission(grantee, Permission.Read);
        acl.grantPermission(grantee, Permission.Write);
        withRead = true;
      } else if (permission.equals("r")) {
        acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
        withRead = true;
      } else if (permission.equals("w")) {
        acl.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Write);
      }
    }
    try {
      if (withRead) {
        ObjectListing objectListing = conn.listObjects(bucketNameToShare);
        AccessControlList aclKeys = null;
        for (S3ObjectSummary elem : objectListing.getObjectSummaries()) {
          aclKeys = conn.getObjectAcl(bucketNameToShare, elem.getKey());
          for (int i = 0; i < canonicalId.length; i++) {
            aclKeys.grantPermission(new CanonicalGrantee(canonicalId[i]), Permission.Read);
          }
          conn.setObjectAcl(bucketNameToShare, elem.getKey(), aclKeys);
        }
      }

      // confirm if acl well
      conn.setBucketAcl(bucketNameToShare, acl);
      AccessControlList newAcl = conn.getBucketAcl(bucketNameToShare);
      Set<Grant> grants = newAcl.getGrants();
      boolean flag = false;
      for (Grant grant : grants) {
        if (grant.getGrantee().getIdentifier().equals(canonicalId[0])) {
          flag = true;
        }
      }
      if (!flag) {
        throw new ServiceSiteException("AWSS3Exception:: ACL");
      }
    } catch (AmazonServiceException e1) {
      throw new ServiceSiteException("AWSS3Exception::" + e1.getMessage());
    } catch (AmazonClientException e2) {
      throw new ClientServiceException("AWSS3Exception::" + e2.getMessage());
    }
    return canonicalId;
  }
 private List<S3ObjectSummary> nestedResources(String resourceName) {
   ObjectListing listing = S3.client().listObjects(S3.bucket(), resourceName);
   List<S3ObjectSummary> resources = listing.getObjectSummaries();
   // exclude the own resource
   if (resources.size() > 0) resources = resources.subList(1, resources.size());
   return resources;
 }
  public void DownloadUsingS3() throws Exception {

    while (true) {
      ObjectListing list =
          s3api.ReadBucket(S3_ACCESS_KEY_ID, S3_SECRET_KEY, S3_ENDPOINT, null, S3_BUCKET);

      System.out.println("bucket files count " + list.getObjectSummaries().size());
      vLogger.LogInfo("emcWorldDownloader: bucket files count " + list.getObjectSummaries().size());

      int s3Count = list.getObjectSummaries().size();
      int localCount = new File(LOCAL_DIR).listFiles().length;
      if (s3Count != localCount) {

        for (S3ObjectSummary obj : list.getObjectSummaries()) {

          if (!(new File(LOCAL_DIR, obj.getKey()).exists())) {
            System.out.println("Downloading - " + obj.getKey());
            vLogger.LogInfo("emcWorldDownloader: Downloading - " + obj.getKey());
            S3ObjectInputStream in =
                s3api.ReadObject(
                    S3_ACCESS_KEY_ID, S3_SECRET_KEY, S3_ENDPOINT, null, S3_BUCKET, obj.getKey());

            File file = new File(LOCAL_DIR + obj.getKey());

            int count = 0;
            byte[] buf = new byte[1024];
            OutputStream out = new FileOutputStream(file);
            while ((count = in.read(buf)) != -1) {
              if (Thread.interrupted()) {
                throw new InterruptedException();
              }
              out.write(buf, 0, count);
            }
            out.close();
            in.close();
          } else {
            System.out.println("Skipping - " + obj.getKey());
            vLogger.LogInfo("emcWorldDownloader: Skipping - " + obj.getKey());
          }
        }
      } else System.out.println("No New Files Yet");

      vLogger.LogInfo("emcWorldDownloader: Skipping - Sleep 60 seconds");
      System.out.println("Sleep for 1 minute");
      Thread.sleep(60000);
    }
  }
Exemple #10
0
 public static List<String> listBucket(String bucketName, String prefix) {
   List<String> ret = new ArrayList<String>();
   ObjectListing listing =
       getInstance()
           .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));
   for (S3ObjectSummary s : listing.getObjectSummaries()) {
     StringBuilder sb = new StringBuilder();
     sb.append("s3://").append(bucketName).append("/").append(s.getKey());
     ret.add(sb.toString());
   }
   while (listing.isTruncated()) {
     listing = getInstance().listNextBatchOfObjects(listing);
     for (S3ObjectSummary s : listing.getObjectSummaries()) {
       StringBuilder sb = new StringBuilder();
       sb.append("s3://").append(bucketName).append("/").append(s.getKey());
       ret.add(sb.toString());
     }
   }
   return ret;
 }
Exemple #11
0
 /**
  * Expect request with start marker and provide in response single item, notify that marker is a
  * next marker to request.
  *
  * @param item Item to respond with
  * @param start Start marker to expect
  * @param marker Next marker
  * @return This instance
  */
 public RegionExpectations expectResponse(
     final String item, final String start, final String marker) {
   final ObjectListing response = new ObjectListing();
   response.setNextMarker(marker);
   final S3ObjectSummary summary = new S3ObjectSummary();
   summary.setKey(item);
   response.getObjectSummaries().add(summary);
   this.responses.add(response);
   this.markers.add(start);
   return this;
 }
  private List<String> getResourceNames(ObjectListing objectListing, Pattern pattern) {
    List<String> resourceNames = new ArrayList<String>();

    for (String commonPrefix : objectListing.getCommonPrefixes()) {
      resourceNames.add(getResourceName(commonPrefix, pattern));
    }

    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
      resourceNames.add(getResourceName(s3ObjectSummary.getKey(), pattern));
    }

    return resourceNames;
  }
 @Override
 public List<BackupMetaData> getAvailableBackups(
     Exhibitor exhibitor, Map<String, String> configValues) throws Exception {
   ListObjectsRequest request = new ListObjectsRequest();
   request.setBucketName(configValues.get(CONFIG_BUCKET.getKey()));
   ObjectListing listing = s3Client.listObjects(request);
   return Lists.transform(
       listing.getObjectSummaries(),
       new Function<S3ObjectSummary, BackupMetaData>() {
         @Override
         public BackupMetaData apply(S3ObjectSummary summary) {
           return fromKey(summary.getKey());
         }
       });
 }
Exemple #14
0
  @Override
  protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;
    pendingExchanges = 0;

    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges = null;

    if (fileName != null) {
      LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);

      S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
      exchanges = createExchanges(s3Object);
    } else {
      LOG.trace("Queueing objects in bucket [{}]...", bucketName);

      ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
      listObjectsRequest.setBucketName(bucketName);
      listObjectsRequest.setPrefix(getConfiguration().getPrefix());
      listObjectsRequest.setMaxKeys(maxMessagesPerPoll);

      ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);

      if (LOG.isTraceEnabled()) {
        LOG.trace(
            "Found {} objects in bucket [{}]...",
            listObjects.getObjectSummaries().size(),
            bucketName);
      }

      exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
  }
Exemple #15
0
 // 列出某个bucket的所有objects
 public static void listObjectInBucket(Bucket bucket) {
   List<Bucket> buckets = conn.listBuckets();
   ObjectListing objects = conn.listObjects(bucket.getName());
   do {
     for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
       System.out.println(
           objectSummary.getKey()
               + "\t"
               + objectSummary.getSize()
               + "\t"
               + StringUtils.fromDate(objectSummary.getLastModified()));
     }
     objects = conn.listNextBatchOfObjects(objects);
   } while (objects.isTruncated());
 }
Exemple #16
0
  /**
   * Tests the S3 access.
   *
   * @throws Exception if test fails
   */
  @Test
  public void s3Test() throws Exception {
    // Create a real object and mock its initClient method
    final S3NutDao dao = spy(new S3NutDao("/path", false, null, -1, "wuic", "login", "pwd", false));

    // Build client mock
    final AmazonS3Client client = mock(AmazonS3Client.class);
    when(dao.initClient()).thenReturn(client);

    // List returned by client
    final ObjectListing list = mock(ObjectListing.class);
    final S3ObjectSummary summary = mock(S3ObjectSummary.class);
    when(summary.getKey()).thenReturn("[cloud].css");
    final S3ObjectSummary summarBis = mock(S3ObjectSummary.class);
    when(summarBis.getKey()).thenReturn("cloud.css");
    when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(list);
    when(list.getObjectSummaries()).thenReturn(Arrays.asList(summary, summarBis));

    // Bytes returned by mocked S3
    final byte[] array = ".cloud { text-align : justify;}".getBytes();
    final S3Object object = mock(S3Object.class);
    when(object.getObjectContent())
        .thenReturn(new S3ObjectInputStream(new ByteArrayInputStream(array), null));
    when(client.getObject(anyString(), anyString())).thenReturn(object);

    // TODO : problem here : we specify '[cloud.css]' but getNuts() returns 'cloud.css' because
    // regex are always activated !
    final NutsHeap nutsHeap = new NutsHeap(Arrays.asList("[cloud].css"), dao, "heap");
    Assert.assertEquals(nutsHeap.getNuts().size(), 1);

    final Engine compressor = new CssYuiCompressorEngine(true, "UTF-8", -1);
    final Engine cacheEngine = new EhCacheEngine(false, null);
    final Engine aggregator = new CGTextAggregatorEngine(true);
    cacheEngine.setNext(compressor);
    compressor.setNext(aggregator);

    final List<Nut> group =
        cacheEngine.parse(new EngineRequest("", "", nutsHeap, new HashMap<NutType, Engine>()));

    Assert.assertFalse(group.isEmpty());
    InputStream is;

    for (Nut res : group) {
      is = res.openStream();
      Assert.assertTrue(IOUtils.readString(new InputStreamReader(is)).length() > 0);
      is.close();
    }
  }
Exemple #17
0
 // 列出有prefix前缀的所有objects
 public static void listObjectInBucketWithPrefix(Bucket bucket, String prefix) {
   ListObjectsRequest listObjectsRequest =
       new ListObjectsRequest().withBucketName(bucket.getName()).withPrefix(prefix);
   ObjectListing objects = conn.listObjects(listObjectsRequest);
   do {
     for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
       System.out.println(
           objectSummary.getKey()
               + "\t"
               + objectSummary.getSize()
               + "\t"
               + StringUtils.fromDate(objectSummary.getLastModified()));
     }
     objects = conn.listNextBatchOfObjects(objects);
   } while (objects.isTruncated());
 }
Exemple #18
0
  public ArrayList<String> getObjectList() {

    s3 =
        new AmazonS3Client(
            new BasicAWSCredentials(
                "AKIAIL5P46R7BWHUK4JA", "YvBz0m95efCfDGRe/uOEwWz/bd09RRZ5ROYF4QY3"));

    objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName));

    ArrayList<String> nodeList = new ArrayList<String>();

    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

      nodeList.add(objectSummary.getKey());
    }
    return nodeList;
  }
Exemple #19
0
  public static void main(final String[] args) {
    DesktopSetup.deploy();
    final RelativePath target = JUtils.newRelativePath();

    final AmazonS3 s3 = new AmazonS3Client(new ProfileCredentialsProvider());
    final ListObjectsRequest request = new ListObjectsRequest().withBucketName("jfix.by");

    final String prefix = "/";
    request.withPrefix("wp-content");
    request.setDelimiter("");

    final ObjectListing objectListing = s3.listObjects(request);
    final List<String> prefixes = Collections.newList(objectListing.getCommonPrefixes());
    prefixes.print("prefixes");

    final List<S3ObjectSummary> summs = Collections.newList(objectListing.getObjectSummaries());
  }
Exemple #20
0
 public void processListing(ObjectListing listing, JsonArray succ, JsonArray fail) {
   for (S3ObjectSummary obj : listing.getObjectSummaries()) {
     try {
       Key k = PersistS3.loadKey(obj);
       JsonObject o = new JsonObject();
       o.addProperty(KEY, k.toString());
       o.addProperty(FILE, obj.getKey());
       o.addProperty(VALUE_SIZE, obj.getSize());
       succ.add(o);
     } catch (IOException e) {
       JsonObject o = new JsonObject();
       o.addProperty(FILE, obj.getKey());
       o.addProperty(ERROR, e.getMessage());
       fail.add(o);
     }
   }
 }
Exemple #21
0
  @Override
  public void remove(String noteId, AuthenticationInfo subject) throws IOException {
    String key = user + "/" + "notebook" + "/" + noteId;
    final ListObjectsRequest listObjectsRequest =
        new ListObjectsRequest().withBucketName(bucketName).withPrefix(key);

    try {
      ObjectListing objects = s3client.listObjects(listObjectsRequest);
      do {
        for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
          s3client.deleteObject(bucketName, objectSummary.getKey());
        }
        objects = s3client.listNextBatchOfObjects(objects);
      } while (objects.isTruncated());
    } catch (AmazonClientException ace) {
      throw new IOException("Unable to remove note in S3: " + ace, ace);
    }
  }
  @Override
  public void recursiveDelete(String keyPrefix, FilenameFilter except) throws IOException {

    if (keyPrefix.charAt(keyPrefix.length() - 1) == '/') {
      // Need to delete the dir too, and to list it, can't specify its trailing slash
      keyPrefix = keyPrefix.substring(0, keyPrefix.length() - 1);
    }

    boolean truncated = true;
    String marker = null;

    String bucket = Namespaces.get().getBucket();
    while (truncated) {

      ListObjectsRequest listRequest =
          new ListObjectsRequest(bucket, keyPrefix, marker, null, null);
      ObjectListing listing;
      try {
        listing = s3Client.listObjects(listRequest);
        Collection<S3ObjectSummary> summaries = listing.getObjectSummaries();
        if (summaries.isEmpty()) {
          return;
        }

        List<DeleteObjectsRequest.KeyVersion> keysToDelete =
            Lists.newArrayListWithCapacity(summaries.size());
        for (S3ObjectSummary summary : summaries) {
          String key = summary.getKey();
          if (except == null || !except.accept(null, key)) {
            keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key));
          }
        }

        DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket);
        deleteObjectsRequest.setKeys(keysToDelete);
        s3Client.deleteObjects(deleteObjectsRequest);
      } catch (AmazonClientException ace) {
        throw new IOException(ace);
      }

      truncated = listing.isTruncated();
      marker = listing.getNextMarker();
    }
  }
  private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {

    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(s3BucketName);
    // listRequest.setGeneralProgressListener(this);
    listRequest.setMaxKeys(1000);

    String nextMarker = null;
    ObjectListing objectListing = null;

    while (true) {

      objectListing = s3Client.listObjects(listRequest);

      List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();

      for (S3ObjectSummary objSummary : objectSummaries) {
        String key = objSummary.getKey();

        TocInfo tocInfo = new TocInfo(key, objSummary.getSize());

        // is it a "dir/" ?
        if (key.lastIndexOf("/") == (key.length() - 1)) {
          tocInfo.isDirectory = true;
        } else {
          tocInfo.isDirectory = false;
        }

        toc.add(tocInfo);
        tocQueue.add(tocInfo);
        tocInfosGenerated++; // increment for logging
      }

      // for pagination
      nextMarker = objectListing.getNextMarker();
      if (nextMarker == null) {
        break;
      } else {
        listRequest.setMarker(nextMarker);
        logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
      }
    }
  }
  @Override
  public InputStream next() {
    if (currObject < currObjects.size()) {
      InputStream ret = s3.objectForKey(bucket, currObjects.get(currObject).getKey());
      currObject++;
      return ret;
    } else if (currList.isTruncated()) {
      currList = s3.nextList(currList);
      currObjects = currList.getObjectSummaries();
      currObject = 0;

      InputStream ret = s3.objectForKey(bucket, currObjects.get(currObject).getKey());

      currObject++;
      return ret;
    }

    return null;
  }
 @Override
 public List<String> list(String prefix, boolean files) throws IOException {
   boolean truncated = true;
   String marker = null;
   List<String> result = Lists.newArrayList();
   while (truncated) {
     if (marker != null) {
       log.info("Next page after {}", marker);
     }
     ListObjectsRequest listRequest =
         new ListObjectsRequest(Namespaces.get().getBucket(), prefix, marker, "/", null);
     ObjectListing listing;
     try {
       listing = s3Client.listObjects(listRequest);
     } catch (AmazonClientException ace) {
       throw new IOException(ace);
     }
     truncated = listing.isTruncated();
     marker = listing.getNextMarker();
     if (!truncated) {
       if (files) {
         for (S3ObjectSummary summary : listing.getObjectSummaries()) {
           String key = summary.getKey();
           if (!key.endsWith("_SUCCESS")) {
             result.add(key);
           }
         }
       } else {
         for (String key : listing.getCommonPrefixes()) {
           if (!key.endsWith("_SUCCESS")) {
             result.add(key);
           }
         }
       }
     }
   }
   Collections.sort(result);
   return result;
 }
  public static void main(String[] args) {

    if (args.length < 2) {
      System.out.println("USAGE: localPath bucketname <bucketPrefix>");
      System.exit(1);
    }

    String localPath = args[0];
    String bucketName = args[1];
    String bucketPrefix = "";

    // add on extra slash
    if (!localPath.endsWith("/")) localPath += "/";

    if (args.length == 3) bucketPrefix = args[2];

    // check local dir, make if it does not exist
    File localDir = new File(localPath);
    if (!localDir.exists()) localDir.mkdirs();

    if (!localDir.isDirectory()) {
      System.out.println("Local Dir is not a dir: " + localPath);
      System.exit(1);
    }

    long totalBytes = 0;
    long start = System.currentTimeMillis();

    AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    ObjectListing listObjects = s3.listObjects(bucketName, bucketPrefix);
    do {
      for (S3ObjectSummary objectSummary : listObjects.getObjectSummaries()) {
        S3Object object = s3.getObject(bucketName, objectSummary.getKey());
        S3ObjectInputStream inputStream = object.getObjectContent();
        if ("gzip".equals(object.getObjectMetadata().getContentEncoding())) {
          InputStream in = null;
          try {

            totalBytes += object.getObjectMetadata().getContentLength();

            in = new GZIPInputStream(inputStream);
            // write this sucker out
            String path = localPath + object.getKey();
            // have to take the gz off since this is not downloading compressed!
            if (path.endsWith(".gz")) path = path.substring(0, path.length() - 3);
            System.out.print("Writing file: " + path);
            File check = new File(path);
            File parentFile = check.getParentFile();
            if (!parentFile.exists()) parentFile.mkdirs();

            FileOutputStream out = new FileOutputStream(path);
            IOUtils.copy(in, out);
            System.out.println(" written.");

          } catch (IOException e) {
            System.out.println("crap");
            e.printStackTrace();
            throw new IllegalStateException("files are too hard", e);
          } finally {
            IOUtils.closeQuietly(in);
          }
        } else {
          System.out.println(
              "unhandled content encoding: " + object.getObjectMetadata().getContentEncoding());
        }
      }

      listObjects = s3.listNextBatchOfObjects(listObjects);
    } while (listObjects.isTruncated());

    long now = System.currentTimeMillis();
    System.out.println(
        (totalBytes / 1000.0 / 1000.0)
            + " mb downloaded in "
            + ((now - start) / 1000)
            + " seconds.");
  }
  /** Deletes content of the repository files in the bucket */
  public void cleanRepositoryFiles(String basePath) {
    Settings settings = internalCluster().getInstance(Settings.class);
    Settings[] buckets = {
      settings.getByPrefix("repositories.s3."),
      settings.getByPrefix("repositories.s3.private-bucket."),
      settings.getByPrefix("repositories.s3.remote-bucket."),
      settings.getByPrefix("repositories.s3.external-bucket.")
    };
    for (Settings bucket : buckets) {
      String endpoint = bucket.get("endpoint", settings.get("repositories.s3.endpoint"));
      String protocol = bucket.get("protocol", settings.get("repositories.s3.protocol"));
      String region = bucket.get("region", settings.get("repositories.s3.region"));
      String accessKey = bucket.get("access_key", settings.get("cloud.aws.access_key"));
      String secretKey = bucket.get("secret_key", settings.get("cloud.aws.secret_key"));
      String bucketName = bucket.get("bucket");

      // We check that settings has been set in elasticsearch.yml integration test file
      // as described in README
      assertThat(
          "Your settings in elasticsearch.yml are incorrects. Check README file.",
          bucketName,
          notNullValue());
      AmazonS3 client =
          internalCluster()
              .getInstance(AwsS3Service.class)
              .client(endpoint, protocol, region, accessKey, secretKey);
      try {
        ObjectListing prevListing = null;
        // From
        // http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html
        // we can do at most 1K objects per delete
        // We don't know the bucket name until first object listing
        DeleteObjectsRequest multiObjectDeleteRequest = null;
        ArrayList<DeleteObjectsRequest.KeyVersion> keys =
            new ArrayList<DeleteObjectsRequest.KeyVersion>();
        while (true) {
          ObjectListing list;
          if (prevListing != null) {
            list = client.listNextBatchOfObjects(prevListing);
          } else {
            list = client.listObjects(bucketName, basePath);
            multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
          }
          for (S3ObjectSummary summary : list.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
            // Every 500 objects batch the delete request
            if (keys.size() > 500) {
              multiObjectDeleteRequest.setKeys(keys);
              client.deleteObjects(multiObjectDeleteRequest);
              multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
              keys.clear();
            }
          }
          if (list.isTruncated()) {
            prevListing = list;
          } else {
            break;
          }
        }
        if (!keys.isEmpty()) {
          multiObjectDeleteRequest.setKeys(keys);
          client.deleteObjects(multiObjectDeleteRequest);
        }
      } catch (Throwable ex) {
        logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
      }
    }
  }
Exemple #28
0
  public static void main(String[] args) throws IOException {
    /*
     * This credentials provider implementation loads your AWS credentials
     * from a properties file at the root of your classpath.
     *
     * Important: Be sure to fill in your AWS access credentials in the
     *            AwsCredentials.properties file before you try to run this
     *            sample.
     * http://aws.amazon.com/security-credentials
     */
    AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    String bucketName = "my-first-s3-bucket-" + UUID.randomUUID();
    String key = "MyObjectKey";

    System.out.println("===========================================");
    System.out.println("Getting Started with Amazon S3");
    System.out.println("===========================================\n");

    try {
      /*
       * Create a new S3 bucket - Amazon S3 bucket names are globally unique,
       * so once a bucket name has been taken by any user, you can't create
       * another bucket with that same name.
       *
       * You can optionally specify a location for your bucket if you want to
       * keep your data closer to your applications or users.
       */
      System.out.println("Creating bucket " + bucketName + "\n");
      s3.createBucket(bucketName);

      /*
       * List the buckets in your account
       */
      System.out.println("Listing buckets");
      for (Bucket bucket : s3.listBuckets()) {
        System.out.println(" - " + bucket.getName());
      }
      System.out.println();

      /*
       * Upload an object to your bucket - You can easily upload a file to
       * S3, or upload directly an InputStream if you know the length of
       * the data in the stream. You can also specify your own metadata
       * when uploading to S3, which allows you set a variety of options
       * like content-type and content-encoding, plus additional metadata
       * specific to your applications.
       */
      System.out.println("Uploading a new object to S3 from a file\n");
      s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile()));

      /*
       * Download an object - When you download an object, you get all of
       * the object's metadata and a stream from which to read the contents.
       * It's important to read the contents of the stream as quickly as
       * possibly since the data is streamed directly from Amazon S3 and your
       * network connection will remain open until you read all the data or
       * close the input stream.
       *
       * GetObjectRequest also supports several other options, including
       * conditional downloading of objects based on modification times,
       * ETags, and selectively downloading a range of an object.
       */
      System.out.println("Downloading an object");
      S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
      System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
      displayTextInputStream(object.getObjectContent());

      /*
       * List objects in your bucket by prefix - There are many options for
       * listing the objects in your bucket.  Keep in mind that buckets with
       * many objects might truncate their results when listing their objects,
       * so be sure to check if the returned object listing is truncated, and
       * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
       * additional results.
       */
      System.out.println("Listing objects");
      ObjectListing objectListing =
          s3.listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        System.out.println(
            " - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
      }
      System.out.println();

      /*
       * Delete an object - Unless versioning has been turned on for your bucket,
       * there is no way to undelete an object, so use caution when deleting objects.
       */
      System.out.println("Deleting an object\n");
      s3.deleteObject(bucketName, key);

      /*
       * Delete a bucket - A bucket must be completely empty before it can be
       * deleted, so remember to delete any objects from your buckets before
       * you try to delete them.
       */
      System.out.println("Deleting bucket " + bucketName + "\n");
      s3.deleteBucket(bucketName);
    } catch (AmazonServiceException ase) {
      System.out.println(
          "Caught an AmazonServiceException, which means your request made it "
              + "to Amazon S3, but was rejected with an error response for some reason.");
      System.out.println("Error Message:    " + ase.getMessage());
      System.out.println("HTTP Status Code: " + ase.getStatusCode());
      System.out.println("AWS Error Code:   " + ase.getErrorCode());
      System.out.println("Error Type:       " + ase.getErrorType());
      System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
      System.out.println(
          "Caught an AmazonClientException, which means the client encountered "
              + "a serious internal problem while trying to communicate with S3, "
              + "such as not being able to access the network.");
      System.out.println("Error Message: " + ace.getMessage());
    }
  }
 private Iterator<LocatedFileStatus> statusFromListing(ObjectListing listing) {
   return Iterators.concat(
       statusFromPrefixes(listing.getCommonPrefixes()),
       statusFromObjects(listing.getObjectSummaries()));
 }