Ejemplo n.º 1
0
  @ResponseBody
  @RequestMapping(value = "/admin/upload.ajax", method = RequestMethod.POST)
  public ResponseEntity<byte[]> upload(
      @RequestParam("restaurantId") String restaurantId,
      @RequestParam("file") CommonsMultipartFile file)
      throws Exception {

    Map<String, Object> model = new HashMap<String, Object>();

    try {
      S3Object object = new S3Object(basePath + "/" + restaurantId);
      object.setDataInputStream(file.getInputStream());
      object.setContentLength(file.getSize());
      object.setContentType(file.getContentType());
      S3Bucket bucket = s3Service.getBucket(bucketName);
      s3Service.putObject(bucket, object);

      Restaurant restaurant = restaurantRepository.findByRestaurantId(restaurantId);
      restaurant.setHasUploadedImage(true);
      restaurantRepository.saveRestaurant(restaurant);

      model.put("success", true);
    } catch (Exception ex) {
      LOGGER.error("", ex);
      model.put("success", false);
      model.put("message", ex.getMessage());
    }

    String json = jsonUtils.serializeAndEscape(model);
    final HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.TEXT_HTML);
    headers.setCacheControl("no-cache");
    return new ResponseEntity<byte[]>(json.getBytes("utf-8"), headers, HttpStatus.OK);
  }
 @Override
 protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
     throws Exception {
   org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
   object.setContentType(contentType);
   object.setDataInputStream(data);
   object.setContentLength(data.available());
   return jetClient.putObject(bucket, object) != null;
 }
 @Override
 public long getLastModified(DataSegment segment) throws SegmentLoadingException {
   S3Coords coords = new S3Coords(segment);
   try {
     S3Object objDetails = s3Client.getObjectDetails(new S3Bucket(coords.bucket), coords.path);
     return objDetails.getLastModifiedDate().getTime();
   } catch (S3ServiceException e) {
     throw new SegmentLoadingException(e, e.getMessage());
   }
 }
Ejemplo n.º 4
0
 public void startElement(String uri, String name, String qName, Attributes attrs) {
   if (name.equals("Contents")) {
     currentObject = new S3Object(null);
     currentObject.setBucketName(bucketName);
   } else if (name.equals("Owner")) {
     currentOwner = new S3Owner();
     currentObject.setOwner(currentOwner);
   } else if (name.equals("CommonPrefixes")) {
     insideCommonPrefixes = true;
   }
 }
  @Override
  public void getSegmentFiles(DataSegment segment, File outDir) throws SegmentLoadingException {
    S3Coords s3Coords = new S3Coords(segment);

    log.info("Pulling index at path[%s] to outDir[%s]", s3Coords, outDir);

    if (!isObjectInBucket(s3Coords)) {
      throw new SegmentLoadingException("IndexFile[%s] does not exist.", s3Coords);
    }

    if (!outDir.exists()) {
      outDir.mkdirs();
    }

    if (!outDir.isDirectory()) {
      throw new ISE("outDir[%s] must be a directory.", outDir);
    }

    long startTime = System.currentTimeMillis();
    S3Object s3Obj = null;

    try {
      s3Obj = s3Client.getObject(new S3Bucket(s3Coords.bucket), s3Coords.path);

      InputStream in = null;
      try {
        in = s3Obj.getDataInputStream();
        final String key = s3Obj.getKey();
        if (key.endsWith(".zip")) {
          CompressionUtils.unzip(in, outDir);
        } else if (key.endsWith(".gz")) {
          final File outFile = new File(outDir, toFilename(key, ".gz"));
          ByteStreams.copy(new GZIPInputStream(in), Files.newOutputStreamSupplier(outFile));
        } else {
          ByteStreams.copy(
              in, Files.newOutputStreamSupplier(new File(outDir, toFilename(key, ""))));
        }
        log.info(
            "Pull of file[%s] completed in %,d millis",
            s3Obj, System.currentTimeMillis() - startTime);
      } catch (IOException e) {
        FileUtils.deleteDirectory(outDir);
        throw new SegmentLoadingException(e, "Problem decompressing object[%s]", s3Obj);
      } finally {
        Closeables.closeQuietly(in);
      }
    } catch (Exception e) {
      throw new SegmentLoadingException(e, e.getMessage());
    } finally {
      S3Utils.closeStreamsQuietly(s3Obj);
    }
  }
Ejemplo n.º 6
0
 private InputStream get(String key) throws IOException {
   try {
     S3Object object = s3Service.getObject(bucket, key);
     return object.getDataInputStream();
   } catch (S3ServiceException e) {
     if ("NoSuchKey".equals(e.getS3ErrorCode())) {
       return null;
     }
     if (e.getCause() instanceof IOException) {
       throw (IOException) e.getCause();
     }
     throw new S3Exception(e);
   }
 }
Ejemplo n.º 7
0
 /**
  * Creates a directory flagged file with the key and folder suffix.
  *
  * @param key the key to create a folder
  * @return true if the operation was successful, false otherwise
  */
 private boolean mkdirsInternal(String key) {
   try {
     String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
     S3Object obj = new S3Object(keyAsFolder);
     obj.setDataInputStream(new ByteArrayInputStream(new byte[0]));
     obj.setContentLength(0);
     obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM);
     mClient.putObject(mBucketName, obj);
     return true;
   } catch (ServiceException se) {
     LOG.error("Failed to create directory: " + key, se);
     return false;
   }
 }
Ejemplo n.º 8
0
  public int run(String[] args) throws Exception {

    if (args.length == 0) {
      System.err.println("Usage: MigrationTool <S3 file system URI>");
      System.err.println("\t<S3 file system URI>\tfilesystem to migrate");
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    URI uri = URI.create(args[0]);

    initialize(uri);

    FileSystemStore newStore = new Jets3tFileSystemStore();
    newStore.initialize(uri, getConf());

    if (get("%2F") != null) {
      System.err.println("Current version number is [unversioned].");
      System.err.println("Target version number is " + newStore.getVersion() + ".");
      Store oldStore = new UnversionedStore();
      migrate(oldStore, newStore);
      return 0;
    } else {
      S3Object root = get("/");
      if (root != null) {
        String version = (String) root.getMetadata("fs-version");
        if (version == null) {
          System.err.println("Can't detect version - exiting.");
        } else {
          String newVersion = newStore.getVersion();
          System.err.println("Current version number is " + version + ".");
          System.err.println("Target version number is " + newVersion + ".");
          if (version.equals(newStore.getVersion())) {
            System.err.println("No migration required.");
            return 0;
          }
          // use version number to create Store
          // Store oldStore = ...
          // migrate(oldStore, newStore);
          System.err.println("Not currently implemented.");
          return 0;
        }
      }
      System.err.println("Can't detect version - exiting.");
      return 0;
    }
  }
Ejemplo n.º 9
0
  /** Event handler for this dialog. */
  public void actionPerformed(ActionEvent e) {
    // Force table to accept any partial edits.
    if (metadataTable.isEditing()) {
      metadataTable.getCellEditor().stopCellEditing();
    }

    // Apply new/modified attributes to the object.
    Map currentObjectMetadata = currentObject.getModifiableMetadata();
    Set obsoleteMetadataItems = currentObjectMetadata.keySet();
    for (int row = 0; row < metadataTable.getRowCount(); row++) {
      String name = (String) objectMetadataTableModel.getValueAt(row, 0);
      String value = (String) objectMetadataTableModel.getValueAt(row, 1);
      currentObject.addMetadata(name, value);
      obsoleteMetadataItems.remove(name);
    }
    // Remove obsolete attributes.
    Iterator obsoleteNamesIter = obsoleteMetadataItems.iterator();
    while (obsoleteNamesIter.hasNext()) {
      currentObject.removeMetadata((String) obsoleteNamesIter.next());
    }

    if (e.getSource().equals(nextObjectButton)) {
      currentObjectIndex++;
      displayObjectProperties();
    } else if (e.getSource().equals(previousObjectButton)) {
      currentObjectIndex--;
      displayObjectProperties();
    } else if ("OK".equals(e.getActionCommand())) {
      modifyActionApproved = isModifyMode();
      this.setVisible(false);
    } else if ("Cancel".equals(e.getActionCommand())) {
      modifyActionApproved = false;
      this.setVisible(false);
    } else if ("addMetadataItem".equals(e.getActionCommand())) {
      int newRowNumber = metadataTable.getRowCount() + 1;
      objectMetadataTableModel.addRow(
          new String[] {"name-" + newRowNumber, "value-" + newRowNumber});
    } else if ("removeMetadataItem".equals(e.getActionCommand())) {
      int[] rows = metadataTable.getSelectedRows();
      for (int i = rows.length - 1; i >= 0; i--) {
        int modelIndex = metadataTableSorter.modelIndex(rows[i]);
        objectMetadataTableModel.removeRow(modelIndex);
      }
    }
  }
Ejemplo n.º 10
0
 @Override
 public long getLastModified(DataSegment segment) throws SegmentLoadingException {
   final S3Coords coords = new S3Coords(segment);
   try {
     final S3Object objDetails =
         S3Utils.retryS3Operation(
             new Callable<S3Object>() {
               @Override
               public S3Object call() throws Exception {
                 return s3Client.getObjectDetails(new S3Bucket(coords.bucket), coords.path);
               }
             });
     return objDetails.getLastModifiedDate().getTime();
   } catch (S3ServiceException | IOException e) {
     throw new SegmentLoadingException(e, e.getMessage());
   } catch (Exception e) {
     throw Throwables.propagate(e);
   }
 }
Ejemplo n.º 11
0
 /**
  * Lists the files in the given path, the paths will be their logical names and not contain the
  * folder suffix.
  *
  * @param path the key to list
  * @param recursive if true will list children directories as well
  * @return an array of the file and folder names in this directory
  * @throws IOException if an I/O error occurs
  */
 private String[] listInternal(String path, boolean recursive) throws IOException {
   try {
     path = stripPrefixIfPresent(path);
     path = PathUtils.normalizePath(path, PATH_SEPARATOR);
     path = path.equals(PATH_SEPARATOR) ? "" : path;
     // Gets all the objects under the path, because we have no idea if there are non Alluxio
     // managed "directories"
     S3Object[] objs = mClient.listObjects(mBucketName, path, "");
     if (recursive) {
       List<String> ret = new ArrayList<>();
       for (S3Object obj : objs) {
         // Remove parent portion of the key
         String child = getChildName(obj.getKey(), path);
         // Prune the special folder suffix
         child = stripFolderSuffixIfPresent(child);
         // Only add if the path is not empty (removes results equal to the path)
         if (!child.isEmpty()) {
           ret.add(child);
         }
       }
       return ret.toArray(new String[ret.size()]);
     }
     // Non recursive list
     Set<String> children = new HashSet<String>();
     for (S3Object obj : objs) {
       // Remove parent portion of the key
       String child = getChildName(obj.getKey(), path);
       // Remove any portion after the path delimiter
       int childNameIndex = child.indexOf(PATH_SEPARATOR);
       child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
       // Prune the special folder suffix
       child = stripFolderSuffixIfPresent(child);
       // Add to the set of children, the set will deduplicate.
       if (!child.isEmpty()) {
         children.add(child);
       }
     }
     return children.toArray(new String[children.size()]);
   } catch (ServiceException e) {
     LOG.error("Failed to list path {}", path, e);
     return null;
   }
 }
Ejemplo n.º 12
0
  public static void main(String[] args) {

    Storage st = new Storage(args[0], args[1], args[2]);

    try {
      String awsAccessKey = "AKIAIXDL4AVDSSMH4JJQ";
      String awsSecretKey = "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";
      AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);

      S3Service s3Service = new RestS3Service(awsCredentials);
      S3Object list[] = s3Service.listObjects("pagerankmam");
      for (int i = 0; i < list.length; i++) {
        if (list[i].getName().startsWith("input/output/")) {
          // Open the file that is the first
          // command line parameter

          // System.out.println(list[i].getName());
          S3Object objectComplete =
              s3Service.getObject("pagerankmam/input/output", list[i].getName().split("/")[2]);
          // System.out.println("S3Object, complete: " + objectComplete);

          // Read the data from the object's DataInputStream using a loop, and print it out.
          // System.out.println("Greeting:");
          BufferedReader reader =
              new BufferedReader(new InputStreamReader(objectComplete.getDataInputStream()));
          String data = null;
          st.connect();
          while ((data = reader.readLine()) != null) {
            String[] results = data.split("\t");
            st.insertPageRankRow(
                Integer.parseInt(results[0]), Double.parseDouble(results[1].split(" ")[0]));
            // System.out.println(Integer.parseInt(results[0])+" -
            // "+Double.parseDouble(results[1].split(" ")[0]));
            // System.out.println(data);
          }
          st.close();
        }
      }
    } catch (Exception e) { // Catch exception if any
      System.err.println("Error: " + e.getMessage());
    }
  }
  @Test
  public void testSimpleGetVersion() throws ServiceException, IOException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);

    S3Object object0 = new S3Object();

    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));

    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(bucket), EasyMock.eq(object0.getKey())))
        .andReturn(object0)
        .once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);

    EasyMock.replay(s3Client);

    String version =
        puller.getVersion(URI.create(String.format("s3://%s/%s", bucket, object0.getKey())));

    EasyMock.verify(s3Client);

    Assert.assertEquals(String.format("%d", new Date(0).getTime()), version);
  }
Ejemplo n.º 14
0
  /**
   * Update the dialog to display the attributes of a single object. The user may choose which
   * object to display by iterating forward and back through the set of objects.
   */
  private void displayObjectProperties() {
    currentObject = destinationObjects[currentObjectIndex];

    // Manage previous/next buttons.
    if (destinationObjects.length > 1) {
      currentObjectLabel.setText((currentObjectIndex + 1) + " of " + destinationObjects.length);
      previousObjectButton.setEnabled(currentObjectIndex > 0);
      nextObjectButton.setEnabled(currentObjectIndex < (destinationObjects.length - 1));
    }

    // Unmodifiable fields.
    objectKeyTextField.setText(currentObject.getKey());
    objectContentLengthTextField.setText(String.valueOf(currentObject.getContentLength()));
    objectLastModifiedTextField.setText(String.valueOf(currentObject.getLastModifiedDate()));
    objectETagTextField.setText(currentObject.getETag());
    bucketLocationTextField.setText(currentObject.getBucketName());

    if (currentObject.getOwner() != null) {
      ownerNameLabel.setVisible(true);
      ownerNameTextField.setVisible(true);
      ownerIdLabel.setVisible(true);
      ownerIdTextField.setVisible(true);
      ownerNameTextField.setText(currentObject.getOwner().getDisplayName());
      ownerIdTextField.setText(currentObject.getOwner().getId());
    } else {
      ownerNameLabel.setVisible(false);
      ownerNameTextField.setVisible(false);
      ownerIdLabel.setVisible(false);
      ownerIdTextField.setVisible(false);
    }

    // Clear old table contents
    while (objectMetadataTableModel.getRowCount() > 0) {
      objectMetadataTableModel.removeRow(0);
    }

    // Display remaining metadata items in the table.
    Iterator mdIter = currentObject.getModifiableMetadata().entrySet().iterator();
    while (mdIter.hasNext()) {
      Map.Entry entry = (Map.Entry) mdIter.next();
      Object name = entry.getKey();
      Object value = entry.getValue();
      objectMetadataTableModel.addRow(new Object[] {name, value});
    }
  }
Ejemplo n.º 15
0
 /**
  * Lists the files in the given path, the paths will be their logical names and not contain the
  * folder suffix
  *
  * @param path the key to list
  * @param recursive if true will list children directories as well
  * @return an array of the file and folder names in this directory
  * @throws IOException
  */
 private String[] listInternal(String path, boolean recursive) throws IOException {
   try {
     path = stripPrefixIfPresent(path);
     path = path.endsWith(PATH_SEPARATOR) ? path : path + PATH_SEPARATOR;
     path = path.equals(PATH_SEPARATOR) ? "" : path;
     // Gets all the objects under the path, because we have no idea if there are non Tachyon
     // managed "directories"
     S3Object[] objs = mClient.listObjects(mBucketName, path, "");
     if (recursive) {
       String[] ret = new String[objs.length];
       for (int i = 0; i < objs.length; i++) {
         // Remove parent portion of the key
         String child = getChildName(objs[i].getKey(), path);
         // Prune the special folder suffix
         child = stripFolderSuffixIfPresent(child);
         ret[i] = child;
       }
       return ret;
     }
     // Non recursive list
     Set<String> children = new HashSet<String>();
     for (S3Object obj : objs) {
       // Remove parent portion of the key
       String child = getChildName(obj.getKey(), path);
       // Remove any portion after the path delimiter
       int childNameIndex = child.indexOf(PATH_SEPARATOR);
       child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
       // Prune the special folder suffix
       child = stripFolderSuffixIfPresent(child);
       // Add to the set of children, the set will deduplicate.
       children.add(child);
     }
     return children.toArray(new String[children.size()]);
   } catch (ServiceException se) {
     LOG.error("Failed to list path " + path);
     return null;
   }
 }
  @Test
  public void testGZUncompress() throws ServiceException, IOException, SegmentLoadingException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    final byte[] value = bucket.getBytes("utf8");

    final File tmpFile = temporaryFolder.newFile("gzTest.gz");

    try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
      outputStream.write(value);
    }

    final S3Object object0 = new S3Object();

    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object0.setDataInputStream(new FileInputStream(tmpFile));

    final File tmpDir = temporaryFolder.newFolder("gzTestDir");

    EasyMock.expect(
            s3Client.getObjectDetails(
                EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
        .andReturn(null)
        .once();
    EasyMock.expect(
            s3Client.getObjectDetails(
                EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
        .andReturn(object0)
        .once();
    EasyMock.expect(
            s3Client.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
        .andReturn(object0)
        .once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);

    EasyMock.replay(s3Client);
    FileUtils.FileCopyResult result =
        puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, object0.getKey()), tmpDir);
    EasyMock.verify(s3Client);

    Assert.assertEquals(value.length, result.size());
    File expected = new File(tmpDir, "renames-0");
    Assert.assertTrue(expected.exists());
    Assert.assertEquals(value.length, expected.length());
  }
Ejemplo n.º 17
0
    protected URLConnection openConnection(URL u)
    throws IOException {
        // This looking for accessKey id and accessKey secret code is based
        // on code from hadoop S3.
        String accessKey = null;
        String secretAccessKey = null;
        String userInfo = u.getUserInfo();
        if (userInfo != null) {
            int index = userInfo.indexOf(':');
            if (index != -1) {
              accessKey = userInfo.substring(0, index);
              secretAccessKey = userInfo.substring(index + 1);
            } else {
              accessKey = userInfo;
            }
        }
        if (accessKey == null) {
          accessKey = System.getProperty("aws.access.key.id");
        }
        if (secretAccessKey == null) {
          secretAccessKey = System.getProperty("aws.access.key.secret");
        }
        if (accessKey == null && secretAccessKey == null) {
          throw new IllegalArgumentException("AWS " +
                "Access Key ID and Secret Access Key " +
                "must be specified as the username " +
                "or password (respectively) of a s3 URL, " +
                "or by setting the " +
                "aws.access.key.id or " +                
                "aws.access.key.secret properties (respectively).");
        } else if (accessKey == null) {
          throw new IllegalArgumentException("AWS " +
                "Access Key ID must be specified " +
                "as the username of a s3 URL, or by setting the " +
                "aws.access.key.id property.");
        } else if (secretAccessKey == null) {
          throw new IllegalArgumentException("AWS " +
                "Secret Access Key must be specified " +
                "as the password of a s3 URL, or by setting the " +
                "aws.access.key.secret property.");        
        }
        
        RestS3Service s3Service;
        try {
            s3Service = new RestS3Service(
                new AWSCredentials(accessKey, secretAccessKey));
        } catch (S3ServiceException e) {
            e.printStackTrace();
            throw new IOException(e.toString());
        }
        InputStream is = null;
        try {
            // This opens the stream to the bucket/key object.
            S3Object s3obj = s3Service.getObject(new S3Bucket(u.getHost()),
                u.getPath().substring(1) /* Skip starting '/' character */);
            is = s3obj.getDataInputStream();
        } catch (S3ServiceException e) {
            e.printStackTrace();
            throw new IOException(e.toString());
        }

        final InputStream inputStream = is;
        return new URLConnection(u) {
            private InputStream is = inputStream;
            @Override
            public InputStream getInputStream() throws IOException {
                return this.is;
            }
            @Override
            public void connect() throws IOException {
                // Nothing to do. When we give back this object, we're
                // connected.
            }
        };
    }
Ejemplo n.º 18
0
    public void endElement(String uri, String name, String qName) {
      String elementText = this.currText.toString();
      // Listing details
      if (name.equals("Name")) {
        bucketName = elementText;
        if (log.isDebugEnabled()) {
          log.debug("Examining listing for bucket: " + bucketName);
        }
      } else if (!insideCommonPrefixes && name.equals("Prefix")) {
        requestPrefix = elementText;
      } else if (name.equals("Marker")) {
        requestMarker = elementText;
      } else if (name.equals("NextMarker")) {
        nextMarker = elementText;
      } else if (name.equals("MaxKeys")) {
        requestMaxKeys = Long.parseLong(elementText);
      } else if (name.equals("IsTruncated")) {
        String isTruncatedStr = elementText.toLowerCase(Locale.getDefault());
        if (isTruncatedStr.startsWith("false")) {
          listingTruncated = false;
        } else if (isTruncatedStr.startsWith("true")) {
          listingTruncated = true;
        } else {
          throw new RuntimeException("Invalid value for IsTruncated field: " + isTruncatedStr);
        }
      }
      // Object details.
      else if (name.equals("Contents")) {
        objects.add(currentObject);
        if (log.isDebugEnabled()) {
          log.debug("Created new S3Object from listing: " + currentObject);
        }
      } else if (name.equals("Key")) {
        currentObject.setKey(elementText);
        lastKey = elementText;
      } else if (name.equals("LastModified")) {
        try {
          currentObject.setLastModifiedDate(ServiceUtils.parseIso8601Date(elementText));
        } catch (ParseException e) {
          throw new RuntimeException("Unexpected date format in list bucket output", e);
        }
      } else if (name.equals("ETag")) {
        currentObject.setETag(elementText);
      } else if (name.equals("Size")) {
        currentObject.setContentLength(Long.parseLong(elementText));
      } else if (name.equals("StorageClass")) {
        currentObject.setStorageClass(elementText);
      }
      // Owner details.
      else if (name.equals("ID")) {
        // Work-around to support Eucalyptus responses, which do not
        // contain Owner elements.
        if (currentOwner == null) {
          currentOwner = new S3Owner();
          currentObject.setOwner(currentOwner);
        }

        currentOwner.setId(elementText);
      } else if (name.equals("DisplayName")) {
        currentOwner.setDisplayName(elementText);
      }
      // Common prefixes.
      else if (insideCommonPrefixes && name.equals("Prefix")) {
        commonPrefixes.add(elementText);
      } else if (name.equals("CommonPrefixes")) {
        insideCommonPrefixes = false;
      }

      this.currText = new StringBuffer();
    }
Ejemplo n.º 19
0
  /*
   * Fetch a file that is in a S3 file system. Return a local File. It accepts "s3://" and "s3n://" prefixes.
   */
  private File s3Fetch(URI uri, Reporter reporter) throws IOException {
    String bucketName = uri.getHost();
    String path = uri.getPath();

    File destFolder = new File(tempDir, bucketName + "/" + path);
    if (destFolder.exists()) {
      FileUtils.deleteDirectory(destFolder);
    }
    destFolder.mkdirs();

    Throttler throttler = new Throttler((double) bytesPerSecThrottle);

    boolean done = false;
    try {
      s3Service = new RestS3Service(getCredentials());
      if (s3Service.checkBucketStatus(bucketName) != RestS3Service.BUCKET_STATUS__MY_BUCKET) {
        throw new IOException("Bucket doesn't exist or is already claimed: " + bucketName);
      }

      if (path.startsWith("/")) {
        path = path.substring(1, path.length());
      }

      for (S3Object object : s3Service.listObjects(new S3Bucket(bucketName), path, "")) {
        long bytesSoFar = 0;

        String fileName = path;
        if (path.contains("/")) {
          fileName = path.substring(path.lastIndexOf("/") + 1, path.length());
        }
        File fileDest = new File(destFolder, fileName);
        log.info("Downloading " + object.getKey() + " to " + fileDest + " ...");

        if (fileDest.exists()) {
          fileDest.delete();
        }

        object = s3Service.getObject(new S3Bucket(bucketName), object.getKey());
        InputStream iS = object.getDataInputStream();
        FileOutputStream writer = new FileOutputStream(fileDest);
        byte[] buffer = new byte[downloadBufferSize];

        int nRead;
        while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
          bytesSoFar += nRead;
          writer.write(buffer, 0, nRead);
          throttler.incrementAndThrottle(nRead);
          if (bytesSoFar >= bytesToReportProgress) {
            reporter.progress(bytesSoFar);
            bytesSoFar = 0l;
          }
        }

        if (reporter != null) {
          reporter.progress(bytesSoFar);
        }

        writer.close();
        iS.close();
        done = true;
      }

      if (!done) {
        throw new IOException("Bucket is empty! " + bucketName + " path: " + path);
      }
    } catch (S3ServiceException e) {
      throw new IOException(e);
    }

    return destFolder;
  }