@ResponseBody @RequestMapping(value = "/admin/upload.ajax", method = RequestMethod.POST) public ResponseEntity<byte[]> upload( @RequestParam("restaurantId") String restaurantId, @RequestParam("file") CommonsMultipartFile file) throws Exception { Map<String, Object> model = new HashMap<String, Object>(); try { S3Object object = new S3Object(basePath + "/" + restaurantId); object.setDataInputStream(file.getInputStream()); object.setContentLength(file.getSize()); object.setContentType(file.getContentType()); S3Bucket bucket = s3Service.getBucket(bucketName); s3Service.putObject(bucket, object); Restaurant restaurant = restaurantRepository.findByRestaurantId(restaurantId); restaurant.setHasUploadedImage(true); restaurantRepository.saveRestaurant(restaurant); model.put("success", true); } catch (Exception ex) { LOGGER.error("", ex); model.put("success", false); model.put("message", ex.getMessage()); } String json = jsonUtils.serializeAndEscape(model); final HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.TEXT_HTML); headers.setCacheControl("no-cache"); return new ResponseEntity<byte[]>(json.getBytes("utf-8"), headers, HttpStatus.OK); }
@Override protected boolean putInputStream(String bucket, String key, InputStream data, String contentType) throws Exception { org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key); object.setContentType(contentType); object.setDataInputStream(data); object.setContentLength(data.available()); return jetClient.putObject(bucket, object) != null; }
@Override public long getLastModified(DataSegment segment) throws SegmentLoadingException { S3Coords coords = new S3Coords(segment); try { S3Object objDetails = s3Client.getObjectDetails(new S3Bucket(coords.bucket), coords.path); return objDetails.getLastModifiedDate().getTime(); } catch (S3ServiceException e) { throw new SegmentLoadingException(e, e.getMessage()); } }
public void startElement(String uri, String name, String qName, Attributes attrs) { if (name.equals("Contents")) { currentObject = new S3Object(null); currentObject.setBucketName(bucketName); } else if (name.equals("Owner")) { currentOwner = new S3Owner(); currentObject.setOwner(currentOwner); } else if (name.equals("CommonPrefixes")) { insideCommonPrefixes = true; } }
@Override public void getSegmentFiles(DataSegment segment, File outDir) throws SegmentLoadingException { S3Coords s3Coords = new S3Coords(segment); log.info("Pulling index at path[%s] to outDir[%s]", s3Coords, outDir); if (!isObjectInBucket(s3Coords)) { throw new SegmentLoadingException("IndexFile[%s] does not exist.", s3Coords); } if (!outDir.exists()) { outDir.mkdirs(); } if (!outDir.isDirectory()) { throw new ISE("outDir[%s] must be a directory.", outDir); } long startTime = System.currentTimeMillis(); S3Object s3Obj = null; try { s3Obj = s3Client.getObject(new S3Bucket(s3Coords.bucket), s3Coords.path); InputStream in = null; try { in = s3Obj.getDataInputStream(); final String key = s3Obj.getKey(); if (key.endsWith(".zip")) { CompressionUtils.unzip(in, outDir); } else if (key.endsWith(".gz")) { final File outFile = new File(outDir, toFilename(key, ".gz")); ByteStreams.copy(new GZIPInputStream(in), Files.newOutputStreamSupplier(outFile)); } else { ByteStreams.copy( in, Files.newOutputStreamSupplier(new File(outDir, toFilename(key, "")))); } log.info( "Pull of file[%s] completed in %,d millis", s3Obj, System.currentTimeMillis() - startTime); } catch (IOException e) { FileUtils.deleteDirectory(outDir); throw new SegmentLoadingException(e, "Problem decompressing object[%s]", s3Obj); } finally { Closeables.closeQuietly(in); } } catch (Exception e) { throw new SegmentLoadingException(e, e.getMessage()); } finally { S3Utils.closeStreamsQuietly(s3Obj); } }
private InputStream get(String key) throws IOException { try { S3Object object = s3Service.getObject(bucket, key); return object.getDataInputStream(); } catch (S3ServiceException e) { if ("NoSuchKey".equals(e.getS3ErrorCode())) { return null; } if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); } throw new S3Exception(e); } }
/** * Creates a directory flagged file with the key and folder suffix. * * @param key the key to create a folder * @return true if the operation was successful, false otherwise */ private boolean mkdirsInternal(String key) { try { String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key)); S3Object obj = new S3Object(keyAsFolder); obj.setDataInputStream(new ByteArrayInputStream(new byte[0])); obj.setContentLength(0); obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM); mClient.putObject(mBucketName, obj); return true; } catch (ServiceException se) { LOG.error("Failed to create directory: " + key, se); return false; } }
public int run(String[] args) throws Exception { if (args.length == 0) { System.err.println("Usage: MigrationTool <S3 file system URI>"); System.err.println("\t<S3 file system URI>\tfilesystem to migrate"); ToolRunner.printGenericCommandUsage(System.err); return -1; } URI uri = URI.create(args[0]); initialize(uri); FileSystemStore newStore = new Jets3tFileSystemStore(); newStore.initialize(uri, getConf()); if (get("%2F") != null) { System.err.println("Current version number is [unversioned]."); System.err.println("Target version number is " + newStore.getVersion() + "."); Store oldStore = new UnversionedStore(); migrate(oldStore, newStore); return 0; } else { S3Object root = get("/"); if (root != null) { String version = (String) root.getMetadata("fs-version"); if (version == null) { System.err.println("Can't detect version - exiting."); } else { String newVersion = newStore.getVersion(); System.err.println("Current version number is " + version + "."); System.err.println("Target version number is " + newVersion + "."); if (version.equals(newStore.getVersion())) { System.err.println("No migration required."); return 0; } // use version number to create Store // Store oldStore = ... // migrate(oldStore, newStore); System.err.println("Not currently implemented."); return 0; } } System.err.println("Can't detect version - exiting."); return 0; } }
/** Event handler for this dialog. */ public void actionPerformed(ActionEvent e) { // Force table to accept any partial edits. if (metadataTable.isEditing()) { metadataTable.getCellEditor().stopCellEditing(); } // Apply new/modified attributes to the object. Map currentObjectMetadata = currentObject.getModifiableMetadata(); Set obsoleteMetadataItems = currentObjectMetadata.keySet(); for (int row = 0; row < metadataTable.getRowCount(); row++) { String name = (String) objectMetadataTableModel.getValueAt(row, 0); String value = (String) objectMetadataTableModel.getValueAt(row, 1); currentObject.addMetadata(name, value); obsoleteMetadataItems.remove(name); } // Remove obsolete attributes. Iterator obsoleteNamesIter = obsoleteMetadataItems.iterator(); while (obsoleteNamesIter.hasNext()) { currentObject.removeMetadata((String) obsoleteNamesIter.next()); } if (e.getSource().equals(nextObjectButton)) { currentObjectIndex++; displayObjectProperties(); } else if (e.getSource().equals(previousObjectButton)) { currentObjectIndex--; displayObjectProperties(); } else if ("OK".equals(e.getActionCommand())) { modifyActionApproved = isModifyMode(); this.setVisible(false); } else if ("Cancel".equals(e.getActionCommand())) { modifyActionApproved = false; this.setVisible(false); } else if ("addMetadataItem".equals(e.getActionCommand())) { int newRowNumber = metadataTable.getRowCount() + 1; objectMetadataTableModel.addRow( new String[] {"name-" + newRowNumber, "value-" + newRowNumber}); } else if ("removeMetadataItem".equals(e.getActionCommand())) { int[] rows = metadataTable.getSelectedRows(); for (int i = rows.length - 1; i >= 0; i--) { int modelIndex = metadataTableSorter.modelIndex(rows[i]); objectMetadataTableModel.removeRow(modelIndex); } } }
@Override public long getLastModified(DataSegment segment) throws SegmentLoadingException { final S3Coords coords = new S3Coords(segment); try { final S3Object objDetails = S3Utils.retryS3Operation( new Callable<S3Object>() { @Override public S3Object call() throws Exception { return s3Client.getObjectDetails(new S3Bucket(coords.bucket), coords.path); } }); return objDetails.getLastModifiedDate().getTime(); } catch (S3ServiceException | IOException e) { throw new SegmentLoadingException(e, e.getMessage()); } catch (Exception e) { throw Throwables.propagate(e); } }
/** * Lists the files in the given path, the paths will be their logical names and not contain the * folder suffix. * * @param path the key to list * @param recursive if true will list children directories as well * @return an array of the file and folder names in this directory * @throws IOException if an I/O error occurs */ private String[] listInternal(String path, boolean recursive) throws IOException { try { path = stripPrefixIfPresent(path); path = PathUtils.normalizePath(path, PATH_SEPARATOR); path = path.equals(PATH_SEPARATOR) ? "" : path; // Gets all the objects under the path, because we have no idea if there are non Alluxio // managed "directories" S3Object[] objs = mClient.listObjects(mBucketName, path, ""); if (recursive) { List<String> ret = new ArrayList<>(); for (S3Object obj : objs) { // Remove parent portion of the key String child = getChildName(obj.getKey(), path); // Prune the special folder suffix child = stripFolderSuffixIfPresent(child); // Only add if the path is not empty (removes results equal to the path) if (!child.isEmpty()) { ret.add(child); } } return ret.toArray(new String[ret.size()]); } // Non recursive list Set<String> children = new HashSet<String>(); for (S3Object obj : objs) { // Remove parent portion of the key String child = getChildName(obj.getKey(), path); // Remove any portion after the path delimiter int childNameIndex = child.indexOf(PATH_SEPARATOR); child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child; // Prune the special folder suffix child = stripFolderSuffixIfPresent(child); // Add to the set of children, the set will deduplicate. if (!child.isEmpty()) { children.add(child); } } return children.toArray(new String[children.size()]); } catch (ServiceException e) { LOG.error("Failed to list path {}", path, e); return null; } }
public static void main(String[] args) { Storage st = new Storage(args[0], args[1], args[2]); try { String awsAccessKey = "AKIAIXDL4AVDSSMH4JJQ"; String awsSecretKey = "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"; AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); S3Service s3Service = new RestS3Service(awsCredentials); S3Object list[] = s3Service.listObjects("pagerankmam"); for (int i = 0; i < list.length; i++) { if (list[i].getName().startsWith("input/output/")) { // Open the file that is the first // command line parameter // System.out.println(list[i].getName()); S3Object objectComplete = s3Service.getObject("pagerankmam/input/output", list[i].getName().split("/")[2]); // System.out.println("S3Object, complete: " + objectComplete); // Read the data from the object's DataInputStream using a loop, and print it out. // System.out.println("Greeting:"); BufferedReader reader = new BufferedReader(new InputStreamReader(objectComplete.getDataInputStream())); String data = null; st.connect(); while ((data = reader.readLine()) != null) { String[] results = data.split("\t"); st.insertPageRankRow( Integer.parseInt(results[0]), Double.parseDouble(results[1].split(" ")[0])); // System.out.println(Integer.parseInt(results[0])+" - // "+Double.parseDouble(results[1].split(" ")[0])); // System.out.println(data); } st.close(); } } } catch (Exception e) { // Catch exception if any System.err.println("Error: " + e.getMessage()); } }
@Test public void testSimpleGetVersion() throws ServiceException, IOException { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); S3Object object0 = new S3Object(); object0.setBucketName(bucket); object0.setKey(keyPrefix + "/renames-0.gz"); object0.setLastModifiedDate(new Date(0)); EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) .andReturn(object0) .once(); S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client); EasyMock.replay(s3Client); String version = puller.getVersion(URI.create(String.format("s3://%s/%s", bucket, object0.getKey()))); EasyMock.verify(s3Client); Assert.assertEquals(String.format("%d", new Date(0).getTime()), version); }
/** * Update the dialog to display the attributes of a single object. The user may choose which * object to display by iterating forward and back through the set of objects. */ private void displayObjectProperties() { currentObject = destinationObjects[currentObjectIndex]; // Manage previous/next buttons. if (destinationObjects.length > 1) { currentObjectLabel.setText((currentObjectIndex + 1) + " of " + destinationObjects.length); previousObjectButton.setEnabled(currentObjectIndex > 0); nextObjectButton.setEnabled(currentObjectIndex < (destinationObjects.length - 1)); } // Unmodifiable fields. objectKeyTextField.setText(currentObject.getKey()); objectContentLengthTextField.setText(String.valueOf(currentObject.getContentLength())); objectLastModifiedTextField.setText(String.valueOf(currentObject.getLastModifiedDate())); objectETagTextField.setText(currentObject.getETag()); bucketLocationTextField.setText(currentObject.getBucketName()); if (currentObject.getOwner() != null) { ownerNameLabel.setVisible(true); ownerNameTextField.setVisible(true); ownerIdLabel.setVisible(true); ownerIdTextField.setVisible(true); ownerNameTextField.setText(currentObject.getOwner().getDisplayName()); ownerIdTextField.setText(currentObject.getOwner().getId()); } else { ownerNameLabel.setVisible(false); ownerNameTextField.setVisible(false); ownerIdLabel.setVisible(false); ownerIdTextField.setVisible(false); } // Clear old table contents while (objectMetadataTableModel.getRowCount() > 0) { objectMetadataTableModel.removeRow(0); } // Display remaining metadata items in the table. Iterator mdIter = currentObject.getModifiableMetadata().entrySet().iterator(); while (mdIter.hasNext()) { Map.Entry entry = (Map.Entry) mdIter.next(); Object name = entry.getKey(); Object value = entry.getValue(); objectMetadataTableModel.addRow(new Object[] {name, value}); } }
/** * Lists the files in the given path, the paths will be their logical names and not contain the * folder suffix * * @param path the key to list * @param recursive if true will list children directories as well * @return an array of the file and folder names in this directory * @throws IOException */ private String[] listInternal(String path, boolean recursive) throws IOException { try { path = stripPrefixIfPresent(path); path = path.endsWith(PATH_SEPARATOR) ? path : path + PATH_SEPARATOR; path = path.equals(PATH_SEPARATOR) ? "" : path; // Gets all the objects under the path, because we have no idea if there are non Tachyon // managed "directories" S3Object[] objs = mClient.listObjects(mBucketName, path, ""); if (recursive) { String[] ret = new String[objs.length]; for (int i = 0; i < objs.length; i++) { // Remove parent portion of the key String child = getChildName(objs[i].getKey(), path); // Prune the special folder suffix child = stripFolderSuffixIfPresent(child); ret[i] = child; } return ret; } // Non recursive list Set<String> children = new HashSet<String>(); for (S3Object obj : objs) { // Remove parent portion of the key String child = getChildName(obj.getKey(), path); // Remove any portion after the path delimiter int childNameIndex = child.indexOf(PATH_SEPARATOR); child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child; // Prune the special folder suffix child = stripFolderSuffixIfPresent(child); // Add to the set of children, the set will deduplicate. children.add(child); } return children.toArray(new String[children.size()]); } catch (ServiceException se) { LOG.error("Failed to list path " + path); return null; } }
@Test public void testGZUncompress() throws ServiceException, IOException, SegmentLoadingException { final String bucket = "bucket"; final String keyPrefix = "prefix/dir/0"; final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); final byte[] value = bucket.getBytes("utf8"); final File tmpFile = temporaryFolder.newFile("gzTest.gz"); try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) { outputStream.write(value); } final S3Object object0 = new S3Object(); object0.setBucketName(bucket); object0.setKey(keyPrefix + "/renames-0.gz"); object0.setLastModifiedDate(new Date(0)); object0.setDataInputStream(new FileInputStream(tmpFile)); final File tmpDir = temporaryFolder.newFolder("gzTestDir"); EasyMock.expect( s3Client.getObjectDetails( EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) .andReturn(null) .once(); EasyMock.expect( s3Client.getObjectDetails( EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) .andReturn(object0) .once(); EasyMock.expect( s3Client.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) .andReturn(object0) .once(); S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client); EasyMock.replay(s3Client); FileUtils.FileCopyResult result = puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, object0.getKey()), tmpDir); EasyMock.verify(s3Client); Assert.assertEquals(value.length, result.size()); File expected = new File(tmpDir, "renames-0"); Assert.assertTrue(expected.exists()); Assert.assertEquals(value.length, expected.length()); }
protected URLConnection openConnection(URL u) throws IOException { // This looking for accessKey id and accessKey secret code is based // on code from hadoop S3. String accessKey = null; String secretAccessKey = null; String userInfo = u.getUserInfo(); if (userInfo != null) { int index = userInfo.indexOf(':'); if (index != -1) { accessKey = userInfo.substring(0, index); secretAccessKey = userInfo.substring(index + 1); } else { accessKey = userInfo; } } if (accessKey == null) { accessKey = System.getProperty("aws.access.key.id"); } if (secretAccessKey == null) { secretAccessKey = System.getProperty("aws.access.key.secret"); } if (accessKey == null && secretAccessKey == null) { throw new IllegalArgumentException("AWS " + "Access Key ID and Secret Access Key " + "must be specified as the username " + "or password (respectively) of a s3 URL, " + "or by setting the " + "aws.access.key.id or " + "aws.access.key.secret properties (respectively)."); } else if (accessKey == null) { throw new IllegalArgumentException("AWS " + "Access Key ID must be specified " + "as the username of a s3 URL, or by setting the " + "aws.access.key.id property."); } else if (secretAccessKey == null) { throw new IllegalArgumentException("AWS " + "Secret Access Key must be specified " + "as the password of a s3 URL, or by setting the " + "aws.access.key.secret property."); } RestS3Service s3Service; try { s3Service = new RestS3Service( new AWSCredentials(accessKey, secretAccessKey)); } catch (S3ServiceException e) { e.printStackTrace(); throw new IOException(e.toString()); } InputStream is = null; try { // This opens the stream to the bucket/key object. S3Object s3obj = s3Service.getObject(new S3Bucket(u.getHost()), u.getPath().substring(1) /* Skip starting '/' character */); is = s3obj.getDataInputStream(); } catch (S3ServiceException e) { e.printStackTrace(); throw new IOException(e.toString()); } final InputStream inputStream = is; return new URLConnection(u) { private InputStream is = inputStream; @Override public InputStream getInputStream() throws IOException { return this.is; } @Override public void connect() throws IOException { // Nothing to do. When we give back this object, we're // connected. } }; }
public void endElement(String uri, String name, String qName) { String elementText = this.currText.toString(); // Listing details if (name.equals("Name")) { bucketName = elementText; if (log.isDebugEnabled()) { log.debug("Examining listing for bucket: " + bucketName); } } else if (!insideCommonPrefixes && name.equals("Prefix")) { requestPrefix = elementText; } else if (name.equals("Marker")) { requestMarker = elementText; } else if (name.equals("NextMarker")) { nextMarker = elementText; } else if (name.equals("MaxKeys")) { requestMaxKeys = Long.parseLong(elementText); } else if (name.equals("IsTruncated")) { String isTruncatedStr = elementText.toLowerCase(Locale.getDefault()); if (isTruncatedStr.startsWith("false")) { listingTruncated = false; } else if (isTruncatedStr.startsWith("true")) { listingTruncated = true; } else { throw new RuntimeException("Invalid value for IsTruncated field: " + isTruncatedStr); } } // Object details. else if (name.equals("Contents")) { objects.add(currentObject); if (log.isDebugEnabled()) { log.debug("Created new S3Object from listing: " + currentObject); } } else if (name.equals("Key")) { currentObject.setKey(elementText); lastKey = elementText; } else if (name.equals("LastModified")) { try { currentObject.setLastModifiedDate(ServiceUtils.parseIso8601Date(elementText)); } catch (ParseException e) { throw new RuntimeException("Unexpected date format in list bucket output", e); } } else if (name.equals("ETag")) { currentObject.setETag(elementText); } else if (name.equals("Size")) { currentObject.setContentLength(Long.parseLong(elementText)); } else if (name.equals("StorageClass")) { currentObject.setStorageClass(elementText); } // Owner details. else if (name.equals("ID")) { // Work-around to support Eucalyptus responses, which do not // contain Owner elements. if (currentOwner == null) { currentOwner = new S3Owner(); currentObject.setOwner(currentOwner); } currentOwner.setId(elementText); } else if (name.equals("DisplayName")) { currentOwner.setDisplayName(elementText); } // Common prefixes. else if (insideCommonPrefixes && name.equals("Prefix")) { commonPrefixes.add(elementText); } else if (name.equals("CommonPrefixes")) { insideCommonPrefixes = false; } this.currText = new StringBuffer(); }
/* * Fetch a file that is in a S3 file system. Return a local File. It accepts "s3://" and "s3n://" prefixes. */ private File s3Fetch(URI uri, Reporter reporter) throws IOException { String bucketName = uri.getHost(); String path = uri.getPath(); File destFolder = new File(tempDir, bucketName + "/" + path); if (destFolder.exists()) { FileUtils.deleteDirectory(destFolder); } destFolder.mkdirs(); Throttler throttler = new Throttler((double) bytesPerSecThrottle); boolean done = false; try { s3Service = new RestS3Service(getCredentials()); if (s3Service.checkBucketStatus(bucketName) != RestS3Service.BUCKET_STATUS__MY_BUCKET) { throw new IOException("Bucket doesn't exist or is already claimed: " + bucketName); } if (path.startsWith("/")) { path = path.substring(1, path.length()); } for (S3Object object : s3Service.listObjects(new S3Bucket(bucketName), path, "")) { long bytesSoFar = 0; String fileName = path; if (path.contains("/")) { fileName = path.substring(path.lastIndexOf("/") + 1, path.length()); } File fileDest = new File(destFolder, fileName); log.info("Downloading " + object.getKey() + " to " + fileDest + " ..."); if (fileDest.exists()) { fileDest.delete(); } object = s3Service.getObject(new S3Bucket(bucketName), object.getKey()); InputStream iS = object.getDataInputStream(); FileOutputStream writer = new FileOutputStream(fileDest); byte[] buffer = new byte[downloadBufferSize]; int nRead; while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) { bytesSoFar += nRead; writer.write(buffer, 0, nRead); throttler.incrementAndThrottle(nRead); if (bytesSoFar >= bytesToReportProgress) { reporter.progress(bytesSoFar); bytesSoFar = 0l; } } if (reporter != null) { reporter.progress(bytesSoFar); } writer.close(); iS.close(); done = true; } if (!done) { throw new IOException("Bucket is empty! " + bucketName + " path: " + path); } } catch (S3ServiceException e) { throw new IOException(e); } return destFolder; }