@ResponseBody
@RequestMapping(value = "/admin/upload.ajax", method = RequestMethod.POST)
public ResponseEntity<byte[]> upload(
@RequestParam("restaurantId") String restaurantId,
@RequestParam("file") CommonsMultipartFile file)
throws Exception {
Map<String, Object> model = new HashMap<String, Object>();
try {
S3Object object = new S3Object(basePath + "/" + restaurantId);
object.setDataInputStream(file.getInputStream());
object.setContentLength(file.getSize());
object.setContentType(file.getContentType());
S3Bucket bucket = s3Service.getBucket(bucketName);
s3Service.putObject(bucket, object);
Restaurant restaurant = restaurantRepository.findByRestaurantId(restaurantId);
restaurant.setHasUploadedImage(true);
restaurantRepository.saveRestaurant(restaurant);
model.put("success", true);
} catch (Exception ex) {
LOGGER.error("", ex);
model.put("success", false);
model.put("message", ex.getMessage());
}
String json = jsonUtils.serializeAndEscape(model);
final HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.TEXT_HTML);
headers.setCacheControl("no-cache");
return new ResponseEntity<byte[]>(json.getBytes("utf-8"), headers, HttpStatus.OK);
}
/**
* Determines if the key represents a folder. If false is returned, it is not guaranteed that the
* path exists.
*
* @param key the key to check
* @return whether the given key identifies a folder
*/
private boolean isFolder(String key) {
// Root is always a folder
if (isRoot(key)) {
return true;
}
try {
String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
mClient.getObjectDetails(mBucketName, keyAsFolder);
// If no exception is thrown, the key exists as a folder
return true;
} catch (ServiceException s) {
// It is possible that the folder has not been encoded as a _$folder$ file
try {
String dir = stripPrefixIfPresent(key);
String dirPrefix = dir.endsWith(PATH_SEPARATOR) ? dir : dir + PATH_SEPARATOR;
// Check if anything begins with <folder_path>/
S3Object[] objs = mClient.listObjects(mBucketName, dirPrefix, "");
// If there are, this is a folder and we can create the necessary metadata
if (objs.length > 0) {
mkdirsInternal(dir);
return true;
} else {
return false;
}
} catch (ServiceException s2) {
return false;
}
}
}
/**
* @param key the key to get the object details of
* @return {@link StorageObject} of the key, or null if the key does not exist
*/
private StorageObject getObjectDetails(String key) {
try {
if (isFolder(key)) {
String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
return mClient.getObjectDetails(mBucketName, keyAsFolder);
} else {
return mClient.getObjectDetails(mBucketName, stripPrefixIfPresent(key));
}
} catch (ServiceException e) {
return null;
}
}
/**
* Internal function to delete a key in S3.
*
* @param key the key to delete
* @return true if successful, false if an exception is thrown
*/
private boolean deleteInternal(String key) {
try {
if (isFolder(key)) {
String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
mClient.deleteObject(mBucketName, keyAsFolder);
} else {
mClient.deleteObject(mBucketName, stripPrefixIfPresent(key));
}
} catch (ServiceException e) {
LOG.error("Failed to delete {}", key, e);
return false;
}
return true;
}
@Override
protected boolean putInputStream(String bucket, String key, InputStream data, String contentType)
throws Exception {
org.jets3t.service.model.S3Object object = new org.jets3t.service.model.S3Object(key);
object.setContentType(contentType);
object.setDataInputStream(data);
object.setContentLength(data.available());
return jetClient.putObject(bucket, object) != null;
}
private S3Object get(String key) {
try {
return s3Service.getObject(bucket, key);
} catch (S3ServiceException e) {
if ("NoSuchKey".equals(e.getS3ErrorCode())) {
return null;
}
}
return null;
}
public static void main(String[] args) {
Storage st = new Storage(args[0], args[1], args[2]);
try {
String awsAccessKey = "AKIAIXDL4AVDSSMH4JJQ";
String awsSecretKey = "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";
AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
S3Service s3Service = new RestS3Service(awsCredentials);
S3Object list[] = s3Service.listObjects("pagerankmam");
for (int i = 0; i < list.length; i++) {
if (list[i].getName().startsWith("input/output/")) {
// Open the file that is the first
// command line parameter
// System.out.println(list[i].getName());
S3Object objectComplete =
s3Service.getObject("pagerankmam/input/output", list[i].getName().split("/")[2]);
// System.out.println("S3Object, complete: " + objectComplete);
// Read the data from the object's DataInputStream using a loop, and print it out.
// System.out.println("Greeting:");
BufferedReader reader =
new BufferedReader(new InputStreamReader(objectComplete.getDataInputStream()));
String data = null;
st.connect();
while ((data = reader.readLine()) != null) {
String[] results = data.split("\t");
st.insertPageRankRow(
Integer.parseInt(results[0]), Double.parseDouble(results[1].split(" ")[0]));
// System.out.println(Integer.parseInt(results[0])+" -
// "+Double.parseDouble(results[1].split(" ")[0]));
// System.out.println(data);
}
st.close();
}
}
} catch (Exception e) { // Catch exception if any
System.err.println("Error: " + e.getMessage());
}
}
/**
* Copies an object to another key.
*
* @param src the source key to copy
* @param dst the destination key to copy to
* @return true if the operation was successful, false otherwise
*/
private boolean copy(String src, String dst) {
try {
src = stripPrefixIfPresent(src);
dst = stripPrefixIfPresent(dst);
LOG.info("Copying {} to {}", src, dst);
S3Object obj = new S3Object(dst);
mClient.copyObject(mBucketName, src, mBucketName, obj, false);
return true;
} catch (ServiceException e) {
LOG.error("Failed to rename file {} to {}", src, dst, e);
return false;
}
}
/**
* Creates a directory flagged file with the key and folder suffix.
*
* @param key the key to create a folder
* @return true if the operation was successful, false otherwise
*/
private boolean mkdirsInternal(String key) {
try {
String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
S3Object obj = new S3Object(keyAsFolder);
obj.setDataInputStream(new ByteArrayInputStream(new byte[0]));
obj.setContentLength(0);
obj.setContentType(Mimetypes.MIMETYPE_BINARY_OCTET_STREAM);
mClient.putObject(mBucketName, obj);
return true;
} catch (ServiceException se) {
LOG.error("Failed to create directory: " + key, se);
return false;
}
}
/**
* Determines if the key represents a folder. If false is returned, it is not guaranteed that the
* path exists.
*
* @param key the key to check
* @return S3Object containing metadata
*/
private boolean isFolder(String key) {
key = key.endsWith(PATH_SEPARATOR) ? key.substring(0, key.length() - 1) : key;
// Root is always a folder
if (isRoot(key)) {
return true;
}
try {
String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
mClient.getObjectDetails(mBucketName, keyAsFolder);
// If no exception is thrown, the key exists as a folder
return true;
} catch (ServiceException se) {
return false;
}
}
/**
* Lists the files in the given path, the paths will be their logical names and not contain the
* folder suffix.
*
* @param path the key to list
* @param recursive if true will list children directories as well
* @return an array of the file and folder names in this directory
* @throws IOException if an I/O error occurs
*/
private String[] listInternal(String path, boolean recursive) throws IOException {
try {
path = stripPrefixIfPresent(path);
path = PathUtils.normalizePath(path, PATH_SEPARATOR);
path = path.equals(PATH_SEPARATOR) ? "" : path;
// Gets all the objects under the path, because we have no idea if there are non Alluxio
// managed "directories"
S3Object[] objs = mClient.listObjects(mBucketName, path, "");
if (recursive) {
List<String> ret = new ArrayList<>();
for (S3Object obj : objs) {
// Remove parent portion of the key
String child = getChildName(obj.getKey(), path);
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
// Only add if the path is not empty (removes results equal to the path)
if (!child.isEmpty()) {
ret.add(child);
}
}
return ret.toArray(new String[ret.size()]);
}
// Non recursive list
Set<String> children = new HashSet<String>();
for (S3Object obj : objs) {
// Remove parent portion of the key
String child = getChildName(obj.getKey(), path);
// Remove any portion after the path delimiter
int childNameIndex = child.indexOf(PATH_SEPARATOR);
child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
// Add to the set of children, the set will deduplicate.
if (!child.isEmpty()) {
children.add(child);
}
}
return children.toArray(new String[children.size()]);
} catch (ServiceException e) {
LOG.error("Failed to list path {}", path, e);
return null;
}
}
/**
* Lists the files in the given path, the paths will be their logical names and not contain the
* folder suffix
*
* @param path the key to list
* @param recursive if true will list children directories as well
* @return an array of the file and folder names in this directory
* @throws IOException
*/
private String[] listInternal(String path, boolean recursive) throws IOException {
try {
path = stripPrefixIfPresent(path);
path = path.endsWith(PATH_SEPARATOR) ? path : path + PATH_SEPARATOR;
path = path.equals(PATH_SEPARATOR) ? "" : path;
// Gets all the objects under the path, because we have no idea if there are non Tachyon
// managed "directories"
S3Object[] objs = mClient.listObjects(mBucketName, path, "");
if (recursive) {
String[] ret = new String[objs.length];
for (int i = 0; i < objs.length; i++) {
// Remove parent portion of the key
String child = getChildName(objs[i].getKey(), path);
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
ret[i] = child;
}
return ret;
}
// Non recursive list
Set<String> children = new HashSet<String>();
for (S3Object obj : objs) {
// Remove parent portion of the key
String child = getChildName(obj.getKey(), path);
// Remove any portion after the path delimiter
int childNameIndex = child.indexOf(PATH_SEPARATOR);
child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
// Add to the set of children, the set will deduplicate.
children.add(child);
}
return children.toArray(new String[children.size()]);
} catch (ServiceException se) {
LOG.error("Failed to list path " + path);
return null;
}
}
/*
* Fetch a file that is in a S3 file system. Return a local File. It accepts "s3://" and "s3n://" prefixes.
*/
private File s3Fetch(URI uri, Reporter reporter) throws IOException {
String bucketName = uri.getHost();
String path = uri.getPath();
File destFolder = new File(tempDir, bucketName + "/" + path);
if (destFolder.exists()) {
FileUtils.deleteDirectory(destFolder);
}
destFolder.mkdirs();
Throttler throttler = new Throttler((double) bytesPerSecThrottle);
boolean done = false;
try {
s3Service = new RestS3Service(getCredentials());
if (s3Service.checkBucketStatus(bucketName) != RestS3Service.BUCKET_STATUS__MY_BUCKET) {
throw new IOException("Bucket doesn't exist or is already claimed: " + bucketName);
}
if (path.startsWith("/")) {
path = path.substring(1, path.length());
}
for (S3Object object : s3Service.listObjects(new S3Bucket(bucketName), path, "")) {
long bytesSoFar = 0;
String fileName = path;
if (path.contains("/")) {
fileName = path.substring(path.lastIndexOf("/") + 1, path.length());
}
File fileDest = new File(destFolder, fileName);
log.info("Downloading " + object.getKey() + " to " + fileDest + " ...");
if (fileDest.exists()) {
fileDest.delete();
}
object = s3Service.getObject(new S3Bucket(bucketName), object.getKey());
InputStream iS = object.getDataInputStream();
FileOutputStream writer = new FileOutputStream(fileDest);
byte[] buffer = new byte[downloadBufferSize];
int nRead;
while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
bytesSoFar += nRead;
writer.write(buffer, 0, nRead);
throttler.incrementAndThrottle(nRead);
if (bytesSoFar >= bytesToReportProgress) {
reporter.progress(bytesSoFar);
bytesSoFar = 0l;
}
}
if (reporter != null) {
reporter.progress(bytesSoFar);
}
writer.close();
iS.close();
done = true;
}
if (!done) {
throw new IOException("Bucket is empty! " + bucketName + " path: " + path);
}
} catch (S3ServiceException e) {
throw new IOException(e);
}
return destFolder;
}
private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception {
log.info("Downloading {}", s3Artifact);
Jets3tProperties jets3tProperties =
Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME);
jets3tProperties.setProperty(
"httpclient.socket-timeout-ms",
Long.toString(configuration.getS3ChunkDownloadTimeoutMillis()));
final S3Service s3 =
new RestS3Service(
getCredentialsForBucket(s3Artifact.getS3Bucket()), null, null, jets3tProperties);
long length = 0;
if (s3Artifact.getFilesize().isPresent()) {
length = s3Artifact.getFilesize().get();
} else {
StorageObject details =
s3.getObjectDetails(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
Preconditions.checkNotNull(
details,
"Couldn't find object at %s/%s",
s3Artifact.getS3Bucket(),
s3Artifact.getS3ObjectKey());
length = details.getContentLength();
}
int numChunks = (int) (length / configuration.getS3ChunkSize());
if (length % configuration.getS3ChunkSize() > 0) {
numChunks++;
}
final long chunkSize = length / numChunks + (length % numChunks);
log.info(
"Downloading {}/{} in {} chunks of {} bytes to {}",
s3Artifact.getS3Bucket(),
s3Artifact.getS3ObjectKey(),
numChunks,
chunkSize,
downloadTo);
final ExecutorService chunkExecutorService =
Executors.newFixedThreadPool(
numChunks,
new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat("S3ArtifactDownloaderChunkThread-%d")
.build());
final List<Future<Path>> futures = Lists.newArrayListWithCapacity(numChunks);
for (int chunk = 0; chunk < numChunks; chunk++) {
futures.add(
chunkExecutorService.submit(
new S3ArtifactChunkDownloader(
configuration,
log,
s3,
s3Artifact,
downloadTo,
chunk,
chunkSize,
length,
exceptionNotifier)));
}
long remainingMillis = configuration.getS3DownloadTimeoutMillis();
boolean failed = false;
for (int chunk = 0; chunk < numChunks; chunk++) {
final Future<Path> future = futures.get(chunk);
if (failed) {
future.cancel(true);
continue;
}
final long start = System.currentTimeMillis();
if (!handleChunk(s3Artifact, future, downloadTo, chunk, start, remainingMillis)) {
failed = true;
}
remainingMillis -= (System.currentTimeMillis() - start);
}
chunkExecutorService.shutdownNow();
Preconditions.checkState(
!failed, "Downloading %s/%s failed", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey());
}