/**
* Determines if the key represents a folder. If false is returned, it is not guaranteed that the
* path exists.
*
* @param key the key to check
* @return whether the given key identifies a folder
*/
private boolean isFolder(String key) {
// Root is always a folder
if (isRoot(key)) {
return true;
}
try {
String keyAsFolder = convertToFolderName(stripPrefixIfPresent(key));
mClient.getObjectDetails(mBucketName, keyAsFolder);
// If no exception is thrown, the key exists as a folder
return true;
} catch (ServiceException s) {
// It is possible that the folder has not been encoded as a _$folder$ file
try {
String dir = stripPrefixIfPresent(key);
String dirPrefix = dir.endsWith(PATH_SEPARATOR) ? dir : dir + PATH_SEPARATOR;
// Check if anything begins with <folder_path>/
S3Object[] objs = mClient.listObjects(mBucketName, dirPrefix, "");
// If there are, this is a folder and we can create the necessary metadata
if (objs.length > 0) {
mkdirsInternal(dir);
return true;
} else {
return false;
}
} catch (ServiceException s2) {
return false;
}
}
}
/**
* Lists the files in the given path, the paths will be their logical names and not contain the
* folder suffix.
*
* @param path the key to list
* @param recursive if true will list children directories as well
* @return an array of the file and folder names in this directory
* @throws IOException if an I/O error occurs
*/
private String[] listInternal(String path, boolean recursive) throws IOException {
try {
path = stripPrefixIfPresent(path);
path = PathUtils.normalizePath(path, PATH_SEPARATOR);
path = path.equals(PATH_SEPARATOR) ? "" : path;
// Gets all the objects under the path, because we have no idea if there are non Alluxio
// managed "directories"
S3Object[] objs = mClient.listObjects(mBucketName, path, "");
if (recursive) {
List<String> ret = new ArrayList<>();
for (S3Object obj : objs) {
// Remove parent portion of the key
String child = getChildName(obj.getKey(), path);
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
// Only add if the path is not empty (removes results equal to the path)
if (!child.isEmpty()) {
ret.add(child);
}
}
return ret.toArray(new String[ret.size()]);
}
// Non recursive list
Set<String> children = new HashSet<String>();
for (S3Object obj : objs) {
// Remove parent portion of the key
String child = getChildName(obj.getKey(), path);
// Remove any portion after the path delimiter
int childNameIndex = child.indexOf(PATH_SEPARATOR);
child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
// Add to the set of children, the set will deduplicate.
if (!child.isEmpty()) {
children.add(child);
}
}
return children.toArray(new String[children.size()]);
} catch (ServiceException e) {
LOG.error("Failed to list path {}", path, e);
return null;
}
}
public static void main(String[] args) {
Storage st = new Storage(args[0], args[1], args[2]);
try {
String awsAccessKey = "AKIAIXDL4AVDSSMH4JJQ";
String awsSecretKey = "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";
AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey);
S3Service s3Service = new RestS3Service(awsCredentials);
S3Object list[] = s3Service.listObjects("pagerankmam");
for (int i = 0; i < list.length; i++) {
if (list[i].getName().startsWith("input/output/")) {
// Open the file that is the first
// command line parameter
// System.out.println(list[i].getName());
S3Object objectComplete =
s3Service.getObject("pagerankmam/input/output", list[i].getName().split("/")[2]);
// System.out.println("S3Object, complete: " + objectComplete);
// Read the data from the object's DataInputStream using a loop, and print it out.
// System.out.println("Greeting:");
BufferedReader reader =
new BufferedReader(new InputStreamReader(objectComplete.getDataInputStream()));
String data = null;
st.connect();
while ((data = reader.readLine()) != null) {
String[] results = data.split("\t");
st.insertPageRankRow(
Integer.parseInt(results[0]), Double.parseDouble(results[1].split(" ")[0]));
// System.out.println(Integer.parseInt(results[0])+" -
// "+Double.parseDouble(results[1].split(" ")[0]));
// System.out.println(data);
}
st.close();
}
}
} catch (Exception e) { // Catch exception if any
System.err.println("Error: " + e.getMessage());
}
}
/**
* Lists the files in the given path, the paths will be their logical names and not contain the
* folder suffix
*
* @param path the key to list
* @param recursive if true will list children directories as well
* @return an array of the file and folder names in this directory
* @throws IOException
*/
private String[] listInternal(String path, boolean recursive) throws IOException {
try {
path = stripPrefixIfPresent(path);
path = path.endsWith(PATH_SEPARATOR) ? path : path + PATH_SEPARATOR;
path = path.equals(PATH_SEPARATOR) ? "" : path;
// Gets all the objects under the path, because we have no idea if there are non Tachyon
// managed "directories"
S3Object[] objs = mClient.listObjects(mBucketName, path, "");
if (recursive) {
String[] ret = new String[objs.length];
for (int i = 0; i < objs.length; i++) {
// Remove parent portion of the key
String child = getChildName(objs[i].getKey(), path);
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
ret[i] = child;
}
return ret;
}
// Non recursive list
Set<String> children = new HashSet<String>();
for (S3Object obj : objs) {
// Remove parent portion of the key
String child = getChildName(obj.getKey(), path);
// Remove any portion after the path delimiter
int childNameIndex = child.indexOf(PATH_SEPARATOR);
child = childNameIndex != -1 ? child.substring(0, childNameIndex) : child;
// Prune the special folder suffix
child = stripFolderSuffixIfPresent(child);
// Add to the set of children, the set will deduplicate.
children.add(child);
}
return children.toArray(new String[children.size()]);
} catch (ServiceException se) {
LOG.error("Failed to list path " + path);
return null;
}
}
/*
* Fetch a file that is in a S3 file system. Return a local File. It accepts "s3://" and "s3n://" prefixes.
*/
private File s3Fetch(URI uri, Reporter reporter) throws IOException {
String bucketName = uri.getHost();
String path = uri.getPath();
File destFolder = new File(tempDir, bucketName + "/" + path);
if (destFolder.exists()) {
FileUtils.deleteDirectory(destFolder);
}
destFolder.mkdirs();
Throttler throttler = new Throttler((double) bytesPerSecThrottle);
boolean done = false;
try {
s3Service = new RestS3Service(getCredentials());
if (s3Service.checkBucketStatus(bucketName) != RestS3Service.BUCKET_STATUS__MY_BUCKET) {
throw new IOException("Bucket doesn't exist or is already claimed: " + bucketName);
}
if (path.startsWith("/")) {
path = path.substring(1, path.length());
}
for (S3Object object : s3Service.listObjects(new S3Bucket(bucketName), path, "")) {
long bytesSoFar = 0;
String fileName = path;
if (path.contains("/")) {
fileName = path.substring(path.lastIndexOf("/") + 1, path.length());
}
File fileDest = new File(destFolder, fileName);
log.info("Downloading " + object.getKey() + " to " + fileDest + " ...");
if (fileDest.exists()) {
fileDest.delete();
}
object = s3Service.getObject(new S3Bucket(bucketName), object.getKey());
InputStream iS = object.getDataInputStream();
FileOutputStream writer = new FileOutputStream(fileDest);
byte[] buffer = new byte[downloadBufferSize];
int nRead;
while ((nRead = iS.read(buffer, 0, buffer.length)) != -1) {
bytesSoFar += nRead;
writer.write(buffer, 0, nRead);
throttler.incrementAndThrottle(nRead);
if (bytesSoFar >= bytesToReportProgress) {
reporter.progress(bytesSoFar);
bytesSoFar = 0l;
}
}
if (reporter != null) {
reporter.progress(bytesSoFar);
}
writer.close();
iS.close();
done = true;
}
if (!done) {
throw new IOException("Bucket is empty! " + bucketName + " path: " + path);
}
} catch (S3ServiceException e) {
throw new IOException(e);
}
return destFolder;
}