/**
* Lists objects from AmazonS3 in chronological order [lexicographical order if 2 files have same
* timestamp] which are later than or equal to the timestamp of the previous offset object
*
* @param s3Client
* @param s3ConfigBean
* @param pathMatcher glob patterns to match file name against
* @param s3Offset current offset which provides the timestamp of the previous object
* @param fetchSize number of objects to fetch in one go
* @return
* @throws AmazonClientException
*/
static List<S3ObjectSummary> listObjectsChronologically(
AmazonS3Client s3Client,
S3ConfigBean s3ConfigBean,
PathMatcher pathMatcher,
AmazonS3Source.S3Offset s3Offset,
int fetchSize)
throws AmazonClientException {
// Algorithm:
// - Full scan all objects that match the file name pattern and which are later than the file in
// the offset
// - Select the oldest "fetchSize" number of files and return them.
TreeSet<S3ObjectSummary> treeSet =
new TreeSet<>(
new Comparator<S3ObjectSummary>() {
@Override
public int compare(S3ObjectSummary o1, S3ObjectSummary o2) {
int result = o1.getLastModified().compareTo(o2.getLastModified());
if (result != 0) {
// same modified time. Use name to sort
return result;
}
return o1.getKey().compareTo(o2.getKey());
}
});
S3Objects s3ObjectSummaries =
S3Objects.withPrefix(s3Client, s3ConfigBean.s3Config.bucket, s3ConfigBean.s3Config.folder)
.withBatchSize(BATCH_SIZE);
for (S3ObjectSummary s : s3ObjectSummaries) {
String fileName =
s.getKey().substring(s3ConfigBean.s3Config.folder.length(), s.getKey().length());
if (!fileName.isEmpty()) {
// fileName can be empty.
// If the user manually creates a folder "myFolder/mySubFolder" in bucket "myBucket" and
// uploads "myObject",
// then the first objects returned here are:
// myFolder/mySubFolder
// myFolder/mySubFolder/myObject
//
// All is good when pipeline is run but preview returns with no data. So we should ignore
// the empty file as it
// has no data
if (pathMatcher.matches(Paths.get(fileName)) && isEligible(s, s3Offset)) {
treeSet.add(s);
}
if (treeSet.size() > fetchSize) {
treeSet.pollLast();
}
}
}
return new ArrayList<>(treeSet);
}
static S3ObjectSummary getObjectSummary(
AmazonS3Client s3Client, String bucket, String objectKey) {
S3ObjectSummary s3ObjectSummary = null;
S3Objects s3ObjectSummaries = S3Objects.withPrefix(s3Client, bucket, objectKey);
for (S3ObjectSummary s : s3ObjectSummaries) {
if (s.getKey().equals(objectKey)) {
s3ObjectSummary = s;
break;
}
}
return s3ObjectSummary;
}
private static void populateFakes3()
throws IOException, InterruptedException, URISyntaxException {
BasicAWSCredentials credentials = new BasicAWSCredentials("foo", "bar");
s3client = new AmazonS3Client(credentials);
s3client.setEndpoint("http://localhost:" + port);
s3client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
createBucket(s3client, BUCKET_NAME);
createBucket(s3client, POSTPROCESS_BUCKET);
createBucket(s3client, ERROR_BUCKET);
// create directory structure
// mybucket/NorthAmerica/USA
// mybucket/NorthAmerica/Canada
//
// write 3 files each under myBucket, myBucket/NorthAmerica, mybucket/NorthAmerica/USA,
// mybucket/NorthAmerica/Canada
// 12 files in total
InputStream in = new ByteArrayInputStream("Hello World".getBytes());
PutObjectRequest putObjectRequest =
new PutObjectRequest(BUCKET_NAME, "file1.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest = new PutObjectRequest(BUCKET_NAME, "file2.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest = new PutObjectRequest(BUCKET_NAME, "file3.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(BUCKET_NAME, "NorthAmerica/file4.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(BUCKET_NAME, "NorthAmerica/file5.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(BUCKET_NAME, "NorthAmerica/file6.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(BUCKET_NAME, "NorthAmerica/USA/file7.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(BUCKET_NAME, "NorthAmerica/USA/file8.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(BUCKET_NAME, "NorthAmerica/USA/file9.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME, "NorthAmerica/Canada/file10.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME, "NorthAmerica/Canada/file11.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
in = new ByteArrayInputStream("Hello World".getBytes());
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME, "NorthAmerica/Canada/file12.log", in, new ObjectMetadata());
s3client.putObject(putObjectRequest);
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME,
"NorthAmerica/logArchive1.zip",
new FileInputStream(new File(Resources.getResource("logArchive.zip").toURI())),
new ObjectMetadata());
s3client.putObject(putObjectRequest);
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME,
"NorthAmerica/logArchive2.zip",
new FileInputStream(new File(Resources.getResource("logArchive.zip").toURI())),
new ObjectMetadata());
s3client.putObject(putObjectRequest);
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME,
"NorthAmerica/logArchive1.tar.gz",
new FileInputStream(new File(Resources.getResource("logArchive.tar.gz").toURI())),
new ObjectMetadata());
s3client.putObject(putObjectRequest);
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME,
"NorthAmerica/logArchive2.tar.gz",
new FileInputStream(new File(Resources.getResource("logArchive.tar.gz").toURI())),
new ObjectMetadata());
s3client.putObject(putObjectRequest);
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME,
"NorthAmerica/testAvro1.tar.gz",
new FileInputStream(new File(Resources.getResource("testAvro.tar.gz").toURI())),
new ObjectMetadata());
s3client.putObject(putObjectRequest);
putObjectRequest =
new PutObjectRequest(
BUCKET_NAME,
"NorthAmerica/testAvro2.tar.gz",
new FileInputStream(new File(Resources.getResource("testAvro.tar.gz").toURI())),
new ObjectMetadata());
s3client.putObject(putObjectRequest);
int count = 0;
if (s3client.doesBucketExist(BUCKET_NAME)) {
for (S3ObjectSummary s : S3Objects.withPrefix(s3client, BUCKET_NAME, "")) {
System.out.println(s.getKey());
count++;
}
}
Assert.assertEquals(18, count); // 12 files + 3 dirs
}