public void postProcessOlderObjectIfNeeded(AmazonS3Source.S3Offset s3Offset) {
// If sdc was shutdown after reading an object but before post processing it, handle it now.
// The scenario is detected as follows:
// 1. the current key must not be null
// 2. offset must be -1
// 3. An object with same key must exist in s3
// 4. The timestamp of the object ins3 must be same as that of the timestamp in offset [It is
// possible that one
// uploads another object with the same name. We can avoid post processing it without
// producing records by
// comparing the timestamp on that object
if (s3Offset.getKey() != null && "-1".equals(s3Offset.getOffset())) {
// conditions 1, 2 are met. Check for 3 and 4.
S3ObjectSummary objectSummary =
AmazonS3Util.getObjectSummary(s3Client, s3ConfigBean.s3Config.bucket, s3Offset.getKey());
if (objectSummary != null
&& objectSummary
.getLastModified()
.compareTo(new Date(Long.parseLong(s3Offset.getTimestamp())))
== 0) {
postProcessOrErrorHandle(
s3Offset.getKey(),
s3ConfigBean.postProcessingConfig.postProcessing,
s3ConfigBean.postProcessingConfig.postProcessBucket,
s3ConfigBean.postProcessingConfig.postProcessPrefix,
s3ConfigBean.postProcessingConfig.archivingOption);
}
}
currentObject = null;
}
private static boolean isEligible(S3ObjectSummary s, AmazonS3Source.S3Offset s3Offset) {
// The object is eligible if
// 1. The timestamp is greater than that of the current object in offset
// 2. The timestamp is same but the name is lexicographically greater than the current object
// [can happen when multiple objects are uploaded in one go]
// 3. Same timestamp, same name [same as the current object in offset], eligible if it was not
// completely processed [offset != -1]
boolean isEligible = false;
if (s.getLastModified().compareTo(new Date(Long.parseLong(s3Offset.getTimestamp()))) > 0) {
isEligible = true;
} else if (s.getLastModified().compareTo(new Date(Long.parseLong(s3Offset.getTimestamp())))
== 0) {
// same timestamp
// compare names
if (s.getKey().compareTo(s3Offset.getKey()) > 0) {
isEligible = true;
} else if (s.getKey().compareTo(s3Offset.getKey()) == 0) {
// same time stamp, same name
// If the current offset is not -1, return the file. It means the previous file was
// partially processed.
if (Long.parseLong(s3Offset.getOffset()) != -1) {
isEligible = true;
}
}
}
return isEligible;
}