@Test
public void testSimpleGetVersion() throws ServiceException, IOException {
String bucket = "bucket";
String keyPrefix = "prefix/dir/0";
RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
S3Object object0 = new S3Object();
object0.setBucketName(bucket);
object0.setKey(keyPrefix + "/renames-0.gz");
object0.setLastModifiedDate(new Date(0));
EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(bucket), EasyMock.eq(object0.getKey())))
.andReturn(object0)
.once();
S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
EasyMock.replay(s3Client);
String version =
puller.getVersion(URI.create(String.format("s3://%s/%s", bucket, object0.getKey())));
EasyMock.verify(s3Client);
Assert.assertEquals(String.format("%d", new Date(0).getTime()), version);
}
@Test
public void testGZUncompressRetries()
throws ServiceException, IOException, SegmentLoadingException {
final String bucket = "bucket";
final String keyPrefix = "prefix/dir/0";
final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
final byte[] value = bucket.getBytes("utf8");
final File tmpFile = temporaryFolder.newFile("gzTest.gz");
try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
outputStream.write(value);
}
S3Object object0 = new S3Object();
object0.setBucketName(bucket);
object0.setKey(keyPrefix + "/renames-0.gz");
object0.setLastModifiedDate(new Date(0));
object0.setDataInputStream(new FileInputStream(tmpFile));
File tmpDir = temporaryFolder.newFolder("gzTestDir");
S3ServiceException exception = new S3ServiceException();
exception.setErrorCode("NoSuchKey");
exception.setResponseCode(404);
EasyMock.expect(
s3Client.getObjectDetails(
EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
.andReturn(null)
.once();
EasyMock.expect(
s3Client.getObjectDetails(
EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
.andReturn(object0)
.once();
EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey())))
.andThrow(exception)
.once();
EasyMock.expect(
s3Client.getObjectDetails(
EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
.andReturn(object0)
.once();
EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey())))
.andReturn(object0)
.once();
S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);
EasyMock.replay(s3Client);
FileUtils.FileCopyResult result =
puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, object0.getKey()), tmpDir);
EasyMock.verify(s3Client);
Assert.assertEquals(value.length, result.size());
File expected = new File(tmpDir, "renames-0");
Assert.assertTrue(expected.exists());
Assert.assertEquals(value.length, expected.length());
}
public void endElement(String uri, String name, String qName) {
String elementText = this.currText.toString();
// Listing details
if (name.equals("Name")) {
bucketName = elementText;
if (log.isDebugEnabled()) {
log.debug("Examining listing for bucket: " + bucketName);
}
} else if (!insideCommonPrefixes && name.equals("Prefix")) {
requestPrefix = elementText;
} else if (name.equals("Marker")) {
requestMarker = elementText;
} else if (name.equals("NextMarker")) {
nextMarker = elementText;
} else if (name.equals("MaxKeys")) {
requestMaxKeys = Long.parseLong(elementText);
} else if (name.equals("IsTruncated")) {
String isTruncatedStr = elementText.toLowerCase(Locale.getDefault());
if (isTruncatedStr.startsWith("false")) {
listingTruncated = false;
} else if (isTruncatedStr.startsWith("true")) {
listingTruncated = true;
} else {
throw new RuntimeException("Invalid value for IsTruncated field: " + isTruncatedStr);
}
}
// Object details.
else if (name.equals("Contents")) {
objects.add(currentObject);
if (log.isDebugEnabled()) {
log.debug("Created new S3Object from listing: " + currentObject);
}
} else if (name.equals("Key")) {
currentObject.setKey(elementText);
lastKey = elementText;
} else if (name.equals("LastModified")) {
try {
currentObject.setLastModifiedDate(ServiceUtils.parseIso8601Date(elementText));
} catch (ParseException e) {
throw new RuntimeException("Unexpected date format in list bucket output", e);
}
} else if (name.equals("ETag")) {
currentObject.setETag(elementText);
} else if (name.equals("Size")) {
currentObject.setContentLength(Long.parseLong(elementText));
} else if (name.equals("StorageClass")) {
currentObject.setStorageClass(elementText);
}
// Owner details.
else if (name.equals("ID")) {
// Work-around to support Eucalyptus responses, which do not
// contain Owner elements.
if (currentOwner == null) {
currentOwner = new S3Owner();
currentObject.setOwner(currentOwner);
}
currentOwner.setId(elementText);
} else if (name.equals("DisplayName")) {
currentOwner.setDisplayName(elementText);
}
// Common prefixes.
else if (insideCommonPrefixes && name.equals("Prefix")) {
commonPrefixes.add(elementText);
} else if (name.equals("CommonPrefixes")) {
insideCommonPrefixes = false;
}
this.currText = new StringBuffer();
}