@Test
public void testByteArrayAllocation() {
GemFireCacheImpl gfc = createCache();
try {
MemoryAllocator ma = gfc.getOffHeapStore();
assertNotNull(ma);
final long offHeapSize = ma.getFreeMemory();
assertEquals(0, ma.getUsedMemory());
byte[] data = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
MemoryChunk mc1 = (MemoryChunk) ma.allocateAndInitialize(data, false, false, null);
assertEquals(data.length + perObjectOverhead(), ma.getUsedMemory());
assertEquals(offHeapSize - (data.length + perObjectOverhead()), ma.getFreeMemory());
byte[] data2 = new byte[data.length];
mc1.readBytes(0, data2);
assertTrue(Arrays.equals(data, data2));
mc1.release();
assertEquals(offHeapSize, ma.getFreeMemory());
assertEquals(0, ma.getUsedMemory());
// try some small byte[] that don't need to be stored off heap.
data = new byte[] {1, 2, 3, 4, 5, 6, 7};
StoredObject so1 = ma.allocateAndInitialize(data, false, false, null);
assertEquals(0, ma.getUsedMemory());
assertEquals(offHeapSize, ma.getFreeMemory());
data2 = new byte[data.length];
data2 = (byte[]) so1.getDeserializedForReading();
assertTrue(Arrays.equals(data, data2));
} finally {
closeCache(gfc, false);
}
}
private void respondGetObjectAndClose(HttpExchange exchange, StoredObject storedObject)
throws IOException {
byte[] response = storedObject.getContent();
addHeader(exchange, "ETag", "\"" + storedObject.md5HexString() + "\"");
addHeader(exchange, HttpHeaders.CONTENT_TYPE, "text/plain");
respondAndClose(exchange, HttpURLConnection.HTTP_OK, response);
}
private StoredObject createCombinedObjectLarge(CombinedStoredObject combinedObject) {
URI location = combinedObject.getLocation();
log.info("starting multipart upload: %s", location);
String bucket = getS3Bucket(location);
String key = getS3ObjectKey(location);
String uploadId =
s3Service
.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucket, key))
.getUploadId();
try {
List<PartETag> parts = newArrayList();
int partNumber = 1;
for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
CopyPartResult part =
s3Service.copyPart(
new CopyPartRequest()
.withUploadId(uploadId)
.withPartNumber(partNumber)
.withDestinationBucketName(bucket)
.withDestinationKey(key)
.withSourceBucketName(getS3Bucket(newCombinedObjectPart.getLocation()))
.withSourceKey(getS3ObjectKey(newCombinedObjectPart.getLocation())));
parts.add(new PartETag(partNumber, part.getETag()));
partNumber++;
}
String etag =
s3Service
.completeMultipartUpload(
new CompleteMultipartUploadRequest(bucket, key, uploadId, parts))
.getETag();
ObjectMetadata newObject = s3Service.getObjectMetadata(bucket, key);
log.info("completed multipart upload: %s", location);
if (!etag.equals(newObject.getETag())) {
// this might happen in rare cases due to S3's eventual consistency
throw new IllegalStateException("completed etag is different from combined object etag");
}
return updateStoredObject(location, newObject);
} catch (AmazonClientException e) {
try {
s3Service.abortMultipartUpload(new AbortMultipartUploadRequest(bucket, key, uploadId));
} catch (AmazonClientException ignored) {
}
throw Throwables.propagate(e);
}
}
public StoredObject getStoredObject(ITransaction transaction, String uri) {
StoredObject so = null;
File file = new File(_root, uri);
if (file.exists()) {
so = new StoredObject();
so.setFolder(file.isDirectory());
so.setLastModified(new Date(file.lastModified()));
so.setCreationDate(new Date(file.lastModified()));
so.setResourceLength(getResourceLength(transaction, uri));
}
return so;
}
private void handlePutObject(HttpExchange exchange, String bucketName, String keyName)
throws IOException {
if (DOUBLE_DOT_PATTERN.matcher(keyName).matches()) {
respondErrorAndClose(exchange, ErrorResponse.INVALID_URI);
return;
}
byte[] content = readRequestBodyFully(exchange);
if (buckets.containsKey(bucketName)) {
Bucket bucket = buckets.get(bucketName);
StoredObject storedObject = new StoredObject(keyName, content);
bucket.put(keyName, storedObject);
addHeader(exchange, HttpHeaders.E_TAG, "\"" + storedObject.md5HexString() + "\"");
respondAndClose(exchange, HttpURLConnection.HTTP_OK);
} else {
respondErrorAndClose(exchange, ErrorResponse.NO_SUCH_BUCKET);
}
}
@Override
public StoredObject createCombinedObject(CombinedStoredObject combinedObject) {
Preconditions.checkNotNull(combinedObject, "combinedObject is null");
Preconditions.checkArgument(
!combinedObject.getSourceParts().isEmpty(), "combinedObject sourceParts is empty");
boolean setIsSmall = combinedObject.getSourceParts().get(0).getSize() < 5 * 1024 * 1024;
// verify size
for (StoredObject newCombinedObjectPart : combinedObject.getSourceParts()) {
boolean fileIsSmall = newCombinedObjectPart.getSize() < 5 * 1024 * 1024;
Preconditions.checkArgument(
fileIsSmall == setIsSmall,
"combinedObject sourceParts contains mixed large and small files");
}
return setIsSmall
? createCombinedObjectSmall(combinedObject)
: createCombinedObjectLarge(combinedObject);
}
public StoredObject getObjectDetails(URI target) {
StoredObject storedObject = new StoredObject(target);
ObjectMetadata metadata =
s3Service.getObjectMetadata(getS3Bucket(target), getS3ObjectKey(target));
return updateStoredObject(storedObject.getLocation(), metadata);
}
