public String putBlob(String container, Blob blob) {
try {
String name = FilenameUtils.getName(blob.getMetadata().getName());
String path = FilenameUtils.getPathNoEndSeparator(blob.getMetadata().getName());
service.uploadInputStream(
blob.getPayload().getInput(),
container,
path,
name,
blob.getPayload().getContentMetadata().getContentLength());
return null;
} catch (UploadException e) {
e.printStackTrace();
return null;
} catch (MethodNotSupportedException e) {
e.printStackTrace();
return null;
} catch (FileNotExistsException e) {
e.printStackTrace();
return null;
}
}
/** Test of getBlob method, of class FilesystemAsyncBlobStore. */
public void testGetBlob() throws IOException {
String blobKey = TestUtils.createRandomBlobKey();
GetOptions options = null;
Blob resultBlob;
blobStore.createContainerInLocation(null, CONTAINER_NAME);
resultBlob = blobStore.getBlob(CONTAINER_NAME, blobKey, options);
assertNull(resultBlob, "Blob exists");
// create blob
TestUtils.createBlobsInContainer(CONTAINER_NAME, blobKey);
resultBlob = blobStore.getBlob(CONTAINER_NAME, blobKey, options);
assertNotNull(resultBlob, "Blob exists");
// checks file content
InputSupplier<FileInputStream> expectedFile =
Files.newInputStreamSupplier(new File(TARGET_CONTAINER_NAME, blobKey));
assertTrue(
ByteStreams.equal(expectedFile, resultBlob.getPayload()),
"Blob payload differs from file content");
// metadata are verified in the test for blobMetadata, so no need to
// perform a complete test here
assertNotNull(resultBlob.getMetadata(), "Metadata null");
MutableBlobMetadata metadata = resultBlob.getMetadata();
assertEquals(blobKey, metadata.getName(), "Wrong blob metadata");
}
private String putDirectoryBlob(final String containerName, final Blob blob) throws IOException {
String blobKey = blob.getMetadata().getName();
ContentMetadata metadata = blob.getMetadata().getContentMetadata();
Long contentLength = metadata.getContentLength();
if (contentLength != null && contentLength != 0) {
throw new IllegalArgumentException("Directory blob cannot have content: " + blobKey);
}
File outputFile = getFileForBlobKey(containerName, blobKey);
Path outputPath = outputFile.toPath();
if (!outputFile.isDirectory() && !outputFile.mkdirs()) {
throw new IOException("Unable to mkdir: " + outputPath);
}
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(outputPath);
if (view != null) {
try {
view.write(XATTR_CONTENT_MD5, ByteBuffer.wrap(DIRECTORY_MD5));
writeCommonMetadataAttr(view, blob);
} catch (IOException e) {
logger.debug("xattrs not supported on %s", outputPath);
}
} else {
logger.warn("xattr not supported on %s", blobKey);
}
return base16().lowerCase().encode(DIRECTORY_MD5);
}
public void execute(String containerName, String directory) {
Blob blob = connection.newBlob(directory + directorySuffix);
blob.setPayload(Payloads.newByteArrayPayload(new byte[] {}));
blob.getPayload().setContentType("application/directory");
blob.getMetadata().setType(StorageType.RELATIVE_PATH);
connection.putBlob(containerName, blob);
}
public CFObject apply(Blob from) {
CFObject object = objectProvider.create(blob2ObjectMd.apply(from.getMetadata()));
if (from.getContentLength() != null) object.setContentLength(from.getContentLength());
object.setData(from.getData());
object.setAllHeaders(from.getAllHeaders());
return object;
}
@Test(groups = {"integration", "live"})
public void testPut() throws IOException, InterruptedException {
String bucketName = getContainerName();
try {
Map<String, Blob> map = createMap(context, bucketName);
Blob blob = context.getBlobStore().newBlob("one");
blob.setPayload(Strings2.toInputStream("apple"));
Payloads.calculateMD5(blob);
Blob old = map.put(blob.getMetadata().getName(), blob);
getOneReturnsAppleAndOldValueIsNull(map, old);
blob.setPayload(Strings2.toInputStream("bear"));
Payloads.calculateMD5(blob);
Blob apple = map.put(blob.getMetadata().getName(), blob);
getOneReturnsBearAndOldValueIsApple(map, apple);
} finally {
returnContainer(bucketName);
}
}
public void testNewBlob() {
String blobKey;
Blob newBlob;
blobKey = TestUtils.createRandomBlobKey("blobtest-", ".txt");
newBlob = storageStrategy.newBlob(blobKey);
assertNotNull(newBlob, "Created blob was null");
assertNotNull(newBlob.getMetadata(), "Created blob metadata were null");
assertEquals(newBlob.getMetadata().getName(), blobKey, "Created blob name is different");
blobKey = TestUtils.createRandomBlobKey("blobtest-", "");
newBlob = storageStrategy.newBlob(blobKey);
assertEquals(newBlob.getMetadata().getName(), blobKey, "Created blob name is different");
blobKey =
TestUtils.createRandomBlobKey("asd" + FS + "asd" + FS + "asdasd" + FS + "afadsf-", "");
newBlob = storageStrategy.newBlob(blobKey);
assertEquals(newBlob.getMetadata().getName(), blobKey, "Created blob name is different");
}
private void writeCommonMetadataAttr(UserDefinedFileAttributeView view, Blob blob)
throws IOException {
ContentMetadata metadata = blob.getMetadata().getContentMetadata();
writeStringAttributeIfPresent(view, XATTR_CACHE_CONTROL, metadata.getCacheControl());
writeStringAttributeIfPresent(
view, XATTR_CONTENT_DISPOSITION, metadata.getContentDisposition());
writeStringAttributeIfPresent(view, XATTR_CONTENT_ENCODING, metadata.getContentEncoding());
writeStringAttributeIfPresent(view, XATTR_CONTENT_LANGUAGE, metadata.getContentLanguage());
writeStringAttributeIfPresent(view, XATTR_CONTENT_TYPE, metadata.getContentType());
Date expires = metadata.getExpires();
if (expires != null) {
ByteBuffer buf = ByteBuffer.allocate(Longs.BYTES).putLong(expires.getTime());
buf.flip();
view.write(XATTR_EXPIRES, buf);
}
for (Map.Entry<String, String> entry : blob.getMetadata().getUserMetadata().entrySet()) {
writeStringAttributeIfPresent(
view, XATTR_USER_METADATA_PREFIX + entry.getKey(), entry.getValue());
}
}
/** {@inheritDoc} */
@Override
public ListenableFuture<BlobMetadata> blobMetadata(String container, String key) {
try {
Blob blob = getBlob(container, key).get();
return Futures.<BlobMetadata>immediateFuture(blob != null ? blob.getMetadata() : null);
} catch (Exception e) {
if (size(filter(getCausalChain(e), KeyNotFoundException.class)) >= 1)
return immediateFuture(null);
return immediateFailedFuture(e);
}
}
public void testRanges() throws IOException {
blobStore.createContainerInLocation(null, CONTAINER_NAME);
String input = "abcdefgh";
Payload payload;
Blob blob = blobStore.blobBuilder("test").payload(new StringPayload(input)).build();
blobStore.putBlob(CONTAINER_NAME, blob);
GetOptions getOptionsRangeStartAt = new GetOptions();
getOptionsRangeStartAt.startAt(1);
Blob blobRangeStartAt =
blobStore.getBlob(CONTAINER_NAME, blob.getMetadata().getName(), getOptionsRangeStartAt);
payload = blobRangeStartAt.getPayload();
try {
assertEquals(input.substring(1), Strings2.toString(payload));
} finally {
Closeables.closeQuietly(payload);
}
GetOptions getOptionsRangeTail = new GetOptions();
getOptionsRangeTail.tail(3);
Blob blobRangeTail =
blobStore.getBlob(CONTAINER_NAME, blob.getMetadata().getName(), getOptionsRangeTail);
payload = blobRangeTail.getPayload();
try {
assertEquals(input.substring(5), Strings2.toString(payload));
} finally {
Closeables.closeQuietly(payload);
}
GetOptions getOptionsFragment = new GetOptions();
getOptionsFragment.range(4, 6);
Blob blobFragment =
blobStore.getBlob(CONTAINER_NAME, blob.getMetadata().getName(), getOptionsFragment);
payload = blobFragment.getPayload();
try {
assertEquals(input.substring(4, 7), Strings2.toString(payload));
} finally {
Closeables.closeQuietly(payload);
}
}
/**
* Calculates the object MD5 and returns it as eTag
*
* @param object
* @return
*/
private String getEtag(Blob object) {
try {
Payloads.calculateMD5(object, crypto.md5());
} catch (IOException ex) {
logger.error(
ex,
"An error occurred calculating MD5 for object with name %s.",
object.getMetadata().getName());
Throwables.propagate(ex);
}
String eTag = CryptoStreams.hex(object.getPayload().getContentMetadata().getContentMD5());
return eTag;
}
public Blob getBlob(String container, String name) {
try {
StorageObject storageObject = service.getStorageObject(container, name);
Blob blob = new BlobImpl(generateJcloudsMetadata(storageObject.getMetadata()));
if (storageObject.getStream() != null) {
blob.setPayload(storageObject.getStream());
blob.getMetadata()
.getContentMetadata()
.setContentLength(storageObject.getMetadata().getLength());
}
return blob;
} catch (FileNotExistsException e) {
e.printStackTrace();
return null;
}
}
/**
* Load the blob with the given key belonging to the container with the given name. There must
* exist a resource on the file system whose complete name is given concatenating the container
* name and the key
*
* @param container it's the name of the container the blob belongs to
* @param key it's the key of the blob
* @return the blob belonging to the given container with the given key
*/
private Blob loadFileBlob(String container, String key) {
logger.debug("Opening blob in container: %s - %s", container, key);
BlobBuilder builder = blobUtils.blobBuilder();
builder.name(key);
File file = storageStrategy.getFileForBlobKey(container, key);
try {
builder.payload(file).calculateMD5();
} catch (IOException e) {
logger.error("An error occurred calculating MD5 for blob %s from container ", key, container);
Throwables.propagateIfPossible(e);
}
Blob blob = builder.build();
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata()
.setETag(CryptoStreams.hex(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;
}
public void testCreateParentIfNeededAsyncNoPath() {
AsyncBlobStore asyncBlobStore = createMock(AsyncBlobStore.class);
String container = "container";
Blob blob = createMock(Blob.class);
MutableBlobMetadata md = createMock(MutableBlobMetadata.class);
expect(blob.getMetadata()).andReturn(md).atLeastOnce();
expect(md.getName()).andReturn("hello").atLeastOnce();
replay(asyncBlobStore);
replay(blob);
replay(md);
createParentIfNeededAsync(asyncBlobStore, container, blob);
verify(asyncBlobStore);
verify(blob);
verify(md);
}
public void testSignPutBlob()
throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
NoSuchMethodException, IOException {
Blob blob = blobFactory.create(null);
blob.getMetadata().setName("name");
blob.setPayload("");
blob.getPayload().getContentMetadata().setContentLength(2l);
blob.getPayload().getContentMetadata().setContentMD5(new byte[] {0, 2, 4, 8});
blob.getPayload().getContentMetadata().setContentType("text/plain");
HttpRequest request = signer.signPutBlob("container", blob);
assertRequestLineEquals(request, "PUT http://storageUrl/container/name HTTP/1.1");
assertNonPayloadHeadersEqual(request, "X-Auth-Token: testtoken\n");
assertContentHeadersEqual(
request, "text/plain", null, null, null, (long) 2l, new byte[] {0, 2, 4, 8});
assertEquals(request.getFilters().size(), 0);
}
/** {@inheritDoc} */
@Override
public ListenableFuture<String> putBlob(String containerName, Blob object) {
String blobKey = object.getMetadata().getName();
logger.debug("Put object with key [%s] to container [%s]", blobKey, containerName);
String eTag = getEtag(object);
try {
// TODO
// must override existing file?
storageStrategy.writePayloadOnFile(containerName, blobKey, object.getPayload());
} catch (IOException e) {
logger.error(
e,
"An error occurred storing the new object with name [%s] to container [%s].",
blobKey,
containerName);
Throwables.propagate(e);
}
return immediateFuture(eTag);
}
public void testSignPutBlob()
throws ArrayIndexOutOfBoundsException, SecurityException, IllegalArgumentException,
NoSuchMethodException, IOException {
Blob blob = blobFactory.create(null);
blob.getMetadata().setName("name");
blob.setPayload("");
blob.getPayload().getContentMetadata().setContentLength(2l);
blob.getPayload().getContentMetadata().setContentMD5(new byte[] {0, 2, 4, 8});
blob.getPayload().getContentMetadata().setContentType("text/plain");
HttpRequest request = signer.signPutBlob("container", blob);
assertRequestLineEquals(
request, "POST https://accesspoint.atmosonline.com/rest/namespace/container/name HTTP/1.1");
assertNonPayloadHeadersEqual(
request,
"Accept: */*\nDate: Thu, 05 Jun 2008 16:38:19 GMT\nx-emc-signature: aLpB1oQaCA27AXT6Nzam7s0f0pI=\nx-emc-uid: identity\n");
assertContentHeadersEqual(
request, "text/plain", null, null, null, (long) 2l, new byte[] {0, 2, 4, 8});
assertEquals(request.getFilters().size(), 0);
}
@Override
public Blob getBlob(final String container, final String key) {
BlobBuilder builder = blobBuilders.get();
builder.name(key);
File file = getFileForBlobKey(container, key);
ByteSource byteSource;
if (getDirectoryBlobSuffix(key) != null) {
logger.debug("%s - %s is a directory", container, key);
byteSource = ByteSource.empty();
} else {
byteSource = Files.asByteSource(file);
}
try {
String cacheControl = null;
String contentDisposition = null;
String contentEncoding = null;
String contentLanguage = null;
String contentType = null;
HashCode hashCode = null;
Date expires = null;
ImmutableMap.Builder<String, String> userMetadata = ImmutableMap.builder();
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath());
if (view != null) {
Set<String> attributes = ImmutableSet.copyOf(view.list());
cacheControl = readStringAttributeIfPresent(view, attributes, XATTR_CACHE_CONTROL);
contentDisposition =
readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_DISPOSITION);
contentEncoding = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_ENCODING);
contentLanguage = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_LANGUAGE);
contentType = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_TYPE);
if (contentType == null && autoDetectContentType) {
contentType = probeContentType(file.toPath());
}
if (attributes.contains(XATTR_CONTENT_MD5)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_CONTENT_MD5));
view.read(XATTR_CONTENT_MD5, buf);
hashCode = HashCode.fromBytes(buf.array());
}
if (attributes.contains(XATTR_EXPIRES)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_EXPIRES));
view.read(XATTR_EXPIRES, buf);
buf.flip();
expires = new Date(buf.asLongBuffer().get());
}
for (String attribute : attributes) {
if (!attribute.startsWith(XATTR_USER_METADATA_PREFIX)) {
continue;
}
String value = readStringAttributeIfPresent(view, attributes, attribute);
userMetadata.put(attribute.substring(XATTR_USER_METADATA_PREFIX.length()), value);
}
builder
.payload(byteSource)
.cacheControl(cacheControl)
.contentDisposition(contentDisposition)
.contentEncoding(contentEncoding)
.contentLanguage(contentLanguage)
.contentLength(byteSource.size())
.contentMD5(hashCode)
.contentType(contentType)
.expires(expires)
.userMetadata(userMetadata.build());
} else {
builder
.payload(byteSource)
.contentLength(byteSource.size())
.contentMD5(byteSource.hash(Hashing.md5()).asBytes());
}
} catch (IOException e) {
throw Throwables.propagate(e);
}
Blob blob = builder.build();
blob.getMetadata().setContainer(container);
blob.getMetadata().setLastModified(new Date(file.lastModified()));
blob.getMetadata().setSize(file.length());
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata()
.setETag(
base16().lowerCase().encode(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;
}
@Override
public String putBlob(final String containerName, final Blob blob) throws IOException {
String blobKey = blob.getMetadata().getName();
Payload payload = blob.getPayload();
filesystemContainerNameValidator.validate(containerName);
filesystemBlobKeyValidator.validate(blobKey);
if (getDirectoryBlobSuffix(blobKey) != null) {
return putDirectoryBlob(containerName, blob);
}
File outputFile = getFileForBlobKey(containerName, blobKey);
// TODO: should we use a known suffix to filter these out during list?
String tmpBlobName = blobKey + "-" + UUID.randomUUID();
File tmpFile = getFileForBlobKey(containerName, tmpBlobName);
Path tmpPath = tmpFile.toPath();
HashingInputStream his = null;
try {
Files.createParentDirs(tmpFile);
his = new HashingInputStream(Hashing.md5(), payload.openStream());
long actualSize = Files.asByteSink(tmpFile).writeFrom(his);
Long expectedSize = blob.getMetadata().getContentMetadata().getContentLength();
if (expectedSize != null && actualSize != expectedSize) {
throw new IOException(
"Content-Length mismatch, actual: " + actualSize + " expected: " + expectedSize);
}
HashCode actualHashCode = his.hash();
HashCode expectedHashCode = payload.getContentMetadata().getContentMD5AsHashCode();
if (expectedHashCode != null && !actualHashCode.equals(expectedHashCode)) {
throw new IOException(
"MD5 hash code mismatch, actual: " + actualHashCode + " expected: " + expectedHashCode);
}
payload.getContentMetadata().setContentMD5(actualHashCode);
if (outputFile.exists()) {
delete(outputFile);
}
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(tmpPath);
if (view != null) {
try {
view.write(XATTR_CONTENT_MD5, ByteBuffer.wrap(actualHashCode.asBytes()));
writeCommonMetadataAttr(view, blob);
} catch (IOException e) {
logger.debug("xattrs not supported on %s", tmpPath);
}
}
setBlobAccess(containerName, tmpBlobName, BlobAccess.PRIVATE);
if (!tmpFile.renameTo(outputFile)) {
throw new RuntimeException("Could not rename file " + tmpFile + " to " + outputFile);
}
return base16().lowerCase().encode(actualHashCode.asBytes());
} catch (IOException ex) {
if (tmpFile != null) {
try {
delete(tmpFile);
} catch (IOException e) {
logger.debug("Could not delete %s: %s", tmpFile, e);
}
}
throw ex;
} finally {
closeQuietly(his);
if (payload != null) {
payload.release();
}
}
}
/** {@inheritDoc} */
@Override
public ListenableFuture<Blob> getBlob(
final String containerName, final String key, GetOptions options) {
logger.debug("Retrieving blob with key %s from container %s", key, containerName);
// If the container doesn't exist, an exception is thrown
if (!containerExistsSyncImpl(containerName)) {
logger.debug("Container %s does not exist", containerName);
return immediateFailedFuture(cnfe(containerName));
}
// If the blob doesn't exist, a null object is returned
if (!storageStrategy.blobExists(containerName, key)) {
logger.debug("Item %s does not exist in container %s", key, containerName);
return immediateFuture(null);
}
Blob blob = loadFileBlob(containerName, key);
if (options != null) {
if (options.getIfMatch() != null) {
if (!blob.getMetadata().getETag().equals(options.getIfMatch()))
return immediateFailedFuture(returnResponseException(412));
}
if (options.getIfNoneMatch() != null) {
if (blob.getMetadata().getETag().equals(options.getIfNoneMatch()))
return immediateFailedFuture(returnResponseException(304));
}
if (options.getIfModifiedSince() != null) {
Date modifiedSince = options.getIfModifiedSince();
if (blob.getMetadata().getLastModified().before(modifiedSince)) {
HttpResponse response = new HttpResponse(304, null, null);
return immediateFailedFuture(
new HttpResponseException(
String.format(
"%1$s is before %2$s", blob.getMetadata().getLastModified(), modifiedSince),
null,
response));
}
}
if (options.getIfUnmodifiedSince() != null) {
Date unmodifiedSince = options.getIfUnmodifiedSince();
if (blob.getMetadata().getLastModified().after(unmodifiedSince)) {
HttpResponse response = new HttpResponse(412, null, null);
return immediateFailedFuture(
new HttpResponseException(
String.format(
"%1$s is after %2$s", blob.getMetadata().getLastModified(), unmodifiedSince),
null,
response));
}
}
if (options.getRanges() != null && options.getRanges().size() > 0) {
byte[] data;
try {
data = toByteArray(blob.getPayload().getInput());
} catch (IOException e) {
return immediateFailedFuture(new RuntimeException(e));
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
for (String s : options.getRanges()) {
if (s.startsWith("-")) {
int length = Integer.parseInt(s.substring(1));
out.write(data, data.length - length, length);
} else if (s.endsWith("-")) {
int offset = Integer.parseInt(s.substring(0, s.length() - 1));
out.write(data, offset, data.length - offset);
} else if (s.contains("-")) {
String[] firstLast = s.split("\\-");
int offset = Integer.parseInt(firstLast[0]);
int last = Integer.parseInt(firstLast[1]);
int length = last - offset + 1; // the range end is included
out.write(data, offset, length);
} else {
return immediateFailedFuture(new IllegalArgumentException("first and last were null!"));
}
}
blob.setPayload(out.toByteArray());
blob.getMetadata().getContentMetadata().setContentLength(new Long(data.length));
}
}
checkNotNull(blob.getPayload(), "payload " + blob);
return immediateFuture(blob);
}