public String signString(String toSign) {
String signature;
try {
signature =
CryptoStreams.base64(CryptoStreams.mac(InputSuppliers.of(toSign), crypto.hmacSHA1(key)));
} catch (Exception e) {
throw new HttpException("error signing request", e);
}
return signature;
}
@Test
@Parameters({"jclouds.blobstore.httpstream.url", "jclouds.blobstore.httpstream.md5"})
public void testCopyUrl(@Optional String httpStreamUrl, @Optional String httpStreamETag)
throws Exception {
httpStreamUrl =
checkNotNull(httpStreamUrl != null ? httpStreamUrl : sysHttpStreamUrl, "httpStreamUrl");
httpStreamETag =
checkNotNull(httpStreamETag != null ? httpStreamETag : sysHttpStreamETag, "httpStreamMd5");
String name = "hello";
URL url = new URL(httpStreamUrl);
byte[] md5 = CryptoStreams.hex(httpStreamETag);
URLConnection connection = url.openConnection();
long length = connection.getContentLength();
InputStream input = connection.getInputStream();
Blob blob =
context
.getBlobStore()
.blobBuilder(name)
.payload(input)
.contentLength(length)
.contentMD5(md5)
.build();
String container = getContainerName();
try {
context.getBlobStore().putBlob(container, blob);
checkMD5(container, name, md5);
} finally {
returnContainer(container);
}
}
@Override
public void setPropertiesFromHttpHeaders(Multimap<String, String> headers) {
boolean chunked =
any(
headers.entries(),
new Predicate<Entry<String, String>>() {
@Override
public boolean apply(Entry<String, String> input) {
return "Transfer-Encoding".equalsIgnoreCase(input.getKey())
&& "chunked".equalsIgnoreCase(input.getValue());
}
});
for (Entry<String, String> header : headers.entries()) {
if (!chunked && CONTENT_LENGTH.equalsIgnoreCase(header.getKey())) {
setContentLength(new Long(header.getValue()));
} else if ("Content-MD5".equalsIgnoreCase(header.getKey())) {
setContentMD5(CryptoStreams.base64(header.getValue()));
} else if (CONTENT_TYPE.equalsIgnoreCase(header.getKey())) {
setContentType(header.getValue());
} else if ("Content-Disposition".equalsIgnoreCase(header.getKey())) {
setContentDisposition(header.getValue());
} else if ("Content-Encoding".equalsIgnoreCase(header.getKey())) {
setContentEncoding(header.getValue());
} else if ("Content-Language".equalsIgnoreCase(header.getKey())) {
setContentLanguage(header.getValue());
}
}
}
public static Account createTestAccount(CloudStackGlobalClient client, String prefix) {
return client
.getAccountClient()
.createAccount(
prefix + "-account",
Account.Type.USER,
"*****@*****.**",
"First",
"Last",
CryptoStreams.md5Hex("password"));
}
public Request apply(HttpRequest request) {
for (HttpRequestFilter filter : request.getFilters()) {
filter.filter(request);
}
RequestBuilder builder = new RequestBuilder(request.getMethod());
builder.setUrl(request.getEndpoint().toASCIIString());
Payload payload = request.getPayload();
if (payload != null) {
boolean chunked = "chunked".equals(request.getFirstHeaderOrNull("Transfer-Encoding"));
if (request.getPayload().getContentMetadata().getContentMD5() != null)
builder.addHeader(
"Content-MD5",
CryptoStreams.base64(request.getPayload().getContentMetadata().getContentMD5()));
if (request.getPayload().getContentMetadata().getContentType() != null)
builder.addHeader(
HttpHeaders.CONTENT_TYPE, request.getPayload().getContentMetadata().getContentType());
if (request.getPayload().getContentMetadata().getContentLanguage() != null)
builder.addHeader(
HttpHeaders.CONTENT_LANGUAGE,
request.getPayload().getContentMetadata().getContentLanguage());
if (request.getPayload().getContentMetadata().getContentEncoding() != null)
builder.addHeader(
HttpHeaders.CONTENT_ENCODING,
request.getPayload().getContentMetadata().getContentEncoding());
if (request.getPayload().getContentMetadata().getContentDisposition() != null)
builder.addHeader(
"Content-Disposition",
request.getPayload().getContentMetadata().getContentDisposition());
if (!chunked) {
Long length =
checkNotNull(
request.getPayload().getContentMetadata().getContentLength(),
"payload.getContentLength");
builder.addHeader(HttpHeaders.CONTENT_LENGTH, length.toString());
}
setPayload(builder, payload);
} else {
builder.addHeader(HttpHeaders.CONTENT_LENGTH, "0");
}
builder.addHeader(HttpHeaders.USER_AGENT, USER_AGENT);
for (String header : request.getHeaders().keySet()) {
for (String value : request.getHeaders().get(header)) {
builder.addHeader(header, value);
}
}
return builder.build();
}
public Request apply(HttpRequest request) {
for (HttpRequestFilter filter : request.getFilters()) {
filter.filter(request);
}
AsyncHttpClient client = new AsyncHttpClient();
AsyncHttpClient.BoundRequestBuilder nativeRequestBuilder;
String endpoint = request.getEndpoint().toASCIIString();
if (request.getMethod().equals(HttpMethod.HEAD)) {
nativeRequestBuilder = client.prepareHead(endpoint);
} else if (request.getMethod().equals(HttpMethod.GET)) {
nativeRequestBuilder = client.prepareGet(endpoint);
} else if (request.getMethod().equals(HttpMethod.DELETE)) {
nativeRequestBuilder = client.prepareDelete(endpoint);
} else if (request.getMethod().equals(HttpMethod.PUT)) {
nativeRequestBuilder = client.preparePut(endpoint);
} else if (request.getMethod().equals(HttpMethod.POST)) {
nativeRequestBuilder = client.preparePost(endpoint);
} else {
throw new UnsupportedOperationException(request.getMethod());
}
Payload payload = request.getPayload();
if (payload != null) {
boolean chunked = "chunked".equals(request.getFirstHeaderOrNull("Transfer-Encoding"));
if (request.getPayload().getContentMD5() != null)
nativeRequestBuilder.addHeader(
"Content-MD5", CryptoStreams.base64(request.getPayload().getContentMD5()));
if (request.getPayload().getContentType() != null)
nativeRequestBuilder.addHeader(
HttpHeaders.CONTENT_TYPE, request.getPayload().getContentType());
if (!chunked) {
Long length =
checkNotNull(request.getPayload().getContentLength(), "payload.getContentLength");
nativeRequestBuilder.addHeader(HttpHeaders.CONTENT_LENGTH, length.toString());
}
setPayload(nativeRequestBuilder, payload);
} else {
nativeRequestBuilder.addHeader(HttpHeaders.CONTENT_LENGTH, "0");
}
nativeRequestBuilder.addHeader(HttpHeaders.USER_AGENT, USER_AGENT);
for (String header : request.getHeaders().keySet()) {
for (String value : request.getHeaders().get(header)) {
nativeRequestBuilder.addHeader(header, value);
}
}
return nativeRequestBuilder.build();
}
/**
* Calculates the object MD5 and returns it as eTag
*
* @param object
* @return
*/
private String getEtag(Blob object) {
try {
Payloads.calculateMD5(object, crypto.md5());
} catch (IOException ex) {
logger.error(
ex,
"An error occurred calculating MD5 for object with name %s.",
object.getMetadata().getName());
Throwables.propagate(ex);
}
String eTag = CryptoStreams.hex(object.getPayload().getContentMetadata().getContentMD5());
return eTag;
}
@Inject
public SignRequest(
SignatureWire signatureWire,
@Named(PROPERTY_IDENTITY) String uid,
@Named(PROPERTY_CREDENTIAL) String encodedKey,
@TimeStamp Provider<String> timeStampProvider,
Crypto crypto,
HttpUtils utils) {
this.signatureWire = signatureWire;
this.uid = uid;
this.key = CryptoStreams.base64(encodedKey);
this.timeStampProvider = timeStampProvider;
this.crypto = crypto;
this.utils = utils;
}
/**
* Load the blob with the given key belonging to the container with the given name. There must
* exist a resource on the file system whose complete name is given concatenating the container
* name and the key
*
* @param container it's the name of the container the blob belongs to
* @param key it's the key of the blob
* @return the blob belonging to the given container with the given key
*/
private Blob loadFileBlob(String container, String key) {
logger.debug("Opening blob in container: %s - %s", container, key);
BlobBuilder builder = blobUtils.blobBuilder();
builder.name(key);
File file = storageStrategy.getFileForBlobKey(container, key);
try {
builder.payload(file).calculateMD5();
} catch (IOException e) {
logger.error("An error occurred calculating MD5 for blob %s from container ", key, container);
Throwables.propagateIfPossible(e);
}
Blob blob = builder.build();
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata()
.setETag(CryptoStreams.hex(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;
}
@Test(
timeOut = 5 * 60 * 1000,
dependsOnMethods = {"testCreateContainer", "testCreatePublicContainer"})
public void testObjectOperations() throws Exception {
String data = "Here is my data";
// Test PUT with string data, ETag hash, and a piece of metadata
AzureBlob object = client.newBlob();
object.getProperties().setName("object");
object.setPayload(data);
Payloads.calculateMD5(object);
object.getProperties().getContentMetadata().setContentType("text/plain");
object.getProperties().getMetadata().put("mykey", "metadata-value");
byte[] md5 = object.getProperties().getContentMetadata().getContentMD5();
String newEtag = client.putBlob(privateContainer, object);
assertEquals(
CryptoStreams.hex(md5),
CryptoStreams.hex(object.getProperties().getContentMetadata().getContentMD5()));
// Test HEAD of missing object
assert client.getBlobProperties(privateContainer, "non-existent-object") == null;
// Test HEAD of object
BlobProperties metadata =
client.getBlobProperties(privateContainer, object.getProperties().getName());
// TODO assertEquals(metadata.getName(), object.getProperties().getName());
// we can't check this while hacking around lack of content-md5, as GET of the first byte will
// show incorrect length 1, the returned size, as opposed to the real length. This is an ok
// tradeoff, as a container list will contain the correct size of the objects in an
// inexpensive fashion
// http://code.google.com/p/jclouds/issues/detail?id=92
// assertEquals(metadata.getSize(), data.length());
assertEquals(metadata.getContentMetadata().getContentType(), "text/plain");
// Azure doesn't return the Content-MD5 on head request..
assertEquals(
CryptoStreams.hex(md5),
CryptoStreams.hex(object.getProperties().getContentMetadata().getContentMD5()));
assertEquals(metadata.getETag(), newEtag);
assertEquals(metadata.getMetadata().entrySet().size(), 1);
assertEquals(metadata.getMetadata().get("mykey"), "metadata-value");
// // Test POST to update object's metadata
// Multimap<String, String> userMetadata = LinkedHashMultimap.create();
// userMetadata.put("New-Metadata-1", "value-1");
// userMetadata.put("New-Metadata-2", "value-2");
// assertTrue(client.setBlobProperties(privateContainer, object.getProperties().getName(),
// userMetadata));
// Test GET of missing object
assert client.getBlob(privateContainer, "non-existent-object") == null;
// Test GET of object (including updated metadata)
AzureBlob getBlob = client.getBlob(privateContainer, object.getProperties().getName());
assertEquals(Utils.toStringAndClose(getBlob.getPayload().getInput()), data);
// TODO assertEquals(getBlob.getName(), object.getProperties().getName());
assertEquals(
getBlob.getPayload().getContentMetadata().getContentLength(), new Long(data.length()));
assertEquals(getBlob.getProperties().getContentMetadata().getContentType(), "text/plain");
assertEquals(
CryptoStreams.hex(md5),
CryptoStreams.hex(getBlob.getProperties().getContentMetadata().getContentMD5()));
assertEquals(newEtag, getBlob.getProperties().getETag());
// wait until we can update metadata
// assertEquals(getBlob.getProperties().getMetadata().entries().size(), 2);
// assertEquals(
// Iterables.getLast(getBlob.getProperties().getMetadata().get("New-Metadata-1")),
// "value-1");
// assertEquals(
// Iterables.getLast(getBlob.getProperties().getMetadata().get("New-Metadata-2")),
// "value-2");
assertEquals(metadata.getMetadata().entrySet().size(), 1);
assertEquals(metadata.getMetadata().get("mykey"), "metadata-value");
// test listing
ListBlobsResponse response =
client.listBlobs(
privateContainer,
ListBlobsOptions.Builder.prefix(
object
.getProperties()
.getName()
.substring(0, object.getProperties().getName().length() - 1))
.maxResults(1)
.includeMetadata());
assertEquals(response.size(), 1);
assertEquals(Iterables.getOnlyElement(response).getName(), object.getProperties().getName());
assertEquals(
Iterables.getOnlyElement(response).getMetadata(),
ImmutableMap.of("mykey", "metadata-value"));
// Test PUT with invalid ETag (as if object's data was corrupted in transit)
String correctEtag = newEtag;
String incorrectEtag = "0" + correctEtag.substring(1);
object.getProperties().setETag(incorrectEtag);
try {
client.putBlob(privateContainer, object);
} catch (Throwable e) {
assertEquals(e.getCause().getClass(), HttpResponseException.class);
assertEquals(((HttpResponseException) e.getCause()).getResponse().getStatusCode(), 422);
}
ByteArrayInputStream bais = new ByteArrayInputStream(data.getBytes("UTF-8"));
object = client.newBlob();
object.getProperties().setName("chunked-object");
object.setPayload(bais);
object.getPayload().getContentMetadata().setContentLength(new Long(data.getBytes().length));
newEtag = client.putBlob(privateContainer, object);
assertEquals(
CryptoStreams.hex(md5),
CryptoStreams.hex(getBlob.getProperties().getContentMetadata().getContentMD5()));
// Test GET with options
// Non-matching ETag
try {
client.getBlob(
privateContainer,
object.getProperties().getName(),
GetOptions.Builder.ifETagDoesntMatch(newEtag));
} catch (Exception e) {
assertEquals(e.getCause().getClass(), HttpResponseException.class);
assertEquals(((HttpResponseException) e.getCause()).getResponse().getStatusCode(), 304);
}
// Matching ETag TODO this shouldn't fail!!!
try {
getBlob =
client.getBlob(
privateContainer,
object.getProperties().getName(),
GetOptions.Builder.ifETagMatches(newEtag));
assertEquals(getBlob.getProperties().getETag(), newEtag);
} catch (HttpResponseException e) {
assertEquals(e.getResponse().getStatusCode(), 412);
}
// Range
// doesn't work per
// http://social.msdn.microsoft.com/Forums/en-US/windowsazure/thread/479fa63f-51df-4b66-96b5-33ae362747b6
// getBlob = client
// .getBlob(privateContainer, object.getProperties().getName(),
// GetOptions.Builder.startAt(8)).get(120,
// TimeUnit.SECONDS);
// assertEquals(Utils.toStringAndClose((InputStream) getBlob.getData()), data.substring(8));
client.deleteBlob(privateContainer, "object");
client.deleteBlob(privateContainer, "chunked-object");
}
public void endElement(String uri, String name, String qName) {
if (inMetadata && !qName.equals("Metadata")) {
currentMetadata.put(qName, currentText.toString().trim());
} else if (qName.equals("Metadata")) {
inMetadata = false;
} else if (qName.equals("MaxResults")) {
maxResults = Integer.parseInt(currentText.toString().trim());
} else if (qName.equals("Marker")) {
marker = currentText.toString().trim();
marker = (marker.equals("")) ? null : marker;
} else if (qName.equals("Prefix")) {
prefix = currentText.toString().trim();
prefix = (prefix.equals("")) ? null : prefix;
} else if (qName.equals("Delimiter")) {
delimiter = currentText.toString().trim();
delimiter = (delimiter.equals("")) ? null : delimiter;
} else if (qName.equals("NextMarker")) {
nextMarker = currentText.toString().trim();
nextMarker = (nextMarker.equals("")) ? null : nextMarker;
} else if (qName.equals("BlobType")) {
currentBlobType = BlobType.fromValue(currentText.toString().trim());
} else if (qName.equals("LeaseStatus")) {
currentLeaseStatus = LeaseStatus.fromValue(currentText.toString().trim());
} else if (qName.equals("Blob")) {
BlobProperties md =
new BlobPropertiesImpl(
currentBlobType,
currentName,
containerUrl.getPath().replace("/", ""),
currentUrl,
currentLastModified,
currentETag,
currentSize,
currentContentType,
currentContentMD5,
currentContentEncoding,
currentContentLanguage,
currentLeaseStatus,
currentMetadata);
blobMetadata.add(md);
currentBlobType = null;
currentName = null;
currentUrl = null;
currentLastModified = null;
currentETag = null;
currentSize = -1;
currentContentType = null;
currentContentEncoding = null;
currentContentLanguage = null;
currentContentMD5 = null;
currentLeaseStatus = null;
currentMetadata = Maps.newHashMap();
} else if (qName.equals("Url")) {
currentUrl = HttpUtils.createUri(currentText.toString().trim());
} else if (qName.equals("Last-Modified")) {
currentLastModified = dateParser.rfc822DateParse(currentText.toString().trim());
} else if (qName.equals("Etag")) {
currentETag = currentText.toString().trim();
} else if (qName.equals("Name")) {
if (inBlob) currentName = currentText.toString().trim();
else if (inBlobPrefix) blobPrefixes.add(currentText.toString().trim());
} else if (qName.equals("Content-Length")) {
currentSize = Long.parseLong(currentText.toString().trim());
} else if (qName.equals("Content-MD5")) {
if (!currentText.toString().trim().equals(""))
currentContentMD5 = CryptoStreams.base64(currentText.toString().trim());
} else if (qName.equals("Content-Type")) {
currentContentType = currentText.toString().trim();
} else if (qName.equals("Content-Encoding")) {
currentContentEncoding = currentText.toString().trim();
if (currentContentEncoding.equals("")) currentContentEncoding = null;
} else if (qName.equals("Content-Language")) {
currentContentLanguage = currentText.toString().trim();
if (currentContentLanguage.equals("")) currentContentLanguage = null;
}
currentText = new StringBuilder();
}