@Test
public void testBlobUploadWithoutMD5Validation()
throws URISyntaxException, StorageException, IOException {
final String pageBlobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testPageBlob");
final CloudPageBlob pageBlobRef = this.container.getPageBlobReference(pageBlobName);
final int length = 2 * 1024;
ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length);
BlobRequestOptions options = new BlobRequestOptions();
options.setDisableContentMD5Validation(false);
options.setStoreBlobContentMD5(false);
pageBlobRef.upload(srcStream, length, null, options, null);
pageBlobRef.downloadAttributes();
pageBlobRef.getProperties().setContentMD5("MDAwMDAwMDA=");
pageBlobRef.uploadProperties(null, options, null);
try {
pageBlobRef.download(new ByteArrayOutputStream(), null, options, null);
fail();
} catch (StorageException ex) {
assertEquals(306, ex.getHttpStatusCode());
assertEquals("InvalidMd5", ex.getErrorCode());
}
options.setDisableContentMD5Validation(true);
pageBlobRef.download(new ByteArrayOutputStream(), null, options, null);
}
/**
* Start copying a blob and then abort
*
* @throws StorageException
* @throws URISyntaxException
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testCopyFromPageBlobAbortTest()
throws StorageException, URISyntaxException, IOException {
final int length = 512;
CloudPageBlob originalBlob =
(CloudPageBlob)
BlobTestHelper.uploadNewBlob(
this.container, BlobType.PAGE_BLOB, "originalBlob", length, null);
CloudPageBlob copyBlob = this.container.getPageBlobReference(originalBlob.getName() + "copyed");
copyBlob.startCopy(originalBlob);
try {
copyBlob.abortCopy(copyBlob.getProperties().getCopyState().getCopyId());
} catch (StorageException e) {
if (!e.getErrorCode().contains("NoPendingCopyOperation")) {
throw e;
}
}
}
@Test
public void testOpenOutputStreamNoArgs() throws URISyntaxException, StorageException {
String blobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testblob");
CloudPageBlob pageBlob = this.container.getPageBlobReference(blobName);
try {
pageBlob.openWriteExisting();
} catch (StorageException ex) {
assertEquals("The specified blob does not exist.", ex.getMessage());
assertEquals(HttpURLConnection.HTTP_NOT_FOUND, ex.getHttpStatusCode());
}
pageBlob.openWriteNew(1024);
pageBlob.openWriteExisting();
CloudPageBlob pageBlob2 = this.container.getPageBlobReference(blobName);
pageBlob2.downloadAttributes();
assertEquals(1024, pageBlob2.getProperties().getLength());
assertEquals(BlobType.PAGE_BLOB, pageBlob2.getProperties().getBlobType());
}
protected static void cleanBlob(String storageAccountName, String storageContainer) {
// Create the blob client
MockCloudBlobClient blobClient = null;
try {
blobClient = createBlobClient(storageAccountName, storageAccountKey);
} catch (InvalidKeyException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
if (blobClient != null) {
MockCloudBlobContainer container = null;
try {
container = blobClient.getContainerReference(storageContainer);
} catch (URISyntaxException e) {
} catch (StorageException e) {
}
try {
container.breakLease(0);
} catch (StorageException e) {
}
try {
container.delete();
} catch (StorageException e) {
}
try {
while (container.exists()) {
Thread.sleep(1000);
}
} catch (StorageException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
/**
* Deletes the directory if it exists using the specified request options and operation context.
*
* @param accessCondition An {@link AccessCondition} object that represents the access conditions
* for the directory.
* @param options A {@link FileRequestOptions} object that specifies any additional options for
* the request. Specifying <code>null</code> will use the default request options from the
* associated service client ( {@link CloudFileClient}).
* @param opContext An {@link OperationContext} object that represents the context for the current
* operation. This object is used to track requests to the storage service, and to provide
* additional runtime information about the operation.
* @return <code>true</code> if the directory existed and was deleted; otherwise, <code>false
* </code>.
* @throws StorageException If a storage service error occurred.
*/
@DoesServiceRequest
public boolean deleteIfExists(
AccessCondition accessCondition, FileRequestOptions options, OperationContext opContext)
throws StorageException {
options = FileRequestOptions.applyDefaults(options, this.fileServiceClient);
boolean exists = this.exists(true /* primaryOnly */, accessCondition, options, opContext);
if (exists) {
try {
this.delete(accessCondition, options, opContext);
return true;
} catch (StorageException e) {
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND
&& StorageErrorCodeStrings.RESOURCE_NOT_FOUND.equals(e.getErrorCode())) {
return false;
} else {
throw e;
}
}
} else {
return false;
}
}
/**
* Creates the directory if it does not exist, using the specified request options and operation
* context.
*
* @param options A {@link FileRequestOptions} object that specifies any additional options for
* the request. Specifying <code>null</code> will use the default request options from the
* associated service client ({@link CloudFileClient}).
* @param opContext An {@link OperationContext} object that represents the context for the current
* operation. This object is used to track requests to the storage service, and to provide
* additional runtime information about the operation.
* @return <code>true</code> if the directory did not already exist and was created; otherwise,
* <code>false</code>.
* @throws StorageException If a storage service error occurred.
*/
@DoesServiceRequest
public boolean createIfNotExists(FileRequestOptions options, OperationContext opContext)
throws StorageException {
options = FileRequestOptions.applyDefaults(options, this.fileServiceClient);
boolean exists =
this.exists(true /* primaryOnly */, null /* accessCondition */, options, opContext);
if (exists) {
return false;
} else {
try {
this.create(options, opContext);
return true;
} catch (StorageException e) {
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_CONFLICT
&& StorageErrorCodeStrings.RESOURCE_ALREADY_EXISTS.equals(e.getErrorCode())) {
return false;
} else {
throw e;
}
}
}
}
@Test
public void testCloudPageBlobUploadFromStreamWithAccessCondition()
throws URISyntaxException, StorageException, IOException {
CloudPageBlob blob1 = this.container.getPageBlobReference("blob1");
AccessCondition accessCondition = AccessCondition.generateIfNoneMatchCondition("\"*\"");
final int length = 6 * 512;
ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length);
blob1.upload(srcStream, length, accessCondition, null, null);
srcStream.reset();
blob1.create(1024);
accessCondition = AccessCondition.generateIfNoneMatchCondition(blob1.getProperties().getEtag());
try {
blob1.upload(srcStream, length, accessCondition, null, null);
} catch (StorageException ex) {
assertEquals(HttpURLConnection.HTTP_PRECON_FAILED, ex.getHttpStatusCode());
}
srcStream.reset();
accessCondition = AccessCondition.generateIfMatchCondition(blob1.getProperties().getEtag());
blob1.upload(srcStream, length, accessCondition, null, null);
srcStream.reset();
CloudPageBlob blob2 = this.container.getPageBlobReference("blob2");
blob2.create(1024);
accessCondition = AccessCondition.generateIfMatchCondition(blob1.getProperties().getEtag());
try {
blob1.upload(srcStream, length, accessCondition, null, null);
} catch (StorageException ex) {
assertEquals(HttpURLConnection.HTTP_PRECON_FAILED, ex.getHttpStatusCode());
}
srcStream.reset();
accessCondition = AccessCondition.generateIfNoneMatchCondition(blob2.getProperties().getEtag());
blob1.upload(srcStream, length, accessCondition, null, null);
}
/**
* Executes an operation and enforces a retrypolicy to handle any potential errors
*
* @param <CLIENT_TYPE> The type of the service client
* @param <PARENT_TYPE> The type of the parent object, i.e. CloudBlobContainer for
* downloadAttributes etc.
* @param <RESULT_TYPE> The type of the expected result
* @param client the service client associated with the request
* @param parentObject the parent object
* @param task the StorageRequest to execute
* @param policyFactory the factory used to generate a new retry policy instance
* @param opContext an object used to track the execution of the operation
* @return the result of the operation
* @throws StorageException an exception representing any error which occurred during the
* operation.
*/
public static <CLIENT_TYPE, PARENT_TYPE, RESULT_TYPE> RESULT_TYPE executeWithRetry(
final CLIENT_TYPE client,
final PARENT_TYPE parentObject,
final StorageRequest<CLIENT_TYPE, PARENT_TYPE, RESULT_TYPE> task,
final RetryPolicyFactory policyFactory,
final OperationContext opContext)
throws StorageException {
RetryPolicy policy = null;
if (policyFactory == null) {
policy = new RetryNoRetry();
} else {
policy = policyFactory.createInstance(opContext);
// if the returned policy is null, set to not retry
if (policy == null) {
policy = new RetryNoRetry();
}
}
int currentRetryCount = 0;
StorageException translatedException = null;
HttpURLConnection request = null;
final long startTime = new Date().getTime();
while (true) {
try {
// 1-4: setup the request
request = setupStorageRequest(client, parentObject, task, currentRetryCount, opContext);
Logger.info(
opContext,
LogConstants.START_REQUEST,
request.getURL(),
request.getRequestProperty(Constants.HeaderConstants.DATE));
// 5. Potentially upload data
if (task.getSendStream() != null) {
Logger.info(opContext, LogConstants.UPLOAD);
final StreamMd5AndLength descriptor =
Utility.writeToOutputStream(
task.getSendStream(),
request.getOutputStream(),
task.getLength(),
false /* rewindStream */,
false /* calculate MD5 */,
opContext,
task.getRequestOptions());
task.validateStreamWrite(descriptor);
Logger.info(opContext, LogConstants.UPLOADDONE);
}
// 6. Process the request - Get response
RequestResult currResult = task.getResult();
currResult.setStartDate(new Date());
Logger.info(opContext, LogConstants.GET_RESPONSE);
currResult.setStatusCode(request.getResponseCode());
currResult.setStatusMessage(request.getResponseMessage());
currResult.setStopDate(new Date());
currResult.setServiceRequestID(BaseResponse.getRequestId(request));
currResult.setEtag(BaseResponse.getEtag(request));
currResult.setRequestDate(BaseResponse.getDate(request));
currResult.setContentMD5(BaseResponse.getContentMD5(request));
// 7. Fire ResponseReceived Event
ExecutionEngine.fireResponseReceivedEvent(opContext, request, task.getResult());
Logger.info(
opContext,
LogConstants.RESPONSE_RECEIVED,
currResult.getStatusCode(),
currResult.getServiceRequestID(),
currResult.getContentMD5(),
currResult.getEtag());
// 8. Pre-process response to check if there was an exception. Do Response parsing (headers
// etc).
Logger.info(opContext, LogConstants.PRE_PROCESS);
RESULT_TYPE result = task.preProcessResponse(parentObject, client, opContext);
Logger.info(opContext, LogConstants.PRE_PROCESS_DONE);
if (!task.isNonExceptionedRetryableFailure()) {
// 9. Post-process response. Read stream from server.
Logger.info(opContext, LogConstants.POST_PROCESS);
result = task.postProcessResponse(request, parentObject, client, opContext, result);
Logger.info(opContext, LogConstants.POST_PROCESS_DONE);
// Success return result and drain the input stream.
if ((task.getResult().getStatusCode() >= 200)
&& (task.getResult().getStatusCode() < 300)) {
if (request != null) {
InputStream inStream = request.getInputStream();
// At this point, we already have a result / exception to return to the user.
// This is just an optimization to improve socket reuse.
try {
Utility.writeToOutputStream(
inStream, null, -1, false, false, null, task.getRequestOptions());
} catch (final IOException ex) {
} catch (StorageException e) {
} finally {
inStream.close();
}
}
}
Logger.info(opContext, LogConstants.COMPLETE);
return result;
} else {
Logger.warn(opContext, LogConstants.UNEXPECTED_RESULT_OR_EXCEPTION);
// The task may have already parsed an exception.
translatedException = task.materializeException(task.getConnection(), opContext);
task.getResult().setException(translatedException);
// throw on non retryable status codes: 501, 505, blob type mismatch
if (task.getResult().getStatusCode() == HttpURLConnection.HTTP_NOT_IMPLEMENTED
|| task.getResult().getStatusCode() == HttpURLConnection.HTTP_VERSION
|| translatedException
.getErrorCode()
.equals(StorageErrorCodeStrings.INVALID_BLOB_TYPE)) {
throw translatedException;
}
}
} catch (final TimeoutException e) {
// Retryable
Logger.warn(
opContext, LogConstants.RETRYABLE_EXCEPTION, e.getClass().getName(), e.getMessage());
translatedException =
StorageException.translateException(task.getConnection(), e, opContext);
task.getResult().setException(translatedException);
} catch (final SocketTimeoutException e) {
// Retryable
Logger.warn(
opContext, LogConstants.RETRYABLE_EXCEPTION, e.getClass().getName(), e.getMessage());
translatedException =
new StorageException(
StorageErrorCodeStrings.OPERATION_TIMED_OUT,
"The operation did not complete in the specified time.",
-1,
null,
e);
task.getResult().setException(translatedException);
} catch (final IOException e) {
// Non Retryable if the inner exception is actually an TimeoutException, otherwise Retryable
if (e.getCause() instanceof TimeoutException) {
translatedException =
new StorageException(
StorageErrorCodeStrings.OPERATION_TIMED_OUT,
SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION,
Constants.HeaderConstants.HTTP_UNUSED_306,
null,
(Exception) e.getCause());
task.getResult().setException(translatedException);
Logger.error(
opContext,
LogConstants.UNRETRYABLE_EXCEPTION,
e.getCause().getClass().getName(),
e.getCause().getMessage());
throw translatedException;
} else {
Logger.warn(
opContext, LogConstants.RETRYABLE_EXCEPTION, e.getClass().getName(), e.getMessage());
translatedException =
StorageException.translateException(task.getConnection(), e, opContext);
task.getResult().setException(translatedException);
}
} catch (final InvalidKeyException e) {
// Non Retryable, just throw
translatedException =
StorageException.translateException(task.getConnection(), e, opContext);
task.getResult().setException(translatedException);
Logger.error(
opContext, LogConstants.UNRETRYABLE_EXCEPTION, e.getClass().getName(), e.getMessage());
throw translatedException;
} catch (final URISyntaxException e) {
// Non Retryable, just throw
translatedException =
StorageException.translateException(task.getConnection(), e, opContext);
task.getResult().setException(translatedException);
Logger.error(
opContext, LogConstants.UNRETRYABLE_EXCEPTION, e.getClass().getName(), e.getMessage());
throw translatedException;
} catch (final TableServiceException e) {
task.getResult().setStatusCode(e.getHttpStatusCode());
task.getResult().setStatusMessage(e.getMessage());
task.getResult().setException(e);
if (!e.isRetryable()) {
Logger.error(
opContext,
LogConstants.UNRETRYABLE_EXCEPTION,
e.getClass().getName(),
e.getMessage());
throw e;
} else {
Logger.warn(
opContext, LogConstants.RETRYABLE_EXCEPTION, e.getClass().getName(), e.getMessage());
translatedException = e;
}
} catch (final StorageException e) {
// Non Retryable, just throw
// do not translate StorageException
task.getResult().setException(e);
Logger.error(
opContext, LogConstants.UNRETRYABLE_EXCEPTION, e.getClass().getName(), e.getMessage());
throw e;
} catch (final Exception e) {
// Non Retryable, just throw
translatedException =
StorageException.translateException(task.getConnection(), e, opContext);
task.getResult().setException(translatedException);
Logger.error(
opContext, LogConstants.UNRETRYABLE_EXCEPTION, e.getClass().getName(), e.getMessage());
throw translatedException;
} finally {
opContext.setClientTimeInMs(new Date().getTime() - startTime);
// 10. Fire RequestCompleted Event
if (task.isSent()) {
ExecutionEngine.fireRequestCompletedEvent(opContext, request, task.getResult());
}
}
// Evaluate Retry Policy
Logger.info(
opContext,
LogConstants.RETRY_CHECK,
currentRetryCount,
task.getResult().getStatusCode(),
translatedException == null ? null : translatedException.getMessage());
task.setCurrentLocation(getNextLocation(task.getCurrentLocation(), task.getLocationMode()));
Logger.info(
opContext, LogConstants.NEXT_LOCATION, task.getCurrentLocation(), task.getLocationMode());
RetryContext retryContext =
new RetryContext(
currentRetryCount++,
task.getResult(),
task.getCurrentLocation(),
task.getLocationMode());
RetryInfo retryInfo = policy.evaluate(retryContext, opContext);
if (retryInfo == null) {
// policy does not allow for retry
Logger.error(
opContext,
LogConstants.DO_NOT_RETRY_POLICY,
translatedException == null ? null : translatedException.getMessage());
throw translatedException;
} else if (Utility.validateMaxExecutionTimeout(
task.getRequestOptions().getOperationExpiryTimeInMs(), retryInfo.getRetryInterval())) {
// maximum execution time would be exceeded by current time plus retry interval delay
TimeoutException timeoutException =
new TimeoutException(SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION);
translatedException =
new StorageException(
StorageErrorCodeStrings.OPERATION_TIMED_OUT,
SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION,
Constants.HeaderConstants.HTTP_UNUSED_306,
null,
timeoutException);
task.initialize(opContext);
task.getResult().setException(translatedException);
Logger.error(
opContext,
LogConstants.DO_NOT_RETRY_TIMEOUT,
translatedException == null ? null : translatedException.getMessage());
throw translatedException;
} else {
// attempt to retry
task.setCurrentLocation(retryInfo.getTargetLocation());
task.setLocationMode(retryInfo.getUpdatedLocationMode());
Logger.info(
opContext, LogConstants.RETRY_INFO, task.getCurrentLocation(), task.getLocationMode());
try {
ExecutionEngine.fireRetryingEvent(
opContext, task.getConnection(), task.getResult(), retryContext);
Logger.info(opContext, LogConstants.RETRY_DELAY, retryInfo.getRetryInterval());
Thread.sleep(retryInfo.getRetryInterval());
} catch (final InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
}
}
}
}
@Test
public void testPageBlobCopyTest()
throws URISyntaxException, StorageException, InterruptedException, IOException {
Calendar calendar = Calendar.getInstance(Utility.UTC_ZONE);
CloudPageBlob source = this.container.getPageBlobReference("source");
byte[] buffer = BlobTestHelper.getRandomBuffer(512);
ByteArrayInputStream stream = new ByteArrayInputStream(buffer);
source.upload(stream, buffer.length);
source.getMetadata().put("Test", "value");
source.uploadMetadata();
CloudPageBlob copy = this.container.getPageBlobReference("copy");
String copyId = copy.startCopy(BlobTestHelper.defiddler(source));
BlobTestHelper.waitForCopy(copy);
assertEquals(CopyStatus.SUCCESS, copy.getCopyState().getStatus());
assertEquals(source.getQualifiedUri().getPath(), copy.getCopyState().getSource().getPath());
assertEquals(buffer.length, copy.getCopyState().getTotalBytes().intValue());
assertEquals(buffer.length, copy.getCopyState().getBytesCopied().intValue());
assertEquals(copyId, copy.getCopyState().getCopyId());
assertTrue(
copy.getCopyState()
.getCompletionTime()
.compareTo(new Date(calendar.get(Calendar.MINUTE) - 1))
> 0);
try {
copy.abortCopy(copy.getCopyState().getCopyId());
} catch (StorageException ex) {
assertEquals(HttpURLConnection.HTTP_CONFLICT, ex.getHttpStatusCode());
}
source.downloadAttributes();
assertNotNull(copy.getProperties().getEtag());
assertFalse(source.getProperties().getEtag().equals(copy.getProperties().getEtag()));
assertTrue(
copy.getProperties()
.getLastModified()
.compareTo(new Date(calendar.get(Calendar.MINUTE) - 1))
> 0);
ByteArrayOutputStream copyStream = new ByteArrayOutputStream();
copy.download(copyStream);
BlobTestHelper.assertStreamsAreEqual(
stream, new ByteArrayInputStream(copyStream.toByteArray()));
copy.downloadAttributes();
BlobProperties prop1 = copy.getProperties();
BlobProperties prop2 = source.getProperties();
assertEquals(prop1.getCacheControl(), prop2.getCacheControl());
assertEquals(prop1.getContentEncoding(), prop2.getContentEncoding());
assertEquals(prop1.getContentDisposition(), prop2.getContentDisposition());
assertEquals(prop1.getContentLanguage(), prop2.getContentLanguage());
assertEquals(prop1.getContentMD5(), prop2.getContentMD5());
assertEquals(prop1.getContentType(), prop2.getContentType());
assertEquals("value", copy.getMetadata().get("Test"));
copy.delete();
}