/**
* Constructs a HttpURLConnection to set the blob's properties, Sign with zero length specified.
*
* @param uri The absolute URI to the blob
* @param timeout The server timeout interval
* @param accessCondition An {@link AccessCondition} object that represents the access conditions
* for the blob.
* @param blobOptions the options to use for the request.
* @param opContext a tracking object for the request
* @return a HttpURLConnection to use to perform the operation.
* @throws IOException if there is an error opening the connection
* @throws URISyntaxException if the resource URI is invalid
* @throws StorageException an exception representing any error which occurred during the
* operation.
* @throws IllegalArgumentException
*/
public static HttpURLConnection putBlockList(
final URI uri,
final int timeout,
final BlobProperties properties,
final AccessCondition accessCondition,
final BlobRequestOptions blobOptions,
final OperationContext opContext)
throws IOException, URISyntaxException, StorageException {
final UriQueryBuilder builder = new UriQueryBuilder();
builder.add("comp", "blocklist");
final HttpURLConnection request =
BlobRequest.createURLConnection(uri, timeout, builder, blobOptions, opContext);
request.setDoOutput(true);
request.setRequestMethod("PUT");
if (accessCondition != null) {
accessCondition.applyConditionToRequest(request);
}
BaseRequest.addOptionalHeader(
request, Constants.HeaderConstants.CACHE_CONTROL_HEADER, properties.getCacheControl());
BaseRequest.addOptionalHeader(
request, BlobConstants.CONTENT_ENCODING_HEADER, properties.getContentEncoding());
BaseRequest.addOptionalHeader(
request, BlobConstants.CONTENT_LANGUAGE_HEADER, properties.getContentLanguage());
BaseRequest.addOptionalHeader(
request, BlobConstants.BLOB_CONTENT_MD5_HEADER, properties.getContentMD5());
BaseRequest.addOptionalHeader(
request, BlobConstants.CONTENT_TYPE_HEADER, properties.getContentType());
return request;
}
/**
* Constructs a HttpURLConnection to upload a blob. Sign with blob length, or -1 for pageblob
* create.
*
* @param uri The absolute URI to the blob
* @param timeout The server timeout interval
* @param properties The properties to set for the blob.
* @param blobType The type of the blob.
* @param pageBlobSize For a page blob, the size of the blob. This parameter is ignored for block
* blobs.
* @param accessCondition An {@link AccessCondition} object that represents the access conditions
* for the blob.
* @param blobOptions the options to use for the request.
* @param opContext a tracking object for the request
* @return a HttpURLConnection to use to perform the operation.
* @throws IOException if there is an error opening the connection
* @throws URISyntaxException if the resource URI is invalid
* @throws StorageException an exception representing any error which occurred during the
* operation.
* @throws IllegalArgumentException
*/
public static HttpURLConnection put(
final URI uri,
final int timeout,
final BlobProperties properties,
final BlobType blobType,
final long pageBlobSize,
final AccessCondition accessCondition,
final BlobRequestOptions blobOptions,
final OperationContext opContext)
throws IOException, URISyntaxException, StorageException {
if (blobType == BlobType.UNSPECIFIED) {
throw new IllegalArgumentException("The blob type cannot be undefined.");
}
final HttpURLConnection request =
BlobRequest.createURLConnection(uri, timeout, null, blobOptions, opContext);
request.setDoOutput(true);
request.setRequestMethod("PUT");
// use set optional header
BaseRequest.addOptionalHeader(
request, Constants.HeaderConstants.CACHE_CONTROL, properties.getCacheControl());
BaseRequest.addOptionalHeader(
request, Constants.HeaderConstants.CONTENT_TYPE, properties.getContentType());
BaseRequest.addOptionalHeader(
request, Constants.HeaderConstants.CONTENT_MD5, properties.getContentMD5());
BaseRequest.addOptionalHeader(
request, Constants.HeaderConstants.CONTENT_LANGUAGE, properties.getContentLanguage());
BaseRequest.addOptionalHeader(
request, Constants.HeaderConstants.CONTENT_ENCODING, properties.getContentEncoding());
if (blobType == BlobType.PAGE_BLOB) {
request.setFixedLengthStreamingMode(0);
request.setRequestProperty(Constants.HeaderConstants.CONTENT_LENGTH, "0");
request.setRequestProperty(BlobConstants.BLOB_TYPE_HEADER, BlobConstants.PAGE_BLOB);
request.setRequestProperty(BlobConstants.SIZE, String.valueOf(pageBlobSize));
properties.setLength(pageBlobSize);
} else {
request.setRequestProperty(BlobConstants.BLOB_TYPE_HEADER, BlobConstants.BLOCK_BLOB);
}
if (accessCondition != null) {
accessCondition.applyConditionToRequest(request);
}
return request;
}
@Override
public List<MessageInfo> recover(
Read read, long startOffset, long endOffset, StoreKeyFactory factory) throws IOException {
ArrayList<MessageInfo> messageRecovered = new ArrayList<MessageInfo>();
try {
while (startOffset < endOffset) {
// read message header
ByteBuffer headerVersion =
ByteBuffer.allocate(MessageFormatRecord.Version_Field_Size_In_Bytes);
if (startOffset + MessageFormatRecord.Version_Field_Size_In_Bytes > endOffset) {
throw new IndexOutOfBoundsException("Unable to read version. Reached end of stream");
}
read.readInto(headerVersion, startOffset);
startOffset += headerVersion.capacity();
headerVersion.flip();
short version = headerVersion.getShort();
switch (version) {
case MessageFormatRecord.Message_Header_Version_V1:
ByteBuffer header =
ByteBuffer.allocate(MessageFormatRecord.MessageHeader_Format_V1.getHeaderSize());
header.putShort(version);
if (startOffset
+ (MessageFormatRecord.MessageHeader_Format_V1.getHeaderSize()
- headerVersion.capacity())
> endOffset) {
throw new IndexOutOfBoundsException("Unable to read version. Reached end of stream");
}
read.readInto(header, startOffset);
startOffset += header.capacity() - headerVersion.capacity();
header.flip();
MessageFormatRecord.MessageHeader_Format_V1 headerFormat =
new MessageFormatRecord.MessageHeader_Format_V1(header);
headerFormat.verifyHeader();
ReadInputStream stream = new ReadInputStream(read, startOffset, endOffset);
StoreKey key = factory.getStoreKey(new DataInputStream(stream));
// read the appropriate type of message based on the relative offset that is set
if (headerFormat.getBlobPropertiesRecordRelativeOffset()
!= MessageFormatRecord.Message_Header_Invalid_Relative_Offset) {
BlobProperties properties = MessageFormatRecord.deserializeBlobProperties(stream);
// we do not use the user metadata or blob during recovery but we still deserialize
// them to check
// for validity
MessageFormatRecord.deserializeUserMetadata(stream);
MessageFormatRecord.deserializeBlob(stream);
MessageInfo info =
new MessageInfo(
key,
header.capacity() + key.sizeInBytes() + headerFormat.getMessageSize(),
Utils.addSecondsToEpochTime(
properties.getCreationTimeInMs(), properties.getTimeToLiveInSeconds()));
messageRecovered.add(info);
} else {
boolean deleteFlag = MessageFormatRecord.deserializeDeleteRecord(stream);
MessageInfo info =
new MessageInfo(
key,
header.capacity() + key.sizeInBytes() + headerFormat.getMessageSize(),
deleteFlag);
messageRecovered.add(info);
}
startOffset = stream.getCurrentPosition();
break;
default:
throw new MessageFormatException(
"Version not known while reading message - " + version,
MessageFormatErrorCodes.Unknown_Format_Version);
}
}
} catch (MessageFormatException e) {
// log in case where we were not able to parse a message. we stop recovery at that point and
// return the
// messages that have been recovered so far.
logger.error("Message format exception while recovering messages");
} catch (IndexOutOfBoundsException e) {
// log in case where were not able to read a complete message. we stop recovery at that point
// and return
// the message that have been recovered so far.
logger.error("Trying to read more than the available bytes");
}
for (MessageInfo messageInfo : messageRecovered) {
logger.info(
"Message Recovered key {} size {} ttl {} deleted {}",
messageInfo.getStoreKey(),
messageInfo.getSize(),
messageInfo.getExpirationTimeInMs(),
messageInfo.isDeleted());
}
return messageRecovered;
}
/**
* Reserved for internal use. Populates the blob from an XMLStreamReader, reader must be at Start
* element of Blob
*
* @param xmlr the XMLStreamReader to read from
* @param serviceClient the CloudBlobClient associated with the objects.
* @throws XMLStreamException if there is an error parsing the stream
* @throws ParseException if there is an error in parsing a date
* @throws URISyntaxException if the uri is invalid
* @throws StorageException
*/
protected static CloudBlob readBlob(
final XMLStreamReader xmlr,
final CloudBlobClient serviceClient,
final CloudBlobContainer container)
throws XMLStreamException, ParseException, URISyntaxException, StorageException {
xmlr.require(XMLStreamConstants.START_ELEMENT, null, BlobConstants.BLOB_ELEMENT);
String blobName = Constants.EMPTY_STRING;
String snapshotID = null;
String urlString = null;
HashMap<String, String> metadata = null;
BlobProperties properties = null;
CopyState copyState = null;
int eventType = xmlr.getEventType();
// check if there are more events in the input stream
while (xmlr.hasNext()) {
eventType = xmlr.next();
final String name = xmlr.getName().toString();
if (eventType == XMLStreamConstants.START_ELEMENT) {
if (name.equals(Constants.URL_ELEMENT)) {
urlString = Utility.readElementFromXMLReader(xmlr, Constants.URL_ELEMENT);
} else if (name.equals(BlobConstants.SNAPSHOT_ELEMENT)) {
snapshotID = Utility.readElementFromXMLReader(xmlr, BlobConstants.SNAPSHOT_ELEMENT);
} else if (name.equals(Constants.NAME_ELEMENT)) {
blobName = Utility.readElementFromXMLReader(xmlr, Constants.NAME_ELEMENT);
} else if (name.equals(BlobConstants.PROPERTIES)) {
properties = BlobDeserializationHelper.readBlobProperties(xmlr);
xmlr.require(XMLStreamConstants.END_ELEMENT, null, BlobConstants.PROPERTIES);
} else if (name.equals(Constants.METADATA_ELEMENT)) {
metadata = DeserializationHelper.parseMetadateFromXML(xmlr);
xmlr.require(XMLStreamConstants.END_ELEMENT, null, Constants.METADATA_ELEMENT);
} else if (name.equals(Constants.COPY_ID_ELEMENT)) {
if (copyState == null) {
copyState = new CopyState();
}
copyState.setCopyId(Utility.readElementFromXMLReader(xmlr, Constants.COPY_ID_ELEMENT));
} else if (name.equals(Constants.COPY_COMPLETION_TIME_ELEMENT)) {
if (copyState == null) {
copyState = new CopyState();
}
copyState.setCompletionTime(
Utility.parseRFC1123DateFromStringInGMT(
Utility.readElementFromXMLReader(xmlr, Constants.COPY_COMPLETION_TIME_ELEMENT)));
} else if (name.equals(Constants.COPY_STATUS_ELEMENT)) {
if (copyState == null) {
copyState = new CopyState();
}
copyState.setStatus(
CopyStatus.parse(
Utility.readElementFromXMLReader(xmlr, Constants.COPY_STATUS_ELEMENT)));
} else if (name.equals(Constants.COPY_SOURCE_ELEMENT)) {
if (copyState == null) {
copyState = new CopyState();
}
copyState.setSource(
new URI(Utility.readElementFromXMLReader(xmlr, Constants.COPY_SOURCE_ELEMENT)));
} else if (name.equals(Constants.COPY_PROGRESS_ELEMENT)) {
if (copyState == null) {
copyState = new CopyState();
}
final String tempString =
Utility.readElementFromXMLReader(xmlr, Constants.COPY_PROGRESS_ELEMENT);
String[] progressSequence = tempString.split("/");
copyState.setBytesCopied(Long.parseLong(progressSequence[0]));
copyState.setTotalBytes(Long.parseLong(progressSequence[1]));
} else if (name.equals(Constants.COPY_STATUS_DESCRIPTION_ELEMENT)) {
if (copyState == null) {
copyState = new CopyState();
}
copyState.setStatusDescription(
Utility.readElementFromXMLReader(xmlr, Constants.COPY_STATUS_DESCRIPTION_ELEMENT));
}
} else if (eventType == XMLStreamConstants.END_ELEMENT
&& name.equals(BlobConstants.BLOB_ELEMENT)) {
break;
}
}
xmlr.require(XMLStreamConstants.END_ELEMENT, null, BlobConstants.BLOB_ELEMENT);
// Assemble and return
if (properties != null) {
CloudBlob retBlob = null;
final int blobNameSectionIndex = urlString.lastIndexOf("/".concat(blobName));
final URI baseUri = new URI(urlString.substring(0, blobNameSectionIndex + 1));
String query = null;
if (blobNameSectionIndex + 1 + blobName.length() < urlString.length()) {
// Snapshot blob URI
// example:http://<yourstorageaccount>.blob.core.windows.net/<yourcontainer>/<yourblobname>?snapshot=2009-12-03T15%3a26%3a19.4466877Z
query = urlString.substring(blobNameSectionIndex + 1 + blobName.length() + 1);
}
final URI blobURI =
new URI(
baseUri.getScheme(),
baseUri.getAuthority(),
baseUri.getRawPath().concat(blobName),
query,
null);
if (properties.getBlobType() == BlobType.BLOCK_BLOB) {
retBlob = new CloudBlockBlob(blobURI, serviceClient, container);
} else if (properties.getBlobType() == BlobType.PAGE_BLOB) {
retBlob = new CloudPageBlob(blobURI, serviceClient, container);
} else {
throw new StorageException(
StorageErrorCodeStrings.INVALID_XML_DOCUMENT,
"The response received is invalid or improperly formatted.",
Constants.HeaderConstants.HTTP_UNUSED_306,
null,
null);
}
retBlob.uri = blobURI;
retBlob.snapshotID = snapshotID;
retBlob.properties = properties;
retBlob.metadata = metadata;
retBlob.copyState = copyState;
return retBlob;
} else {
throw new StorageException(
StorageErrorCodeStrings.INVALID_XML_DOCUMENT,
"The response received is invalid or improperly formatted.",
Constants.HeaderConstants.HTTP_UNUSED_306,
null,
null);
}
}
/**
* Populates the object from the XMLStreamReader, reader must be at Start element of Properties
*
* @param xmlr the XMLStreamReader object
* @return the BlobProperties that was read.
* @throws XMLStreamException if there is a parsing exception
* @throws ParseException if a date value is not correctly encoded
* @throws StorageException
* @throws URISyntaxException
*/
protected static BlobProperties readBlobProperties(final XMLStreamReader xmlr)
throws XMLStreamException, ParseException, StorageException, URISyntaxException {
xmlr.require(XMLStreamConstants.START_ELEMENT, null, BlobConstants.PROPERTIES);
int eventType = xmlr.getEventType();
final BlobProperties properties = new BlobProperties();
while (xmlr.hasNext()) {
eventType = xmlr.next();
final String name = xmlr.getName().toString();
if (eventType == XMLStreamConstants.START_ELEMENT) {
if (name.equals(Constants.LAST_MODIFIED_ELEMENT)) {
properties.setLastModified(
Utility.parseRFC1123DateFromStringInGMT(
Utility.readElementFromXMLReader(xmlr, Constants.LAST_MODIFIED_ELEMENT)));
} else if (name.equals(Constants.ETAG_ELEMENT)) {
properties.setEtag(Utility.readElementFromXMLReader(xmlr, Constants.ETAG_ELEMENT));
} else if (name.equals(Constants.HeaderConstants.CONTENT_LENGTH)) {
final String tempString =
Utility.readElementFromXMLReader(xmlr, Constants.HeaderConstants.CONTENT_LENGTH);
properties.setLength(Long.parseLong(tempString));
} else if (name.equals(Constants.HeaderConstants.CONTENT_TYPE)) {
properties.setContentType(
Utility.readElementFromXMLReader(xmlr, Constants.HeaderConstants.CONTENT_TYPE));
} else if (name.equals(Constants.HeaderConstants.CONTENT_ENCODING)) {
properties.setContentEncoding(
Utility.readElementFromXMLReader(xmlr, Constants.HeaderConstants.CONTENT_ENCODING));
} else if (name.equals(Constants.HeaderConstants.CONTENT_LANGUAGE)) {
properties.setContentLanguage(
Utility.readElementFromXMLReader(xmlr, Constants.HeaderConstants.CONTENT_LANGUAGE));
} else if (name.equals(Constants.HeaderConstants.CONTENT_MD5)) {
properties.setContentMD5(
Utility.readElementFromXMLReader(xmlr, Constants.HeaderConstants.CONTENT_MD5));
} else if (name.equals(Constants.HeaderConstants.CACHE_CONTROL)) {
properties.setCacheControl(
Utility.readElementFromXMLReader(xmlr, Constants.HeaderConstants.CACHE_CONTROL));
} else if (name.equals(Constants.HeaderConstants.CACHE_CONTROL)) {
properties.setCacheControl(
Utility.readElementFromXMLReader(xmlr, Constants.HeaderConstants.CACHE_CONTROL));
} else if (name.equals(BlobConstants.SEQUENCE_NUMBER)) {
Utility.readElementFromXMLReader(xmlr, BlobConstants.SEQUENCE_NUMBER);
} else if (name.equals(BlobConstants.BLOB_TYPE_ELEMENT)) {
final String tempString =
Utility.readElementFromXMLReader(xmlr, BlobConstants.BLOB_TYPE_ELEMENT);
if (tempString.equals(BlobConstants.BLOCK_BLOB_VALUE)) {
properties.setBlobType(BlobType.BLOCK_BLOB);
} else if (tempString.equals(BlobConstants.PAGE_BLOB_VALUE)) {
properties.setBlobType(BlobType.PAGE_BLOB);
} else {
throw new StorageException(
StorageErrorCodeStrings.INVALID_XML_DOCUMENT,
"The response received is invalid or improperly formatted.",
Constants.HeaderConstants.HTTP_UNUSED_306,
null,
null);
}
} else if (name.equals(Constants.LEASE_STATUS_ELEMENT)) {
final String tempString =
Utility.readElementFromXMLReader(xmlr, Constants.LEASE_STATUS_ELEMENT);
if (tempString.equals(Constants.LOCKED_VALUE.toLowerCase())) {
properties.setLeaseStatus(LeaseStatus.LOCKED);
} else if (tempString.equals(Constants.UNLOCKED_VALUE.toLowerCase())) {
properties.setLeaseStatus(LeaseStatus.UNLOCKED);
} else {
throw new StorageException(
StorageErrorCodeStrings.INVALID_XML_DOCUMENT,
"The response received is invalid or improperly formatted.",
Constants.HeaderConstants.HTTP_UNUSED_306,
null,
null);
}
} else if (name.equals(Constants.LEASE_STATE_ELEMENT)) {
properties.setLeaseState(
LeaseState.parse(
Utility.readElementFromXMLReader(xmlr, Constants.LEASE_STATE_ELEMENT)));
} else if (name.equals(Constants.LEASE_DURATION_ELEMENT)) {
properties.setLeaseDuration(
LeaseDuration.parse(
Utility.readElementFromXMLReader(xmlr, Constants.LEASE_DURATION_ELEMENT)));
}
} else if (eventType == XMLStreamConstants.END_ELEMENT) {
// expect end of properties
xmlr.require(XMLStreamConstants.END_ELEMENT, null, BlobConstants.PROPERTIES);
break;
}
}
return properties;
}
@Test
public void testPageBlobCopyFromSnapshot()
throws StorageException, IOException, URISyntaxException, InterruptedException {
CloudPageBlob source = this.container.getPageBlobReference("source");
byte[] buffer = BlobTestHelper.getRandomBuffer(512);
ByteArrayInputStream stream = new ByteArrayInputStream(buffer);
source.upload(stream, buffer.length);
source.getMetadata().put("Test", "value");
source.uploadMetadata();
CloudPageBlob snapshot = (CloudPageBlob) source.createSnapshot();
// Modify source
byte[] buffer2 = BlobTestHelper.getRandomBuffer(512);
ByteArrayInputStream stream2 = new ByteArrayInputStream(buffer2);
source.getMetadata().put("Test", "newvalue");
source.uploadMetadata();
source.getProperties().setContentMD5(null);
source.upload(stream2, buffer.length);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
source.download(outputStream);
ByteArrayOutputStream snapshotStream = new ByteArrayOutputStream();
snapshot.download(snapshotStream);
BlobTestHelper.assertStreamsAreEqual(
stream2, new ByteArrayInputStream(outputStream.toByteArray()));
BlobTestHelper.assertStreamsAreEqual(
stream, new ByteArrayInputStream(snapshotStream.toByteArray()));
source.downloadAttributes();
snapshot.downloadAttributes();
assertFalse(source.getMetadata().get("Test").equals(snapshot.getMetadata().get("Test")));
CloudPageBlob copy = this.container.getPageBlobReference("copy");
String copyId = copy.startCopy(BlobTestHelper.defiddler(snapshot));
BlobTestHelper.waitForCopy(copy);
ByteArrayOutputStream copyStream = new ByteArrayOutputStream();
copy.download(copyStream);
assertEquals(CopyStatus.SUCCESS, copy.getCopyState().getStatus());
BlobTestHelper.assertStreamsAreEqual(
stream, new ByteArrayInputStream(copyStream.toByteArray()));
assertEquals(copyId, copy.getProperties().getCopyState().getCopyId());
copy.downloadAttributes();
BlobProperties prop1 = copy.getProperties();
BlobProperties prop2 = snapshot.getProperties();
assertEquals(prop1.getCacheControl(), prop2.getCacheControl());
assertEquals(prop1.getContentEncoding(), prop2.getContentEncoding());
assertEquals(prop1.getContentDisposition(), prop2.getContentDisposition());
assertEquals(prop1.getContentLanguage(), prop2.getContentLanguage());
assertEquals(prop1.getContentMD5(), prop2.getContentMD5());
assertEquals(prop1.getContentType(), prop2.getContentType());
assertEquals("value", copy.getMetadata().get("Test"));
copy.delete();
}
@Test
public void testPageBlobCopyWithMetadataOverride()
throws URISyntaxException, StorageException, IOException, InterruptedException {
Calendar calendar = Calendar.getInstance(Utility.UTC_ZONE);
CloudPageBlob source = this.container.getPageBlobReference("source");
byte[] buffer = BlobTestHelper.getRandomBuffer(512);
ByteArrayInputStream stream = new ByteArrayInputStream(buffer);
source.upload(stream, buffer.length);
source.getMetadata().put("Test", "value");
source.uploadMetadata();
CloudPageBlob copy = this.container.getPageBlobReference("copy");
copy.getMetadata().put("Test2", "value2");
String copyId = copy.startCopy(BlobTestHelper.defiddler(source));
BlobTestHelper.waitForCopy(copy);
assertEquals(CopyStatus.SUCCESS, copy.getCopyState().getStatus());
assertEquals(source.getQualifiedUri().getPath(), copy.getCopyState().getSource().getPath());
assertEquals(buffer.length, copy.getCopyState().getTotalBytes().intValue());
assertEquals(buffer.length, copy.getCopyState().getBytesCopied().intValue());
assertEquals(copyId, copy.getCopyState().getCopyId());
assertTrue(
copy.getCopyState()
.getCompletionTime()
.compareTo(new Date(calendar.get(Calendar.MINUTE) - 1))
> 0);
ByteArrayOutputStream copyStream = new ByteArrayOutputStream();
copy.download(copyStream);
BlobTestHelper.assertStreamsAreEqual(
stream, new ByteArrayInputStream(copyStream.toByteArray()));
copy.downloadAttributes();
source.downloadAttributes();
BlobProperties prop1 = copy.getProperties();
BlobProperties prop2 = source.getProperties();
assertEquals(prop1.getCacheControl(), prop2.getCacheControl());
assertEquals(prop1.getContentEncoding(), prop2.getContentEncoding());
assertEquals(prop1.getContentDisposition(), prop2.getContentDisposition());
assertEquals(prop1.getContentLanguage(), prop2.getContentLanguage());
assertEquals(prop1.getContentMD5(), prop2.getContentMD5());
assertEquals(prop1.getContentType(), prop2.getContentType());
assertEquals("value2", copy.getMetadata().get("Test2"));
assertFalse(copy.getMetadata().containsKey("Test"));
copy.delete();
}
