@Test
public void testReadGenerationChanged() throws IOException {
BlobId blobId = BlobId.of(BUCKET_NAME, BLOB_NAME);
reader = new BlobReadChannel(options, blobId, EMPTY_RPC_OPTIONS);
byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
expect(storageRpcMock.read(blobId.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag1", firstResult));
expect(
storageRpcMock.read(
blobId.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag2", secondResult));
replay(storageRpcMock);
reader.read(firstReadBuffer);
try {
reader.read(secondReadBuffer);
fail("Expected ReadChannel read to throw StorageException");
} catch (StorageException ex) {
StringBuilder messageBuilder = new StringBuilder();
messageBuilder.append("Blob ").append(blobId).append(" was updated while reading");
assertEquals(messageBuilder.toString(), ex.getMessage());
}
}
@Override
protected Restorable<?>[] restorableObjects() {
StorageOptions options = StorageOptions.newBuilder().setProjectId("p2").build();
ReadChannel reader = new BlobReadChannel(options, BlobId.of("b", "n"), EMPTY_RPC_OPTIONS);
// avoid closing when you don't want partial writes to GCS upon failure
@SuppressWarnings("resource")
BlobWriteChannel writer =
new BlobWriteChannel(
options, BlobInfo.newBuilder(BlobId.of("b", "n")).build(), "upload-id");
return new Restorable<?>[] {reader, writer};
}
@Test
public void testCopyBlob() {
String sourceBlobName = "test-copy-blob-source";
BlobId source = BlobId.of(BUCKET, sourceBlobName);
ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v");
BlobInfo blob = BlobInfo.builder(source).contentType(CONTENT_TYPE).metadata(metadata).build();
assertNotNull(storage.create(blob, BLOB_BYTE_CONTENT));
String targetBlobName = "test-copy-blob-target";
Storage.CopyRequest req = Storage.CopyRequest.of(source, BlobId.of(BUCKET, targetBlobName));
CopyWriter copyWriter = storage.copy(req);
assertEquals(BUCKET, copyWriter.result().bucket());
assertEquals(targetBlobName, copyWriter.result().name());
assertEquals(CONTENT_TYPE, copyWriter.result().contentType());
assertEquals(metadata, copyWriter.result().metadata());
assertTrue(copyWriter.isDone());
assertTrue(storage.delete(BUCKET, sourceBlobName));
assertTrue(storage.delete(BUCKET, targetBlobName));
}
@Test
public void attachmentsShouldBeRetrievedWhenSome() throws Exception {
String payload = "payload";
BlobId blodId = BlobId.of("id1");
String type = "content";
Attachment expectedAttachment =
Attachment.builder()
.blobId(blodId)
.size(payload.length())
.type(type)
.cid("cid")
.isInline(true)
.build();
MetaDataWithContent testMail =
MetaDataWithContent.builder()
.uid(MessageUid.of(2))
.flags(new Flags(Flag.SEEN))
.size(0)
.internalDate(INTERNAL_DATE)
.content(
new ByteArrayInputStream(
IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("spamMail.eml"))))
.attachments(
ImmutableList.of(
MessageAttachment.builder()
.attachment(
org.apache.james.mailbox.model.Attachment.builder()
.attachmentId(AttachmentId.from(blodId.getRawValue()))
.bytes(payload.getBytes())
.type(type)
.build())
.cid(Cid.from("cid"))
.isInline(true)
.build()))
.mailboxId(MAILBOX_ID)
.messageId(MessageId.of("user|box|2"))
.build();
Message testee = messageFactory.fromMetaDataWithContent(testMail);
assertThat(testee.getAttachments()).hasSize(1);
assertThat(testee.getAttachments().get(0)).isEqualToComparingFieldByField(expectedAttachment);
}
@Test
public void testReadFinish() throws IOException {
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
byte[] result = {};
ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", result));
replay(storageRpcMock);
assertEquals(-1, reader.read(readBuffer));
}
@Test
public void testSaveAndRestore() throws IOException {
byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer firstReadBuffer = ByteBuffer.allocate(42);
ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", firstResult));
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", secondResult));
replay(storageRpcMock);
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
reader.read(firstReadBuffer);
RestorableState<ReadChannel> readerState = reader.capture();
ReadChannel restoredReader = readerState.restore();
restoredReader.read(secondReadBuffer);
assertArrayEquals(
Arrays.copyOf(firstResult, firstReadBuffer.capacity()), firstReadBuffer.array());
assertArrayEquals(secondResult, secondReadBuffer.array());
}
@Test
public void testSeek() throws IOException {
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
reader.seek(42);
byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", result));
replay(storageRpcMock);
reader.read(readBuffer);
assertArrayEquals(result, readBuffer.array());
}
@Test
public void testReadBig() throws IOException {
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
reader.chunkSize(CUSTOM_CHUNK_SIZE);
byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
ByteBuffer secondReadBuffer = ByteBuffer.allocate(42);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", firstResult));
expect(
storageRpcMock.read(
BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, CUSTOM_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", secondResult));
replay(storageRpcMock);
reader.read(firstReadBuffer);
reader.read(secondReadBuffer);
assertArrayEquals(firstResult, firstReadBuffer.array());
assertArrayEquals(
Arrays.copyOf(secondResult, secondReadBuffer.capacity()), secondReadBuffer.array());
}
@Test
public void testCopyBlobFail() {
String sourceBlobName = "test-copy-blob-source-fail";
BlobId source = BlobId.of(BUCKET, sourceBlobName);
assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT));
String targetBlobName = "test-copy-blob-target-fail";
BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build();
Storage.CopyRequest req =
Storage.CopyRequest.builder()
.source(source)
.sourceOptions(Storage.BlobSourceOption.generationMatch(-1L))
.target(target)
.build();
try {
storage.copy(req);
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
assertTrue(storage.delete(BUCKET, sourceBlobName));
}
@Test
public void headersShouldBeSetIntoMessage() throws Exception {
String headers =
"From: user <user@domain>\n"
+ "Subject: test subject\n"
+ "To: user1 <user1@domain>, user2 <user2@domain>\n"
+ "Cc: usercc <usercc@domain>\n"
+ "Bcc: userbcc <userbcc@domain>\n"
+ "Reply-To: \"user to reply to\" <user.reply.to@domain>\n"
+ "In-Reply-To: <*****@*****.**>\n"
+ "Other-header: other header value";
MetaDataWithContent testMail =
MetaDataWithContent.builder()
.uid(MessageUid.of(2))
.flags(new Flags(Flag.SEEN))
.size(headers.length())
.internalDate(INTERNAL_DATE)
.content(new ByteArrayInputStream(headers.getBytes(Charsets.UTF_8)))
.attachments(ImmutableList.of())
.mailboxId(MAILBOX_ID)
.messageId(MessageId.of("user|box|2"))
.build();
Emailer user = Emailer.builder().name("user").email("user@domain").build();
Emailer user1 = Emailer.builder().name("user1").email("user1@domain").build();
Emailer user2 = Emailer.builder().name("user2").email("user2@domain").build();
Emailer usercc = Emailer.builder().name("usercc").email("usercc@domain").build();
Emailer userbcc = Emailer.builder().name("userbcc").email("userbcc@domain").build();
Emailer userRT =
Emailer.builder().name("user to reply to").email("user.reply.to@domain").build();
ImmutableMap<String, String> headersMap =
ImmutableMap.<String, String>builder()
.put("Cc", "usercc <usercc@domain>")
.put("Bcc", "userbcc <userbcc@domain>")
.put("Subject", "test subject")
.put("From", "user <user@domain>")
.put("To", "user1 <user1@domain>, user2 <user2@domain>")
.put("Reply-To", "\"user to reply to\" <user.reply.to@domain>")
.put("In-Reply-To", "<*****@*****.**>")
.put("Other-header", "other header value")
.put("Date", "Tue, 14 Jul 2015 12:30:42 +0000")
.put("MIME-Version", "1.0")
.build();
Message testee = messageFactory.fromMetaDataWithContent(testMail);
Message expected =
Message.builder()
.id(MessageId.of("user|box|2"))
.blobId(BlobId.of("2"))
.threadId("user|box|2")
.mailboxIds(ImmutableList.of(MAILBOX_ID))
.inReplyToMessageId("<*****@*****.**>")
.headers(headersMap)
.from(user)
.to(ImmutableList.of(user1, user2))
.cc(ImmutableList.of(usercc))
.bcc(ImmutableList.of(userbcc))
.replyTo(ImmutableList.of(userRT))
.subject("test subject")
.date(ZONED_DATE)
.size(headers.length())
.preview("(Empty)")
.textBody("")
.build();
assertThat(testee).isEqualToComparingFieldByField(expected);
}
/** Retrieve metadata for the given blob. */
public Builder get(String bucket, String blob, BlobGetOption... options) {
toGet.put(BlobId.of(bucket, blob), Lists.newArrayList(options));
return this;
}
/** Delete the given blob. */
public Builder delete(String bucket, String blob, BlobSourceOption... options) {
toDelete.put(BlobId.of(bucket, blob), Lists.newArrayList(options));
return this;
}
public class BlobReadChannelTest {
private static final String BUCKET_NAME = "b";
private static final String BLOB_NAME = "n";
private static final BlobId BLOB_ID = BlobId.of(BUCKET_NAME, BLOB_NAME, -1L);
private static final Map<StorageRpc.Option, ?> EMPTY_RPC_OPTIONS = ImmutableMap.of();
private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024;
private static final int CUSTOM_CHUNK_SIZE = 2 * 1024 * 1024;
private static final Random RANDOM = new Random();
private StorageOptions options;
private StorageRpcFactory rpcFactoryMock;
private StorageRpc storageRpcMock;
private BlobReadChannel reader;
@Before
public void setUp() {
rpcFactoryMock = createMock(StorageRpcFactory.class);
storageRpcMock = createMock(StorageRpc.class);
expect(rpcFactoryMock.create(anyObject(StorageOptions.class))).andReturn(storageRpcMock);
replay(rpcFactoryMock);
options =
StorageOptions.builder()
.projectId("projectId")
.serviceRpcFactory(rpcFactoryMock)
.retryParams(RetryParams.noRetries())
.build();
}
@After
public void tearDown() throws Exception {
verify(rpcFactoryMock, storageRpcMock);
}
@Test
public void testCreate() {
replay(storageRpcMock);
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
assertTrue(reader.isOpen());
}
@Test
public void testReadBuffered() throws IOException {
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer firstReadBuffer = ByteBuffer.allocate(42);
ByteBuffer secondReadBuffer = ByteBuffer.allocate(42);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", result));
replay(storageRpcMock);
reader.read(firstReadBuffer);
reader.read(secondReadBuffer);
assertArrayEquals(Arrays.copyOf(result, firstReadBuffer.capacity()), firstReadBuffer.array());
assertArrayEquals(
Arrays.copyOfRange(
result,
firstReadBuffer.capacity(),
firstReadBuffer.capacity() + secondReadBuffer.capacity()),
secondReadBuffer.array());
}
@Test
public void testReadBig() throws IOException {
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
reader.chunkSize(CUSTOM_CHUNK_SIZE);
byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
ByteBuffer secondReadBuffer = ByteBuffer.allocate(42);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", firstResult));
expect(
storageRpcMock.read(
BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, CUSTOM_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", secondResult));
replay(storageRpcMock);
reader.read(firstReadBuffer);
reader.read(secondReadBuffer);
assertArrayEquals(firstResult, firstReadBuffer.array());
assertArrayEquals(
Arrays.copyOf(secondResult, secondReadBuffer.capacity()), secondReadBuffer.array());
}
@Test
public void testReadFinish() throws IOException {
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
byte[] result = {};
ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", result));
replay(storageRpcMock);
assertEquals(-1, reader.read(readBuffer));
}
@Test
public void testSeek() throws IOException {
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
reader.seek(42);
byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", result));
replay(storageRpcMock);
reader.read(readBuffer);
assertArrayEquals(result, readBuffer.array());
}
@Test
public void testClose() {
replay(storageRpcMock);
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
assertTrue(reader.isOpen());
reader.close();
assertTrue(!reader.isOpen());
}
@Test
public void testReadClosed() throws IOException {
replay(storageRpcMock);
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
reader.close();
try {
ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
reader.read(readBuffer);
fail("Expected BlobReadChannel read to throw ClosedChannelException");
} catch (ClosedChannelException ex) {
// expected
}
}
@Test
public void testReadGenerationChanged() throws IOException {
BlobId blobId = BlobId.of(BUCKET_NAME, BLOB_NAME);
reader = new BlobReadChannel(options, blobId, EMPTY_RPC_OPTIONS);
byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
expect(storageRpcMock.read(blobId.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag1", firstResult));
expect(
storageRpcMock.read(
blobId.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag2", secondResult));
replay(storageRpcMock);
reader.read(firstReadBuffer);
try {
reader.read(secondReadBuffer);
fail("Expected ReadChannel read to throw StorageException");
} catch (StorageException ex) {
StringBuilder messageBuilder = new StringBuilder();
messageBuilder.append("Blob ").append(blobId).append(" was updated while reading");
assertEquals(messageBuilder.toString(), ex.getMessage());
}
}
@Test
public void testSaveAndRestore() throws IOException {
byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE);
byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE);
ByteBuffer firstReadBuffer = ByteBuffer.allocate(42);
ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE);
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", firstResult));
expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE))
.andReturn(StorageRpc.Tuple.of("etag", secondResult));
replay(storageRpcMock);
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
reader.read(firstReadBuffer);
RestorableState<ReadChannel> readerState = reader.capture();
ReadChannel restoredReader = readerState.restore();
restoredReader.read(secondReadBuffer);
assertArrayEquals(
Arrays.copyOf(firstResult, firstReadBuffer.capacity()), firstReadBuffer.array());
assertArrayEquals(secondResult, secondReadBuffer.array());
}
@Test
public void testStateEquals() {
replay(storageRpcMock);
reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
@SuppressWarnings("resource") // avoid closing when you don't want partial writes to GCS
ReadChannel secondReader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS);
RestorableState<ReadChannel> state = reader.capture();
RestorableState<ReadChannel> secondState = secondReader.capture();
assertEquals(state, secondState);
assertEquals(state.hashCode(), secondState.hashCode());
assertEquals(state.toString(), secondState.toString());
}
private static byte[] randomByteArray(int size) {
byte[] byteArray = new byte[size];
RANDOM.nextBytes(byteArray);
return byteArray;
}
}
