@Test public void testReadGenerationChanged() throws IOException { BlobId blobId = BlobId.of(BUCKET_NAME, BLOB_NAME); reader = new BlobReadChannel(options, blobId, EMPTY_RPC_OPTIONS); byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(blobId.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag1", firstResult)); expect( storageRpcMock.read( blobId.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag2", secondResult)); replay(storageRpcMock); reader.read(firstReadBuffer); try { reader.read(secondReadBuffer); fail("Expected ReadChannel read to throw StorageException"); } catch (StorageException ex) { StringBuilder messageBuilder = new StringBuilder(); messageBuilder.append("Blob ").append(blobId).append(" was updated while reading"); assertEquals(messageBuilder.toString(), ex.getMessage()); } }
@Override protected Restorable<?>[] restorableObjects() { StorageOptions options = StorageOptions.newBuilder().setProjectId("p2").build(); ReadChannel reader = new BlobReadChannel(options, BlobId.of("b", "n"), EMPTY_RPC_OPTIONS); // avoid closing when you don't want partial writes to GCS upon failure @SuppressWarnings("resource") BlobWriteChannel writer = new BlobWriteChannel( options, BlobInfo.newBuilder(BlobId.of("b", "n")).build(), "upload-id"); return new Restorable<?>[] {reader, writer}; }
@Test public void testCopyBlob() { String sourceBlobName = "test-copy-blob-source"; BlobId source = BlobId.of(BUCKET, sourceBlobName); ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v"); BlobInfo blob = BlobInfo.builder(source).contentType(CONTENT_TYPE).metadata(metadata).build(); assertNotNull(storage.create(blob, BLOB_BYTE_CONTENT)); String targetBlobName = "test-copy-blob-target"; Storage.CopyRequest req = Storage.CopyRequest.of(source, BlobId.of(BUCKET, targetBlobName)); CopyWriter copyWriter = storage.copy(req); assertEquals(BUCKET, copyWriter.result().bucket()); assertEquals(targetBlobName, copyWriter.result().name()); assertEquals(CONTENT_TYPE, copyWriter.result().contentType()); assertEquals(metadata, copyWriter.result().metadata()); assertTrue(copyWriter.isDone()); assertTrue(storage.delete(BUCKET, sourceBlobName)); assertTrue(storage.delete(BUCKET, targetBlobName)); }
@Test public void attachmentsShouldBeRetrievedWhenSome() throws Exception { String payload = "payload"; BlobId blodId = BlobId.of("id1"); String type = "content"; Attachment expectedAttachment = Attachment.builder() .blobId(blodId) .size(payload.length()) .type(type) .cid("cid") .isInline(true) .build(); MetaDataWithContent testMail = MetaDataWithContent.builder() .uid(MessageUid.of(2)) .flags(new Flags(Flag.SEEN)) .size(0) .internalDate(INTERNAL_DATE) .content( new ByteArrayInputStream( IOUtils.toByteArray(ClassLoader.getSystemResourceAsStream("spamMail.eml")))) .attachments( ImmutableList.of( MessageAttachment.builder() .attachment( org.apache.james.mailbox.model.Attachment.builder() .attachmentId(AttachmentId.from(blodId.getRawValue())) .bytes(payload.getBytes()) .type(type) .build()) .cid(Cid.from("cid")) .isInline(true) .build())) .mailboxId(MAILBOX_ID) .messageId(MessageId.of("user|box|2")) .build(); Message testee = messageFactory.fromMetaDataWithContent(testMail); assertThat(testee.getAttachments()).hasSize(1); assertThat(testee.getAttachments().get(0)).isEqualToComparingFieldByField(expectedAttachment); }
@Test public void testReadFinish() throws IOException { reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); byte[] result = {}; ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", result)); replay(storageRpcMock); assertEquals(-1, reader.read(readBuffer)); }
@Test public void testSaveAndRestore() throws IOException { byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(42); ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", firstResult)); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", secondResult)); replay(storageRpcMock); reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); reader.read(firstReadBuffer); RestorableState<ReadChannel> readerState = reader.capture(); ReadChannel restoredReader = readerState.restore(); restoredReader.read(secondReadBuffer); assertArrayEquals( Arrays.copyOf(firstResult, firstReadBuffer.capacity()), firstReadBuffer.array()); assertArrayEquals(secondResult, secondReadBuffer.array()); }
@Test public void testSeek() throws IOException { reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); reader.seek(42); byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", result)); replay(storageRpcMock); reader.read(readBuffer); assertArrayEquals(result, readBuffer.array()); }
@Test public void testReadBig() throws IOException { reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); reader.chunkSize(CUSTOM_CHUNK_SIZE); byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); ByteBuffer secondReadBuffer = ByteBuffer.allocate(42); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", firstResult)); expect( storageRpcMock.read( BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, CUSTOM_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", secondResult)); replay(storageRpcMock); reader.read(firstReadBuffer); reader.read(secondReadBuffer); assertArrayEquals(firstResult, firstReadBuffer.array()); assertArrayEquals( Arrays.copyOf(secondResult, secondReadBuffer.capacity()), secondReadBuffer.array()); }
@Test public void testCopyBlobFail() { String sourceBlobName = "test-copy-blob-source-fail"; BlobId source = BlobId.of(BUCKET, sourceBlobName); assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); String targetBlobName = "test-copy-blob-target-fail"; BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); Storage.CopyRequest req = Storage.CopyRequest.builder() .source(source) .sourceOptions(Storage.BlobSourceOption.generationMatch(-1L)) .target(target) .build(); try { storage.copy(req); fail("StorageException was expected"); } catch (StorageException ex) { // expected } assertTrue(storage.delete(BUCKET, sourceBlobName)); }
@Test public void headersShouldBeSetIntoMessage() throws Exception { String headers = "From: user <user@domain>\n" + "Subject: test subject\n" + "To: user1 <user1@domain>, user2 <user2@domain>\n" + "Cc: usercc <usercc@domain>\n" + "Bcc: userbcc <userbcc@domain>\n" + "Reply-To: \"user to reply to\" <user.reply.to@domain>\n" + "In-Reply-To: <*****@*****.**>\n" + "Other-header: other header value"; MetaDataWithContent testMail = MetaDataWithContent.builder() .uid(MessageUid.of(2)) .flags(new Flags(Flag.SEEN)) .size(headers.length()) .internalDate(INTERNAL_DATE) .content(new ByteArrayInputStream(headers.getBytes(Charsets.UTF_8))) .attachments(ImmutableList.of()) .mailboxId(MAILBOX_ID) .messageId(MessageId.of("user|box|2")) .build(); Emailer user = Emailer.builder().name("user").email("user@domain").build(); Emailer user1 = Emailer.builder().name("user1").email("user1@domain").build(); Emailer user2 = Emailer.builder().name("user2").email("user2@domain").build(); Emailer usercc = Emailer.builder().name("usercc").email("usercc@domain").build(); Emailer userbcc = Emailer.builder().name("userbcc").email("userbcc@domain").build(); Emailer userRT = Emailer.builder().name("user to reply to").email("user.reply.to@domain").build(); ImmutableMap<String, String> headersMap = ImmutableMap.<String, String>builder() .put("Cc", "usercc <usercc@domain>") .put("Bcc", "userbcc <userbcc@domain>") .put("Subject", "test subject") .put("From", "user <user@domain>") .put("To", "user1 <user1@domain>, user2 <user2@domain>") .put("Reply-To", "\"user to reply to\" <user.reply.to@domain>") .put("In-Reply-To", "<*****@*****.**>") .put("Other-header", "other header value") .put("Date", "Tue, 14 Jul 2015 12:30:42 +0000") .put("MIME-Version", "1.0") .build(); Message testee = messageFactory.fromMetaDataWithContent(testMail); Message expected = Message.builder() .id(MessageId.of("user|box|2")) .blobId(BlobId.of("2")) .threadId("user|box|2") .mailboxIds(ImmutableList.of(MAILBOX_ID)) .inReplyToMessageId("<*****@*****.**>") .headers(headersMap) .from(user) .to(ImmutableList.of(user1, user2)) .cc(ImmutableList.of(usercc)) .bcc(ImmutableList.of(userbcc)) .replyTo(ImmutableList.of(userRT)) .subject("test subject") .date(ZONED_DATE) .size(headers.length()) .preview("(Empty)") .textBody("") .build(); assertThat(testee).isEqualToComparingFieldByField(expected); }
/** Retrieve metadata for the given blob. */ public Builder get(String bucket, String blob, BlobGetOption... options) { toGet.put(BlobId.of(bucket, blob), Lists.newArrayList(options)); return this; }
/** Delete the given blob. */ public Builder delete(String bucket, String blob, BlobSourceOption... options) { toDelete.put(BlobId.of(bucket, blob), Lists.newArrayList(options)); return this; }
public class BlobReadChannelTest { private static final String BUCKET_NAME = "b"; private static final String BLOB_NAME = "n"; private static final BlobId BLOB_ID = BlobId.of(BUCKET_NAME, BLOB_NAME, -1L); private static final Map<StorageRpc.Option, ?> EMPTY_RPC_OPTIONS = ImmutableMap.of(); private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; private static final int CUSTOM_CHUNK_SIZE = 2 * 1024 * 1024; private static final Random RANDOM = new Random(); private StorageOptions options; private StorageRpcFactory rpcFactoryMock; private StorageRpc storageRpcMock; private BlobReadChannel reader; @Before public void setUp() { rpcFactoryMock = createMock(StorageRpcFactory.class); storageRpcMock = createMock(StorageRpc.class); expect(rpcFactoryMock.create(anyObject(StorageOptions.class))).andReturn(storageRpcMock); replay(rpcFactoryMock); options = StorageOptions.builder() .projectId("projectId") .serviceRpcFactory(rpcFactoryMock) .retryParams(RetryParams.noRetries()) .build(); } @After public void tearDown() throws Exception { verify(rpcFactoryMock, storageRpcMock); } @Test public void testCreate() { replay(storageRpcMock); reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); assertTrue(reader.isOpen()); } @Test public void testReadBuffered() throws IOException { reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(42); ByteBuffer secondReadBuffer = ByteBuffer.allocate(42); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", result)); replay(storageRpcMock); reader.read(firstReadBuffer); reader.read(secondReadBuffer); assertArrayEquals(Arrays.copyOf(result, firstReadBuffer.capacity()), firstReadBuffer.array()); assertArrayEquals( Arrays.copyOfRange( result, firstReadBuffer.capacity(), firstReadBuffer.capacity() + secondReadBuffer.capacity()), secondReadBuffer.array()); } @Test public void testReadBig() throws IOException { reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); reader.chunkSize(CUSTOM_CHUNK_SIZE); byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); ByteBuffer secondReadBuffer = ByteBuffer.allocate(42); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", firstResult)); expect( storageRpcMock.read( BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, CUSTOM_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", secondResult)); replay(storageRpcMock); reader.read(firstReadBuffer); reader.read(secondReadBuffer); assertArrayEquals(firstResult, firstReadBuffer.array()); assertArrayEquals( Arrays.copyOf(secondResult, secondReadBuffer.capacity()), secondReadBuffer.array()); } @Test public void testReadFinish() throws IOException { reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); byte[] result = {}; ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", result)); replay(storageRpcMock); assertEquals(-1, reader.read(readBuffer)); } @Test public void testSeek() throws IOException { reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); reader.seek(42); byte[] result = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", result)); replay(storageRpcMock); reader.read(readBuffer); assertArrayEquals(result, readBuffer.array()); } @Test public void testClose() { replay(storageRpcMock); reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); assertTrue(reader.isOpen()); reader.close(); assertTrue(!reader.isOpen()); } @Test public void testReadClosed() throws IOException { replay(storageRpcMock); reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); reader.close(); try { ByteBuffer readBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); reader.read(readBuffer); fail("Expected BlobReadChannel read to throw ClosedChannelException"); } catch (ClosedChannelException ex) { // expected } } @Test public void testReadGenerationChanged() throws IOException { BlobId blobId = BlobId.of(BUCKET_NAME, BLOB_NAME); reader = new BlobReadChannel(options, blobId, EMPTY_RPC_OPTIONS); byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(blobId.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag1", firstResult)); expect( storageRpcMock.read( blobId.toPb(), EMPTY_RPC_OPTIONS, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag2", secondResult)); replay(storageRpcMock); reader.read(firstReadBuffer); try { reader.read(secondReadBuffer); fail("Expected ReadChannel read to throw StorageException"); } catch (StorageException ex) { StringBuilder messageBuilder = new StringBuilder(); messageBuilder.append("Blob ").append(blobId).append(" was updated while reading"); assertEquals(messageBuilder.toString(), ex.getMessage()); } } @Test public void testSaveAndRestore() throws IOException { byte[] firstResult = randomByteArray(DEFAULT_CHUNK_SIZE); byte[] secondResult = randomByteArray(DEFAULT_CHUNK_SIZE); ByteBuffer firstReadBuffer = ByteBuffer.allocate(42); ByteBuffer secondReadBuffer = ByteBuffer.allocate(DEFAULT_CHUNK_SIZE); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 0, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", firstResult)); expect(storageRpcMock.read(BLOB_ID.toPb(), EMPTY_RPC_OPTIONS, 42, DEFAULT_CHUNK_SIZE)) .andReturn(StorageRpc.Tuple.of("etag", secondResult)); replay(storageRpcMock); reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); reader.read(firstReadBuffer); RestorableState<ReadChannel> readerState = reader.capture(); ReadChannel restoredReader = readerState.restore(); restoredReader.read(secondReadBuffer); assertArrayEquals( Arrays.copyOf(firstResult, firstReadBuffer.capacity()), firstReadBuffer.array()); assertArrayEquals(secondResult, secondReadBuffer.array()); } @Test public void testStateEquals() { replay(storageRpcMock); reader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); @SuppressWarnings("resource") // avoid closing when you don't want partial writes to GCS ReadChannel secondReader = new BlobReadChannel(options, BLOB_ID, EMPTY_RPC_OPTIONS); RestorableState<ReadChannel> state = reader.capture(); RestorableState<ReadChannel> secondState = secondReader.capture(); assertEquals(state, secondState); assertEquals(state.hashCode(), secondState.hashCode()); assertEquals(state.toString(), secondState.toString()); } private static byte[] randomByteArray(int size) { byte[] byteArray = new byte[size]; RANDOM.nextBytes(byteArray); return byteArray; } }