private static ConnectorPageSource getPageSource( OrcStorageManager manager, List<Long> columnIds, List<Type> columnTypes, UUID uuid, TupleDomain<RaptorColumnHandle> tupleDomain) { return manager.getPageSource( uuid, OptionalInt.empty(), columnIds, columnTypes, tupleDomain, READER_ATTRIBUTES); }
@Test public void testRewriter() throws Exception { OrcStorageManager manager = createOrcStorageManager(); long transactionId = TRANSACTION_ID; List<Long> columnIds = ImmutableList.of(3L, 7L); List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10)); // create file with 2 rows StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build(); sink.appendPages(pages); List<ShardInfo> shards = getFutureValue(sink.commit()); assertEquals(shardRecorder.getShards().size(), 1); // delete one row BitSet rowsToDelete = new BitSet(); rowsToDelete.set(0); Collection<Slice> fragments = manager.rewriteShard( transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete); Slice shardDelta = Iterables.getOnlyElement(fragments); ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes()); ShardInfo shardInfo = Iterables.getOnlyElement(shardDeltas.getNewShards()); // check that output file has one row assertEquals(shardInfo.getRowCount(), 1); // check that storage file is same as backup file File storageFile = storageService.getStorageFile(shardInfo.getShardUuid()); File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid()); assertFileEquals(storageFile, backupFile); // verify recorded shard List<RecordedShard> recordedShards = shardRecorder.getShards(); assertEquals(recordedShards.size(), 2); assertEquals(recordedShards.get(1).getTransactionId(), TRANSACTION_ID); assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid()); }
@Test public void testWriter() throws Exception { OrcStorageManager manager = createOrcStorageManager(); List<Long> columnIds = ImmutableList.of(3L, 7L); List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10)); StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes); List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build(); sink.appendPages(pages); // shard is not recorded until flush assertEquals(shardRecorder.getShards().size(), 0); sink.flush(); // shard is recorded after flush List<RecordedShard> recordedShards = shardRecorder.getShards(); assertEquals(recordedShards.size(), 1); List<ShardInfo> shards = getFutureValue(sink.commit()); assertEquals(shards.size(), 1); ShardInfo shardInfo = Iterables.getOnlyElement(shards); UUID shardUuid = shardInfo.getShardUuid(); File file = storageService.getStorageFile(shardUuid); File backupFile = fileBackupStore.getBackupFile(shardUuid); assertEquals(recordedShards.get(0).getTransactionId(), TRANSACTION_ID); assertEquals(recordedShards.get(0).getShardUuid(), shardUuid); assertEquals(shardInfo.getRowCount(), 2); assertEquals(shardInfo.getCompressedSize(), file.length()); // verify primary and backup shard exist assertFile(file, "primary shard"); assertFile(backupFile, "backup shard"); assertFileEquals(file, backupFile); // remove primary shard to force recovery from backup assertTrue(file.delete()); assertTrue(file.getParentFile().delete()); assertFalse(file.exists()); recoveryManager.restoreFromBackup(shardUuid, OptionalLong.empty()); try (OrcDataSource dataSource = manager.openShard(shardUuid, READER_ATTRIBUTES)) { OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes); assertEquals(reader.nextBatch(), 2); Block column0 = reader.readBlock(BIGINT, 0); assertEquals(column0.isNull(0), false); assertEquals(column0.isNull(1), false); assertEquals(BIGINT.getLong(column0, 0), 123L); assertEquals(BIGINT.getLong(column0, 1), 456L); Block column1 = reader.readBlock(createVarcharType(10), 1); assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello")); assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("bye")); assertEquals(reader.nextBatch(), -1); } }