@Test
public void testWriterRollback() throws Exception {
// verify staging directory is empty
File staging = new File(new File(temporary, "data"), "staging");
assertDirectory(staging);
assertEquals(staging.list(), new String[] {});
// create a shard in staging
OrcStorageManager manager = createOrcStorageManager();
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
sink.appendPages(pages);
sink.flush();
// verify shard exists in staging
String[] files = staging.list();
assertEquals(files.length, 1);
assertTrue(files[0].endsWith(".orc"));
// rollback should cleanup staging files
sink.rollback();
assertEquals(staging.list(), new String[] {});
}
@Test
public void testMaxShardRows() throws Exception {
OrcStorageManager manager = createOrcStorageManager(2, new DataSize(2, MEGABYTE));
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
sink.appendPages(pages);
assertTrue(sink.isFull());
}
@Test
public void testMaxFileSize() throws Exception {
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(5));
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
// Set maxFileSize to 1 byte, so adding any page makes the StoragePageSink full
OrcStorageManager manager = createOrcStorageManager(20, new DataSize(1, BYTE));
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
sink.appendPages(pages);
assertTrue(sink.isFull());
}
private List<ColumnStats> columnStats(List<Type> columnTypes, Object[]... rows) {
ImmutableList.Builder<Long> list = ImmutableList.builder();
for (long i = 1; i <= columnTypes.size(); i++) {
list.add(i);
}
List<Long> columnIds = list.build();
OrcStorageManager manager = createOrcStorageManager();
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
sink.appendPages(rowPagesBuilder(columnTypes).rows(rows).build());
List<ShardInfo> shards = getFutureValue(sink.commit());
assertEquals(shards.size(), 1);
return Iterables.getOnlyElement(shards).getColumnStats();
}
@Test
public void testRewriter() throws Exception {
OrcStorageManager manager = createOrcStorageManager();
long transactionId = TRANSACTION_ID;
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));
// create file with 2 rows
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
sink.appendPages(pages);
List<ShardInfo> shards = getFutureValue(sink.commit());
assertEquals(shardRecorder.getShards().size(), 1);
// delete one row
BitSet rowsToDelete = new BitSet();
rowsToDelete.set(0);
Collection<Slice> fragments =
manager.rewriteShard(
transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete);
Slice shardDelta = Iterables.getOnlyElement(fragments);
ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes());
ShardInfo shardInfo = Iterables.getOnlyElement(shardDeltas.getNewShards());
// check that output file has one row
assertEquals(shardInfo.getRowCount(), 1);
// check that storage file is same as backup file
File storageFile = storageService.getStorageFile(shardInfo.getShardUuid());
File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid());
assertFileEquals(storageFile, backupFile);
// verify recorded shard
List<RecordedShard> recordedShards = shardRecorder.getShards();
assertEquals(recordedShards.size(), 2);
assertEquals(recordedShards.get(1).getTransactionId(), TRANSACTION_ID);
assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid());
}
@Test
public void testReader() throws Exception {
OrcStorageManager manager = createOrcStorageManager();
List<Long> columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L);
List<Type> columnTypes =
ImmutableList.<Type>of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE);
byte[] bytes1 = octets(0x00, 0xFE, 0xFF);
byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80);
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
Object[][] doubles = {
{881L, "-inf", null, null, null, Double.NEGATIVE_INFINITY},
{882L, "+inf", null, null, null, Double.POSITIVE_INFINITY},
{883L, "nan", null, null, null, Double.NaN},
{884L, "min", null, null, null, Double.MIN_VALUE},
{885L, "max", null, null, null, Double.MAX_VALUE},
{886L, "pzero", null, null, null, 0.0},
{887L, "nzero", null, null, null, -0.0},
};
List<Page> pages =
rowPagesBuilder(columnTypes)
.row(123L, "hello", wrappedBuffer(bytes1), sqlDate(2001, 8, 22).getDays(), true, 123.45)
.row(null, null, null, null, null, null)
.row(456L, "bye", wrappedBuffer(bytes3), sqlDate(2005, 4, 22).getDays(), false, 987.65)
.rows(doubles)
.build();
sink.appendPages(pages);
List<ShardInfo> shards = getFutureValue(sink.commit());
assertEquals(shards.size(), 1);
UUID uuid = Iterables.getOnlyElement(shards).getShardUuid();
MaterializedResult expected =
resultBuilder(SESSION, columnTypes)
.row(123L, "hello", sqlBinary(bytes1), sqlDate(2001, 8, 22), true, 123.45)
.row(null, null, null, null, null, null)
.row(456L, "bye", sqlBinary(bytes3), sqlDate(2005, 4, 22), false, 987.65)
.rows(doubles)
.build();
// no tuple domain (all)
TupleDomain<RaptorColumnHandle> tupleDomain = TupleDomain.all();
try (ConnectorPageSource pageSource =
getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
assertEquals(result.getRowCount(), expected.getRowCount());
assertEquals(result, expected);
}
// tuple domain within the column range
tupleDomain =
TupleDomain.fromFixedValues(
ImmutableMap.<RaptorColumnHandle, NullableValue>builder()
.put(
new RaptorColumnHandle("test", "c1", 2, BIGINT), NullableValue.of(BIGINT, 124L))
.build());
try (ConnectorPageSource pageSource =
getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
assertEquals(result.getRowCount(), expected.getRowCount());
}
// tuple domain outside the column range
tupleDomain =
TupleDomain.fromFixedValues(
ImmutableMap.<RaptorColumnHandle, NullableValue>builder()
.put(
new RaptorColumnHandle("test", "c1", 2, BIGINT), NullableValue.of(BIGINT, 122L))
.build());
try (ConnectorPageSource pageSource =
getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
assertEquals(result.getRowCount(), 0);
}
}
@Test
public void testWriter() throws Exception {
OrcStorageManager manager = createOrcStorageManager();
List<Long> columnIds = ImmutableList.of(3L, 7L);
List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));
StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
sink.appendPages(pages);
// shard is not recorded until flush
assertEquals(shardRecorder.getShards().size(), 0);
sink.flush();
// shard is recorded after flush
List<RecordedShard> recordedShards = shardRecorder.getShards();
assertEquals(recordedShards.size(), 1);
List<ShardInfo> shards = getFutureValue(sink.commit());
assertEquals(shards.size(), 1);
ShardInfo shardInfo = Iterables.getOnlyElement(shards);
UUID shardUuid = shardInfo.getShardUuid();
File file = storageService.getStorageFile(shardUuid);
File backupFile = fileBackupStore.getBackupFile(shardUuid);
assertEquals(recordedShards.get(0).getTransactionId(), TRANSACTION_ID);
assertEquals(recordedShards.get(0).getShardUuid(), shardUuid);
assertEquals(shardInfo.getRowCount(), 2);
assertEquals(shardInfo.getCompressedSize(), file.length());
// verify primary and backup shard exist
assertFile(file, "primary shard");
assertFile(backupFile, "backup shard");
assertFileEquals(file, backupFile);
// remove primary shard to force recovery from backup
assertTrue(file.delete());
assertTrue(file.getParentFile().delete());
assertFalse(file.exists());
recoveryManager.restoreFromBackup(shardUuid, OptionalLong.empty());
try (OrcDataSource dataSource = manager.openShard(shardUuid, READER_ATTRIBUTES)) {
OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);
assertEquals(reader.nextBatch(), 2);
Block column0 = reader.readBlock(BIGINT, 0);
assertEquals(column0.isNull(0), false);
assertEquals(column0.isNull(1), false);
assertEquals(BIGINT.getLong(column0, 0), 123L);
assertEquals(BIGINT.getLong(column0, 1), 456L);
Block column1 = reader.readBlock(createVarcharType(10), 1);
assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("bye"));
assertEquals(reader.nextBatch(), -1);
}
}