Exemplo n.º 1
0
  @Test
  public void testWriterRollback() throws Exception {
    // verify staging directory is empty
    File staging = new File(new File(temporary, "data"), "staging");
    assertDirectory(staging);
    assertEquals(staging.list(), new String[] {});

    // create a shard in staging
    OrcStorageManager manager = createOrcStorageManager();

    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));

    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);

    sink.flush();

    // verify shard exists in staging
    String[] files = staging.list();
    assertEquals(files.length, 1);
    assertTrue(files[0].endsWith(".orc"));

    // rollback should cleanup staging files
    sink.rollback();

    assertEquals(staging.list(), new String[] {});
  }
Exemplo n.º 2
0
  @Test
  public void testMaxShardRows() throws Exception {
    OrcStorageManager manager = createOrcStorageManager(2, new DataSize(2, MEGABYTE));

    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));

    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);
    assertTrue(sink.isFull());
  }
Exemplo n.º 3
0
  @Test
  public void testMaxFileSize() throws Exception {
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(5));

    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();

    // Set maxFileSize to 1 byte, so adding any page makes the StoragePageSink full
    OrcStorageManager manager = createOrcStorageManager(20, new DataSize(1, BYTE));
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    sink.appendPages(pages);
    assertTrue(sink.isFull());
  }
Exemplo n.º 4
0
  private List<ColumnStats> columnStats(List<Type> columnTypes, Object[]... rows) {
    ImmutableList.Builder<Long> list = ImmutableList.builder();
    for (long i = 1; i <= columnTypes.size(); i++) {
      list.add(i);
    }
    List<Long> columnIds = list.build();

    OrcStorageManager manager = createOrcStorageManager();
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    sink.appendPages(rowPagesBuilder(columnTypes).rows(rows).build());
    List<ShardInfo> shards = getFutureValue(sink.commit());

    assertEquals(shards.size(), 1);
    return Iterables.getOnlyElement(shards).getColumnStats();
  }
Exemplo n.º 5
0
  @Test
  public void testRewriter() throws Exception {
    OrcStorageManager manager = createOrcStorageManager();

    long transactionId = TRANSACTION_ID;
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));

    // create file with 2 rows
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);
    List<ShardInfo> shards = getFutureValue(sink.commit());

    assertEquals(shardRecorder.getShards().size(), 1);

    // delete one row
    BitSet rowsToDelete = new BitSet();
    rowsToDelete.set(0);
    Collection<Slice> fragments =
        manager.rewriteShard(
            transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete);

    Slice shardDelta = Iterables.getOnlyElement(fragments);
    ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes());
    ShardInfo shardInfo = Iterables.getOnlyElement(shardDeltas.getNewShards());

    // check that output file has one row
    assertEquals(shardInfo.getRowCount(), 1);

    // check that storage file is same as backup file
    File storageFile = storageService.getStorageFile(shardInfo.getShardUuid());
    File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid());
    assertFileEquals(storageFile, backupFile);

    // verify recorded shard
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    assertEquals(recordedShards.size(), 2);
    assertEquals(recordedShards.get(1).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid());
  }
Exemplo n.º 6
0
  @Test
  public void testReader() throws Exception {
    OrcStorageManager manager = createOrcStorageManager();

    List<Long> columnIds = ImmutableList.of(2L, 4L, 6L, 7L, 8L, 9L);
    List<Type> columnTypes =
        ImmutableList.<Type>of(BIGINT, createVarcharType(10), VARBINARY, DATE, BOOLEAN, DOUBLE);

    byte[] bytes1 = octets(0x00, 0xFE, 0xFF);
    byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80);

    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);

    Object[][] doubles = {
      {881L, "-inf", null, null, null, Double.NEGATIVE_INFINITY},
      {882L, "+inf", null, null, null, Double.POSITIVE_INFINITY},
      {883L, "nan", null, null, null, Double.NaN},
      {884L, "min", null, null, null, Double.MIN_VALUE},
      {885L, "max", null, null, null, Double.MAX_VALUE},
      {886L, "pzero", null, null, null, 0.0},
      {887L, "nzero", null, null, null, -0.0},
    };

    List<Page> pages =
        rowPagesBuilder(columnTypes)
            .row(123L, "hello", wrappedBuffer(bytes1), sqlDate(2001, 8, 22).getDays(), true, 123.45)
            .row(null, null, null, null, null, null)
            .row(456L, "bye", wrappedBuffer(bytes3), sqlDate(2005, 4, 22).getDays(), false, 987.65)
            .rows(doubles)
            .build();

    sink.appendPages(pages);
    List<ShardInfo> shards = getFutureValue(sink.commit());

    assertEquals(shards.size(), 1);
    UUID uuid = Iterables.getOnlyElement(shards).getShardUuid();

    MaterializedResult expected =
        resultBuilder(SESSION, columnTypes)
            .row(123L, "hello", sqlBinary(bytes1), sqlDate(2001, 8, 22), true, 123.45)
            .row(null, null, null, null, null, null)
            .row(456L, "bye", sqlBinary(bytes3), sqlDate(2005, 4, 22), false, 987.65)
            .rows(doubles)
            .build();

    // no tuple domain (all)
    TupleDomain<RaptorColumnHandle> tupleDomain = TupleDomain.all();

    try (ConnectorPageSource pageSource =
        getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
      MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
      assertEquals(result.getRowCount(), expected.getRowCount());
      assertEquals(result, expected);
    }

    // tuple domain within the column range
    tupleDomain =
        TupleDomain.fromFixedValues(
            ImmutableMap.<RaptorColumnHandle, NullableValue>builder()
                .put(
                    new RaptorColumnHandle("test", "c1", 2, BIGINT), NullableValue.of(BIGINT, 124L))
                .build());

    try (ConnectorPageSource pageSource =
        getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
      MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
      assertEquals(result.getRowCount(), expected.getRowCount());
    }

    // tuple domain outside the column range
    tupleDomain =
        TupleDomain.fromFixedValues(
            ImmutableMap.<RaptorColumnHandle, NullableValue>builder()
                .put(
                    new RaptorColumnHandle("test", "c1", 2, BIGINT), NullableValue.of(BIGINT, 122L))
                .build());

    try (ConnectorPageSource pageSource =
        getPageSource(manager, columnIds, columnTypes, uuid, tupleDomain)) {
      MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, columnTypes);
      assertEquals(result.getRowCount(), 0);
    }
  }
Exemplo n.º 7
0
  @Test
  public void testWriter() throws Exception {
    OrcStorageManager manager = createOrcStorageManager();

    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));

    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);

    // shard is not recorded until flush
    assertEquals(shardRecorder.getShards().size(), 0);

    sink.flush();

    // shard is recorded after flush
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    assertEquals(recordedShards.size(), 1);

    List<ShardInfo> shards = getFutureValue(sink.commit());

    assertEquals(shards.size(), 1);
    ShardInfo shardInfo = Iterables.getOnlyElement(shards);

    UUID shardUuid = shardInfo.getShardUuid();
    File file = storageService.getStorageFile(shardUuid);
    File backupFile = fileBackupStore.getBackupFile(shardUuid);

    assertEquals(recordedShards.get(0).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(0).getShardUuid(), shardUuid);

    assertEquals(shardInfo.getRowCount(), 2);
    assertEquals(shardInfo.getCompressedSize(), file.length());

    // verify primary and backup shard exist
    assertFile(file, "primary shard");
    assertFile(backupFile, "backup shard");

    assertFileEquals(file, backupFile);

    // remove primary shard to force recovery from backup
    assertTrue(file.delete());
    assertTrue(file.getParentFile().delete());
    assertFalse(file.exists());

    recoveryManager.restoreFromBackup(shardUuid, OptionalLong.empty());

    try (OrcDataSource dataSource = manager.openShard(shardUuid, READER_ATTRIBUTES)) {
      OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);

      assertEquals(reader.nextBatch(), 2);

      Block column0 = reader.readBlock(BIGINT, 0);
      assertEquals(column0.isNull(0), false);
      assertEquals(column0.isNull(1), false);
      assertEquals(BIGINT.getLong(column0, 0), 123L);
      assertEquals(BIGINT.getLong(column0, 1), 456L);

      Block column1 = reader.readBlock(createVarcharType(10), 1);
      assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
      assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("bye"));

      assertEquals(reader.nextBatch(), -1);
    }
  }