예제 #1
0
  private void doCreateSampledTable() throws InterruptedException {
    // begin creating the table
    List<ColumnMetadata> columns =
        ImmutableList.<ColumnMetadata>builder()
            .add(new ColumnMetadata("sales", BIGINT, 1, false))
            .build();

    ConnectorTableMetadata tableMetadata =
        new ConnectorTableMetadata(temporaryCreateSampledTable, columns, tableOwner, true);
    ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, tableMetadata);

    // write the records
    RecordSink sink = recordSinkProvider.getRecordSink(outputHandle);

    sink.beginRecord(8);
    sink.appendLong(2);
    sink.finishRecord();

    sink.beginRecord(5);
    sink.appendLong(3);
    sink.finishRecord();

    sink.beginRecord(7);
    sink.appendLong(4);
    sink.finishRecord();

    String fragment = sink.commit();

    // commit the table
    metadata.commitCreateTable(outputHandle, ImmutableList.of(fragment));

    // load the new table
    ConnectorTableHandle tableHandle = getTableHandle(temporaryCreateSampledTable);
    List<ConnectorColumnHandle> columnHandles =
        ImmutableList.<ConnectorColumnHandle>builder()
            .addAll(metadata.getColumnHandles(tableHandle).values())
            .add(metadata.getSampleWeightColumnHandle(tableHandle))
            .build();
    assertEquals(columnHandles.size(), 2);

    // verify the metadata
    tableMetadata = metadata.getTableMetadata(getTableHandle(temporaryCreateSampledTable));
    assertEquals(tableMetadata.getOwner(), tableOwner);

    Map<String, ColumnMetadata> columnMap =
        uniqueIndex(tableMetadata.getColumns(), columnNameGetter());
    assertEquals(columnMap.size(), 1);

    assertPrimitiveField(columnMap, 0, "sales", BIGINT, false);

    // verify the data
    ConnectorPartitionResult partitionResult =
        splitManager.getPartitions(tableHandle, TupleDomain.<ConnectorColumnHandle>all());
    assertEquals(partitionResult.getPartitions().size(), 1);
    ConnectorSplitSource splitSource =
        splitManager.getPartitionSplits(tableHandle, partitionResult.getPartitions());
    ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));

    try (RecordCursor cursor = recordSetProvider.getRecordSet(split, columnHandles).cursor()) {
      assertRecordCursorType(cursor, "rcfile-binary");

      assertTrue(cursor.advanceNextPosition());
      assertEquals(cursor.getLong(0), 2);
      assertEquals(cursor.getLong(1), 8);

      assertTrue(cursor.advanceNextPosition());
      assertEquals(cursor.getLong(0), 3);
      assertEquals(cursor.getLong(1), 5);

      assertTrue(cursor.advanceNextPosition());
      assertEquals(cursor.getLong(0), 4);
      assertEquals(cursor.getLong(1), 7);

      assertFalse(cursor.advanceNextPosition());
    }
  }
예제 #2
0
  private void doCreateTable() throws InterruptedException {
    // begin creating the table
    List<ColumnMetadata> columns =
        ImmutableList.<ColumnMetadata>builder()
            .add(new ColumnMetadata("id", BIGINT, 1, false))
            .add(new ColumnMetadata("t_string", VARCHAR, 2, false))
            .add(new ColumnMetadata("t_bigint", BIGINT, 3, false))
            .add(new ColumnMetadata("t_double", DOUBLE, 4, false))
            .add(new ColumnMetadata("t_boolean", BOOLEAN, 5, false))
            .build();

    ConnectorTableMetadata tableMetadata =
        new ConnectorTableMetadata(temporaryCreateTable, columns, tableOwner);
    ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, tableMetadata);

    // write the records
    RecordSink sink = recordSinkProvider.getRecordSink(outputHandle);

    sink.beginRecord(1);
    sink.appendLong(1);
    sink.appendString("hello".getBytes(UTF_8));
    sink.appendLong(123);
    sink.appendDouble(43.5);
    sink.appendBoolean(true);
    sink.finishRecord();

    sink.beginRecord(1);
    sink.appendLong(2);
    sink.appendNull();
    sink.appendNull();
    sink.appendNull();
    sink.appendNull();
    sink.finishRecord();

    sink.beginRecord(1);
    sink.appendLong(3);
    sink.appendString("bye".getBytes(UTF_8));
    sink.appendLong(456);
    sink.appendDouble(98.1);
    sink.appendBoolean(false);
    sink.finishRecord();

    String fragment = sink.commit();

    // commit the table
    metadata.commitCreateTable(outputHandle, ImmutableList.of(fragment));

    // load the new table
    ConnectorTableHandle tableHandle = getTableHandle(temporaryCreateTable);
    List<ConnectorColumnHandle> columnHandles =
        ImmutableList.copyOf(metadata.getColumnHandles(tableHandle).values());

    // verify the metadata
    tableMetadata = metadata.getTableMetadata(getTableHandle(temporaryCreateTable));
    assertEquals(tableMetadata.getOwner(), tableOwner);

    Map<String, ColumnMetadata> columnMap =
        uniqueIndex(tableMetadata.getColumns(), columnNameGetter());

    assertPrimitiveField(columnMap, 0, "id", BIGINT, false);
    assertPrimitiveField(columnMap, 1, "t_string", VARCHAR, false);
    assertPrimitiveField(columnMap, 2, "t_bigint", BIGINT, false);
    assertPrimitiveField(columnMap, 3, "t_double", DOUBLE, false);
    assertPrimitiveField(columnMap, 4, "t_boolean", BOOLEAN, false);

    // verify the data
    ConnectorPartitionResult partitionResult =
        splitManager.getPartitions(tableHandle, TupleDomain.<ConnectorColumnHandle>all());
    assertEquals(partitionResult.getPartitions().size(), 1);
    ConnectorSplitSource splitSource =
        splitManager.getPartitionSplits(tableHandle, partitionResult.getPartitions());
    ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));

    try (RecordCursor cursor = recordSetProvider.getRecordSet(split, columnHandles).cursor()) {
      assertRecordCursorType(cursor, "rcfile-binary");

      assertTrue(cursor.advanceNextPosition());
      assertEquals(cursor.getLong(0), 1);
      assertEquals(cursor.getSlice(1).toStringUtf8(), "hello");
      assertEquals(cursor.getLong(2), 123);
      assertEquals(cursor.getDouble(3), 43.5);
      assertEquals(cursor.getBoolean(4), true);

      assertTrue(cursor.advanceNextPosition());
      assertEquals(cursor.getLong(0), 2);
      assertTrue(cursor.isNull(1));
      assertTrue(cursor.isNull(2));
      assertTrue(cursor.isNull(3));
      assertTrue(cursor.isNull(4));

      assertTrue(cursor.advanceNextPosition());
      assertEquals(cursor.getLong(0), 3);
      assertEquals(cursor.getSlice(1).toStringUtf8(), "bye");
      assertEquals(cursor.getLong(2), 456);
      assertEquals(cursor.getDouble(3), 98.1);
      assertEquals(cursor.getBoolean(4), false);

      assertFalse(cursor.advanceNextPosition());
    }
  }