Exemple #1
0
 private static Object nativeContainerToOrcValue(Type type, Object nativeValue) {
   if (nativeValue == null) {
     return null;
   }
   if (type instanceof DecimalType) {
     BigInteger unscaledValue;
     DecimalType decimalType = (DecimalType) type;
     if (decimalType.isShort()) {
       unscaledValue = BigInteger.valueOf((long) nativeValue);
     } else {
       unscaledValue = Decimals.decodeUnscaledValue((Slice) nativeValue);
     }
     return HiveDecimal.create(unscaledValue, decimalType.getScale());
   }
   if (type.getJavaType() == boolean.class) {
     return nativeValue;
   }
   if (type.getJavaType() == long.class) {
     return nativeValue;
   }
   if (type.getJavaType() == double.class) {
     return nativeValue;
   }
   if (type.getJavaType() == Slice.class) {
     Slice slice = (Slice) nativeValue;
     return type instanceof VarcharType ? slice.toStringUtf8() : slice.getBytes();
   }
   if (isArrayType(type)) {
     Block arrayBlock = (Block) nativeValue;
     Type elementType = type.getTypeParameters().get(0);
     List<Object> list = new ArrayList<>();
     for (int i = 0; i < arrayBlock.getPositionCount(); i++) {
       list.add(
           nativeContainerToOrcValue(
               elementType, getNativeContainerValue(elementType, arrayBlock, i)));
     }
     return list;
   }
   if (isMapType(type)) {
     Block mapBlock = (Block) nativeValue;
     Type keyType = type.getTypeParameters().get(0);
     Type valueType = type.getTypeParameters().get(1);
     Map<Object, Object> map = new HashMap<>();
     for (int i = 0; i < mapBlock.getPositionCount(); i += 2) {
       Object key =
           nativeContainerToOrcValue(keyType, getNativeContainerValue(keyType, mapBlock, i));
       Object value =
           nativeContainerToOrcValue(
               valueType, getNativeContainerValue(valueType, mapBlock, i + 1));
       map.put(key, value);
     }
     return map;
   }
   throw new PrestoException(INTERNAL_ERROR, "Unimplemented type: " + type);
 }
  @Test
  public void testRewriter() throws Exception {
    OrcStorageManager manager = createOrcStorageManager();

    long transactionId = TRANSACTION_ID;
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.<Type>of(BIGINT, createVarcharType(10));

    // create file with 2 rows
    StoragePageSink sink = createStoragePageSink(manager, columnIds, columnTypes);
    List<Page> pages = rowPagesBuilder(columnTypes).row(123L, "hello").row(456L, "bye").build();
    sink.appendPages(pages);
    List<ShardInfo> shards = getFutureValue(sink.commit());

    assertEquals(shardRecorder.getShards().size(), 1);

    // delete one row
    BitSet rowsToDelete = new BitSet();
    rowsToDelete.set(0);
    Collection<Slice> fragments =
        manager.rewriteShard(
            transactionId, OptionalInt.empty(), shards.get(0).getShardUuid(), rowsToDelete);

    Slice shardDelta = Iterables.getOnlyElement(fragments);
    ShardDelta shardDeltas = jsonCodec(ShardDelta.class).fromJson(shardDelta.getBytes());
    ShardInfo shardInfo = Iterables.getOnlyElement(shardDeltas.getNewShards());

    // check that output file has one row
    assertEquals(shardInfo.getRowCount(), 1);

    // check that storage file is same as backup file
    File storageFile = storageService.getStorageFile(shardInfo.getShardUuid());
    File backupFile = fileBackupStore.getBackupFile(shardInfo.getShardUuid());
    assertFileEquals(storageFile, backupFile);

    // verify recorded shard
    List<RecordedShard> recordedShards = shardRecorder.getShards();
    assertEquals(recordedShards.size(), 2);
    assertEquals(recordedShards.get(1).getTransactionId(), TRANSACTION_ID);
    assertEquals(recordedShards.get(1).getShardUuid(), shardInfo.getShardUuid());
  }