Exemplo n.º 1
0
  private static Block serializeList(
      Type type, BlockBuilder builder, Object object, ListObjectInspector inspector) {
    List<?> list = inspector.getList(object);
    if (list == null) {
      requireNonNull(builder, "parent builder is null").appendNull();
      return null;
    }

    List<Type> typeParameters = type.getTypeParameters();
    checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter");
    Type elementType = typeParameters.get(0);
    ObjectInspector elementInspector = inspector.getListElementObjectInspector();
    BlockBuilder currentBuilder;
    if (builder != null) {
      currentBuilder = builder.beginBlockEntry();
    } else {
      currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size());
    }

    for (Object element : list) {
      serializeObject(elementType, currentBuilder, element, elementInspector);
    }

    if (builder != null) {
      builder.closeEntry();
      return null;
    } else {
      Block resultBlock = currentBuilder.build();
      return resultBlock;
    }
  }
Exemplo n.º 2
0
 public static Block arrayBlockOf(Type elementType, Object... values) {
   BlockBuilder blockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), 1024);
   for (Object value : values) {
     appendToBlockBuilder(elementType, value, blockBuilder);
   }
   return blockBuilder.build();
 }
Exemplo n.º 3
0
 private static Block createVarWidthValueBlock(int positionCount, int mapSize) {
   Type valueType = createUnboundedVarcharType();
   BlockBuilder valueBlockBuilder =
       valueType.createBlockBuilder(new BlockBuilderStatus(), positionCount * mapSize);
   for (int i = 0; i < positionCount * mapSize; i++) {
     int wordLength = ThreadLocalRandom.current().nextInt(5, 10);
     valueType.writeSlice(valueBlockBuilder, utf8Slice(randomString(wordLength)));
   }
   return valueBlockBuilder.build();
 }
  protected void checkPageSource(
      ConnectorPageSource pageSource, List<TestColumn> testColumns, List<Type> types)
      throws IOException {
    try {
      MaterializedResult result = materializeSourceDataStream(SESSION, pageSource, types);

      for (MaterializedRow row : result) {
        for (int i = 0, testColumnsSize = testColumns.size(); i < testColumnsSize; i++) {
          TestColumn testColumn = testColumns.get(i);
          Type type = types.get(i);

          Object actualValue = row.getField(i);
          Object expectedValue = testColumn.getExpectedValue();
          if (actualValue == null) {
            assertEquals(null, expectedValue, String.format("Expected non-null for column %d", i));
          } else if (testColumn.getObjectInspector().getTypeName().equals("float")
              || testColumn.getObjectInspector().getTypeName().equals("double")) {
            assertEquals((double) actualValue, (double) expectedValue, EPSILON);
          } else if (testColumn.getObjectInspector().getTypeName().equals("date")) {
            SqlDate expectedDate = new SqlDate(((Long) expectedValue).intValue());
            assertEquals(actualValue, expectedDate);
          } else if (testColumn.getObjectInspector().getTypeName().equals("timestamp")) {
            SqlTimestamp expectedTimestamp =
                new SqlTimestamp((Long) expectedValue, SESSION.getTimeZoneKey());
            assertEquals(actualValue, expectedTimestamp);
          } else if (testColumn.getObjectInspector().getCategory() == Category.PRIMITIVE) {
            if (expectedValue instanceof Slice) {
              expectedValue = ((Slice) expectedValue).toStringUtf8();
            }

            if (actualValue instanceof Slice) {
              actualValue = ((Slice) actualValue).toStringUtf8();
            }
            if (actualValue instanceof SqlVarbinary) {
              actualValue = new String(((SqlVarbinary) actualValue).getBytes(), UTF_8);
            }
            assertEquals(actualValue, expectedValue, String.format("Wrong value for column %d", i));
          } else {
            BlockBuilder builder = type.createBlockBuilder(new BlockBuilderStatus(), 1);
            type.writeObject(builder, expectedValue);
            expectedValue = type.getObjectValue(SESSION, builder.build(), 0);
            assertEquals(
                actualValue,
                expectedValue,
                String.format("Wrong value for column %s", testColumn.getName()));
          }
        }
      }
    } finally {
      pageSource.close();
    }
  }
  private Block createZeroBlock(Type type, int rowsCount, Slice constantSlice) {
    checkArgument(isSupportedType(type), "Unsupported type [%s]", type);

    Slice slice;
    // do not exceed varchar limit
    if (isVarcharType(type)) {
      slice =
          constantSlice.slice(
              0, Math.min(((VarcharType) type).getLength(), constantSlice.length()));
    } else if (isLongDecimal(type)) {
      slice = encodeScaledValue(ZERO);
    } else {
      slice = constantSlice;
    }

    BlockBuilder builder;
    if (type instanceof FixedWidthType) {
      builder = type.createBlockBuilder(new BlockBuilderStatus(), rowsCount);
    } else {
      builder = type.createBlockBuilder(new BlockBuilderStatus(), rowsCount, slice.length());
    }

    for (int i = 0; i < rowsCount; i++) {
      Class<?> javaType = type.getJavaType();
      if (javaType == boolean.class) {
        type.writeBoolean(builder, false);
      } else if (javaType == long.class) {
        type.writeLong(builder, 0);
      } else if (javaType == double.class) {
        type.writeDouble(builder, 0.0);
      } else if (javaType == Slice.class) {
        requireNonNull(slice, "slice is null");
        type.writeSlice(builder, slice, 0, slice.length());
      } else {
        throw new UnsupportedOperationException("Unknown javaType: " + javaType.getName());
      }
    }
    return builder.build();
  }
Exemplo n.º 6
0
  public ParquetPageSource(
      ParquetReader parquetReader,
      ParquetDataSource dataSource,
      MessageType fileSchema,
      MessageType requestedSchema,
      long totalBytes,
      Properties splitSchema,
      List<HiveColumnHandle> columns,
      TupleDomain<HiveColumnHandle> effectivePredicate,
      TypeManager typeManager,
      boolean useParquetColumnNames) {
    checkArgument(totalBytes >= 0, "totalBytes is negative");
    requireNonNull(splitSchema, "splitSchema is null");
    requireNonNull(columns, "columns is null");
    requireNonNull(effectivePredicate, "effectivePredicate is null");

    this.parquetReader = parquetReader;
    this.dataSource = dataSource;
    this.requestedSchema = requestedSchema;
    this.totalBytes = totalBytes;

    int size = columns.size();
    this.constantBlocks = new Block[size];
    this.hiveColumnIndexes = new int[size];

    ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
    for (int columnIndex = 0; columnIndex < size; columnIndex++) {
      HiveColumnHandle column = columns.get(columnIndex);
      checkState(column.getColumnType() == REGULAR, "column type must be regular");

      String name = column.getName();
      Type type = typeManager.getType(column.getTypeSignature());

      namesBuilder.add(name);
      typesBuilder.add(type);

      hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex();

      if (getParquetType(column, fileSchema, useParquetColumnNames) == null) {
        BlockBuilder blockBuilder =
            type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH);
        for (int i = 0; i < MAX_VECTOR_LENGTH; i++) {
          blockBuilder.appendNull();
        }
        constantBlocks[columnIndex] = blockBuilder.build();
      }
    }
    types = typesBuilder.build();
    columnNames = namesBuilder.build();
  }
Exemplo n.º 7
0
  private static Block toBlock(Type type, SliceInput input, int length) {
    BlockBuilder builder = type.createBlockBuilder(new BlockBuilderStatus(), 1, length);
    if (type.getJavaType() == long.class) {
      type.writeLong(builder, input.readLong());
    } else if (type.getJavaType() == double.class) {
      type.writeDouble(builder, input.readDouble());
    } else if (type.getJavaType() == Slice.class) {
      type.writeSlice(builder, input.readSlice(length));
    } else if (type.getJavaType() == boolean.class) {
      type.writeBoolean(builder, input.readByte() != 0);
    } else {
      throw new IllegalArgumentException("Unsupported type: " + type.getJavaType().getSimpleName());
    }

    return builder.build();
  }
Exemplo n.º 8
0
 @Override
 public void start() {
   if (builder == null) {
     if (nullBuilder == null
         || (nullBuilder.getPositionCount() >= NULL_BUILDER_POSITIONS_THRESHOLD
             && nullBuilder.getSizeInBytes() >= NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD)) {
       nullBuilder =
           mapType.createBlockBuilder(
               new BlockBuilderStatus(), NULL_BUILDER_POSITIONS_THRESHOLD);
     }
     currentEntryBuilder = nullBuilder.beginBlockEntry();
   } else {
     currentEntryBuilder = builder.beginBlockEntry();
   }
   entryConverter.beforeValue(currentEntryBuilder);
 }
Exemplo n.º 9
0
  @NotNull
  public static Object evaluatePropertyValue(
      Expression expression, Type expectedType, Session session, Metadata metadata) {
    Object value =
        evaluateConstantExpression(expression, expectedType, metadata, session, ImmutableSet.of());

    // convert to object value type of SQL type
    BlockBuilder blockBuilder = expectedType.createBlockBuilder(new BlockBuilderStatus(), 1);
    BlockUtils.appendObject(expectedType, blockBuilder, value);
    Object objectValue = expectedType.getObjectValue(session.toConnectorSession(), blockBuilder, 0);

    if (objectValue == null) {
      throw new PrestoException(
          INVALID_SESSION_PROPERTY, "Session property value must not be null");
    }
    return objectValue;
  }
Exemplo n.º 10
0
  public static Object evaluatePropertyValue(
      Expression expression,
      Type expectedType,
      Session session,
      Metadata metadata,
      List<Expression> parameters) {
    Expression rewritten =
        ExpressionTreeRewriter.rewriteWith(new ParameterRewriter(parameters), expression);
    Object value =
        evaluateConstantExpression(rewritten, expectedType, metadata, session, parameters);

    // convert to object value type of SQL type
    BlockBuilder blockBuilder = expectedType.createBlockBuilder(new BlockBuilderStatus(), 1);
    writeNativeValue(expectedType, blockBuilder, value);
    Object objectValue = expectedType.getObjectValue(session.toConnectorSession(), blockBuilder, 0);

    if (objectValue == null) {
      throw new PrestoException(
          INVALID_SESSION_PROPERTY, "Session property value must not be null");
    }
    return objectValue;
  }
Exemplo n.º 11
0
 private static Object toTypeObjectValue(Session session, Type type, Object value) {
   BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), 1);
   writeNativeValue(type, blockBuilder, value);
   return type.getObjectValue(session.toConnectorSession(), blockBuilder, 0);
 }
Exemplo n.º 12
0
  public static Block intersect(Type type, Block leftArray, Block rightArray) {
    int leftPositionCount = leftArray.getPositionCount();
    int rightPositionCount = rightArray.getPositionCount();

    if (leftPositionCount == 0) {
      return leftArray;
    }
    if (rightPositionCount == 0) {
      return rightArray;
    }

    int[] leftPositions = new int[leftPositionCount];
    int[] rightPositions = new int[rightPositionCount];

    for (int i = 0; i < leftPositionCount; i++) {
      leftPositions[i] = i;
    }
    for (int i = 0; i < rightPositionCount; i++) {
      rightPositions[i] = i;
    }
    IntArrays.quickSort(leftPositions, IntBlockCompare(type, leftArray));
    IntArrays.quickSort(rightPositions, IntBlockCompare(type, rightArray));

    int entrySize;
    if (leftPositionCount < rightPositionCount) {
      entrySize = (int) Math.ceil(leftArray.getSizeInBytes() / (double) leftPositionCount);
    } else {
      entrySize = (int) Math.ceil(rightArray.getSizeInBytes() / (double) rightPositionCount);
    }
    BlockBuilder resultBlockBuilder =
        type.createBlockBuilder(
            new BlockBuilderStatus(),
            Math.min(leftArray.getPositionCount(), rightArray.getPositionCount()),
            entrySize);

    int leftCurrentPosition = 0;
    int rightCurrentPosition = 0;
    int leftBasePosition;
    int rightBasePosition;

    while (leftCurrentPosition < leftPositionCount && rightCurrentPosition < rightPositionCount) {
      leftBasePosition = leftCurrentPosition;
      rightBasePosition = rightCurrentPosition;
      int compareValue =
          type.compareTo(
              leftArray,
              leftPositions[leftCurrentPosition],
              rightArray,
              rightPositions[rightCurrentPosition]);
      if (compareValue > 0) {
        rightCurrentPosition++;
      } else if (compareValue < 0) {
        leftCurrentPosition++;
      } else {
        type.appendTo(leftArray, leftPositions[leftCurrentPosition], resultBlockBuilder);
        leftCurrentPosition++;
        rightCurrentPosition++;
        while (leftCurrentPosition < leftPositionCount
            && type.equalTo(
                leftArray,
                leftPositions[leftBasePosition],
                leftArray,
                leftPositions[leftCurrentPosition])) {
          leftCurrentPosition++;
        }
        while (rightCurrentPosition < rightPositionCount
            && type.equalTo(
                rightArray,
                rightPositions[rightBasePosition],
                rightArray,
                rightPositions[rightCurrentPosition])) {
          rightCurrentPosition++;
        }
      }
    }

    return resultBlockBuilder.build();
  }