public MethodHandleAndConstructor specialize( Signature boundSignature, Map<String, Type> boundTypeParameters, TypeManager typeManager, FunctionRegistry functionRegistry) { for (Map.Entry<String, Class<?>> entry : specializedTypeParameters.entrySet()) { if (!entry .getValue() .isAssignableFrom(boundTypeParameters.get(entry.getKey()).getJavaType())) { return null; } } Class<?> returnContainerType = getNullAwareContainerType( typeManager.getType(boundSignature.getReturnType()).getJavaType(), nullable); if (!returnContainerType.equals(methodHandle.type().returnType())) { return null; } for (int i = 0; i < boundSignature.getArgumentTypes().size(); i++) { Class<?> argumentContainerType = getNullAwareContainerType( typeManager.getType(boundSignature.getArgumentTypes().get(i)).getJavaType(), nullableArguments.get(i)); if (!argumentNativeContainerTypes.get(i).isAssignableFrom(argumentContainerType)) { return null; } } MethodHandle methodHandle = this.methodHandle; for (ImplementationDependency dependency : dependencies) { methodHandle = methodHandle.bindTo( dependency.resolve(boundTypeParameters, typeManager, functionRegistry)); } MethodHandle constructor = null; if (this.constructor.isPresent()) { constructor = this.constructor.get(); for (ImplementationDependency dependency : constructorDependencies) { constructor = constructor.bindTo( dependency.resolve(boundTypeParameters, typeManager, functionRegistry)); } } return new MethodHandleAndConstructor(methodHandle, Optional.ofNullable(constructor)); }
private static void verifyMethodSignature( Method method, TypeSignature returnTypeName, List<TypeSignature> argumentTypeNames, TypeManager typeManager) { Type returnType = typeManager.getType(returnTypeName); checkNotNull(returnType, "returnType is null"); List<Type> argumentTypes = resolveTypes(argumentTypeNames, typeManager); checkArgument( Primitives.unwrap(method.getReturnType()) == returnType.getJavaType(), "Expected method %s return type to be %s (%s)", method, returnType.getJavaType().getName(), returnType); // skip Session argument Class<?>[] parameterTypes = method.getParameterTypes(); Annotation[][] annotations = method.getParameterAnnotations(); if (parameterTypes.length > 0 && parameterTypes[0] == ConnectorSession.class) { parameterTypes = Arrays.copyOfRange(parameterTypes, 1, parameterTypes.length); annotations = Arrays.copyOfRange(annotations, 1, annotations.length); } for (int i = 0; i < parameterTypes.length; i++) { Class<?> actualType = parameterTypes[i]; Type expectedType = argumentTypes.get(i); boolean nullable = !FluentIterable.from(Arrays.asList(annotations[i])).filter(Nullable.class).isEmpty(); // Only allow boxing for functions that need to see nulls if (Primitives.isWrapperType(actualType)) { checkArgument( nullable, "Method %s has parameter with type %s that is missing @Nullable", method, actualType); } if (nullable) { checkArgument( !NON_NULLABLE_ARGUMENT_TYPES.contains(actualType), "Method %s has parameter type %s, but @Nullable is not supported on this type", method, actualType); } checkArgument( Primitives.unwrap(actualType) == expectedType.getJavaType(), "Expected method %s parameter %s type to be %s (%s)", method, i, expectedType.getJavaType().getName(), expectedType); } }
public ParquetPageSource( ParquetReader parquetReader, ParquetDataSource dataSource, MessageType fileSchema, MessageType requestedSchema, long totalBytes, Properties splitSchema, List<HiveColumnHandle> columns, TupleDomain<HiveColumnHandle> effectivePredicate, TypeManager typeManager, boolean useParquetColumnNames) { checkArgument(totalBytes >= 0, "totalBytes is negative"); requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(columns, "columns is null"); requireNonNull(effectivePredicate, "effectivePredicate is null"); this.parquetReader = parquetReader; this.dataSource = dataSource; this.requestedSchema = requestedSchema; this.totalBytes = totalBytes; int size = columns.size(); this.constantBlocks = new Block[size]; this.hiveColumnIndexes = new int[size]; ImmutableList.Builder<String> namesBuilder = ImmutableList.builder(); ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder(); for (int columnIndex = 0; columnIndex < size; columnIndex++) { HiveColumnHandle column = columns.get(columnIndex); checkState(column.getColumnType() == REGULAR, "column type must be regular"); String name = column.getName(); Type type = typeManager.getType(column.getTypeSignature()); namesBuilder.add(name); typesBuilder.add(type); hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex(); if (getParquetType(column, fileSchema, useParquetColumnNames) == null) { BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH); for (int i = 0; i < MAX_VECTOR_LENGTH; i++) { blockBuilder.appendNull(); } constantBlocks[columnIndex] = blockBuilder.build(); } } types = typesBuilder.build(); columnNames = namesBuilder.build(); }
@Override public TableColumn map(int index, ResultSet r, StatementContext ctx) throws SQLException { QualifiedTableName table = new QualifiedTableName( r.getString("catalog_name"), r.getString("schema_name"), r.getString("table_name")); String typeName = r.getString("data_type"); Type type = typeManager.getType(typeName); checkArgument(type != null, "Unknown type %s", typeName); return new TableColumn( table, r.getString("column_name"), r.getInt("ordinal_position"), type, r.getLong("column_id")); }
private static Function<HiveColumnHandle, ColumnMetadata> columnMetadataGetter( Table table, final TypeManager typeManager) { ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); for (FieldSchema field : concat(table.getSd().getCols(), table.getPartitionKeys())) { if (field.getComment() != null) { builder.put(field.getName(), field.getComment()); } } final Map<String, String> columnComment = builder.build(); return input -> new ColumnMetadata( input.getName(), typeManager.getType(input.getTypeSignature()), input.isPartitionKey(), columnComment.get(input.getName()), false); }
public GenericHiveRecordCursor( RecordReader<K, V> recordReader, long totalBytes, Properties splitSchema, List<HivePartitionKey> partitionKeys, List<HiveColumnHandle> columns, DateTimeZone hiveStorageTimeZone, TypeManager typeManager) { requireNonNull(recordReader, "recordReader is null"); checkArgument(totalBytes >= 0, "totalBytes is negative"); requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(partitionKeys, "partitionKeys is null"); requireNonNull(columns, "columns is null"); requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null"); this.recordReader = recordReader; this.totalBytes = totalBytes; this.key = recordReader.createKey(); this.value = recordReader.createValue(); this.hiveStorageTimeZone = hiveStorageTimeZone; this.deserializer = getDeserializer(splitSchema); this.rowInspector = getTableObjectInspector(deserializer); int size = columns.size(); String[] names = new String[size]; this.types = new Type[size]; this.hiveTypes = new HiveType[size]; this.structFields = new StructField[size]; this.fieldInspectors = new ObjectInspector[size]; this.isPartitionColumn = new boolean[size]; this.loaded = new boolean[size]; this.booleans = new boolean[size]; this.longs = new long[size]; this.doubles = new double[size]; this.slices = new Slice[size]; this.objects = new Object[size]; this.nulls = new boolean[size]; // initialize data columns for (int i = 0; i < columns.size(); i++) { HiveColumnHandle column = columns.get(i); names[i] = column.getName(); types[i] = typeManager.getType(column.getTypeSignature()); hiveTypes[i] = column.getHiveType(); if (!column.isPartitionKey()) { StructField field = rowInspector.getStructFieldRef(column.getName()); structFields[i] = field; fieldInspectors[i] = field.getFieldObjectInspector(); } isPartitionColumn[i] = column.isPartitionKey(); } // parse requested partition columns Map<String, HivePartitionKey> partitionKeysByName = uniqueIndex(partitionKeys, HivePartitionKey::getName); for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { HiveColumnHandle column = columns.get(columnIndex); if (column.isPartitionKey()) { HivePartitionKey partitionKey = partitionKeysByName.get(column.getName()); checkArgument(partitionKey != null, "Unknown partition key %s", column.getName()); byte[] bytes = partitionKey.getValue().getBytes(UTF_8); String name = names[columnIndex]; Type type = types[columnIndex]; if (HiveUtil.isHiveNull(bytes)) { nulls[columnIndex] = true; } else if (BOOLEAN.equals(type)) { booleans[columnIndex] = booleanPartitionKey(partitionKey.getValue(), name); } else if (BIGINT.equals(type)) { longs[columnIndex] = bigintPartitionKey(partitionKey.getValue(), name); } else if (INTEGER.equals(type)) { longs[columnIndex] = integerPartitionKey(partitionKey.getValue(), name); } else if (SMALLINT.equals(type)) { longs[columnIndex] = smallintPartitionKey(partitionKey.getValue(), name); } else if (TINYINT.equals(type)) { longs[columnIndex] = tinyintPartitionKey(partitionKey.getValue(), name); } else if (DOUBLE.equals(type)) { doubles[columnIndex] = doublePartitionKey(partitionKey.getValue(), name); } else if (isVarcharType(type)) { slices[columnIndex] = varcharPartitionKey(partitionKey.getValue(), name, type); } else if (DATE.equals(type)) { longs[columnIndex] = datePartitionKey(partitionKey.getValue(), name); } else if (TIMESTAMP.equals(type)) { longs[columnIndex] = timestampPartitionKey(partitionKey.getValue(), hiveStorageTimeZone, name); } else if (isShortDecimal(type)) { longs[columnIndex] = shortDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name); } else if (isLongDecimal(type)) { slices[columnIndex] = longDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name); } else { throw new PrestoException( NOT_SUPPORTED, format( "Unsupported column type %s for partition key: %s", type.getDisplayName(), name)); } } } }
private static Type type(TypeManager typeManager, SqlType explicitType) { Type type = typeManager.getType(parseTypeSignature(explicitType.value())); checkNotNull(type, "No type found for '%s'", explicitType.value()); return type; }
public static OrcPageSource createOrcPageSource( MetadataReader metadataReader, Configuration configuration, Path path, long start, long length, List<HiveColumnHandle> columns, List<HivePartitionKey> partitionKeys, TupleDomain<HiveColumnHandle> effectivePredicate, DateTimeZone hiveStorageTimeZone, TypeManager typeManager, DataSize maxMergeDistance, DataSize maxBufferSize, DataSize streamBufferSize) { OrcDataSource orcDataSource; try { FileSystem fileSystem = path.getFileSystem(configuration); long size = fileSystem.getFileStatus(path).getLen(); FSDataInputStream inputStream = fileSystem.open(path); orcDataSource = new HdfsOrcDataSource( path.toString(), size, maxMergeDistance, maxBufferSize, streamBufferSize, inputStream); } catch (Exception e) { if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed") || e instanceof FileNotFoundException) { throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, e); } throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, splitError(e, path, start, length), e); } ImmutableSet.Builder<Integer> includedColumns = ImmutableSet.builder(); ImmutableList.Builder<ColumnReference<HiveColumnHandle>> columnReferences = ImmutableList.builder(); for (HiveColumnHandle column : columns) { if (!column.isPartitionKey()) { includedColumns.add(column.getHiveColumnIndex()); Type type = typeManager.getType(column.getTypeSignature()); columnReferences.add(new ColumnReference<>(column, column.getHiveColumnIndex(), type)); } } OrcPredicate predicate = new TupleDomainOrcPredicate<>(effectivePredicate, columnReferences.build()); try { OrcReader reader = new OrcReader(orcDataSource, metadataReader); OrcRecordReader recordReader = reader.createRecordReader( includedColumns.build(), predicate, start, length, hiveStorageTimeZone); return new OrcPageSource( recordReader, orcDataSource, partitionKeys, columns, hiveStorageTimeZone, typeManager); } catch (Exception e) { try { orcDataSource.close(); } catch (IOException ignored) { } if (e instanceof PrestoException) { throw (PrestoException) e; } String message = splitError(e, path, start, length); if (e.getClass().getSimpleName().equals("BlockMissingException")) { throw new PrestoException(HIVE_MISSING_DATA, message, e); } throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, message, e); } }
public ParquetHiveRecordCursor( Configuration configuration, Path path, long start, long length, Properties splitSchema, List<HivePartitionKey> partitionKeys, List<HiveColumnHandle> columns, boolean useParquetColumnNames, TypeManager typeManager) { requireNonNull(path, "path is null"); checkArgument(length >= 0, "totalBytes is negative"); requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(partitionKeys, "partitionKeys is null"); requireNonNull(columns, "columns is null"); this.totalBytes = length; int size = columns.size(); this.names = new String[size]; this.types = new Type[size]; this.isPartitionColumn = new boolean[size]; this.booleans = new boolean[size]; this.longs = new long[size]; this.doubles = new double[size]; this.slices = new Slice[size]; this.objects = new Object[size]; this.nulls = new boolean[size]; this.nullsRowDefault = new boolean[size]; for (int i = 0; i < columns.size(); i++) { HiveColumnHandle column = columns.get(i); names[i] = column.getName(); types[i] = typeManager.getType(column.getTypeSignature()); isPartitionColumn[i] = column.isPartitionKey(); nullsRowDefault[i] = !column.isPartitionKey(); } this.recordReader = createParquetRecordReader( configuration, path, start, length, columns, useParquetColumnNames); // parse requested partition columns Map<String, HivePartitionKey> partitionKeysByName = uniqueIndex(partitionKeys, HivePartitionKey::getName); for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) { HiveColumnHandle column = columns.get(columnIndex); if (column.isPartitionKey()) { HivePartitionKey partitionKey = partitionKeysByName.get(column.getName()); checkArgument(partitionKey != null, "Unknown partition key %s", column.getName()); byte[] bytes = partitionKey.getValue().getBytes(UTF_8); String name = names[columnIndex]; Type type = types[columnIndex]; if (HiveUtil.isHiveNull(bytes)) { nullsRowDefault[columnIndex] = true; } else if (type.equals(BOOLEAN)) { booleans[columnIndex] = booleanPartitionKey(partitionKey.getValue(), name); } else if (type.equals(BIGINT)) { longs[columnIndex] = bigintPartitionKey(partitionKey.getValue(), name); } else if (type.equals(DOUBLE)) { doubles[columnIndex] = doublePartitionKey(partitionKey.getValue(), name); } else if (type.equals(VARCHAR)) { slices[columnIndex] = Slices.wrappedBuffer(bytes); } else { throw new PrestoException( NOT_SUPPORTED, format( "Unsupported column type %s for partition key: %s", type.getDisplayName(), name)); } } } }
@Override public Type resolve( Map<String, Type> types, TypeManager typeManager, FunctionRegistry functionRegistry) { return typeManager.getType(signature.bindParameters(types)); }