public MethodHandleAndConstructor specialize(
Signature boundSignature,
Map<String, Type> boundTypeParameters,
TypeManager typeManager,
FunctionRegistry functionRegistry) {
for (Map.Entry<String, Class<?>> entry : specializedTypeParameters.entrySet()) {
if (!entry
.getValue()
.isAssignableFrom(boundTypeParameters.get(entry.getKey()).getJavaType())) {
return null;
}
}
Class<?> returnContainerType =
getNullAwareContainerType(
typeManager.getType(boundSignature.getReturnType()).getJavaType(), nullable);
if (!returnContainerType.equals(methodHandle.type().returnType())) {
return null;
}
for (int i = 0; i < boundSignature.getArgumentTypes().size(); i++) {
Class<?> argumentContainerType =
getNullAwareContainerType(
typeManager.getType(boundSignature.getArgumentTypes().get(i)).getJavaType(),
nullableArguments.get(i));
if (!argumentNativeContainerTypes.get(i).isAssignableFrom(argumentContainerType)) {
return null;
}
}
MethodHandle methodHandle = this.methodHandle;
for (ImplementationDependency dependency : dependencies) {
methodHandle =
methodHandle.bindTo(
dependency.resolve(boundTypeParameters, typeManager, functionRegistry));
}
MethodHandle constructor = null;
if (this.constructor.isPresent()) {
constructor = this.constructor.get();
for (ImplementationDependency dependency : constructorDependencies) {
constructor =
constructor.bindTo(
dependency.resolve(boundTypeParameters, typeManager, functionRegistry));
}
}
return new MethodHandleAndConstructor(methodHandle, Optional.ofNullable(constructor));
}
private static void verifyMethodSignature(
Method method,
TypeSignature returnTypeName,
List<TypeSignature> argumentTypeNames,
TypeManager typeManager) {
Type returnType = typeManager.getType(returnTypeName);
checkNotNull(returnType, "returnType is null");
List<Type> argumentTypes = resolveTypes(argumentTypeNames, typeManager);
checkArgument(
Primitives.unwrap(method.getReturnType()) == returnType.getJavaType(),
"Expected method %s return type to be %s (%s)",
method,
returnType.getJavaType().getName(),
returnType);
// skip Session argument
Class<?>[] parameterTypes = method.getParameterTypes();
Annotation[][] annotations = method.getParameterAnnotations();
if (parameterTypes.length > 0 && parameterTypes[0] == ConnectorSession.class) {
parameterTypes = Arrays.copyOfRange(parameterTypes, 1, parameterTypes.length);
annotations = Arrays.copyOfRange(annotations, 1, annotations.length);
}
for (int i = 0; i < parameterTypes.length; i++) {
Class<?> actualType = parameterTypes[i];
Type expectedType = argumentTypes.get(i);
boolean nullable =
!FluentIterable.from(Arrays.asList(annotations[i])).filter(Nullable.class).isEmpty();
// Only allow boxing for functions that need to see nulls
if (Primitives.isWrapperType(actualType)) {
checkArgument(
nullable,
"Method %s has parameter with type %s that is missing @Nullable",
method,
actualType);
}
if (nullable) {
checkArgument(
!NON_NULLABLE_ARGUMENT_TYPES.contains(actualType),
"Method %s has parameter type %s, but @Nullable is not supported on this type",
method,
actualType);
}
checkArgument(
Primitives.unwrap(actualType) == expectedType.getJavaType(),
"Expected method %s parameter %s type to be %s (%s)",
method,
i,
expectedType.getJavaType().getName(),
expectedType);
}
}
public ParquetPageSource(
ParquetReader parquetReader,
ParquetDataSource dataSource,
MessageType fileSchema,
MessageType requestedSchema,
long totalBytes,
Properties splitSchema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
TypeManager typeManager,
boolean useParquetColumnNames) {
checkArgument(totalBytes >= 0, "totalBytes is negative");
requireNonNull(splitSchema, "splitSchema is null");
requireNonNull(columns, "columns is null");
requireNonNull(effectivePredicate, "effectivePredicate is null");
this.parquetReader = parquetReader;
this.dataSource = dataSource;
this.requestedSchema = requestedSchema;
this.totalBytes = totalBytes;
int size = columns.size();
this.constantBlocks = new Block[size];
this.hiveColumnIndexes = new int[size];
ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
for (int columnIndex = 0; columnIndex < size; columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
checkState(column.getColumnType() == REGULAR, "column type must be regular");
String name = column.getName();
Type type = typeManager.getType(column.getTypeSignature());
namesBuilder.add(name);
typesBuilder.add(type);
hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex();
if (getParquetType(column, fileSchema, useParquetColumnNames) == null) {
BlockBuilder blockBuilder =
type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH);
for (int i = 0; i < MAX_VECTOR_LENGTH; i++) {
blockBuilder.appendNull();
}
constantBlocks[columnIndex] = blockBuilder.build();
}
}
types = typesBuilder.build();
columnNames = namesBuilder.build();
}
@Override
public TableColumn map(int index, ResultSet r, StatementContext ctx) throws SQLException {
QualifiedTableName table =
new QualifiedTableName(
r.getString("catalog_name"), r.getString("schema_name"), r.getString("table_name"));
String typeName = r.getString("data_type");
Type type = typeManager.getType(typeName);
checkArgument(type != null, "Unknown type %s", typeName);
return new TableColumn(
table,
r.getString("column_name"),
r.getInt("ordinal_position"),
type,
r.getLong("column_id"));
}
private static Function<HiveColumnHandle, ColumnMetadata> columnMetadataGetter(
Table table, final TypeManager typeManager) {
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
for (FieldSchema field : concat(table.getSd().getCols(), table.getPartitionKeys())) {
if (field.getComment() != null) {
builder.put(field.getName(), field.getComment());
}
}
final Map<String, String> columnComment = builder.build();
return input ->
new ColumnMetadata(
input.getName(),
typeManager.getType(input.getTypeSignature()),
input.isPartitionKey(),
columnComment.get(input.getName()),
false);
}
@Override
public List<PropertyMetadata<?>> getTableProperties() {
return ImmutableList.of(
integerSessionProperty(
SPLIT_COUNT_PROPERTY, "Number of splits generated by this table", 0, false),
integerSessionProperty(
PAGES_PER_SPLIT_PROPERTY,
"Number of pages per each split generated by this table",
0,
false),
integerSessionProperty(
ROWS_PER_PAGE_PROPERTY,
"Number of rows per each page generated by this table",
0,
false),
integerSessionProperty(
FIELD_LENGTH_PROPERTY,
"Overwrite default length (16) of variable length columns, such as VARCHAR or VARBINARY",
16,
false),
new PropertyMetadata<>(
DISTRIBUTED_ON,
"Distribution columns",
typeManager.getParameterizedType(
ARRAY,
ImmutableList.of(createUnboundedVarcharType().getTypeSignature()),
ImmutableList.of()),
List.class,
ImmutableList.of(),
false,
value ->
ImmutableList.copyOf(
((List<String>) value)
.stream()
.map(name -> name.toLowerCase(ENGLISH))
.collect(Collectors.toList()))));
}
public GenericHiveRecordCursor(
RecordReader<K, V> recordReader,
long totalBytes,
Properties splitSchema,
List<HivePartitionKey> partitionKeys,
List<HiveColumnHandle> columns,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager) {
requireNonNull(recordReader, "recordReader is null");
checkArgument(totalBytes >= 0, "totalBytes is negative");
requireNonNull(splitSchema, "splitSchema is null");
requireNonNull(partitionKeys, "partitionKeys is null");
requireNonNull(columns, "columns is null");
requireNonNull(hiveStorageTimeZone, "hiveStorageTimeZone is null");
this.recordReader = recordReader;
this.totalBytes = totalBytes;
this.key = recordReader.createKey();
this.value = recordReader.createValue();
this.hiveStorageTimeZone = hiveStorageTimeZone;
this.deserializer = getDeserializer(splitSchema);
this.rowInspector = getTableObjectInspector(deserializer);
int size = columns.size();
String[] names = new String[size];
this.types = new Type[size];
this.hiveTypes = new HiveType[size];
this.structFields = new StructField[size];
this.fieldInspectors = new ObjectInspector[size];
this.isPartitionColumn = new boolean[size];
this.loaded = new boolean[size];
this.booleans = new boolean[size];
this.longs = new long[size];
this.doubles = new double[size];
this.slices = new Slice[size];
this.objects = new Object[size];
this.nulls = new boolean[size];
// initialize data columns
for (int i = 0; i < columns.size(); i++) {
HiveColumnHandle column = columns.get(i);
names[i] = column.getName();
types[i] = typeManager.getType(column.getTypeSignature());
hiveTypes[i] = column.getHiveType();
if (!column.isPartitionKey()) {
StructField field = rowInspector.getStructFieldRef(column.getName());
structFields[i] = field;
fieldInspectors[i] = field.getFieldObjectInspector();
}
isPartitionColumn[i] = column.isPartitionKey();
}
// parse requested partition columns
Map<String, HivePartitionKey> partitionKeysByName =
uniqueIndex(partitionKeys, HivePartitionKey::getName);
for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
if (column.isPartitionKey()) {
HivePartitionKey partitionKey = partitionKeysByName.get(column.getName());
checkArgument(partitionKey != null, "Unknown partition key %s", column.getName());
byte[] bytes = partitionKey.getValue().getBytes(UTF_8);
String name = names[columnIndex];
Type type = types[columnIndex];
if (HiveUtil.isHiveNull(bytes)) {
nulls[columnIndex] = true;
} else if (BOOLEAN.equals(type)) {
booleans[columnIndex] = booleanPartitionKey(partitionKey.getValue(), name);
} else if (BIGINT.equals(type)) {
longs[columnIndex] = bigintPartitionKey(partitionKey.getValue(), name);
} else if (INTEGER.equals(type)) {
longs[columnIndex] = integerPartitionKey(partitionKey.getValue(), name);
} else if (SMALLINT.equals(type)) {
longs[columnIndex] = smallintPartitionKey(partitionKey.getValue(), name);
} else if (TINYINT.equals(type)) {
longs[columnIndex] = tinyintPartitionKey(partitionKey.getValue(), name);
} else if (DOUBLE.equals(type)) {
doubles[columnIndex] = doublePartitionKey(partitionKey.getValue(), name);
} else if (isVarcharType(type)) {
slices[columnIndex] = varcharPartitionKey(partitionKey.getValue(), name, type);
} else if (DATE.equals(type)) {
longs[columnIndex] = datePartitionKey(partitionKey.getValue(), name);
} else if (TIMESTAMP.equals(type)) {
longs[columnIndex] =
timestampPartitionKey(partitionKey.getValue(), hiveStorageTimeZone, name);
} else if (isShortDecimal(type)) {
longs[columnIndex] =
shortDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name);
} else if (isLongDecimal(type)) {
slices[columnIndex] =
longDecimalPartitionKey(partitionKey.getValue(), (DecimalType) type, name);
} else {
throw new PrestoException(
NOT_SUPPORTED,
format(
"Unsupported column type %s for partition key: %s", type.getDisplayName(), name));
}
}
}
}
private static Type type(TypeManager typeManager, SqlType explicitType) {
Type type = typeManager.getType(parseTypeSignature(explicitType.value()));
checkNotNull(type, "No type found for '%s'", explicitType.value());
return type;
}
public static OrcPageSource createOrcPageSource(
MetadataReader metadataReader,
Configuration configuration,
Path path,
long start,
long length,
List<HiveColumnHandle> columns,
List<HivePartitionKey> partitionKeys,
TupleDomain<HiveColumnHandle> effectivePredicate,
DateTimeZone hiveStorageTimeZone,
TypeManager typeManager,
DataSize maxMergeDistance,
DataSize maxBufferSize,
DataSize streamBufferSize) {
OrcDataSource orcDataSource;
try {
FileSystem fileSystem = path.getFileSystem(configuration);
long size = fileSystem.getFileStatus(path).getLen();
FSDataInputStream inputStream = fileSystem.open(path);
orcDataSource =
new HdfsOrcDataSource(
path.toString(),
size,
maxMergeDistance,
maxBufferSize,
streamBufferSize,
inputStream);
} catch (Exception e) {
if (nullToEmpty(e.getMessage()).trim().equals("Filesystem closed")
|| e instanceof FileNotFoundException) {
throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, e);
}
throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, splitError(e, path, start, length), e);
}
ImmutableSet.Builder<Integer> includedColumns = ImmutableSet.builder();
ImmutableList.Builder<ColumnReference<HiveColumnHandle>> columnReferences =
ImmutableList.builder();
for (HiveColumnHandle column : columns) {
if (!column.isPartitionKey()) {
includedColumns.add(column.getHiveColumnIndex());
Type type = typeManager.getType(column.getTypeSignature());
columnReferences.add(new ColumnReference<>(column, column.getHiveColumnIndex(), type));
}
}
OrcPredicate predicate =
new TupleDomainOrcPredicate<>(effectivePredicate, columnReferences.build());
try {
OrcReader reader = new OrcReader(orcDataSource, metadataReader);
OrcRecordReader recordReader =
reader.createRecordReader(
includedColumns.build(), predicate, start, length, hiveStorageTimeZone);
return new OrcPageSource(
recordReader, orcDataSource, partitionKeys, columns, hiveStorageTimeZone, typeManager);
} catch (Exception e) {
try {
orcDataSource.close();
} catch (IOException ignored) {
}
if (e instanceof PrestoException) {
throw (PrestoException) e;
}
String message = splitError(e, path, start, length);
if (e.getClass().getSimpleName().equals("BlockMissingException")) {
throw new PrestoException(HIVE_MISSING_DATA, message, e);
}
throw new PrestoException(HIVE_CANNOT_OPEN_SPLIT, message, e);
}
}
public ParquetHiveRecordCursor(
Configuration configuration,
Path path,
long start,
long length,
Properties splitSchema,
List<HivePartitionKey> partitionKeys,
List<HiveColumnHandle> columns,
boolean useParquetColumnNames,
TypeManager typeManager) {
requireNonNull(path, "path is null");
checkArgument(length >= 0, "totalBytes is negative");
requireNonNull(splitSchema, "splitSchema is null");
requireNonNull(partitionKeys, "partitionKeys is null");
requireNonNull(columns, "columns is null");
this.totalBytes = length;
int size = columns.size();
this.names = new String[size];
this.types = new Type[size];
this.isPartitionColumn = new boolean[size];
this.booleans = new boolean[size];
this.longs = new long[size];
this.doubles = new double[size];
this.slices = new Slice[size];
this.objects = new Object[size];
this.nulls = new boolean[size];
this.nullsRowDefault = new boolean[size];
for (int i = 0; i < columns.size(); i++) {
HiveColumnHandle column = columns.get(i);
names[i] = column.getName();
types[i] = typeManager.getType(column.getTypeSignature());
isPartitionColumn[i] = column.isPartitionKey();
nullsRowDefault[i] = !column.isPartitionKey();
}
this.recordReader =
createParquetRecordReader(
configuration, path, start, length, columns, useParquetColumnNames);
// parse requested partition columns
Map<String, HivePartitionKey> partitionKeysByName =
uniqueIndex(partitionKeys, HivePartitionKey::getName);
for (int columnIndex = 0; columnIndex < columns.size(); columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
if (column.isPartitionKey()) {
HivePartitionKey partitionKey = partitionKeysByName.get(column.getName());
checkArgument(partitionKey != null, "Unknown partition key %s", column.getName());
byte[] bytes = partitionKey.getValue().getBytes(UTF_8);
String name = names[columnIndex];
Type type = types[columnIndex];
if (HiveUtil.isHiveNull(bytes)) {
nullsRowDefault[columnIndex] = true;
} else if (type.equals(BOOLEAN)) {
booleans[columnIndex] = booleanPartitionKey(partitionKey.getValue(), name);
} else if (type.equals(BIGINT)) {
longs[columnIndex] = bigintPartitionKey(partitionKey.getValue(), name);
} else if (type.equals(DOUBLE)) {
doubles[columnIndex] = doublePartitionKey(partitionKey.getValue(), name);
} else if (type.equals(VARCHAR)) {
slices[columnIndex] = Slices.wrappedBuffer(bytes);
} else {
throw new PrestoException(
NOT_SUPPORTED,
format(
"Unsupported column type %s for partition key: %s", type.getDisplayName(), name));
}
}
}
}
@Override
public Type resolve(
Map<String, Type> types, TypeManager typeManager, FunctionRegistry functionRegistry) {
return typeManager.getType(signature.bindParameters(types));
}
