public static Signature nullIfSignature(Type returnType, Type firstType, Type secondType) {
return new Signature(
NULL_IF,
returnType.getTypeSignature(),
firstType.getTypeSignature(),
secondType.getTypeSignature());
}
public static Signature subscriptSignature(Type returnType, Type leftType, Type rightType) {
return internalOperator(
SUBSCRIPT.name(),
returnType.getTypeSignature(),
leftType.getTypeSignature(),
rightType.getTypeSignature());
}
@Test
public void testRowTypeLookup() throws Exception {
functionAssertions.getMetadata().getType(parseTypeSignature("row<bigint>('a')"));
Type type = functionAssertions.getMetadata().getType(parseTypeSignature("row<bigint>('b')"));
assertEquals(type.getTypeSignature().getParameters().size(), 1);
assertEquals(
type.getTypeSignature().getParameters().get(0).getNamedTypeSignature().getName(), "b");
}
public static Signature betweenSignature(Type valueType, Type minType, Type maxType) {
return internalOperator(
"BETWEEN",
parseTypeSignature(StandardTypes.BOOLEAN),
valueType.getTypeSignature(),
minType.getTypeSignature(),
maxType.getTypeSignature());
}
public static Signature arithmeticExpressionSignature(
ArithmeticBinaryExpression.Type expressionType,
Type returnType,
Type leftType,
Type rightType) {
return internalOperator(
expressionType.name(),
returnType.getTypeSignature(),
leftType.getTypeSignature(),
rightType.getTypeSignature());
}
@Override
public FunctionInfo specialize(
Map<String, Type> types,
int arity,
TypeManager typeManager,
FunctionRegistry functionRegistry) {
Type valueType = types.get("T");
Signature signature =
new Signature(NAME, AGGREGATE, valueType.getTypeSignature(), valueType.getTypeSignature());
InternalAggregationFunction aggregation = generateAggregation(valueType);
return new FunctionInfo(signature, getDescription(), aggregation);
}
public static Signature arrayConstructorSignature(
Type returnType, List<? extends Type> argumentTypes) {
return internalFunction(
ARRAY_CONSTRUCTOR,
returnType.getTypeSignature(),
Lists.transform(argumentTypes, Type::getTypeSignature));
}
public ParquetListConverter(Type prestoType, String columnName, GroupType listType) {
checkArgument(
listType.getFieldCount() == 1,
"Expected LIST column '%s' to only have one field, but has %s fields",
columnName,
listType.getFieldCount());
checkArgument(ARRAY.equals(prestoType.getTypeSignature().getBase()));
this.arrayType = prestoType;
// The Parquet specification requires that the element value of a
// LIST type be wrapped in an inner repeated group, like so:
//
// optional group listField (LIST) {
// repeated group list {
// optional int element
// }
// }
//
// However, some parquet libraries don't follow this spec. The
// compatibility rules used here are specified in the Parquet
// documentation at http://git.io/vOpNz.
parquet.schema.Type elementType = listType.getType(0);
if (isElementType(elementType, listType.getName())) {
elementConverter =
createConverter(
prestoType.getTypeParameters().get(0), columnName + ".element", elementType);
} else {
elementConverter =
new ParquetListEntryConverter(
prestoType.getTypeParameters().get(0), columnName, elementType.asGroupType());
}
}
public static Signature comparisonExpressionSignature(
ComparisonExpression.Type expressionType, Type leftType, Type rightType) {
for (OperatorType operatorType : OperatorType.values()) {
if (operatorType.name().equals(expressionType.name())) {
return internalOperator(
expressionType.name(),
parseTypeSignature(StandardTypes.BOOLEAN),
leftType.getTypeSignature(),
rightType.getTypeSignature());
}
}
return internalFunction(
expressionType.name(),
parseTypeSignature(StandardTypes.BOOLEAN),
leftType.getTypeSignature(),
rightType.getTypeSignature());
}
@Override
protected Type visitArrayConstructor(ArrayConstructor node, AnalysisContext context) {
Type type =
coerceToSingleType(
context, "All ARRAY elements must be the same type: %s", node.getValues());
Type arrayType =
typeManager.getParameterizedType(
ARRAY.getName(), ImmutableList.of(type.getTypeSignature()), ImmutableList.of());
expressionTypes.put(node, arrayType);
return arrayType;
}
@Override
public FunctionInfo specialize(
Map<String, Type> types,
int arity,
TypeManager typeManager,
FunctionRegistry functionRegistry) {
checkArgument(types.size() == 1, "Can only construct arrays from exactly matching types");
ImmutableList.Builder<Class<?>> builder = ImmutableList.builder();
Type type = types.get("E");
for (int i = 0; i < arity; i++) {
if (type.getJavaType().isPrimitive()) {
builder.add(Primitives.wrap(type.getJavaType()));
} else {
builder.add(type.getJavaType());
}
}
ImmutableList<Class<?>> stackTypes = builder.build();
Class<?> clazz = generateArrayConstructor(stackTypes, type);
MethodHandle methodHandle;
try {
Method method =
clazz.getMethod("arrayConstructor", stackTypes.toArray(new Class<?>[stackTypes.size()]));
methodHandle = lookup().unreflect(method);
} catch (ReflectiveOperationException e) {
throw Throwables.propagate(e);
}
List<Boolean> nullableParameters =
ImmutableList.copyOf(Collections.nCopies(stackTypes.size(), true));
return new FunctionInfo(
arrayConstructorSignature(
parameterizedTypeName("array", type.getTypeSignature()),
Collections.nCopies(arity, type.getTypeSignature())),
"Constructs an array of the given elements",
true,
methodHandle,
true,
false,
nullableParameters);
}
private boolean processScalarFunction(Method method) throws IllegalAccessException {
ScalarFunction scalarFunction = method.getAnnotation(ScalarFunction.class);
if (scalarFunction == null) {
return false;
}
checkValidMethod(method);
MethodHandle methodHandle = lookup().unreflect(method);
String name = scalarFunction.value();
if (name.isEmpty()) {
name = camelToSnake(method.getName());
}
SqlType returnTypeAnnotation = method.getAnnotation(SqlType.class);
checkArgument(
returnTypeAnnotation != null,
"Method %s return type does not have a @SqlType annotation",
method);
Type returnType = type(typeManager, returnTypeAnnotation);
Signature signature =
new Signature(
name.toLowerCase(ENGLISH),
returnType.getTypeSignature(),
Lists.transform(parameterTypes(typeManager, method), Type::getTypeSignature));
verifyMethodSignature(
method, signature.getReturnType(), signature.getArgumentTypes(), typeManager);
List<Boolean> nullableArguments = getNullableArguments(method);
scalar(
signature,
methodHandle,
scalarFunction.deterministic(),
getDescription(method),
scalarFunction.hidden(),
method.isAnnotationPresent(Nullable.class),
nullableArguments);
for (String alias : scalarFunction.alias()) {
scalar(
signature.withAlias(alias.toLowerCase(ENGLISH)),
methodHandle,
scalarFunction.deterministic(),
getDescription(method),
scalarFunction.hidden(),
method.isAnnotationPresent(Nullable.class),
nullableArguments);
}
return true;
}
@Override
public FunctionInfo specialize(
Map<String, Type> types,
int arity,
TypeManager typeManager,
FunctionRegistry functionRegistry) {
Type type = types.get("T");
TypeSignature typeSignature = type.getTypeSignature();
return operatorInfo(
NOT_EQUAL,
RETURN_TYPE,
ImmutableList.of(typeSignature, typeSignature),
METHOD_HANDLE.bindTo(type),
false,
ImmutableList.of(false, false));
}
public ParquetStructConverter(Type prestoType, String columnName, GroupType entryType) {
checkArgument(ROW.equals(prestoType.getTypeSignature().getBase()));
List<Type> prestoTypeParameters = prestoType.getTypeParameters();
List<parquet.schema.Type> fieldTypes = entryType.getFields();
checkArgument(prestoTypeParameters.size() == fieldTypes.size());
this.rowType = prestoType;
ImmutableList.Builder<BlockConverter> converters = ImmutableList.builder();
for (int i = 0; i < prestoTypeParameters.size(); i++) {
parquet.schema.Type fieldType = fieldTypes.get(i);
converters.add(
createConverter(
prestoTypeParameters.get(i), columnName + "." + fieldType.getName(), fieldType));
}
this.converters = converters.build();
}
@Override
public FunctionInfo specialize(
Map<String, Type> types,
int arity,
TypeManager typeManager,
FunctionRegistry functionRegistry) {
checkArgument(arity == 1, "Expected arity to be 1");
Type type = types.get("T");
MethodHandle methodHandle = METHOD_HANDLE.bindTo(type);
return operatorInfo(
OperatorType.CAST,
parseTypeSignature(StandardTypes.JSON),
ImmutableList.of(type.getTypeSignature()),
methodHandle,
false,
ImmutableList.of(false));
}
public ParquetMapEntryConverter(Type prestoType, String columnName, GroupType entryType) {
checkArgument(StandardTypes.MAP.equals(prestoType.getTypeSignature().getBase()));
// original version of parquet used null for entry due to a bug
if (entryType.getOriginalType() != null) {
checkArgument(
entryType.getOriginalType() == MAP_KEY_VALUE,
"Expected MAP column '%s' field to be type %s, but is %s",
columnName,
MAP_KEY_VALUE,
entryType);
}
GroupType entryGroupType = entryType.asGroupType();
checkArgument(
entryGroupType.getFieldCount() == 2,
"Expected MAP column '%s' entry to have two fields, but has %s fields",
columnName,
entryGroupType.getFieldCount());
checkArgument(
entryGroupType.getFieldName(0).equals("key"),
"Expected MAP column '%s' entry field 0 to be named 'key', but is named %s",
columnName,
entryGroupType.getFieldName(0));
checkArgument(
entryGroupType.getFieldName(1).equals("value"),
"Expected MAP column '%s' entry field 1 to be named 'value', but is named %s",
columnName,
entryGroupType.getFieldName(1));
checkArgument(
entryGroupType.getType(0).isPrimitive(),
"Expected MAP column '%s' entry field 0 to be primitive, but is named %s",
columnName,
entryGroupType.getType(0));
keyConverter =
createConverter(
prestoType.getTypeParameters().get(0),
columnName + ".key",
entryGroupType.getFields().get(0));
valueConverter =
createConverter(
prestoType.getTypeParameters().get(1),
columnName + ".value",
entryGroupType.getFields().get(1));
}
private FunctionListBuilder operator(
OperatorType operatorType,
Type returnType,
List<Type> parameterTypes,
MethodHandle function,
boolean nullable,
List<Boolean> nullableArguments) {
FunctionInfo operatorInfo =
operatorInfo(
operatorType,
returnType.getTypeSignature(),
Lists.transform(parameterTypes, Type::getTypeSignature),
function,
nullable,
nullableArguments);
functions.add(operatorInfo);
return this;
}
private RelationPlan processAndCoerceIfNecessary(Relation node, Void context) {
Type[] coerceToTypes = analysis.getRelationCoercion(node);
RelationPlan plan = node.accept(this, context);
if (coerceToTypes == null) {
return plan;
}
List<Symbol> oldSymbols = plan.getOutputSymbols();
TupleDescriptor oldDescriptor = plan.getDescriptor().withOnlyVisibleFields();
verify(coerceToTypes.length == oldSymbols.size());
ImmutableList.Builder<Symbol> newSymbols = new ImmutableList.Builder<>();
Field[] newFields = new Field[coerceToTypes.length];
ImmutableMap.Builder<Symbol, Expression> assignments = new ImmutableMap.Builder<>();
for (int i = 0; i < coerceToTypes.length; i++) {
Symbol inputSymbol = oldSymbols.get(i);
Type inputType = symbolAllocator.getTypes().get(inputSymbol);
Type outputType = coerceToTypes[i];
if (outputType != inputType) {
Cast cast =
new Cast(
new QualifiedNameReference(inputSymbol.toQualifiedName()),
outputType.getTypeSignature().toString());
Symbol outputSymbol = symbolAllocator.newSymbol(cast, outputType);
assignments.put(outputSymbol, cast);
newSymbols.add(outputSymbol);
} else {
assignments.put(inputSymbol, new QualifiedNameReference(inputSymbol.toQualifiedName()));
newSymbols.add(inputSymbol);
}
Field oldField = oldDescriptor.getFieldByIndex(i);
newFields[i] =
new Field(
oldField.getRelationAlias(),
oldField.getName(),
coerceToTypes[i],
oldField.isHidden());
}
ProjectNode projectNode =
new ProjectNode(idAllocator.getNextId(), plan.getRoot(), assignments.build());
return new RelationPlan(
projectNode, new TupleDescriptor(newFields), newSymbols.build(), plan.getSampleWeight());
}
public FunctionListBuilder window(
String name,
Type returnType,
List<? extends Type> argumentTypes,
Class<? extends WindowFunction> functionClass) {
WindowFunctionSupplier windowFunctionSupplier =
new ReflectionWindowFunctionSupplier<>(
new Signature(
name,
returnType.getTypeSignature(),
Lists.transform(ImmutableList.copyOf(argumentTypes), Type::getTypeSignature)),
functionClass);
functions.add(
new FunctionInfo(
windowFunctionSupplier.getSignature(),
windowFunctionSupplier.getDescription(),
windowFunctionSupplier));
return this;
}
private boolean processScalarOperator(Method method) throws IllegalAccessException {
ScalarOperator scalarOperator = method.getAnnotation(ScalarOperator.class);
if (scalarOperator == null) {
return false;
}
checkValidMethod(method);
MethodHandle methodHandle = lookup().unreflect(method);
OperatorType operatorType = scalarOperator.value();
List<Type> parameterTypes = parameterTypes(typeManager, method);
Type returnType;
if (operatorType == OperatorType.HASH_CODE) {
// todo hack for hashCode... should be int
returnType = BIGINT;
} else {
SqlType explicitType = method.getAnnotation(SqlType.class);
checkArgument(
explicitType != null,
"Method %s return type does not have a @SqlType annotation",
method);
returnType = type(typeManager, explicitType);
verifyMethodSignature(
method,
returnType.getTypeSignature(),
Lists.transform(parameterTypes, Type::getTypeSignature),
typeManager);
}
List<Boolean> nullableArguments = getNullableArguments(method);
operator(
operatorType,
returnType,
parameterTypes,
methodHandle,
method.isAnnotationPresent(Nullable.class),
nullableArguments);
return true;
}
protected List<HiveColumnHandle> getColumnHandles(List<TestColumn> testColumns) {
List<HiveColumnHandle> columns = new ArrayList<>();
int nextHiveColumnIndex = 0;
for (int i = 0; i < testColumns.size(); i++) {
TestColumn testColumn = testColumns.get(i);
int columnIndex = testColumn.isPartitionKey() ? -1 : nextHiveColumnIndex++;
ObjectInspector inspector = testColumn.getObjectInspector();
HiveType hiveType = HiveType.getHiveType(inspector);
Type type = getType(inspector, TYPE_MANAGER);
columns.add(
new HiveColumnHandle(
"client_id",
testColumn.getName(),
i,
hiveType,
type.getTypeSignature(),
columnIndex,
testColumn.isPartitionKey()));
}
return columns;
}
private Type instantiateParametricType(TypeSignature signature) {
List<TypeParameter> parameters = new ArrayList<>();
for (TypeSignatureParameter parameter : signature.getParameters()) {
TypeParameter typeParameter = TypeParameter.of(parameter, this);
if (typeParameter == null) {
return null;
}
parameters.add(typeParameter);
}
ParametricType parametricType =
parametricTypes.get(signature.getBase().toLowerCase(Locale.ENGLISH));
if (parametricType == null) {
return null;
}
Type instantiatedType = parametricType.createType(parameters);
checkState(
instantiatedType.getTypeSignature().equals(signature),
"Instantiated parametric type name (%s) does not match expected name (%s)",
instantiatedType,
signature);
return instantiatedType;
}
public static Signature tryCastSignature(Type returnType, Type valueType) {
return internalFunction(TRY_CAST, returnType.getTypeSignature(), valueType.getTypeSignature());
}
@Override
public Optional<Type> getCommonSuperType(Type firstType, Type secondType) {
return getCommonSuperTypeSignature(firstType.getTypeSignature(), secondType.getTypeSignature())
.map(this::getType);
}
public static boolean canCoerce(Type actualType, Type expectedType) {
return canCoerce(actualType.getTypeSignature(), expectedType.getTypeSignature());
}
public void addType(Type type) {
verifyTypeClass(type);
Type existingType = types.putIfAbsent(type.getTypeSignature(), type);
checkState(
existingType == null || existingType.equals(type), "Type %s is already registered", type);
}
public static Signature arithmeticNegationSignature(Type returnType, Type valueType) {
return internalOperator(
"NEGATION", returnType.getTypeSignature(), valueType.getTypeSignature());
}
public static Signature castSignature(Type returnType, Type valueType) {
// Name has already been mangled, so don't use internalOperator
return internalFunction(CAST, returnType.getTypeSignature(), valueType.getTypeSignature());
}
public static Signature coalesceSignature(Type returnType, List<Type> argumentTypes) {
return internalFunction(
COALESCE,
returnType.getTypeSignature(),
Lists.transform(argumentTypes, Type::getTypeSignature));
}
// **************** functions that need to do special null handling ****************
public static Signature isNullSignature(Type argumentType) {
return internalFunction(
IS_NULL, parseTypeSignature(StandardTypes.BOOLEAN), argumentType.getTypeSignature());
}