@Override
public SubPlanBuilder visitUnion(UnionNode node, Void context) {
if (createSingleNodePlan) {
ImmutableList.Builder<PlanNode> sourceBuilder = ImmutableList.builder();
for (PlanNode source : node.getSources()) {
sourceBuilder.add(source.accept(this, context).getRoot());
}
UnionNode unionNode =
new UnionNode(node.getId(), sourceBuilder.build(), node.getSymbolMapping());
return createSingleNodePlan(unionNode);
} else {
ImmutableList.Builder<SubPlan> sourceBuilder = ImmutableList.builder();
ImmutableList.Builder<PlanFragmentId> fragmentIdBuilder = ImmutableList.builder();
for (int i = 0; i < node.getSources().size(); i++) {
PlanNode subPlan = node.getSources().get(i);
SubPlanBuilder current = subPlan.accept(this, context);
current.setRoot(
new SinkNode(idAllocator.getNextId(), current.getRoot(), node.sourceOutputLayout(i)));
fragmentIdBuilder.add(current.getId());
sourceBuilder.add(current.build());
}
ExchangeNode exchangeNode =
new ExchangeNode(
idAllocator.getNextId(), fragmentIdBuilder.build(), node.getOutputSymbols());
return createSingleNodePlan(exchangeNode).setChildren(sourceBuilder.build());
}
}
@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) {
checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty");
SchemaTableName schemaTableName = tableMetadata.getTable();
String schemaName = schemaTableName.getSchemaName();
String tableName = schemaTableName.getTableName();
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
buildColumnInfo(tableMetadata, columnNames, columnTypes);
ImmutableList.Builder<FieldSchema> partitionKeys = ImmutableList.builder();
ImmutableList.Builder<FieldSchema> columns = ImmutableList.builder();
List<String> names = columnNames.build();
List<String> typeNames =
columnTypes
.build()
.stream()
.map(HiveType::toHiveType)
.map(HiveType::getHiveTypeName)
.collect(toList());
for (int i = 0; i < names.size(); i++) {
if (tableMetadata.getColumns().get(i).isPartitionKey()) {
partitionKeys.add(new FieldSchema(names.get(i), typeNames.get(i), null));
} else {
columns.add(new FieldSchema(names.get(i), typeNames.get(i), null));
}
}
Path targetPath = getTargetPath(schemaName, tableName, schemaTableName);
HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(session, this.hiveStorageFormat);
SerDeInfo serdeInfo = new SerDeInfo();
serdeInfo.setName(tableName);
serdeInfo.setSerializationLib(hiveStorageFormat.getSerDe());
StorageDescriptor sd = new StorageDescriptor();
sd.setLocation(targetPath.toString());
sd.setCols(columns.build());
sd.setSerdeInfo(serdeInfo);
sd.setInputFormat(hiveStorageFormat.getInputFormat());
sd.setOutputFormat(hiveStorageFormat.getOutputFormat());
Table table = new Table();
table.setDbName(schemaName);
table.setTableName(tableName);
table.setOwner(tableMetadata.getOwner());
table.setTableType(TableType.MANAGED_TABLE.toString());
String tableComment = "Created by Presto";
table.setParameters(ImmutableMap.of("comment", tableComment));
table.setPartitionKeys(partitionKeys.build());
table.setSd(sd);
metastore.createTable(table);
}
@Override
public PlanNode visitValues(ValuesNode node, RewriteContext<Set<Symbol>> context) {
ImmutableList.Builder<Symbol> rewrittenOutputSymbolsBuilder = ImmutableList.builder();
ImmutableList.Builder<ImmutableList.Builder<Expression>> rowBuildersBuilder =
ImmutableList.builder();
// Initialize builder for each row
for (int i = 0; i < node.getRows().size(); i++) {
rowBuildersBuilder.add(ImmutableList.builder());
}
ImmutableList<ImmutableList.Builder<Expression>> rowBuilders = rowBuildersBuilder.build();
for (int i = 0; i < node.getOutputSymbols().size(); i++) {
Symbol outputSymbol = node.getOutputSymbols().get(i);
// If output symbol is used
if (context.get().contains(outputSymbol)) {
rewrittenOutputSymbolsBuilder.add(outputSymbol);
// Add the value of the output symbol for each row
for (int j = 0; j < node.getRows().size(); j++) {
rowBuilders.get(j).add(node.getRows().get(j).get(i));
}
}
}
List<List<Expression>> rewrittenRows =
rowBuilders.stream().map((rowBuilder) -> rowBuilder.build()).collect(toImmutableList());
return new ValuesNode(node.getId(), rewrittenOutputSymbolsBuilder.build(), rewrittenRows);
}
public JoinLeafPredicateInfo(
SqlKind comparisonType,
List<List<RexNode>> joinKeyExprs,
List<Set<Integer>> projsJoinKeysInChildSchema,
List<Set<Integer>> projsJoinKeysInJoinSchema) {
this.comparisonType = comparisonType;
ImmutableList.Builder<ImmutableList<RexNode>> joinKeyExprsBuilder = ImmutableList.builder();
for (int i = 0; i < joinKeyExprs.size(); i++) {
joinKeyExprsBuilder.add(ImmutableList.copyOf(joinKeyExprs.get(i)));
}
this.joinKeyExprs = joinKeyExprsBuilder.build();
ImmutableList.Builder<ImmutableSet<Integer>> projsJoinKeysInChildSchemaBuilder =
ImmutableList.builder();
for (int i = 0; i < projsJoinKeysInChildSchema.size(); i++) {
projsJoinKeysInChildSchemaBuilder.add(
ImmutableSet.copyOf(projsJoinKeysInChildSchema.get(i)));
}
this.projsJoinKeysInChildSchema = projsJoinKeysInChildSchemaBuilder.build();
ImmutableList.Builder<ImmutableSet<Integer>> projsJoinKeysInJoinSchemaBuilder =
ImmutableList.builder();
for (int i = 0; i < projsJoinKeysInJoinSchema.size(); i++) {
projsJoinKeysInJoinSchemaBuilder.add(ImmutableSet.copyOf(projsJoinKeysInJoinSchema.get(i)));
}
this.projsJoinKeysInJoinSchema = projsJoinKeysInJoinSchemaBuilder.build();
}
private static ImmutableList<Token> buildTokens(List<Tok> toks) {
ImmutableList.Builder<Token> tokens = ImmutableList.builder();
int k = 0;
int kN = toks.size();
while (k < kN) {
// Remaining non-tokens before the token go here.
ImmutableList.Builder<Tok> toksBefore = ImmutableList.builder();
while (!toks.get(k).isToken()) {
toksBefore.add(toks.get(k++));
}
Tok tok = toks.get(k++);
// Non-tokens starting on the same line go here too.
ImmutableList.Builder<Tok> toksAfter = ImmutableList.builder();
while (k < kN && !"\n".equals(toks.get(k).getText()) && !toks.get(k).isToken()) {
Tok nonTokenAfter = toks.get(k++);
toksAfter.add(nonTokenAfter);
if (nonTokenAfter.getText().contains("\n")) {
break;
}
}
tokens.add(new Token(toksBefore.build(), tok, toksAfter.build()));
}
return tokens.build();
}
@Override
public List<String> onTabComplete(
CommandSender sebder, Command command, String label, String[] args) {
if (args.length == 0) {
return ImmutableList.of("module", "debug", "reload");
}
if (args.length == 1) {
Builder<String> list = ImmutableList.builder();
for (String string : Arrays.asList("module", "debug", "reload")) {
if (string.startsWith(args[0])) {
list.add(string);
}
}
return list.build();
}
if (args.length == 2) {
Builder<String> list = ImmutableList.builder();
for (String string : Arrays.asList("unload", "load", "reload", "debug")) {
if (string.startsWith(args[0])) {
list.add(string);
}
}
return list.build();
}
return ImmutableList.of();
}
private void logEndpoints() {
final StringBuilder stringBuilder = new StringBuilder(1024).append("\n\n");
final ImmutableList.Builder<Class<?>> builder = ImmutableList.builder();
for (Object o : config.getSingletons()) {
if (o.getClass().isAnnotationPresent(Path.class)) {
builder.add(o.getClass());
}
}
for (Class<?> klass : config.getClasses()) {
if (klass.isAnnotationPresent(Path.class)) {
builder.add(klass);
}
}
for (Class<?> klass : builder.build()) {
final String path = klass.getAnnotation(Path.class).value();
final ImmutableList.Builder<String> endpoints = ImmutableList.builder();
for (AnnotatedMethod method : annotatedMethods(klass)) {
for (HttpMethod verb : method.getMetaMethodAnnotations(HttpMethod.class)) {
endpoints.add(
String.format(" %-7s %s (%s)", verb.value(), path, klass.getCanonicalName()));
}
}
for (String line : Ordering.natural().sortedCopy(endpoints.build())) {
stringBuilder.append(line).append('\n');
}
}
LOG.info(stringBuilder.toString());
}
private static <T> ImmutableList<ComponentAndSize<T>> resized(
ImmutableList<ComponentSizeAndWeight<T>> sizes, int sizeOfAll) {
int weightOfAll = weights(sizes);
ImmutableList<ComponentAndSize<T>> weighted = weighted(sizes, sizeOfAll, weightOfAll);
ImmutableList<ComponentAndSize<T>> toBig = toBig(weighted);
if (!toBig.isEmpty()) {
ImmutableList<ComponentSizeAndWeight<T>> left = withoutMatching(sizes, components(toBig));
ImmutableList<ComponentAndSize<T>> maxOfToBig = sizeFixedToLimit(toBig);
int sizeOfToBig = sumOrMax(maxOfToBig);
Builder<ComponentAndSize<T>> builder = ImmutableList.<ComponentAndSize<T>>builder();
builder.addAll(maxOfToBig);
builder.addAll(resized(left, sizeOfAll - sizeOfToBig));
return builder.build();
} else {
ImmutableList<ComponentAndSize<T>> toSmall = toSmall(weighted);
if (!toSmall.isEmpty()) {
ImmutableList<ComponentSizeAndWeight<T>> left = withoutMatching(sizes, components(toSmall));
ImmutableList<ComponentAndSize<T>> maxOfToSmall = sizeFixedToLimit(toSmall);
int sizeOfToBig = sumOrMax(maxOfToSmall);
Builder<ComponentAndSize<T>> builder = ImmutableList.<ComponentAndSize<T>>builder();
builder.addAll(maxOfToSmall);
builder.addAll(resized(left, sizeOfAll - sizeOfToBig));
return builder.build();
}
}
return weighted;
}
private List<CassandraPartition> getCassandraPartitions(
CassandraTable table, TupleDomain<ColumnHandle> tupleDomain) {
if (tupleDomain.isNone()) {
return ImmutableList.of();
}
Set<List<Comparable<?>>> partitionKeysSet = getPartitionKeysSet(table, tupleDomain);
// empty filter means, all partitions
if (partitionKeysSet.isEmpty()) {
return schemaProvider.getAllPartitions(table);
}
ImmutableList.Builder<ListenableFuture<List<CassandraPartition>>> getPartitionResults =
ImmutableList.builder();
for (List<Comparable<?>> partitionKeys : partitionKeysSet) {
getPartitionResults.add(
executor.submit(() -> schemaProvider.getPartitions(table, partitionKeys)));
}
ImmutableList.Builder<CassandraPartition> partitions = ImmutableList.builder();
for (ListenableFuture<List<CassandraPartition>> result : getPartitionResults.build()) {
try {
partitions.addAll(result.get());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw Throwables.propagate(e);
} catch (ExecutionException e) {
throw new PrestoException(EXTERNAL, "Error fetching cassandra partitions", e);
}
}
return partitions.build();
}
static byte[] encode(RSAPrivateCrtKey key) {
List<BigInteger> seq =
ImmutableList.<BigInteger>builder()
.add(BigInteger.valueOf(0)) // version
.add(key.getModulus())
.add(key.getPublicExponent())
.add(key.getPrivateExponent())
.add(key.getPrimeP())
.add(key.getPrimeQ())
.add(key.getPrimeExponentP())
.add(key.getPrimeExponentQ())
.add(key.getCrtCoefficient())
.build();
int length = 0;
for (BigInteger part : seq) {
byte[] bytes = part.toByteArray();
length += 1 + calculateBodyLength(bytes.length) + bytes.length;
}
Builder<Byte> output = ImmutableList.<Byte>builder();
output.add((byte) (SEQUENCE | CONSTRUCTED));
writeLength(output, length);
for (BigInteger part : seq) {
byte[] bytes = part.toByteArray();
output.add((byte) TAG);
writeLength(output, bytes.length);
output.addAll(Bytes.asList(bytes));
}
return Bytes.toArray(output.build());
}
private Builder() {
created = ImmutableList.builder();
updated = ImmutableList.builder();
destroyed = ImmutableList.builder();
notCreated = ImmutableMap.builder();
notUpdated = ImmutableMap.builder();
notDestroyed = ImmutableMap.builder();
}
private void assertCompDir(Path compDir, Optional<String> failure) throws Exception {
ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot().toPath());
CxxPlatform platform = DefaultCxxPlatforms.build(new CxxBuckConfig(new FakeBuckConfig()));
// Build up the paths to various files the archive step will use.
ImmutableList<String> compiler =
platform.getCc().getCommandPrefix(new SourcePathResolver(new BuildRuleResolver()));
Path output = filesystem.resolve(Paths.get("output.o"));
Path relativeInput = Paths.get("input.c");
Path input = filesystem.resolve(relativeInput);
filesystem.writeContentsToPath("int main() {}", relativeInput);
ImmutableList.Builder<String> preprocessorCommand = ImmutableList.builder();
preprocessorCommand.addAll(compiler);
ImmutableList.Builder<String> compilerCommand = ImmutableList.builder();
compilerCommand.addAll(compiler);
compilerCommand.add("-g");
DebugPathSanitizer sanitizer =
new DebugPathSanitizer(200, File.separatorChar, compDir, ImmutableBiMap.<Path, Path>of());
// Build an archive step.
CxxPreprocessAndCompileStep step =
new CxxPreprocessAndCompileStep(
CxxPreprocessAndCompileStep.Operation.COMPILE_MUNGE_DEBUGINFO,
output,
relativeInput,
CxxSource.Type.C,
Optional.of(preprocessorCommand.build()),
Optional.of(compilerCommand.build()),
ImmutableMap.<Path, Path>of(),
sanitizer);
// Execute the archive step and verify it ran successfully.
ExecutionContext executionContext =
TestExecutionContext.newBuilder()
.setProjectFilesystem(new ProjectFilesystem(tmp.getRoot().toPath()))
.build();
TestConsole console = (TestConsole) executionContext.getConsole();
int exitCode = step.execute(executionContext);
if (failure.isPresent()) {
assertNotEquals("compile step succeeded", 0, exitCode);
assertThat(
console.getTextWrittenToStdErr(),
console.getTextWrittenToStdErr(),
Matchers.containsString(failure.get()));
} else {
assertEquals("compile step failed: " + console.getTextWrittenToStdErr(), 0, exitCode);
// Verify that we find the expected compilation dir embedded in the file.
String contents = new String(Files.readAllBytes(output));
assertThat(contents, Matchers.containsString(sanitizer.getCompilationDirectory()));
}
// Cleanup.
Files.delete(input);
Files.deleteIfExists(output);
}
private <T> CachedRuleSource doExtract(final Class<T> source) {
final ModelType<T> type = ModelType.of(source);
DefaultMethodModelRuleExtractionContext context =
new DefaultMethodModelRuleExtractionContext(type, this);
// TODO - exceptions thrown here should point to some extensive documentation on the concept of
// class rule sources
StructSchema<T> schema = getSchema(source, context);
if (schema == null) {
throw new InvalidModelRuleDeclarationException(context.problems.format());
}
// sort for determinism
Set<Method> methods = new TreeSet<Method>(Ordering.usingToString());
methods.addAll(Arrays.asList(source.getDeclaredMethods()));
ImmutableList.Builder<ModelProperty<?>> implicitInputs = ImmutableList.builder();
ModelProperty<?> target = null;
for (ModelProperty<?> property : schema.getProperties()) {
if (property.isAnnotationPresent(RuleTarget.class)) {
target = property;
} else if (property.isAnnotationPresent(RuleInput.class)
&& !(property.getSchema() instanceof ScalarValueSchema)) {
implicitInputs.add(property);
}
for (WeaklyTypeReferencingMethod<?, ?> method : property.getAccessors()) {
methods.remove(method.getMethod());
}
}
ImmutableList.Builder<ExtractedRuleDetails> rules = ImmutableList.builder();
for (Method method : methods) {
MethodRuleDefinition<?, ?> ruleDefinition =
DefaultMethodRuleDefinition.create(source, method);
ExtractedModelRule rule = getMethodHandler(ruleDefinition, method, context);
if (rule != null) {
rules.add(new ExtractedRuleDetails(ruleDefinition, rule));
}
}
if (context.hasProblems()) {
throw new InvalidModelRuleDeclarationException(context.problems.format());
}
StructBindings<T> bindings = structBindingsStore.getBindings(schema);
if (schema.getProperties().isEmpty()) {
return new StatelessRuleSource(
rules.build(),
Modifier.isAbstract(source.getModifiers())
? new AbstractRuleSourceFactory<T>(schema, bindings, proxyFactory)
: new ConcreteRuleSourceFactory<T>(type));
} else {
return new ParameterizedRuleSource(
rules.build(), target, implicitInputs.build(), schema, bindings, proxyFactory);
}
}
private ImmutableList<SourcePath> generateCCompilation(ImmutableList<SourcePath> cInput) {
ImmutableList.Builder<SourcePath> objects = ImmutableList.builder();
ImmutableList.Builder<String> cCompileFlags = ImmutableList.builder();
cCompileFlags.addAll(ocamlContext.getCCompileFlags());
cCompileFlags.addAll(ocamlContext.getCommonCFlags());
CxxPreprocessorInput cxxPreprocessorInput = ocamlContext.getCxxPreprocessorInput();
for (SourcePath cSrc : cInput) {
String name = pathResolver.getAbsolutePath(cSrc).toFile().getName();
BuildTarget target = createCCompileBuildTarget(params.getBuildTarget(), name);
BuildRuleParams cCompileParams =
params.copyWithChanges(
target,
/* declaredDeps */ Suppliers.ofInstance(
ImmutableSortedSet.<BuildRule>naturalOrder()
// Depend on the rule that generates the sources and headers we're compiling.
.addAll(
pathResolver.filterBuildRuleInputs(
ImmutableList.<SourcePath>builder()
.add(cSrc)
.addAll(
cxxPreprocessorInput
.getIncludes()
.getNameToPathMap()
.values())
.build()))
// Also add in extra deps from the preprocessor input, such as the symlink
// tree rules.
.addAll(
BuildRules.toBuildRulesFor(
params.getBuildTarget(), resolver, cxxPreprocessorInput.getRules()))
.addAll(params.getDeclaredDeps().get())
.build()),
/* extraDeps */ params.getExtraDeps());
Path outputPath = ocamlContext.getCOutput(pathResolver.getRelativePath(cSrc));
OCamlCCompile compileRule =
new OCamlCCompile(
cCompileParams,
pathResolver,
new OCamlCCompileStep.Args(
cCompiler.getEnvironment(pathResolver),
cCompiler.getCommandPrefix(pathResolver),
ocamlContext.getOcamlCompiler().get(),
outputPath,
cSrc,
cCompileFlags.build(),
ImmutableMap.copyOf(cxxPreprocessorInput.getIncludes().getNameToPathMap())));
resolver.addToIndex(compileRule);
objects.add(new BuildTargetSourcePath(compileRule.getBuildTarget()));
}
return objects.build();
}
@Test
public void testPickLayoutPartitionedWithGroup() throws Exception {
Comparator<ActualProperties> preference =
streamingExecutionPreference(
PreferredProperties.partitionedWithLocal(
ImmutableSet.of(symbol("a")), ImmutableList.of(grouped("a"))));
List<ActualProperties> input =
ImmutableList.<ActualProperties>builder()
.add(builder().global(streamPartitionedOn("a")).build())
.add(builder().global(singleStreamPartition()).build())
.add(
builder()
.global(arbitraryPartition())
.local(ImmutableList.of(grouped("a", "b")))
.build())
.add(builder().global(arbitraryPartition()).build())
.add(builder().global(hashDistributedOn("a")).build())
.add(
builder()
.global(singleStream())
.local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST)))
.build())
.add(
builder()
.global(singleStreamPartition())
.local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST)))
.build())
.add(builder().global(hashDistributedOn("a")).build())
.build();
List<ActualProperties> expected =
ImmutableList.<ActualProperties>builder()
.add(
builder()
.global(singleStream())
.local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST)))
.build())
.add(
builder()
.global(singleStreamPartition())
.local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST)))
.build())
.add(builder().global(streamPartitionedOn("a")).build())
.add(builder().global(singleStreamPartition()).build())
.add(builder().global(hashDistributedOn("a")).build())
.add(builder().global(hashDistributedOn("a")).build())
.add(
builder()
.global(arbitraryPartition())
.local(ImmutableList.of(grouped("a", "b")))
.build())
.add(builder().global(arbitraryPartition()).build())
.build();
Assert.assertEquals(stableSort(input, preference), expected);
}
@Override
public HiveOutputTableHandle beginCreateTable(
ConnectorSession session, ConnectorTableMetadata tableMetadata) {
verifyJvmTimeZone();
checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty");
HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(session, this.hiveStorageFormat);
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
// get the root directory for the database
SchemaTableName schemaTableName = tableMetadata.getTable();
String schemaName = schemaTableName.getSchemaName();
String tableName = schemaTableName.getTableName();
buildColumnInfo(tableMetadata, columnNames, columnTypes);
Path targetPath = getTargetPath(schemaName, tableName, schemaTableName);
if (!useTemporaryDirectory(targetPath)) {
return new HiveOutputTableHandle(
connectorId,
schemaName,
tableName,
columnNames.build(),
columnTypes.build(),
tableMetadata.getOwner(),
targetPath.toString(),
targetPath.toString(),
session,
hiveStorageFormat);
}
// use a per-user temporary directory to avoid permission problems
// TODO: this should use Hadoop UserGroupInformation
String temporaryPrefix = "/tmp/presto-" + StandardSystemProperty.USER_NAME.value();
// create a temporary directory on the same filesystem
Path temporaryRoot = new Path(targetPath, temporaryPrefix);
Path temporaryPath = new Path(temporaryRoot, randomUUID().toString());
createDirectories(temporaryPath);
return new HiveOutputTableHandle(
connectorId,
schemaName,
tableName,
columnNames.build(),
columnTypes.build(),
tableMetadata.getOwner(),
targetPath.toString(),
temporaryPath.toString(),
session,
hiveStorageFormat);
}
@Override
public Expression visitJoin(JoinNode node, Void context) {
Expression leftPredicate = node.getLeft().accept(this, context);
Expression rightPredicate = node.getRight().accept(this, context);
List<Expression> joinConjuncts = new ArrayList<>();
for (JoinNode.EquiJoinClause clause : node.getCriteria()) {
joinConjuncts.add(
new ComparisonExpression(
ComparisonExpression.Type.EQUAL,
new QualifiedNameReference(clause.getLeft().toQualifiedName()),
new QualifiedNameReference(clause.getRight().toQualifiedName())));
}
switch (node.getType()) {
case INNER:
case CROSS:
return combineConjuncts(
ImmutableList.<Expression>builder()
.add(leftPredicate)
.add(rightPredicate)
.addAll(joinConjuncts)
.build());
case LEFT:
return combineConjuncts(
ImmutableList.<Expression>builder()
.add(leftPredicate)
.addAll(
transform(
extractConjuncts(rightPredicate),
expressionOrNullSymbols(in(node.getRight().getOutputSymbols()))))
.addAll(
transform(
joinConjuncts,
expressionOrNullSymbols(in(node.getRight().getOutputSymbols()))))
.build());
case RIGHT:
return combineConjuncts(
ImmutableList.<Expression>builder()
.add(rightPredicate)
.addAll(
transform(
extractConjuncts(leftPredicate),
expressionOrNullSymbols(in(node.getLeft().getOutputSymbols()))))
.addAll(
transform(
joinConjuncts,
expressionOrNullSymbols(in(node.getLeft().getOutputSymbols()))))
.build());
default:
throw new UnsupportedOperationException("Unknown join type: " + node.getType());
}
}
@Override
protected RelationPlan visitUnnest(Unnest node, Void context) {
TupleDescriptor descriptor = analysis.getOutputDescriptor(node);
ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder();
for (Field field : descriptor.getVisibleFields()) {
Symbol symbol = symbolAllocator.newSymbol(field);
outputSymbolsBuilder.add(symbol);
}
List<Symbol> unnestedSymbols = outputSymbolsBuilder.build();
// If we got here, then we must be unnesting a constant, and not be in a join (where there could
// be column references)
ImmutableList.Builder<Symbol> argumentSymbols = ImmutableList.builder();
ImmutableList.Builder<Expression> values = ImmutableList.builder();
ImmutableMap.Builder<Symbol, List<Symbol>> unnestSymbols = ImmutableMap.builder();
Iterator<Symbol> unnestedSymbolsIterator = unnestedSymbols.iterator();
for (Expression expression : node.getExpressions()) {
Object constantValue =
evaluateConstantExpression(expression, analysis.getCoercions(), metadata, session);
Type type = analysis.getType(expression);
values.add(LiteralInterpreter.toExpression(constantValue, type));
Symbol inputSymbol = symbolAllocator.newSymbol(expression, type);
argumentSymbols.add(inputSymbol);
if (type instanceof ArrayType) {
unnestSymbols.put(inputSymbol, ImmutableList.of(unnestedSymbolsIterator.next()));
} else if (type instanceof MapType) {
unnestSymbols.put(
inputSymbol,
ImmutableList.of(unnestedSymbolsIterator.next(), unnestedSymbolsIterator.next()));
} else {
throw new IllegalArgumentException("Unsupported type for UNNEST: " + type);
}
}
Optional<Symbol> ordinalitySymbol =
node.isWithOrdinality() ? Optional.of(unnestedSymbolsIterator.next()) : Optional.empty();
checkState(
!unnestedSymbolsIterator.hasNext(),
"Not all output symbols were matched with input symbols");
ValuesNode valuesNode =
new ValuesNode(
idAllocator.getNextId(),
argumentSymbols.build(),
ImmutableList.<List<Expression>>of(values.build()));
UnnestNode unnestNode =
new UnnestNode(
idAllocator.getNextId(),
valuesNode,
ImmutableList.<Symbol>of(),
unnestSymbols.build(),
ordinalitySymbol);
return new RelationPlan(unnestNode, descriptor, unnestedSymbols, Optional.empty());
}
@Override
public void generateMethods(
ClassDefinition classDefinition,
CallSiteBinder callSiteBinder,
RowExpression filter,
List<RowExpression> projections) {
CachedInstanceBinder cachedInstanceBinder =
new CachedInstanceBinder(classDefinition, callSiteBinder);
ImmutableList.Builder<MethodDefinition> projectMethods = ImmutableList.builder();
ImmutableList.Builder<MethodDefinition> projectColumnarMethods = ImmutableList.builder();
ImmutableList.Builder<MethodDefinition> projectDictionaryMethods = ImmutableList.builder();
for (int i = 0; i < projections.size(); i++) {
MethodDefinition project =
generateProjectMethod(
classDefinition,
callSiteBinder,
cachedInstanceBinder,
"project_" + i,
projections.get(i));
MethodDefinition projectColumnar =
generateProjectColumnarMethod(
classDefinition, callSiteBinder, "projectColumnar_" + i, projections.get(i), project);
MethodDefinition projectDictionary =
generateProjectDictionaryMethod(
classDefinition,
"projectDictionary_" + i,
projections.get(i),
project,
projectColumnar);
projectMethods.add(project);
projectColumnarMethods.add(projectColumnar);
projectDictionaryMethods.add(projectDictionary);
}
List<MethodDefinition> projectMethodDefinitions = projectMethods.build();
List<MethodDefinition> projectColumnarMethodDefinitions = projectColumnarMethods.build();
List<MethodDefinition> projectDictionaryMethodDefinitions = projectDictionaryMethods.build();
generateProcessMethod(classDefinition, filter, projections, projectMethodDefinitions);
generateGetNonLazyPageMethod(classDefinition, filter, projections);
generateProcessColumnarMethod(classDefinition, projections, projectColumnarMethodDefinitions);
generateProcessColumnarDictionaryMethod(
classDefinition,
projections,
projectColumnarMethodDefinitions,
projectDictionaryMethodDefinitions);
generateFilterPageMethod(classDefinition, filter);
generateFilterMethod(classDefinition, callSiteBinder, cachedInstanceBinder, filter);
generateConstructor(classDefinition, cachedInstanceBinder, projections.size());
}
/** Returns a {@link ComposedCombineFn} with an additional {@link CombineFn}. */
public <InputT, OutputT> ComposedCombineFn<DataT> with(
SimpleFunction<DataT, InputT> extractInputFn,
CombineFn<InputT, ?, OutputT> combineFn,
TupleTag<OutputT> outputTag) {
checkUniqueness(outputTags, outputTag);
return new ComposedCombineFn<>(
ImmutableList.<SerializableFunction<DataT, ?>>builder()
.addAll(extractInputFns)
.add(extractInputFn)
.build(),
ImmutableList.<CombineFn<?, ?, ?>>builder().addAll(combineFns).add(combineFn).build(),
ImmutableList.<TupleTag<?>>builder().addAll(outputTags).add(outputTag).build());
}
public Operator createAlignmentOperator(
OperatorContext operatorContext,
List<ConnectorColumnHandle> columns,
RaptorSplit raptorSplit) {
ImmutableList.Builder<Type> types = ImmutableList.builder();
ImmutableList.Builder<Iterable<Block>> channels = ImmutableList.builder();
for (ConnectorColumnHandle column : columns) {
RaptorColumnHandle raptorColumnHandle = checkType(column, RaptorColumnHandle.class, "column");
types.add(raptorColumnHandle.getColumnType());
channels.add(storageManager.getBlocks(raptorSplit.getShardUuid(), column));
}
return new AlignmentOperator(operatorContext, types.build(), channels.build());
}
public ParquetPageSource(
ParquetReader parquetReader,
ParquetDataSource dataSource,
MessageType fileSchema,
MessageType requestedSchema,
long totalBytes,
Properties splitSchema,
List<HiveColumnHandle> columns,
TupleDomain<HiveColumnHandle> effectivePredicate,
TypeManager typeManager,
boolean useParquetColumnNames) {
checkArgument(totalBytes >= 0, "totalBytes is negative");
requireNonNull(splitSchema, "splitSchema is null");
requireNonNull(columns, "columns is null");
requireNonNull(effectivePredicate, "effectivePredicate is null");
this.parquetReader = parquetReader;
this.dataSource = dataSource;
this.requestedSchema = requestedSchema;
this.totalBytes = totalBytes;
int size = columns.size();
this.constantBlocks = new Block[size];
this.hiveColumnIndexes = new int[size];
ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
for (int columnIndex = 0; columnIndex < size; columnIndex++) {
HiveColumnHandle column = columns.get(columnIndex);
checkState(column.getColumnType() == REGULAR, "column type must be regular");
String name = column.getName();
Type type = typeManager.getType(column.getTypeSignature());
namesBuilder.add(name);
typesBuilder.add(type);
hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex();
if (getParquetType(column, fileSchema, useParquetColumnNames) == null) {
BlockBuilder blockBuilder =
type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH);
for (int i = 0; i < MAX_VECTOR_LENGTH; i++) {
blockBuilder.appendNull();
}
constantBlocks[columnIndex] = blockBuilder.build();
}
}
types = typesBuilder.build();
columnNames = namesBuilder.build();
}
public OperatorFactory createTableScanOperator(
Session session, int operatorId, String tableName, String... columnNames) {
checkArgument(session.getCatalog().isPresent(), "catalog not set");
checkArgument(session.getSchema().isPresent(), "schema not set");
// look up the table
QualifiedTableName qualifiedTableName =
new QualifiedTableName(session.getCatalog().get(), session.getSchema().get(), tableName);
TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName).orElse(null);
checkArgument(tableHandle != null, "Table %s does not exist", qualifiedTableName);
// lookup the columns
Map<String, ColumnHandle> allColumnHandles = metadata.getColumnHandles(session, tableHandle);
ImmutableList.Builder<ColumnHandle> columnHandlesBuilder = ImmutableList.builder();
ImmutableList.Builder<Type> columnTypesBuilder = ImmutableList.builder();
for (String columnName : columnNames) {
ColumnHandle columnHandle = allColumnHandles.get(columnName);
checkArgument(
columnHandle != null, "Table %s does not have a column %s", tableName, columnName);
columnHandlesBuilder.add(columnHandle);
ColumnMetadata columnMetadata =
metadata.getColumnMetadata(session, tableHandle, columnHandle);
columnTypesBuilder.add(columnMetadata.getType());
}
List<ColumnHandle> columnHandles = columnHandlesBuilder.build();
List<Type> columnTypes = columnTypesBuilder.build();
// get the split for this table
List<TableLayoutResult> layouts =
metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty());
Split split = getLocalQuerySplit(layouts.get(0).getLayout().getHandle());
return new OperatorFactory() {
@Override
public List<Type> getTypes() {
return columnTypes;
}
@Override
public Operator createOperator(DriverContext driverContext) {
OperatorContext operatorContext =
driverContext.addOperatorContext(operatorId, "BenchmarkSource");
ConnectorPageSource pageSource =
pageSourceManager.createPageSource(session, split, columnHandles);
return new PageSourceOperator(pageSource, columnTypes, operatorContext);
}
@Override
public void close() {}
};
}
IndexShardRoutingTable(
ShardId shardId, List<ShardRouting> shards, boolean primaryAllocatedPostApi) {
this.shardId = shardId;
this.shuffler = new RotationShardShuffler(ThreadLocalRandom.current().nextInt());
this.shards = ImmutableList.copyOf(shards);
this.primaryAllocatedPostApi = primaryAllocatedPostApi;
ShardRouting primary = null;
ImmutableList.Builder<ShardRouting> replicas = ImmutableList.builder();
ImmutableList.Builder<ShardRouting> activeShards = ImmutableList.builder();
ImmutableList.Builder<ShardRouting> assignedShards = ImmutableList.builder();
ImmutableList.Builder<ShardRouting> allInitializingShards = ImmutableList.builder();
boolean allShardsStarted = true;
for (ShardRouting shard : shards) {
if (shard.primary()) {
primary = shard;
} else {
replicas.add(shard);
}
if (shard.active()) {
activeShards.add(shard);
}
if (shard.initializing()) {
allInitializingShards.add(shard);
}
if (shard.relocating()) {
// create the target initializing shard routing on the node the shard is relocating to
allInitializingShards.add(shard.buildTargetRelocatingShard());
}
if (shard.assignedToNode()) {
assignedShards.add(shard);
}
if (shard.state() != ShardRoutingState.STARTED) {
allShardsStarted = false;
}
}
this.allShardsStarted = allShardsStarted;
this.primary = primary;
if (primary != null) {
this.primaryAsList = ImmutableList.of(primary);
} else {
this.primaryAsList = ImmutableList.of();
}
this.replicas = replicas.build();
this.activeShards = activeShards.build();
this.assignedShards = assignedShards.build();
this.allInitializingShards = allInitializingShards.build();
}
/**
* Creates a new attribute service using the given providers and user provided default attribute
* values.
*/
public AttributeService(
Iterable<? extends AttributeProvider> providers, Map<String, ?> userProvidedDefaults) {
ImmutableMap.Builder<String, AttributeProvider> byViewNameBuilder = ImmutableMap.builder();
ImmutableMap.Builder<Class<?>, AttributeProvider> byViewTypeBuilder = ImmutableMap.builder();
ImmutableMap.Builder<Class<?>, AttributeProvider> byAttributesTypeBuilder =
ImmutableMap.builder();
ImmutableList.Builder<FileAttribute<?>> defaultAttributesBuilder = ImmutableList.builder();
for (AttributeProvider provider : providers) {
byViewNameBuilder.put(provider.name(), provider);
byViewTypeBuilder.put(provider.viewType(), provider);
if (provider.attributesType() != null) {
byAttributesTypeBuilder.put(provider.attributesType(), provider);
}
for (Map.Entry<String, ?> entry : provider.defaultValues(userProvidedDefaults).entrySet()) {
defaultAttributesBuilder.add(new SimpleFileAttribute<>(entry.getKey(), entry.getValue()));
}
}
this.providersByName = byViewNameBuilder.build();
this.providersByViewType = byViewTypeBuilder.build();
this.providersByAttributesType = byAttributesTypeBuilder.build();
this.defaultValues = defaultAttributesBuilder.build();
}
private ImmutableList<Path> conditionallyCopy(ImmutableList<Path> roots) throws IOException {
final Builder<Path> builder = ImmutableList.builder();
for (Path root : roots) {
Preconditions.checkArgument(
root.startsWith(workingDirectory),
root + " must start with root " + workingDirectory + " from " + roots);
Preconditions.checkArgument(
!root.equals(workingDirectory),
"Cannot deduplicate root directory: " + root + " from " + roots);
if (!seen.containsKey(root)) {
seen.put(root, null);
final Path newRoot = out.resolve(workingDirectory.relativize(root));
Files.walkFileTree(
root,
ImmutableSet.of(FileVisitOption.FOLLOW_LINKS),
Integer.MAX_VALUE,
new ConditionalCopyVisitor(newRoot, root, seen, hashFunction));
builder.add(newRoot);
} else {
// Duplicated directories are ok -- multiple files from different libraries
// can reside in the same directory, but duplicate files should not be seen mulitple times.
}
}
return builder.build();
}
/**
* Returns a {@link ComposedKeyedCombineFnWithContext} with an additional {@link
* PerKeyCombineFn}.
*/
public <InputT, OutputT> ComposedKeyedCombineFnWithContext<DataT, K> with(
SimpleFunction<DataT, InputT> extractInputFn,
PerKeyCombineFn<K, InputT, ?, OutputT> perKeyCombineFn,
TupleTag<OutputT> outputTag) {
checkUniqueness(outputTags, outputTag);
return new ComposedKeyedCombineFnWithContext<>(
ImmutableList.<SerializableFunction<DataT, ?>>builder()
.addAll(extractInputFns)
.add(extractInputFn)
.build(),
ImmutableList.<KeyedCombineFnWithContext<K, ?, ?, ?>>builder()
.addAll(keyedCombineFns)
.add(toFnWithContext(perKeyCombineFn))
.build(),
ImmutableList.<TupleTag<?>>builder().addAll(outputTags).add(outputTag).build());
}
public static Optional<TypeSignature> getCommonSuperTypeSignature(
TypeSignature firstType, TypeSignature secondType) {
// Special handling for UnknownType is necessary because we forbid cast between types with
// different number of type parameters.
// Without this, cast from null to map<bigint, bigint> will not be allowed.
if (UnknownType.NAME.equals(firstType.getBase())) {
return Optional.of(secondType);
}
if (UnknownType.NAME.equals(secondType.getBase())) {
return Optional.of(firstType);
}
List<TypeSignatureParameter> firstTypeTypeParameters = firstType.getParameters();
List<TypeSignatureParameter> secondTypeTypeParameters = secondType.getParameters();
if (firstTypeTypeParameters.size() != secondTypeTypeParameters.size()) {
return Optional.empty();
}
Optional<String> commonSuperTypeBase =
getCommonSuperTypeBase(firstType.getBase(), secondType.getBase());
if (!commonSuperTypeBase.isPresent()) {
return Optional.empty();
}
ImmutableList.Builder<TypeSignatureParameter> typeParameters = ImmutableList.builder();
for (int i = 0; i < firstTypeTypeParameters.size(); i++) {
TypeSignatureParameter firstParameter = firstTypeTypeParameters.get(i);
TypeSignatureParameter secondParameter = secondTypeTypeParameters.get(i);
if (firstParameter.getKind() == secondParameter.getKind()
&& firstParameter.getKind() == ParameterKind.LONG_LITERAL) {
typeParameters.add(
TypeSignatureParameter.of(
Math.max(firstParameter.getLongLiteral(), secondParameter.getLongLiteral())));
} else if (isCovariantParameterPosition(commonSuperTypeBase.get(), i)) {
Optional<TypeSignature> firstParameterSignature =
firstParameter.getTypeSignatureOrNamedTypeSignature();
Optional<TypeSignature> secondParameterSignature =
secondParameter.getTypeSignatureOrNamedTypeSignature();
if (!firstParameterSignature.isPresent() || !secondParameterSignature.isPresent()) {
return Optional.empty();
}
Optional<TypeSignature> commonSuperType =
getCommonSuperTypeSignature(
firstParameterSignature.get(), secondParameterSignature.get());
if (!commonSuperType.isPresent()) {
return Optional.empty();
}
typeParameters.add(TypeSignatureParameter.of(commonSuperType.get()));
} else {
if (!firstParameter.equals(secondParameter)) {
return Optional.empty();
}
typeParameters.add(firstParameter);
}
}
return Optional.of(new TypeSignature(commonSuperTypeBase.get(), typeParameters.build()));
}
/**
* Parse a string delimited by comma into a list of object instances depending on
*
* <pre>itemParser</pre>
*
* .
*
* @param str String delimited by comma
* @param itemParser A function to transform a string to an object.
* @param <T> Type to be transformed from a string
* @return List of object instances
*/
static <T> List<T> parseList(String str, Function<String, T> itemParser) {
if (!str.contains(",")) { // if just one item
return ImmutableList.of(itemParser.apply(str));
}
final ImmutableList.Builder<T> fields = ImmutableList.builder();
final Stack<Character> stack = new Stack<>();
int paramStartIdx = 0;
for (int i = 0; i < str.length(); i++) {
char c = str.charAt(i);
if (c == '<' || c == '[' || c == '(') {
stack.push(c);
} else if (c == '>') {
Assert.assertCondition(stack.pop() == '<', "Bad signature: '%s'", str);
} else if (c == ']') {
Assert.assertCondition(stack.pop() == '[', "Bad signature: '%s'", str);
} else if (c == ')') {
Assert.assertCondition(stack.pop() == '(', "Bad signature: '%s'", str);
} else if (c == ',') {
if (stack.isEmpty()) { // ensure outermost type parameters
fields.add(itemParser.apply(str.substring(paramStartIdx, i)));
paramStartIdx = i + 1;
}
}
}
Assert.assertCondition(stack.empty(), "Bad signature: '%s'", str);
if (paramStartIdx < str.length()) {
fields.add(itemParser.apply(str.substring(paramStartIdx, str.length())));
}
return fields.build();
}
private StoredObject createCombinedObjectSmall(CombinedStoredObject combinedObject) {
ImmutableList.Builder<InputSupplier<InputStream>> builder = ImmutableList.builder();
List<URI> sourceParts =
Lists.transform(combinedObject.getSourceParts(), StoredObject.GET_LOCATION_FUNCTION);
for (URI sourcePart : sourceParts) {
builder.add(getInputSupplier(sourcePart));
}
InputSupplier<InputStream> source = ByteStreams.join(builder.build());
File tempFile = null;
try {
tempFile =
File.createTempFile(
S3StorageHelper.getS3FileName(combinedObject.getLocation()), ".small.s3.data");
Files.copy(source, tempFile);
StoredObject result = putObject(combinedObject.getLocation(), tempFile);
return result;
} catch (IOException e) {
throw Throwables.propagate(e);
} finally {
if (tempFile != null) {
tempFile.delete();
}
}
}
