@Override public SubPlanBuilder visitUnion(UnionNode node, Void context) { if (createSingleNodePlan) { ImmutableList.Builder<PlanNode> sourceBuilder = ImmutableList.builder(); for (PlanNode source : node.getSources()) { sourceBuilder.add(source.accept(this, context).getRoot()); } UnionNode unionNode = new UnionNode(node.getId(), sourceBuilder.build(), node.getSymbolMapping()); return createSingleNodePlan(unionNode); } else { ImmutableList.Builder<SubPlan> sourceBuilder = ImmutableList.builder(); ImmutableList.Builder<PlanFragmentId> fragmentIdBuilder = ImmutableList.builder(); for (int i = 0; i < node.getSources().size(); i++) { PlanNode subPlan = node.getSources().get(i); SubPlanBuilder current = subPlan.accept(this, context); current.setRoot( new SinkNode(idAllocator.getNextId(), current.getRoot(), node.sourceOutputLayout(i))); fragmentIdBuilder.add(current.getId()); sourceBuilder.add(current.build()); } ExchangeNode exchangeNode = new ExchangeNode( idAllocator.getNextId(), fragmentIdBuilder.build(), node.getOutputSymbols()); return createSingleNodePlan(exchangeNode).setChildren(sourceBuilder.build()); } }
@Override public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) { checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty"); SchemaTableName schemaTableName = tableMetadata.getTable(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); ImmutableList.Builder<String> columnNames = ImmutableList.builder(); ImmutableList.Builder<Type> columnTypes = ImmutableList.builder(); buildColumnInfo(tableMetadata, columnNames, columnTypes); ImmutableList.Builder<FieldSchema> partitionKeys = ImmutableList.builder(); ImmutableList.Builder<FieldSchema> columns = ImmutableList.builder(); List<String> names = columnNames.build(); List<String> typeNames = columnTypes .build() .stream() .map(HiveType::toHiveType) .map(HiveType::getHiveTypeName) .collect(toList()); for (int i = 0; i < names.size(); i++) { if (tableMetadata.getColumns().get(i).isPartitionKey()) { partitionKeys.add(new FieldSchema(names.get(i), typeNames.get(i), null)); } else { columns.add(new FieldSchema(names.get(i), typeNames.get(i), null)); } } Path targetPath = getTargetPath(schemaName, tableName, schemaTableName); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(session, this.hiveStorageFormat); SerDeInfo serdeInfo = new SerDeInfo(); serdeInfo.setName(tableName); serdeInfo.setSerializationLib(hiveStorageFormat.getSerDe()); StorageDescriptor sd = new StorageDescriptor(); sd.setLocation(targetPath.toString()); sd.setCols(columns.build()); sd.setSerdeInfo(serdeInfo); sd.setInputFormat(hiveStorageFormat.getInputFormat()); sd.setOutputFormat(hiveStorageFormat.getOutputFormat()); Table table = new Table(); table.setDbName(schemaName); table.setTableName(tableName); table.setOwner(tableMetadata.getOwner()); table.setTableType(TableType.MANAGED_TABLE.toString()); String tableComment = "Created by Presto"; table.setParameters(ImmutableMap.of("comment", tableComment)); table.setPartitionKeys(partitionKeys.build()); table.setSd(sd); metastore.createTable(table); }
@Override public PlanNode visitValues(ValuesNode node, RewriteContext<Set<Symbol>> context) { ImmutableList.Builder<Symbol> rewrittenOutputSymbolsBuilder = ImmutableList.builder(); ImmutableList.Builder<ImmutableList.Builder<Expression>> rowBuildersBuilder = ImmutableList.builder(); // Initialize builder for each row for (int i = 0; i < node.getRows().size(); i++) { rowBuildersBuilder.add(ImmutableList.builder()); } ImmutableList<ImmutableList.Builder<Expression>> rowBuilders = rowBuildersBuilder.build(); for (int i = 0; i < node.getOutputSymbols().size(); i++) { Symbol outputSymbol = node.getOutputSymbols().get(i); // If output symbol is used if (context.get().contains(outputSymbol)) { rewrittenOutputSymbolsBuilder.add(outputSymbol); // Add the value of the output symbol for each row for (int j = 0; j < node.getRows().size(); j++) { rowBuilders.get(j).add(node.getRows().get(j).get(i)); } } } List<List<Expression>> rewrittenRows = rowBuilders.stream().map((rowBuilder) -> rowBuilder.build()).collect(toImmutableList()); return new ValuesNode(node.getId(), rewrittenOutputSymbolsBuilder.build(), rewrittenRows); }
public JoinLeafPredicateInfo( SqlKind comparisonType, List<List<RexNode>> joinKeyExprs, List<Set<Integer>> projsJoinKeysInChildSchema, List<Set<Integer>> projsJoinKeysInJoinSchema) { this.comparisonType = comparisonType; ImmutableList.Builder<ImmutableList<RexNode>> joinKeyExprsBuilder = ImmutableList.builder(); for (int i = 0; i < joinKeyExprs.size(); i++) { joinKeyExprsBuilder.add(ImmutableList.copyOf(joinKeyExprs.get(i))); } this.joinKeyExprs = joinKeyExprsBuilder.build(); ImmutableList.Builder<ImmutableSet<Integer>> projsJoinKeysInChildSchemaBuilder = ImmutableList.builder(); for (int i = 0; i < projsJoinKeysInChildSchema.size(); i++) { projsJoinKeysInChildSchemaBuilder.add( ImmutableSet.copyOf(projsJoinKeysInChildSchema.get(i))); } this.projsJoinKeysInChildSchema = projsJoinKeysInChildSchemaBuilder.build(); ImmutableList.Builder<ImmutableSet<Integer>> projsJoinKeysInJoinSchemaBuilder = ImmutableList.builder(); for (int i = 0; i < projsJoinKeysInJoinSchema.size(); i++) { projsJoinKeysInJoinSchemaBuilder.add(ImmutableSet.copyOf(projsJoinKeysInJoinSchema.get(i))); } this.projsJoinKeysInJoinSchema = projsJoinKeysInJoinSchemaBuilder.build(); }
private static ImmutableList<Token> buildTokens(List<Tok> toks) { ImmutableList.Builder<Token> tokens = ImmutableList.builder(); int k = 0; int kN = toks.size(); while (k < kN) { // Remaining non-tokens before the token go here. ImmutableList.Builder<Tok> toksBefore = ImmutableList.builder(); while (!toks.get(k).isToken()) { toksBefore.add(toks.get(k++)); } Tok tok = toks.get(k++); // Non-tokens starting on the same line go here too. ImmutableList.Builder<Tok> toksAfter = ImmutableList.builder(); while (k < kN && !"\n".equals(toks.get(k).getText()) && !toks.get(k).isToken()) { Tok nonTokenAfter = toks.get(k++); toksAfter.add(nonTokenAfter); if (nonTokenAfter.getText().contains("\n")) { break; } } tokens.add(new Token(toksBefore.build(), tok, toksAfter.build())); } return tokens.build(); }
@Override public List<String> onTabComplete( CommandSender sebder, Command command, String label, String[] args) { if (args.length == 0) { return ImmutableList.of("module", "debug", "reload"); } if (args.length == 1) { Builder<String> list = ImmutableList.builder(); for (String string : Arrays.asList("module", "debug", "reload")) { if (string.startsWith(args[0])) { list.add(string); } } return list.build(); } if (args.length == 2) { Builder<String> list = ImmutableList.builder(); for (String string : Arrays.asList("unload", "load", "reload", "debug")) { if (string.startsWith(args[0])) { list.add(string); } } return list.build(); } return ImmutableList.of(); }
private void logEndpoints() { final StringBuilder stringBuilder = new StringBuilder(1024).append("\n\n"); final ImmutableList.Builder<Class<?>> builder = ImmutableList.builder(); for (Object o : config.getSingletons()) { if (o.getClass().isAnnotationPresent(Path.class)) { builder.add(o.getClass()); } } for (Class<?> klass : config.getClasses()) { if (klass.isAnnotationPresent(Path.class)) { builder.add(klass); } } for (Class<?> klass : builder.build()) { final String path = klass.getAnnotation(Path.class).value(); final ImmutableList.Builder<String> endpoints = ImmutableList.builder(); for (AnnotatedMethod method : annotatedMethods(klass)) { for (HttpMethod verb : method.getMetaMethodAnnotations(HttpMethod.class)) { endpoints.add( String.format(" %-7s %s (%s)", verb.value(), path, klass.getCanonicalName())); } } for (String line : Ordering.natural().sortedCopy(endpoints.build())) { stringBuilder.append(line).append('\n'); } } LOG.info(stringBuilder.toString()); }
private static <T> ImmutableList<ComponentAndSize<T>> resized( ImmutableList<ComponentSizeAndWeight<T>> sizes, int sizeOfAll) { int weightOfAll = weights(sizes); ImmutableList<ComponentAndSize<T>> weighted = weighted(sizes, sizeOfAll, weightOfAll); ImmutableList<ComponentAndSize<T>> toBig = toBig(weighted); if (!toBig.isEmpty()) { ImmutableList<ComponentSizeAndWeight<T>> left = withoutMatching(sizes, components(toBig)); ImmutableList<ComponentAndSize<T>> maxOfToBig = sizeFixedToLimit(toBig); int sizeOfToBig = sumOrMax(maxOfToBig); Builder<ComponentAndSize<T>> builder = ImmutableList.<ComponentAndSize<T>>builder(); builder.addAll(maxOfToBig); builder.addAll(resized(left, sizeOfAll - sizeOfToBig)); return builder.build(); } else { ImmutableList<ComponentAndSize<T>> toSmall = toSmall(weighted); if (!toSmall.isEmpty()) { ImmutableList<ComponentSizeAndWeight<T>> left = withoutMatching(sizes, components(toSmall)); ImmutableList<ComponentAndSize<T>> maxOfToSmall = sizeFixedToLimit(toSmall); int sizeOfToBig = sumOrMax(maxOfToSmall); Builder<ComponentAndSize<T>> builder = ImmutableList.<ComponentAndSize<T>>builder(); builder.addAll(maxOfToSmall); builder.addAll(resized(left, sizeOfAll - sizeOfToBig)); return builder.build(); } } return weighted; }
private List<CassandraPartition> getCassandraPartitions( CassandraTable table, TupleDomain<ColumnHandle> tupleDomain) { if (tupleDomain.isNone()) { return ImmutableList.of(); } Set<List<Comparable<?>>> partitionKeysSet = getPartitionKeysSet(table, tupleDomain); // empty filter means, all partitions if (partitionKeysSet.isEmpty()) { return schemaProvider.getAllPartitions(table); } ImmutableList.Builder<ListenableFuture<List<CassandraPartition>>> getPartitionResults = ImmutableList.builder(); for (List<Comparable<?>> partitionKeys : partitionKeysSet) { getPartitionResults.add( executor.submit(() -> schemaProvider.getPartitions(table, partitionKeys))); } ImmutableList.Builder<CassandraPartition> partitions = ImmutableList.builder(); for (ListenableFuture<List<CassandraPartition>> result : getPartitionResults.build()) { try { partitions.addAll(result.get()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw Throwables.propagate(e); } catch (ExecutionException e) { throw new PrestoException(EXTERNAL, "Error fetching cassandra partitions", e); } } return partitions.build(); }
static byte[] encode(RSAPrivateCrtKey key) { List<BigInteger> seq = ImmutableList.<BigInteger>builder() .add(BigInteger.valueOf(0)) // version .add(key.getModulus()) .add(key.getPublicExponent()) .add(key.getPrivateExponent()) .add(key.getPrimeP()) .add(key.getPrimeQ()) .add(key.getPrimeExponentP()) .add(key.getPrimeExponentQ()) .add(key.getCrtCoefficient()) .build(); int length = 0; for (BigInteger part : seq) { byte[] bytes = part.toByteArray(); length += 1 + calculateBodyLength(bytes.length) + bytes.length; } Builder<Byte> output = ImmutableList.<Byte>builder(); output.add((byte) (SEQUENCE | CONSTRUCTED)); writeLength(output, length); for (BigInteger part : seq) { byte[] bytes = part.toByteArray(); output.add((byte) TAG); writeLength(output, bytes.length); output.addAll(Bytes.asList(bytes)); } return Bytes.toArray(output.build()); }
private Builder() { created = ImmutableList.builder(); updated = ImmutableList.builder(); destroyed = ImmutableList.builder(); notCreated = ImmutableMap.builder(); notUpdated = ImmutableMap.builder(); notDestroyed = ImmutableMap.builder(); }
private void assertCompDir(Path compDir, Optional<String> failure) throws Exception { ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot().toPath()); CxxPlatform platform = DefaultCxxPlatforms.build(new CxxBuckConfig(new FakeBuckConfig())); // Build up the paths to various files the archive step will use. ImmutableList<String> compiler = platform.getCc().getCommandPrefix(new SourcePathResolver(new BuildRuleResolver())); Path output = filesystem.resolve(Paths.get("output.o")); Path relativeInput = Paths.get("input.c"); Path input = filesystem.resolve(relativeInput); filesystem.writeContentsToPath("int main() {}", relativeInput); ImmutableList.Builder<String> preprocessorCommand = ImmutableList.builder(); preprocessorCommand.addAll(compiler); ImmutableList.Builder<String> compilerCommand = ImmutableList.builder(); compilerCommand.addAll(compiler); compilerCommand.add("-g"); DebugPathSanitizer sanitizer = new DebugPathSanitizer(200, File.separatorChar, compDir, ImmutableBiMap.<Path, Path>of()); // Build an archive step. CxxPreprocessAndCompileStep step = new CxxPreprocessAndCompileStep( CxxPreprocessAndCompileStep.Operation.COMPILE_MUNGE_DEBUGINFO, output, relativeInput, CxxSource.Type.C, Optional.of(preprocessorCommand.build()), Optional.of(compilerCommand.build()), ImmutableMap.<Path, Path>of(), sanitizer); // Execute the archive step and verify it ran successfully. ExecutionContext executionContext = TestExecutionContext.newBuilder() .setProjectFilesystem(new ProjectFilesystem(tmp.getRoot().toPath())) .build(); TestConsole console = (TestConsole) executionContext.getConsole(); int exitCode = step.execute(executionContext); if (failure.isPresent()) { assertNotEquals("compile step succeeded", 0, exitCode); assertThat( console.getTextWrittenToStdErr(), console.getTextWrittenToStdErr(), Matchers.containsString(failure.get())); } else { assertEquals("compile step failed: " + console.getTextWrittenToStdErr(), 0, exitCode); // Verify that we find the expected compilation dir embedded in the file. String contents = new String(Files.readAllBytes(output)); assertThat(contents, Matchers.containsString(sanitizer.getCompilationDirectory())); } // Cleanup. Files.delete(input); Files.deleteIfExists(output); }
private <T> CachedRuleSource doExtract(final Class<T> source) { final ModelType<T> type = ModelType.of(source); DefaultMethodModelRuleExtractionContext context = new DefaultMethodModelRuleExtractionContext(type, this); // TODO - exceptions thrown here should point to some extensive documentation on the concept of // class rule sources StructSchema<T> schema = getSchema(source, context); if (schema == null) { throw new InvalidModelRuleDeclarationException(context.problems.format()); } // sort for determinism Set<Method> methods = new TreeSet<Method>(Ordering.usingToString()); methods.addAll(Arrays.asList(source.getDeclaredMethods())); ImmutableList.Builder<ModelProperty<?>> implicitInputs = ImmutableList.builder(); ModelProperty<?> target = null; for (ModelProperty<?> property : schema.getProperties()) { if (property.isAnnotationPresent(RuleTarget.class)) { target = property; } else if (property.isAnnotationPresent(RuleInput.class) && !(property.getSchema() instanceof ScalarValueSchema)) { implicitInputs.add(property); } for (WeaklyTypeReferencingMethod<?, ?> method : property.getAccessors()) { methods.remove(method.getMethod()); } } ImmutableList.Builder<ExtractedRuleDetails> rules = ImmutableList.builder(); for (Method method : methods) { MethodRuleDefinition<?, ?> ruleDefinition = DefaultMethodRuleDefinition.create(source, method); ExtractedModelRule rule = getMethodHandler(ruleDefinition, method, context); if (rule != null) { rules.add(new ExtractedRuleDetails(ruleDefinition, rule)); } } if (context.hasProblems()) { throw new InvalidModelRuleDeclarationException(context.problems.format()); } StructBindings<T> bindings = structBindingsStore.getBindings(schema); if (schema.getProperties().isEmpty()) { return new StatelessRuleSource( rules.build(), Modifier.isAbstract(source.getModifiers()) ? new AbstractRuleSourceFactory<T>(schema, bindings, proxyFactory) : new ConcreteRuleSourceFactory<T>(type)); } else { return new ParameterizedRuleSource( rules.build(), target, implicitInputs.build(), schema, bindings, proxyFactory); } }
private ImmutableList<SourcePath> generateCCompilation(ImmutableList<SourcePath> cInput) { ImmutableList.Builder<SourcePath> objects = ImmutableList.builder(); ImmutableList.Builder<String> cCompileFlags = ImmutableList.builder(); cCompileFlags.addAll(ocamlContext.getCCompileFlags()); cCompileFlags.addAll(ocamlContext.getCommonCFlags()); CxxPreprocessorInput cxxPreprocessorInput = ocamlContext.getCxxPreprocessorInput(); for (SourcePath cSrc : cInput) { String name = pathResolver.getAbsolutePath(cSrc).toFile().getName(); BuildTarget target = createCCompileBuildTarget(params.getBuildTarget(), name); BuildRuleParams cCompileParams = params.copyWithChanges( target, /* declaredDeps */ Suppliers.ofInstance( ImmutableSortedSet.<BuildRule>naturalOrder() // Depend on the rule that generates the sources and headers we're compiling. .addAll( pathResolver.filterBuildRuleInputs( ImmutableList.<SourcePath>builder() .add(cSrc) .addAll( cxxPreprocessorInput .getIncludes() .getNameToPathMap() .values()) .build())) // Also add in extra deps from the preprocessor input, such as the symlink // tree rules. .addAll( BuildRules.toBuildRulesFor( params.getBuildTarget(), resolver, cxxPreprocessorInput.getRules())) .addAll(params.getDeclaredDeps().get()) .build()), /* extraDeps */ params.getExtraDeps()); Path outputPath = ocamlContext.getCOutput(pathResolver.getRelativePath(cSrc)); OCamlCCompile compileRule = new OCamlCCompile( cCompileParams, pathResolver, new OCamlCCompileStep.Args( cCompiler.getEnvironment(pathResolver), cCompiler.getCommandPrefix(pathResolver), ocamlContext.getOcamlCompiler().get(), outputPath, cSrc, cCompileFlags.build(), ImmutableMap.copyOf(cxxPreprocessorInput.getIncludes().getNameToPathMap()))); resolver.addToIndex(compileRule); objects.add(new BuildTargetSourcePath(compileRule.getBuildTarget())); } return objects.build(); }
@Test public void testPickLayoutPartitionedWithGroup() throws Exception { Comparator<ActualProperties> preference = streamingExecutionPreference( PreferredProperties.partitionedWithLocal( ImmutableSet.of(symbol("a")), ImmutableList.of(grouped("a")))); List<ActualProperties> input = ImmutableList.<ActualProperties>builder() .add(builder().global(streamPartitionedOn("a")).build()) .add(builder().global(singleStreamPartition()).build()) .add( builder() .global(arbitraryPartition()) .local(ImmutableList.of(grouped("a", "b"))) .build()) .add(builder().global(arbitraryPartition()).build()) .add(builder().global(hashDistributedOn("a")).build()) .add( builder() .global(singleStream()) .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) .build()) .add( builder() .global(singleStreamPartition()) .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) .build()) .add(builder().global(hashDistributedOn("a")).build()) .build(); List<ActualProperties> expected = ImmutableList.<ActualProperties>builder() .add( builder() .global(singleStream()) .local(ImmutableList.of(constant("a"), sorted("b", ASC_NULLS_FIRST))) .build()) .add( builder() .global(singleStreamPartition()) .local(ImmutableList.of(sorted("a", ASC_NULLS_FIRST))) .build()) .add(builder().global(streamPartitionedOn("a")).build()) .add(builder().global(singleStreamPartition()).build()) .add(builder().global(hashDistributedOn("a")).build()) .add(builder().global(hashDistributedOn("a")).build()) .add( builder() .global(arbitraryPartition()) .local(ImmutableList.of(grouped("a", "b"))) .build()) .add(builder().global(arbitraryPartition()).build()) .build(); Assert.assertEquals(stableSort(input, preference), expected); }
@Override public HiveOutputTableHandle beginCreateTable( ConnectorSession session, ConnectorTableMetadata tableMetadata) { verifyJvmTimeZone(); checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty"); HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(session, this.hiveStorageFormat); ImmutableList.Builder<String> columnNames = ImmutableList.builder(); ImmutableList.Builder<Type> columnTypes = ImmutableList.builder(); // get the root directory for the database SchemaTableName schemaTableName = tableMetadata.getTable(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); buildColumnInfo(tableMetadata, columnNames, columnTypes); Path targetPath = getTargetPath(schemaName, tableName, schemaTableName); if (!useTemporaryDirectory(targetPath)) { return new HiveOutputTableHandle( connectorId, schemaName, tableName, columnNames.build(), columnTypes.build(), tableMetadata.getOwner(), targetPath.toString(), targetPath.toString(), session, hiveStorageFormat); } // use a per-user temporary directory to avoid permission problems // TODO: this should use Hadoop UserGroupInformation String temporaryPrefix = "/tmp/presto-" + StandardSystemProperty.USER_NAME.value(); // create a temporary directory on the same filesystem Path temporaryRoot = new Path(targetPath, temporaryPrefix); Path temporaryPath = new Path(temporaryRoot, randomUUID().toString()); createDirectories(temporaryPath); return new HiveOutputTableHandle( connectorId, schemaName, tableName, columnNames.build(), columnTypes.build(), tableMetadata.getOwner(), targetPath.toString(), temporaryPath.toString(), session, hiveStorageFormat); }
@Override public Expression visitJoin(JoinNode node, Void context) { Expression leftPredicate = node.getLeft().accept(this, context); Expression rightPredicate = node.getRight().accept(this, context); List<Expression> joinConjuncts = new ArrayList<>(); for (JoinNode.EquiJoinClause clause : node.getCriteria()) { joinConjuncts.add( new ComparisonExpression( ComparisonExpression.Type.EQUAL, new QualifiedNameReference(clause.getLeft().toQualifiedName()), new QualifiedNameReference(clause.getRight().toQualifiedName()))); } switch (node.getType()) { case INNER: case CROSS: return combineConjuncts( ImmutableList.<Expression>builder() .add(leftPredicate) .add(rightPredicate) .addAll(joinConjuncts) .build()); case LEFT: return combineConjuncts( ImmutableList.<Expression>builder() .add(leftPredicate) .addAll( transform( extractConjuncts(rightPredicate), expressionOrNullSymbols(in(node.getRight().getOutputSymbols())))) .addAll( transform( joinConjuncts, expressionOrNullSymbols(in(node.getRight().getOutputSymbols())))) .build()); case RIGHT: return combineConjuncts( ImmutableList.<Expression>builder() .add(rightPredicate) .addAll( transform( extractConjuncts(leftPredicate), expressionOrNullSymbols(in(node.getLeft().getOutputSymbols())))) .addAll( transform( joinConjuncts, expressionOrNullSymbols(in(node.getLeft().getOutputSymbols())))) .build()); default: throw new UnsupportedOperationException("Unknown join type: " + node.getType()); } }
@Override protected RelationPlan visitUnnest(Unnest node, Void context) { TupleDescriptor descriptor = analysis.getOutputDescriptor(node); ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder(); for (Field field : descriptor.getVisibleFields()) { Symbol symbol = symbolAllocator.newSymbol(field); outputSymbolsBuilder.add(symbol); } List<Symbol> unnestedSymbols = outputSymbolsBuilder.build(); // If we got here, then we must be unnesting a constant, and not be in a join (where there could // be column references) ImmutableList.Builder<Symbol> argumentSymbols = ImmutableList.builder(); ImmutableList.Builder<Expression> values = ImmutableList.builder(); ImmutableMap.Builder<Symbol, List<Symbol>> unnestSymbols = ImmutableMap.builder(); Iterator<Symbol> unnestedSymbolsIterator = unnestedSymbols.iterator(); for (Expression expression : node.getExpressions()) { Object constantValue = evaluateConstantExpression(expression, analysis.getCoercions(), metadata, session); Type type = analysis.getType(expression); values.add(LiteralInterpreter.toExpression(constantValue, type)); Symbol inputSymbol = symbolAllocator.newSymbol(expression, type); argumentSymbols.add(inputSymbol); if (type instanceof ArrayType) { unnestSymbols.put(inputSymbol, ImmutableList.of(unnestedSymbolsIterator.next())); } else if (type instanceof MapType) { unnestSymbols.put( inputSymbol, ImmutableList.of(unnestedSymbolsIterator.next(), unnestedSymbolsIterator.next())); } else { throw new IllegalArgumentException("Unsupported type for UNNEST: " + type); } } Optional<Symbol> ordinalitySymbol = node.isWithOrdinality() ? Optional.of(unnestedSymbolsIterator.next()) : Optional.empty(); checkState( !unnestedSymbolsIterator.hasNext(), "Not all output symbols were matched with input symbols"); ValuesNode valuesNode = new ValuesNode( idAllocator.getNextId(), argumentSymbols.build(), ImmutableList.<List<Expression>>of(values.build())); UnnestNode unnestNode = new UnnestNode( idAllocator.getNextId(), valuesNode, ImmutableList.<Symbol>of(), unnestSymbols.build(), ordinalitySymbol); return new RelationPlan(unnestNode, descriptor, unnestedSymbols, Optional.empty()); }
@Override public void generateMethods( ClassDefinition classDefinition, CallSiteBinder callSiteBinder, RowExpression filter, List<RowExpression> projections) { CachedInstanceBinder cachedInstanceBinder = new CachedInstanceBinder(classDefinition, callSiteBinder); ImmutableList.Builder<MethodDefinition> projectMethods = ImmutableList.builder(); ImmutableList.Builder<MethodDefinition> projectColumnarMethods = ImmutableList.builder(); ImmutableList.Builder<MethodDefinition> projectDictionaryMethods = ImmutableList.builder(); for (int i = 0; i < projections.size(); i++) { MethodDefinition project = generateProjectMethod( classDefinition, callSiteBinder, cachedInstanceBinder, "project_" + i, projections.get(i)); MethodDefinition projectColumnar = generateProjectColumnarMethod( classDefinition, callSiteBinder, "projectColumnar_" + i, projections.get(i), project); MethodDefinition projectDictionary = generateProjectDictionaryMethod( classDefinition, "projectDictionary_" + i, projections.get(i), project, projectColumnar); projectMethods.add(project); projectColumnarMethods.add(projectColumnar); projectDictionaryMethods.add(projectDictionary); } List<MethodDefinition> projectMethodDefinitions = projectMethods.build(); List<MethodDefinition> projectColumnarMethodDefinitions = projectColumnarMethods.build(); List<MethodDefinition> projectDictionaryMethodDefinitions = projectDictionaryMethods.build(); generateProcessMethod(classDefinition, filter, projections, projectMethodDefinitions); generateGetNonLazyPageMethod(classDefinition, filter, projections); generateProcessColumnarMethod(classDefinition, projections, projectColumnarMethodDefinitions); generateProcessColumnarDictionaryMethod( classDefinition, projections, projectColumnarMethodDefinitions, projectDictionaryMethodDefinitions); generateFilterPageMethod(classDefinition, filter); generateFilterMethod(classDefinition, callSiteBinder, cachedInstanceBinder, filter); generateConstructor(classDefinition, cachedInstanceBinder, projections.size()); }
/** Returns a {@link ComposedCombineFn} with an additional {@link CombineFn}. */ public <InputT, OutputT> ComposedCombineFn<DataT> with( SimpleFunction<DataT, InputT> extractInputFn, CombineFn<InputT, ?, OutputT> combineFn, TupleTag<OutputT> outputTag) { checkUniqueness(outputTags, outputTag); return new ComposedCombineFn<>( ImmutableList.<SerializableFunction<DataT, ?>>builder() .addAll(extractInputFns) .add(extractInputFn) .build(), ImmutableList.<CombineFn<?, ?, ?>>builder().addAll(combineFns).add(combineFn).build(), ImmutableList.<TupleTag<?>>builder().addAll(outputTags).add(outputTag).build()); }
public Operator createAlignmentOperator( OperatorContext operatorContext, List<ConnectorColumnHandle> columns, RaptorSplit raptorSplit) { ImmutableList.Builder<Type> types = ImmutableList.builder(); ImmutableList.Builder<Iterable<Block>> channels = ImmutableList.builder(); for (ConnectorColumnHandle column : columns) { RaptorColumnHandle raptorColumnHandle = checkType(column, RaptorColumnHandle.class, "column"); types.add(raptorColumnHandle.getColumnType()); channels.add(storageManager.getBlocks(raptorSplit.getShardUuid(), column)); } return new AlignmentOperator(operatorContext, types.build(), channels.build()); }
public ParquetPageSource( ParquetReader parquetReader, ParquetDataSource dataSource, MessageType fileSchema, MessageType requestedSchema, long totalBytes, Properties splitSchema, List<HiveColumnHandle> columns, TupleDomain<HiveColumnHandle> effectivePredicate, TypeManager typeManager, boolean useParquetColumnNames) { checkArgument(totalBytes >= 0, "totalBytes is negative"); requireNonNull(splitSchema, "splitSchema is null"); requireNonNull(columns, "columns is null"); requireNonNull(effectivePredicate, "effectivePredicate is null"); this.parquetReader = parquetReader; this.dataSource = dataSource; this.requestedSchema = requestedSchema; this.totalBytes = totalBytes; int size = columns.size(); this.constantBlocks = new Block[size]; this.hiveColumnIndexes = new int[size]; ImmutableList.Builder<String> namesBuilder = ImmutableList.builder(); ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder(); for (int columnIndex = 0; columnIndex < size; columnIndex++) { HiveColumnHandle column = columns.get(columnIndex); checkState(column.getColumnType() == REGULAR, "column type must be regular"); String name = column.getName(); Type type = typeManager.getType(column.getTypeSignature()); namesBuilder.add(name); typesBuilder.add(type); hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex(); if (getParquetType(column, fileSchema, useParquetColumnNames) == null) { BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH); for (int i = 0; i < MAX_VECTOR_LENGTH; i++) { blockBuilder.appendNull(); } constantBlocks[columnIndex] = blockBuilder.build(); } } types = typesBuilder.build(); columnNames = namesBuilder.build(); }
public OperatorFactory createTableScanOperator( Session session, int operatorId, String tableName, String... columnNames) { checkArgument(session.getCatalog().isPresent(), "catalog not set"); checkArgument(session.getSchema().isPresent(), "schema not set"); // look up the table QualifiedTableName qualifiedTableName = new QualifiedTableName(session.getCatalog().get(), session.getSchema().get(), tableName); TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName).orElse(null); checkArgument(tableHandle != null, "Table %s does not exist", qualifiedTableName); // lookup the columns Map<String, ColumnHandle> allColumnHandles = metadata.getColumnHandles(session, tableHandle); ImmutableList.Builder<ColumnHandle> columnHandlesBuilder = ImmutableList.builder(); ImmutableList.Builder<Type> columnTypesBuilder = ImmutableList.builder(); for (String columnName : columnNames) { ColumnHandle columnHandle = allColumnHandles.get(columnName); checkArgument( columnHandle != null, "Table %s does not have a column %s", tableName, columnName); columnHandlesBuilder.add(columnHandle); ColumnMetadata columnMetadata = metadata.getColumnMetadata(session, tableHandle, columnHandle); columnTypesBuilder.add(columnMetadata.getType()); } List<ColumnHandle> columnHandles = columnHandlesBuilder.build(); List<Type> columnTypes = columnTypesBuilder.build(); // get the split for this table List<TableLayoutResult> layouts = metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty()); Split split = getLocalQuerySplit(layouts.get(0).getLayout().getHandle()); return new OperatorFactory() { @Override public List<Type> getTypes() { return columnTypes; } @Override public Operator createOperator(DriverContext driverContext) { OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, "BenchmarkSource"); ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, columnHandles); return new PageSourceOperator(pageSource, columnTypes, operatorContext); } @Override public void close() {} }; }
IndexShardRoutingTable( ShardId shardId, List<ShardRouting> shards, boolean primaryAllocatedPostApi) { this.shardId = shardId; this.shuffler = new RotationShardShuffler(ThreadLocalRandom.current().nextInt()); this.shards = ImmutableList.copyOf(shards); this.primaryAllocatedPostApi = primaryAllocatedPostApi; ShardRouting primary = null; ImmutableList.Builder<ShardRouting> replicas = ImmutableList.builder(); ImmutableList.Builder<ShardRouting> activeShards = ImmutableList.builder(); ImmutableList.Builder<ShardRouting> assignedShards = ImmutableList.builder(); ImmutableList.Builder<ShardRouting> allInitializingShards = ImmutableList.builder(); boolean allShardsStarted = true; for (ShardRouting shard : shards) { if (shard.primary()) { primary = shard; } else { replicas.add(shard); } if (shard.active()) { activeShards.add(shard); } if (shard.initializing()) { allInitializingShards.add(shard); } if (shard.relocating()) { // create the target initializing shard routing on the node the shard is relocating to allInitializingShards.add(shard.buildTargetRelocatingShard()); } if (shard.assignedToNode()) { assignedShards.add(shard); } if (shard.state() != ShardRoutingState.STARTED) { allShardsStarted = false; } } this.allShardsStarted = allShardsStarted; this.primary = primary; if (primary != null) { this.primaryAsList = ImmutableList.of(primary); } else { this.primaryAsList = ImmutableList.of(); } this.replicas = replicas.build(); this.activeShards = activeShards.build(); this.assignedShards = assignedShards.build(); this.allInitializingShards = allInitializingShards.build(); }
/** * Creates a new attribute service using the given providers and user provided default attribute * values. */ public AttributeService( Iterable<? extends AttributeProvider> providers, Map<String, ?> userProvidedDefaults) { ImmutableMap.Builder<String, AttributeProvider> byViewNameBuilder = ImmutableMap.builder(); ImmutableMap.Builder<Class<?>, AttributeProvider> byViewTypeBuilder = ImmutableMap.builder(); ImmutableMap.Builder<Class<?>, AttributeProvider> byAttributesTypeBuilder = ImmutableMap.builder(); ImmutableList.Builder<FileAttribute<?>> defaultAttributesBuilder = ImmutableList.builder(); for (AttributeProvider provider : providers) { byViewNameBuilder.put(provider.name(), provider); byViewTypeBuilder.put(provider.viewType(), provider); if (provider.attributesType() != null) { byAttributesTypeBuilder.put(provider.attributesType(), provider); } for (Map.Entry<String, ?> entry : provider.defaultValues(userProvidedDefaults).entrySet()) { defaultAttributesBuilder.add(new SimpleFileAttribute<>(entry.getKey(), entry.getValue())); } } this.providersByName = byViewNameBuilder.build(); this.providersByViewType = byViewTypeBuilder.build(); this.providersByAttributesType = byAttributesTypeBuilder.build(); this.defaultValues = defaultAttributesBuilder.build(); }
private ImmutableList<Path> conditionallyCopy(ImmutableList<Path> roots) throws IOException { final Builder<Path> builder = ImmutableList.builder(); for (Path root : roots) { Preconditions.checkArgument( root.startsWith(workingDirectory), root + " must start with root " + workingDirectory + " from " + roots); Preconditions.checkArgument( !root.equals(workingDirectory), "Cannot deduplicate root directory: " + root + " from " + roots); if (!seen.containsKey(root)) { seen.put(root, null); final Path newRoot = out.resolve(workingDirectory.relativize(root)); Files.walkFileTree( root, ImmutableSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new ConditionalCopyVisitor(newRoot, root, seen, hashFunction)); builder.add(newRoot); } else { // Duplicated directories are ok -- multiple files from different libraries // can reside in the same directory, but duplicate files should not be seen mulitple times. } } return builder.build(); }
/** * Returns a {@link ComposedKeyedCombineFnWithContext} with an additional {@link * PerKeyCombineFn}. */ public <InputT, OutputT> ComposedKeyedCombineFnWithContext<DataT, K> with( SimpleFunction<DataT, InputT> extractInputFn, PerKeyCombineFn<K, InputT, ?, OutputT> perKeyCombineFn, TupleTag<OutputT> outputTag) { checkUniqueness(outputTags, outputTag); return new ComposedKeyedCombineFnWithContext<>( ImmutableList.<SerializableFunction<DataT, ?>>builder() .addAll(extractInputFns) .add(extractInputFn) .build(), ImmutableList.<KeyedCombineFnWithContext<K, ?, ?, ?>>builder() .addAll(keyedCombineFns) .add(toFnWithContext(perKeyCombineFn)) .build(), ImmutableList.<TupleTag<?>>builder().addAll(outputTags).add(outputTag).build()); }
public static Optional<TypeSignature> getCommonSuperTypeSignature( TypeSignature firstType, TypeSignature secondType) { // Special handling for UnknownType is necessary because we forbid cast between types with // different number of type parameters. // Without this, cast from null to map<bigint, bigint> will not be allowed. if (UnknownType.NAME.equals(firstType.getBase())) { return Optional.of(secondType); } if (UnknownType.NAME.equals(secondType.getBase())) { return Optional.of(firstType); } List<TypeSignatureParameter> firstTypeTypeParameters = firstType.getParameters(); List<TypeSignatureParameter> secondTypeTypeParameters = secondType.getParameters(); if (firstTypeTypeParameters.size() != secondTypeTypeParameters.size()) { return Optional.empty(); } Optional<String> commonSuperTypeBase = getCommonSuperTypeBase(firstType.getBase(), secondType.getBase()); if (!commonSuperTypeBase.isPresent()) { return Optional.empty(); } ImmutableList.Builder<TypeSignatureParameter> typeParameters = ImmutableList.builder(); for (int i = 0; i < firstTypeTypeParameters.size(); i++) { TypeSignatureParameter firstParameter = firstTypeTypeParameters.get(i); TypeSignatureParameter secondParameter = secondTypeTypeParameters.get(i); if (firstParameter.getKind() == secondParameter.getKind() && firstParameter.getKind() == ParameterKind.LONG_LITERAL) { typeParameters.add( TypeSignatureParameter.of( Math.max(firstParameter.getLongLiteral(), secondParameter.getLongLiteral()))); } else if (isCovariantParameterPosition(commonSuperTypeBase.get(), i)) { Optional<TypeSignature> firstParameterSignature = firstParameter.getTypeSignatureOrNamedTypeSignature(); Optional<TypeSignature> secondParameterSignature = secondParameter.getTypeSignatureOrNamedTypeSignature(); if (!firstParameterSignature.isPresent() || !secondParameterSignature.isPresent()) { return Optional.empty(); } Optional<TypeSignature> commonSuperType = getCommonSuperTypeSignature( firstParameterSignature.get(), secondParameterSignature.get()); if (!commonSuperType.isPresent()) { return Optional.empty(); } typeParameters.add(TypeSignatureParameter.of(commonSuperType.get())); } else { if (!firstParameter.equals(secondParameter)) { return Optional.empty(); } typeParameters.add(firstParameter); } } return Optional.of(new TypeSignature(commonSuperTypeBase.get(), typeParameters.build())); }
/** * Parse a string delimited by comma into a list of object instances depending on * * <pre>itemParser</pre> * * . * * @param str String delimited by comma * @param itemParser A function to transform a string to an object. * @param <T> Type to be transformed from a string * @return List of object instances */ static <T> List<T> parseList(String str, Function<String, T> itemParser) { if (!str.contains(",")) { // if just one item return ImmutableList.of(itemParser.apply(str)); } final ImmutableList.Builder<T> fields = ImmutableList.builder(); final Stack<Character> stack = new Stack<>(); int paramStartIdx = 0; for (int i = 0; i < str.length(); i++) { char c = str.charAt(i); if (c == '<' || c == '[' || c == '(') { stack.push(c); } else if (c == '>') { Assert.assertCondition(stack.pop() == '<', "Bad signature: '%s'", str); } else if (c == ']') { Assert.assertCondition(stack.pop() == '[', "Bad signature: '%s'", str); } else if (c == ')') { Assert.assertCondition(stack.pop() == '(', "Bad signature: '%s'", str); } else if (c == ',') { if (stack.isEmpty()) { // ensure outermost type parameters fields.add(itemParser.apply(str.substring(paramStartIdx, i))); paramStartIdx = i + 1; } } } Assert.assertCondition(stack.empty(), "Bad signature: '%s'", str); if (paramStartIdx < str.length()) { fields.add(itemParser.apply(str.substring(paramStartIdx, str.length()))); } return fields.build(); }
private StoredObject createCombinedObjectSmall(CombinedStoredObject combinedObject) { ImmutableList.Builder<InputSupplier<InputStream>> builder = ImmutableList.builder(); List<URI> sourceParts = Lists.transform(combinedObject.getSourceParts(), StoredObject.GET_LOCATION_FUNCTION); for (URI sourcePart : sourceParts) { builder.add(getInputSupplier(sourcePart)); } InputSupplier<InputStream> source = ByteStreams.join(builder.build()); File tempFile = null; try { tempFile = File.createTempFile( S3StorageHelper.getS3FileName(combinedObject.getLocation()), ".small.s3.data"); Files.copy(source, tempFile); StoredObject result = putObject(combinedObject.getLocation(), tempFile); return result; } catch (IOException e) { throw Throwables.propagate(e); } finally { if (tempFile != null) { tempFile.delete(); } } }