@Override
 public SubPlanBuilder visitUnion(UnionNode node, Void context) {
   if (createSingleNodePlan) {
     ImmutableList.Builder<PlanNode> sourceBuilder = ImmutableList.builder();
     for (PlanNode source : node.getSources()) {
       sourceBuilder.add(source.accept(this, context).getRoot());
     }
     UnionNode unionNode =
         new UnionNode(node.getId(), sourceBuilder.build(), node.getSymbolMapping());
     return createSingleNodePlan(unionNode);
   } else {
     ImmutableList.Builder<SubPlan> sourceBuilder = ImmutableList.builder();
     ImmutableList.Builder<PlanFragmentId> fragmentIdBuilder = ImmutableList.builder();
     for (int i = 0; i < node.getSources().size(); i++) {
       PlanNode subPlan = node.getSources().get(i);
       SubPlanBuilder current = subPlan.accept(this, context);
       current.setRoot(
           new SinkNode(idAllocator.getNextId(), current.getRoot(), node.sourceOutputLayout(i)));
       fragmentIdBuilder.add(current.getId());
       sourceBuilder.add(current.build());
     }
     ExchangeNode exchangeNode =
         new ExchangeNode(
             idAllocator.getNextId(), fragmentIdBuilder.build(), node.getOutputSymbols());
     return createSingleNodePlan(exchangeNode).setChildren(sourceBuilder.build());
   }
 }
 private List<KnapsackState> get(String name) throws IOException {
   ImmutableList.Builder<KnapsackState> builder = ImmutableList.builder();
   try {
     logger.debug("get knapsack states: {}", name);
     final Client client = injector.getInstance(Client.class);
     createIndexIfNotExist(client);
     GetResponse getResponse =
         client.prepareGet(INDEX_NAME, MAPPING_NAME, name).execute().actionGet();
     if (!getResponse.isExists()) {
       return builder.build();
     }
     XContentParser parser = xContent(JSON).createParser(getResponse.getSourceAsBytes());
     while (parser.nextToken() != START_ARRAY) {
       // forward
     }
     while (parser.nextToken() != END_ARRAY) {
       KnapsackState state = new KnapsackState();
       builder.add(state.fromXContent(parser));
     }
     return builder.build();
   } catch (Throwable t) {
     logger.error("get settings failed", t);
     return null;
   }
 }
  @Override
  public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) {
    checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty");

    SchemaTableName schemaTableName = tableMetadata.getTable();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();

    ImmutableList.Builder<String> columnNames = ImmutableList.builder();
    ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();

    buildColumnInfo(tableMetadata, columnNames, columnTypes);

    ImmutableList.Builder<FieldSchema> partitionKeys = ImmutableList.builder();
    ImmutableList.Builder<FieldSchema> columns = ImmutableList.builder();

    List<String> names = columnNames.build();
    List<String> typeNames =
        columnTypes
            .build()
            .stream()
            .map(HiveType::toHiveType)
            .map(HiveType::getHiveTypeName)
            .collect(toList());

    for (int i = 0; i < names.size(); i++) {
      if (tableMetadata.getColumns().get(i).isPartitionKey()) {
        partitionKeys.add(new FieldSchema(names.get(i), typeNames.get(i), null));
      } else {
        columns.add(new FieldSchema(names.get(i), typeNames.get(i), null));
      }
    }

    Path targetPath = getTargetPath(schemaName, tableName, schemaTableName);

    HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(session, this.hiveStorageFormat);
    SerDeInfo serdeInfo = new SerDeInfo();
    serdeInfo.setName(tableName);
    serdeInfo.setSerializationLib(hiveStorageFormat.getSerDe());

    StorageDescriptor sd = new StorageDescriptor();
    sd.setLocation(targetPath.toString());

    sd.setCols(columns.build());
    sd.setSerdeInfo(serdeInfo);
    sd.setInputFormat(hiveStorageFormat.getInputFormat());
    sd.setOutputFormat(hiveStorageFormat.getOutputFormat());

    Table table = new Table();
    table.setDbName(schemaName);
    table.setTableName(tableName);
    table.setOwner(tableMetadata.getOwner());
    table.setTableType(TableType.MANAGED_TABLE.toString());
    String tableComment = "Created by Presto";
    table.setParameters(ImmutableMap.of("comment", tableComment));
    table.setPartitionKeys(partitionKeys.build());
    table.setSd(sd);

    metastore.createTable(table);
  }
Exemple #4
0
 private ImmutableList<ColumnIdent> getPrimaryKey() {
   Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
   if (metaMap != null) {
     ImmutableList.Builder<ColumnIdent> builder = ImmutableList.builder();
     Object pKeys = metaMap.get("primary_keys");
     if (pKeys != null) {
       if (pKeys instanceof String) {
         builder.add(ColumnIdent.fromPath((String) pKeys));
         return builder.build();
       } else if (pKeys instanceof Collection) {
         Collection keys = (Collection) pKeys;
         if (!keys.isEmpty()) {
           for (Object pkey : keys) {
             builder.add(ColumnIdent.fromPath(pkey.toString()));
           }
           return builder.build();
         }
       }
     }
   }
   if (getCustomRoutingCol() == null && partitionedByList.isEmpty()) {
     hasAutoGeneratedPrimaryKey = true;
     return ImmutableList.of(ID_IDENT);
   }
   return ImmutableList.of();
 }
  private static ImmutableList<Token> buildTokens(List<Tok> toks) {
    ImmutableList.Builder<Token> tokens = ImmutableList.builder();
    int k = 0;
    int kN = toks.size();

    while (k < kN) {
      // Remaining non-tokens before the token go here.
      ImmutableList.Builder<Tok> toksBefore = ImmutableList.builder();

      while (!toks.get(k).isToken()) {
        toksBefore.add(toks.get(k++));
      }
      Tok tok = toks.get(k++);

      // Non-tokens starting on the same line go here too.
      ImmutableList.Builder<Tok> toksAfter = ImmutableList.builder();
      while (k < kN && !"\n".equals(toks.get(k).getText()) && !toks.get(k).isToken()) {
        Tok nonTokenAfter = toks.get(k++);
        toksAfter.add(nonTokenAfter);
        if (nonTokenAfter.getText().contains("\n")) {
          break;
        }
      }
      tokens.add(new Token(toksBefore.build(), tok, toksAfter.build()));
    }
    return tokens.build();
  }
  private <T> CachedRuleSource doExtract(final Class<T> source) {
    final ModelType<T> type = ModelType.of(source);
    DefaultMethodModelRuleExtractionContext context =
        new DefaultMethodModelRuleExtractionContext(type, this);

    // TODO - exceptions thrown here should point to some extensive documentation on the concept of
    // class rule sources

    StructSchema<T> schema = getSchema(source, context);
    if (schema == null) {
      throw new InvalidModelRuleDeclarationException(context.problems.format());
    }

    // sort for determinism
    Set<Method> methods = new TreeSet<Method>(Ordering.usingToString());
    methods.addAll(Arrays.asList(source.getDeclaredMethods()));

    ImmutableList.Builder<ModelProperty<?>> implicitInputs = ImmutableList.builder();
    ModelProperty<?> target = null;
    for (ModelProperty<?> property : schema.getProperties()) {
      if (property.isAnnotationPresent(RuleTarget.class)) {
        target = property;
      } else if (property.isAnnotationPresent(RuleInput.class)
          && !(property.getSchema() instanceof ScalarValueSchema)) {
        implicitInputs.add(property);
      }
      for (WeaklyTypeReferencingMethod<?, ?> method : property.getAccessors()) {
        methods.remove(method.getMethod());
      }
    }

    ImmutableList.Builder<ExtractedRuleDetails> rules = ImmutableList.builder();
    for (Method method : methods) {
      MethodRuleDefinition<?, ?> ruleDefinition =
          DefaultMethodRuleDefinition.create(source, method);
      ExtractedModelRule rule = getMethodHandler(ruleDefinition, method, context);
      if (rule != null) {
        rules.add(new ExtractedRuleDetails(ruleDefinition, rule));
      }
    }

    if (context.hasProblems()) {
      throw new InvalidModelRuleDeclarationException(context.problems.format());
    }

    StructBindings<T> bindings = structBindingsStore.getBindings(schema);

    if (schema.getProperties().isEmpty()) {
      return new StatelessRuleSource(
          rules.build(),
          Modifier.isAbstract(source.getModifiers())
              ? new AbstractRuleSourceFactory<T>(schema, bindings, proxyFactory)
              : new ConcreteRuleSourceFactory<T>(type));
    } else {
      return new ParameterizedRuleSource(
          rules.build(), target, implicitInputs.build(), schema, bindings, proxyFactory);
    }
  }
  private void assertCompDir(Path compDir, Optional<String> failure) throws Exception {
    ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot().toPath());
    CxxPlatform platform = DefaultCxxPlatforms.build(new CxxBuckConfig(new FakeBuckConfig()));

    // Build up the paths to various files the archive step will use.
    ImmutableList<String> compiler =
        platform.getCc().getCommandPrefix(new SourcePathResolver(new BuildRuleResolver()));
    Path output = filesystem.resolve(Paths.get("output.o"));
    Path relativeInput = Paths.get("input.c");
    Path input = filesystem.resolve(relativeInput);
    filesystem.writeContentsToPath("int main() {}", relativeInput);

    ImmutableList.Builder<String> preprocessorCommand = ImmutableList.builder();
    preprocessorCommand.addAll(compiler);

    ImmutableList.Builder<String> compilerCommand = ImmutableList.builder();
    compilerCommand.addAll(compiler);
    compilerCommand.add("-g");

    DebugPathSanitizer sanitizer =
        new DebugPathSanitizer(200, File.separatorChar, compDir, ImmutableBiMap.<Path, Path>of());

    // Build an archive step.
    CxxPreprocessAndCompileStep step =
        new CxxPreprocessAndCompileStep(
            CxxPreprocessAndCompileStep.Operation.COMPILE_MUNGE_DEBUGINFO,
            output,
            relativeInput,
            CxxSource.Type.C,
            Optional.of(preprocessorCommand.build()),
            Optional.of(compilerCommand.build()),
            ImmutableMap.<Path, Path>of(),
            sanitizer);

    // Execute the archive step and verify it ran successfully.
    ExecutionContext executionContext =
        TestExecutionContext.newBuilder()
            .setProjectFilesystem(new ProjectFilesystem(tmp.getRoot().toPath()))
            .build();
    TestConsole console = (TestConsole) executionContext.getConsole();
    int exitCode = step.execute(executionContext);
    if (failure.isPresent()) {
      assertNotEquals("compile step succeeded", 0, exitCode);
      assertThat(
          console.getTextWrittenToStdErr(),
          console.getTextWrittenToStdErr(),
          Matchers.containsString(failure.get()));
    } else {
      assertEquals("compile step failed: " + console.getTextWrittenToStdErr(), 0, exitCode);
      // Verify that we find the expected compilation dir embedded in the file.
      String contents = new String(Files.readAllBytes(output));
      assertThat(contents, Matchers.containsString(sanitizer.getCompilationDirectory()));
    }

    // Cleanup.
    Files.delete(input);
    Files.deleteIfExists(output);
  }
  private ImmutableList<SourcePath> generateCCompilation(ImmutableList<SourcePath> cInput) {

    ImmutableList.Builder<SourcePath> objects = ImmutableList.builder();

    ImmutableList.Builder<String> cCompileFlags = ImmutableList.builder();
    cCompileFlags.addAll(ocamlContext.getCCompileFlags());
    cCompileFlags.addAll(ocamlContext.getCommonCFlags());

    CxxPreprocessorInput cxxPreprocessorInput = ocamlContext.getCxxPreprocessorInput();

    for (SourcePath cSrc : cInput) {
      String name = pathResolver.getAbsolutePath(cSrc).toFile().getName();
      BuildTarget target = createCCompileBuildTarget(params.getBuildTarget(), name);

      BuildRuleParams cCompileParams =
          params.copyWithChanges(
              target,
              /* declaredDeps */ Suppliers.ofInstance(
                  ImmutableSortedSet.<BuildRule>naturalOrder()
                      // Depend on the rule that generates the sources and headers we're compiling.
                      .addAll(
                          pathResolver.filterBuildRuleInputs(
                              ImmutableList.<SourcePath>builder()
                                  .add(cSrc)
                                  .addAll(
                                      cxxPreprocessorInput
                                          .getIncludes()
                                          .getNameToPathMap()
                                          .values())
                                  .build()))
                      // Also add in extra deps from the preprocessor input, such as the symlink
                      // tree rules.
                      .addAll(
                          BuildRules.toBuildRulesFor(
                              params.getBuildTarget(), resolver, cxxPreprocessorInput.getRules()))
                      .addAll(params.getDeclaredDeps().get())
                      .build()),
              /* extraDeps */ params.getExtraDeps());

      Path outputPath = ocamlContext.getCOutput(pathResolver.getRelativePath(cSrc));
      OCamlCCompile compileRule =
          new OCamlCCompile(
              cCompileParams,
              pathResolver,
              new OCamlCCompileStep.Args(
                  cCompiler.getEnvironment(pathResolver),
                  cCompiler.getCommandPrefix(pathResolver),
                  ocamlContext.getOcamlCompiler().get(),
                  outputPath,
                  cSrc,
                  cCompileFlags.build(),
                  ImmutableMap.copyOf(cxxPreprocessorInput.getIncludes().getNameToPathMap())));
      resolver.addToIndex(compileRule);
      objects.add(new BuildTargetSourcePath(compileRule.getBuildTarget()));
    }
    return objects.build();
  }
  @Override
  public HiveOutputTableHandle beginCreateTable(
      ConnectorSession session, ConnectorTableMetadata tableMetadata) {
    verifyJvmTimeZone();

    checkArgument(!isNullOrEmpty(tableMetadata.getOwner()), "Table owner is null or empty");

    HiveStorageFormat hiveStorageFormat = getHiveStorageFormat(session, this.hiveStorageFormat);

    ImmutableList.Builder<String> columnNames = ImmutableList.builder();
    ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();

    // get the root directory for the database
    SchemaTableName schemaTableName = tableMetadata.getTable();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();

    buildColumnInfo(tableMetadata, columnNames, columnTypes);

    Path targetPath = getTargetPath(schemaName, tableName, schemaTableName);

    if (!useTemporaryDirectory(targetPath)) {
      return new HiveOutputTableHandle(
          connectorId,
          schemaName,
          tableName,
          columnNames.build(),
          columnTypes.build(),
          tableMetadata.getOwner(),
          targetPath.toString(),
          targetPath.toString(),
          session,
          hiveStorageFormat);
    }

    // use a per-user temporary directory to avoid permission problems
    // TODO: this should use Hadoop UserGroupInformation
    String temporaryPrefix = "/tmp/presto-" + StandardSystemProperty.USER_NAME.value();

    // create a temporary directory on the same filesystem
    Path temporaryRoot = new Path(targetPath, temporaryPrefix);
    Path temporaryPath = new Path(temporaryRoot, randomUUID().toString());
    createDirectories(temporaryPath);

    return new HiveOutputTableHandle(
        connectorId,
        schemaName,
        tableName,
        columnNames.build(),
        columnTypes.build(),
        tableMetadata.getOwner(),
        targetPath.toString(),
        temporaryPath.toString(),
        session,
        hiveStorageFormat);
  }
 /**
  * Splits a constructor in two if the constructor contains a parameter of type UUID. The first
  * constructor is unchanged, while the second constructor has all UUID parameters converted into
  * strings. If the constructor does not have a UUID parameter, then the original constructor is
  * returned.
  */
 protected static ImmutableList<RequestConstructor> splitUuidConstructor(
     final RequestConstructor constructor) {
   final ImmutableList.Builder<RequestConstructor> builder = ImmutableList.builder();
   builder.add(constructor);
   if (!containsType(constructor.getParameters(), "UUID")) {
     return builder.build();
   }
   builder.add(convertUuidConstructorToStringConstructor(constructor));
   return builder.build();
 }
Exemple #11
0
 static {
   ImmutableList.Builder<SqlAggFunction> builder = ImmutableList.builder();
   builder.add(SqlStdOperatorTable.COUNT);
   builder.add(SqlStdOperatorTable.SUM);
   builder.add(SqlStdOperatorTable.SUM0);
   builder.add(SqlStdOperatorTable.MIN);
   builder.add(SqlStdOperatorTable.MAX);
   AGG_FUNCS = builder.build();
   builder.add(SqlStdOperatorTable.SINGLE_VALUE);
   MYSQL_AGG_FUNCS = builder.build();
 }
Exemple #12
0
  @Override
  protected RelationPlan visitUnnest(Unnest node, Void context) {
    TupleDescriptor descriptor = analysis.getOutputDescriptor(node);
    ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder();
    for (Field field : descriptor.getVisibleFields()) {
      Symbol symbol = symbolAllocator.newSymbol(field);
      outputSymbolsBuilder.add(symbol);
    }
    List<Symbol> unnestedSymbols = outputSymbolsBuilder.build();

    // If we got here, then we must be unnesting a constant, and not be in a join (where there could
    // be column references)
    ImmutableList.Builder<Symbol> argumentSymbols = ImmutableList.builder();
    ImmutableList.Builder<Expression> values = ImmutableList.builder();
    ImmutableMap.Builder<Symbol, List<Symbol>> unnestSymbols = ImmutableMap.builder();
    Iterator<Symbol> unnestedSymbolsIterator = unnestedSymbols.iterator();
    for (Expression expression : node.getExpressions()) {
      Object constantValue =
          evaluateConstantExpression(expression, analysis.getCoercions(), metadata, session);
      Type type = analysis.getType(expression);
      values.add(LiteralInterpreter.toExpression(constantValue, type));
      Symbol inputSymbol = symbolAllocator.newSymbol(expression, type);
      argumentSymbols.add(inputSymbol);
      if (type instanceof ArrayType) {
        unnestSymbols.put(inputSymbol, ImmutableList.of(unnestedSymbolsIterator.next()));
      } else if (type instanceof MapType) {
        unnestSymbols.put(
            inputSymbol,
            ImmutableList.of(unnestedSymbolsIterator.next(), unnestedSymbolsIterator.next()));
      } else {
        throw new IllegalArgumentException("Unsupported type for UNNEST: " + type);
      }
    }
    Optional<Symbol> ordinalitySymbol =
        node.isWithOrdinality() ? Optional.of(unnestedSymbolsIterator.next()) : Optional.empty();
    checkState(
        !unnestedSymbolsIterator.hasNext(),
        "Not all output symbols were matched with input symbols");
    ValuesNode valuesNode =
        new ValuesNode(
            idAllocator.getNextId(),
            argumentSymbols.build(),
            ImmutableList.<List<Expression>>of(values.build()));

    UnnestNode unnestNode =
        new UnnestNode(
            idAllocator.getNextId(),
            valuesNode,
            ImmutableList.<Symbol>of(),
            unnestSymbols.build(),
            ordinalitySymbol);
    return new RelationPlan(unnestNode, descriptor, unnestedSymbols, Optional.empty());
  }
 public SetMessagesResponse build() {
   return new SetMessagesResponse(
       accountId,
       oldState,
       newState,
       created.build(),
       updated.build(),
       destroyed.build(),
       notCreated.build(),
       notUpdated.build(),
       notDestroyed.build());
 }
  @Override
  public void generateMethods(
      ClassDefinition classDefinition,
      CallSiteBinder callSiteBinder,
      RowExpression filter,
      List<RowExpression> projections) {
    CachedInstanceBinder cachedInstanceBinder =
        new CachedInstanceBinder(classDefinition, callSiteBinder);
    ImmutableList.Builder<MethodDefinition> projectMethods = ImmutableList.builder();
    ImmutableList.Builder<MethodDefinition> projectColumnarMethods = ImmutableList.builder();
    ImmutableList.Builder<MethodDefinition> projectDictionaryMethods = ImmutableList.builder();
    for (int i = 0; i < projections.size(); i++) {
      MethodDefinition project =
          generateProjectMethod(
              classDefinition,
              callSiteBinder,
              cachedInstanceBinder,
              "project_" + i,
              projections.get(i));
      MethodDefinition projectColumnar =
          generateProjectColumnarMethod(
              classDefinition, callSiteBinder, "projectColumnar_" + i, projections.get(i), project);
      MethodDefinition projectDictionary =
          generateProjectDictionaryMethod(
              classDefinition,
              "projectDictionary_" + i,
              projections.get(i),
              project,
              projectColumnar);

      projectMethods.add(project);
      projectColumnarMethods.add(projectColumnar);
      projectDictionaryMethods.add(projectDictionary);
    }

    List<MethodDefinition> projectMethodDefinitions = projectMethods.build();
    List<MethodDefinition> projectColumnarMethodDefinitions = projectColumnarMethods.build();
    List<MethodDefinition> projectDictionaryMethodDefinitions = projectDictionaryMethods.build();

    generateProcessMethod(classDefinition, filter, projections, projectMethodDefinitions);
    generateGetNonLazyPageMethod(classDefinition, filter, projections);
    generateProcessColumnarMethod(classDefinition, projections, projectColumnarMethodDefinitions);
    generateProcessColumnarDictionaryMethod(
        classDefinition,
        projections,
        projectColumnarMethodDefinitions,
        projectDictionaryMethodDefinitions);

    generateFilterPageMethod(classDefinition, filter);
    generateFilterMethod(classDefinition, callSiteBinder, cachedInstanceBinder, filter);
    generateConstructor(classDefinition, cachedInstanceBinder, projections.size());
  }
  public ParquetPageSource(
      ParquetReader parquetReader,
      ParquetDataSource dataSource,
      MessageType fileSchema,
      MessageType requestedSchema,
      long totalBytes,
      Properties splitSchema,
      List<HiveColumnHandle> columns,
      TupleDomain<HiveColumnHandle> effectivePredicate,
      TypeManager typeManager,
      boolean useParquetColumnNames) {
    checkArgument(totalBytes >= 0, "totalBytes is negative");
    requireNonNull(splitSchema, "splitSchema is null");
    requireNonNull(columns, "columns is null");
    requireNonNull(effectivePredicate, "effectivePredicate is null");

    this.parquetReader = parquetReader;
    this.dataSource = dataSource;
    this.requestedSchema = requestedSchema;
    this.totalBytes = totalBytes;

    int size = columns.size();
    this.constantBlocks = new Block[size];
    this.hiveColumnIndexes = new int[size];

    ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
    for (int columnIndex = 0; columnIndex < size; columnIndex++) {
      HiveColumnHandle column = columns.get(columnIndex);
      checkState(column.getColumnType() == REGULAR, "column type must be regular");

      String name = column.getName();
      Type type = typeManager.getType(column.getTypeSignature());

      namesBuilder.add(name);
      typesBuilder.add(type);

      hiveColumnIndexes[columnIndex] = column.getHiveColumnIndex();

      if (getParquetType(column, fileSchema, useParquetColumnNames) == null) {
        BlockBuilder blockBuilder =
            type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH);
        for (int i = 0; i < MAX_VECTOR_LENGTH; i++) {
          blockBuilder.appendNull();
        }
        constantBlocks[columnIndex] = blockBuilder.build();
      }
    }
    types = typesBuilder.build();
    columnNames = namesBuilder.build();
  }
  public OperatorFactory createTableScanOperator(
      Session session, int operatorId, String tableName, String... columnNames) {
    checkArgument(session.getCatalog().isPresent(), "catalog not set");
    checkArgument(session.getSchema().isPresent(), "schema not set");

    // look up the table
    QualifiedTableName qualifiedTableName =
        new QualifiedTableName(session.getCatalog().get(), session.getSchema().get(), tableName);
    TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName).orElse(null);
    checkArgument(tableHandle != null, "Table %s does not exist", qualifiedTableName);

    // lookup the columns
    Map<String, ColumnHandle> allColumnHandles = metadata.getColumnHandles(session, tableHandle);
    ImmutableList.Builder<ColumnHandle> columnHandlesBuilder = ImmutableList.builder();
    ImmutableList.Builder<Type> columnTypesBuilder = ImmutableList.builder();
    for (String columnName : columnNames) {
      ColumnHandle columnHandle = allColumnHandles.get(columnName);
      checkArgument(
          columnHandle != null, "Table %s does not have a column %s", tableName, columnName);
      columnHandlesBuilder.add(columnHandle);
      ColumnMetadata columnMetadata =
          metadata.getColumnMetadata(session, tableHandle, columnHandle);
      columnTypesBuilder.add(columnMetadata.getType());
    }
    List<ColumnHandle> columnHandles = columnHandlesBuilder.build();
    List<Type> columnTypes = columnTypesBuilder.build();

    // get the split for this table
    List<TableLayoutResult> layouts =
        metadata.getLayouts(session, tableHandle, Constraint.alwaysTrue(), Optional.empty());
    Split split = getLocalQuerySplit(layouts.get(0).getLayout().getHandle());

    return new OperatorFactory() {
      @Override
      public List<Type> getTypes() {
        return columnTypes;
      }

      @Override
      public Operator createOperator(DriverContext driverContext) {
        OperatorContext operatorContext =
            driverContext.addOperatorContext(operatorId, "BenchmarkSource");
        ConnectorPageSource pageSource =
            pageSourceManager.createPageSource(session, split, columnHandles);
        return new PageSourceOperator(pageSource, columnTypes, operatorContext);
      }

      @Override
      public void close() {}
    };
  }
  IndexShardRoutingTable(
      ShardId shardId, List<ShardRouting> shards, boolean primaryAllocatedPostApi) {
    this.shardId = shardId;
    this.shuffler = new RotationShardShuffler(ThreadLocalRandom.current().nextInt());
    this.shards = ImmutableList.copyOf(shards);
    this.primaryAllocatedPostApi = primaryAllocatedPostApi;

    ShardRouting primary = null;
    ImmutableList.Builder<ShardRouting> replicas = ImmutableList.builder();
    ImmutableList.Builder<ShardRouting> activeShards = ImmutableList.builder();
    ImmutableList.Builder<ShardRouting> assignedShards = ImmutableList.builder();
    ImmutableList.Builder<ShardRouting> allInitializingShards = ImmutableList.builder();
    boolean allShardsStarted = true;
    for (ShardRouting shard : shards) {
      if (shard.primary()) {
        primary = shard;
      } else {
        replicas.add(shard);
      }
      if (shard.active()) {
        activeShards.add(shard);
      }
      if (shard.initializing()) {
        allInitializingShards.add(shard);
      }
      if (shard.relocating()) {
        // create the target initializing shard routing on the node the shard is relocating to
        allInitializingShards.add(shard.buildTargetRelocatingShard());
      }
      if (shard.assignedToNode()) {
        assignedShards.add(shard);
      }
      if (shard.state() != ShardRoutingState.STARTED) {
        allShardsStarted = false;
      }
    }
    this.allShardsStarted = allShardsStarted;

    this.primary = primary;
    if (primary != null) {
      this.primaryAsList = ImmutableList.of(primary);
    } else {
      this.primaryAsList = ImmutableList.of();
    }
    this.replicas = replicas.build();
    this.activeShards = activeShards.build();
    this.assignedShards = assignedShards.build();
    this.allInitializingShards = allInitializingShards.build();
  }
Exemple #18
0
 /** @return the current stack trace as a list of functions. */
 public ImmutableList<BaseFunction> getStackTrace() {
   ImmutableList.Builder<BaseFunction> builder = new ImmutableList.Builder<>();
   for (Continuation k = continuation; k != null; k = k.continuation) {
     builder.add(k.function);
   }
   return builder.build().reverse();
 }
  /**
   * Parse a string delimited by comma into a list of object instances depending on
   *
   * <pre>itemParser</pre>
   *
   * .
   *
   * @param str String delimited by comma
   * @param itemParser A function to transform a string to an object.
   * @param <T> Type to be transformed from a string
   * @return List of object instances
   */
  static <T> List<T> parseList(String str, Function<String, T> itemParser) {
    if (!str.contains(",")) { // if just one item
      return ImmutableList.of(itemParser.apply(str));
    }

    final ImmutableList.Builder<T> fields = ImmutableList.builder();
    final Stack<Character> stack = new Stack<>();
    int paramStartIdx = 0;
    for (int i = 0; i < str.length(); i++) {
      char c = str.charAt(i);

      if (c == '<' || c == '[' || c == '(') {
        stack.push(c);
      } else if (c == '>') {
        Assert.assertCondition(stack.pop() == '<', "Bad signature: '%s'", str);
      } else if (c == ']') {
        Assert.assertCondition(stack.pop() == '[', "Bad signature: '%s'", str);
      } else if (c == ')') {
        Assert.assertCondition(stack.pop() == '(', "Bad signature: '%s'", str);
      } else if (c == ',') {
        if (stack.isEmpty()) { // ensure outermost type parameters
          fields.add(itemParser.apply(str.substring(paramStartIdx, i)));
          paramStartIdx = i + 1;
        }
      }
    }

    Assert.assertCondition(stack.empty(), "Bad signature: '%s'", str);
    if (paramStartIdx < str.length()) {
      fields.add(itemParser.apply(str.substring(paramStartIdx, str.length())));
    }

    return fields.build();
  }
 /**
  * Gets the list of Arguments needed to create the request constructor. This includes all non-void
  * required parameters, and arguments described within the request header.
  */
 @Override
 public ImmutableList<Arguments> toConstructorArgumentsList(final Ds3Request ds3Request) {
   final ImmutableList.Builder<Arguments> builder = ImmutableList.builder();
   builder.addAll(getRequiredArgsFromRequestHeader(ds3Request));
   builder.addAll(removeVoidArguments(toArgumentsList(ds3Request.getRequiredQueryParams())));
   return builder.build();
 }
Exemple #21
0
 public static ImmutableList<Mod> split(String cs) {
   ImmutableList.Builder<Mod> builder = ImmutableList.builder();
   for (String ms : Splitters.csplitter(cs)) {
     builder.add(Mod.parseMod(ms));
   }
   return builder.build();
 }
 private List<JavaType> convertAsmTypes(org.objectweb.asm.Type[] asmTypes) {
   ImmutableList.Builder<JavaType> result = ImmutableList.builder();
   for (org.objectweb.asm.Type asmType : asmTypes) {
     result.add(convertAsmType(asmType));
   }
   return result.build();
 }
Exemple #23
0
 /**
  * @author gabizou - February 7th, 2016
  *     <p>This will short circuit all other patches such that we control the entities being loaded
  *     by chunkloading and can throw our bulk entity event. This will bypass Forge's hook for
  *     individual entity events, but the SpongeModEventManager will still successfully throw the
  *     appropriate event and cancel the entities otherwise contained.
  * @param entities The entities being loaded
  * @param callbackInfo The callback info
  */
 @Final
 @Inject(method = "loadEntities", at = @At("HEAD"), cancellable = true)
 private void spongeLoadEntities(
     Collection<net.minecraft.entity.Entity> entities, CallbackInfo callbackInfo) {
   if (entities.isEmpty()) {
     // just return, no entities to load!
     callbackInfo.cancel();
     return;
   }
   List<Entity> entityList = new ArrayList<>();
   ImmutableList.Builder<EntitySnapshot> snapshotBuilder = ImmutableList.builder();
   for (net.minecraft.entity.Entity entity : entities) {
     entityList.add((Entity) entity);
     snapshotBuilder.add(((Entity) entity).createSnapshot());
   }
   SpawnCause cause = SpawnCause.builder().type(InternalSpawnTypes.CHUNK_LOAD).build();
   List<NamedCause> causes = new ArrayList<>();
   causes.add(NamedCause.source(cause));
   causes.add(NamedCause.of("World", this));
   SpawnEntityEvent.ChunkLoad chunkLoad =
       SpongeEventFactory.createSpawnEntityEventChunkLoad(
           Cause.of(causes), entityList, snapshotBuilder.build(), this);
   SpongeImpl.postEvent(chunkLoad);
   if (!chunkLoad.isCancelled()) {
     for (Entity successful : chunkLoad.getEntities()) {
       this.loadedEntityList.add((net.minecraft.entity.Entity) successful);
       this.onEntityAdded((net.minecraft.entity.Entity) successful);
     }
   }
   callbackInfo.cancel();
 }
  private void testCorrectnessOfErrorFunction(List<Number> inputList) throws Exception {
    int inRange = 0;
    int numberOfRuns = 1000;
    double sampleRatio = 1 / (double) WEIGHT;
    double actual = getExpectedValue(inputList);
    Random rand = new Random(1);

    for (int i = 0; i < numberOfRuns; i++) {
      // Compute Sampled Value using sampledList (numberOfRuns times)
      ImmutableList.Builder<Number> sampledList = ImmutableList.builder();
      for (Number x : inputList) {
        if (rand.nextDouble() < sampleRatio) {
          sampledList.add(x);
        }
      }

      BlockBuilder builder = getType().createBlockBuilder(new BlockBuilderStatus());
      for (Number sample : sampledList.build()) {
        if (getType() == BIGINT) {
          BIGINT.writeLong(builder, sample.longValue());
        } else if (getType() == DOUBLE) {
          DOUBLE.writeDouble(builder, sample.doubleValue());
        } else {
          throw new AssertionError("Can only handle longs and doubles");
        }
      }
      Page page = new Page(builder.build());
      page = OperatorAssertion.appendSampleWeight(ImmutableList.of(page), WEIGHT).get(0);
      Accumulator accumulator =
          getFunction()
              .bind(
                  ImmutableList.of(0),
                  Optional.<Integer>absent(),
                  Optional.of(page.getChannelCount() - 1),
                  getConfidence())
              .createAccumulator();

      accumulator.addInput(page);
      Block result = accumulator.evaluateFinal();

      String approxValue =
          BlockAssertions.toValues(accumulator.getFinalType(), result).get(0).toString();
      double approx = Double.parseDouble(approxValue.split(" ")[0]);
      double error = Double.parseDouble(approxValue.split(" ")[2]);

      // Check if actual answer lies within [approxAnswer - error, approxAnswer + error]
      if (Math.abs(approx - actual) <= error) {
        inRange++;
      }
    }

    BinomialDistribution binomial = new BinomialDistribution(numberOfRuns, getConfidence());
    int lowerBound = binomial.inverseCumulativeProbability(0.01);
    int upperBound = binomial.inverseCumulativeProbability(0.99);
    assertTrue(
        lowerBound < inRange && inRange < upperBound,
        String.format(
            "%d out of %d passed. Expected [%d, %d]",
            inRange, numberOfRuns, lowerBound, upperBound));
  }
Exemple #25
0
  private int executeCCompilation(
      ExecutionContext context, ImmutableList.Builder<Path> linkerInputs)
      throws IOException, InterruptedException {

    ImmutableList.Builder<String> cCompileFlags = ImmutableList.builder();
    cCompileFlags.addAll(ocamlContext.getCCompileFlags());
    cCompileFlags.addAll(ocamlContext.getCommonCFlags());

    CxxPreprocessorInput cxxPreprocessorInput = ocamlContext.getCxxPreprocessorInput();

    for (SourcePath cSrc : ocamlContext.getCInput()) {
      Path outputPath = ocamlContext.getCOutput(resolver.getAbsolutePath(cSrc));
      linkerInputs.add(outputPath);
      Step compileStep =
          new OCamlCCompileStep(
              resolver,
              filesystem.getRootPath(),
              new OCamlCCompileStep.Args(
                  cCompilerEnvironment,
                  cCompiler,
                  ocamlContext.getOcamlCompiler().get(),
                  outputPath,
                  cSrc,
                  cCompileFlags.build(),
                  ImmutableMap.copyOf(cxxPreprocessorInput.getIncludes().getNameToPathMap())));
      int compileExitCode = compileStep.execute(context);
      if (compileExitCode != 0) {
        return compileExitCode;
      }
    }
    return 0;
  }
Exemple #26
0
  private void logEndpoints() {
    final StringBuilder stringBuilder = new StringBuilder(1024).append("\n\n");

    final ImmutableList.Builder<Class<?>> builder = ImmutableList.builder();
    for (Object o : config.getSingletons()) {
      if (o.getClass().isAnnotationPresent(Path.class)) {
        builder.add(o.getClass());
      }
    }
    for (Class<?> klass : config.getClasses()) {
      if (klass.isAnnotationPresent(Path.class)) {
        builder.add(klass);
      }
    }

    for (Class<?> klass : builder.build()) {
      final String path = klass.getAnnotation(Path.class).value();
      final ImmutableList.Builder<String> endpoints = ImmutableList.builder();
      for (AnnotatedMethod method : annotatedMethods(klass)) {
        for (HttpMethod verb : method.getMetaMethodAnnotations(HttpMethod.class)) {
          endpoints.add(
              String.format("    %-7s %s (%s)", verb.value(), path, klass.getCanonicalName()));
        }
      }

      for (String line : Ordering.natural().sortedCopy(endpoints.build())) {
        stringBuilder.append(line).append('\n');
      }
    }

    LOG.info(stringBuilder.toString());
  }
Exemple #27
0
  public static Optional<TypeSignature> getCommonSuperTypeSignature(
      TypeSignature firstType, TypeSignature secondType) {
    // Special handling for UnknownType is necessary because we forbid cast between types with
    // different number of type parameters.
    // Without this, cast from null to map<bigint, bigint> will not be allowed.
    if (UnknownType.NAME.equals(firstType.getBase())) {
      return Optional.of(secondType);
    }
    if (UnknownType.NAME.equals(secondType.getBase())) {
      return Optional.of(firstType);
    }

    List<TypeSignatureParameter> firstTypeTypeParameters = firstType.getParameters();
    List<TypeSignatureParameter> secondTypeTypeParameters = secondType.getParameters();
    if (firstTypeTypeParameters.size() != secondTypeTypeParameters.size()) {
      return Optional.empty();
    }

    Optional<String> commonSuperTypeBase =
        getCommonSuperTypeBase(firstType.getBase(), secondType.getBase());
    if (!commonSuperTypeBase.isPresent()) {
      return Optional.empty();
    }

    ImmutableList.Builder<TypeSignatureParameter> typeParameters = ImmutableList.builder();
    for (int i = 0; i < firstTypeTypeParameters.size(); i++) {
      TypeSignatureParameter firstParameter = firstTypeTypeParameters.get(i);
      TypeSignatureParameter secondParameter = secondTypeTypeParameters.get(i);

      if (firstParameter.getKind() == secondParameter.getKind()
          && firstParameter.getKind() == ParameterKind.LONG_LITERAL) {
        typeParameters.add(
            TypeSignatureParameter.of(
                Math.max(firstParameter.getLongLiteral(), secondParameter.getLongLiteral())));
      } else if (isCovariantParameterPosition(commonSuperTypeBase.get(), i)) {
        Optional<TypeSignature> firstParameterSignature =
            firstParameter.getTypeSignatureOrNamedTypeSignature();
        Optional<TypeSignature> secondParameterSignature =
            secondParameter.getTypeSignatureOrNamedTypeSignature();
        if (!firstParameterSignature.isPresent() || !secondParameterSignature.isPresent()) {
          return Optional.empty();
        }

        Optional<TypeSignature> commonSuperType =
            getCommonSuperTypeSignature(
                firstParameterSignature.get(), secondParameterSignature.get());
        if (!commonSuperType.isPresent()) {
          return Optional.empty();
        }
        typeParameters.add(TypeSignatureParameter.of(commonSuperType.get()));
      } else {
        if (!firstParameter.equals(secondParameter)) {
          return Optional.empty();
        }
        typeParameters.add(firstParameter);
      }
    }

    return Optional.of(new TypeSignature(commonSuperTypeBase.get(), typeParameters.build()));
  }
  protected List<String> buildArguments() {
    ImmutableList.Builder<String> builder = ImmutableList.builder();

    builder.add("-host").add(settings.getHost());
    builder.add("-port").add(String.format("%s", launcherPort));
    if (settings.getDisplay() != null && settings.getDisplay() > 0) {
      builder.add("-display").add(String.format(":%s", settings.getDisplay()));
    }
    if (settings.logging().getLevel() != Level.OFF) {
      builder.add("-console"); // TODO(andreastt): Allow for file logging
      builder
          .add("-verbosity")
          .add(toLauncherLoggingLevel(settings.logging().getLevel()).toString());
    }
    if (settings.getProduct() != OperaProduct.ALL) {
      builder.add("-profile").add(settings.getProduct().getDescriptionString());
    }
    if (settings.getBackend() != null && !settings.getBackend().isEmpty()) {
      builder.add("-backend").add(settings.getBackend());
    }
    if (settings.hasDetach()) {
      builder.add("-noquit");
    }
    builder.add("-bin").add(settings.getBinary().getAbsolutePath());

    // The launcher will pass on any extra arguments to Opera
    for (OperaArgument argument : settings.arguments()) {
      builder.add(settings.arguments().sign() + argument.getArgument());
      if (argument.getValue() != null && !argument.getValue().isEmpty()) {
        builder.add(argument.getValue());
      }
    }

    return builder.build();
  }
Exemple #29
0
  private ImmutableList<Artifact> generatedOutputArtifacts(FileType newFileType) {
    ImmutableList.Builder<Artifact> builder = new ImmutableList.Builder<>();
    for (Artifact protoFile : getFilteredProtoSources()) {
      String protoFileName = FileSystemUtils.removeExtension(protoFile.getFilename());
      String generatedOutputName;
      if (attributes.outputsCpp()) {
        generatedOutputName = protoFileName;
      } else if (usesProtobufLibrary()) {
        // The protobuf library generates filenames with some slight modifications.
        generatedOutputName = generateProtobufFilename(protoFileName);
      } else {
        String lowerUnderscoreBaseName = protoFileName.replace('-', '_').toLowerCase();
        generatedOutputName = LOWER_UNDERSCORE.to(UPPER_CAMEL, lowerUnderscoreBaseName);
      }

      PathFragment generatedFilePath =
          new PathFragment(
              protoFile.getRootRelativePath().getParentDirectory(),
              new PathFragment(generatedOutputName));

      PathFragment outputFile =
          FileSystemUtils.appendExtension(generatedFilePath, newFileType.getExtensions().get(0));

      if (outputFile != null) {
        builder.add(
            ruleContext.getUniqueDirectoryArtifact(
                UNIQUE_DIRECTORY_NAME, outputFile, ruleContext.getBinOrGenfilesDirectory()));
      }
    }
    return builder.build();
  }
  @Override
  public Records getNext(int maxNumberOfRecords) {
    ensureBuffered();

    if (!it.hasNext() && buffer.isEndOfShard()) {
      return new Records(ImmutableList.<Record>of(), true);
    }

    ImmutableList.Builder<Record> recs = new ImmutableList.Builder<>();
    int recsSize = 0;

    while (recsSize < maxNumberOfRecords) {
      if (it.hasNext()) {
        recs.add(it.next());
        recsSize++;
      } else if (!it.hasNext() && !buffer.isEndOfShard()) {
        rebuffer();
        // No more data in shard.
        if (!it.hasNext()) {
          break;
        }
      } else {
        // No more records, end of shard.
        break;
      }
    }

    return new Records(recs.build(), false);
  }