Beispiel #1
0
  /**
   * Use the Catalog to set the workers for fragments that have scans, and verify that the workers
   * are consistent with existing constraints.
   *
   * @see #assignWorkersToFragments(List, ConstructArgs)
   * @param fragments the fragments of the plan
   * @param args other arguments necessary for query construction
   * @throws CatalogException if there is an error getting information from the Catalog
   */
  private static void setAndVerifyScans(
      final List<PlanFragmentEncoding> fragments, final ConstructArgs args)
      throws CatalogException {
    Server server = args.getServer();

    for (PlanFragmentEncoding fragment : fragments) {
      for (OperatorEncoding<?> operator : fragment.operators) {
        Set<Integer> scanWorkers;
        String scanRelation;

        if (operator instanceof TableScanEncoding) {
          TableScanEncoding scan = ((TableScanEncoding) operator);
          scanRelation = scan.relationKey.toString();
          scanWorkers = server.getWorkersForRelation(scan.relationKey, scan.storedRelationId);
        } else if (operator instanceof TempTableScanEncoding) {
          TempTableScanEncoding scan = ((TempTableScanEncoding) operator);
          scanRelation = "temporary relation " + scan.table;
          scanWorkers =
              server
                  .getQueryManager()
                  .getWorkersForTempRelation(
                      args.getQueryId(), RelationKey.ofTemp(args.getQueryId(), scan.table));
        } else {
          continue;
        }
        Preconditions.checkArgument(
            scanWorkers != null, "Unable to find workers that store %s", scanRelation);
        /*
         * Note: the current assumption is that all the partitions need to be scanned. This will not be true if we have
         * data replication, or allow to scan only a subset of the partitions. Revise if needed.
         */
        setOrVerifyFragmentWorkers(fragment, scanWorkers, "Setting workers for " + scanRelation);
      }
    }
  }
Beispiel #2
0
  /**
   * Given an abstract execution plan, assign the workers to the fragments.
   *
   * <p>This assignment follows the following five rules, in precedence order:
   *
   * <ol>
   *   <li>Obey user-overrides of fragment workers.
   *   <li>Fragments that scan tables must use the workers that contain the data.
   *   <li>Edge constraints between fragments. E.g., a {@link LocalMultiwayProducerEncoding} must
   *       use the same set of workers as its consumer.
   *   <li>Singleton constraints: Fragments with a {@link CollectConsumerEncoding} or a {@link
   *       SingletonEncoding} must run on a single worker. If none is still set, choose an arbitrary
   *       worker.
   *   <li>Unspecified: Any fragments that still have unspecified worker sets will use all workers
   *       in the cluster.
   * </ol>
   *
   * @param fragments
   * @param args
   * @throws CatalogException if there is an error getting information about existing relations from
   *     the catalog
   */
  private static void assignWorkersToFragments(
      final List<PlanFragmentEncoding> fragments, final ConstructArgs args)
      throws CatalogException {

    /* 1. Honor user overrides. Note this is unchecked, but we may find constraint violations later. */
    for (PlanFragmentEncoding fragment : fragments) {
      if (fragment.overrideWorkers != null && fragment.overrideWorkers.size() > 0) {
        /* The workers are set in the plan. */
        fragment.workers = fragment.overrideWorkers;
      }
    }

    /* 2. Use scans to set workers, and verify constraints. */
    setAndVerifyScans(fragments, args);

    /* 3. Verify and propagate worker assignments using LocalMultiwayProducer/Consumer constraints. */
    verifyAndPropagateLocalEdgeConstraints(fragments);

    /* 4. Use singletons to set worker, and verify constraints. */
    setAndVerifySingletonConstraints(fragments, args);

    /* 5. Again, verify and propagate worker assignments using LocalMultiwayProducer/Consumer constraints. */
    verifyAndPropagateLocalEdgeConstraints(fragments);

    /* Last-1. For all remaining fragments, fill them in with all workers. */
    Server server = args.getServer();
    ImmutableList<Integer> allWorkers = ImmutableList.copyOf(server.getAliveWorkers());
    for (PlanFragmentEncoding fragment : fragments) {
      if (fragment.workers == null) {
        fragment.workers = allWorkers;
      }
    }
    // We don't need to verify and propagate LocalMultiwayProducer/Consumer constraints again since
    // all the new ones
    // have all workers.

    /* Fill in the #realOperatorIDs and the #realWorkerIDs fields for the producers and consumers. */
    fillInRealOperatorAndWorkerIDs(fragments);
  }
Beispiel #3
0
  /**
   * Use the Catalog to set the workers for fragments that have scans, and verify that the workers
   * are consistent with existing constraints.
   *
   * @see #assignWorkersToFragments(List, ConstructArgs)
   * @param fragments the fragments of the plan
   * @param args other arguments necessary for query construction
   * @throws CatalogException if there is an error getting information from the Catalog
   */
  private static void setAndVerifySingletonConstraints(
      final List<PlanFragmentEncoding> fragments, final ConstructArgs args) {
    List<Integer> singletonWorkers =
        ImmutableList.of(args.getServer().getAliveWorkers().iterator().next());

    for (PlanFragmentEncoding fragment : fragments) {
      for (OperatorEncoding<?> operator : fragment.operators) {
        if (operator instanceof CollectConsumerEncoding
            || operator instanceof SingletonEncoding
            || operator instanceof EOSControllerEncoding
            || operator instanceof BinaryFileScanEncoding
            || operator instanceof FileScanEncoding
            || operator instanceof NChiladaFileScanEncoding
            || operator instanceof SeaFlowFileScanEncoding
            || operator instanceof TipsyFileScanEncoding) {
          if (fragment.workers == null) {
            String encodingTypeName = operator.getClass().getSimpleName();
            String operatorTypeName =
                encodingTypeName.substring(0, encodingTypeName.indexOf("Encoding"));
            LOGGER.warn(
                "{} operator can only be instantiated on a single worker, assigning to random worker",
                operatorTypeName);
            fragment.workers = singletonWorkers;
          } else {
            Preconditions.checkArgument(
                fragment.workers.size() == 1,
                "Fragment %s has a singleton operator %s, but workers %s",
                fragment.fragmentIndex,
                operator.opId,
                fragment.workers);
          }
          /* We only need to verify singleton-ness once per fragment. */
          break;
        }
      }
    }
  }