Ejemplo n.º 1
0
  public static JsonSubQuery setDoWhileCondition(final String condition) {
    ImmutableList.Builder<PlanFragmentEncoding> fragments = ImmutableList.builder();
    int opId = 0;

    /* The worker part: scan the relation and send it to master. */
    // scan the relation
    TempTableScanEncoding scan = new TempTableScanEncoding();
    scan.opId = opId++;
    scan.opName = "Scan[" + condition + "]";
    scan.table = condition;
    // send it to master
    CollectProducerEncoding producer = new CollectProducerEncoding();
    producer.argChild = scan.opId;
    producer.opName = "CollectProducer[" + scan.opName + "]";
    producer.opId = opId++;
    // make a fragment
    PlanFragmentEncoding workerFragment = new PlanFragmentEncoding();
    workerFragment.operators = ImmutableList.of(scan, producer);
    // add it to the list
    fragments.add(workerFragment);

    /* The master part: collect the tuples, update the variable. */
    // collect the tuples
    CollectConsumerEncoding consumer = new CollectConsumerEncoding();
    consumer.argOperatorId = producer.opId;
    consumer.opId = opId++;
    consumer.opName = "CollectConsumer";
    // update the variable
    SetGlobalEncoding setGlobal = new SetGlobalEncoding();
    setGlobal.opId = opId++;
    setGlobal.opName = "SetGlobal[" + condition + "]";
    setGlobal.argChild = consumer.opId;
    setGlobal.key = condition;
    // the fragment, and it must only run at the master.
    PlanFragmentEncoding masterFragment = new PlanFragmentEncoding();
    masterFragment.operators = ImmutableList.of(consumer, setGlobal);
    masterFragment.overrideWorkers = ImmutableList.of(MyriaConstants.MASTER_ID);
    fragments.add(masterFragment);

    // Done!
    return new JsonSubQuery(fragments.build());
  }
Ejemplo n.º 2
0
  /**
   * Instantiate the server's desired physical plan from a list of JSON encodings of fragments. This
   * list must contain a self-consistent, complete query. All fragments will be executed in
   * parallel.
   *
   * @param fragments the JSON-encoded query fragments to be executed in parallel
   * @param server the server on which the query will be executed
   * @return the physical plan
   * @throws CatalogException if there is an error instantiating the plan
   */
  public static Map<Integer, SubQueryPlan> instantiate(
      final List<PlanFragmentEncoding> fragments, final ConstructArgs args)
      throws CatalogException {

    // Assign fragment index before everything else
    int idx = 0;
    for (PlanFragmentEncoding fragment : fragments) {
      fragment.setFragmentIndex(idx++);
    }

    /* Sanity check the edges between fragments. */
    sanityCheckEdges(fragments);

    assignWorkersToFragments(fragments, args);

    Map<Integer, PlanFragmentEncoding> op2OwnerFragmentMapping = Maps.newHashMap();
    for (PlanFragmentEncoding fragment : fragments) {
      for (OperatorEncoding<?> op : fragment.operators) {
        op2OwnerFragmentMapping.put(op.opId, fragment);
      }
    }

    Map<Integer, SubQueryPlan> plan = Maps.newHashMap();
    Map<PlanFragmentEncoding, RootOperator> instantiatedFragments = Maps.newHashMap();
    Map<Integer, Operator> allOperators = Maps.newHashMap();
    for (PlanFragmentEncoding fragment : fragments) {
      RootOperator op =
          instantiateFragment(
              fragment, args, instantiatedFragments, op2OwnerFragmentMapping, allOperators);
      for (Integer worker : fragment.workers) {
        SubQueryPlan workerPlan = plan.get(worker);
        if (workerPlan == null) {
          workerPlan = new SubQueryPlan();
          plan.put(worker, workerPlan);
        }
        workerPlan.addRootOp(op);
      }
    }
    return plan;
  }
Ejemplo n.º 3
0
  /**
   * Given an abstract execution plan, assign the workers to the fragments.
   *
   * <p>This assignment follows the following five rules, in precedence order:
   *
   * <ol>
   *   <li>Obey user-overrides of fragment workers.
   *   <li>Fragments that scan tables must use the workers that contain the data.
   *   <li>Edge constraints between fragments. E.g., a {@link LocalMultiwayProducerEncoding} must
   *       use the same set of workers as its consumer.
   *   <li>Singleton constraints: Fragments with a {@link CollectConsumerEncoding} or a {@link
   *       SingletonEncoding} must run on a single worker. If none is still set, choose an arbitrary
   *       worker.
   *   <li>Unspecified: Any fragments that still have unspecified worker sets will use all workers
   *       in the cluster.
   * </ol>
   *
   * @param fragments
   * @param args
   * @throws CatalogException if there is an error getting information about existing relations from
   *     the catalog
   */
  private static void assignWorkersToFragments(
      final List<PlanFragmentEncoding> fragments, final ConstructArgs args)
      throws CatalogException {

    /* 1. Honor user overrides. Note this is unchecked, but we may find constraint violations later. */
    for (PlanFragmentEncoding fragment : fragments) {
      if (fragment.overrideWorkers != null && fragment.overrideWorkers.size() > 0) {
        /* The workers are set in the plan. */
        fragment.workers = fragment.overrideWorkers;
      }
    }

    /* 2. Use scans to set workers, and verify constraints. */
    setAndVerifyScans(fragments, args);

    /* 3. Verify and propagate worker assignments using LocalMultiwayProducer/Consumer constraints. */
    verifyAndPropagateLocalEdgeConstraints(fragments);

    /* 4. Use singletons to set worker, and verify constraints. */
    setAndVerifySingletonConstraints(fragments, args);

    /* 5. Again, verify and propagate worker assignments using LocalMultiwayProducer/Consumer constraints. */
    verifyAndPropagateLocalEdgeConstraints(fragments);

    /* Last-1. For all remaining fragments, fill them in with all workers. */
    Server server = args.getServer();
    ImmutableList<Integer> allWorkers = ImmutableList.copyOf(server.getAliveWorkers());
    for (PlanFragmentEncoding fragment : fragments) {
      if (fragment.workers == null) {
        fragment.workers = allWorkers;
      }
    }
    // We don't need to verify and propagate LocalMultiwayProducer/Consumer constraints again since
    // all the new ones
    // have all workers.

    /* Fill in the #realOperatorIDs and the #realWorkerIDs fields for the producers and consumers. */
    fillInRealOperatorAndWorkerIDs(fragments);
  }
Ejemplo n.º 4
0
  /**
   * Use the Catalog to set the workers for fragments that have scans, and verify that the workers
   * are consistent with existing constraints.
   *
   * @see #assignWorkersToFragments(List, ConstructArgs)
   * @param fragments the fragments of the plan
   * @param args other arguments necessary for query construction
   * @throws CatalogException if there is an error getting information from the Catalog
   */
  private static void setAndVerifySingletonConstraints(
      final List<PlanFragmentEncoding> fragments, final ConstructArgs args) {
    List<Integer> singletonWorkers =
        ImmutableList.of(args.getServer().getAliveWorkers().iterator().next());

    for (PlanFragmentEncoding fragment : fragments) {
      for (OperatorEncoding<?> operator : fragment.operators) {
        if (operator instanceof CollectConsumerEncoding
            || operator instanceof SingletonEncoding
            || operator instanceof EOSControllerEncoding
            || operator instanceof BinaryFileScanEncoding
            || operator instanceof FileScanEncoding
            || operator instanceof NChiladaFileScanEncoding
            || operator instanceof SeaFlowFileScanEncoding
            || operator instanceof TipsyFileScanEncoding) {
          if (fragment.workers == null) {
            String encodingTypeName = operator.getClass().getSimpleName();
            String operatorTypeName =
                encodingTypeName.substring(0, encodingTypeName.indexOf("Encoding"));
            LOGGER.warn(
                "{} operator can only be instantiated on a single worker, assigning to random worker",
                operatorTypeName);
            fragment.workers = singletonWorkers;
          } else {
            Preconditions.checkArgument(
                fragment.workers.size() == 1,
                "Fragment %s has a singleton operator %s, but workers %s",
                fragment.fragmentIndex,
                operator.opId,
                fragment.workers);
          }
          /* We only need to verify singleton-ness once per fragment. */
          break;
        }
      }
    }
  }
Ejemplo n.º 5
0
 /**
  * Helper function for setting the workers of a fragment. If the workers are not yet set, always
  * succeeds. If the workers are set, ensures that the new value exactly matches the old value.
  *
  * @param fragment the fragment
  * @param workers the workers this fragment should be assigned to
  * @return <code>true</code> if the workers were newly assigned
  * @throws IllegalArgumentException if the fragment already has workers, and the new set does not
  *     match
  */
 private static boolean setOrVerifyFragmentWorkers(
     @Nonnull final PlanFragmentEncoding fragment,
     @Nonnull final Collection<Integer> workers,
     @Nonnull final String currentTask) {
   Preconditions.checkNotNull(fragment, "fragment");
   Preconditions.checkNotNull(workers, "workers");
   Preconditions.checkNotNull(currentTask, "currentTask");
   if (fragment.workers == null) {
     fragment.workers = ImmutableList.copyOf(workers);
     return true;
   } else {
     Preconditions.checkArgument(
         HashMultiset.create(fragment.workers).equals(HashMultiset.create(workers)),
         "During %s, cannot change workers for fragment %s from %s to %s",
         currentTask,
         fragment.fragmentIndex,
         fragment.workers,
         workers);
     return false;
   }
 }