public static JsonSubQuery setDoWhileCondition(final String condition) { ImmutableList.Builder<PlanFragmentEncoding> fragments = ImmutableList.builder(); int opId = 0; /* The worker part: scan the relation and send it to master. */ // scan the relation TempTableScanEncoding scan = new TempTableScanEncoding(); scan.opId = opId++; scan.opName = "Scan[" + condition + "]"; scan.table = condition; // send it to master CollectProducerEncoding producer = new CollectProducerEncoding(); producer.argChild = scan.opId; producer.opName = "CollectProducer[" + scan.opName + "]"; producer.opId = opId++; // make a fragment PlanFragmentEncoding workerFragment = new PlanFragmentEncoding(); workerFragment.operators = ImmutableList.of(scan, producer); // add it to the list fragments.add(workerFragment); /* The master part: collect the tuples, update the variable. */ // collect the tuples CollectConsumerEncoding consumer = new CollectConsumerEncoding(); consumer.argOperatorId = producer.opId; consumer.opId = opId++; consumer.opName = "CollectConsumer"; // update the variable SetGlobalEncoding setGlobal = new SetGlobalEncoding(); setGlobal.opId = opId++; setGlobal.opName = "SetGlobal[" + condition + "]"; setGlobal.argChild = consumer.opId; setGlobal.key = condition; // the fragment, and it must only run at the master. PlanFragmentEncoding masterFragment = new PlanFragmentEncoding(); masterFragment.operators = ImmutableList.of(consumer, setGlobal); masterFragment.overrideWorkers = ImmutableList.of(MyriaConstants.MASTER_ID); fragments.add(masterFragment); // Done! return new JsonSubQuery(fragments.build()); }
/** * Instantiate the server's desired physical plan from a list of JSON encodings of fragments. This * list must contain a self-consistent, complete query. All fragments will be executed in * parallel. * * @param fragments the JSON-encoded query fragments to be executed in parallel * @param server the server on which the query will be executed * @return the physical plan * @throws CatalogException if there is an error instantiating the plan */ public static Map<Integer, SubQueryPlan> instantiate( final List<PlanFragmentEncoding> fragments, final ConstructArgs args) throws CatalogException { // Assign fragment index before everything else int idx = 0; for (PlanFragmentEncoding fragment : fragments) { fragment.setFragmentIndex(idx++); } /* Sanity check the edges between fragments. */ sanityCheckEdges(fragments); assignWorkersToFragments(fragments, args); Map<Integer, PlanFragmentEncoding> op2OwnerFragmentMapping = Maps.newHashMap(); for (PlanFragmentEncoding fragment : fragments) { for (OperatorEncoding<?> op : fragment.operators) { op2OwnerFragmentMapping.put(op.opId, fragment); } } Map<Integer, SubQueryPlan> plan = Maps.newHashMap(); Map<PlanFragmentEncoding, RootOperator> instantiatedFragments = Maps.newHashMap(); Map<Integer, Operator> allOperators = Maps.newHashMap(); for (PlanFragmentEncoding fragment : fragments) { RootOperator op = instantiateFragment( fragment, args, instantiatedFragments, op2OwnerFragmentMapping, allOperators); for (Integer worker : fragment.workers) { SubQueryPlan workerPlan = plan.get(worker); if (workerPlan == null) { workerPlan = new SubQueryPlan(); plan.put(worker, workerPlan); } workerPlan.addRootOp(op); } } return plan; }
/** * Given an abstract execution plan, assign the workers to the fragments. * * <p>This assignment follows the following five rules, in precedence order: * * <ol> * <li>Obey user-overrides of fragment workers. * <li>Fragments that scan tables must use the workers that contain the data. * <li>Edge constraints between fragments. E.g., a {@link LocalMultiwayProducerEncoding} must * use the same set of workers as its consumer. * <li>Singleton constraints: Fragments with a {@link CollectConsumerEncoding} or a {@link * SingletonEncoding} must run on a single worker. If none is still set, choose an arbitrary * worker. * <li>Unspecified: Any fragments that still have unspecified worker sets will use all workers * in the cluster. * </ol> * * @param fragments * @param args * @throws CatalogException if there is an error getting information about existing relations from * the catalog */ private static void assignWorkersToFragments( final List<PlanFragmentEncoding> fragments, final ConstructArgs args) throws CatalogException { /* 1. Honor user overrides. Note this is unchecked, but we may find constraint violations later. */ for (PlanFragmentEncoding fragment : fragments) { if (fragment.overrideWorkers != null && fragment.overrideWorkers.size() > 0) { /* The workers are set in the plan. */ fragment.workers = fragment.overrideWorkers; } } /* 2. Use scans to set workers, and verify constraints. */ setAndVerifyScans(fragments, args); /* 3. Verify and propagate worker assignments using LocalMultiwayProducer/Consumer constraints. */ verifyAndPropagateLocalEdgeConstraints(fragments); /* 4. Use singletons to set worker, and verify constraints. */ setAndVerifySingletonConstraints(fragments, args); /* 5. Again, verify and propagate worker assignments using LocalMultiwayProducer/Consumer constraints. */ verifyAndPropagateLocalEdgeConstraints(fragments); /* Last-1. For all remaining fragments, fill them in with all workers. */ Server server = args.getServer(); ImmutableList<Integer> allWorkers = ImmutableList.copyOf(server.getAliveWorkers()); for (PlanFragmentEncoding fragment : fragments) { if (fragment.workers == null) { fragment.workers = allWorkers; } } // We don't need to verify and propagate LocalMultiwayProducer/Consumer constraints again since // all the new ones // have all workers. /* Fill in the #realOperatorIDs and the #realWorkerIDs fields for the producers and consumers. */ fillInRealOperatorAndWorkerIDs(fragments); }
/** * Use the Catalog to set the workers for fragments that have scans, and verify that the workers * are consistent with existing constraints. * * @see #assignWorkersToFragments(List, ConstructArgs) * @param fragments the fragments of the plan * @param args other arguments necessary for query construction * @throws CatalogException if there is an error getting information from the Catalog */ private static void setAndVerifySingletonConstraints( final List<PlanFragmentEncoding> fragments, final ConstructArgs args) { List<Integer> singletonWorkers = ImmutableList.of(args.getServer().getAliveWorkers().iterator().next()); for (PlanFragmentEncoding fragment : fragments) { for (OperatorEncoding<?> operator : fragment.operators) { if (operator instanceof CollectConsumerEncoding || operator instanceof SingletonEncoding || operator instanceof EOSControllerEncoding || operator instanceof BinaryFileScanEncoding || operator instanceof FileScanEncoding || operator instanceof NChiladaFileScanEncoding || operator instanceof SeaFlowFileScanEncoding || operator instanceof TipsyFileScanEncoding) { if (fragment.workers == null) { String encodingTypeName = operator.getClass().getSimpleName(); String operatorTypeName = encodingTypeName.substring(0, encodingTypeName.indexOf("Encoding")); LOGGER.warn( "{} operator can only be instantiated on a single worker, assigning to random worker", operatorTypeName); fragment.workers = singletonWorkers; } else { Preconditions.checkArgument( fragment.workers.size() == 1, "Fragment %s has a singleton operator %s, but workers %s", fragment.fragmentIndex, operator.opId, fragment.workers); } /* We only need to verify singleton-ness once per fragment. */ break; } } } }
/** * Helper function for setting the workers of a fragment. If the workers are not yet set, always * succeeds. If the workers are set, ensures that the new value exactly matches the old value. * * @param fragment the fragment * @param workers the workers this fragment should be assigned to * @return <code>true</code> if the workers were newly assigned * @throws IllegalArgumentException if the fragment already has workers, and the new set does not * match */ private static boolean setOrVerifyFragmentWorkers( @Nonnull final PlanFragmentEncoding fragment, @Nonnull final Collection<Integer> workers, @Nonnull final String currentTask) { Preconditions.checkNotNull(fragment, "fragment"); Preconditions.checkNotNull(workers, "workers"); Preconditions.checkNotNull(currentTask, "currentTask"); if (fragment.workers == null) { fragment.workers = ImmutableList.copyOf(workers); return true; } else { Preconditions.checkArgument( HashMultiset.create(fragment.workers).equals(HashMultiset.create(workers)), "During %s, cannot change workers for fragment %s from %s to %s", currentTask, fragment.fragmentIndex, fragment.workers, workers); return false; } }