public static JsonSubQuery setDoWhileCondition(final String condition) {
ImmutableList.Builder<PlanFragmentEncoding> fragments = ImmutableList.builder();
int opId = 0;
/* The worker part: scan the relation and send it to master. */
// scan the relation
TempTableScanEncoding scan = new TempTableScanEncoding();
scan.opId = opId++;
scan.opName = "Scan[" + condition + "]";
scan.table = condition;
// send it to master
CollectProducerEncoding producer = new CollectProducerEncoding();
producer.argChild = scan.opId;
producer.opName = "CollectProducer[" + scan.opName + "]";
producer.opId = opId++;
// make a fragment
PlanFragmentEncoding workerFragment = new PlanFragmentEncoding();
workerFragment.operators = ImmutableList.of(scan, producer);
// add it to the list
fragments.add(workerFragment);
/* The master part: collect the tuples, update the variable. */
// collect the tuples
CollectConsumerEncoding consumer = new CollectConsumerEncoding();
consumer.argOperatorId = producer.opId;
consumer.opId = opId++;
consumer.opName = "CollectConsumer";
// update the variable
SetGlobalEncoding setGlobal = new SetGlobalEncoding();
setGlobal.opId = opId++;
setGlobal.opName = "SetGlobal[" + condition + "]";
setGlobal.argChild = consumer.opId;
setGlobal.key = condition;
// the fragment, and it must only run at the master.
PlanFragmentEncoding masterFragment = new PlanFragmentEncoding();
masterFragment.operators = ImmutableList.of(consumer, setGlobal);
masterFragment.overrideWorkers = ImmutableList.of(MyriaConstants.MASTER_ID);
fragments.add(masterFragment);
// Done!
return new JsonSubQuery(fragments.build());
}
/**
* Instantiate the server's desired physical plan from a list of JSON encodings of fragments. This
* list must contain a self-consistent, complete query. All fragments will be executed in
* parallel.
*
* @param fragments the JSON-encoded query fragments to be executed in parallel
* @param server the server on which the query will be executed
* @return the physical plan
* @throws CatalogException if there is an error instantiating the plan
*/
public static Map<Integer, SubQueryPlan> instantiate(
final List<PlanFragmentEncoding> fragments, final ConstructArgs args)
throws CatalogException {
// Assign fragment index before everything else
int idx = 0;
for (PlanFragmentEncoding fragment : fragments) {
fragment.setFragmentIndex(idx++);
}
/* Sanity check the edges between fragments. */
sanityCheckEdges(fragments);
assignWorkersToFragments(fragments, args);
Map<Integer, PlanFragmentEncoding> op2OwnerFragmentMapping = Maps.newHashMap();
for (PlanFragmentEncoding fragment : fragments) {
for (OperatorEncoding<?> op : fragment.operators) {
op2OwnerFragmentMapping.put(op.opId, fragment);
}
}
Map<Integer, SubQueryPlan> plan = Maps.newHashMap();
Map<PlanFragmentEncoding, RootOperator> instantiatedFragments = Maps.newHashMap();
Map<Integer, Operator> allOperators = Maps.newHashMap();
for (PlanFragmentEncoding fragment : fragments) {
RootOperator op =
instantiateFragment(
fragment, args, instantiatedFragments, op2OwnerFragmentMapping, allOperators);
for (Integer worker : fragment.workers) {
SubQueryPlan workerPlan = plan.get(worker);
if (workerPlan == null) {
workerPlan = new SubQueryPlan();
plan.put(worker, workerPlan);
}
workerPlan.addRootOp(op);
}
}
return plan;
}
/**
* Given an abstract execution plan, assign the workers to the fragments.
*
* <p>This assignment follows the following five rules, in precedence order:
*
* <ol>
* <li>Obey user-overrides of fragment workers.
* <li>Fragments that scan tables must use the workers that contain the data.
* <li>Edge constraints between fragments. E.g., a {@link LocalMultiwayProducerEncoding} must
* use the same set of workers as its consumer.
* <li>Singleton constraints: Fragments with a {@link CollectConsumerEncoding} or a {@link
* SingletonEncoding} must run on a single worker. If none is still set, choose an arbitrary
* worker.
* <li>Unspecified: Any fragments that still have unspecified worker sets will use all workers
* in the cluster.
* </ol>
*
* @param fragments
* @param args
* @throws CatalogException if there is an error getting information about existing relations from
* the catalog
*/
private static void assignWorkersToFragments(
final List<PlanFragmentEncoding> fragments, final ConstructArgs args)
throws CatalogException {
/* 1. Honor user overrides. Note this is unchecked, but we may find constraint violations later. */
for (PlanFragmentEncoding fragment : fragments) {
if (fragment.overrideWorkers != null && fragment.overrideWorkers.size() > 0) {
/* The workers are set in the plan. */
fragment.workers = fragment.overrideWorkers;
}
}
/* 2. Use scans to set workers, and verify constraints. */
setAndVerifyScans(fragments, args);
/* 3. Verify and propagate worker assignments using LocalMultiwayProducer/Consumer constraints. */
verifyAndPropagateLocalEdgeConstraints(fragments);
/* 4. Use singletons to set worker, and verify constraints. */
setAndVerifySingletonConstraints(fragments, args);
/* 5. Again, verify and propagate worker assignments using LocalMultiwayProducer/Consumer constraints. */
verifyAndPropagateLocalEdgeConstraints(fragments);
/* Last-1. For all remaining fragments, fill them in with all workers. */
Server server = args.getServer();
ImmutableList<Integer> allWorkers = ImmutableList.copyOf(server.getAliveWorkers());
for (PlanFragmentEncoding fragment : fragments) {
if (fragment.workers == null) {
fragment.workers = allWorkers;
}
}
// We don't need to verify and propagate LocalMultiwayProducer/Consumer constraints again since
// all the new ones
// have all workers.
/* Fill in the #realOperatorIDs and the #realWorkerIDs fields for the producers and consumers. */
fillInRealOperatorAndWorkerIDs(fragments);
}
/**
* Use the Catalog to set the workers for fragments that have scans, and verify that the workers
* are consistent with existing constraints.
*
* @see #assignWorkersToFragments(List, ConstructArgs)
* @param fragments the fragments of the plan
* @param args other arguments necessary for query construction
* @throws CatalogException if there is an error getting information from the Catalog
*/
private static void setAndVerifySingletonConstraints(
final List<PlanFragmentEncoding> fragments, final ConstructArgs args) {
List<Integer> singletonWorkers =
ImmutableList.of(args.getServer().getAliveWorkers().iterator().next());
for (PlanFragmentEncoding fragment : fragments) {
for (OperatorEncoding<?> operator : fragment.operators) {
if (operator instanceof CollectConsumerEncoding
|| operator instanceof SingletonEncoding
|| operator instanceof EOSControllerEncoding
|| operator instanceof BinaryFileScanEncoding
|| operator instanceof FileScanEncoding
|| operator instanceof NChiladaFileScanEncoding
|| operator instanceof SeaFlowFileScanEncoding
|| operator instanceof TipsyFileScanEncoding) {
if (fragment.workers == null) {
String encodingTypeName = operator.getClass().getSimpleName();
String operatorTypeName =
encodingTypeName.substring(0, encodingTypeName.indexOf("Encoding"));
LOGGER.warn(
"{} operator can only be instantiated on a single worker, assigning to random worker",
operatorTypeName);
fragment.workers = singletonWorkers;
} else {
Preconditions.checkArgument(
fragment.workers.size() == 1,
"Fragment %s has a singleton operator %s, but workers %s",
fragment.fragmentIndex,
operator.opId,
fragment.workers);
}
/* We only need to verify singleton-ness once per fragment. */
break;
}
}
}
}
/**
* Helper function for setting the workers of a fragment. If the workers are not yet set, always
* succeeds. If the workers are set, ensures that the new value exactly matches the old value.
*
* @param fragment the fragment
* @param workers the workers this fragment should be assigned to
* @return <code>true</code> if the workers were newly assigned
* @throws IllegalArgumentException if the fragment already has workers, and the new set does not
* match
*/
private static boolean setOrVerifyFragmentWorkers(
@Nonnull final PlanFragmentEncoding fragment,
@Nonnull final Collection<Integer> workers,
@Nonnull final String currentTask) {
Preconditions.checkNotNull(fragment, "fragment");
Preconditions.checkNotNull(workers, "workers");
Preconditions.checkNotNull(currentTask, "currentTask");
if (fragment.workers == null) {
fragment.workers = ImmutableList.copyOf(workers);
return true;
} else {
Preconditions.checkArgument(
HashMultiset.create(fragment.workers).equals(HashMultiset.create(workers)),
"During %s, cannot change workers for fragment %s from %s to %s",
currentTask,
fragment.fragmentIndex,
fragment.workers,
workers);
return false;
}
}