private void parseAndRunLogicalPlan(String json) {
try {
LogicalPlan logicalPlan = context.getPlanReader().readLogicalPlan(json);
if (logicalPlan.getProperties().resultMode == ResultMode.LOGICAL) {
fail(
"Failure running plan. You requested a result mode of LOGICAL and submitted a logical plan. In this case you're output mode must be PHYSICAL or EXEC.",
new Exception());
}
if (logger.isDebugEnabled())
logger.debug("Logical {}", logicalPlan.unparse(context.getConfig()));
PhysicalPlan physicalPlan = convert(logicalPlan);
if (logicalPlan.getProperties().resultMode == ResultMode.PHYSICAL) {
returnPhysical(physicalPlan);
return;
}
if (logger.isDebugEnabled())
logger.debug(
"Physical {}", context.getConfig().getMapper().writeValueAsString(physicalPlan));
runPhysicalPlan(physicalPlan);
} catch (IOException e) {
fail("Failure while parsing logical plan.", e);
} catch (OptimizerException e) {
fail("Failure while converting logical plan to physical plan.", e);
}
}
private PhysicalPlan convert(final LogicalPlan plan) throws OptimizerException {
if (logger.isDebugEnabled()) {
logger.debug("Converting logical plan {}.", plan.toJsonStringSafe(queryContext.getConfig()));
}
return new BasicOptimizer(queryContext, initiatingClient)
.optimize(new BasicOptimizer.BasicOptimizationContext(queryContext), plan);
}
private void setupSortMemoryAllocations(final PhysicalPlan plan) {
// look for external sorts
final List<ExternalSort> sortList = new LinkedList<>();
for (final PhysicalOperator op : plan.getSortedOperators()) {
if (op instanceof ExternalSort) {
sortList.add((ExternalSort) op);
}
}
// if there are any sorts, compute the maximum allocation, and set it on them
if (sortList.size() > 0) {
final OptionManager optionManager = queryContext.getOptions();
final long maxWidthPerNode =
optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
long maxAllocPerNode =
Math.min(
DrillConfig.getMaxDirectMemory(),
queryContext.getConfig().getLong(ExecConstants.TOP_LEVEL_MAX_ALLOC));
maxAllocPerNode =
Math.min(
maxAllocPerNode,
optionManager.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val);
final long maxSortAlloc = maxAllocPerNode / (sortList.size() * maxWidthPerNode);
logger.debug("Max sort alloc: {}", maxSortAlloc);
for (final ExternalSort externalSort : sortList) {
externalSort.setMaxAllocation(maxSortAlloc);
}
}
}
protected void log(final String name, final PhysicalPlan plan, final Logger logger)
throws JsonProcessingException {
if (logger.isDebugEnabled()) {
String planText = plan.unparse(context.getConfig().getMapper().writer());
logger.debug(name + " : \n" + planText);
}
}
private void log(final PhysicalPlan plan) {
if (logger.isDebugEnabled()) {
try {
final String planText = queryContext.getConfig().getMapper().writeValueAsString(plan);
logger.debug("Physical {}", planText);
} catch (final IOException e) {
logger.warn("Error while attempting to log physical plan.", e);
}
}
}
private void returnPhysical(final PhysicalPlan plan) throws ExecutionSetupException {
final String jsonPlan = plan.unparse(queryContext.getConfig().getMapper().writer());
runPhysicalPlan(
DirectPlan.createDirectPlan(queryContext, new PhysicalFromLogicalExplain(jsonPlan)));
}
private void log(final LogicalPlan plan) {
if (logger.isDebugEnabled()) {
logger.debug("Logical {}", plan.unparse(queryContext.getConfig()));
}
}
private PhysicalPlan convert(LogicalPlan plan) throws OptimizerException {
if (logger.isDebugEnabled())
logger.debug("Converting logical plan {}.", plan.toJsonStringSafe(context.getConfig()));
return new BasicOptimizer(DrillConfig.create(), context)
.optimize(new BasicOptimizer.BasicOptimizationContext(), plan);
}
private void runPhysicalPlan(PhysicalPlan plan) {
if (plan.getProperties().resultMode != ResultMode.EXEC) {
fail(
String.format(
"Failure running plan. You requested a result mode of %s and a physical plan can only be output as EXEC",
plan.getProperties().resultMode),
new Exception());
}
PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
MakeFragmentsVisitor makeFragmentsVisitor = new MakeFragmentsVisitor();
Fragment rootFragment;
try {
rootFragment = rootOperator.accept(makeFragmentsVisitor, null);
} catch (FragmentSetupException e) {
fail("Failure while fragmenting query.", e);
return;
}
PlanningSet planningSet = StatsCollector.collectStats(rootFragment);
SimpleParallelizer parallelizer = new SimpleParallelizer();
try {
QueryWorkUnit work =
parallelizer.getFragments(
context.getCurrentEndpoint(),
queryId,
context.getActiveEndpoints(),
context.getPlanReader(),
rootFragment,
planningSet,
context.getConfig().getInt(ExecConstants.GLOBAL_MAX_WIDTH),
context.getConfig().getInt(ExecConstants.MAX_WIDTH_PER_ENDPOINT));
this.context
.getWorkBus()
.setFragmentStatusListener(
work.getRootFragment().getHandle().getQueryId(), fragmentManager);
List<PlanFragment> leafFragments = Lists.newArrayList();
List<PlanFragment> intermediateFragments = Lists.newArrayList();
// store fragments in distributed grid.
logger.debug("Storing fragments");
for (PlanFragment f : work.getFragments()) {
// store all fragments in grid since they are part of handshake.
context.getCache().storeFragment(f);
if (f.getLeafFragment()) {
leafFragments.add(f);
} else {
intermediateFragments.add(f);
}
}
logger.debug("Fragments stored.");
logger.debug("Submitting fragments to run.");
fragmentManager.runFragments(
bee,
work.getRootFragment(),
work.getRootOperator(),
initiatingClient,
leafFragments,
intermediateFragments);
logger.debug("Fragments running.");
} catch (ExecutionSetupException | RpcException e) {
fail("Failure while setting up query.", e);
}
}
private void returnPhysical(PhysicalPlan plan) {
String jsonPlan = plan.unparse(context.getConfig().getMapper().writer());
runPhysicalPlan(DirectPlan.createDirectPlan(context, new PhysicalFromLogicalExplain(jsonPlan)));
}