Ejemplo n.º 1
0
  private void parseAndRunLogicalPlan(String json) {

    try {
      LogicalPlan logicalPlan = context.getPlanReader().readLogicalPlan(json);

      if (logicalPlan.getProperties().resultMode == ResultMode.LOGICAL) {
        fail(
            "Failure running plan.  You requested a result mode of LOGICAL and submitted a logical plan.  In this case you're output mode must be PHYSICAL or EXEC.",
            new Exception());
      }
      if (logger.isDebugEnabled())
        logger.debug("Logical {}", logicalPlan.unparse(context.getConfig()));
      PhysicalPlan physicalPlan = convert(logicalPlan);

      if (logicalPlan.getProperties().resultMode == ResultMode.PHYSICAL) {
        returnPhysical(physicalPlan);
        return;
      }

      if (logger.isDebugEnabled())
        logger.debug(
            "Physical {}", context.getConfig().getMapper().writeValueAsString(physicalPlan));
      runPhysicalPlan(physicalPlan);
    } catch (IOException e) {
      fail("Failure while parsing logical plan.", e);
    } catch (OptimizerException e) {
      fail("Failure while converting logical plan to physical plan.", e);
    }
  }
Ejemplo n.º 2
0
 private PhysicalPlan convert(final LogicalPlan plan) throws OptimizerException {
   if (logger.isDebugEnabled()) {
     logger.debug("Converting logical plan {}.", plan.toJsonStringSafe(queryContext.getConfig()));
   }
   return new BasicOptimizer(queryContext, initiatingClient)
       .optimize(new BasicOptimizer.BasicOptimizationContext(queryContext), plan);
 }
Ejemplo n.º 3
0
  private void setupSortMemoryAllocations(final PhysicalPlan plan) {
    // look for external sorts
    final List<ExternalSort> sortList = new LinkedList<>();
    for (final PhysicalOperator op : plan.getSortedOperators()) {
      if (op instanceof ExternalSort) {
        sortList.add((ExternalSort) op);
      }
    }

    // if there are any sorts, compute the maximum allocation, and set it on them
    if (sortList.size() > 0) {
      final OptionManager optionManager = queryContext.getOptions();
      final long maxWidthPerNode =
          optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
      long maxAllocPerNode =
          Math.min(
              DrillConfig.getMaxDirectMemory(),
              queryContext.getConfig().getLong(ExecConstants.TOP_LEVEL_MAX_ALLOC));
      maxAllocPerNode =
          Math.min(
              maxAllocPerNode,
              optionManager.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val);
      final long maxSortAlloc = maxAllocPerNode / (sortList.size() * maxWidthPerNode);
      logger.debug("Max sort alloc: {}", maxSortAlloc);

      for (final ExternalSort externalSort : sortList) {
        externalSort.setMaxAllocation(maxSortAlloc);
      }
    }
  }
Ejemplo n.º 4
0
 protected void log(final String name, final PhysicalPlan plan, final Logger logger)
     throws JsonProcessingException {
   if (logger.isDebugEnabled()) {
     String planText = plan.unparse(context.getConfig().getMapper().writer());
     logger.debug(name + " : \n" + planText);
   }
 }
Ejemplo n.º 5
0
 private void log(final PhysicalPlan plan) {
   if (logger.isDebugEnabled()) {
     try {
       final String planText = queryContext.getConfig().getMapper().writeValueAsString(plan);
       logger.debug("Physical {}", planText);
     } catch (final IOException e) {
       logger.warn("Error while attempting to log physical plan.", e);
     }
   }
 }
Ejemplo n.º 6
0
 private void returnPhysical(final PhysicalPlan plan) throws ExecutionSetupException {
   final String jsonPlan = plan.unparse(queryContext.getConfig().getMapper().writer());
   runPhysicalPlan(
       DirectPlan.createDirectPlan(queryContext, new PhysicalFromLogicalExplain(jsonPlan)));
 }
Ejemplo n.º 7
0
 private void log(final LogicalPlan plan) {
   if (logger.isDebugEnabled()) {
     logger.debug("Logical {}", plan.unparse(queryContext.getConfig()));
   }
 }
Ejemplo n.º 8
0
 private PhysicalPlan convert(LogicalPlan plan) throws OptimizerException {
   if (logger.isDebugEnabled())
     logger.debug("Converting logical plan {}.", plan.toJsonStringSafe(context.getConfig()));
   return new BasicOptimizer(DrillConfig.create(), context)
       .optimize(new BasicOptimizer.BasicOptimizationContext(), plan);
 }
Ejemplo n.º 9
0
  private void runPhysicalPlan(PhysicalPlan plan) {

    if (plan.getProperties().resultMode != ResultMode.EXEC) {
      fail(
          String.format(
              "Failure running plan.  You requested a result mode of %s and a physical plan can only be output as EXEC",
              plan.getProperties().resultMode),
          new Exception());
    }
    PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();

    MakeFragmentsVisitor makeFragmentsVisitor = new MakeFragmentsVisitor();
    Fragment rootFragment;
    try {
      rootFragment = rootOperator.accept(makeFragmentsVisitor, null);
    } catch (FragmentSetupException e) {
      fail("Failure while fragmenting query.", e);
      return;
    }

    PlanningSet planningSet = StatsCollector.collectStats(rootFragment);
    SimpleParallelizer parallelizer = new SimpleParallelizer();

    try {
      QueryWorkUnit work =
          parallelizer.getFragments(
              context.getCurrentEndpoint(),
              queryId,
              context.getActiveEndpoints(),
              context.getPlanReader(),
              rootFragment,
              planningSet,
              context.getConfig().getInt(ExecConstants.GLOBAL_MAX_WIDTH),
              context.getConfig().getInt(ExecConstants.MAX_WIDTH_PER_ENDPOINT));

      this.context
          .getWorkBus()
          .setFragmentStatusListener(
              work.getRootFragment().getHandle().getQueryId(), fragmentManager);
      List<PlanFragment> leafFragments = Lists.newArrayList();
      List<PlanFragment> intermediateFragments = Lists.newArrayList();

      // store fragments in distributed grid.
      logger.debug("Storing fragments");
      for (PlanFragment f : work.getFragments()) {

        // store all fragments in grid since they are part of handshake.

        context.getCache().storeFragment(f);
        if (f.getLeafFragment()) {
          leafFragments.add(f);
        } else {
          intermediateFragments.add(f);
        }
      }

      logger.debug("Fragments stored.");

      logger.debug("Submitting fragments to run.");
      fragmentManager.runFragments(
          bee,
          work.getRootFragment(),
          work.getRootOperator(),
          initiatingClient,
          leafFragments,
          intermediateFragments);
      logger.debug("Fragments running.");

    } catch (ExecutionSetupException | RpcException e) {
      fail("Failure while setting up query.", e);
    }
  }
Ejemplo n.º 10
0
 private void returnPhysical(PhysicalPlan plan) {
   String jsonPlan = plan.unparse(context.getConfig().getMapper().writer());
   runPhysicalPlan(DirectPlan.createDirectPlan(context, new PhysicalFromLogicalExplain(jsonPlan)));
 }