private QueryWorkUnit generateWorkUnit(
      DrillbitEndpoint foremanNode,
      QueryId queryId,
      PhysicalPlanReader reader,
      Fragment rootNode,
      PlanningSet planningSet)
      throws ExecutionSetupException {

    List<PlanFragment> fragments = Lists.newArrayList();

    PlanFragment rootFragment = null;
    FragmentRoot rootOperator = null;

    long queryStartTime = System.currentTimeMillis();

    // now we generate all the individual plan fragments and associated assignments. Note, we need
    // all endpoints
    // assigned before we can materialize, so we start a new loop here rather than utilizing the
    // previous one.
    for (Wrapper wrapper : planningSet) {
      Fragment node = wrapper.getNode();
      Stats stats = node.getStats();
      final PhysicalOperator physicalOperatorRoot = node.getRoot();
      boolean isRootNode = rootNode == node;

      if (isRootNode && wrapper.getWidth() != 1)
        throw new FragmentSetupException(
            String.format(
                "Failure while trying to setup fragment.  The root fragment must always have parallelization one.  In the current case, the width was set to %d.",
                wrapper.getWidth()));
      // a fragment is self driven if it doesn't rely on any other exchanges.
      boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;

      // Create a minorFragment for each major fragment.
      for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
        IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
        PhysicalOperator op = physicalOperatorRoot.accept(materializer, iNode);
        Preconditions.checkArgument(op instanceof FragmentRoot);
        FragmentRoot root = (FragmentRoot) op;

        // get plan as JSON
        String plan;
        try {
          plan = reader.writeJson(root);
        } catch (JsonProcessingException e) {
          throw new FragmentSetupException(
              "Failure while trying to convert fragment into json.", e);
        }

        FragmentHandle handle =
            FragmentHandle //
                .newBuilder() //
                .setMajorFragmentId(wrapper.getMajorFragmentId()) //
                .setMinorFragmentId(minorFragmentId) //
                .setQueryId(queryId) //
                .build();
        PlanFragment fragment =
            PlanFragment.newBuilder() //
                .setCpuCost(stats.getCpuCost()) //
                .setDiskCost(stats.getDiskCost()) //
                .setForeman(foremanNode) //
                .setMemoryCost(stats.getMemoryCost()) //
                .setNetworkCost(stats.getNetworkCost()) //
                .setFragmentJson(plan) //
                .setHandle(handle) //
                .setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
                .setLeafFragment(isLeafFragment) //
                .setQueryStartTime(queryStartTime)
                .build();

        if (isRootNode) {
          logger.debug("Root fragment:\n {}", fragment);
          rootFragment = fragment;
          rootOperator = root;
        } else {
          logger.debug("Remote fragment:\n {}", fragment);
          fragments.add(fragment);
        }
      }
    }

    return new QueryWorkUnit(rootOperator, rootFragment, fragments);
  }
Exemplo n.º 2
0
  protected QueryWorkUnit generateWorkUnit(
      OptionList options,
      DrillbitEndpoint foremanNode,
      QueryId queryId,
      PhysicalPlanReader reader,
      Fragment rootNode,
      PlanningSet planningSet,
      UserSession session,
      QueryContextInformation queryContextInfo)
      throws ExecutionSetupException {
    List<PlanFragment> fragments = Lists.newArrayList();

    PlanFragment rootFragment = null;
    FragmentRoot rootOperator = null;

    // now we generate all the individual plan fragments and associated assignments. Note, we need
    // all endpoints
    // assigned before we can materialize, so we start a new loop here rather than utilizing the
    // previous one.
    for (Wrapper wrapper : planningSet) {
      Fragment node = wrapper.getNode();
      final PhysicalOperator physicalOperatorRoot = node.getRoot();
      boolean isRootNode = rootNode == node;

      if (isRootNode && wrapper.getWidth() != 1) {
        throw new ForemanSetupException(
            String.format(
                "Failure while trying to setup fragment. "
                    + "The root fragment must always have parallelization one. In the current case, the width was set to %d.",
                wrapper.getWidth()));
      }
      // a fragment is self driven if it doesn't rely on any other exchanges.
      boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;

      // Create a minorFragment for each major fragment.
      for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
        IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
        wrapper.resetAllocation();
        PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode);
        Preconditions.checkArgument(op instanceof FragmentRoot);
        FragmentRoot root = (FragmentRoot) op;

        // get plan as JSON
        String plan;
        String optionsData;
        try {
          plan = reader.writeJson(root);
          optionsData = reader.writeJson(options);
        } catch (JsonProcessingException e) {
          throw new ForemanSetupException("Failure while trying to convert fragment into json.", e);
        }

        FragmentHandle handle =
            FragmentHandle //
                .newBuilder() //
                .setMajorFragmentId(wrapper.getMajorFragmentId()) //
                .setMinorFragmentId(minorFragmentId) //
                .setQueryId(queryId) //
                .build();

        PlanFragment fragment =
            PlanFragment.newBuilder() //
                .setForeman(foremanNode) //
                .setFragmentJson(plan) //
                .setHandle(handle) //
                .setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
                .setLeafFragment(isLeafFragment) //
                .setContext(queryContextInfo)
                .setMemInitial(wrapper.getInitialAllocation()) //
                .setMemMax(wrapper.getMaxAllocation())
                .setOptionsJson(optionsData)
                .setCredentials(session.getCredentials())
                .addAllCollector(CountRequiredFragments.getCollectors(root))
                .build();

        if (isRootNode) {
          logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
          rootFragment = fragment;
          rootOperator = root;
        } else {
          logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
          fragments.add(fragment);
        }
      }
    }

    return new QueryWorkUnit(rootOperator, rootFragment, fragments);
  }