@Override public Void visitOp(PhysicalOperator op, List<Collector> collectors) throws RuntimeException { for (PhysicalOperator o : op) { o.accept(this, collectors); } return null; }
private QueryWorkUnit getQueryWorkUnit(final PhysicalPlan plan) throws ExecutionSetupException { final PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next(); final Fragment rootFragment = rootOperator.accept(MakeFragmentsVisitor.INSTANCE, null); final SimpleParallelizer parallelizer = new SimpleParallelizer(queryContext); final QueryWorkUnit queryWorkUnit = parallelizer.getFragments( queryContext.getOptions().getOptionList(), queryContext.getCurrentEndpoint(), queryId, queryContext.getActiveEndpoints(), drillbitContext.getPlanReader(), rootFragment, initiatingClient.getSession(), queryContext.getQueryContextInfo()); if (logger.isTraceEnabled()) { final StringBuilder sb = new StringBuilder(); sb.append("PlanFragments for query "); sb.append(queryId); sb.append('\n'); final List<PlanFragment> planFragments = queryWorkUnit.getFragments(); final int fragmentCount = planFragments.size(); int fragmentIndex = 0; for (final PlanFragment planFragment : planFragments) { final FragmentHandle fragmentHandle = planFragment.getHandle(); sb.append("PlanFragment("); sb.append(++fragmentIndex); sb.append('/'); sb.append(fragmentCount); sb.append(") major_fragment_id "); sb.append(fragmentHandle.getMajorFragmentId()); sb.append(" minor_fragment_id "); sb.append(fragmentHandle.getMinorFragmentId()); sb.append('\n'); final DrillbitEndpoint endpointAssignment = planFragment.getAssignment(); sb.append(" DrillbitEndpoint address "); sb.append(endpointAssignment.getAddress()); sb.append('\n'); String jsonString = "<<malformed JSON>>"; sb.append(" fragment_json: "); final ObjectMapper objectMapper = new ObjectMapper(); try { final Object json = objectMapper.readValue(planFragment.getFragmentJson(), Object.class); jsonString = objectMapper.defaultPrettyPrintingWriter().writeValueAsString(json); } catch (final Exception e) { // we've already set jsonString to a fallback value } sb.append(jsonString); logger.trace(sb.toString()); } } return queryWorkUnit; }
public Fragment getRootFragment(PhysicalPlanReader reader, String file) throws FragmentSetupException, IOException { MakeFragmentsVisitor f = new MakeFragmentsVisitor(); PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8)); PhysicalOperator o = plan.getSortedOperators(false).iterator().next(); return o.accept(f, null); }
@Override public Void visitOp(PhysicalOperator op, Collection<PhysicalOperator> collection) throws RuntimeException { collection.add(op); for (PhysicalOperator o : op) { o.accept(this, collection); } return null; }
@Override public Void visitOp(PhysicalOperator op, Wrapper wrapper) { if (op instanceof HasAffinity) { wrapper.addEndpointAffinity(((HasAffinity) op).getOperatorAffinity()); } Stats stats = wrapper.getStats(); stats.addCost(op.getCost()); for (PhysicalOperator child : op) { child.accept(this, wrapper); } return null; }
private QueryWorkUnit generateWorkUnit( DrillbitEndpoint foremanNode, QueryId queryId, PhysicalPlanReader reader, Fragment rootNode, PlanningSet planningSet) throws ExecutionSetupException { List<PlanFragment> fragments = Lists.newArrayList(); PlanFragment rootFragment = null; FragmentRoot rootOperator = null; long queryStartTime = System.currentTimeMillis(); // now we generate all the individual plan fragments and associated assignments. Note, we need // all endpoints // assigned before we can materialize, so we start a new loop here rather than utilizing the // previous one. for (Wrapper wrapper : planningSet) { Fragment node = wrapper.getNode(); Stats stats = node.getStats(); final PhysicalOperator physicalOperatorRoot = node.getRoot(); boolean isRootNode = rootNode == node; if (isRootNode && wrapper.getWidth() != 1) throw new FragmentSetupException( String.format( "Failure while trying to setup fragment. The root fragment must always have parallelization one. In the current case, the width was set to %d.", wrapper.getWidth())); // a fragment is self driven if it doesn't rely on any other exchanges. boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0; // Create a minorFragment for each major fragment. for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) { IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper); PhysicalOperator op = physicalOperatorRoot.accept(materializer, iNode); Preconditions.checkArgument(op instanceof FragmentRoot); FragmentRoot root = (FragmentRoot) op; // get plan as JSON String plan; try { plan = reader.writeJson(root); } catch (JsonProcessingException e) { throw new FragmentSetupException( "Failure while trying to convert fragment into json.", e); } FragmentHandle handle = FragmentHandle // .newBuilder() // .setMajorFragmentId(wrapper.getMajorFragmentId()) // .setMinorFragmentId(minorFragmentId) // .setQueryId(queryId) // .build(); PlanFragment fragment = PlanFragment.newBuilder() // .setCpuCost(stats.getCpuCost()) // .setDiskCost(stats.getDiskCost()) // .setForeman(foremanNode) // .setMemoryCost(stats.getMemoryCost()) // .setNetworkCost(stats.getNetworkCost()) // .setFragmentJson(plan) // .setHandle(handle) // .setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) // .setLeafFragment(isLeafFragment) // .setQueryStartTime(queryStartTime) .build(); if (isRootNode) { logger.debug("Root fragment:\n {}", fragment); rootFragment = fragment; rootOperator = root; } else { logger.debug("Remote fragment:\n {}", fragment); fragments.add(fragment); } } } return new QueryWorkUnit(rootOperator, rootFragment, fragments); }
private void runPhysicalPlan(PhysicalPlan plan) { if (plan.getProperties().resultMode != ResultMode.EXEC) { fail( String.format( "Failure running plan. You requested a result mode of %s and a physical plan can only be output as EXEC", plan.getProperties().resultMode), new Exception()); } PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next(); MakeFragmentsVisitor makeFragmentsVisitor = new MakeFragmentsVisitor(); Fragment rootFragment; try { rootFragment = rootOperator.accept(makeFragmentsVisitor, null); } catch (FragmentSetupException e) { fail("Failure while fragmenting query.", e); return; } PlanningSet planningSet = StatsCollector.collectStats(rootFragment); SimpleParallelizer parallelizer = new SimpleParallelizer(); try { QueryWorkUnit work = parallelizer.getFragments( context.getCurrentEndpoint(), queryId, context.getActiveEndpoints(), context.getPlanReader(), rootFragment, planningSet, context.getConfig().getInt(ExecConstants.GLOBAL_MAX_WIDTH), context.getConfig().getInt(ExecConstants.MAX_WIDTH_PER_ENDPOINT)); this.context .getWorkBus() .setFragmentStatusListener( work.getRootFragment().getHandle().getQueryId(), fragmentManager); List<PlanFragment> leafFragments = Lists.newArrayList(); List<PlanFragment> intermediateFragments = Lists.newArrayList(); // store fragments in distributed grid. logger.debug("Storing fragments"); for (PlanFragment f : work.getFragments()) { // store all fragments in grid since they are part of handshake. context.getCache().storeFragment(f); if (f.getLeafFragment()) { leafFragments.add(f); } else { intermediateFragments.add(f); } } logger.debug("Fragments stored."); logger.debug("Submitting fragments to run."); fragmentManager.runFragments( bee, work.getRootFragment(), work.getRootOperator(), initiatingClient, leafFragments, intermediateFragments); logger.debug("Fragments running."); } catch (ExecutionSetupException | RpcException e) { fail("Failure while setting up query.", e); } }
public static List<PhysicalOperator> getPops(PhysicalOperator root) { List<PhysicalOperator> ops = Lists.newArrayList(); PopCollector c = new PopCollector(); root.accept(c, ops); return ops; }
public static List<Collector> getCollectors(PhysicalOperator root) { List<Collector> collectors = Lists.newArrayList(); root.accept(INSTANCE, collectors); return collectors; }
protected QueryWorkUnit generateWorkUnit( OptionList options, DrillbitEndpoint foremanNode, QueryId queryId, PhysicalPlanReader reader, Fragment rootNode, PlanningSet planningSet, UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException { List<PlanFragment> fragments = Lists.newArrayList(); PlanFragment rootFragment = null; FragmentRoot rootOperator = null; // now we generate all the individual plan fragments and associated assignments. Note, we need // all endpoints // assigned before we can materialize, so we start a new loop here rather than utilizing the // previous one. for (Wrapper wrapper : planningSet) { Fragment node = wrapper.getNode(); final PhysicalOperator physicalOperatorRoot = node.getRoot(); boolean isRootNode = rootNode == node; if (isRootNode && wrapper.getWidth() != 1) { throw new ForemanSetupException( String.format( "Failure while trying to setup fragment. " + "The root fragment must always have parallelization one. In the current case, the width was set to %d.", wrapper.getWidth())); } // a fragment is self driven if it doesn't rely on any other exchanges. boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0; // Create a minorFragment for each major fragment. for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) { IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper); wrapper.resetAllocation(); PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode); Preconditions.checkArgument(op instanceof FragmentRoot); FragmentRoot root = (FragmentRoot) op; // get plan as JSON String plan; String optionsData; try { plan = reader.writeJson(root); optionsData = reader.writeJson(options); } catch (JsonProcessingException e) { throw new ForemanSetupException("Failure while trying to convert fragment into json.", e); } FragmentHandle handle = FragmentHandle // .newBuilder() // .setMajorFragmentId(wrapper.getMajorFragmentId()) // .setMinorFragmentId(minorFragmentId) // .setQueryId(queryId) // .build(); PlanFragment fragment = PlanFragment.newBuilder() // .setForeman(foremanNode) // .setFragmentJson(plan) // .setHandle(handle) // .setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) // .setLeafFragment(isLeafFragment) // .setContext(queryContextInfo) .setMemInitial(wrapper.getInitialAllocation()) // .setMemMax(wrapper.getMaxAllocation()) .setOptionsJson(optionsData) .setCredentials(session.getCredentials()) .addAllCollector(CountRequiredFragments.getCollectors(root)) .build(); if (isRootNode) { logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString())); rootFragment = fragment; rootOperator = root; } else { logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString())); fragments.add(fragment); } } } return new QueryWorkUnit(rootOperator, rootFragment, fragments); }