@Override public Void visitSendingExchange(Exchange exchange, Wrapper wrapper) throws RuntimeException { Stats stats = wrapper.getStats(); stats.addCost(exchange.getAggregateSendCost()); stats.addMaxWidth(exchange.getMaxSendWidth()); return super.visitSendingExchange(exchange, wrapper); }
/** * Helper method for parallelizing a given fragment. Dependent fragments are parallelized first * before parallelizing the given fragment. */ private void parallelizeFragment( Wrapper fragmentWrapper, PlanningSet planningSet, Collection<DrillbitEndpoint> activeEndpoints) throws PhysicalOperatorSetupException { // If the fragment is already parallelized, return. if (fragmentWrapper.isEndpointsAssignmentDone()) { return; } // First parallelize fragments on which this fragment depends on. final List<Wrapper> fragmentDependencies = fragmentWrapper.getFragmentDependencies(); if (fragmentDependencies != null && fragmentDependencies.size() > 0) { for (Wrapper dependency : fragmentDependencies) { parallelizeFragment(dependency, planningSet, activeEndpoints); } } // Find stats. Stats include various factors including cost of physical operators, // parallelizability of // work in physical operator and affinity of physical operator to certain nodes. fragmentWrapper.getNode().getRoot().accept(new StatsCollector(planningSet), fragmentWrapper); fragmentWrapper .getStats() .getDistributionAffinity() .getFragmentParallelizer() .parallelizeFragment(fragmentWrapper, this, activeEndpoints); }
@Override public Void visitOp(PhysicalOperator op, Wrapper wrapper) { if (op instanceof HasAffinity) { wrapper.addEndpointAffinity(((HasAffinity) op).getOperatorAffinity()); } Stats stats = wrapper.getStats(); stats.addCost(op.getCost()); for (PhysicalOperator child : op) { child.accept(this, wrapper); } return null; }
private void assignEndpoints( Collection<DrillbitEndpoint> allNodes, PlanningSet planningSet, int globalMaxWidth, int maxWidthPerEndpoint) throws PhysicalOperatorSetupException { // First we determine the amount of parallelization for a fragment. This will be between 1 and // maxWidth based on // cost. (Later could also be based on cluster operation.) then we decide endpoints based on // affinity (later this // could be based on endpoint load) for (Wrapper wrapper : planningSet) { Stats stats = wrapper.getStats(); // figure out width. int width = Math.min(stats.getMaxWidth(), globalMaxWidth); float diskCost = stats.getDiskCost(); // logger.debug("Frag max width: {} and diskCost: {}", stats.getMaxWidth(), diskCost); // TODO: right now we'll just assume that each task is cost 1 so we'll set the breadth at the // lesser of the number // of tasks or the maximum width of the fragment. if (diskCost < width) { // width = (int) diskCost; } width = Math.min(width, maxWidthPerEndpoint * allNodes.size()); if (width < 1) width = 1; // logger.debug("Setting width {} on fragment {}", width, wrapper); wrapper.setWidth(width); // figure out endpoint assignments. also informs the exchanges about their respective // endpoints. wrapper.assignEndpoints(allNodes); } }
@Override public Void visitStore(Store store, Wrapper wrapper) { Stats stats = wrapper.getStats(); stats.addMaxWidth(store.getMaxWidth()); return super.visitStore(store, wrapper); }
@Override public Void visitGroupScan(GroupScan groupScan, Wrapper wrapper) { Stats stats = wrapper.getStats(); stats.addMaxWidth(groupScan.getMaxParallelizationWidth()); return super.visitGroupScan(groupScan, wrapper); }
@Override public Void visitReceivingExchange(Exchange exchange, Wrapper wrapper) throws RuntimeException { wrapper.getStats().addCost(exchange.getAggregateReceiveCost()); // no traversal since it would cross fragment boundary. return null; }