private FragmentWritableBatch(
boolean isLast,
QueryId queryId,
int sendMajorFragmentId,
int sendMinorFragmentId,
int receiveMajorFragmentId,
int receiveMinorFragmentId,
RecordBatchDef def,
ByteBuf... buffers) {
this.buffers = buffers;
FragmentHandle handle =
FragmentHandle //
.newBuilder() //
.setMajorFragmentId(receiveMajorFragmentId) //
.setMinorFragmentId(receiveMinorFragmentId) //
.setQueryId(queryId) //
.build();
this.header =
FragmentRecordBatch //
.newBuilder() //
.setIsLastBatch(isLast) //
.setDef(def) //
.setHandle(handle) //
.setSendingMajorFragmentId(sendMajorFragmentId) //
.setSendingMinorFragmentId(sendMinorFragmentId) //
.build();
}
private QueryWorkUnit generateWorkUnit(
DrillbitEndpoint foremanNode,
QueryId queryId,
PhysicalPlanReader reader,
Fragment rootNode,
PlanningSet planningSet)
throws ExecutionSetupException {
List<PlanFragment> fragments = Lists.newArrayList();
PlanFragment rootFragment = null;
FragmentRoot rootOperator = null;
long queryStartTime = System.currentTimeMillis();
// now we generate all the individual plan fragments and associated assignments. Note, we need
// all endpoints
// assigned before we can materialize, so we start a new loop here rather than utilizing the
// previous one.
for (Wrapper wrapper : planningSet) {
Fragment node = wrapper.getNode();
Stats stats = node.getStats();
final PhysicalOperator physicalOperatorRoot = node.getRoot();
boolean isRootNode = rootNode == node;
if (isRootNode && wrapper.getWidth() != 1)
throw new FragmentSetupException(
String.format(
"Failure while trying to setup fragment. The root fragment must always have parallelization one. In the current case, the width was set to %d.",
wrapper.getWidth()));
// a fragment is self driven if it doesn't rely on any other exchanges.
boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
// Create a minorFragment for each major fragment.
for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
PhysicalOperator op = physicalOperatorRoot.accept(materializer, iNode);
Preconditions.checkArgument(op instanceof FragmentRoot);
FragmentRoot root = (FragmentRoot) op;
// get plan as JSON
String plan;
try {
plan = reader.writeJson(root);
} catch (JsonProcessingException e) {
throw new FragmentSetupException(
"Failure while trying to convert fragment into json.", e);
}
FragmentHandle handle =
FragmentHandle //
.newBuilder() //
.setMajorFragmentId(wrapper.getMajorFragmentId()) //
.setMinorFragmentId(minorFragmentId) //
.setQueryId(queryId) //
.build();
PlanFragment fragment =
PlanFragment.newBuilder() //
.setCpuCost(stats.getCpuCost()) //
.setDiskCost(stats.getDiskCost()) //
.setForeman(foremanNode) //
.setMemoryCost(stats.getMemoryCost()) //
.setNetworkCost(stats.getNetworkCost()) //
.setFragmentJson(plan) //
.setHandle(handle) //
.setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
.setLeafFragment(isLeafFragment) //
.setQueryStartTime(queryStartTime)
.build();
if (isRootNode) {
logger.debug("Root fragment:\n {}", fragment);
rootFragment = fragment;
rootOperator = root;
} else {
logger.debug("Remote fragment:\n {}", fragment);
fragments.add(fragment);
}
}
}
return new QueryWorkUnit(rootOperator, rootFragment, fragments);
}
protected QueryWorkUnit generateWorkUnit(
OptionList options,
DrillbitEndpoint foremanNode,
QueryId queryId,
PhysicalPlanReader reader,
Fragment rootNode,
PlanningSet planningSet,
UserSession session,
QueryContextInformation queryContextInfo)
throws ExecutionSetupException {
List<PlanFragment> fragments = Lists.newArrayList();
PlanFragment rootFragment = null;
FragmentRoot rootOperator = null;
// now we generate all the individual plan fragments and associated assignments. Note, we need
// all endpoints
// assigned before we can materialize, so we start a new loop here rather than utilizing the
// previous one.
for (Wrapper wrapper : planningSet) {
Fragment node = wrapper.getNode();
final PhysicalOperator physicalOperatorRoot = node.getRoot();
boolean isRootNode = rootNode == node;
if (isRootNode && wrapper.getWidth() != 1) {
throw new ForemanSetupException(
String.format(
"Failure while trying to setup fragment. "
+ "The root fragment must always have parallelization one. In the current case, the width was set to %d.",
wrapper.getWidth()));
}
// a fragment is self driven if it doesn't rely on any other exchanges.
boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
// Create a minorFragment for each major fragment.
for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
wrapper.resetAllocation();
PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode);
Preconditions.checkArgument(op instanceof FragmentRoot);
FragmentRoot root = (FragmentRoot) op;
// get plan as JSON
String plan;
String optionsData;
try {
plan = reader.writeJson(root);
optionsData = reader.writeJson(options);
} catch (JsonProcessingException e) {
throw new ForemanSetupException("Failure while trying to convert fragment into json.", e);
}
FragmentHandle handle =
FragmentHandle //
.newBuilder() //
.setMajorFragmentId(wrapper.getMajorFragmentId()) //
.setMinorFragmentId(minorFragmentId) //
.setQueryId(queryId) //
.build();
PlanFragment fragment =
PlanFragment.newBuilder() //
.setForeman(foremanNode) //
.setFragmentJson(plan) //
.setHandle(handle) //
.setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
.setLeafFragment(isLeafFragment) //
.setContext(queryContextInfo)
.setMemInitial(wrapper.getInitialAllocation()) //
.setMemMax(wrapper.getMaxAllocation())
.setOptionsJson(optionsData)
.setCredentials(session.getCredentials())
.addAllCollector(CountRequiredFragments.getCollectors(root))
.build();
if (isRootNode) {
logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
rootFragment = fragment;
rootOperator = root;
} else {
logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
fragments.add(fragment);
}
}
}
return new QueryWorkUnit(rootOperator, rootFragment, fragments);
}
