public ExternalSortBatch(ExternalSort popConfig, FragmentContext context, RecordBatch incoming)
throws OutOfMemoryException {
super(popConfig, context, true);
this.incoming = incoming;
DrillConfig config = context.getConfig();
Configuration conf = new Configuration();
conf.set("fs.default.name", config.getString(ExecConstants.EXTERNAL_SORT_SPILL_FILESYSTEM));
try {
this.fs = FileSystem.get(conf);
} catch (IOException e) {
throw new RuntimeException(e);
}
SPILL_BATCH_GROUP_SIZE = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_GROUP_SIZE);
SPILL_THRESHOLD = config.getInt(ExecConstants.EXTERNAL_SORT_SPILL_THRESHOLD);
dirs = Iterators.cycle(config.getStringList(ExecConstants.EXTERNAL_SORT_SPILL_DIRS));
oAllocator = oContext.getAllocator();
copierAllocator =
oAllocator.newChildAllocator(
oAllocator.getName() + ":copier",
PriorityQueueCopier.INITIAL_ALLOCATION,
PriorityQueueCopier.MAX_ALLOCATION);
FragmentHandle handle = context.getHandle();
fileName =
String.format(
"%s_majorfragment%s_minorfragment%s_operator%s",
QueryIdHelper.getQueryId(handle.getQueryId()),
handle.getMajorFragmentId(),
handle.getMinorFragmentId(),
popConfig.getOperatorId());
}
private QueryWorkUnit getQueryWorkUnit(final PhysicalPlan plan) throws ExecutionSetupException {
final PhysicalOperator rootOperator = plan.getSortedOperators(false).iterator().next();
final Fragment rootFragment = rootOperator.accept(MakeFragmentsVisitor.INSTANCE, null);
final SimpleParallelizer parallelizer = new SimpleParallelizer(queryContext);
final QueryWorkUnit queryWorkUnit =
parallelizer.getFragments(
queryContext.getOptions().getOptionList(),
queryContext.getCurrentEndpoint(),
queryId,
queryContext.getActiveEndpoints(),
drillbitContext.getPlanReader(),
rootFragment,
initiatingClient.getSession(),
queryContext.getQueryContextInfo());
if (logger.isTraceEnabled()) {
final StringBuilder sb = new StringBuilder();
sb.append("PlanFragments for query ");
sb.append(queryId);
sb.append('\n');
final List<PlanFragment> planFragments = queryWorkUnit.getFragments();
final int fragmentCount = planFragments.size();
int fragmentIndex = 0;
for (final PlanFragment planFragment : planFragments) {
final FragmentHandle fragmentHandle = planFragment.getHandle();
sb.append("PlanFragment(");
sb.append(++fragmentIndex);
sb.append('/');
sb.append(fragmentCount);
sb.append(") major_fragment_id ");
sb.append(fragmentHandle.getMajorFragmentId());
sb.append(" minor_fragment_id ");
sb.append(fragmentHandle.getMinorFragmentId());
sb.append('\n');
final DrillbitEndpoint endpointAssignment = planFragment.getAssignment();
sb.append(" DrillbitEndpoint address ");
sb.append(endpointAssignment.getAddress());
sb.append('\n');
String jsonString = "<<malformed JSON>>";
sb.append(" fragment_json: ");
final ObjectMapper objectMapper = new ObjectMapper();
try {
final Object json = objectMapper.readValue(planFragment.getFragmentJson(), Object.class);
jsonString = objectMapper.defaultPrettyPrintingWriter().writeValueAsString(json);
} catch (final Exception e) {
// we've already set jsonString to a fallback value
}
sb.append(jsonString);
logger.trace(sb.toString());
}
}
return queryWorkUnit;
}
private void sendFinalState() {
final FragmentState outcome = fragmentState.get();
if (outcome == FragmentState.FAILED) {
final FragmentHandle handle = getContext().getHandle();
final UserException uex =
UserException.systemError(deferredException.getAndClear())
.addIdentity(getContext().getIdentity())
.addContext(
"Fragment", handle.getMajorFragmentId() + ":" + handle.getMinorFragmentId())
.build(logger);
statusReporter.fail(uex);
} else {
statusReporter.stateChanged(outcome);
}
}
private FragmentWritableBatch(
boolean isLast,
QueryId queryId,
int sendMajorFragmentId,
int sendMinorFragmentId,
int receiveMajorFragmentId,
int receiveMinorFragmentId,
RecordBatchDef def,
ByteBuf... buffers) {
this.buffers = buffers;
FragmentHandle handle =
FragmentHandle //
.newBuilder() //
.setMajorFragmentId(receiveMajorFragmentId) //
.setMinorFragmentId(receiveMinorFragmentId) //
.setQueryId(queryId) //
.build();
this.header =
FragmentRecordBatch //
.newBuilder() //
.setIsLastBatch(isLast) //
.setDef(def) //
.setHandle(handle) //
.setSendingMajorFragmentId(sendMajorFragmentId) //
.setSendingMinorFragmentId(sendMinorFragmentId) //
.build();
}
public RecordWriter getRecordWriter(FragmentContext context, ParquetWriter writer)
throws IOException, OutOfMemoryException {
Map<String, String> options = Maps.newHashMap();
options.put("location", writer.getLocation());
FragmentHandle handle = context.getHandle();
String fragmentId =
String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
options.put("prefix", fragmentId);
options.put(
FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).connection);
options.put(
ExecConstants.PARQUET_BLOCK_SIZE,
context.getOptions().getOption(ExecConstants.PARQUET_BLOCK_SIZE).num_val.toString());
options.put(
ExecConstants.PARQUET_PAGE_SIZE,
context.getOptions().getOption(ExecConstants.PARQUET_PAGE_SIZE).num_val.toString());
options.put(
ExecConstants.PARQUET_DICT_PAGE_SIZE,
context.getOptions().getOption(ExecConstants.PARQUET_DICT_PAGE_SIZE).num_val.toString());
options.put(
ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE,
context.getOptions().getOption(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE).string_val);
options.put(
ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING,
context
.getOptions()
.getOption(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING)
.bool_val
.toString());
RecordWriter recordWriter = new ParquetRecordWriter(context, writer);
recordWriter.init(options);
return recordWriter;
}
@Override
public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer)
throws IOException {
Map<String, String> options = Maps.newHashMap();
options.put("location", writer.getLocation());
FragmentHandle handle = context.getHandle();
String fragmentId =
String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
options.put("prefix", fragmentId);
options.put("separator", ((TextFormatConfig) getConfig()).getDelimiter());
options.put(
FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).connection);
options.put("extension", ((TextFormatConfig) getConfig()).getExtensions().get(0));
RecordWriter recordWriter = new DrillTextRecordWriter(context.getAllocator());
recordWriter.init(options);
return recordWriter;
}
@Test
public void project(
@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection)
throws Exception {
new NonStrictExpectations() {
{
bitContext.getMetrics();
result = new MetricRegistry("test");
bitContext.getAllocator();
result = BufferAllocator.getAllocator(c);
}
};
PhysicalPlanReader reader =
new PhysicalPlanReader(
c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
PhysicalPlan plan =
reader.readPhysicalPlan(
Files.toString(FileUtils.getResourceAsFile("/project/test1.json"), Charsets.UTF_8));
FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
FragmentContext context =
new FragmentContext(
bitContext, FragmentHandle.getDefaultInstance(), connection, null, registry);
SimpleRootExec exec =
new SimpleRootExec(
ImplCreator.getExec(
context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
while (exec.next()) {
BigIntVector c1 =
exec.getValueVectorById(
new SchemaPath("col1", ExpressionPosition.UNKNOWN), BigIntVector.class);
BigIntVector c2 =
exec.getValueVectorById(
new SchemaPath("col2", ExpressionPosition.UNKNOWN), BigIntVector.class);
int x = 0;
BigIntVector.Accessor a1, a2;
a1 = c1.getAccessor();
a2 = c2.getAccessor();
for (int i = 0; i < c1.getAccessor().getValueCount(); i++) {
assertEquals(a1.get(i) + 1, a2.get(i));
x += a1.get(i);
}
System.out.println(x);
}
}
private QueryWorkUnit generateWorkUnit(
DrillbitEndpoint foremanNode,
QueryId queryId,
PhysicalPlanReader reader,
Fragment rootNode,
PlanningSet planningSet)
throws ExecutionSetupException {
List<PlanFragment> fragments = Lists.newArrayList();
PlanFragment rootFragment = null;
FragmentRoot rootOperator = null;
long queryStartTime = System.currentTimeMillis();
// now we generate all the individual plan fragments and associated assignments. Note, we need
// all endpoints
// assigned before we can materialize, so we start a new loop here rather than utilizing the
// previous one.
for (Wrapper wrapper : planningSet) {
Fragment node = wrapper.getNode();
Stats stats = node.getStats();
final PhysicalOperator physicalOperatorRoot = node.getRoot();
boolean isRootNode = rootNode == node;
if (isRootNode && wrapper.getWidth() != 1)
throw new FragmentSetupException(
String.format(
"Failure while trying to setup fragment. The root fragment must always have parallelization one. In the current case, the width was set to %d.",
wrapper.getWidth()));
// a fragment is self driven if it doesn't rely on any other exchanges.
boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
// Create a minorFragment for each major fragment.
for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
PhysicalOperator op = physicalOperatorRoot.accept(materializer, iNode);
Preconditions.checkArgument(op instanceof FragmentRoot);
FragmentRoot root = (FragmentRoot) op;
// get plan as JSON
String plan;
try {
plan = reader.writeJson(root);
} catch (JsonProcessingException e) {
throw new FragmentSetupException(
"Failure while trying to convert fragment into json.", e);
}
FragmentHandle handle =
FragmentHandle //
.newBuilder() //
.setMajorFragmentId(wrapper.getMajorFragmentId()) //
.setMinorFragmentId(minorFragmentId) //
.setQueryId(queryId) //
.build();
PlanFragment fragment =
PlanFragment.newBuilder() //
.setCpuCost(stats.getCpuCost()) //
.setDiskCost(stats.getDiskCost()) //
.setForeman(foremanNode) //
.setMemoryCost(stats.getMemoryCost()) //
.setNetworkCost(stats.getNetworkCost()) //
.setFragmentJson(plan) //
.setHandle(handle) //
.setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
.setLeafFragment(isLeafFragment) //
.setQueryStartTime(queryStartTime)
.build();
if (isRootNode) {
logger.debug("Root fragment:\n {}", fragment);
rootFragment = fragment;
rootOperator = root;
} else {
logger.debug("Remote fragment:\n {}", fragment);
fragments.add(fragment);
}
}
}
return new QueryWorkUnit(rootOperator, rootFragment, fragments);
}
protected QueryWorkUnit generateWorkUnit(
OptionList options,
DrillbitEndpoint foremanNode,
QueryId queryId,
PhysicalPlanReader reader,
Fragment rootNode,
PlanningSet planningSet,
UserSession session,
QueryContextInformation queryContextInfo)
throws ExecutionSetupException {
List<PlanFragment> fragments = Lists.newArrayList();
PlanFragment rootFragment = null;
FragmentRoot rootOperator = null;
// now we generate all the individual plan fragments and associated assignments. Note, we need
// all endpoints
// assigned before we can materialize, so we start a new loop here rather than utilizing the
// previous one.
for (Wrapper wrapper : planningSet) {
Fragment node = wrapper.getNode();
final PhysicalOperator physicalOperatorRoot = node.getRoot();
boolean isRootNode = rootNode == node;
if (isRootNode && wrapper.getWidth() != 1) {
throw new ForemanSetupException(
String.format(
"Failure while trying to setup fragment. "
+ "The root fragment must always have parallelization one. In the current case, the width was set to %d.",
wrapper.getWidth()));
}
// a fragment is self driven if it doesn't rely on any other exchanges.
boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
// Create a minorFragment for each major fragment.
for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
IndexedFragmentNode iNode = new IndexedFragmentNode(minorFragmentId, wrapper);
wrapper.resetAllocation();
PhysicalOperator op = physicalOperatorRoot.accept(Materializer.INSTANCE, iNode);
Preconditions.checkArgument(op instanceof FragmentRoot);
FragmentRoot root = (FragmentRoot) op;
// get plan as JSON
String plan;
String optionsData;
try {
plan = reader.writeJson(root);
optionsData = reader.writeJson(options);
} catch (JsonProcessingException e) {
throw new ForemanSetupException("Failure while trying to convert fragment into json.", e);
}
FragmentHandle handle =
FragmentHandle //
.newBuilder() //
.setMajorFragmentId(wrapper.getMajorFragmentId()) //
.setMinorFragmentId(minorFragmentId) //
.setQueryId(queryId) //
.build();
PlanFragment fragment =
PlanFragment.newBuilder() //
.setForeman(foremanNode) //
.setFragmentJson(plan) //
.setHandle(handle) //
.setAssignment(wrapper.getAssignedEndpoint(minorFragmentId)) //
.setLeafFragment(isLeafFragment) //
.setContext(queryContextInfo)
.setMemInitial(wrapper.getInitialAllocation()) //
.setMemMax(wrapper.getMaxAllocation())
.setOptionsJson(optionsData)
.setCredentials(session.getCredentials())
.addAllCollector(CountRequiredFragments.getCollectors(root))
.build();
if (isRootNode) {
logger.debug("Root fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
rootFragment = fragment;
rootOperator = root;
} else {
logger.debug("Remote fragment:\n {}", DrillStringUtils.unescapeJava(fragment.toString()));
fragments.add(fragment);
}
}
}
return new QueryWorkUnit(rootOperator, rootFragment, fragments);
}
@Override
public void run() {
// if a cancel thread has already entered this executor, we have not reason to continue.
if (!hasCloseoutThread.compareAndSet(false, true)) {
return;
}
final Thread myThread = Thread.currentThread();
myThreadRef.set(myThread);
final String originalThreadName = myThread.getName();
final FragmentHandle fragmentHandle = fragmentContext.getHandle();
final DrillbitContext drillbitContext = fragmentContext.getDrillbitContext();
final ClusterCoordinator clusterCoordinator = drillbitContext.getClusterCoordinator();
final DrillbitStatusListener drillbitStatusListener = new FragmentDrillbitStatusListener();
final String newThreadName = QueryIdHelper.getExecutorThreadName(fragmentHandle);
try {
myThread.setName(newThreadName);
// if we didn't get the root operator when the executor was created, create it now.
final FragmentRoot rootOperator =
this.rootOperator != null
? this.rootOperator
: drillbitContext.getPlanReader().readFragmentOperator(fragment.getFragmentJson());
root = ImplCreator.getExec(fragmentContext, rootOperator);
if (root == null) {
return;
}
clusterCoordinator.addDrillbitStatusListener(drillbitStatusListener);
updateState(FragmentState.RUNNING);
acceptExternalEvents.countDown();
injector.injectPause(fragmentContext.getExecutionControls(), "fragment-running", logger);
final DrillbitEndpoint endpoint = drillbitContext.getEndpoint();
logger.debug(
"Starting fragment {}:{} on {}:{}",
fragmentHandle.getMajorFragmentId(),
fragmentHandle.getMinorFragmentId(),
endpoint.getAddress(),
endpoint.getUserPort());
final UserGroupInformation queryUserUgi =
fragmentContext.isImpersonationEnabled()
? ImpersonationUtil.createProxyUgi(fragmentContext.getQueryUserName())
: ImpersonationUtil.getProcessUserUGI();
queryUserUgi.doAs(
new PrivilegedExceptionAction<Void>() {
public Void run() throws Exception {
injector.injectChecked(
fragmentContext.getExecutionControls(), "fragment-execution", IOException.class);
/*
* Run the query until root.next returns false OR we no longer need to continue.
*/
while (shouldContinue() && root.next()) {
// loop
}
return null;
}
});
} catch (OutOfMemoryError | OutOfMemoryException e) {
if (!(e instanceof OutOfMemoryError) || "Direct buffer memory".equals(e.getMessage())) {
fail(UserException.memoryError(e).build(logger));
} else {
// we have a heap out of memory error. The JVM in unstable, exit.
CatastrophicFailure.exit(
e, "Unable to handle out of memory condition in FragmentExecutor.", -2);
}
} catch (AssertionError | Exception e) {
fail(e);
} finally {
// no longer allow this thread to be interrupted. We synchronize here to make sure that cancel
// can't set an
// interruption after we have moved beyond this block.
synchronized (myThreadRef) {
myThreadRef.set(null);
Thread.interrupted();
}
// We need to sure we countDown at least once. We'll do it here to guarantee that.
acceptExternalEvents.countDown();
// here we could be in FAILED, RUNNING, or CANCELLATION_REQUESTED
cleanup(FragmentState.FINISHED);
clusterCoordinator.removeDrillbitStatusListener(drillbitStatusListener);
myThread.setName(originalThreadName);
}
}