/** Configures the Reduce plan, the POPackage operator and the reporter thread */
@SuppressWarnings("unchecked")
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
inIllustrator = inIllustrator(context);
if (inIllustrator) pack = getPack(context);
Configuration jConf = context.getConfiguration();
SpillableMemoryManager.configure(ConfigurationUtil.toProperties(jConf));
context
.getConfiguration()
.set(
PigConstants.TASK_INDEX,
Integer.toString(context.getTaskAttemptID().getTaskID().getId()));
sJobContext = context;
sJobConfInternal.set(context.getConfiguration());
sJobConf = context.getConfiguration();
try {
PigContext.setPackageImportList(
(ArrayList<String>) ObjectSerializer.deserialize(jConf.get("udf.import.list")));
pigContext = (PigContext) ObjectSerializer.deserialize(jConf.get("pig.pigContext"));
// This attempts to fetch all of the generated code from the distributed cache, and resolve
// it
SchemaTupleBackend.initialize(jConf, pigContext);
if (rp == null)
rp = (PhysicalPlan) ObjectSerializer.deserialize(jConf.get("pig.reducePlan"));
stores = PlanHelper.getPhysicalOperators(rp, POStore.class);
if (!inIllustrator)
pack = (POPackage) ObjectSerializer.deserialize(jConf.get("pig.reduce.package"));
// To be removed
if (rp.isEmpty()) log.debug("Reduce Plan empty!");
else {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
rp.explain(baos);
log.debug(baos.toString());
}
pigReporter = new ProgressableReporter();
if (!(rp.isEmpty())) {
roots = rp.getRoots().toArray(new PhysicalOperator[1]);
leaf = rp.getLeaves().get(0);
}
// Get the UDF specific context
MapRedUtil.setupUDFContext(jConf);
} catch (IOException ioe) {
String msg = "Problem while configuring reduce plan.";
throw new RuntimeException(msg, ioe);
}
log.info(
"Aliases being processed per job phase (AliasName[line,offset]): "
+ jConf.get("pig.alias.location"));
Utils.setDefaultTimeZone(PigMapReduce.sJobConfInternal.get());
}
// return: false-more output
// true- end of processing
public boolean processOnePackageOutput(Context oc) throws IOException, InterruptedException {
Result res = pack.getNextTuple();
if (res.returnStatus == POStatus.STATUS_OK) {
Tuple packRes = (Tuple) res.result;
if (rp.isEmpty()) {
oc.write(null, packRes);
return false;
}
for (int i = 0; i < roots.length; i++) {
roots[i].attachInput(packRes);
}
runPipeline(leaf);
}
if (res.returnStatus == POStatus.STATUS_NULL) {
return false;
}
if (res.returnStatus == POStatus.STATUS_ERR) {
int errCode = 2093;
String msg = "Encountered error in package operator while processing group.";
throw new ExecException(msg, errCode, PigException.BUG);
}
if (res.returnStatus == POStatus.STATUS_EOP) {
return true;
}
return false;
}