public void close() {
if (oc == null) {
l4j.trace("Close called. no row processed by map.");
}
try {
mo.close(abort);
if (fetchOperators != null) {
mapredLocalWork localWork = mo.getConf().getMapLocalWork();
for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
Operator<? extends Serializable> forwardOp =
localWork.getAliasToWork().get(entry.getKey());
forwardOp.close(abort);
}
}
if (l4j.isInfoEnabled()) {
long used_memory = memoryMXBean.getHeapMemoryUsage().getUsed();
l4j.info("ExecMapper: processed " + numRows + " rows: used memory = " + used_memory);
}
reportStats rps = new reportStats(rp);
mo.preorderMap(rps);
oc = null;
mo.setOutputCollector(oc);
return;
} catch (Exception e) {
if (!abort) {
l4j.error("Hit error while closing operators - failing tree");
throw new RuntimeException("Error while closing operators", e);
}
}
}
private void processOldMapLocalWork() {
if (fetchOperators != null) {
try {
mapredLocalWork localWork = mo.getConf().getMapLocalWork();
int fetchOpNum = 0;
for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
int fetchOpRows = 0;
String alias = entry.getKey();
FetchOperator fetchOp = entry.getValue();
Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);
while (true) {
InspectableObject row = fetchOp.getNextRow();
if (row == null) {
break;
}
fetchOpRows++;
forwardOp.process(row.o, 0);
}
if (l4j.isInfoEnabled()) {
l4j.info(
"fetch "
+ fetchOpNum++
+ " processed "
+ fetchOpRows
+ " used mem: "
+ memoryMXBean.getHeapMemoryUsage().getUsed());
}
}
} catch (Throwable e) {
abort = true;
if (e instanceof OutOfMemoryError) {
throw (OutOfMemoryError) e;
} else {
throw new RuntimeException("Map local work failed", e);
}
}
}
}
private void processMapLocalWork() {
l4j.info("Begin to process map side computing!");
if (fetchOperators != null) {
try {
int fetchOpNum = 0;
for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
int fetchOpRows = 0;
String alias = entry.getKey();
FetchOperator fetchOp = entry.getValue();
l4j.info("The table processed in the fetch operator: " + alias);
fetchOp.clearFetchContext();
setUpFetchOpContext(fetchOp, alias);
Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(alias);
while (true) {
InspectableObject row = fetchOp.getNextRow();
if (row == null) {
forwardOp.close(false);
break;
}
fetchOpRows++;
forwardOp.process(row.o, 0);
if (forwardOp.getDone()) {
done = true;
break;
}
}
if (l4j.isInfoEnabled()) {
l4j.info(
"fetch "
+ fetchOpNum++
+ " processed "
+ fetchOpRows
+ " used mem: "
+ memoryMXBean.getHeapMemoryUsage().getUsed());
}
}
} catch (Throwable e) {
abort = true;
if (e instanceof OutOfMemoryError) {
throw (OutOfMemoryError) e;
} else {
throw new RuntimeException("Map local work failed", e);
}
}
}
}
public void configure(JobConf job) {
memoryMXBean = ManagementFactory.getMemoryMXBean();
l4j.info("maximum memory = " + memoryMXBean.getHeapMemoryUsage().getMax());
try {
l4j.info(
"conf classpath = " + Arrays.asList(((URLClassLoader) job.getClassLoader()).getURLs()));
l4j.info(
"thread classpath = "
+ Arrays.asList(
((URLClassLoader) Thread.currentThread().getContextClassLoader()).getURLs()));
} catch (Exception e) {
l4j.info("cannot get classpath: " + e.getMessage());
}
try {
jc = job;
int estCountBucketSize = jc.getInt("hive.exec.estdistinct.bucketsize.log", 15);
int estCountBufferSize = jc.getInt("hive.exec.estdistinct.buffsize.log", 8);
GenericUDAFCardinalityEstimation.initParams(estCountBucketSize, estCountBufferSize);
boolean isChangSizeZero2Null =
jc.getBoolean(
HiveConf.ConfVars.HIVE_UDTF_EXPLODE_CHANGE_ZERO_SIZE_2_NULL.varname,
HiveConf.ConfVars.HIVE_UDTF_EXPLODE_CHANGE_ZERO_SIZE_2_NULL.defaultBoolVal);
boolean isChangeNull2Null =
jc.getBoolean(
HiveConf.ConfVars.HIVE_UDTF_EXPLODE_CHANGE_NULL_2_NULL.varname,
HiveConf.ConfVars.HIVE_UDTF_EXPLODE_CHANGE_NULL_2_NULL.defaultBoolVal);
GenericUDTFExplode.isChangSizeZero2Null = isChangSizeZero2Null;
GenericUDTFExplode.isChangNull2Null = isChangeNull2Null;
GenericUDTFPosExplode.isChangSizeZero2Null = isChangSizeZero2Null;
GenericUDTFPosExplode.isChangNull2Null = isChangeNull2Null;
mapredWork mrwork = Utilities.getMapRedWork(job);
mo = new MapOperator();
mo.setConf(mrwork);
mo.setChildren(job);
l4j.info(mo.dump(0));
mo.initialize(jc, null);
localWork = mrwork.getMapLocalWork();
if (localWork == null) {
return;
}
fetchOperators = new HashMap<String, FetchOperator>();
for (Map.Entry<String, fetchWork> entry : localWork.getAliasToFetchWork().entrySet()) {
fetchOperators.put(entry.getKey(), new FetchOperator(entry.getValue(), job));
l4j.info("fetchoperator for " + entry.getKey() + " created");
}
for (Map.Entry<String, FetchOperator> entry : fetchOperators.entrySet()) {
Operator<? extends Serializable> forwardOp = localWork.getAliasToWork().get(entry.getKey());
forwardOp.initialize(
jc, new ObjectInspector[] {entry.getValue().getOutputObjectInspector()});
l4j.info("fetchoperator for " + entry.getKey() + " initialized");
}
} catch (Throwable e) {
abort = true;
if (e instanceof OutOfMemoryError) {
throw (OutOfMemoryError) e;
} else {
throw new RuntimeException("Map operator initialization failed", e);
}
}
}