private static int executeCmd(String[] args, String outFile, String errFile) throws Exception {
LOG.info("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
PrintStream out =
outFile == null
? SessionState.getConsole().getChildOutStream()
: new PrintStream(new FileOutputStream(outFile), true);
PrintStream err =
errFile == null
? SessionState.getConsole().getChildErrStream()
: new PrintStream(new FileOutputStream(errFile), true);
Process executor = Runtime.getRuntime().exec(args);
StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out);
outPrinter.start();
errPrinter.start();
int result = executor.waitFor();
outPrinter.join();
errPrinter.join();
if (outFile != null) {
out.close();
}
if (errFile != null) {
err.close();
}
return result;
}
@Override
public int execute(DriverContext driverContext) {
Context ctx = driverContext.getCtx();
boolean ctxCreated = false;
try {
if (ctx == null) {
ctx = new Context(conf);
ctxCreated = true;
}
// estimate number of reducers
setNumberOfReducers();
// auto-determine local mode if allowed
if (!ctx.isLocalOnlyExecutionMode() && conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO)) {
if (inputSummary == null) {
inputSummary = Utilities.getInputSummary(driverContext.getCtx(), work, null);
}
// set the values of totalInputFileSize and totalInputNumFiles, estimating them
// if percentage block sampling is being used
estimateInputSize();
// at this point the number of reducers is precisely defined in the plan
int numReducers = work.getNumReduceTasks();
if (LOG.isDebugEnabled()) {
LOG.debug(
"Task: "
+ getId()
+ ", Summary: "
+ totalInputFileSize
+ ","
+ totalInputNumFiles
+ ","
+ numReducers);
}
String reason =
MapRedTask.isEligibleForLocalMode(
conf, numReducers, totalInputFileSize, totalInputNumFiles);
if (reason == null) {
// clone configuration before modifying it on per-task basis
cloneConf();
conf.setVar(HiveConf.ConfVars.HADOOPJT, "local");
console.printInfo("Selecting local mode for task: " + getId());
this.setLocalMode(true);
} else {
console.printInfo("Cannot run job locally: " + reason);
this.setLocalMode(false);
}
}
runningViaChild =
"local".equals(conf.getVar(HiveConf.ConfVars.HADOOPJT))
|| conf.getBoolVar(HiveConf.ConfVars.SUBMITVIACHILD);
if (!runningViaChild) {
// we are not running this mapred task via child jvm
// so directly invoke ExecDriver
return super.execute(driverContext);
}
// we need to edit the configuration to setup cmdline. clone it first
cloneConf();
// propagate input format if necessary
super.setInputAttributes(conf);
// enable assertion
String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN);
String hiveJar = conf.getJar();
String libJarsOption;
String addedJars = Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR);
conf.setVar(ConfVars.HIVEADDEDJARS, addedJars);
String auxJars = conf.getAuxJars();
// Put auxjars and addedjars together into libjars
if (StringUtils.isEmpty(addedJars)) {
if (StringUtils.isEmpty(auxJars)) {
libJarsOption = " ";
} else {
libJarsOption = " -libjars " + auxJars + " ";
}
} else {
if (StringUtils.isEmpty(auxJars)) {
libJarsOption = " -libjars " + addedJars + " ";
} else {
libJarsOption = " -libjars " + addedJars + "," + auxJars + " ";
}
}
// Generate the hiveConfArgs after potentially adding the jars
String hiveConfArgs = generateCmdLine(conf);
// write out the plan to a local file
Path planPath = new Path(ctx.getLocalTmpFileURI(), "plan.xml");
OutputStream out = FileSystem.getLocal(conf).create(planPath);
MapredWork plan = getWork();
LOG.info("Generating plan file " + planPath.toString());
Utilities.serializeMapRedWork(plan, out);
String isSilent = "true".equalsIgnoreCase(System.getProperty("test.silent")) ? "-nolog" : "";
String jarCmd;
if (ShimLoader.getHadoopShims().usesJobShell()) {
jarCmd = libJarsOption + hiveJar + " " + ExecDriver.class.getName();
} else {
jarCmd = hiveJar + " " + ExecDriver.class.getName() + libJarsOption;
}
String cmdLine =
hadoopExec
+ " jar "
+ jarCmd
+ " -plan "
+ planPath.toString()
+ " "
+ isSilent
+ " "
+ hiveConfArgs;
String workDir = (new File(".")).getCanonicalPath();
String files = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE);
if (!files.isEmpty()) {
cmdLine = cmdLine + " -files " + files;
workDir = (new Path(ctx.getLocalTmpFileURI())).toUri().getPath();
if (!(new File(workDir)).mkdir()) {
throw new IOException("Cannot create tmp working dir: " + workDir);
}
for (String f : StringUtils.split(files, ',')) {
Path p = new Path(f);
String target = p.toUri().getPath();
String link = workDir + Path.SEPARATOR + p.getName();
if (FileUtil.symLink(target, link) != 0) {
throw new IOException("Cannot link to added file: " + target + " from: " + link);
}
}
}
LOG.info("Executing: " + cmdLine);
Process executor = null;
// Inherit Java system variables
String hadoopOpts;
StringBuilder sb = new StringBuilder();
Properties p = System.getProperties();
for (String element : HIVE_SYS_PROP) {
if (p.containsKey(element)) {
sb.append(" -D" + element + "=" + p.getProperty(element));
}
}
hadoopOpts = sb.toString();
// Inherit the environment variables
String[] env;
Map<String, String> variables = new HashMap(System.getenv());
// The user can specify the hadoop memory
if ("local".equals(conf.getVar(HiveConf.ConfVars.HADOOPJT))) {
// if we are running in local mode - then the amount of memory used
// by the child jvm can no longer default to the memory used by the
// parent jvm
int hadoopMem = conf.getIntVar(HiveConf.ConfVars.HIVEHADOOPMAXMEM);
if (hadoopMem == 0) {
// remove env var that would default child jvm to use parent's memory
// as default. child jvm would use default memory for a hadoop client
variables.remove(HADOOP_MEM_KEY);
} else {
// user specified the memory for local mode hadoop run
variables.put(HADOOP_MEM_KEY, String.valueOf(hadoopMem));
}
} else {
// nothing to do - we are not running in local mode - only submitting
// the job via a child process. in this case it's appropriate that the
// child jvm use the same memory as the parent jvm
}
if (variables.containsKey(HADOOP_OPTS_KEY)) {
variables.put(HADOOP_OPTS_KEY, variables.get(HADOOP_OPTS_KEY) + hadoopOpts);
} else {
variables.put(HADOOP_OPTS_KEY, hadoopOpts);
}
if (variables.containsKey(HIVE_DEBUG_RECURSIVE)) {
configureDebugVariablesForChildJVM(variables);
}
env = new String[variables.size()];
int pos = 0;
for (Map.Entry<String, String> entry : variables.entrySet()) {
String name = entry.getKey();
String value = entry.getValue();
env[pos++] = name + "=" + value;
}
// Run ExecDriver in another JVM
executor = Runtime.getRuntime().exec(cmdLine, env, new File(workDir));
StreamPrinter outPrinter =
new StreamPrinter(
executor.getInputStream(), null, SessionState.getConsole().getChildOutStream());
StreamPrinter errPrinter =
new StreamPrinter(
executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream());
outPrinter.start();
errPrinter.start();
int exitVal = jobExecHelper.progressLocal(executor, getId());
if (exitVal != 0) {
LOG.error("Execution failed with exit status: " + exitVal);
} else {
LOG.info("Execution completed successfully");
}
return exitVal;
} catch (Exception e) {
e.printStackTrace();
LOG.error("Exception: " + e.getMessage());
return (1);
} finally {
try {
// creating the context can create a bunch of files. So make
// sure to clear it out
if (ctxCreated) {
ctx.clear();
}
} catch (Exception e) {
LOG.error("Exception: " + e.getMessage());
}
}
}
public void run(
SessionState sess,
Set<ReadEntity> inputs,
Set<WriteEntity> outputs,
LineageInfo linfo,
UserGroupInformation ugi)
throws Exception {
LogHelper console = SessionState.getConsole();
if (console == null) {
return;
}
if (sess != null) {
console.printError("POSTHOOK: query: " + sess.getCmd().trim());
console.printError("POSTHOOK: type: " + sess.getCommandType());
}
PreExecutePrinter.printEntities(console, inputs, "POSTHOOK: Input: ");
PreExecutePrinter.printEntities(console, outputs, "POSTHOOK: Output: ");
// Also print out the generic lineage information if there is any
if (linfo != null) {
LinkedList<Map.Entry<DependencyKey, Dependency>> entry_list =
new LinkedList<Map.Entry<DependencyKey, Dependency>>(linfo.entrySet());
Collections.sort(entry_list, new DependencyKeyComp());
Iterator<Map.Entry<DependencyKey, Dependency>> iter = entry_list.iterator();
while (iter.hasNext()) {
Map.Entry<DependencyKey, Dependency> it = iter.next();
Dependency dep = it.getValue();
DependencyKey depK = it.getKey();
if (dep == null) {
continue;
}
StringBuilder sb = new StringBuilder();
sb.append("POSTHOOK: Lineage: ");
if (depK.getDataContainer().isPartition()) {
Partition part = depK.getDataContainer().getPartition();
sb.append(part.getTableName());
sb.append(" PARTITION(");
int i = 0;
for (FieldSchema fs : depK.getDataContainer().getTable().getPartitionKeys()) {
if (i != 0) {
sb.append(",");
}
sb.append(fs.getName() + "=" + part.getValues().get(i++));
}
sb.append(")");
} else {
sb.append(depK.getDataContainer().getTable().getTableName());
}
sb.append("." + depK.getFieldSchema().getName() + " " + dep.getType() + " ");
sb.append("[");
for (BaseColumnInfo col : dep.getBaseCols()) {
sb.append(
"("
+ col.getTabAlias().getTable().getTableName()
+ ")"
+ col.getTabAlias().getAlias()
+ "."
+ col.getColumn()
+ ", ");
}
sb.append("]");
console.printError(sb.toString());
}
}
}
