private RunningJob submitAction(Context context, Namespace ns) throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
WorkflowAction action = context.getAction();
ae.prepareActionDir(getFileSystem(), context);
ae.submitLauncher(getFileSystem(), context, action);
String jobId = action.getExternalId();
String jobTracker = action.getTrackerUri();
String consoleUrl = action.getConsoleUrl();
assertNotNull(jobId);
assertNotNull(jobTracker);
assertNotNull(consoleUrl);
Element e = XmlUtils.parseXml(action.getConf());
XConfiguration conf =
new XConfiguration(
new StringReader(XmlUtils.prettyPrint(e.getChild("configuration", ns)).toString()));
conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker", ns));
conf.set("fs.default.name", e.getChildTextTrim("name-node", ns));
conf.set("user.name", context.getProtoActionConf().get("user.name"));
conf.set("group.name", getTestGroup());
JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker);
XConfiguration.copy(conf, jobConf);
String user = jobConf.get("user.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, jobConf);
final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
assertNotNull(runningJob);
return runningJob;
}
private void copyTTlogs(FileSystem fs, Path path, WorkflowAction action) throws Exception {
List<String> ttLogUrls = getTTlogURL(action.getExternalId());
if (ttLogUrls != null) {
int index = 1;
for (String ttLogURL : ttLogUrls) {
LOG.info("Fetching log for action: {} from url: {}", action.getExternalId(), ttLogURL);
InputStream in = getURLinputStream(new URL(ttLogURL));
OutputStream out =
fs.create(
new Path(
path,
action.getName()
+ "_"
+ action.getType()
+ "_"
+ getMappedStatus(action.getStatus())
+ "-"
+ index
+ ".log"));
IOUtils.copyBytes(in, out, 4096, true);
LOG.info("Copied log to {}", path);
index++;
}
}
}
public static WorkflowJob.Status queryJobStatus(String oozieJobid)
throws OozieClientException, InterruptedException {
OozieClient wc = new AuthOozieClient("http://10.25.22.19:8080/oozie", "ldap");
wc.setHeader("j_username", "V_PA011_HADOOP_CORE");
wc.setHeader("j_password", "{DES}/+J5e0Ed2/PABTqzeVGQ1g==");
List<WorkflowAction> walist;
int wacount = 0;
while (wacount < 2) {
walist = wc.getJobInfo(oozieJobid).getActions();
wacount = walist.size();
Thread.sleep(20000);
System.out.println("#######-" + wacount);
for (WorkflowAction wa : walist) {
System.out.println("#######" + wa.getName());
if (wa.getConsoleUrl().contains("jobdetails.jsp?jobid=")) {
System.out.println("#######" + wa.getConsoleUrl());
}
}
}
return wc.getJobInfo(oozieJobid).getStatus();
}
public int run(WorkflowExecutionContext context) {
try {
OozieClient client = new OozieClient(context.getWorkflowEngineUrl());
WorkflowJob jobInfo;
try {
jobInfo = client.getJobInfo(context.getUserSubflowId());
} catch (OozieClientException e) {
LOG.error("Error getting jobinfo for: {}", context.getUserSubflowId(), e);
return 0;
}
// Assumption is - Each wf run will have a directory
// the corresponding job logs are stored within the respective dir
Path path =
new Path(context.getLogDir() + "/" + String.format("%03d", context.getWorkflowRunId()));
FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri(), getConf());
if (EntityType.FEED.name().equalsIgnoreCase(context.getEntityType())
|| notUserWorkflowEngineIsOozie(context.getUserWorkflowEngine())) {
// if replication wf, retention wf or PIG Process
copyOozieLog(client, fs, path, jobInfo.getId());
List<WorkflowAction> workflowActions = jobInfo.getActions();
for (int i = 0; i < workflowActions.size(); i++) {
if (FALCON_ACTIONS.contains(workflowActions.get(i).getName())) {
copyTTlogs(fs, path, jobInfo.getActions().get(i));
break;
}
}
} else {
String flowId;
// if process wf with pig, hive
if (context.getUserWorkflowEngine().equals("pig")
|| context.getUserWorkflowEngine().equals("hive")) {
flowId = jobInfo.getId();
} else {
// if process wf with oozie engine
flowId = jobInfo.getExternalId();
}
copyOozieLog(client, fs, path, flowId);
WorkflowJob subflowInfo = client.getJobInfo(flowId);
List<WorkflowAction> actions = subflowInfo.getActions();
for (WorkflowAction action : actions) {
if (isActionTypeSupported(action)) {
LOG.info(
"Copying hadoop TT log for action: {} of type: {}",
action.getName(),
action.getType());
copyTTlogs(fs, path, action);
} else {
LOG.info(
"Ignoring hadoop TT log for non supported action: {} of type: {}",
action.getName(),
action.getType());
}
}
}
} catch (Exception e) {
// JobLogMover doesn't throw exception, a failed log mover will not fail the user workflow
LOG.error("Exception in log mover:", e);
}
return 0;
}
private boolean isActionTypeSupported(WorkflowAction action) {
return action.getType().equals("pig")
|| action.getType().equals("hive")
|| action.getType().equals("java")
|| action.getType().equals("map-reduce");
}