public boolean update(String yarnAppId, String jobId, Map<String, String> tags, AppInfo app) {
String path = this.zkRoot + "/" + yarnAppId + "/" + jobId;
Map<String, String> appInfo = new HashMap<>();
appInfo.put("id", app.getId());
appInfo.put("user", app.getUser());
appInfo.put("name", app.getName());
appInfo.put("queue", app.getQueue());
appInfo.put("state", app.getState());
appInfo.put("finalStatus", app.getFinalStatus());
appInfo.put("progress", app.getProgress() + "");
appInfo.put("trackingUI", app.getTrackingUI());
appInfo.put("diagnostics", app.getDiagnostics());
appInfo.put("trackingUrl", app.getTrackingUrl());
appInfo.put("clusterId", app.getClusterId());
appInfo.put("applicationType", app.getApplicationType());
appInfo.put("startedTime", app.getStartedTime() + "");
appInfo.put("finishedTime", app.getFinishedTime() + "");
appInfo.put("elapsedTime", app.getElapsedTime() + "");
appInfo.put(
"amContainerLogs", app.getAmContainerLogs() == null ? "" : app.getAmContainerLogs());
appInfo.put(
"amHostHttpAddress", app.getAmHostHttpAddress() == null ? "" : app.getAmHostHttpAddress());
appInfo.put("allocatedMB", app.getAllocatedMB() + "");
appInfo.put("allocatedVCores", app.getAllocatedVCores() + "");
appInfo.put("runningContainers", app.getRunningContainers() + "");
Map<String, String> fields = new HashMap<>();
fields.put(ENTITY_TAGS_KEY, (new JSONObject(tags)).toString());
fields.put(APP_INFO_KEY, (new JSONObject(appInfo)).toString());
try {
lock.acquire();
JSONObject object = new JSONObject(fields);
if (curator.checkExists().forPath(path) == null) {
curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(path);
}
curator.setData().forPath(path, object.toString().getBytes("UTF-8"));
} catch (Exception e) {
LOG.error("failed to update job {} for yarn app {} ", jobId, yarnAppId);
} finally {
try {
lock.release();
} catch (Exception e) {
LOG.error("fail releasing lock", e);
}
}
return true;
}
@SuppressWarnings("unchecked")
@Override
public void nextTuple() {
// LOG.info("Start to run tuple");
try {
Calendar calendar = Calendar.getInstance();
long fetchTime = calendar.getTimeInMillis();
calendar.setTimeInMillis(this.lastFinishAppTime);
if (fetchTime - this.lastFinishAppTime > this.config.stormConfig.spoutCrawlInterval) {
LOG.info("Last finished time = {}", calendar.getTime());
List<AppInfo> appInfos =
rmFetch.getResource(
Constants.ResourceType.COMPLETE_SPARK_JOB, Long.toString(lastFinishAppTime));
if (appInfos != null) {
LOG.info("Get " + appInfos.size() + " from yarn resource manager.");
for (AppInfo app : appInfos) {
String appId = app.getId();
if (!zkState.hasApplication(appId)) {
zkState.addFinishedApplication(
appId,
app.getQueue(),
app.getState(),
app.getFinalStatus(),
app.getUser(),
app.getName());
}
}
}
this.lastFinishAppTime = fetchTime;
zkState.updateLastUpdateTime(fetchTime);
}
List<String> appIds = zkState.loadApplications(10);
for (String appId : appIds) {
collector.emit(new Values(appId), appId);
LOG.info("emit " + appId);
zkState.updateApplicationStatus(appId, ZKStateConstant.AppStatus.SENT_FOR_PARSE);
}
if (appIds.isEmpty()) {
this.takeRest(5);
} else {
LOG.info("{} apps sent.", appIds.size());
}
} catch (Exception e) {
LOG.error("Fail to run next tuple", e);
}
}
