@SuppressWarnings("unchecked")
@Override
public void nextTuple() {
// LOG.info("Start to run tuple");
try {
Calendar calendar = Calendar.getInstance();
long fetchTime = calendar.getTimeInMillis();
calendar.setTimeInMillis(this.lastFinishAppTime);
if (fetchTime - this.lastFinishAppTime > this.config.stormConfig.spoutCrawlInterval) {
LOG.info("Last finished time = {}", calendar.getTime());
List<AppInfo> appInfos =
rmFetch.getResource(
Constants.ResourceType.COMPLETE_SPARK_JOB, Long.toString(lastFinishAppTime));
if (appInfos != null) {
LOG.info("Get " + appInfos.size() + " from yarn resource manager.");
for (AppInfo app : appInfos) {
String appId = app.getId();
if (!zkState.hasApplication(appId)) {
zkState.addFinishedApplication(
appId,
app.getQueue(),
app.getState(),
app.getFinalStatus(),
app.getUser(),
app.getName());
}
}
}
this.lastFinishAppTime = fetchTime;
zkState.updateLastUpdateTime(fetchTime);
}
List<String> appIds = zkState.loadApplications(10);
for (String appId : appIds) {
collector.emit(new Values(appId), appId);
LOG.info("emit " + appId);
zkState.updateApplicationStatus(appId, ZKStateConstant.AppStatus.SENT_FOR_PARSE);
}
if (appIds.isEmpty()) {
this.takeRest(5);
} else {
LOG.info("{} apps sent.", appIds.size());
}
} catch (Exception e) {
LOG.error("Fail to run next tuple", e);
}
}
public Map<String, Pair<Map<String, String>, AppInfo>> recoverYarnApp(String yarnAppId)
throws Exception {
Map<String, Pair<Map<String, String>, AppInfo>> result = new HashMap<>();
String path = this.zkRoot + "/" + yarnAppId;
try {
lock.acquire();
if (curator.checkExists().forPath(path) == null) {
return result;
}
List<String> jobIds = curator.getChildren().forPath(path);
if (jobIds.size() == 0) {
LOG.info("delete empty path {}", path);
delete(yarnAppId);
}
for (String jobId : jobIds) {
String jobPath = path + "/" + jobId;
LOG.info("recover path {}", jobPath);
String fields = new String(curator.getData().forPath(jobPath), "UTF-8");
if (fields.length() == 0) {
// LOG.info("delete empty path {}", jobPath);
// delete(yarnAppId, jobId);
continue;
}
JSONObject object = new JSONObject(fields);
Map<String, Map<String, String>> parseResult = parse(object);
Map<String, String> appInfoMap = parseResult.get(APP_INFO_KEY);
AppInfo appInfo = new AppInfo();
appInfo.setId(appInfoMap.get("id"));
appInfo.setUser(appInfoMap.get("user"));
appInfo.setName(appInfoMap.get("name"));
appInfo.setQueue(appInfoMap.get("queue"));
appInfo.setState(appInfoMap.get("state"));
appInfo.setFinalStatus(appInfoMap.get("finalStatus"));
appInfo.setProgress(Double.parseDouble(appInfoMap.get("progress")));
appInfo.setTrackingUI(appInfoMap.get("trackingUI"));
appInfo.setDiagnostics(appInfoMap.get("diagnostics"));
appInfo.setTrackingUrl(appInfoMap.get("trackingUrl"));
appInfo.setClusterId(appInfoMap.get("clusterId"));
appInfo.setApplicationType(appInfoMap.get("applicationType"));
appInfo.setStartedTime(Long.parseLong(appInfoMap.get("startedTime")));
appInfo.setFinishedTime(Long.parseLong(appInfoMap.get("finishedTime")));
appInfo.setElapsedTime(Long.parseLong(appInfoMap.get("elapsedTime")));
appInfo.setAmContainerLogs(
appInfoMap.get("amContainerLogs") == null ? "" : appInfoMap.get("amContainerLogs"));
appInfo.setAmHostHttpAddress(
appInfoMap.get("amHostHttpAddress") == null ? "" : appInfoMap.get("amHostHttpAddress"));
appInfo.setAllocatedMB(Long.parseLong(appInfoMap.get("allocatedMB")));
appInfo.setAllocatedVCores(Integer.parseInt(appInfoMap.get("allocatedVCores")));
appInfo.setRunningContainers(Integer.parseInt(appInfoMap.get("runningContainers")));
Map<String, String> tags = parseResult.get(ENTITY_TAGS_KEY);
result.put(jobId, Pair.of(tags, appInfo));
}
} catch (Exception e) {
LOG.error("fail to recoverYarnApp", e);
throw new RuntimeException(e);
} finally {
try {
lock.release();
} catch (Exception e) {
LOG.error("fail releasing lock", e);
}
}
return result;
}
public boolean update(String yarnAppId, String jobId, Map<String, String> tags, AppInfo app) {
String path = this.zkRoot + "/" + yarnAppId + "/" + jobId;
Map<String, String> appInfo = new HashMap<>();
appInfo.put("id", app.getId());
appInfo.put("user", app.getUser());
appInfo.put("name", app.getName());
appInfo.put("queue", app.getQueue());
appInfo.put("state", app.getState());
appInfo.put("finalStatus", app.getFinalStatus());
appInfo.put("progress", app.getProgress() + "");
appInfo.put("trackingUI", app.getTrackingUI());
appInfo.put("diagnostics", app.getDiagnostics());
appInfo.put("trackingUrl", app.getTrackingUrl());
appInfo.put("clusterId", app.getClusterId());
appInfo.put("applicationType", app.getApplicationType());
appInfo.put("startedTime", app.getStartedTime() + "");
appInfo.put("finishedTime", app.getFinishedTime() + "");
appInfo.put("elapsedTime", app.getElapsedTime() + "");
appInfo.put(
"amContainerLogs", app.getAmContainerLogs() == null ? "" : app.getAmContainerLogs());
appInfo.put(
"amHostHttpAddress", app.getAmHostHttpAddress() == null ? "" : app.getAmHostHttpAddress());
appInfo.put("allocatedMB", app.getAllocatedMB() + "");
appInfo.put("allocatedVCores", app.getAllocatedVCores() + "");
appInfo.put("runningContainers", app.getRunningContainers() + "");
Map<String, String> fields = new HashMap<>();
fields.put(ENTITY_TAGS_KEY, (new JSONObject(tags)).toString());
fields.put(APP_INFO_KEY, (new JSONObject(appInfo)).toString());
try {
lock.acquire();
JSONObject object = new JSONObject(fields);
if (curator.checkExists().forPath(path) == null) {
curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(path);
}
curator.setData().forPath(path, object.toString().getBytes("UTF-8"));
} catch (Exception e) {
LOG.error("failed to update job {} for yarn app {} ", jobId, yarnAppId);
} finally {
try {
lock.release();
} catch (Exception e) {
LOG.error("fail releasing lock", e);
}
}
return true;
}
