@SuppressWarnings("unchecked") @Override public void nextTuple() { // LOG.info("Start to run tuple"); try { Calendar calendar = Calendar.getInstance(); long fetchTime = calendar.getTimeInMillis(); calendar.setTimeInMillis(this.lastFinishAppTime); if (fetchTime - this.lastFinishAppTime > this.config.stormConfig.spoutCrawlInterval) { LOG.info("Last finished time = {}", calendar.getTime()); List<AppInfo> appInfos = rmFetch.getResource( Constants.ResourceType.COMPLETE_SPARK_JOB, Long.toString(lastFinishAppTime)); if (appInfos != null) { LOG.info("Get " + appInfos.size() + " from yarn resource manager."); for (AppInfo app : appInfos) { String appId = app.getId(); if (!zkState.hasApplication(appId)) { zkState.addFinishedApplication( appId, app.getQueue(), app.getState(), app.getFinalStatus(), app.getUser(), app.getName()); } } } this.lastFinishAppTime = fetchTime; zkState.updateLastUpdateTime(fetchTime); } List<String> appIds = zkState.loadApplications(10); for (String appId : appIds) { collector.emit(new Values(appId), appId); LOG.info("emit " + appId); zkState.updateApplicationStatus(appId, ZKStateConstant.AppStatus.SENT_FOR_PARSE); } if (appIds.isEmpty()) { this.takeRest(5); } else { LOG.info("{} apps sent.", appIds.size()); } } catch (Exception e) { LOG.error("Fail to run next tuple", e); } }
public Map<String, Pair<Map<String, String>, AppInfo>> recoverYarnApp(String yarnAppId) throws Exception { Map<String, Pair<Map<String, String>, AppInfo>> result = new HashMap<>(); String path = this.zkRoot + "/" + yarnAppId; try { lock.acquire(); if (curator.checkExists().forPath(path) == null) { return result; } List<String> jobIds = curator.getChildren().forPath(path); if (jobIds.size() == 0) { LOG.info("delete empty path {}", path); delete(yarnAppId); } for (String jobId : jobIds) { String jobPath = path + "/" + jobId; LOG.info("recover path {}", jobPath); String fields = new String(curator.getData().forPath(jobPath), "UTF-8"); if (fields.length() == 0) { // LOG.info("delete empty path {}", jobPath); // delete(yarnAppId, jobId); continue; } JSONObject object = new JSONObject(fields); Map<String, Map<String, String>> parseResult = parse(object); Map<String, String> appInfoMap = parseResult.get(APP_INFO_KEY); AppInfo appInfo = new AppInfo(); appInfo.setId(appInfoMap.get("id")); appInfo.setUser(appInfoMap.get("user")); appInfo.setName(appInfoMap.get("name")); appInfo.setQueue(appInfoMap.get("queue")); appInfo.setState(appInfoMap.get("state")); appInfo.setFinalStatus(appInfoMap.get("finalStatus")); appInfo.setProgress(Double.parseDouble(appInfoMap.get("progress"))); appInfo.setTrackingUI(appInfoMap.get("trackingUI")); appInfo.setDiagnostics(appInfoMap.get("diagnostics")); appInfo.setTrackingUrl(appInfoMap.get("trackingUrl")); appInfo.setClusterId(appInfoMap.get("clusterId")); appInfo.setApplicationType(appInfoMap.get("applicationType")); appInfo.setStartedTime(Long.parseLong(appInfoMap.get("startedTime"))); appInfo.setFinishedTime(Long.parseLong(appInfoMap.get("finishedTime"))); appInfo.setElapsedTime(Long.parseLong(appInfoMap.get("elapsedTime"))); appInfo.setAmContainerLogs( appInfoMap.get("amContainerLogs") == null ? "" : appInfoMap.get("amContainerLogs")); appInfo.setAmHostHttpAddress( appInfoMap.get("amHostHttpAddress") == null ? "" : appInfoMap.get("amHostHttpAddress")); appInfo.setAllocatedMB(Long.parseLong(appInfoMap.get("allocatedMB"))); appInfo.setAllocatedVCores(Integer.parseInt(appInfoMap.get("allocatedVCores"))); appInfo.setRunningContainers(Integer.parseInt(appInfoMap.get("runningContainers"))); Map<String, String> tags = parseResult.get(ENTITY_TAGS_KEY); result.put(jobId, Pair.of(tags, appInfo)); } } catch (Exception e) { LOG.error("fail to recoverYarnApp", e); throw new RuntimeException(e); } finally { try { lock.release(); } catch (Exception e) { LOG.error("fail releasing lock", e); } } return result; }
public boolean update(String yarnAppId, String jobId, Map<String, String> tags, AppInfo app) { String path = this.zkRoot + "/" + yarnAppId + "/" + jobId; Map<String, String> appInfo = new HashMap<>(); appInfo.put("id", app.getId()); appInfo.put("user", app.getUser()); appInfo.put("name", app.getName()); appInfo.put("queue", app.getQueue()); appInfo.put("state", app.getState()); appInfo.put("finalStatus", app.getFinalStatus()); appInfo.put("progress", app.getProgress() + ""); appInfo.put("trackingUI", app.getTrackingUI()); appInfo.put("diagnostics", app.getDiagnostics()); appInfo.put("trackingUrl", app.getTrackingUrl()); appInfo.put("clusterId", app.getClusterId()); appInfo.put("applicationType", app.getApplicationType()); appInfo.put("startedTime", app.getStartedTime() + ""); appInfo.put("finishedTime", app.getFinishedTime() + ""); appInfo.put("elapsedTime", app.getElapsedTime() + ""); appInfo.put( "amContainerLogs", app.getAmContainerLogs() == null ? "" : app.getAmContainerLogs()); appInfo.put( "amHostHttpAddress", app.getAmHostHttpAddress() == null ? "" : app.getAmHostHttpAddress()); appInfo.put("allocatedMB", app.getAllocatedMB() + ""); appInfo.put("allocatedVCores", app.getAllocatedVCores() + ""); appInfo.put("runningContainers", app.getRunningContainers() + ""); Map<String, String> fields = new HashMap<>(); fields.put(ENTITY_TAGS_KEY, (new JSONObject(tags)).toString()); fields.put(APP_INFO_KEY, (new JSONObject(appInfo)).toString()); try { lock.acquire(); JSONObject object = new JSONObject(fields); if (curator.checkExists().forPath(path) == null) { curator.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(path); } curator.setData().forPath(path, object.toString().getBytes("UTF-8")); } catch (Exception e) { LOG.error("failed to update job {} for yarn app {} ", jobId, yarnAppId); } finally { try { lock.release(); } catch (Exception e) { LOG.error("fail releasing lock", e); } } return true; }