@SuppressWarnings("unchecked")
@Override
public void nextTuple() {
// LOG.info("Start to run tuple");
try {
Calendar calendar = Calendar.getInstance();
long fetchTime = calendar.getTimeInMillis();
calendar.setTimeInMillis(this.lastFinishAppTime);
if (fetchTime - this.lastFinishAppTime > this.config.stormConfig.spoutCrawlInterval) {
LOG.info("Last finished time = {}", calendar.getTime());
List<AppInfo> appInfos =
rmFetch.getResource(
Constants.ResourceType.COMPLETE_SPARK_JOB, Long.toString(lastFinishAppTime));
if (appInfos != null) {
LOG.info("Get " + appInfos.size() + " from yarn resource manager.");
for (AppInfo app : appInfos) {
String appId = app.getId();
if (!zkState.hasApplication(appId)) {
zkState.addFinishedApplication(
appId,
app.getQueue(),
app.getState(),
app.getFinalStatus(),
app.getUser(),
app.getName());
}
}
}
this.lastFinishAppTime = fetchTime;
zkState.updateLastUpdateTime(fetchTime);
}
List<String> appIds = zkState.loadApplications(10);
for (String appId : appIds) {
collector.emit(new Values(appId), appId);
LOG.info("emit " + appId);
zkState.updateApplicationStatus(appId, ZKStateConstant.AppStatus.SENT_FOR_PARSE);
}
if (appIds.isEmpty()) {
this.takeRest(5);
} else {
LOG.info("{} apps sent.", appIds.size());
}
} catch (Exception e) {
LOG.error("Fail to run next tuple", e);
}
}
