@Override
public void open(
Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
rmFetch = new RMResourceFetcher(config.jobHistoryConfig.rms);
this.collector = spoutOutputCollector;
this.zkState = new JobHistoryZKStateManager(config);
this.lastFinishAppTime = zkState.readLastFinishedTimestamp();
zkState.resetApplications();
}
@SuppressWarnings("unchecked")
@Override
public void nextTuple() {
// LOG.info("Start to run tuple");
try {
Calendar calendar = Calendar.getInstance();
long fetchTime = calendar.getTimeInMillis();
calendar.setTimeInMillis(this.lastFinishAppTime);
if (fetchTime - this.lastFinishAppTime > this.config.stormConfig.spoutCrawlInterval) {
LOG.info("Last finished time = {}", calendar.getTime());
List<AppInfo> appInfos =
rmFetch.getResource(
Constants.ResourceType.COMPLETE_SPARK_JOB, Long.toString(lastFinishAppTime));
if (appInfos != null) {
LOG.info("Get " + appInfos.size() + " from yarn resource manager.");
for (AppInfo app : appInfos) {
String appId = app.getId();
if (!zkState.hasApplication(appId)) {
zkState.addFinishedApplication(
appId,
app.getQueue(),
app.getState(),
app.getFinalStatus(),
app.getUser(),
app.getName());
}
}
}
this.lastFinishAppTime = fetchTime;
zkState.updateLastUpdateTime(fetchTime);
}
List<String> appIds = zkState.loadApplications(10);
for (String appId : appIds) {
collector.emit(new Values(appId), appId);
LOG.info("emit " + appId);
zkState.updateApplicationStatus(appId, ZKStateConstant.AppStatus.SENT_FOR_PARSE);
}
if (appIds.isEmpty()) {
this.takeRest(5);
} else {
LOG.info("{} apps sent.", appIds.size());
}
} catch (Exception e) {
LOG.error("Fail to run next tuple", e);
}
}
@Override
public void fail(Object msgId) {
// Sleep 3 seconds and retry.
// Utils.sleep(3000);
collector.emit(new Values(msgId), msgId);
zkState.updateApplicationStatus((String) msgId, ZKStateConstant.AppStatus.FAILED);
LOG.warn("fail {}", msgId.toString());
}
@Override
public void close() {
super.close();
zkState.close();
}
@Override
public void ack(Object msgId) {
zkState.updateApplicationStatus((String) msgId, ZKStateConstant.AppStatus.FINISHED);
LOG.info("ack {}", msgId.toString());
}