@Override
public synchronized void checkJobStatus() throws AnalysisException {
// 通过外部事件激发重新载入配置
if (jobBuilder.isNeedRebuild()) {
if (logger.isInfoEnabled()) {
logger.info("check job status need to rebuild");
}
jobs = jobBuilder.rebuild(jobs);
if (jobs == null || (jobs != null && jobs.size() == 0))
throw new AnalysisException("jobs should not be empty!");
}
checkTaskStatus();
mergeAndExportJobs();
// 任务全部完成并且没有新加任务的情况下，休息1s
for (Job job : jobs.values()) {
if (!job.isExported().get() || job.getRebuildTag() == 2) {
return;
} else {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
logger.error(e);
}
}
}
}
@Override
public void init() throws AnalysisException {
// 获得任务数量
jobBuilder.setConfig(config);
jobExporter.setConfig(config);
jobResultMerger.setConfig(config);
jobBuilder.init();
jobExporter.init();
jobResultMerger.init();
jobs = jobBuilder.build();
for (Job job : jobs.values()) {
job.reset(null);
}
if (jobs == null || (jobs != null && jobs.size() == 0))
throw new AnalysisException("jobs should not be empty!");
jobTaskPool = new ConcurrentHashMap<String, JobTask>();
undoTaskQueue = new LinkedBlockingDeque<JobTask>();
statusPool = new ConcurrentHashMap<String, JobTaskStatus>();
jobTaskResultsQueuePool = new HashMap<String, BlockingQueue<JobTaskResult>>();
branchResultQueuePool = new HashMap<String, BlockingQueue<JobMergedResult>>();
for (String jobName : jobs.keySet()) {
jobTaskResultsQueuePool.put(jobName, new LinkedBlockingQueue<JobTaskResult>());
branchResultQueuePool.put(jobName, new LinkedBlockingQueue<JobMergedResult>());
}
eventProcessThreadPool =
new ThreadPoolExecutor(
this.config.getMaxJobEventWorker(),
this.config.getMaxJobEventWorker(),
0,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("jobManagerEventProcess_worker"));
masterDataRecoverWorker =
new MasterDataRecoverWorker(
config.getMasterName(), config.getTempStoreDataDir(), jobs, this.config);
masterDataRecoverWorker.start();
addJobsToPool();
if (logger.isInfoEnabled())
logger.info("jobManager init end, MaxJobEventWorker size : " + config.getMaxJobEventWorker());
}
@Override
public void releaseResource() {
stopped = true;
try {
// 导出所有结果，暂时不导出中间data，后面看是否需要
// 添加中间结果导出，不导出中间结果，会有部分数据丢失
if (jobs != null)
for (Job j : jobs.values()) {
// 结果导出不重要，可以考虑去掉
while (!j.getTrunkExported().get()) Thread.sleep(3000);
if (!j.isExported().get()) {
jobExporter.exportReport(j, false);
logger.info("releaseResouce now, export job : " + j.getJobName());
}
}
if (eventProcessThreadPool != null) eventProcessThreadPool.shutdown();
if (masterDataRecoverWorker != null) masterDataRecoverWorker.stopWorker();
} catch (Throwable e) {
logger.error("error when stop the node", e);
} finally {
if (jobs != null) jobs.clear();
if (jobTaskPool != null) jobTaskPool.clear();
if (undoTaskQueue != null) undoTaskQueue.clear();
if (statusPool != null) statusPool.clear();
if (jobTaskResultsQueuePool != null) jobTaskResultsQueuePool.clear();
if (branchResultQueuePool != null) branchResultQueuePool.clear();
if (jobBuilder != null) jobBuilder.releaseResource();
if (jobExporter != null) jobExporter.releaseResource();
if (jobResultMerger != null) jobResultMerger.releaseResource();
logger.info("jobManager releaseResource end");
}
}