Example #1
0
  @Override
  public synchronized void checkJobStatus() throws AnalysisException {

    // 通过外部事件激发重新载入配置
    if (jobBuilder.isNeedRebuild()) {
      if (logger.isInfoEnabled()) {
        logger.info("check job status need to rebuild");
      }
      jobs = jobBuilder.rebuild(jobs);

      if (jobs == null || (jobs != null && jobs.size() == 0))
        throw new AnalysisException("jobs should not be empty!");
    }

    checkTaskStatus();

    mergeAndExportJobs();

    // 任务全部完成并且没有新加任务的情况下,休息1s
    for (Job job : jobs.values()) {
      if (!job.isExported().get() || job.getRebuildTag() == 2) {
        return;
      } else {
        try {
          Thread.sleep(1000);
        } catch (InterruptedException e) {
          logger.error(e);
        }
      }
    }
  }
Example #2
0
  @Override
  public void init() throws AnalysisException {
    // 获得任务数量
    jobBuilder.setConfig(config);
    jobExporter.setConfig(config);
    jobResultMerger.setConfig(config);

    jobBuilder.init();
    jobExporter.init();
    jobResultMerger.init();

    jobs = jobBuilder.build();
    for (Job job : jobs.values()) {
      job.reset(null);
    }

    if (jobs == null || (jobs != null && jobs.size() == 0))
      throw new AnalysisException("jobs should not be empty!");

    jobTaskPool = new ConcurrentHashMap<String, JobTask>();
    undoTaskQueue = new LinkedBlockingDeque<JobTask>();
    statusPool = new ConcurrentHashMap<String, JobTaskStatus>();
    jobTaskResultsQueuePool = new HashMap<String, BlockingQueue<JobTaskResult>>();
    branchResultQueuePool = new HashMap<String, BlockingQueue<JobMergedResult>>();

    for (String jobName : jobs.keySet()) {
      jobTaskResultsQueuePool.put(jobName, new LinkedBlockingQueue<JobTaskResult>());
      branchResultQueuePool.put(jobName, new LinkedBlockingQueue<JobMergedResult>());
    }

    eventProcessThreadPool =
        new ThreadPoolExecutor(
            this.config.getMaxJobEventWorker(),
            this.config.getMaxJobEventWorker(),
            0,
            TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>(),
            new NamedThreadFactory("jobManagerEventProcess_worker"));

    masterDataRecoverWorker =
        new MasterDataRecoverWorker(
            config.getMasterName(), config.getTempStoreDataDir(), jobs, this.config);
    masterDataRecoverWorker.start();

    addJobsToPool();

    if (logger.isInfoEnabled())
      logger.info("jobManager init end, MaxJobEventWorker size : " + config.getMaxJobEventWorker());
  }
Example #3
0
  @Override
  public void releaseResource() {
    stopped = true;

    try {
      // 导出所有结果,暂时不导出中间data,后面看是否需要
      // 添加中间结果导出,不导出中间结果,会有部分数据丢失
      if (jobs != null)
        for (Job j : jobs.values()) {
          // 结果导出不重要,可以考虑去掉
          while (!j.getTrunkExported().get()) Thread.sleep(3000);
          if (!j.isExported().get()) {
            jobExporter.exportReport(j, false);
            logger.info("releaseResouce now, export job : " + j.getJobName());
          }
        }
      if (eventProcessThreadPool != null) eventProcessThreadPool.shutdown();

      if (masterDataRecoverWorker != null) masterDataRecoverWorker.stopWorker();
    } catch (Throwable e) {
      logger.error("error when stop the node", e);
    } finally {
      if (jobs != null) jobs.clear();

      if (jobTaskPool != null) jobTaskPool.clear();
      if (undoTaskQueue != null) undoTaskQueue.clear();

      if (statusPool != null) statusPool.clear();

      if (jobTaskResultsQueuePool != null) jobTaskResultsQueuePool.clear();

      if (branchResultQueuePool != null) branchResultQueuePool.clear();

      if (jobBuilder != null) jobBuilder.releaseResource();

      if (jobExporter != null) jobExporter.releaseResource();

      if (jobResultMerger != null) jobResultMerger.releaseResource();

      logger.info("jobManager releaseResource end");
    }
  }