/**
   * 根据停止的节点修复死锁
   *
   * @param node
   */
  public void fixedDeadNodeJob(Node node) {
    try {
      // 1. 判断这个节点的channel是否存在
      ChannelWrapper channelWrapper =
          application
              .getChannelManager()
              .getChannel(node.getGroup(), node.getNodeType(), node.getIdentity());
      if (channelWrapper == null
          || channelWrapper.getChannel() == null
          || channelWrapper.isClosed()) {
        //                List<JobPo> jobPos =
        // application.getExecutingJobQueue().getJobs(node.getIdentity());
        //                if (CollectionUtils.isNotEmpty(jobPos)) {
        //                    for (JobPo jobPo : jobPos) {
        //                        fixedDeadJob(jobPo);
        //                    }
        //                }

        List<TaskExecutingBean> executingTasks =
            InjectorHolder.getInstance(DTaskProvider.class)
                .loadTaskExecutingTasksBytaskTrackerIdentity(node.getIdentity());
        if (CollectionUtils.isNotEmpty(executingTasks)) {
          for (TaskExecutingBean taskExectuing : executingTasks) {
            fixedDeadJob(taskExectuing);
          }
        }
      }
    } catch (Exception t) {
      LOGGER.error(t.getMessage(), t);
    }
  }
  private void fix() throws RemotingSendException {
    // 查询出所有死掉的任务 (其实可以直接在数据库中fix的, 查询出来主要是为了日志打印)
    // 一般来说这个是没有多大的,我就不分页去查询了
    List<JobPo> jobPos = new ArrayList<JobPo>();
    ; // application.getExecutingJobQueue().getDeadJobs(SystemClock.now() - MAX_DEAD_CHECK_TIME);
    List<TaskExecutingBean> taskExecutingList =
        InjectorHolder.getInstance(DTaskProvider.class)
            .loadDeadTaskExecutingTasks(before(MAX_DEAD_CHECK_TIME));
    if (CollectionUtils.isNotEmpty(taskExecutingList)) {
      for (TaskExecutingBean taskExecuting : taskExecutingList) {
        jobPos.add(JobDomainConverter.convertTaskExecutingToJobpo(taskExecuting));
      }
    }

    if (jobPos != null && jobPos.size() > 0) {
      List<Node> nodes = application.getSubscribedNodeManager().getNodeList(NodeType.TASK_TRACKER);
      HashSet<String /*identity*/> identities = new HashSet<String>();
      if (CollectionUtils.isNotEmpty(nodes)) {
        for (Node node : nodes) {
          identities.add(node.getIdentity());
        }
      }

      Map<TaskTrackerNode /*执行的TaskTracker节点 identity*/, List<JobPo /*jobId*/>> timeoutMap =
          new HashMap<TaskTrackerNode, List<JobPo>>();
      for (JobPo jobPo : jobPos) {
        if (!identities.contains(jobPo.getTaskTrackerIdentity())) {
          fixedDeadJob(jobPo);
        } else {
          // 如果节点存在,并且超时了, 那么去主动询问taskTracker 这个任务是否在执行中
          if (SystemClock.now() - jobPo.getGmtModified() > MAX_TIME_OUT) {
            TaskTrackerNode taskTrackerNode =
                new TaskTrackerNode(
                    jobPo.getTaskTrackerIdentity(), jobPo.getTaskTrackerNodeGroup());
            List<JobPo> jobPosList = timeoutMap.get(taskTrackerNode);
            if (jobPosList == null) {
              jobPosList = new ArrayList<JobPo>();
              timeoutMap.put(taskTrackerNode, jobPosList);
            }
            jobPosList.add(jobPo);
          }
        }
      }

      if (CollectionUtils.isNotEmpty(timeoutMap)) {
        RemotingServerDelegate remotingServer = application.getRemotingServer();
        for (Map.Entry<TaskTrackerNode, List<JobPo>> entry : timeoutMap.entrySet()) {
          TaskTrackerNode taskTrackerNode = entry.getKey();
          ChannelWrapper channelWrapper =
              application
                  .getChannelManager()
                  .getChannel(
                      taskTrackerNode.getNodeGroup(),
                      NodeType.TASK_TRACKER,
                      taskTrackerNode.getIdentity());
          if (channelWrapper != null
              && channelWrapper.getChannel() != null
              && channelWrapper.isOpen()) {
            JobAskRequest requestBody =
                application.getCommandBodyWrapper().wrapper(new JobAskRequest());

            final List<JobPo> jobPoList = entry.getValue();
            List<String> jobIds = new ArrayList<String>(jobPoList.size());
            for (JobPo jobPo : jobPoList) {
              jobIds.add(jobPo.getJobId());
            }
            requestBody.setJobIds(jobIds);
            RemotingCommand request =
                RemotingCommand.createRequestCommand(
                    JobProtos.RequestCode.JOB_ASK.code(), requestBody);
            remotingServer.invokeAsync(
                channelWrapper.getChannel(),
                request,
                new InvokeCallback() {
                  @Override
                  public void operationComplete(ResponseFuture responseFuture) {
                    RemotingCommand response = responseFuture.getResponseCommand();
                    if (response != null
                        && RemotingProtos.ResponseCode.SUCCESS.code() == response.getCode()) {
                      JobAskResponse responseBody = response.getBody();
                      List<String> deadJobIds = responseBody.getJobIds();
                      if (CollectionUtils.isNotEmpty(deadJobIds)) {
                        try {
                          Thread.sleep(1000L); // 睡了1秒再修复, 防止任务刚好执行完正在传输中. 1s可以让完成的正常完成
                        } catch (InterruptedException e) {
                          e.printStackTrace();
                        }
                        for (JobPo jobPo : jobPoList) {
                          if (deadJobIds.contains(jobPo.getJobId())) {
                            fixedDeadJob(jobPo);
                          }
                        }
                      }
                    }
                  }
                });
          }
        }
      }
    }
  }