Пример #1
0
  public NodeMonitor() {
    childListener =
        new IZkChildListener() {

          public void handleChildChange(String parentPath, List<String> currentChilds)
              throws Exception {
            if (currentChilds != null) {
              initNodes(currentChilds);
            }
          }
        };
    List<String> childs =
        zookeeper.subscribeChildChanges(ArbitrateConstants.NODE_NID_ROOT, childListener);
    if (childs == null) { // 如果为null,代表系统节点为初始化
      try {
        zookeeper.createPersistent(ArbitrateConstants.NODE_NID_ROOT, true);
      } catch (ZkNodeExistsException e) {
        // ignore
      }

      childs = zookeeper.getChildren(ArbitrateConstants.NODE_NID_ROOT);
    }

    initNodes(childs);
    // syncNodes();// 开始监视node节点的变化
    MonitorScheduler.register(this);
  }
Пример #2
0
 private void initNodes() {
   // 获取一下当前存活的所有node节点,再根据对应pipeline关联的node,检查是否在对应的存活列表里
   List<String> nodes = zookeeper.getChildren(ArbitrateConstants.NODE_NID_ROOT);
   initNodes(nodes);
 }
Пример #3
0
  public List<ProcessStat> listProcesses(Long channelId, Long pipelineId) {
    List<ProcessStat> processStats = new ArrayList<ProcessStat>();
    String processRoot = ManagePathUtils.getProcessRoot(channelId, pipelineId);
    IZkConnection connection = zookeeper.getConnection();
    // zkclient会将获取stat信息和正常的操作分开,使用原生的zk进行优化
    ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();

    // 获取所有的process列表
    List<String> processNodes = zookeeper.getChildren(processRoot);
    List<Long> processIds = new ArrayList<Long>();
    for (String processNode : processNodes) {
      processIds.add(ManagePathUtils.getProcessId(processNode));
    }

    Collections.sort(processIds);

    for (int i = 0; i < processIds.size(); i++) {
      Long processId = processIds.get(i);
      // 当前的process可能会有变化
      ProcessStat processStat = new ProcessStat();
      processStat.setPipelineId(pipelineId);
      processStat.setProcessId(processId);

      List<StageStat> stageStats = new ArrayList<StageStat>();
      processStat.setStageStats(stageStats);
      try {
        String processPath = ManagePathUtils.getProcess(channelId, pipelineId, processId);
        Stat zkProcessStat = new Stat();
        List<String> stages = orginZk.getChildren(processPath, false, zkProcessStat);
        Collections.sort(stages, new StageComparator());

        StageStat prev = null;
        for (String stage : stages) { // 循环每个process下的stage
          String stagePath = processPath + "/" + stage;
          Stat zkStat = new Stat();

          StageStat stageStat = new StageStat();
          stageStat.setPipelineId(pipelineId);
          stageStat.setProcessId(processId);

          byte[] bytes = orginZk.getData(stagePath, false, zkStat);
          if (bytes != null && bytes.length > 0) {
            // 特殊处理zookeeper里的data信息,manager没有对应node中PipeKey的对象,所以导致反序列化会失败,需要特殊处理,删除'@'符号
            String json = StringUtils.remove(new String(bytes, "UTF-8"), '@');
            EtlEventData data = JsonUtils.unmarshalFromString(json, EtlEventData.class);
            stageStat.setNumber(data.getNumber());
            stageStat.setSize(data.getSize());

            Map exts = new HashMap();
            if (!CollectionUtils.isEmpty(data.getExts())) {
              exts.putAll(data.getExts());
            }
            exts.put("currNid", data.getCurrNid());
            exts.put("nextNid", data.getNextNid());
            exts.put("desc", data.getDesc());
            stageStat.setExts(exts);
          }
          if (prev != null) { // 对应的start时间为上一个节点的结束时间
            stageStat.setStartTime(prev.getEndTime());
          } else {
            stageStat.setStartTime(zkProcessStat.getMtime()); // process的最后修改时间,select
            // await成功后会设置USED标志位
          }
          stageStat.setEndTime(zkStat.getMtime());
          if (ArbitrateConstants.NODE_SELECTED.equals(stage)) {
            stageStat.setStage(StageType.SELECT);
          } else if (ArbitrateConstants.NODE_EXTRACTED.equals(stage)) {
            stageStat.setStage(StageType.EXTRACT);
          } else if (ArbitrateConstants.NODE_TRANSFORMED.equals(stage)) {
            stageStat.setStage(StageType.TRANSFORM);
            // } else if
            // (ArbitrateConstants.NODE_LOADED.equals(stage)) {
            // stageStat.setStage(StageType.LOAD);
          }

          prev = stageStat;
          stageStats.add(stageStat);
        }

        // 添加一个当前正在处理的
        StageStat currentStageStat = new StageStat();
        currentStageStat.setPipelineId(pipelineId);
        currentStageStat.setProcessId(processId);
        if (prev == null) {
          byte[] bytes = orginZk.getData(processPath, false, zkProcessStat);
          if (bytes == null || bytes.length == 0) {
            continue; // 直接认为未使用,忽略之
          }

          ProcessNodeEventData nodeData =
              JsonUtils.unmarshalFromByte(bytes, ProcessNodeEventData.class);
          if (nodeData.getStatus().isUnUsed()) { // process未使用,直接忽略
            continue; // 跳过该process
          } else {
            currentStageStat.setStage(StageType.SELECT); // select操作
            currentStageStat.setStartTime(zkProcessStat.getMtime());
          }
        } else {
          // 判断上一个节点,确定当前的stage
          StageType stage = prev.getStage();
          if (stage.isSelect()) {
            currentStageStat.setStage(StageType.EXTRACT);
          } else if (stage.isExtract()) {
            currentStageStat.setStage(StageType.TRANSFORM);
          } else if (stage.isTransform()) {
            currentStageStat.setStage(StageType.LOAD);
          } else if (stage.isLoad()) { // 已经是最后一个节点了
            continue;
          }

          currentStageStat.setStartTime(prev.getEndTime()); // 开始时间为上一个节点的结束时间
        }

        if (currentStageStat.getStage().isLoad()) { // load必须为第一个process节点
          if (i == 0) {
            stageStats.add(currentStageStat);
          }
        } else {
          stageStats.add(currentStageStat); // 其他情况都添加
        }

      } catch (NoNodeException e) {
        // ignore
      } catch (KeeperException e) {
        throw new ArbitrateException(e);
      } catch (InterruptedException e) {
        // ignore
      } catch (UnsupportedEncodingException e) {
        // ignore
      }

      processStats.add(processStat);
    }

    return processStats;
  }