Esempio n. 1
0
  public NodeMonitor() {
    childListener =
        new IZkChildListener() {

          public void handleChildChange(String parentPath, List<String> currentChilds)
              throws Exception {
            if (currentChilds != null) {
              initNodes(currentChilds);
            }
          }
        };
    List<String> childs =
        zookeeper.subscribeChildChanges(ArbitrateConstants.NODE_NID_ROOT, childListener);
    if (childs == null) { // 如果为null,代表系统节点为初始化
      try {
        zookeeper.createPersistent(ArbitrateConstants.NODE_NID_ROOT, true);
      } catch (ZkNodeExistsException e) {
        // ignore
      }

      childs = zookeeper.getChildren(ArbitrateConstants.NODE_NID_ROOT);
    }

    initNodes(childs);
    // syncNodes();// 开始监视node节点的变化
    MonitorScheduler.register(this);
  }
Esempio n. 2
0
  // 模拟创建一个process
  protected Long initProcess() {
    String path =
        zookeeper.create(processPath + "/", new byte[0], CreateMode.PERSISTENT_SEQUENTIAL);

    // 创建为顺序的节点
    String processNode = StringUtils.substringAfterLast(path, "/");
    return StagePathUtils.getProcessId(processNode); // 添加到当前的process列表
  }
Esempio n. 3
0
  // 模拟创建一个终结节点
  protected void initTermin(Long processId) {
    TerminEventData data = new TerminEventData();
    data.setPipelineId(pipelineId);
    data.setProcessId(processId);
    data.setType(TerminType.NORMAL);

    byte[] bytes = JsonUtils.marshalToByte(data);
    zookeeper.create(StagePathUtils.getTermin(pipelineId, processId), bytes, CreateMode.PERSISTENT);
  }
 public MainStemEventData mainstemData(Long channelId, Long pipelineId) {
   String path = ManagePathUtils.getMainStem(channelId, pipelineId);
   try {
     byte[] bytes = zookeeper.readData(path);
     return JsonUtils.unmarshalFromByte(bytes, MainStemEventData.class);
   } catch (ZkException e) {
     return null;
   }
 }
  @AfterMethod
  public void tearDown() {
    // 删除mainStem节点
    String path = StagePathUtils.getPipeline(pipelineId) + "/" + ArbitrateConstants.NODE_MAINSTEM;
    zookeeper.delete(path);

    nodeEvent.destory(one.getId());
    // 关闭pipeline
    pipelineEvent.destory(channelId, pipelineId);
    channelEvent.destory(channelId);
  }
Esempio n. 6
0
  @AfterClass
  public void destory() {
    ArbitrateFactory.destory(pipelineId);
    // 删除mainStem节点
    String path = pipelinePath + "/" + ArbitrateConstants.NODE_MAINSTEM;

    zookeeper.delete(path);

    pipelineEvent.destory(channelId, pipelineId);
    channelEvent.destory(channelId);
  }
  public Long getNextProcessId(Long channelId, Long pipelineId) {
    String processRoot = ManagePathUtils.getProcessRoot(channelId, pipelineId);
    IZkConnection connection = zookeeper.getConnection();
    // zkclient会将获取stat信息和正常的操作分开,使用原生的zk进行优化
    ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();

    Stat processParentStat = new Stat();
    // 获取所有的process列表
    try {
      orginZk.getChildren(processRoot, false, processParentStat);
      return (Long) ((processParentStat.getCversion() + processParentStat.getNumChildren()) / 2L);
    } catch (Exception e) {
      return -1L;
    }
  }
 public PositionEventData getCanalCursor(String destination, short clientId) {
   String path = String.format(CANAL_CURSOR_PATH, destination, String.valueOf(clientId));
   try {
     IZkConnection connection = zookeeper.getConnection();
     // zkclient会将获取stat信息和正常的操作分开,使用原生的zk进行优化
     ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();
     Stat stat = new Stat();
     byte[] bytes = orginZk.getData(path, false, stat);
     PositionEventData eventData = new PositionEventData();
     eventData.setCreateTime(new Date(stat.getCtime()));
     eventData.setModifiedTime(new Date(stat.getMtime()));
     eventData.setPosition(new String(bytes, "UTF-8"));
     return eventData;
   } catch (Exception e) {
     return null;
   }
 }
Esempio n. 9
0
  public void destory() {
    listeners.clear();

    zookeeper.unsubscribeChildChanges(ArbitrateConstants.NODE_NID_ROOT, childListener);
    MonitorScheduler.unRegister(this);
  }
Esempio n. 10
0
 private void initNodes() {
   // 获取一下当前存活的所有node节点,再根据对应pipeline关联的node,检查是否在对应的存活列表里
   List<String> nodes = zookeeper.getChildren(ArbitrateConstants.NODE_NID_ROOT);
   initNodes(nodes);
 }
Esempio n. 11
0
 // 模拟销毁一个process
 protected void destoryProcess(Long processId) {
   String path = processPath + "/" + StagePathUtils.getProcessNode(processId);
   zookeeper.delete(path);
 }
Esempio n. 12
0
 // 模拟创建一个stage节点
 protected void initStage(Long processId, String stageNode) {
   String path = processPath + "/" + StagePathUtils.getProcessNode(processId) + "/" + stageNode;
   zookeeper.create(path, new byte[0], CreateMode.PERSISTENT);
 }
  public List<ProcessStat> listProcesses(Long channelId, Long pipelineId) {
    List<ProcessStat> processStats = new ArrayList<ProcessStat>();
    String processRoot = ManagePathUtils.getProcessRoot(channelId, pipelineId);
    IZkConnection connection = zookeeper.getConnection();
    // zkclient会将获取stat信息和正常的操作分开,使用原生的zk进行优化
    ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();

    // 获取所有的process列表
    List<String> processNodes = zookeeper.getChildren(processRoot);
    List<Long> processIds = new ArrayList<Long>();
    for (String processNode : processNodes) {
      processIds.add(ManagePathUtils.getProcessId(processNode));
    }

    Collections.sort(processIds);

    for (int i = 0; i < processIds.size(); i++) {
      Long processId = processIds.get(i);
      // 当前的process可能会有变化
      ProcessStat processStat = new ProcessStat();
      processStat.setPipelineId(pipelineId);
      processStat.setProcessId(processId);

      List<StageStat> stageStats = new ArrayList<StageStat>();
      processStat.setStageStats(stageStats);
      try {
        String processPath = ManagePathUtils.getProcess(channelId, pipelineId, processId);
        Stat zkProcessStat = new Stat();
        List<String> stages = orginZk.getChildren(processPath, false, zkProcessStat);
        Collections.sort(stages, new StageComparator());

        StageStat prev = null;
        for (String stage : stages) { // 循环每个process下的stage
          String stagePath = processPath + "/" + stage;
          Stat zkStat = new Stat();

          StageStat stageStat = new StageStat();
          stageStat.setPipelineId(pipelineId);
          stageStat.setProcessId(processId);

          byte[] bytes = orginZk.getData(stagePath, false, zkStat);
          if (bytes != null && bytes.length > 0) {
            // 特殊处理zookeeper里的data信息,manager没有对应node中PipeKey的对象,所以导致反序列化会失败,需要特殊处理,删除'@'符号
            String json = StringUtils.remove(new String(bytes, "UTF-8"), '@');
            EtlEventData data = JsonUtils.unmarshalFromString(json, EtlEventData.class);
            stageStat.setNumber(data.getNumber());
            stageStat.setSize(data.getSize());

            Map exts = new HashMap();
            if (!CollectionUtils.isEmpty(data.getExts())) {
              exts.putAll(data.getExts());
            }
            exts.put("currNid", data.getCurrNid());
            exts.put("nextNid", data.getNextNid());
            exts.put("desc", data.getDesc());
            stageStat.setExts(exts);
          }
          if (prev != null) { // 对应的start时间为上一个节点的结束时间
            stageStat.setStartTime(prev.getEndTime());
          } else {
            stageStat.setStartTime(zkProcessStat.getMtime()); // process的最后修改时间,select
            // await成功后会设置USED标志位
          }
          stageStat.setEndTime(zkStat.getMtime());
          if (ArbitrateConstants.NODE_SELECTED.equals(stage)) {
            stageStat.setStage(StageType.SELECT);
          } else if (ArbitrateConstants.NODE_EXTRACTED.equals(stage)) {
            stageStat.setStage(StageType.EXTRACT);
          } else if (ArbitrateConstants.NODE_TRANSFORMED.equals(stage)) {
            stageStat.setStage(StageType.TRANSFORM);
            // } else if
            // (ArbitrateConstants.NODE_LOADED.equals(stage)) {
            // stageStat.setStage(StageType.LOAD);
          }

          prev = stageStat;
          stageStats.add(stageStat);
        }

        // 添加一个当前正在处理的
        StageStat currentStageStat = new StageStat();
        currentStageStat.setPipelineId(pipelineId);
        currentStageStat.setProcessId(processId);
        if (prev == null) {
          byte[] bytes = orginZk.getData(processPath, false, zkProcessStat);
          if (bytes == null || bytes.length == 0) {
            continue; // 直接认为未使用,忽略之
          }

          ProcessNodeEventData nodeData =
              JsonUtils.unmarshalFromByte(bytes, ProcessNodeEventData.class);
          if (nodeData.getStatus().isUnUsed()) { // process未使用,直接忽略
            continue; // 跳过该process
          } else {
            currentStageStat.setStage(StageType.SELECT); // select操作
            currentStageStat.setStartTime(zkProcessStat.getMtime());
          }
        } else {
          // 判断上一个节点,确定当前的stage
          StageType stage = prev.getStage();
          if (stage.isSelect()) {
            currentStageStat.setStage(StageType.EXTRACT);
          } else if (stage.isExtract()) {
            currentStageStat.setStage(StageType.TRANSFORM);
          } else if (stage.isTransform()) {
            currentStageStat.setStage(StageType.LOAD);
          } else if (stage.isLoad()) { // 已经是最后一个节点了
            continue;
          }

          currentStageStat.setStartTime(prev.getEndTime()); // 开始时间为上一个节点的结束时间
        }

        if (currentStageStat.getStage().isLoad()) { // load必须为第一个process节点
          if (i == 0) {
            stageStats.add(currentStageStat);
          }
        } else {
          stageStats.add(currentStageStat); // 其他情况都添加
        }

      } catch (NoNodeException e) {
        // ignore
      } catch (KeeperException e) {
        throw new ArbitrateException(e);
      } catch (InterruptedException e) {
        // ignore
      } catch (UnsupportedEncodingException e) {
        // ignore
      }

      processStats.add(processStat);
    }

    return processStats;
  }
 public void removeCanal(String destination) {
   String path = String.format(CANAL_PATH, destination);
   zookeeper.deleteRecursive(path);
 }
Esempio n. 15
0
  // 模拟销毁一个stage节点
  protected void destoryStage(Long processId, String stageNode) {
    String path = processPath + "/" + StagePathUtils.getProcessNode(processId) + "/" + stageNode;

    zookeeper.delete(path);
  }
 @Override
 public void removeCanal(String destination, short clientId) {
   String path = String.format(CANAL_DATA_PATH, destination, String.valueOf(clientId));
   zookeeper.deleteRecursive(path);
 }
 public void removeCanalCursor(String destination, short clientId) {
   String path = String.format(CANAL_CURSOR_PATH, destination, String.valueOf(clientId));
   zookeeper.delete(path);
 }
Esempio n. 18
0
  // 模拟创建一个带数据的stage节点
  protected void initStage(Long processId, String stageNode, EtlEventData event) {
    String path = processPath + "/" + StagePathUtils.getProcessNode(processId) + "/" + stageNode;

    byte[] datas = JsonUtils.marshalToByte(event);
    zookeeper.create(path, datas, CreateMode.PERSISTENT);
  }
Esempio n. 19
0
 // 模拟删除一个终结节点
 protected void destoryTermin(Long processId) {
   zookeeper.delete(StagePathUtils.getTermin(pipelineId, processId));
 }
Esempio n. 20
0
  @BeforeClass
  public void init() {
    // 初始化节点
    // mock 配置信息数据
    local.setStatus(NodeStatus.START);
    Mockit.setUpMock(
        ArbitrateConfigUtils.class,
        new Object() {

          @SuppressWarnings("unused")
          @Mock
          public Channel getChannelByChannelId(Long channelId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
          }

          @SuppressWarnings("unused")
          @Mock
          public Channel getChannel(Long pipelineId) {
            Channel channel = new Channel();
            channel.setId(channelId);
            Pipeline pipeline = new Pipeline();
            pipeline.setId(pipelineId);
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            channel.setPipelines(Arrays.asList(pipeline));
            return channel;
          }

          @SuppressWarnings("unused")
          @Mock
          public Pipeline getPipeline(Long pipelineId) {
            Pipeline pipeline = new Pipeline();
            pipeline.setSelectNodes(Arrays.asList(local));
            pipeline.setExtractNodes(Arrays.asList(local));
            pipeline.setLoadNodes(Arrays.asList(local));
            return pipeline;
          }

          @SuppressWarnings("unused")
          @Mock
          public Pipeline getOppositePipeline(Long pipelineId) {
            return null; // 没有反向同步
          }

          @SuppressWarnings("unused")
          @Mock
          public Long getCurrentNid() {
            return nid;
          }

          @SuppressWarnings("unused")
          @Mock
          public int getParallelism(Long pipelineId) {
            return 3; // 并行度
          }
        });

    Mockit.setUpMock(
        ArbitrateCommmunicationClient.class,
        new Object() {

          @SuppressWarnings("unused")
          @Mock
          public Object callManager(final Event event) {
            // do nothing
            return null;
          }

          @SuppressWarnings("unused")
          @Mock
          public void callManager(final Event event, final Callback callback) {
            // do nothing
          }
        });

    zookeeper = getZookeeper();
    local.setId(nid);
    nodeEvent = new NodeArbitrateEvent();
    channelEvent = new ChannelArbitrateEvent();
    pipelineEvent = new PipelineArbitrateEvent();

    pipelinePath = StagePathUtils.getPipeline(channelId, pipelineId);
    processPath = StagePathUtils.getProcessRoot(channelId, pipelineId);
    channelEvent.init(channelId);
    pipelineEvent.init(channelId, pipelineId);
    channelEvent.start(channelId);

    String path = pipelinePath + "/" + ArbitrateConstants.NODE_MAINSTEM;
    MainStemEventData eventData = new MainStemEventData();
    eventData.setStatus(MainStemEventData.Status.OVERTAKE);
    eventData.setNid(nid);
    byte[] bytes = JsonUtils.marshalToByte(eventData); // 初始化的数据对象

    zookeeper.create(path, bytes, CreateMode.EPHEMERAL);
  }