public NodeMonitor() {
childListener =
new IZkChildListener() {
public void handleChildChange(String parentPath, List<String> currentChilds)
throws Exception {
if (currentChilds != null) {
initNodes(currentChilds);
}
}
};
List<String> childs =
zookeeper.subscribeChildChanges(ArbitrateConstants.NODE_NID_ROOT, childListener);
if (childs == null) { // 如果为null，代表系统节点为初始化
try {
zookeeper.createPersistent(ArbitrateConstants.NODE_NID_ROOT, true);
} catch (ZkNodeExistsException e) {
// ignore
}
childs = zookeeper.getChildren(ArbitrateConstants.NODE_NID_ROOT);
}
initNodes(childs);
// syncNodes();// 开始监视node节点的变化
MonitorScheduler.register(this);
}
// 模拟创建一个process
protected Long initProcess() {
String path =
zookeeper.create(processPath + "/", new byte[0], CreateMode.PERSISTENT_SEQUENTIAL);
// 创建为顺序的节点
String processNode = StringUtils.substringAfterLast(path, "/");
return StagePathUtils.getProcessId(processNode); // 添加到当前的process列表
}
// 模拟创建一个终结节点
protected void initTermin(Long processId) {
TerminEventData data = new TerminEventData();
data.setPipelineId(pipelineId);
data.setProcessId(processId);
data.setType(TerminType.NORMAL);
byte[] bytes = JsonUtils.marshalToByte(data);
zookeeper.create(StagePathUtils.getTermin(pipelineId, processId), bytes, CreateMode.PERSISTENT);
}
public MainStemEventData mainstemData(Long channelId, Long pipelineId) {
String path = ManagePathUtils.getMainStem(channelId, pipelineId);
try {
byte[] bytes = zookeeper.readData(path);
return JsonUtils.unmarshalFromByte(bytes, MainStemEventData.class);
} catch (ZkException e) {
return null;
}
}
@AfterMethod
public void tearDown() {
// 删除mainStem节点
String path = StagePathUtils.getPipeline(pipelineId) + "/" + ArbitrateConstants.NODE_MAINSTEM;
zookeeper.delete(path);
nodeEvent.destory(one.getId());
// 关闭pipeline
pipelineEvent.destory(channelId, pipelineId);
channelEvent.destory(channelId);
}
@AfterClass
public void destory() {
ArbitrateFactory.destory(pipelineId);
// 删除mainStem节点
String path = pipelinePath + "/" + ArbitrateConstants.NODE_MAINSTEM;
zookeeper.delete(path);
pipelineEvent.destory(channelId, pipelineId);
channelEvent.destory(channelId);
}
public Long getNextProcessId(Long channelId, Long pipelineId) {
String processRoot = ManagePathUtils.getProcessRoot(channelId, pipelineId);
IZkConnection connection = zookeeper.getConnection();
// zkclient会将获取stat信息和正常的操作分开，使用原生的zk进行优化
ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();
Stat processParentStat = new Stat();
// 获取所有的process列表
try {
orginZk.getChildren(processRoot, false, processParentStat);
return (Long) ((processParentStat.getCversion() + processParentStat.getNumChildren()) / 2L);
} catch (Exception e) {
return -1L;
}
}
public PositionEventData getCanalCursor(String destination, short clientId) {
String path = String.format(CANAL_CURSOR_PATH, destination, String.valueOf(clientId));
try {
IZkConnection connection = zookeeper.getConnection();
// zkclient会将获取stat信息和正常的操作分开，使用原生的zk进行优化
ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();
Stat stat = new Stat();
byte[] bytes = orginZk.getData(path, false, stat);
PositionEventData eventData = new PositionEventData();
eventData.setCreateTime(new Date(stat.getCtime()));
eventData.setModifiedTime(new Date(stat.getMtime()));
eventData.setPosition(new String(bytes, "UTF-8"));
return eventData;
} catch (Exception e) {
return null;
}
}
public void destory() {
listeners.clear();
zookeeper.unsubscribeChildChanges(ArbitrateConstants.NODE_NID_ROOT, childListener);
MonitorScheduler.unRegister(this);
}
private void initNodes() {
// 获取一下当前存活的所有node节点，再根据对应pipeline关联的node，检查是否在对应的存活列表里
List<String> nodes = zookeeper.getChildren(ArbitrateConstants.NODE_NID_ROOT);
initNodes(nodes);
}
// 模拟销毁一个process
protected void destoryProcess(Long processId) {
String path = processPath + "/" + StagePathUtils.getProcessNode(processId);
zookeeper.delete(path);
}
// 模拟创建一个stage节点
protected void initStage(Long processId, String stageNode) {
String path = processPath + "/" + StagePathUtils.getProcessNode(processId) + "/" + stageNode;
zookeeper.create(path, new byte[0], CreateMode.PERSISTENT);
}
public List<ProcessStat> listProcesses(Long channelId, Long pipelineId) {
List<ProcessStat> processStats = new ArrayList<ProcessStat>();
String processRoot = ManagePathUtils.getProcessRoot(channelId, pipelineId);
IZkConnection connection = zookeeper.getConnection();
// zkclient会将获取stat信息和正常的操作分开，使用原生的zk进行优化
ZooKeeper orginZk = ((ZooKeeperx) connection).getZookeeper();
// 获取所有的process列表
List<String> processNodes = zookeeper.getChildren(processRoot);
List<Long> processIds = new ArrayList<Long>();
for (String processNode : processNodes) {
processIds.add(ManagePathUtils.getProcessId(processNode));
}
Collections.sort(processIds);
for (int i = 0; i < processIds.size(); i++) {
Long processId = processIds.get(i);
// 当前的process可能会有变化
ProcessStat processStat = new ProcessStat();
processStat.setPipelineId(pipelineId);
processStat.setProcessId(processId);
List<StageStat> stageStats = new ArrayList<StageStat>();
processStat.setStageStats(stageStats);
try {
String processPath = ManagePathUtils.getProcess(channelId, pipelineId, processId);
Stat zkProcessStat = new Stat();
List<String> stages = orginZk.getChildren(processPath, false, zkProcessStat);
Collections.sort(stages, new StageComparator());
StageStat prev = null;
for (String stage : stages) { // 循环每个process下的stage
String stagePath = processPath + "/" + stage;
Stat zkStat = new Stat();
StageStat stageStat = new StageStat();
stageStat.setPipelineId(pipelineId);
stageStat.setProcessId(processId);
byte[] bytes = orginZk.getData(stagePath, false, zkStat);
if (bytes != null && bytes.length > 0) {
// 特殊处理zookeeper里的data信息，manager没有对应node中PipeKey的对象，所以导致反序列化会失败，需要特殊处理，删除'@'符号
String json = StringUtils.remove(new String(bytes, "UTF-8"), '@');
EtlEventData data = JsonUtils.unmarshalFromString(json, EtlEventData.class);
stageStat.setNumber(data.getNumber());
stageStat.setSize(data.getSize());
Map exts = new HashMap();
if (!CollectionUtils.isEmpty(data.getExts())) {
exts.putAll(data.getExts());
}
exts.put("currNid", data.getCurrNid());
exts.put("nextNid", data.getNextNid());
exts.put("desc", data.getDesc());
stageStat.setExts(exts);
}
if (prev != null) { // 对应的start时间为上一个节点的结束时间
stageStat.setStartTime(prev.getEndTime());
} else {
stageStat.setStartTime(zkProcessStat.getMtime()); // process的最后修改时间,select
// await成功后会设置USED标志位
}
stageStat.setEndTime(zkStat.getMtime());
if (ArbitrateConstants.NODE_SELECTED.equals(stage)) {
stageStat.setStage(StageType.SELECT);
} else if (ArbitrateConstants.NODE_EXTRACTED.equals(stage)) {
stageStat.setStage(StageType.EXTRACT);
} else if (ArbitrateConstants.NODE_TRANSFORMED.equals(stage)) {
stageStat.setStage(StageType.TRANSFORM);
// } else if
// (ArbitrateConstants.NODE_LOADED.equals(stage)) {
// stageStat.setStage(StageType.LOAD);
}
prev = stageStat;
stageStats.add(stageStat);
}
// 添加一个当前正在处理的
StageStat currentStageStat = new StageStat();
currentStageStat.setPipelineId(pipelineId);
currentStageStat.setProcessId(processId);
if (prev == null) {
byte[] bytes = orginZk.getData(processPath, false, zkProcessStat);
if (bytes == null || bytes.length == 0) {
continue; // 直接认为未使用，忽略之
}
ProcessNodeEventData nodeData =
JsonUtils.unmarshalFromByte(bytes, ProcessNodeEventData.class);
if (nodeData.getStatus().isUnUsed()) { // process未使用,直接忽略
continue; // 跳过该process
} else {
currentStageStat.setStage(StageType.SELECT); // select操作
currentStageStat.setStartTime(zkProcessStat.getMtime());
}
} else {
// 判断上一个节点，确定当前的stage
StageType stage = prev.getStage();
if (stage.isSelect()) {
currentStageStat.setStage(StageType.EXTRACT);
} else if (stage.isExtract()) {
currentStageStat.setStage(StageType.TRANSFORM);
} else if (stage.isTransform()) {
currentStageStat.setStage(StageType.LOAD);
} else if (stage.isLoad()) { // 已经是最后一个节点了
continue;
}
currentStageStat.setStartTime(prev.getEndTime()); // 开始时间为上一个节点的结束时间
}
if (currentStageStat.getStage().isLoad()) { // load必须为第一个process节点
if (i == 0) {
stageStats.add(currentStageStat);
}
} else {
stageStats.add(currentStageStat); // 其他情况都添加
}
} catch (NoNodeException e) {
// ignore
} catch (KeeperException e) {
throw new ArbitrateException(e);
} catch (InterruptedException e) {
// ignore
} catch (UnsupportedEncodingException e) {
// ignore
}
processStats.add(processStat);
}
return processStats;
}
public void removeCanal(String destination) {
String path = String.format(CANAL_PATH, destination);
zookeeper.deleteRecursive(path);
}
// 模拟销毁一个stage节点
protected void destoryStage(Long processId, String stageNode) {
String path = processPath + "/" + StagePathUtils.getProcessNode(processId) + "/" + stageNode;
zookeeper.delete(path);
}
@Override
public void removeCanal(String destination, short clientId) {
String path = String.format(CANAL_DATA_PATH, destination, String.valueOf(clientId));
zookeeper.deleteRecursive(path);
}
public void removeCanalCursor(String destination, short clientId) {
String path = String.format(CANAL_CURSOR_PATH, destination, String.valueOf(clientId));
zookeeper.delete(path);
}
// 模拟创建一个带数据的stage节点
protected void initStage(Long processId, String stageNode, EtlEventData event) {
String path = processPath + "/" + StagePathUtils.getProcessNode(processId) + "/" + stageNode;
byte[] datas = JsonUtils.marshalToByte(event);
zookeeper.create(path, datas, CreateMode.PERSISTENT);
}
// 模拟删除一个终结节点
protected void destoryTermin(Long processId) {
zookeeper.delete(StagePathUtils.getTermin(pipelineId, processId));
}
@BeforeClass
public void init() {
// 初始化节点
// mock 配置信息数据
local.setStatus(NodeStatus.START);
Mockit.setUpMock(
ArbitrateConfigUtils.class,
new Object() {
@SuppressWarnings("unused")
@Mock
public Channel getChannelByChannelId(Long channelId) {
Channel channel = new Channel();
channel.setId(channelId);
Pipeline pipeline = new Pipeline();
pipeline.setId(pipelineId);
pipeline.setSelectNodes(Arrays.asList(local));
pipeline.setExtractNodes(Arrays.asList(local));
pipeline.setLoadNodes(Arrays.asList(local));
channel.setPipelines(Arrays.asList(pipeline));
return channel;
}
@SuppressWarnings("unused")
@Mock
public Channel getChannel(Long pipelineId) {
Channel channel = new Channel();
channel.setId(channelId);
Pipeline pipeline = new Pipeline();
pipeline.setId(pipelineId);
pipeline.setSelectNodes(Arrays.asList(local));
pipeline.setExtractNodes(Arrays.asList(local));
pipeline.setLoadNodes(Arrays.asList(local));
channel.setPipelines(Arrays.asList(pipeline));
return channel;
}
@SuppressWarnings("unused")
@Mock
public Pipeline getPipeline(Long pipelineId) {
Pipeline pipeline = new Pipeline();
pipeline.setSelectNodes(Arrays.asList(local));
pipeline.setExtractNodes(Arrays.asList(local));
pipeline.setLoadNodes(Arrays.asList(local));
return pipeline;
}
@SuppressWarnings("unused")
@Mock
public Pipeline getOppositePipeline(Long pipelineId) {
return null; // 没有反向同步
}
@SuppressWarnings("unused")
@Mock
public Long getCurrentNid() {
return nid;
}
@SuppressWarnings("unused")
@Mock
public int getParallelism(Long pipelineId) {
return 3; // 并行度
}
});
Mockit.setUpMock(
ArbitrateCommmunicationClient.class,
new Object() {
@SuppressWarnings("unused")
@Mock
public Object callManager(final Event event) {
// do nothing
return null;
}
@SuppressWarnings("unused")
@Mock
public void callManager(final Event event, final Callback callback) {
// do nothing
}
});
zookeeper = getZookeeper();
local.setId(nid);
nodeEvent = new NodeArbitrateEvent();
channelEvent = new ChannelArbitrateEvent();
pipelineEvent = new PipelineArbitrateEvent();
pipelinePath = StagePathUtils.getPipeline(channelId, pipelineId);
processPath = StagePathUtils.getProcessRoot(channelId, pipelineId);
channelEvent.init(channelId);
pipelineEvent.init(channelId, pipelineId);
channelEvent.start(channelId);
String path = pipelinePath + "/" + ArbitrateConstants.NODE_MAINSTEM;
MainStemEventData eventData = new MainStemEventData();
eventData.setStatus(MainStemEventData.Status.OVERTAKE);
eventData.setNid(nid);
byte[] bytes = JsonUtils.marshalToByte(eventData); // 初始化的数据对象
zookeeper.create(path, bytes, CreateMode.EPHEMERAL);
}