@Override
public void initialize() {
log.debug("Going to initialize required znode structure in zookeeper");
for (NodeType type : NodeType.values()) {
String child = CoordinationUtil.nodeNameOf(type);
rootNode.createChild(znode().withPath(child));
log.debug("Created node {}", child);
}
rootNode.createChild(znode().withPath(CoordinationUtil.STATUSES_NODE_NAME));
log.debug("Created node {}", CoordinationUtil.STATUSES_NODE_NAME);
log.debug("Successfully initialized");
}
@Override
public void waitForInitialization() {
log.info("Waiting for coordination znode structure structure initialization");
while (true) {
boolean initialized =
rootNode.exists() && rootNode.hasChild(CoordinationUtil.STATUSES_NODE_NAME);
if (initialized) {
log.info("Coordination znode structure initialized");
break;
}
try {
Thread.sleep(INITIALIZATION_SLEEP_PERIOD);
log.info("Znode structure is not initialized. Waiting {} ms", INITIALIZATION_SLEEP_PERIOD);
} catch (InterruptedException e) {
log.warn("Sleep interrupted", e);
}
}
}
private static <C extends Command<R>, R extends Serializable> List<QueueEntry<C, R>> getEntries(
ZNode queueNode, Watcher watcher) {
List<QueueEntry<C, R>> result = Lists.newLinkedList();
List<ZNode> children = queueNode.firstLevelChildren(watcher);
Collections.sort(
children,
new Comparator<ZNode>() {
@Override
public int compare(ZNode first, ZNode second) {
return first.getPath().compareTo(second.getPath());
}
});
for (ZNode child : children) {
QueueEntry<C, R> entry = child.getObject(QueueEntry.class);
child.remove();
result.add(entry);
}
return result;
}
private static <C extends Command<R>, R extends Serializable> void executeCommand(
CommandExecutor<C, R> executor,
ZNode executorNode,
final QueueEntry<C, R> entry,
final NodeContext nodeContext) {
String relativePath = entry.getResultPath().substring(executorNode.getPath().length() + 1);
final ZNode output = executorNode.child(relativePath);
final NodeCommandExecutionListener<C> listener = entry.getListener();
try {
C command = entry.getCommand();
listener.onCommandExecutionStarted(command, nodeContext);
R result = executor.execute(command, nodeContext);
log.debug("Command {} executed", command);
listener.onCommandExecuted(command);
output.setObject(CommandExecutionResult.success(result));
} catch (Throwable throwable) {
// todo add fail event
log.error("error during task execution", throwable);
output.setObject(CommandExecutionResult.fail(throwable));
}
}
@Override
public Set<NodeId> getAvailableNodes(NodeType type) {
Set<NodeId> result = Sets.newHashSet();
ZNode typeNode = rootNode.child(CoordinationUtil.nodeNameOf(type));
for (ZNode node : typeNode.children()) {
if (node.hasChild(CoordinationUtil.AVAILABLE_NODE_NAME)) {
result.add(NodeId.of(type, node.getShortPath()));
}
}
return result;
}
@Override
public void waitForReady() {
while (true) {
try {
rootNode.exists();
break;
} catch (Throwable e) {
// do nothing
}
try {
Thread.sleep(INITIALIZATION_SLEEP_PERIOD);
log.info("Znode structure is not initialized. Waiting {} ms", INITIALIZATION_SLEEP_PERIOD);
} catch (InterruptedException e) {
log.warn("Sleep interrupted", e);
}
}
}
@Override
public void process(WatchedEvent event) {
if (event.getType() != Event.EventType.NodeChildrenChanged) {
return;
}
Runnable runnable =
new Runnable() {
public void run() {
lock.lock();
try {
List<ZNode> children = node.children();
Collection<NodeId> newIds = Sets.newHashSet();
for (ZNode child : children) {
newIds.add(child.getObject(NodeId.class));
}
Collection<NodeId> copy = Sets.newHashSet(newIds);
newIds.removeAll(currentIds);
currentIds.removeAll(copy);
for (NodeId newId : newIds) {
statusChangeListener.onNodeStatusChanged(newId, NodeStatus.AVAILABLE);
}
for (NodeId newId : currentIds) {
statusChangeListener.onNodeStatusChanged(newId, NodeStatus.DISCONNECTED);
}
currentIds = copy;
} finally {
lock.unlock();
}
}
};
new Thread(runnable).run();
node.addChildrenWatcher(this);
}
@Override
public boolean canExecuteCommands(NodeId nodeId, Set<Qualifier<?>> qualifiers) {
ZNode typeNode = rootNode.child(CoordinationUtil.nodeNameOf(nodeId.getType()));
String identifier = nodeId.getIdentifier();
if (!typeNode.hasChild(identifier)) {
throw new CoordinatorException("Node with id " + nodeId + " is not found");
}
ZNode node = typeNode.child(identifier);
if (!node.hasChild(CoordinationUtil.AVAILABLE_NODE_NAME)) {
return false;
}
for (Qualifier<?> qualifier : qualifiers) {
if (!node.hasChild(nodeNameOf(qualifier))) {
return false;
}
}
return true;
}
private <C extends Command<R>, R extends Serializable> void registerExecutor(
final NodeContext nodeContext, final CommandExecutor<C, R> executor, ZNode node) {
final ZNode executorNode =
node.createChild(znode().withPath(nodeNameOf(executor.getQualifier())));
final ZNode queueNode = executorNode.createChild(znode().withPath("queue"));
executorNode.createChild(znode().withPath("result"));
log.debug("Created znodes for executor {}", executorNode.getPath());
queueNode.addChildrenWatcher(
new Watcher() {
@Override
public void process(WatchedEvent event) {
if (event.getType() != Event.EventType.NodeChildrenChanged) {
return;
}
synchronized (lock) {
if (log.isDebugEnabled()) {
log.debug(
"Children changed {} event type {}", queueNode.getPath(), event.getType());
}
List<QueueEntry<C, R>> entries = getEntries(queueNode, this);
for (final QueueEntry<C, R> entry : entries) {
Runnable run =
new Runnable() {
@Override
public void run() {
executeCommand(executor, executorNode, entry, nodeContext);
}
};
ZookeeperCoordinator.this.executor.execute(run);
}
}
}
});
}
@Override
public void registerNode(
NodeContext nodeContext, Set<Worker> workers, final StatusChangeListener listener)
throws CoordinatorException {
log.info("Going to register node {} with {} workers", nodeContext.getId(), workers.size());
ZNode typeNode = rootNode.child(CoordinationUtil.nodeNameOf(nodeContext.getId().getType()));
ZNode node = typeNode.createChild(znode().withPath(nodeContext.getId().getIdentifier()));
Set<CommandExecutor<?, ?>> executors = Sets.newHashSet();
Set<Qualifier<?>> qualifiers = Sets.newHashSet();
for (Worker worker : workers) {
for (CommandExecutor<?, ?> executor : worker.getExecutors()) {
Qualifier<?> qualifier = executor.getQualifier();
if (qualifiers.contains(qualifier)) {
throw new CoordinatorException(
"Executor for qualifier " + qualifier + " is already registered");
}
executors.add(executor);
}
}
for (CommandExecutor<?, ?> executor : executors) {
registerExecutor(nodeContext, executor, node);
}
rootNode.addNodeWatcher(
new Watcher() {
@Override
public void process(WatchedEvent event) {
if (event.getState() == Event.KeeperState.Disconnected) {
listener.onCoordinatorDisconnected();
}
if (event.getState() == Event.KeeperState.SyncConnected) {
listener.onCoordinatorConnected();
}
}
});
ZNode statuses = rootNode.child(CoordinationUtil.STATUSES_NODE_NAME);
statuses.createChild(znode().ephemeralSequential().withDataObject(nodeContext.getId()));
Lock lock = new ReentrantLock();
lock.lock();
try {
Collection<NodeId> nodeIds = Sets.newHashSet();
StatusWatcher statusWatcher = new StatusWatcher(statuses, lock, nodeIds, listener);
List<ZNode> nodes = statuses.children(statusWatcher);
for (ZNode zNode : nodes) {
nodeIds.add(zNode.getObject(NodeId.class));
}
} finally {
lock.unlock();
}
node.createChild(znode().withPath(CoordinationUtil.AVAILABLE_NODE_NAME));
}