@Override
protected void doSubscribe(Node node, NotifyListener listener) {
List<NodeType> listenNodeTypes = node.getListenNodeTypes();
if (CollectionUtils.isEmpty(listenNodeTypes)) {
return;
}
for (NodeType listenNodeType : listenNodeTypes) {
String listenNodePath = NodeRegistryUtils.getNodeTypePath(clusterName, listenNodeType);
Notifier notifier = notifiers.get(listenNodePath);
if (notifier == null) {
Notifier newNotifier = new Notifier(listenNodePath);
notifiers.putIfAbsent(listenNodePath, newNotifier);
notifier = notifiers.get(listenNodePath);
if (notifier == newNotifier) {
notifier.start();
}
}
boolean success = false;
NodeRegistryException exception = null;
for (Map.Entry<String, JedisPool> entry : jedisPools.entrySet()) {
JedisPool jedisPool = entry.getValue();
try {
Jedis jedis = jedisPool.getResource();
try {
doNotify(
jedis,
Collections.singletonList(listenNodePath),
Collections.singletonList(listener));
success = true;
break; // 只需读一个服务器的数据
} finally {
jedis.close();
}
} catch (Throwable t) {
exception =
new NodeRegistryException(
"Failed to unregister node to redis registry. registry: "
+ entry.getKey()
+ ", node: "
+ node
+ ", cause: "
+ t.getMessage(),
t);
}
}
if (exception != null) {
if (success) {
LOGGER.warn(exception.getMessage(), exception);
} else {
throw exception;
}
}
}
}
private static void testPotato(
Class<? extends Collection> implClazz, Class<? extends List> argClazz) throws Throwable {
try {
System.out.printf("implClazz=%s, argClazz=%s\n", implClazz.getName(), argClazz.getName());
final int iterations = 100000;
final List<Integer> list = (List<Integer>) argClazz.newInstance();
final Integer one = Integer.valueOf(1);
final List<Integer> oneElementList = Collections.singletonList(one);
final Constructor<? extends Collection> constr = implClazz.getConstructor(Collection.class);
final Thread t =
new CheckedThread() {
public void realRun() {
for (int i = 0; i < iterations; i++) {
list.add(one);
list.remove(one);
}
}
};
t.setDaemon(true);
t.start();
for (int i = 0; i < iterations; i++) {
Collection<?> coll = constr.newInstance(list);
Object[] elts = coll.toArray();
check(elts.length == 0 || (elts.length == 1 && elts[0] == one));
}
} catch (Throwable t) {
unexpected(t);
}
}
private void doNotify(Jedis jedis, String key) {
for (Map.Entry<Node, Set<NotifyListener>> entry :
new HashMap<Node, Set<NotifyListener>>(getSubscribed()).entrySet()) {
doNotify(
jedis, Collections.singletonList(key), new HashSet<NotifyListener>(entry.getValue()));
}
}
/** Created by jacobo on 5/02/15. */
public class ThreadRunner {
private ExecutorService executorService;
private List<ReadNode> readNodes = new LinkedList<>();
private List<Node> taskNodes = new LinkedList<>();
private List<WriterNode> writerNodes = new LinkedList<>();
private List<Node> nodes = new LinkedList<>();
private final int batchSize;
private final Object syncObject = new Object();
private static final List<Object> SINGLETON_LIST = Collections.singletonList(new Object());
private static final List POISON_PILL = new LinkedList();
public ThreadRunner(ExecutorService executorService, int batchSize) {
this.executorService = executorService;
this.batchSize = batchSize;
}
public <I, O> TaskNode<I, O> newTaskNode(List<Task<I, O>> tasks) {
TaskNode<I, O> taskNode = new TaskNode<>(tasks, "task-node-" + taskNodes.size());
taskNodes.add(taskNode);
return taskNode;
}
public <T> SimpleTaskNode<T> newSimpleTaskNode(org.opencb.commons.run.Task<T> task, int n) {
List<org.opencb.commons.run.Task<T>> tasks = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
tasks.add(task);
}
SimpleTaskNode<T> taskNode = new SimpleTaskNode<>(tasks, "task-node-" + taskNodes.size());
taskNodes.add(taskNode);
return taskNode;
}
public <T> SimpleTaskNode<T> newSimpleTaskNode(List<org.opencb.commons.run.Task<T>> tasks) {
SimpleTaskNode<T> taskNode = new SimpleTaskNode<>(tasks, "task-node-" + taskNodes.size());
taskNodes.add(taskNode);
return taskNode;
}
public <I, O> TaskNode<I, O> newTaskNode(Task<I, O> task, int n) {
List<Task<I, O>> tasks = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
tasks.add(task);
}
TaskNode<I, O> taskNode = new TaskNode<>(tasks, "task-node-" + taskNodes.size());
taskNodes.add(taskNode);
return taskNode;
}
public <O> ReadNode<O> newReaderNode(List<DataReader<O>> readers) {
ReadNode<O> readNode = new ReadNode<>(readers, "reader-node-" + readNodes.size());
readNodes.add(readNode);
return readNode;
}
public <O> ReadNode<O> newReaderNode(DataReader<O> reader, int n) {
List<DataReader<O>> readers = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
readers.add(reader);
}
ReadNode<O> readNode = new ReadNode<>(readers, "reader-node-" + readNodes.size());
readNodes.add(readNode);
return readNode;
}
public <I> WriterNode<I> newWriterNode(List<DataWriter<I>> writers) {
WriterNode<I> writerNode = new WriterNode<>(writers, "writer-node-" + writerNodes.size());
writerNodes.add(writerNode);
return writerNode;
}
public <I> WriterNode<I> newWriterNode(DataWriter<I> writer, int n) {
List<DataWriter<I>> writers = new ArrayList<>(n);
for (int i = 0; i < n; i++) {
writers.add(writer);
}
WriterNode<I> writerNode = new WriterNode<>(writers, "writer-node-" + writerNodes.size());
writerNodes.add(writerNode);
return writerNode;
}
public void run() {
start();
join();
}
public void start() {
nodes.addAll(readNodes);
nodes.addAll(taskNodes);
nodes.addAll(writerNodes);
for (Node node : nodes) {
node.init();
node.pre();
}
for (ReadNode readNode : readNodes) {
readNode.start();
}
}
public void join() {
boolean allFinalized;
synchronized (syncObject) {
do {
allFinalized = true;
for (Node node : nodes) {
if (!node.isFinished()) {
System.out.println(
"Node "
+ node.name
+ " is not finished pending:"
+ node.pendingJobs
+ " lastBatch:"
+ node.lastBatch);
allFinalized = false;
break;
} /*else {
System.out.println("Node " + node.name + " is finished");
}*/
}
if (!allFinalized) {
try {
System.out.println("WAIT");
syncObject.wait();
System.out.println("NOTIFY");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} while (!allFinalized);
}
for (Node node : nodes) {
node.post();
}
executorService.shutdown();
}
public abstract static class Task<I, O> {
public boolean pre() {
return true;
}
public abstract List<O> apply(List<I> batch) throws IOException;
public boolean post() {
return true;
}
}
public class ReadNode<O> extends Node<Object, O, DataReader<O>> {
private ReadNode(List<DataReader<O>> tasks, String name) {
super(tasks, name);
}
public void start() {
submit(SINGLETON_LIST);
}
@Override
List<O> doJob(List<Object> b) {
// System.out.println(name + " - read start - " );
List<O> reddenBatch = super.doJob(b);
if (reddenBatch != null) {
// System.out.println(name + " - read end - " + reddenBatch.size());
if (!reddenBatch.isEmpty()) {
// System.out.println(name + " - non empty list! - " +
// reddenBatch.size());
start();
} else {
// System.out.println("Empty list! Lets submit the last batch " +
// !isLastBatchSent());
if (!isLastBatchSent()) {
submit(POISON_PILL);
}
}
} else {
// System.out.println(name + " - read end NULL taskQueue.size : " +
// taskQueue.size());
}
return reddenBatch;
}
@Override
protected List<O> execute(DataReader<O> reader, List<Object> ignored) {
List<O> read = reader.read(batchSize);
return read;
}
@Override
protected void pre() {
for (DataReader<O> reader : tasks) {
reader.open();
reader.pre();
}
}
@Override
protected void post() {
for (DataReader<O> reader : tasks) {
reader.post();
reader.close();
}
}
}
public class TaskNode<I, O> extends Node<I, O, Task<I, O>> {
private TaskNode(List<Task<I, O>> tasks, String name) {
super(tasks, name);
}
@Override
protected List<O> execute(Task<I, O> task, List<I> batch) {
try {
return task.apply(batch);
} catch (IOException e) {
e.printStackTrace();
}
return Collections.emptyList();
}
@Override
protected void pre() {
for (Task<I, O> task : tasks) {
task.pre();
}
}
@Override
protected void post() {
for (Task<I, O> task : tasks) {
task.post();
}
}
}
public class SimpleTaskNode<I> extends Node<I, I, org.opencb.commons.run.Task<I>> {
private SimpleTaskNode(List<org.opencb.commons.run.Task<I>> tasks, String name) {
super(tasks, name);
}
@Override
protected void pre() {
for (org.opencb.commons.run.Task<I> task : tasks) {
task.pre();
}
}
@Override
protected void post() {
for (org.opencb.commons.run.Task<I> task : tasks) {
task.post();
}
}
@Override
protected List<I> execute(org.opencb.commons.run.Task<I> task, List<I> batch) {
try {
task.apply(batch);
} catch (IOException e) {
e.printStackTrace();
}
return batch;
}
}
public class WriterNode<I> extends Node<I, Object, DataWriter<I>> {
private WriterNode(List<DataWriter<I>> tasks, String name) {
super(tasks, name);
}
@Override
protected void pre() {
for (DataWriter<I> writer : tasks) {
writer.open();
writer.pre();
}
}
@Override
protected void post() {
for (DataWriter<I> writer : tasks) {
writer.post();
writer.close();
}
}
@Override
protected List<Object> execute(DataWriter<I> writer, List<I> batch) {
writer.write(batch);
return SINGLETON_LIST;
}
}
abstract class Node<I, O, EXECUTOR> {
protected final List<EXECUTOR> tasks;
protected final String name;
private final BlockingQueue<EXECUTOR> taskQueue;
private List<Node<O, ?, ?>> nodes;
private int pendingJobs;
private boolean lastBatch;
private boolean lastBatchSent;
public Node(List<EXECUTOR> tasks, String name) {
this.tasks = tasks;
this.name = name;
taskQueue = new ArrayBlockingQueue<>(tasks.size(), false, tasks);
nodes = new LinkedList<>();
}
/* package */ void init() {
pendingJobs = 0;
lastBatch = false;
lastBatchSent = false;
}
protected abstract void pre();
protected abstract void post();
/*package*/ List<O> doJob(List<I> batch) {
List<O> generatedBatch;
assert lastBatchSent == false;
if (batch == POISON_PILL) {
lastBatch = true;
synchronized (name) {
pendingJobs--;
}
// System.out.println(name + " - lastBatch");
generatedBatch = Collections.emptyList();
} else {
EXECUTOR task = taskQueue.poll();
boolean nextNodesAvailable = true;
for (Node<O, ?, ?> node : nodes) {
nextNodesAvailable &= node.isAvailable();
}
if (task == null) { // No available task
resubmit(batch);
generatedBatch = null;
} else if (!nextNodesAvailable) { // Next nodes have to many batches.
try {
taskQueue.put(task);
} catch (InterruptedException e) {
e.printStackTrace();
}
resubmit(batch);
generatedBatch = null;
} else { // Execute
generatedBatch = execute(task, batch);
// System.out.println(name + " - end job - " + generatedBatch.size());
for (Node<O, ?, ?> node : nodes) {
node.submit(generatedBatch);
}
try {
taskQueue.put(task);
} catch (InterruptedException e) {
e.printStackTrace();
}
synchronized (name) {
pendingJobs--;
}
// System.out.println(name + " - pendingJobs " + pendingJobs);
}
}
if (isFinished()) {
if (!lastBatchSent) {
for (Node<O, ?, ?> node : nodes) {
node.submit(POISON_PILL);
}
lastBatchSent = true;
}
System.out.println("Node '" + name + "' is finished");
synchronized (syncObject) {
syncObject.notify();
}
} else {
System.out.println("Node '" + name + "' pendingJobs " + pendingJobs);
}
return generatedBatch;
}
protected abstract List<O> execute(EXECUTOR task, List<I> batch);
private void resubmit(final List<I> batch) {
executorService.submit(
new Runnable() {
public void run() {
doJob(batch);
}
});
}
/*package*/ void submit(final List<I> batch) {
// System.out.println("Submitting batch: pendingJobs = " + pendingJobs + " - " +
// "[" + (isAvailable()? " " : "*") + "]" + name + " - " + Thread.currentThread().getName());
pendingJobs++;
resubmit(batch);
}
public boolean isAvailable() {
return pendingJobs < tasks.size();
}
public boolean isFinished() {
return pendingJobs == 0 && lastBatch;
}
public boolean isLastBatchSent() {
return lastBatchSent;
}
public Node<I, O, EXECUTOR> append(Node<O, ?, ?> node) {
nodes.add(node);
return this;
}
}
}
/** @throws Exception If failed. */
public void testEmptyProjections() throws Exception {
final GridClientCompute dflt = client.compute();
Collection<? extends GridClientNode> nodes = dflt.nodes();
assertEquals(NODES_CNT, nodes.size());
Iterator<? extends GridClientNode> iter = nodes.iterator();
final GridClientCompute singleNodePrj = dflt.projection(Collections.singletonList(iter.next()));
final GridClientNode second = iter.next();
final GridClientPredicate<GridClientNode> noneFilter =
new GridClientPredicate<GridClientNode>() {
@Override
public boolean apply(GridClientNode node) {
return false;
}
};
final GridClientPredicate<GridClientNode> targetFilter =
new GridClientPredicate<GridClientNode>() {
@Override
public boolean apply(GridClientNode node) {
return node.nodeId().equals(second.nodeId());
}
};
GridTestUtils.assertThrows(
log(),
new Callable<Object>() {
@Override
public Object call() throws Exception {
return dflt.projection(noneFilter).log(-1, -1);
}
},
GridServerUnreachableException.class,
null);
GridTestUtils.assertThrows(
log(),
new Callable<Object>() {
@Override
public Object call() throws Exception {
return singleNodePrj.projection(second);
}
},
GridClientException.class,
null);
GridTestUtils.assertThrows(
log(),
new Callable<Object>() {
@Override
public Object call() throws Exception {
return singleNodePrj.projection(targetFilter);
}
},
GridClientException.class,
null);
}
/**
* Create a new broker connection.
*
* <p>If <a href="http://www.rabbitmq.com/api-guide.html#recovery">automatic connection
* recovery</a> is enabled, the connection returned by this method will be {@link Recoverable}.
* Reconnection attempts will always use the address configured on {@link ConnectionFactory}.
*
* @param connectionName arbitrary sring for connection name client property
* @return an interface to the connection
* @throws IOException if it encounters a problem
*/
public Connection newConnection(String connectionName) throws IOException, TimeoutException {
return newConnection(
this.sharedExecutor,
Collections.singletonList(new Address(getHost(), getPort())),
connectionName);
}
/**
* Create a new broker connection.
*
* <p>If <a href="http://www.rabbitmq.com/api-guide.html#recovery">automatic connection
* recovery</a> is enabled, the connection returned by this method will be {@link Recoverable}.
* Reconnection attempts will always use the address configured on {@link ConnectionFactory}.
*
* @param executor thread execution service for consumers on the connection
* @return an interface to the connection
* @throws IOException if it encounters a problem
*/
public Connection newConnection(ExecutorService executor) throws IOException, TimeoutException {
return newConnection(executor, Collections.singletonList(new Address(getHost(), getPort())));
}