@AfterMethod
public void tearDown() {
serviceA.close();
serviceB.close();
threadPool.shutdown();
}
public NodesFaultDetection(
Settings settings, ThreadPool threadPool, TransportService transportService) {
super(settings);
this.threadPool = threadPool;
this.transportService = transportService;
this.connectOnNetworkDisconnect =
componentSettings.getAsBoolean("connect_on_network_disconnect", true);
this.pingInterval = componentSettings.getAsTime("ping_interval", timeValueSeconds(1));
this.pingRetryTimeout = componentSettings.getAsTime("ping_timeout", timeValueSeconds(30));
this.pingRetryCount = componentSettings.getAsInt("ping_retries", 3);
this.registerConnectionListener =
componentSettings.getAsBoolean("register_connection_listener", true);
logger.debug(
"[node ] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]",
pingInterval,
pingRetryTimeout,
pingRetryCount);
transportService.registerHandler(PingRequestHandler.ACTION, new PingRequestHandler());
this.connectionListener = new FDConnectionListener();
if (registerConnectionListener) {
transportService.addConnectionListener(connectionListener);
}
}
@Test
public void testTimeoutSendExceptionWithNeverSendingBackResponse() throws Exception {
serviceA.registerHandler(
"sayHelloTimeoutNoResponse",
new BaseTransportRequestHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void messageReceived(StringMessage request, TransportChannel channel) {
System.out.println("got message: " + request.message);
assertThat("moshe", equalTo(request.message));
// don't send back a response
// try {
// channel.sendResponse(new StringMessage("hello " +
// request.message));
// } catch (IOException e) {
// e.printStackTrace();
// assertThat(e.getMessage(), false, equalTo(true));
// }
}
});
TransportFuture<StringMessage> res =
serviceB.submitRequest(
serviceANode,
"sayHelloTimeoutNoResponse",
new StringMessage("moshe"),
options().withTimeout(100),
new BaseTransportResponseHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void handleResponse(StringMessage response) {
assertThat("got response instead of exception", false, equalTo(true));
}
@Override
public void handleException(RemoteTransportException exp) {
assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class));
}
});
try {
StringMessage message = res.txGet();
assertThat("exception should be thrown", false, equalTo(true));
} catch (Exception e) {
assertThat(e, instanceOf(ReceiveTimeoutTransportException.class));
}
serviceA.removeHandler("sayHelloTimeoutNoResponse");
System.out.println("after ...");
}
@BeforeMethod
public void setUp() {
threadPool = new CachedThreadPool();
timerService = new TimerService(threadPool);
build();
serviceA.connectToNode(serviceBNode);
serviceB.connectToNode(serviceANode);
}
@Test
public void testErrorMessage() {
serviceA.registerHandler(
"sayHelloException",
new BaseTransportRequestHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void messageReceived(StringMessage request, TransportChannel channel)
throws Exception {
System.out.println("got message: " + request.message);
assertThat("moshe", equalTo(request.message));
throw new RuntimeException("bad message !!!");
}
});
TransportFuture<StringMessage> res =
serviceB.submitRequest(
serviceANode,
"sayHelloException",
new StringMessage("moshe"),
new BaseTransportResponseHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void handleResponse(StringMessage response) {
assertThat("got response instead of exception", false, equalTo(true));
}
@Override
public void handleException(RemoteTransportException exp) {
assertThat("bad message !!!", equalTo(exp.getCause().getMessage()));
}
});
try {
res.txGet();
assertThat("exception should be thrown", false, equalTo(true));
} catch (Exception e) {
assertThat("bad message !!!", equalTo(e.getCause().getMessage()));
}
serviceA.removeHandler("sayHelloException");
System.out.println("after ...");
}
private void handleTransportDisconnect(DiscoveryNode node) {
if (!latestNodes.nodeExists(node.id())) {
return;
}
NodeFD nodeFD = nodesFD.remove(node);
if (nodeFD == null) {
return;
}
if (!running) {
return;
}
nodeFD.running = false;
if (connectOnNetworkDisconnect) {
try {
transportService.connectToNode(node);
nodesFD.put(node, new NodeFD());
threadPool.schedule(pingInterval, ThreadPool.Names.SAME, new SendPingRequest(node));
} catch (Exception e) {
logger.trace("[node ] [{}] transport disconnected (with verified connect)", node);
notifyNodeFailure(node, "transport disconnected (with verified connect)");
}
} else {
logger.trace("[node ] [{}] transport disconnected", node);
notifyNodeFailure(node, "transport disconnected");
}
}
@Test
public void testDisconnectListener() throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
TransportConnectionListener disconnectListener =
new TransportConnectionListener() {
@Override
public void onNodeConnected(DiscoveryNode node) {
throw new RuntimeException("Should not be called");
}
@Override
public void onNodeDisconnected(DiscoveryNode node) {
latch.countDown();
}
};
serviceA.addConnectionListener(disconnectListener);
serviceB.close();
assertThat(latch.await(5, TimeUnit.SECONDS), equalTo(true));
}
private void handleTransportDisconnect(DiscoveryNode node) {
synchronized (masterNodeMutex) {
if (!node.equals(this.masterNode)) {
return;
}
if (connectOnNetworkDisconnect) {
try {
transportService.connectToNode(node);
} catch (Exception e) {
logger.trace("[master] [{}] transport disconnected (with verified connect)", masterNode);
notifyMasterFailure(masterNode, "transport disconnected (with verified connect)");
}
} else {
logger.trace("[master] [{}] transport disconnected", node);
notifyMasterFailure(node, "transport disconnected");
}
}
}
private void innerStart(final DiscoveryNode masterNode) {
this.masterNode = masterNode;
this.retryCount = 0;
this.notifiedMasterFailure.set(false);
// try and connect to make sure we are connected
try {
transportService.connectToNode(masterNode);
} catch (final Exception e) {
// notify master failure (which stops also) and bail..
notifyMasterFailure(masterNode, "failed to perform initial connect [" + e.getMessage() + "]");
return;
}
if (masterPinger != null) {
masterPinger.stop();
}
this.masterPinger = new MasterPinger();
// start the ping process
threadPool.schedule(masterPinger, pingInterval);
}
public void close() {
stop();
transportService.removeHandler(PingRequestHandler.ACTION);
transportService.removeConnectionListener(connectionListener);
}
public void close() {
stop("closing");
this.listeners.clear();
transportService.removeConnectionListener(connectionListener);
transportService.removeHandler(MasterPingRequestHandler.ACTION);
}
@Test
public void testHelloWorld() {
serviceA.registerHandler(
"sayHello",
new BaseTransportRequestHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void messageReceived(StringMessage request, TransportChannel channel) {
System.out.println("got message: " + request.message);
assertThat("moshe", equalTo(request.message));
try {
channel.sendResponse(new StringMessage("hello " + request.message));
} catch (IOException e) {
e.printStackTrace();
assertThat(e.getMessage(), false, equalTo(true));
}
}
});
TransportFuture<StringMessage> res =
serviceB.submitRequest(
serviceANode,
"sayHello",
new StringMessage("moshe"),
new BaseTransportResponseHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void handleResponse(StringMessage response) {
System.out.println("got response: " + response.message);
assertThat("hello moshe", equalTo(response.message));
}
@Override
public void handleException(RemoteTransportException exp) {
exp.printStackTrace();
assertThat(
"got exception instead of a response: " + exp.getMessage(),
false,
equalTo(true));
}
});
try {
StringMessage message = res.get();
assertThat("hello moshe", equalTo(message.message));
} catch (Exception e) {
assertThat(e.getMessage(), false, equalTo(true));
}
serviceA.removeHandler("sayHello");
System.out.println("after ...");
}
@Test
public void testTimeoutSendExceptionWithDelayedResponse() throws Exception {
serviceA.registerHandler(
"sayHelloTimeoutDelayedResponse",
new BaseTransportRequestHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void messageReceived(StringMessage request, TransportChannel channel) {
System.out.println("got message: " + request.message);
TimeValue sleep = TimeValue.parseTimeValue(request.message, null);
try {
Thread.sleep(sleep.millis());
} catch (InterruptedException e) {
// ignore
}
try {
channel.sendResponse(new StringMessage("hello " + request.message));
} catch (IOException e) {
e.printStackTrace();
assertThat(e.getMessage(), false, equalTo(true));
}
}
});
TransportFuture<StringMessage> res =
serviceB.submitRequest(
serviceANode,
"sayHelloTimeoutDelayedResponse",
new StringMessage("300ms"),
options().withTimeout(100),
new BaseTransportResponseHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void handleResponse(StringMessage response) {
assertThat("got response instead of exception", false, equalTo(true));
}
@Override
public void handleException(RemoteTransportException exp) {
assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class));
}
});
try {
StringMessage message = res.txGet();
assertThat("exception should be thrown", false, equalTo(true));
} catch (Exception e) {
assertThat(e, instanceOf(ReceiveTimeoutTransportException.class));
}
// sleep for 400 millis to make sure we get back the response
Thread.sleep(400);
for (int i = 0; i < 10; i++) {
final int counter = i;
// now, try and send another request, this times, with a short timeout
res =
serviceB.submitRequest(
serviceANode,
"sayHelloTimeoutDelayedResponse",
new StringMessage(counter + "ms"),
options().withTimeout(100),
new BaseTransportResponseHandler<StringMessage>() {
@Override
public StringMessage newInstance() {
return new StringMessage();
}
@Override
public void handleResponse(StringMessage response) {
System.out.println("got response: " + response.message);
assertThat("hello " + counter + "ms", equalTo(response.message));
}
@Override
public void handleException(RemoteTransportException exp) {
exp.printStackTrace();
assertThat(
"got exception instead of a response for "
+ counter
+ ": "
+ exp.getDetailedMessage(),
false,
equalTo(true));
}
});
StringMessage message = res.txGet();
assertThat(message.message, equalTo("hello " + counter + "ms"));
}
serviceA.removeHandler("sayHelloTimeoutDelayedResponse");
System.out.println("after ...");
}