/**
* jestClient.
*
* @return a {@link io.searchbox.client.http.JestHttpClient} object.
*/
@Bean
public JestHttpClient jestClient() {
// Construct a new Jest client according to configuration via factory
JestClientFactory factory = new JestClientFactory();
HttpClientConfig httpClientConfig =
new HttpClientConfig.Builder(getConnectionURL())
.multiThreaded(true)
.connTimeout(Constants.CONNTIMEOUT)
.readTimeout(Constants.READTIMEOUT)
.maxTotalConnection(Constants.MAXTOTALCONNECTION)
// .requestCompressionEnabled(true)
.gson(new GsonBuilder().setDateFormat(DATESTYLE).create())
.build();
factory.setHttpClientConfig(httpClientConfig);
JestHttpClient jestHttpClient = (JestHttpClient) factory.getObject();
CloseableHttpClient closableHttpClient = HttpClientBuilder.create().build();
jestHttpClient.setHttpClient(closableHttpClient);
return jestHttpClient;
}
@Test
public void testNoIdleConnectionReaper() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(3);
assertEquals(
"All nodes in cluster should have HTTP endpoint exposed",
3,
cluster().httpAddresses().length);
factory.setHttpClientConfig(
new HttpClientConfig.Builder("http://localhost:" + cluster().httpAddresses()[0].getPort())
.multiThreaded(true)
.discoveryEnabled(true)
.discoveryFrequency(100l, TimeUnit.MILLISECONDS)
.maxTotalConnection(75)
.defaultMaxTotalConnectionPerRoute(75)
.build());
JestHttpClient jestClient = (JestHttpClient) factory.getObject();
assertNotNull(jestClient);
Thread.sleep(
300L); // Allow nodechecker to do it's thing and use at least one connection in the pool
// Ask for the cluster health just to use some connections and create a little white noise
int maxPoolSize = getPoolSize(jestClient);
for (int x = 0; x < 5; ++x) {
jestClient.execute(new Health.Builder().build());
maxPoolSize = Math.max(maxPoolSize, getPoolSize(jestClient));
}
Thread.sleep(3000); // Allow for a quiesce period of no activity (except for nodechecker)
int newPoolSize = getPoolSize(jestClient);
// These two values being equal proves that connections returned to the pool stick around for
// some non-zero
// duration of time while they wait to be re-leased. It's impractical to prove in an
// integration test that they
// can in fact stay around for over an hour without ever being used (by which time the server
// has most certainly
// closed the connection).
assertEquals(maxPoolSize, newPoolSize);
jestClient.shutdownClient();
}
@Test
public void testIdleConnectionReaper() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(3);
assertEquals(
"All nodes in cluster should have HTTP endpoint exposed",
3,
cluster().httpAddresses().length);
factory.setHttpClientConfig(
new HttpClientConfig.Builder("http://localhost:" + cluster().httpAddresses()[0].getPort())
.multiThreaded(true)
.discoveryEnabled(true)
.discoveryFrequency(100l, TimeUnit.MILLISECONDS)
.maxConnectionIdleTime(1500L, TimeUnit.MILLISECONDS)
.maxTotalConnection(75)
.defaultMaxTotalConnectionPerRoute(75)
.build());
JestHttpClient jestClient = (JestHttpClient) factory.getObject();
assertNotNull(jestClient);
Thread.sleep(
300L); // Allow nodechecker to do it's thing and use at least one connection in the pool
// Ask for the cluster health just to use some connections
int maxPoolSize = getPoolSize(jestClient);
for (int x = 0; x < 5; ++x) {
jestClient.execute(new Health.Builder().build());
maxPoolSize = Math.max(maxPoolSize, getPoolSize(jestClient));
}
Thread.sleep(3200); // Allow cxn reaper a chance to do it's thing
int newPoolSize = getPoolSize(jestClient);
// The new pool size should be much less than the maxPoolSize since the idle connection reaper
// will have run
// twice in the time between maxPoolSize's last calculation and now. There should really only
// be 1-2 connections
// in the pool at this point since our idle timeout is set so low for this test.
assertTrue(maxPoolSize > newPoolSize);
jestClient.shutdownClient();
}
/** cleanup. */
@PreDestroy
public void closeClients() {
LOGGER.info("Spring Container is destroy! Customer clean up");
try {
jestClient.shutdownClient();
mongoClient.close();
} catch (Exception e) {
LOGGER.error("Exception while closing {}", e.getMessage(), e);
}
LOGGER.info("Spring Container is destroy! Customer clean up completed");
}
@Test
public void testDiscoveryWithFiltering() throws InterruptedException, IOException {
// wait for 3 active nodes
internalCluster().ensureAtLeastNumDataNodes(3);
// spin up two more client nodes with additional attributes
Settings settings =
Settings.builder()
.put(internalCluster().getDefaultSettings())
.put("node.master", false) // for example, a client node
.put("node.data", false)
.put("node.type", "aardvark") // put some arbitrary attribute to filter by
.build();
String clientNode1 = internalCluster().startNode(settings);
String clientNode2 = internalCluster().startNode(settings);
assertNotEquals("client nodes should be different", clientNode1, clientNode2);
assertEquals(
"All nodes in cluster should have HTTP endpoint exposed",
5,
cluster().httpAddresses().length);
factory.setHttpClientConfig(
new HttpClientConfig.Builder("http://localhost:" + cluster().httpAddresses()[0].getPort())
.discoveryEnabled(true)
.discoveryFilter("type:aardvark")
.discoveryFrequency(500l, TimeUnit.MILLISECONDS)
.build());
JestHttpClient jestClient = (JestHttpClient) factory.getObject();
assertNotNull(jestClient);
// wait for NodeChecker to do the discovery
Thread.sleep(3000);
assertEquals(
"Only 2 nodes should be discovered and be in the client's server list",
2,
jestClient.getServerPoolSize());
jestClient.shutdownClient();
}
@Test
public void testDiscovery() throws InterruptedException, IOException {
// wait for 4 active nodes
internalCluster().ensureAtLeastNumDataNodes(4);
assertEquals(
"All nodes in cluster should have HTTP endpoint exposed",
4,
cluster().httpAddresses().length);
factory.setHttpClientConfig(
new HttpClientConfig.Builder("http://localhost:" + cluster().httpAddresses()[0].getPort())
.discoveryEnabled(true)
.discoveryFrequency(500l, TimeUnit.MILLISECONDS)
.build());
JestHttpClient jestClient = (JestHttpClient) factory.getObject();
assertNotNull(jestClient);
// wait for NodeChecker to do the discovery
Thread.sleep(3000);
assertEquals(
"All 4 nodes should be discovered and be in the client's server list",
4,
jestClient.getServerPoolSize());
internalCluster().ensureAtMostNumDataNodes(3);
int numServers = 0;
int retries = 0;
while (numServers != 3 && retries < 30) {
numServers = jestClient.getServerPoolSize();
retries++;
Thread.sleep(1000);
}
assertEquals("Only 3 nodes should be in Jest's list", 3, jestClient.getServerPoolSize());
jestClient.shutdownClient();
}
/**
* Forgive me these sins. This is the only way I can think of to determine the *actual* size of
* the connection pool without wrapping large quantities of the underlying client.
*
* <p>This whole method is cheating and full of bad examples. Don't copy this. You've been warned.
*/
private int getPoolSize(JestHttpClient client) throws Exception {
try {
Field fieldHttpClient = client.getClass().getDeclaredField("httpClient");
fieldHttpClient.setAccessible(true);
Object objInternalHttpClient = fieldHttpClient.get(client);
Field fieldConnectionManager =
objInternalHttpClient.getClass().getDeclaredField("connManager");
fieldConnectionManager.setAccessible(true);
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager =
(PoolingHttpClientConnectionManager) fieldConnectionManager.get(objInternalHttpClient);
PoolStats poolStats = poolingHttpClientConnectionManager.getTotalStats();
return poolStats.getAvailable() + poolStats.getLeased();
} catch (Exception e) {
e.printStackTrace();
}
return -1;
}
