protected RequestConfig configureRequest(RequestConfig.Builder rcb, Settings settings)
throws HTTPException {
// Configure the RequestConfig
for (Prop key : settings.getKeys()) {
Object value = settings.getParameter(key);
boolean tf = (value instanceof Boolean ? (Boolean) value : false);
if (key == Prop.ALLOW_CIRCULAR_REDIRECTS) {
rcb.setCircularRedirectsAllowed(tf);
} else if (key == Prop.HANDLE_REDIRECTS) {
rcb.setRedirectsEnabled(tf);
rcb.setRelativeRedirectsAllowed(tf);
} else if (key == Prop.MAX_REDIRECTS) {
rcb.setMaxRedirects((Integer) value);
} else if (key == Prop.SO_TIMEOUT) {
rcb.setSocketTimeout((Integer) value);
} else if (key == Prop.CONN_TIMEOUT) {
rcb.setConnectTimeout((Integer) value);
} else if (key == Prop.CONN_REQ_TIMEOUT) {
rcb.setConnectionRequestTimeout((Integer) value);
} else if (key == Prop.MAX_THREADS) {
connmgr.setMaxTotal((Integer) value);
connmgr.setDefaultMaxPerRoute((Integer) value);
} /* else ignore */
}
RequestConfig cfg = rcb.build();
return cfg;
}
MyDefaultApacheHttpClient4Config() {
PoolingHttpClientConnectionManager cm;
if (systemSSL) {
cm = createSystemSslCM();
} else if (trustStoreFileName != null) {
cm = createCustomSslCM();
} else {
cm = new PoolingHttpClientConnectionManager();
}
if (proxyHost != null) {
addProxyConfiguration();
}
DiscoveryJerseyProvider discoveryJerseyProvider =
new DiscoveryJerseyProvider(encoderWrapper, decoderWrapper);
// getSingletons().add(discoveryJerseyProvider);
register(discoveryJerseyProvider);
// Common properties to all clients
cm.setDefaultMaxPerRoute(maxConnectionsPerHost);
cm.setMaxTotal(maxTotalConnections);
property(ApacheClientProperties.CONNECTION_MANAGER, cm);
String fullUserAgentName =
(userAgent == null ? clientName : userAgent) + "/v" + buildVersion();
property(CoreProtocolPNames.USER_AGENT, fullUserAgentName);
// To pin a client to specific server in case redirect happens, we handle redirects directly
// (see DiscoveryClient.makeRemoteCall methods).
property(ClientProperties.FOLLOW_REDIRECTS, Boolean.FALSE);
property(ClientPNames.HANDLE_REDIRECTS, Boolean.FALSE);
}
/**
* @param maxTotal maxTotal
* @param maxPerRoute maxPerRoute
* @param timeout timeout
* @param retryExecutionCount retryExecutionCount
* @return
*/
public static CloseableHttpClient createHttpClient(
int maxTotal, int maxPerRoute, int timeout, int retryExecutionCount) {
try {
SSLContext sslContext = SSLContexts.custom().useSSL().build();
SSLConnectionSocketFactory sf =
new SSLConnectionSocketFactory(
sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager =
new PoolingHttpClientConnectionManager();
poolingHttpClientConnectionManager.setMaxTotal(maxTotal);
poolingHttpClientConnectionManager.setDefaultMaxPerRoute(maxPerRoute);
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(timeout).build();
poolingHttpClientConnectionManager.setDefaultSocketConfig(socketConfig);
return HttpClientBuilder.create()
.setConnectionManager(poolingHttpClientConnectionManager)
.setSSLSocketFactory(sf)
.setRetryHandler(new HttpRequestRetryHandlerImpl(retryExecutionCount))
.build();
} catch (KeyManagementException e) {
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
return null;
}
/**
* Sets the timeouts of the HTTP client.
*
* @param connectionTimeout timeout until connection established in milliseconds. Zero means no
* timeout.
* @param socketTimeout timeout for waiting for data in milliseconds. Zero means no timeout.
* @param maxRequests maximum number of connections to a particuar host
*/
public static void setParams(int connectionTimeout, int socketTimeout, int maxRequests) {
PrudentHttpEntityResolver.maxRequests = maxRequests;
PoolingHttpClientConnectionManager phcConnMgr;
Registry<ConnectionSocketFactory> registry = //
RegistryBuilder.<ConnectionSocketFactory>create() //
.register("http", PlainConnectionSocketFactory.getSocketFactory()) //
.register("https", SSLConnectionSocketFactory.getSocketFactory()) //
.build();
HttpClientBuilder builder = HttpClients.custom();
builder.setRedirectStrategy(new LaxRedirectStrategy());
builder.setMaxConnPerRoute(maxRequests);
builder.setMaxConnTotal(200);
if ("true".equals(System.getProperty("nu.validator.xml.promiscuous-ssl", "false"))) { //
try {
SSLContext promiscuousSSLContext =
new SSLContextBuilder() //
.loadTrustMaterial(
null,
new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] arg0, String arg1)
throws CertificateException {
return true;
}
})
.build();
builder.setSslcontext(promiscuousSSLContext);
HostnameVerifier verifier = //
SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;
SSLConnectionSocketFactory promiscuousSSLConnSocketFactory = //
new SSLConnectionSocketFactory(promiscuousSSLContext, verifier);
registry =
RegistryBuilder.<ConnectionSocketFactory>create() //
.register("https", promiscuousSSLConnSocketFactory) //
.register("http", PlainConnectionSocketFactory.getSocketFactory()) //
.build();
} catch (KeyManagementException
| KeyStoreException
| NoSuchAlgorithmException
| NumberFormatException e) {
e.printStackTrace();
}
}
phcConnMgr = new PoolingHttpClientConnectionManager(registry);
phcConnMgr.setDefaultMaxPerRoute(maxRequests);
phcConnMgr.setMaxTotal(200);
builder.setConnectionManager(phcConnMgr);
RequestConfig.Builder config = RequestConfig.custom();
config.setCircularRedirectsAllowed(true);
config.setMaxRedirects(20); // Gecko default
config.setConnectTimeout(connectionTimeout);
config.setCookieSpec(CookieSpecs.BEST_MATCH);
config.setSocketTimeout(socketTimeout);
client = builder.setDefaultRequestConfig(config.build()).build();
}
private static HttpClientConnectionManager makeConnectionManager(int maxConnections) {
// Jersey requires an instance of the ClientConnectionManager interface which is deprecated in
// the latest
// version of the Apache HttpComponents. In a future version, this implementation should be
// updated to
// the PoolingHttpClientConnectionManager and the HttpClientConnectionManager.
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(maxConnections);
connectionManager.setDefaultMaxPerRoute(maxConnections);
return connectionManager;
}
@Bean
RestTemplate restTemplate() {
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);
HttpClientBuilder httpClient =
HttpClientBuilder.create().setConnectionManager(connectionManager);
HttpComponentsClientHttpRequestFactory httpRequestFactory =
new HttpComponentsClientHttpRequestFactory(httpClient.build());
return new RestTemplate(httpRequestFactory);
}
@Bean
public Client webClient() {
final PoolingHttpClientConnectionManager poolingConnectionManager =
new PoolingHttpClientConnectionManager();
poolingConnectionManager.setMaxTotal(webClientConnectionPoolMaxTotal);
poolingConnectionManager.setDefaultMaxPerRoute(webClientConnectionPoolDefaultMaxPerRoute);
final ClientConfig clientConfig = new ClientConfig();
clientConfig.property(READ_TIMEOUT, 2000);
clientConfig.property(CONNECT_TIMEOUT, 500);
clientConfig.property(CONNECTION_MANAGER, poolingConnectionManager);
clientConfig.connectorProvider(new ApacheConnectorProvider());
return newClient(clientConfig).register(jacksonJsonProvider());
}
private MozuHttpClientPool() {
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
// Increase max total connection to 200
cm.setMaxTotal(MozuConfig.getMaxHttpClientConnections());
// Increase default max connection per route to 20
cm.setDefaultMaxPerRoute(MozuConfig.getDefaultHttpClientMaxPerRoute());
ConnectionKeepAliveStrategy myStrategy =
new ConnectionKeepAliveStrategy() {
@Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
HeaderElementIterator it =
new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String param = he.getName();
String value = he.getValue();
if (value != null && param.equalsIgnoreCase("timeout")) {
return Long.parseLong(value) * 1000;
}
}
return 10 * 1000;
}
};
RequestConfig requestConfig =
RequestConfig.custom()
.setConnectionRequestTimeout(MozuConfig.getHttpClientTimeoutMillis())
.setSocketTimeout(MozuConfig.getHttpClientTimeoutMillis())
.setConnectionRequestTimeout(MozuConfig.getHttpClientTimeoutMillis())
.build();
// Build the client.
threadSafeClient =
HttpClients.custom()
.setConnectionManager(cm)
.useSystemProperties()
.disableConnectionState()
.setDefaultRequestConfig(requestConfig)
.setKeepAliveStrategy(myStrategy)
.build();
monitor = new IdleConnectionMonitorThread(cm);
monitor.setDaemon(true);
monitor.start();
}
@Override
protected void configureClient(ClientConfig config) {
// using AHC as a test client connector to avoid issues with HttpUrlConnection socket
// management.
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
// adjusting max. connections just to be safe - the testEventSourceReconnect is quite greedy...
cm.setMaxTotal(MAX_LISTENERS * MAX_ITEMS);
cm.setDefaultMaxPerRoute(MAX_LISTENERS * MAX_ITEMS);
config
.register(SseFeature.class)
.property(ApacheClientProperties.CONNECTION_MANAGER, cm)
.property(ClientProperties.READ_TIMEOUT, 2000)
.connector(new ApacheConnector(config));
}
@Override
public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Exception {
LOG.debug("HTTP connection manager stats {}", CONNECTION_MANAGER.getTotalStats());
HttpGet httpget = new HttpGet(url);
httpget.setConfig(requestConfig);
if (md != null) {
String ifModifiedSince = md.getFirstValue("cachedLastModified");
if (StringUtils.isNotBlank(ifModifiedSince)) {
httpget.addHeader("If-Modified-Since", ifModifiedSince);
}
String ifNoneMatch = md.getFirstValue("cachedEtag");
if (StringUtils.isNotBlank(ifNoneMatch)) {
httpget.addHeader("If-None-Match", ifNoneMatch);
}
}
// no need to release the connection explicitly as this is handled
// automatically. The client itself must be closed though.
try (CloseableHttpClient httpclient = builder.build()) {
return httpclient.execute(httpget, this);
}
}
@Override
public void run() {
try {
while (!shutdown) {
synchronized (this) {
wait(5000);
// Close expired connections
connMgr.closeExpiredConnections();
// Optionally, close connections that have been idle longer than 30 sec
connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
}
}
} catch (InterruptedException ex) {
// terminate
}
}
private static PoolingHttpClientConnectionManager bulidConnectionManager() {
MessageConstraints messageConstraints =
MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build();
ConnectionConfig connectionConfig =
ConnectionConfig.custom()
.setMalformedInputAction(CodingErrorAction.IGNORE)
.setUnmappableInputAction(CodingErrorAction.IGNORE)
.setCharset(Consts.UTF_8)
.setMessageConstraints(messageConstraints)
.build();
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(600);
connectionManager.setDefaultMaxPerRoute(200);
connectionManager.setDefaultConnectionConfig(connectionConfig);
connectionManager.setValidateAfterInactivity(1000);
return connectionManager;
}
protected PoolingHttpClientConnectionManager getPoolingHttpClientConnectionManager() {
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = null;
if (_keyStore != null) {
poolingHttpClientConnectionManager =
new PoolingHttpClientConnectionManager(
getSocketFactoryRegistry(), null, null, null, 60000, TimeUnit.MILLISECONDS);
} else {
poolingHttpClientConnectionManager =
new PoolingHttpClientConnectionManager(60000, TimeUnit.MILLISECONDS);
}
poolingHttpClientConnectionManager.setMaxTotal(20);
return poolingHttpClientConnectionManager;
}
@Override
public void run() {
try {
if (this != null) logger.log(Level.SEVERE, "Thread Running: " + getName());
client.execute(get);
if (connManager != null) {
logger.log(Level.SEVERE, "Leased Connections " + connManager.getTotalStats().getLeased());
leasedConn = connManager.getTotalStats().getLeased();
logger.log(
Level.SEVERE, "Available Connections " + connManager.getTotalStats().getAvailable());
}
} catch (final ClientProtocolException ex) {
} catch (final IOException ex) {
}
}
public HttpClientGenerator() {
Registry<ConnectionSocketFactory> reg =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", SSLConnectionSocketFactory.getSocketFactory())
.build();
connectionManager = new PoolingHttpClientConnectionManager(reg);
connectionManager.setDefaultMaxPerRoute(100);
}
private static synchronized void setup() {
if (properties == null) {
properties = new Properties();
try {
InputStream is = ConanUtils.class.getResourceAsStream("/sampletabconverters.properties");
properties.load(is);
} catch (IOException e) {
log.error("Unable to read resource sampletabconverters.properties", e);
}
}
if (conman == null) {
conman = new PoolingHttpClientConnectionManager();
conman.setMaxTotal(128);
conman.setDefaultMaxPerRoute(64);
conman.setValidateAfterInactivity(0);
ConnectionKeepAliveStrategy keepAliveStrategy =
new ConnectionKeepAliveStrategy() {
@Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
// see if the user provides a live time
HeaderElementIterator it =
new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String param = he.getName();
String value = he.getValue();
if (value != null && param.equalsIgnoreCase("timeout")) {
return Long.parseLong(value) * 1000;
}
}
// default to one second live time
return 1 * 1000;
}
};
httpClient =
HttpClients.custom()
.setKeepAliveStrategy(keepAliveStrategy)
.setConnectionManager(conman)
.build();
}
}
/**
* Forgive me these sins. This is the only way I can think of to determine the *actual* size of
* the connection pool without wrapping large quantities of the underlying client.
*
* <p>This whole method is cheating and full of bad examples. Don't copy this. You've been warned.
*/
private int getPoolSize(JestHttpClient client) throws Exception {
try {
Field fieldHttpClient = client.getClass().getDeclaredField("httpClient");
fieldHttpClient.setAccessible(true);
Object objInternalHttpClient = fieldHttpClient.get(client);
Field fieldConnectionManager =
objInternalHttpClient.getClass().getDeclaredField("connManager");
fieldConnectionManager.setAccessible(true);
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager =
(PoolingHttpClientConnectionManager) fieldConnectionManager.get(objInternalHttpClient);
PoolStats poolStats = poolingHttpClientConnectionManager.getTotalStats();
return poolStats.getAvailable() + poolStats.getLeased();
} catch (Exception e) {
e.printStackTrace();
}
return -1;
}
@Scheduled(fixedRate = 20000)
public void httpConnectionRelease() {
PoolingHttpClientConnectionManager connectionManager =
(PoolingHttpClientConnectionManager)
ContextLoader.getCurrentWebApplicationContext().getBean("httpPoolManager");
if (logger.isInfoEnabled()) {
logger.info(
"release start connect count:=" + connectionManager.getTotalStats().getAvailable());
}
// Close expired connections
connectionManager.closeExpiredConnections();
// Optionally, close connections
// that have been idle longer than readTimeout*2 MILLISECONDS
connectionManager.closeIdleConnections(60000, TimeUnit.MILLISECONDS);
if (logger.isInfoEnabled()) {
logger.info("release end connect count:=" + connectionManager.getTotalStats().getAvailable());
logger.info("release end connect count:=" + connectionManager.getTotalStats().getMax());
}
}
@Override
public void run() {
try {
// Holds the stop request that stopped the process.
Stop stopRequest;
// Every 5 seconds.
while ((stopRequest = stopSignal.poll(5, TimeUnit.SECONDS)) == null) {
// Close expired connections
cm.closeExpiredConnections();
// Optionally, close connections that have been idle too long.
cm.closeIdleConnections(60, TimeUnit.SECONDS);
// Look at pool stats.
logger.debug("Stats: {}", cm.getTotalStats());
}
// Acknowledge the stop request.
stopRequest.stopped();
} catch (InterruptedException ex) {
// terminate
}
}
public static HttpClient createHttpClient(int maxTotal, int maxPerRoute) {
try {
SSLContext sslContext = SSLContexts.custom().useSSL().build();
SSLConnectionSocketFactory sf =
new SSLConnectionSocketFactory(
sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager =
new PoolingHttpClientConnectionManager();
poolingHttpClientConnectionManager.setMaxTotal(maxTotal);
poolingHttpClientConnectionManager.setDefaultMaxPerRoute(maxPerRoute);
return HttpClientBuilder.create()
.setConnectionManager(poolingHttpClientConnectionManager)
.setSSLSocketFactory(sf)
.build();
} catch (KeyManagementException e) {
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
return null;
}
protected static synchronized void kill() {
if (sessionList != null) {
for (HTTPSession session : sessionList) {
session.close();
}
sessionList.clear();
// Rebuild the connection manager
connmgr.shutdown();
connmgr = new PoolingHttpClientConnectionManager(sslregistry);
setGlobalThreadCount(DFALTTHREADCOUNT);
}
}
private Config getConfig(final String url, final ProcessContext context) {
final String baseUrl = getBaseUrl(url);
Config config = configMap.get(baseUrl);
if (config != null) {
return config;
}
final PoolingHttpClientConnectionManager conMan;
final SSLContextService sslContextService =
context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
if (sslContextService == null) {
conMan = new PoolingHttpClientConnectionManager();
} else {
final SSLContext sslContext;
try {
sslContext = createSSLContext(sslContextService);
} catch (final Exception e) {
throw new ProcessException(e);
}
final SSLConnectionSocketFactory sslsf =
new SSLConnectionSocketFactory(
sslContext,
new String[] {"TLSv1"},
null,
SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
final Registry<ConnectionSocketFactory> socketFactoryRegistry =
RegistryBuilder.<ConnectionSocketFactory>create().register("https", sslsf).build();
conMan = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
}
conMan.setDefaultMaxPerRoute(context.getMaxConcurrentTasks());
conMan.setMaxTotal(context.getMaxConcurrentTasks());
config = new Config(conMan);
final Config existingConfig = configMap.putIfAbsent(baseUrl, config);
return (existingConfig == null) ? config : existingConfig;
}
protected PoolingHttpClientConnectionManager newConnectionManager() {
try {
final SSLContext sslContext = SSLContext.getInstance("SSL");
sslContext.init(
null,
new TrustManager[] {
new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s)
throws CertificateException {}
@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s)
throws CertificateException {}
@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
}
},
new SecureRandom());
final Registry<ConnectionSocketFactory> registry =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslContext))
.build();
connectionManager = new PoolingHttpClientConnectionManager(registry);
connectionManager.setMaxTotal(
Integer.parseInt(System.getProperty("zuul.max.host.connections", "200")));
connectionManager.setDefaultMaxPerRoute(
Integer.parseInt(System.getProperty("zuul.max.host.connections", "20")));
return connectionManager;
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
@Override
public void initialize() throws InitializationException {
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(100);
m_httpClient = HttpClients.custom().setConnectionManager(cm).build();
Builder b = RequestConfig.custom();
Properties globalConfig = m_env.getGlobalConfig();
// TODO config
b.setConnectTimeout(
Integer.valueOf(globalConfig.getProperty("gateway.subcription.connect.timeout", "2000")));
b.setSocketTimeout(
Integer.valueOf(globalConfig.getProperty("gateway.subscription.socket.timeout", "5000")));
m_requestConfig = b.build();
failedMeterGlobal =
HermesMetricsRegistry.getMetricRegistry()
.meter(
MetricRegistry.name(
SubscriptionPushService.class, "MessageSubscription", "Failed"));
requestMeterGlobal =
HermesMetricsRegistry.getMetricRegistry()
.meter(
MetricRegistry.name(
SubscriptionPushService.class, "MessageSubscription", "Request"));
requestSizeHistogramGlobal =
HermesMetricsRegistry.getMetricRegistry()
.histogram(
MetricRegistry.name(
SubscriptionPushService.class, "MessageSubscription", "BodySize"));
pushTimerGlobal =
HermesMetricsRegistry.getMetricRegistry()
.timer(MetricRegistry.name(SubscriptionPushService.class, "MessageSubscription"));
failedMeterByTopic = new HashMap<>();
requestMeterByTopic = new HashMap<>();
requestSizeHistogramByTopic = new HashMap<>();
pushTimerByTopic = new HashMap<>();
}
/**
* Key store 类型HttpClient
*
* @param keystore
* @param keyPassword
* @param supportedProtocols
* @param maxTotal
* @param maxPerRoute
* @return
*/
public static HttpClient createKeyMaterialHttpClient(
KeyStore keystore,
String keyPassword,
String[] supportedProtocols,
int maxTotal,
int maxPerRoute) {
try {
SSLContext sslContext =
SSLContexts.custom()
.useSSL()
.loadKeyMaterial(keystore, keyPassword.toCharArray())
.build();
SSLConnectionSocketFactory sf =
new SSLConnectionSocketFactory(
sslContext,
supportedProtocols,
null,
SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager =
new PoolingHttpClientConnectionManager();
poolingHttpClientConnectionManager.setMaxTotal(maxTotal);
poolingHttpClientConnectionManager.setDefaultMaxPerRoute(maxPerRoute);
return HttpClientBuilder.create()
.setConnectionManager(poolingHttpClientConnectionManager)
.setSSLSocketFactory(sf)
.build();
} catch (KeyManagementException e) {
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
} catch (UnrecoverableKeyException e) {
e.printStackTrace();
} catch (KeyStoreException e) {
e.printStackTrace();
}
return null;
}
public SpiderSupporter(int infoType, String url, String indexSelector) {
cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(200);
cm.setDefaultMaxPerRoute(20);
cookieStore = new BasicCookieStore();
defaultRequestConfig =
RequestConfig.custom()
.setCookieSpec(CookieSpecs.DEFAULT)
.setExpectContinueEnabled(true)
.build();
httpclient =
HttpClients.custom()
.setConnectionManager(cm)
.setDefaultCookieStore(cookieStore)
// .setProxy(new HttpHost("127.0.0.1", 8888)) //fiddler proxy
.setConnectionManagerShared(true)
.setDefaultRequestConfig(defaultRequestConfig)
.setKeepAliveStrategy(new ChangePerResponse())
.build();
this.INFO_TYPE = infoType;
this.URL = url; // 索引所在页的地址
indexExtractor = new IndexExtractor(URL, httpclient, indexSelector);
}
/**
* 个性化配置连接管理器
*
* @param maxConnectionsPerHost 设置默认的连接到每个主机的最大连接数
* @param maxTotalConnections 设置整个管理连接器的最大连接数
* @param conTimeOutMs 连接超时
* @param soTimeOutMs socket超时
* @param routeCfgList 特殊路由配置列表，若无请填null
* @param proxy 代理设置，若无请填null
*/
public QHttpClient(
int maxConnectionsPerHost,
int maxTotalConnections,
int conTimeOutMs,
int soTimeOutMs,
List<RouteCfg> routeCfgList,
HttpHost proxy) {
// 使用默认的 socket factories 注册 "http" & "https" protocol scheme
Registry<ConnectionSocketFactory> reg =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", SSLConnectionSocketFactory.getSocketFactory())
.build();
PoolingHttpClientConnectionManager connectionManager =
new PoolingHttpClientConnectionManager(reg);
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(soTimeOutMs).build();
connectionManager.setDefaultMaxPerRoute(maxConnectionsPerHost);
connectionManager.setMaxTotal(maxTotalConnections);
connectionManager.setDefaultSocketConfig(socketConfig);
// 对特定路由修改最大连接数
if (null != routeCfgList) {
for (RouteCfg routeCfg : routeCfgList) {
HttpHost localhost = new HttpHost(routeCfg.getHost(), routeCfg.getPort());
connectionManager.setMaxPerRoute(new HttpRoute(localhost), routeCfg.getMaxConnetions());
}
}
// 设置代理
DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
httpClient =
HttpClients.custom()
.setConnectionManager(connectionManager)
.setRoutePlanner(routePlanner)
.build();
}
public void shutdown() throws InterruptedException {
logger.trace("Shutting down client pool");
// Signal the stop to the thread.
Stop stop = new Stop();
stopSignal.add(stop);
// Wait for the stop to complete.
stop.waitForStopped();
// Close the pool - Added
try {
threadSafeClient.close();
} catch (IOException ioe) {
logger.info("IO Exception while closing HttpClient connecntions.");
}
// Close the connection manager.
cm.close();
logger.trace("Client pool shut down");
}
HttpClient() {
try {
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();
connManager.setDefaultSocketConfig(socketConfig);
MessageConstraints messageConstraints =
MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build();
ConnectionConfig connectionConfig =
ConnectionConfig.custom()
.setMalformedInputAction(CodingErrorAction.IGNORE)
.setUnmappableInputAction(CodingErrorAction.IGNORE)
.setCharset(Consts.UTF_8)
.setMessageConstraints(messageConstraints)
.build();
connManager.setDefaultConnectionConfig(connectionConfig);
connManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
connManager.setDefaultMaxPerRoute(MAX_ROUTE_CONNECTIONS);
connManager.setMaxPerRoute(new HttpRoute(DEFAULT_TARGETHOST), 50);
HttpRequestRetryHandler retryHandler =
new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(
IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 5) return false;
if (exception instanceof InterruptedIOException) return false;
if (exception instanceof UnknownHostException) return false;
if (exception instanceof ConnectTimeoutException) return false;
if (exception instanceof SSLException) return false;
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) return true;
return false;
}
};
httpClient =
HttpClients.custom()
.setConnectionManager(connManager)
.setRetryHandler(retryHandler)
.build();
} catch (Exception e) {
logger.error("HttpClientUtil error : [{}].", e.getMessage(), e);
}
}
public void shutdown() {
connectionManager.shutdown();
}