/**
* @param maxTotal maxTotal
* @param maxPerRoute maxPerRoute
* @param timeout timeout
* @param retryExecutionCount retryExecutionCount
* @return
*/
public static CloseableHttpClient createHttpClient(
int maxTotal, int maxPerRoute, int timeout, int retryExecutionCount) {
try {
SSLContext sslContext = SSLContexts.custom().useSSL().build();
SSLConnectionSocketFactory sf =
new SSLConnectionSocketFactory(
sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager =
new PoolingHttpClientConnectionManager();
poolingHttpClientConnectionManager.setMaxTotal(maxTotal);
poolingHttpClientConnectionManager.setDefaultMaxPerRoute(maxPerRoute);
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(timeout).build();
poolingHttpClientConnectionManager.setDefaultSocketConfig(socketConfig);
return HttpClientBuilder.create()
.setConnectionManager(poolingHttpClientConnectionManager)
.setSSLSocketFactory(sf)
.setRetryHandler(new HttpRequestRetryHandlerImpl(retryExecutionCount))
.build();
} catch (KeyManagementException e) {
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
return null;
}
/**
* Key store 类型HttpClient
*
* @param keystore keystore
* @param keyPassword keyPassword
* @param supportedProtocols supportedProtocols
* @param timeout timeout
* @param retryExecutionCount retryExecutionCount
* @return CloseableHttpClient
*/
public static CloseableHttpClient createKeyMaterialHttpClient(
KeyStore keystore,
String keyPassword,
String[] supportedProtocols,
int timeout,
int retryExecutionCount) {
try {
SSLContext sslContext =
SSLContexts.custom()
.useSSL()
.loadKeyMaterial(keystore, keyPassword.toCharArray())
.build();
SSLConnectionSocketFactory sf =
new SSLConnectionSocketFactory(
sslContext,
supportedProtocols,
null,
SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(timeout).build();
return HttpClientBuilder.create()
.setDefaultSocketConfig(socketConfig)
.setSSLSocketFactory(sf)
.setRetryHandler(new HttpRequestRetryHandlerImpl(retryExecutionCount))
.build();
} catch (KeyManagementException e) {
e.printStackTrace();
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
} catch (UnrecoverableKeyException e) {
e.printStackTrace();
} catch (KeyStoreException e) {
e.printStackTrace();
}
return null;
}
private CloseableHttpClient generateClient(Site site) {
HttpClientBuilder httpClientBuilder =
HttpClients.custom().setConnectionManager(connectionManager);
if (site != null && site.getUserAgent() != null) {
httpClientBuilder.setUserAgent(site.getUserAgent());
} else {
httpClientBuilder.setUserAgent("");
}
if (site == null || site.isUseGzip()) {
httpClientBuilder.addInterceptorFirst(
new HttpRequestInterceptor() {
public void process(final HttpRequest request, final HttpContext context)
throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
}
SocketConfig socketConfig =
SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
if (site != null) {
httpClientBuilder.setRetryHandler(
new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
}
generateCookie(httpClientBuilder, site);
return httpClientBuilder.build();
}
HttpClient() {
try {
PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();
connManager.setDefaultSocketConfig(socketConfig);
MessageConstraints messageConstraints =
MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build();
ConnectionConfig connectionConfig =
ConnectionConfig.custom()
.setMalformedInputAction(CodingErrorAction.IGNORE)
.setUnmappableInputAction(CodingErrorAction.IGNORE)
.setCharset(Consts.UTF_8)
.setMessageConstraints(messageConstraints)
.build();
connManager.setDefaultConnectionConfig(connectionConfig);
connManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
connManager.setDefaultMaxPerRoute(MAX_ROUTE_CONNECTIONS);
connManager.setMaxPerRoute(new HttpRoute(DEFAULT_TARGETHOST), 50);
HttpRequestRetryHandler retryHandler =
new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(
IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 5) return false;
if (exception instanceof InterruptedIOException) return false;
if (exception instanceof UnknownHostException) return false;
if (exception instanceof ConnectTimeoutException) return false;
if (exception instanceof SSLException) return false;
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) return true;
return false;
}
};
httpClient =
HttpClients.custom()
.setConnectionManager(connManager)
.setRetryHandler(retryHandler)
.build();
} catch (Exception e) {
logger.error("HttpClientUtil error : [{}].", e.getMessage(), e);
}
}
/**
* 个性化配置连接管理器
*
* @param maxConnectionsPerHost 设置默认的连接到每个主机的最大连接数
* @param maxTotalConnections 设置整个管理连接器的最大连接数
* @param conTimeOutMs 连接超时
* @param soTimeOutMs socket超时
* @param routeCfgList 特殊路由配置列表，若无请填null
* @param proxy 代理设置，若无请填null
*/
public QHttpClient(
int maxConnectionsPerHost,
int maxTotalConnections,
int conTimeOutMs,
int soTimeOutMs,
List<RouteCfg> routeCfgList,
HttpHost proxy) {
// 使用默认的 socket factories 注册 "http" & "https" protocol scheme
Registry<ConnectionSocketFactory> reg =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", SSLConnectionSocketFactory.getSocketFactory())
.build();
PoolingHttpClientConnectionManager connectionManager =
new PoolingHttpClientConnectionManager(reg);
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(soTimeOutMs).build();
connectionManager.setDefaultMaxPerRoute(maxConnectionsPerHost);
connectionManager.setMaxTotal(maxTotalConnections);
connectionManager.setDefaultSocketConfig(socketConfig);
// 对特定路由修改最大连接数
if (null != routeCfgList) {
for (RouteCfg routeCfg : routeCfgList) {
HttpHost localhost = new HttpHost(routeCfg.getHost(), routeCfg.getPort());
connectionManager.setMaxPerRoute(new HttpRoute(localhost), routeCfg.getMaxConnetions());
}
}
// 设置代理
DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
httpClient =
HttpClients.custom()
.setConnectionManager(connectionManager)
.setRoutePlanner(routePlanner)
.build();
}
public static final void main(String[] args) throws Exception {
// Use custom message parser / writer to customize the way HTTP
// messages are parsed from and written out to the data stream.
HttpMessageParserFactory<HttpResponse> responseParserFactory =
new DefaultHttpResponseParserFactory() {
@Override
public HttpMessageParser<HttpResponse> create(
SessionInputBuffer buffer, MessageConstraints constraints) {
LineParser lineParser =
new BasicLineParser() {
@Override
public Header parseHeader(final CharArrayBuffer buffer) {
try {
return super.parseHeader(buffer);
} catch (ParseException ex) {
return new BasicHeader(buffer.toString(), null);
}
}
};
return new DefaultHttpResponseParser(
buffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints) {
@Override
protected boolean reject(final CharArrayBuffer line, int count) {
// try to ignore all garbage preceding a status line infinitely
return false;
}
};
}
};
HttpMessageWriterFactory<HttpRequest> requestWriterFactory =
new DefaultHttpRequestWriterFactory();
// Use a custom connection factory to customize the process of
// initialization of outgoing HTTP connections. Beside standard connection
// configuration parameters HTTP connection factory can define message
// parser / writer routines to be employed by individual connections.
HttpConnectionFactory<HttpRoute, ManagedHttpClientConnection> connFactory =
new ManagedHttpClientConnectionFactory(requestWriterFactory, responseParserFactory);
// Client HTTP connection objects when fully initialized can be bound to
// an arbitrary network socket. The process of network socket initialization,
// its connection to a remote address and binding to a local one is controlled
// by a connection socket factory.
// SSL context for secure connections can be created either based on
// system or application specific properties.
SSLContext sslcontext = SSLContexts.createSystemDefault();
// Create a registry of custom connection socket factories for supported
// protocol schemes.
Registry<ConnectionSocketFactory> socketFactoryRegistry =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
// Use custom DNS resolver to override the system DNS resolution.
DnsResolver dnsResolver =
new SystemDefaultDnsResolver() {
@Override
public InetAddress[] resolve(final String host) throws UnknownHostException {
if (host.equalsIgnoreCase("myhost")) {
return new InetAddress[] {InetAddress.getByAddress(new byte[] {127, 0, 0, 1})};
} else {
return super.resolve(host);
}
}
};
// Create a connection manager with custom configuration.
PoolingHttpClientConnectionManager connManager =
new PoolingHttpClientConnectionManager(socketFactoryRegistry, connFactory, dnsResolver);
// Create socket configuration
SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();
// Configure the connection manager to use socket configuration either
// by default or for a specific host.
connManager.setDefaultSocketConfig(socketConfig);
connManager.setSocketConfig(new HttpHost("somehost", 80), socketConfig);
// Validate connections after 1 sec of inactivity
connManager.setValidateAfterInactivity(1000);
// Create message constraints
MessageConstraints messageConstraints =
MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build();
// Create connection configuration
ConnectionConfig connectionConfig =
ConnectionConfig.custom()
.setMalformedInputAction(CodingErrorAction.IGNORE)
.setUnmappableInputAction(CodingErrorAction.IGNORE)
.setCharset(Consts.UTF_8)
.setMessageConstraints(messageConstraints)
.build();
// Configure the connection manager to use connection configuration either
// by default or for a specific host.
connManager.setDefaultConnectionConfig(connectionConfig);
connManager.setConnectionConfig(new HttpHost("somehost", 80), ConnectionConfig.DEFAULT);
// Configure total max or per route limits for persistent connections
// that can be kept in the pool or leased by the connection manager.
connManager.setMaxTotal(100);
connManager.setDefaultMaxPerRoute(10);
connManager.setMaxPerRoute(new HttpRoute(new HttpHost("somehost", 80)), 20);
// Use custom cookie store if necessary.
CookieStore cookieStore = new BasicCookieStore();
// Use custom credentials provider if necessary.
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// Create global request configuration
RequestConfig defaultRequestConfig =
RequestConfig.custom()
.setCookieSpec(CookieSpecs.DEFAULT)
.setExpectContinueEnabled(true)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
.build();
// Create an HttpClient with the given custom dependencies and configuration.
CloseableHttpClient httpclient =
HttpClients.custom()
.setConnectionManager(connManager)
.setDefaultCookieStore(cookieStore)
.setDefaultCredentialsProvider(credentialsProvider)
.setProxy(new HttpHost("myproxy", 8080))
.setDefaultRequestConfig(defaultRequestConfig)
.build();
try {
HttpGet httpget = new HttpGet("http://httpbin.org/get");
// Request configuration can be overridden at the request level.
// They will take precedence over the one set at the client level.
RequestConfig requestConfig =
RequestConfig.copy(defaultRequestConfig)
.setSocketTimeout(5000)
.setConnectTimeout(5000)
.setConnectionRequestTimeout(5000)
.setProxy(new HttpHost("myotherproxy", 8080))
.build();
httpget.setConfig(requestConfig);
// Execution context can be customized locally.
HttpClientContext context = HttpClientContext.create();
// Contextual attributes set the local context level will take
// precedence over those set at the client level.
context.setCookieStore(cookieStore);
context.setCredentialsProvider(credentialsProvider);
System.out.println("executing request " + httpget.getURI());
CloseableHttpResponse response = httpclient.execute(httpget, context);
try {
System.out.println("----------------------------------------");
System.out.println(response.getStatusLine());
System.out.println(EntityUtils.toString(response.getEntity()));
System.out.println("----------------------------------------");
// Once the request has been executed the local context can
// be used to examine updated state and various objects affected
// by the request execution.
// Last executed request
context.getRequest();
// Execution route
context.getHttpRoute();
// Target auth state
context.getTargetAuthState();
// Proxy auth state
context.getTargetAuthState();
// Cookie origin
context.getCookieOrigin();
// Cookie spec used
context.getCookieSpec();
// User security token
context.getUserToken();
} finally {
response.close();
}
} finally {
httpclient.close();
}
}
public EasyHttpClient() {
CookieSpecProvider easySpecProvider =
new CookieSpecProvider() {
public CookieSpec create(HttpContext context) {
return new BrowserCompatSpec() {
public void validate(Cookie cookie, CookieOrigin origin)
throws MalformedCookieException {}
};
}
};
SSLContext sslContext = SSLContexts.createDefault(); // 忽略证书主机名验证
SSLConnectionSocketFactory sslsf =
new SSLConnectionSocketFactory(
sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
Registry<ConnectionSocketFactory> reg =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", sslsf)
.build();
connectionManager = new PoolingHttpClientConnectionManager(reg);
connectionManager.setDefaultMaxPerRoute(100); // 同一个路由允许最大连接数
// connectionManager.setMaxTotal(poolSize);
HttpHost localhost = new HttpHost("locahost", 80); // 如果是多网卡，这里选择出口IP？
connectionManager.setMaxPerRoute(new HttpRoute(localhost), 50);
Registry<CookieSpecProvider> r =
RegistryBuilder.<CookieSpecProvider>create()
.register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
.register(CookieSpecs.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory())
.register("easy", easySpecProvider)
.build();
RequestConfig requestConfig =
RequestConfig.custom()
.setCookieSpec("easy")
.setSocketTimeout(10000)
.setConnectTimeout(10000)
.build();
ConnectionConfig connectioncfg = ConnectionConfig.custom().setCharset(Charsets.UTF_8).build();
SocketConfig socketConfig =
SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
HttpRequestRetryHandler myRetryHandler =
new HttpRequestRetryHandler() {
public boolean retryRequest(
IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 3) {
return false;
}
if (exception instanceof SSLHandshakeException) {
return false;
} else {
return true;
}
}
};
HttpClientBuilder builder =
HttpClients.custom() // 默认
.setConnectionManager(connectionManager) // 链接管理器
.setDefaultSocketConfig(socketConfig) // socket管理器
.setRetryHandler(myRetryHandler) // 重试3次
.setDefaultConnectionConfig(connectioncfg) // 链接配置，如默认字符编码
.setDefaultCookieSpecRegistry(r) // cookie策略
.setUserAgent(Chorme_User_Agent) // 浏览器请求头
.setDefaultRequestConfig(requestConfig) // 链接配置，超时等
.setDefaultCookieStore(cookieStore); // cookie
builder.addInterceptorFirst(
new HttpRequestInterceptor() {
public void process(final HttpRequest request, final HttpContext context)
throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
httpclient = builder.build();
}