public HttpClient(int maxConPerHost, int conTimeOutMs, int soTimeOutMs, int maxSize) {
connectionManager = new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams params = connectionManager.getParams();
params.setDefaultMaxConnectionsPerHost(maxConPerHost);
params.setConnectionTimeout(conTimeOutMs);
params.setSoTimeout(soTimeOutMs);
HttpClientParams clientParams = new HttpClientParams();
// 忽略cookie 避免 Cookie rejected 警告
clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
client = new org.apache.commons.httpclient.HttpClient(clientParams, connectionManager);
Protocol myhttps = new Protocol("https", new MySSLSocketFactory(), 443);
Protocol.registerProtocol("https", myhttps);
this.maxSize = maxSize;
// 支持proxy
if (proxyHost != null && !proxyHost.equals("")) {
client.getHostConfiguration().setProxy(proxyHost, proxyPort);
client.getParams().setAuthenticationPreemptive(true);
if (proxyAuthUser != null && !proxyAuthUser.equals("")) {
client
.getState()
.setProxyCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword));
log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword);
}
}
}
private void configureHttpClient() {
MultiThreadedHttpConnectionManager connMgr = new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams connParams = connMgr.getParams();
connParams.setMaxTotalConnections(maxConnections);
connParams.setMaxConnectionsPerHost(HostConfiguration.ANY_HOST_CONFIGURATION, maxConnections);
connMgr.setParams(connParams);
hc = new HttpClient(connMgr);
// NOTE: These didn't seem to help in my initial testing
// hc.getParams().setParameter("http.tcp.nodelay", true);
// hc.getParams().setParameter("http.connection.stalecheck", false);
if (proxyHost != null) {
HostConfiguration hostConfig = new HostConfiguration();
hostConfig.setProxy(proxyHost, proxyPort);
hc.setHostConfiguration(hostConfig);
log.info("Proxy Host set to " + proxyHost + ":" + proxyPort);
if (proxyUser != null && !proxyUser.trim().equals("")) {
if (proxyDomain != null) {
hc.getState()
.setProxyCredentials(
new AuthScope(proxyHost, proxyPort),
new NTCredentials(proxyUser, proxyPassword, proxyHost, proxyDomain));
} else {
hc.getState()
.setProxyCredentials(
new AuthScope(proxyHost, proxyPort),
new UsernamePasswordCredentials(proxyUser, proxyPassword));
}
}
}
}
private static MultiThreadedHttpConnectionManager getMultiThreadedConnManager() {
if (mConnManager == null) {
mConnManager = new MultiThreadedHttpConnectionManager();
mConnManager.getParams().setDefaultMaxConnectionsPerHost(5);
mConnManager.getParams().setMaxTotalConnections(5);
}
return mConnManager;
}
public HttpImpl() {
// Mimic behavior found in
// http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html
if (Validator.isNotNull(_NON_PROXY_HOSTS)) {
String nonProxyHostsRegEx = _NON_PROXY_HOSTS;
nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\.", "\\\\.");
nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\*", ".*?");
nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\|", ")|(");
nonProxyHostsRegEx = "(" + nonProxyHostsRegEx + ")";
_nonProxyHostsPattern = Pattern.compile(nonProxyHostsRegEx);
}
MultiThreadedHttpConnectionManager httpConnectionManager =
new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams httpConnectionManagerParams = httpConnectionManager.getParams();
httpConnectionManagerParams.setConnectionTimeout(_TIMEOUT);
httpConnectionManagerParams.setDefaultMaxConnectionsPerHost(
new Integer(_MAX_CONNECTIONS_PER_HOST));
httpConnectionManagerParams.setMaxTotalConnections(new Integer(_MAX_TOTAL_CONNECTIONS));
httpConnectionManagerParams.setSoTimeout(_TIMEOUT);
_httpClient.setHttpConnectionManager(httpConnectionManager);
_proxyHttpClient.setHttpConnectionManager(httpConnectionManager);
if (hasProxyConfig() && Validator.isNotNull(_PROXY_USERNAME)) {
List<String> authPrefs = new ArrayList<String>();
if (_PROXY_AUTH_TYPE.equals("username-password")) {
_proxyCredentials = new UsernamePasswordCredentials(_PROXY_USERNAME, _PROXY_PASSWORD);
authPrefs.add(AuthPolicy.BASIC);
authPrefs.add(AuthPolicy.DIGEST);
authPrefs.add(AuthPolicy.NTLM);
} else if (_PROXY_AUTH_TYPE.equals("ntlm")) {
_proxyCredentials =
new NTCredentials(
_PROXY_USERNAME, _PROXY_PASSWORD, _PROXY_NTLM_HOST, _PROXY_NTLM_DOMAIN);
authPrefs.add(AuthPolicy.NTLM);
authPrefs.add(AuthPolicy.BASIC);
authPrefs.add(AuthPolicy.DIGEST);
}
HttpClientParams httpClientParams = _proxyHttpClient.getParams();
httpClientParams.setParameter(AuthPolicy.AUTH_SCHEME_PRIORITY, authPrefs);
}
}
private synchronized MultiThreadedHttpConnectionManager getConnectionManager() {
if (connectionManager == null) {
connectionManager = new MultiThreadedHttpConnectionManager();
final HttpConnectionManagerParams params = connectionManager.getParams();
params.setDefaultMaxConnectionsPerHost(perHostParallelFetches);
params.setMaxTotalConnections(globalParallelFetches);
params.setSoTimeout(socketTimeout);
params.setConnectionTimeout(connectionTimeout);
}
return connectionManager;
}
/** 初始化httpclient */
static {
int connectionTimeout = 30000;
int soTimeout = 30000;
try {
connectionTimeout =
Integer.parseInt(System.getProperty("sun.net.client.defaultConnectTimeout", "30000"));
} catch (Exception e) {
}
try {
soTimeout =
Integer.parseInt(System.getProperty("sun.net.client.defaultReadTimeout", "30000"));
} catch (Exception e) {
}
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
connectionManager.getParams().setDefaultMaxConnectionsPerHost(10);
connectionManager.getParams().setMaxTotalConnections(300);
connectionManager.getParams().setConnectionTimeout(connectionTimeout);
connectionManager.getParams().setSoTimeout(soTimeout);
client.setHttpConnectionManager(connectionManager);
// 忽略cookie 避免 Cookie rejected 警告
HttpClientParams clientParams = new HttpClientParams();
clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
client.setParams(clientParams);
// 支持https
Protocol myhttps = new Protocol("https", new SSLSocketFactory(), 443);
Protocol.registerProtocol("https", myhttps);
// 设置代理
if (ProxyClient.getProxy() != null) {
client.getHostConfiguration().setProxy(ProxyClient.getHost(), ProxyClient.getPort());
client.getParams().setAuthenticationPreemptive(true);
if (ProxyClient.getUsername() != null && !ProxyClient.getUsername().trim().equals("")) {
client
.getState()
.setProxyCredentials(
AuthScope.ANY,
new UsernamePasswordCredentials(
ProxyClient.getUsername().trim(), ProxyClient.getPassword().trim()));
}
}
}
@PostConstruct
public void init() {
connectionManager = new MultiThreadedHttpConnectionManager();
connectionManager
.getParams()
.setDefaultMaxConnectionsPerHost(settings.getSolrMaxConnectionsPerHost());
connectionManager.getParams().setMaxTotalConnections(settings.getSolrMaxTotalConnections());
httpClient = new HttpClient(connectionManager);
eventWorker = new EventWorker();
eventWorkerThread = new Thread(eventWorker, "IndexerWorkerEventWorker");
eventWorkerThread.start();
synchronized (indexUpdatersLock) {
Collection<IndexDefinition> indexes = indexerModel.getIndexes(listener);
for (IndexDefinition index : indexes) {
if (shouldRunIndexUpdater(index)) {
addIndexUpdater(index);
}
}
}
}
/**
* 构造Http客户端对象 <功能详细描述>
*
* @return [参数说明]
* @return HttpClient [返回类型说明]
* @exception throws [违例类型] [违例说明]
* @see [类、类#方法、类#成员]
*/
public HttpClient getHttpClient() {
/** 链接的超时数,默认为5秒,此处要做成可配置 */
final int CONNECTION_TIME_OUT = 5000;
final int SOCKET_TIME_OUT = 5000;
/** 每个主机的最大并行链接数，默认为10 */
final int MAX_CONNECTIONS_PER_HOST = 10;
/** 客户端总并行链接最大数，默认为50 */
final int MAX_TOTAL_CONNECTIONS = 50;
// 此处运用连接池技术。
MultiThreadedHttpConnectionManager manager = new MultiThreadedHttpConnectionManager();
// 设定参数：与每个主机的最大连接数
manager.getParams().setDefaultMaxConnectionsPerHost(MAX_CONNECTIONS_PER_HOST);
// 设定参数：客户端的总连接数
manager.getParams().setMaxTotalConnections(MAX_TOTAL_CONNECTIONS);
// 使用连接池技术创建HttpClient对象
HttpClient httpClient = new HttpClient(manager);
// 设置超时时间
httpClient.getParams().setConnectionManagerTimeout(CONNECTION_TIME_OUT); // 从连接池中获取连接超时设置
httpClient
.getHttpConnectionManager()
.getParams()
.setConnectionTimeout(CONNECTION_TIME_OUT); // 建立连接超时设置
httpClient
.getHttpConnectionManager()
.getParams()
.setSoTimeout(SOCKET_TIME_OUT); // socket上没有数据流动超时设置
return httpClient;
}
private void initHttpConnectionManager() throws ServletException {
httpConnectionManager = new MultiThreadedHttpConnectionManager();
// settings may be overridden from ode-axis2.properties using the same properties as HttpClient
// /!\ If the size of the conn pool is smaller than the size of the thread pool, the thread pool
// might get starved.
int max_per_host =
Integer.parseInt(
_odeConfig.getProperty(
HttpConnectionManagerParams.MAX_HOST_CONNECTIONS,
"" + _odeConfig.getPoolMaxSize()));
int max_total =
Integer.parseInt(
_odeConfig.getProperty(
HttpConnectionManagerParams.MAX_TOTAL_CONNECTIONS,
"" + _odeConfig.getPoolMaxSize()));
if (__log.isDebugEnabled()) {
__log.debug(HttpConnectionManagerParams.MAX_HOST_CONNECTIONS + "=" + max_per_host);
__log.debug(HttpConnectionManagerParams.MAX_TOTAL_CONNECTIONS + "=" + max_total);
}
if (max_per_host < 1 || max_total < 1) {
String errmsg =
HttpConnectionManagerParams.MAX_HOST_CONNECTIONS
+ " and "
+ HttpConnectionManagerParams.MAX_TOTAL_CONNECTIONS
+ " must be positive integers!";
__log.error(errmsg);
throw new ServletException(errmsg);
}
httpConnectionManager.getParams().setDefaultMaxConnectionsPerHost(max_per_host);
httpConnectionManager.getParams().setMaxTotalConnections(max_total);
// Register the connection manager to a idle check thread
idleConnectionTimeoutThread = new IdleConnectionTimeoutThread();
idleConnectionTimeoutThread.setName("Http_Idle_Connection_Timeout_Thread");
long idleConnectionTimeout =
Long.parseLong(_odeConfig.getProperty("http.idle.connection.timeout", "30000"));
long idleConnectionCheckInterval =
Long.parseLong(_odeConfig.getProperty("http.idle.connection.check.interval", "30000"));
if (__log.isDebugEnabled()) {
__log.debug("http.idle.connection.timeout=" + idleConnectionTimeout);
__log.debug("http.idle.connection.check.interval=" + idleConnectionCheckInterval);
}
idleConnectionTimeoutThread.setConnectionTimeout(idleConnectionTimeout);
idleConnectionTimeoutThread.setTimeoutInterval(idleConnectionCheckInterval);
idleConnectionTimeoutThread.addConnectionManager(httpConnectionManager);
idleConnectionTimeoutThread.start();
}
private HttpClient createHttpClient() {
// see http://hc.apache.org/httpclient-3.x/threading.html
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams params = connectionManager.getParams();
params.setMaxConnectionsPerHost(
HostConfiguration.ANY_HOST_CONFIGURATION,
PreferenceHelper.getInt(ServerConstants.HTTP_MAX_CONN_HOST_CONF_KEY, 50));
params.setMaxTotalConnections(
PreferenceHelper.getInt(ServerConstants.HTTP_MAX_CONN_TOTAL_CONF_KEY, 150));
params.setConnectionTimeout(
PreferenceHelper.getInt(ServerConstants.HTTP_CONN_TIMEOUT_CONF_KEY, 15000)); // 15s
params.setSoTimeout(
PreferenceHelper.getInt(ServerConstants.HTTP_SO_TIMEOUT_CONF_KEY, 30000)); // 30s
connectionManager.setParams(params);
HttpClientParams clientParams = new HttpClientParams();
clientParams.setConnectionManagerTimeout(
PreferenceHelper.getInt(
ServerConstants.HTTP_CONN_MGR_TIMEOUT_CONF_KEY, 300000)); // 5 minutes
return new HttpClient(clientParams, connectionManager);
}
@Test
public void testAsyncRangeRequests()
throws IOException, URISyntaxException, InterruptedException {
final URL testResourceUrl = new URL(VAULT_BASE_URI.toURL(), "asyncRangeRequestTestFile.txt");
final MultiThreadedHttpConnectionManager cm = new MultiThreadedHttpConnectionManager();
cm.getParams().setDefaultMaxConnectionsPerHost(50);
final HttpClient client = new HttpClient(cm);
// prepare 8MiB test data:
final byte[] plaintextData = new byte[2097152 * Integer.BYTES];
final ByteBuffer bbIn = ByteBuffer.wrap(plaintextData);
for (int i = 0; i < 2097152; i++) {
bbIn.putInt(i);
}
// put request:
final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString());
putMethod.setRequestEntity(new ByteArrayRequestEntity(plaintextData));
final int putResponse = client.executeMethod(putMethod);
putMethod.releaseConnection();
Assert.assertEquals(201, putResponse);
// multiple async range requests:
final List<ForkJoinTask<?>> tasks = new ArrayList<>();
final Random generator = new Random(System.currentTimeMillis());
final AtomicBoolean success = new AtomicBoolean(true);
// 10 full interrupted requests:
for (int i = 0; i < 10; i++) {
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
final int statusCode = client.executeMethod(getMethod);
if (statusCode != 200) {
LOG.error("Invalid status code for interrupted full request");
success.set(false);
}
getMethod.getResponseBodyAsStream().read();
getMethod.getResponseBodyAsStream().close();
getMethod.releaseConnection();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 50 crappy interrupted range requests:
for (int i = 0; i < 50; i++) {
final int lower = generator.nextInt(plaintextData.length);
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
final int statusCode = client.executeMethod(getMethod);
if (statusCode != 206) {
LOG.error("Invalid status code for interrupted range request");
success.set(false);
}
getMethod.getResponseBodyAsStream().read();
getMethod.getResponseBodyAsStream().close();
getMethod.releaseConnection();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 50 normal open range requests:
for (int i = 0; i < 50; i++) {
final int lower = generator.nextInt(plaintextData.length - 512);
final int upper = plaintextData.length - 1;
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
final int statusCode = client.executeMethod(getMethod);
final byte[] responseBody = new byte[upper - lower + 10];
final int bytesRead =
IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
getMethod.releaseConnection();
if (statusCode != 206) {
LOG.error("Invalid status code for open range request");
success.set(false);
} else if (upper - lower + 1 != bytesRead) {
LOG.error("Invalid response length for open range request");
success.set(false);
} else if (!Arrays.equals(
expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
LOG.error("Invalid response body for open range request");
success.set(false);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 200 normal closed range requests:
for (int i = 0; i < 200; i++) {
final int pos1 = generator.nextInt(plaintextData.length - 512);
final int pos2 = pos1 + 512;
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final int lower = Math.min(pos1, pos2);
final int upper = Math.max(pos1, pos2);
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-" + upper);
final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
final int statusCode = client.executeMethod(getMethod);
final byte[] responseBody = new byte[upper - lower + 1];
final int bytesRead =
IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
getMethod.releaseConnection();
if (statusCode != 206) {
LOG.error("Invalid status code for closed range request");
success.set(false);
} else if (upper - lower + 1 != bytesRead) {
LOG.error("Invalid response length for closed range request");
success.set(false);
} else if (!Arrays.equals(
expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
LOG.error("Invalid response body for closed range request");
success.set(false);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
Collections.shuffle(tasks, generator);
final ForkJoinPool pool = new ForkJoinPool(4);
for (ForkJoinTask<?> task : tasks) {
pool.execute(task);
}
for (ForkJoinTask<?> task : tasks) {
task.join();
}
pool.shutdown();
cm.shutdown();
Assert.assertTrue(success.get());
}
private void configure() {
HttpConnectionManagerParams params = connectionManager.getParams();
params.setConnectionTimeout(this.connectionTimeOut);
params.setMaxTotalConnections(this.connectionMaxTotal);
params.setDefaultMaxConnectionsPerHost(this.connectionMaxPerHost);
}
public static void setGlobalConnectionTimeout(int timeout) {
connmgr.getParams().setConnectionTimeout(timeout);
}
public static int getGlobalThreadCount() {
return connmgr.getParams().getMaxTotalConnections();
}
public static void setGlobalThreadCount(int nthreads) {
connmgr.getParams().setMaxTotalConnections(nthreads);
connmgr.getParams().setDefaultMaxConnectionsPerHost(nthreads);
}
