/**
* Get方法传送消息
*
* @param url 连接的URL
* @param queryString 请求参数串
* @return 服务器返回的信息
* @throws Exception
*/
public String httpGet(String url, String queryString) throws Exception {
String responseData = null;
if (queryString != null && !queryString.equals("")) {
url += "?" + queryString;
}
RequestBuilder requestBuilder = RequestBuilder.get().setUri(url);
log.info("QHttpClient httpGet [1] url = " + url);
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
requestBuilder.setConfig(requestConfig);
HttpResponse response;
HttpGet httpGet = (HttpGet) requestBuilder.build();
response = httpClient.execute(httpGet);
try {
log.info("QHttpClient httpGet [2] StatusLine : " + response.getStatusLine());
responseData = EntityUtils.toString(response.getEntity());
log.info("QHttpClient httpGet [3] Response = " + responseData);
} catch (Exception e) {
e.printStackTrace();
} finally {
httpGet.abort();
}
return responseData;
}
/**
* 构造请求的方法，如post，get，header等<br>
* 设置请求参数，如超时时间
*
* @param url 请求的URL
* @param method 请求的方法
* @param postdata post的数据
* @param headers 请求头
* @return
*/
protected HttpUriRequest getHttpUriRequest(
String url, String method, Map<String, String> postdata, Map<String, String> headers) {
RequestBuilder requestBuilder = selectRequestMethod(method, postdata, headers).setUri(url);
requestBuilder.addHeader("Accept", "*/*");
requestBuilder.addHeader("Connection", "keep-alive");
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
int timeout = 45000; // 超时时间
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(timeout)
.setSocketTimeout(timeout)
.setConnectTimeout(timeout)
// .setRedirectsEnabled(false)//禁止自动跳转
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (null != host && !"".equals(host) && port > 10) {
HttpHost proxy = new HttpHost(host, port);
requestConfigBuilder.setProxy(proxy);
}
requestBuilder.setConfig(requestConfigBuilder.build());
return requestBuilder.build();
}
/**
* Called primarily from HTTPMethod to do the bulk of the execution. Assumes HTTPMethod has
* inserted its headers into request.
*
* @param method
* @param methoduri
* @param rb
* @return Request+Response pair
* @throws HTTPException
*/
ExecState execute(HTTPMethod method, URI methoduri, RequestBuilder rb) throws HTTPException {
this.execution = new ExecState();
this.requestURI = methoduri;
AuthScope methodscope = HTTPAuthUtil.uriToAuthScope(methoduri);
AuthScope target = HTTPAuthUtil.authscopeUpgrade(this.scope, methodscope);
synchronized (this) { // keep coverity happy
// Merge Settings;
Settings merged = HTTPUtil.merge(globalsettings, localsettings);
if (!this.cachevalid) {
RequestConfig.Builder rcb = RequestConfig.custom();
this.cachedconfig = configureRequest(rcb, merged);
HttpClientBuilder cb = HttpClients.custom();
configClient(cb, merged);
setAuthenticationAndProxy(cb);
this.cachedclient = cb.build();
rb.setConfig(this.cachedconfig);
this.cachevalid = true;
}
}
this.execution.request = (HttpRequestBase) rb.build();
try {
HttpHost targethost = HTTPAuthUtil.authscopeToHost(target);
this.execution.response =
cachedclient.execute(targethost, this.execution.request, this.sessioncontext);
} catch (IOException ioe) {
throw new HTTPException(ioe);
}
return this.execution;
}
/**
* Post方法传送消息
*
* @param url 连接的URL
* @param queryString 请求参数串
* @return 服务器返回的信息
* @throws Exception
*/
public String httpPost(String url, String queryString) throws Exception {
String responseData = null;
URI tmpUri = new URI(url);
URI uri =
new URI(
tmpUri.getScheme(),
null,
tmpUri.getHost(),
tmpUri.getPort(),
tmpUri.getPath(),
queryString,
null);
log.info("QHttpClient httpPost [1] url = " + uri.toURL());
RequestBuilder requestBuilder = RequestBuilder.post().setUri(uri);
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
requestBuilder.setConfig(requestConfig);
HttpPost httpPost = (HttpPost) requestBuilder.build();
if (queryString != null && !queryString.equals("")) {
StringEntity reqEntity = new StringEntity(queryString);
// 设置类型
reqEntity.setContentType("application/x-www-form-urlencoded");
// 设置请求的数据
httpPost.setEntity(reqEntity);
}
try {
HttpResponse response = httpClient.execute(httpPost);
log.info("QHttpClient httpPost [2] StatusLine = " + response.getStatusLine());
responseData = EntityUtils.toString(response.getEntity());
log.info("QHttpClient httpPost [3] responseData = " + responseData);
} catch (Exception e) {
e.printStackTrace();
} finally {
httpPost.abort();
}
return responseData;
}
@Override
public Page download(Request request, Task task) {
Site site = null;
if (task != null) {
site = task.getSite();
}
Set<Integer> acceptStatCode;
String charset = null;
Map<String, String> headers = null;
if (site != null) {
acceptStatCode = site.getAcceptStatCode();
charset = site.getCharset();
headers = site.getHeaders();
} else {
acceptStatCode = Sets.newHashSet(200);
}
logger.info("downloading page " + request.getUrl());
RequestBuilder requestBuilder = RequestBuilder.get().setUri(request.getUrl());
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut())
.setSocketTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (site != null && site.getHttpProxy() != null) {
requestConfigBuilder.setProxy(site.getHttpProxy());
}
requestBuilder.setConfig(requestConfigBuilder.build());
CloseableHttpResponse httpResponse = null;
try {
httpResponse = getHttpClient(site).execute(requestBuilder.build());
int statusCode = httpResponse.getStatusLine().getStatusCode();
if (acceptStatCode.contains(statusCode)) {
// charset
if (charset == null) {
String value = httpResponse.getEntity().getContentType().getValue();
charset = UrlUtils.getCharset(value);
}
return handleResponse(request, charset, httpResponse, task);
} else {
logger.warn("code error " + statusCode + "\t" + request.getUrl());
return null;
}
} catch (IOException e) {
logger.warn("download page " + request.getUrl() + " error", e);
if (site.getCycleRetryTimes() > 0) {
return addToCycleRetry(request, site);
}
return null;
} finally {
try {
if (httpResponse != null) {
// ensure the connection is released back to pool
EntityUtils.consume(httpResponse.getEntity());
}
} catch (IOException e) {
logger.warn("close response fail", e);
}
}
}