/**
* 构造请求的方法，如post，get，header等<br>
* 设置请求参数，如超时时间
*
* @param url 请求的URL
* @param method 请求的方法
* @param postdata post的数据
* @param headers 请求头
* @return
*/
protected HttpUriRequest getHttpUriRequest(
String url, String method, Map<String, String> postdata, Map<String, String> headers) {
RequestBuilder requestBuilder = selectRequestMethod(method, postdata, headers).setUri(url);
requestBuilder.addHeader("Accept", "*/*");
requestBuilder.addHeader("Connection", "keep-alive");
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
int timeout = 45000; // 超时时间
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(timeout)
.setSocketTimeout(timeout)
.setConnectTimeout(timeout)
// .setRedirectsEnabled(false)//禁止自动跳转
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (null != host && !"".equals(host) && port > 10) {
HttpHost proxy = new HttpHost(host, port);
requestConfigBuilder.setProxy(proxy);
}
requestBuilder.setConfig(requestConfigBuilder.build());
return requestBuilder.build();
}
/**
* Get方法传送消息
*
* @param url 连接的URL
* @param queryString 请求参数串
* @return 服务器返回的信息
* @throws Exception
*/
public String httpGet(String url, String queryString) throws Exception {
String responseData = null;
if (queryString != null && !queryString.equals("")) {
url += "?" + queryString;
}
RequestBuilder requestBuilder = RequestBuilder.get().setUri(url);
log.info("QHttpClient httpGet [1] url = " + url);
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
requestBuilder.setConfig(requestConfig);
HttpResponse response;
HttpGet httpGet = (HttpGet) requestBuilder.build();
response = httpClient.execute(httpGet);
try {
log.info("QHttpClient httpGet [2] StatusLine : " + response.getStatusLine());
responseData = EntityUtils.toString(response.getEntity());
log.info("QHttpClient httpGet [3] Response = " + responseData);
} catch (Exception e) {
e.printStackTrace();
} finally {
httpGet.abort();
}
return responseData;
}
/**
* Called primarily from HTTPMethod to do the bulk of the execution. Assumes HTTPMethod has
* inserted its headers into request.
*
* @param method
* @param methoduri
* @param rb
* @return Request+Response pair
* @throws HTTPException
*/
ExecState execute(HTTPMethod method, URI methoduri, RequestBuilder rb) throws HTTPException {
this.execution = new ExecState();
this.requestURI = methoduri;
AuthScope methodscope = HTTPAuthUtil.uriToAuthScope(methoduri);
AuthScope target = HTTPAuthUtil.authscopeUpgrade(this.scope, methodscope);
synchronized (this) { // keep coverity happy
// Merge Settings;
Settings merged = HTTPUtil.merge(globalsettings, localsettings);
if (!this.cachevalid) {
RequestConfig.Builder rcb = RequestConfig.custom();
this.cachedconfig = configureRequest(rcb, merged);
HttpClientBuilder cb = HttpClients.custom();
configClient(cb, merged);
setAuthenticationAndProxy(cb);
this.cachedclient = cb.build();
rb.setConfig(this.cachedconfig);
this.cachevalid = true;
}
}
this.execution.request = (HttpRequestBase) rb.build();
try {
HttpHost targethost = HTTPAuthUtil.authscopeToHost(target);
this.execution.response =
cachedclient.execute(targethost, this.execution.request, this.sessioncontext);
} catch (IOException ioe) {
throw new HTTPException(ioe);
}
return this.execution;
}
/**
* Build a new Apache's {@link org.apache.http.client.methods.HttpUriRequest} from Jersey's {@link
* org.glassfish.jersey.client.ClientRequest}
*
* <p>Convert a method, URI, body, headers and override a user-agent if necessary
*
* @param jerseyRequest representation of an HTTP request in Jersey
* @return a new {@link org.apache.http.client.methods.HttpUriRequest}
*/
private HttpUriRequest buildApacheRequest(ClientRequest jerseyRequest) {
final RequestBuilder builder =
RequestBuilder.create(jerseyRequest.getMethod())
.setUri(jerseyRequest.getUri())
.setEntity(getHttpEntity(jerseyRequest));
for (String headerName : jerseyRequest.getHeaders().keySet()) {
builder.addHeader(headerName, jerseyRequest.getHeaderString(headerName));
}
final Optional<RequestConfig> requestConfig = addJerseyRequestConfig(jerseyRequest);
requestConfig.ifPresent(builder::setConfig);
return builder.build();
}
private void assertBuild(final Charset charset) throws Exception {
final RequestBuilder requestBuilder = RequestBuilder.create("GET").setCharset(charset);
requestBuilder.setUri("https://somehost.com/stuff");
requestBuilder.addParameters(createParameters());
final String encodedData1 = URLEncoder.encode("\"1\u00aa position\"", charset.displayName());
final String encodedData2 = URLEncoder.encode("Jos\u00e9 Abra\u00e3o", charset.displayName());
final String uriExpected =
String.format(
"https://somehost.com/stuff?parameter1=value1¶meter2=%s¶meter3=%s",
encodedData1, encodedData2);
final HttpUriRequest request = requestBuilder.build();
Assert.assertEquals(uriExpected, request.getURI().toString());
}
private <B, R> R executeRequest(
RequestSpec requestSpec,
PathParams pathParams,
B requestBody,
Class<R> responseType,
int expectedStatus) {
String url =
String.format(
ADMIN_URL_PREFIX + requestSpec.path(pathParams), scheme, host, port, urlPathPrefix);
RequestBuilder requestBuilder =
RequestBuilder.create(requestSpec.method().getName()).setUri(url);
if (requestBody != null) {
requestBuilder.setEntity(jsonStringEntity(Json.write(requestBody)));
}
String responseBodyString = safelyExecuteRequest(url, expectedStatus, requestBuilder.build());
return responseType == Void.class ? null : Json.read(responseBodyString, responseType);
}
/**
* Post方法传送消息
*
* @param url 连接的URL
* @param queryString 请求参数串
* @return 服务器返回的信息
* @throws Exception
*/
public String httpPost(String url, String queryString) throws Exception {
String responseData = null;
URI tmpUri = new URI(url);
URI uri =
new URI(
tmpUri.getScheme(),
null,
tmpUri.getHost(),
tmpUri.getPort(),
tmpUri.getPath(),
queryString,
null);
log.info("QHttpClient httpPost [1] url = " + uri.toURL());
RequestBuilder requestBuilder = RequestBuilder.post().setUri(uri);
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
requestBuilder.setConfig(requestConfig);
HttpPost httpPost = (HttpPost) requestBuilder.build();
if (queryString != null && !queryString.equals("")) {
StringEntity reqEntity = new StringEntity(queryString);
// 设置类型
reqEntity.setContentType("application/x-www-form-urlencoded");
// 设置请求的数据
httpPost.setEntity(reqEntity);
}
try {
HttpResponse response = httpClient.execute(httpPost);
log.info("QHttpClient httpPost [2] StatusLine = " + response.getStatusLine());
responseData = EntityUtils.toString(response.getEntity());
log.info("QHttpClient httpPost [3] responseData = " + responseData);
} catch (Exception e) {
e.printStackTrace();
} finally {
httpPost.abort();
}
return responseData;
}
@Override
public Page download(Request request, Task task) {
Site site = null;
if (task != null) {
site = task.getSite();
}
Set<Integer> acceptStatCode;
String charset = null;
Map<String, String> headers = null;
if (site != null) {
acceptStatCode = site.getAcceptStatCode();
charset = site.getCharset();
headers = site.getHeaders();
} else {
acceptStatCode = Sets.newHashSet(200);
}
logger.info("downloading page " + request.getUrl());
RequestBuilder requestBuilder = RequestBuilder.get().setUri(request.getUrl());
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut())
.setSocketTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (site != null && site.getHttpProxy() != null) {
requestConfigBuilder.setProxy(site.getHttpProxy());
}
requestBuilder.setConfig(requestConfigBuilder.build());
CloseableHttpResponse httpResponse = null;
try {
httpResponse = getHttpClient(site).execute(requestBuilder.build());
int statusCode = httpResponse.getStatusLine().getStatusCode();
if (acceptStatCode.contains(statusCode)) {
// charset
if (charset == null) {
String value = httpResponse.getEntity().getContentType().getValue();
charset = UrlUtils.getCharset(value);
}
return handleResponse(request, charset, httpResponse, task);
} else {
logger.warn("code error " + statusCode + "\t" + request.getUrl());
return null;
}
} catch (IOException e) {
logger.warn("download page " + request.getUrl() + " error", e);
if (site.getCycleRetryTimes() > 0) {
return addToCycleRetry(request, site);
}
return null;
} finally {
try {
if (httpResponse != null) {
// ensure the connection is released back to pool
EntityUtils.consume(httpResponse.getEntity());
}
} catch (IOException e) {
logger.warn("close response fail", e);
}
}
}
/** 传入加入参数post 参数的url */
public CrawlResultPojo crawl4Post(UrlPojo urlPojo) {
if (urlPojo == null) {
return null;
}
CrawlResultPojo crawlResultPojo = new CrawlResultPojo();
CloseableHttpResponse response1 = null;
BufferedReader br = null;
try {
RequestBuilder rb = RequestBuilder.post().setUri(new URI(urlPojo.getUrl()));
/* HttpUriRequest urlRequest = RequestBuilder.post()
.setUri(new URI(urlPojo.getUrl()))
.addParameter("IDToken1", "username")
.addParameter("IDToken2", "password")
.build();*/
Map<String, Object> paramsMap = urlPojo.getParamsMap();
if (paramsMap != null) {
for (Entry<String, Object> entry : paramsMap.entrySet()) {
rb.addParameter(entry.getKey(), entry.getValue().toString());
}
}
HttpUriRequest httpRequst = rb.build();
response1 = httpclient.execute(httpRequst);
HttpEntity entity = response1.getEntity();
InputStreamReader isr = new InputStreamReader(entity.getContent(), "utf-8");
br = new BufferedReader(isr);
String line = null;
StringBuilder stringBuilder = new StringBuilder();
while ((line = br.readLine()) != null) {
stringBuilder.append(line + "\n");
}
crawlResultPojo.setSuccess(true);
crawlResultPojo.setPageContent(stringBuilder.toString());
return crawlResultPojo;
} catch (Exception e) {
e.printStackTrace();
crawlResultPojo.setSuccess(false);
} finally {
if (response1 != null) {
try {
response1.close();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
if (br != null) {
try {
br.close();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
}
return crawlResultPojo;
}