/**
* Get方法传送消息
*
* @param url 连接的URL
* @param queryString 请求参数串
* @return 服务器返回的信息
* @throws Exception
*/
public String httpGet(String url, String queryString) throws Exception {
String responseData = null;
if (queryString != null && !queryString.equals("")) {
url += "?" + queryString;
}
RequestBuilder requestBuilder = RequestBuilder.get().setUri(url);
log.info("QHttpClient httpGet [1] url = " + url);
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
requestBuilder.setConfig(requestConfig);
HttpResponse response;
HttpGet httpGet = (HttpGet) requestBuilder.build();
response = httpClient.execute(httpGet);
try {
log.info("QHttpClient httpGet [2] StatusLine : " + response.getStatusLine());
responseData = EntityUtils.toString(response.getEntity());
log.info("QHttpClient httpGet [3] Response = " + responseData);
} catch (Exception e) {
e.printStackTrace();
} finally {
httpGet.abort();
}
return responseData;
}
/**
* Called primarily from HTTPMethod to do the bulk of the execution. Assumes HTTPMethod has
* inserted its headers into request.
*
* @param method
* @param methoduri
* @param rb
* @return Request+Response pair
* @throws HTTPException
*/
ExecState execute(HTTPMethod method, URI methoduri, RequestBuilder rb) throws HTTPException {
this.execution = new ExecState();
this.requestURI = methoduri;
AuthScope methodscope = HTTPAuthUtil.uriToAuthScope(methoduri);
AuthScope target = HTTPAuthUtil.authscopeUpgrade(this.scope, methodscope);
synchronized (this) { // keep coverity happy
// Merge Settings;
Settings merged = HTTPUtil.merge(globalsettings, localsettings);
if (!this.cachevalid) {
RequestConfig.Builder rcb = RequestConfig.custom();
this.cachedconfig = configureRequest(rcb, merged);
HttpClientBuilder cb = HttpClients.custom();
configClient(cb, merged);
setAuthenticationAndProxy(cb);
this.cachedclient = cb.build();
rb.setConfig(this.cachedconfig);
this.cachevalid = true;
}
}
this.execution.request = (HttpRequestBase) rb.build();
try {
HttpHost targethost = HTTPAuthUtil.authscopeToHost(target);
this.execution.response =
cachedclient.execute(targethost, this.execution.request, this.sessioncontext);
} catch (IOException ioe) {
throw new HTTPException(ioe);
}
return this.execution;
}
public RequestBuilder listVirtualDisks() {
RequestBuilder requestBuilder = RequestBuilder.get();
addCommonHeaders(requestBuilder);
requestBuilder.setUri(
String.format(
VHD_RESOURCES,
this.provider.getContext().getEndpoint(),
this.provider.getContext().getAccountNumber()));
return requestBuilder;
}
public RequestBuilder listTemplates() {
RequestBuilder requestBuilder = RequestBuilder.get();
addCommonHeaders(requestBuilder);
requestBuilder.setUri(
String.format(
VMTEMPLATES_RESOURCES,
this.provider.getContext().getEndpoint(),
this.provider.getContext().getAccountNumber()));
return requestBuilder;
}
@Override
public RequestBuilder transform(HttpServletRequest request)
throws NoSuchRequestHandlingMethodException, URISyntaxException, IOException {
RequestBuilder requestBuilder = predecessor.transform(request);
String requestContent = request.getReader().lines().collect(Collectors.joining(""));
if (!requestContent.isEmpty()) {
StringEntity entity = new StringEntity(requestContent, ContentType.APPLICATION_JSON);
requestBuilder.setEntity(entity);
}
return requestBuilder;
}
/**
* Build a new Apache's {@link org.apache.http.client.methods.HttpUriRequest} from Jersey's {@link
* org.glassfish.jersey.client.ClientRequest}
*
* <p>Convert a method, URI, body, headers and override a user-agent if necessary
*
* @param jerseyRequest representation of an HTTP request in Jersey
* @return a new {@link org.apache.http.client.methods.HttpUriRequest}
*/
private HttpUriRequest buildApacheRequest(ClientRequest jerseyRequest) {
final RequestBuilder builder =
RequestBuilder.create(jerseyRequest.getMethod())
.setUri(jerseyRequest.getUri())
.setEntity(getHttpEntity(jerseyRequest));
for (String headerName : jerseyRequest.getHeaders().keySet()) {
builder.addHeader(headerName, jerseyRequest.getHeaderString(headerName));
}
final Optional<RequestConfig> requestConfig = addJerseyRequestConfig(jerseyRequest);
requestConfig.ifPresent(builder::setConfig);
return builder.build();
}
public PingInfo ping(String host, int restPort, boolean noProxy) throws Exception {
PingInfo myPingInfo = pingInfoProvider.createPingInfo();
RequestConfig.Builder requestConfigBuilder =
httpClientProvider.createRequestConfigBuilder(host, noProxy);
String url = String.format(URL_PATTERN, host, restPort) + PingHandler.URL;
HttpUriRequest request =
RequestBuilder.post(url)
.setConfig(requestConfigBuilder.build())
.addParameter(PingHandler.PING_INFO_INPUT_NAME, gson.toJson(myPingInfo))
.build();
CloseableHttpResponse response = httpClientProvider.executeRequest(request);
try {
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
EntityUtils.consumeQuietly(response.getEntity());
throw new Exception(statusLine.getStatusCode() + " " + statusLine.getReasonPhrase());
}
HttpEntity entity = response.getEntity();
String content = EntityUtils.toString(entity);
EntityUtils.consumeQuietly(entity);
PingInfo receivedPingInfo = gson.fromJson(content, PingInfo.class);
receivedPingInfo.getAgentId().setHost(request.getURI().getHost());
return receivedPingInfo;
} finally {
response.close();
}
}
private void assertBuild(final Charset charset) throws Exception {
final RequestBuilder requestBuilder = RequestBuilder.create("GET").setCharset(charset);
requestBuilder.setUri("https://somehost.com/stuff");
requestBuilder.addParameters(createParameters());
final String encodedData1 = URLEncoder.encode("\"1\u00aa position\"", charset.displayName());
final String encodedData2 = URLEncoder.encode("Jos\u00e9 Abra\u00e3o", charset.displayName());
final String uriExpected =
String.format(
"https://somehost.com/stuff?parameter1=value1¶meter2=%s¶meter3=%s",
encodedData1, encodedData2);
final HttpUriRequest request = requestBuilder.build();
Assert.assertEquals(uriExpected, request.getURI().toString());
}
/**
* 发货通知
*
* @param access_token
* @param delivernotifyJson
* @return
*/
private static BaseResult payDelivernotify(String access_token, String delivernotifyJson) {
HttpUriRequest httpUriRequest =
RequestBuilder.post()
.setHeader(jsonHeader)
.setUri(BASE_URI + "/pay/delivernotify")
.addParameter("access_token", access_token)
.setEntity(new StringEntity(delivernotifyJson, Charset.forName("utf-8")))
.build();
return LocalHttpClient.executeJsonResult(httpUriRequest, BaseResult.class);
}
/**
* 订单查询
*
* @param access_token
* @param orderqueryJson
* @return
*/
private static OrderInfo payOrderquery(String access_token, String orderqueryJson) {
HttpUriRequest httpUriRequest =
RequestBuilder.post()
.setHeader(jsonHeader)
.setUri(BASE_URI + "/pay/orderquery")
.addParameter("access_token", access_token)
.setEntity(new StringEntity(orderqueryJson, Charset.forName("utf-8")))
.build();
return LocalHttpClient.executeJsonResult(httpUriRequest, OrderInfo.class);
}
private <B, R> R executeRequest(
RequestSpec requestSpec,
PathParams pathParams,
B requestBody,
Class<R> responseType,
int expectedStatus) {
String url =
String.format(
ADMIN_URL_PREFIX + requestSpec.path(pathParams), scheme, host, port, urlPathPrefix);
RequestBuilder requestBuilder =
RequestBuilder.create(requestSpec.method().getName()).setUri(url);
if (requestBody != null) {
requestBuilder.setEntity(jsonStringEntity(Json.write(requestBody)));
}
String responseBodyString = safelyExecuteRequest(url, expectedStatus, requestBuilder.build());
return responseType == Void.class ? null : Json.read(responseBodyString, responseType);
}
/**
* 标记客户的投诉处理状态
*
* @param access_token
* @param openid
* @param feedbackid
* @return
*/
public static BaseResult payfeedbackUpdate(
String access_token, String openid, String feedbackid) {
HttpUriRequest httpUriRequest =
RequestBuilder.post()
.setUri(BASE_URI + "/payfeedback/update")
.addParameter("access_token", access_token)
.addParameter("openid", openid)
.addParameter("feedbackid", feedbackid)
.build();
return LocalHttpClient.executeJsonResult(httpUriRequest, BaseResult.class);
}
/**
* Post方法传送消息
*
* @param url 连接的URL
* @param queryString 请求参数串
* @return 服务器返回的信息
* @throws Exception
*/
public String httpPost(String url, String queryString) throws Exception {
String responseData = null;
URI tmpUri = new URI(url);
URI uri =
new URI(
tmpUri.getScheme(),
null,
tmpUri.getHost(),
tmpUri.getPort(),
tmpUri.getPath(),
queryString,
null);
log.info("QHttpClient httpPost [1] url = " + uri.toURL());
RequestBuilder requestBuilder = RequestBuilder.post().setUri(uri);
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
requestBuilder.setConfig(requestConfig);
HttpPost httpPost = (HttpPost) requestBuilder.build();
if (queryString != null && !queryString.equals("")) {
StringEntity reqEntity = new StringEntity(queryString);
// 设置类型
reqEntity.setContentType("application/x-www-form-urlencoded");
// 设置请求的数据
httpPost.setEntity(reqEntity);
}
try {
HttpResponse response = httpClient.execute(httpPost);
log.info("QHttpClient httpPost [2] StatusLine = " + response.getStatusLine());
responseData = EntityUtils.toString(response.getEntity());
log.info("QHttpClient httpPost [3] responseData = " + responseData);
} catch (Exception e) {
e.printStackTrace();
} finally {
httpPost.abort();
}
return responseData;
}
/**
* 构造请求的方法，如post，get，header等<br>
* 设置请求参数，如超时时间
*
* @param url 请求的URL
* @param method 请求的方法
* @param postdata post的数据
* @param headers 请求头
* @return
*/
protected HttpUriRequest getHttpUriRequest(
String url, String method, Map<String, String> postdata, Map<String, String> headers) {
RequestBuilder requestBuilder = selectRequestMethod(method, postdata, headers).setUri(url);
requestBuilder.addHeader("Accept", "*/*");
requestBuilder.addHeader("Connection", "keep-alive");
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
int timeout = 45000; // 超时时间
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(timeout)
.setSocketTimeout(timeout)
.setConnectTimeout(timeout)
// .setRedirectsEnabled(false)//禁止自动跳转
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (null != host && !"".equals(host) && port > 10) {
HttpHost proxy = new HttpHost(host, port);
requestConfigBuilder.setProxy(proxy);
}
requestBuilder.setConfig(requestConfigBuilder.build());
return requestBuilder.build();
}
@Override
public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context)
throws ProtocolException {
RequestLine line = request.getRequestLine();
String location = getLocation(request, response, context).toString();
Log.i(
TAG,
"Following redirection: " + line.getMethod() + " " + line.getUri() + " -> " + location);
return RequestBuilder.copy(request)
.setUri(location)
.removeHeaders(
"Content-Length") // Content-Length will be set again automatically, if required;
// remove it now to avoid duplicate header
.build();
}
@Override
public void logout() {
final HttpUriRequest logoutGet = RequestBuilder.get().setUri(NYT_LOGOUT_URL).build();
try (final CloseableHttpResponse getResponse = this.getHttpClient().execute(logoutGet)) {
// successful NYT logout should give 200 status
final int responseStatus = getResponse.getStatusLine().getStatusCode();
if (responseStatus != 200) {
final String errorMessage =
String.format("did not detect expected 200, got %d instead", responseStatus);
throw new LogoutException(errorMessage);
}
// successful NYT logout should delete a few cookies like this:
// Set-Cookie: NYT-S=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT; path=/;
// domain=.nytimes.com
final Header[] cookies = getResponse.getHeaders("Set-Cookie");
if (cookies.length < 1) {
throw new LogoutException("no cookie deletions detected, logout might have failed");
}
final Stream<Header> cookieStream = Arrays.stream(cookies);
final Predicate<Header> deletedCheck = c -> c.getValue().contains("deleted");
if (!cookieStream.allMatch(deletedCheck)) {
final List<Header> unexpectedCookies =
cookieStream.filter(deletedCheck).collect(Collectors.toList());
LOG.error("unexpected cookies={}", unexpectedCookies);
throw new LogoutException("unexpected cookie(s) set, loguout might have failed");
}
LOG.info("successfully logged out of nyt");
} catch (IOException | LogoutException e) {
LOG.error("error while logging out of nyt, e={}", e);
}
}
private JSONObject makePostRequest(
String userName, String password, String resourceID, String distributor) {
JSONObject reply;
try {
BasicNameValuePair rID = new BasicNameValuePair(REQUEST_PARAMETER_RESOURCE_ID, resourceID);
BasicNameValuePair rDistributor =
new BasicNameValuePair(REQUEST_PARAMETER_DISTRIBUTOR, distributor);
HttpClient client = HttpClients.custom().build();
HttpUriRequest request =
RequestBuilder.post()
.setUri(SERVICE_URI + "?" + rID + "&" + rDistributor)
.setHeader(HttpHeaders.ACCEPT, CONTENT_TYPE_APPLICATION_JSON)
.setHeader(HttpHeaders.CONTENT_TYPE, CONTENT_TYPE_APPLICATION_JSON)
.setHeader(
HttpHeaders.AUTHORIZATION,
HTTP_AUTHORIZATION_METHOD + encryptCredentials(userName, password))
.setEntity(createRequestPayload())
.build();
HttpResponse response = client.execute(request);
reply = parseResponse(response);
} catch (UnsupportedEncodingException e) {
LOGGER.error(
"Abort provisional exchange for resource "
+ resourceID
+ " failed due to: Invalid encoding");
reply = createFailureReply();
} catch (IOException e) {
LOGGER.error(
"Abort provisional exchange for resource "
+ resourceID
+ " failed due to: IO error, could not connect to the service");
reply = createFailureReply();
}
return reply;
}
/** 传入加入参数post 参数的url */
public CrawlResultPojo crawl4Post(UrlPojo urlPojo) {
if (urlPojo == null) {
return null;
}
CrawlResultPojo crawlResultPojo = new CrawlResultPojo();
CloseableHttpResponse response1 = null;
BufferedReader br = null;
try {
RequestBuilder rb = RequestBuilder.post().setUri(new URI(urlPojo.getUrl()));
/* HttpUriRequest urlRequest = RequestBuilder.post()
.setUri(new URI(urlPojo.getUrl()))
.addParameter("IDToken1", "username")
.addParameter("IDToken2", "password")
.build();*/
Map<String, Object> paramsMap = urlPojo.getParamsMap();
if (paramsMap != null) {
for (Entry<String, Object> entry : paramsMap.entrySet()) {
rb.addParameter(entry.getKey(), entry.getValue().toString());
}
}
HttpUriRequest httpRequst = rb.build();
response1 = httpclient.execute(httpRequst);
HttpEntity entity = response1.getEntity();
InputStreamReader isr = new InputStreamReader(entity.getContent(), "utf-8");
br = new BufferedReader(isr);
String line = null;
StringBuilder stringBuilder = new StringBuilder();
while ((line = br.readLine()) != null) {
stringBuilder.append(line + "\n");
}
crawlResultPojo.setSuccess(true);
crawlResultPojo.setPageContent(stringBuilder.toString());
return crawlResultPojo;
} catch (Exception e) {
e.printStackTrace();
crawlResultPojo.setSuccess(false);
} finally {
if (response1 != null) {
try {
response1.close();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
if (br != null) {
try {
br.close();
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
}
return crawlResultPojo;
}
@Override
public boolean authenticate() {
if (!super.authenticate()) {
LOG.error(
String.format(
"blank username or password detected, no %s xword will be downloaded",
this.getType()));
return false;
}
final HttpUriRequest loginGet = RequestBuilder.get().setUri(NYT_LOGIN_URL).build();
final String loginPage;
try (final CloseableHttpResponse getResponse = this.getHttpClient().execute(loginGet)) {
loginPage = EntityUtils.toString(getResponse.getEntity());
} catch (final IOException e) {
LOG.error("error while navigating to NYT login page", e);
return false;
}
final String token;
final String expires;
try {
final TagNode node = this.getCleaner().clean(loginPage);
final Object[] foundNodes = node.evaluateXPath("//input[@name='token']");
if (foundNodes.length != 1) {
this.throwLoginException(
"unexpected login page, found %d hidden token input elements, expected 1",
foundNodes.length);
}
final TagNode hiddenTokenInput = (TagNode) foundNodes[0];
token = hiddenTokenInput.getAttributeByName("value");
LOG.debug("found hidden input token {}", token);
final Object[] foundExpiresNodes = node.evaluateXPath("//input[@name='expires']");
if (foundExpiresNodes.length != 1) {
this.throwLoginException(
"unexpected login page, found %d hidden token expiration input elements, expected 1",
foundNodes.length);
}
final TagNode hiddenTokenExpiresInput = (TagNode) foundExpiresNodes[0];
expires = hiddenTokenExpiresInput.getAttributeByName("value");
LOG.debug("found hidden input token expiration {}", expires);
} catch (LoginException | XPatherException e) {
LOG.error("error while pulling login tokens from NYT login page", e);
return false;
}
// @formatter:off
final HttpUriRequest loginPost =
RequestBuilder.post()
.setUri("https://myaccount.nytimes.com/auth/login")
.addParameter("is_continue", Boolean.FALSE.toString())
.addParameter("token", token)
.addParameter("expires", expires)
.addParameter("userid", this.getLoginInfo().getUsername())
.addParameter("password", this.getLoginInfo().getPassword())
.addParameter("remember", Boolean.TRUE.toString())
.build();
// @formatter:on
try (CloseableHttpResponse postResponse = this.getHttpClient().execute(loginPost)) {
// successful NYT login should give 302 status
final int responseStatus = postResponse.getStatusLine().getStatusCode();
if (responseStatus != 302) {
final String errorMessage =
String.format("did not detect expected 302 redirect, got %d instead", responseStatus);
throw new LoginException(errorMessage);
}
// successful NYT login redirects to the NYT homepage
final Header location = postResponse.getFirstHeader("Location");
// have seen this redirect both with and without the final portion
final Pattern expectedRedirectLocation =
Pattern.compile("http://www.nytimes.com(\\?login=email)*");
final String actualRedirectLocation = location.getValue();
final Matcher matcher = expectedRedirectLocation.matcher(actualRedirectLocation);
if (!matcher.matches()) {
final String errorMessage =
String.format(
"redirect to unexpected URL, expected %s, found Location=%s instead",
expectedRedirectLocation, actualRedirectLocation);
throw new LoginException(errorMessage);
}
// successful NYT login should set a few cookies
final Header[] cookies = postResponse.getHeaders("Set-Cookie");
if (cookies.length < 1) {
throw new LoginException("no post login cookies set, login likely failed");
}
} catch (final IOException | LoginException e) {
LOG.error("error while logging in, e={}", e.getMessage());
return false;
}
LOG.info("successfully logged in to nyt");
return true;
}
public static void save() {
URI uri;
try {
uri = new URI(url);
HttpUriRequest request =
RequestBuilder.post()
.setUri(uri)
.addParameter("productCode", "")
// .addParameter("formName", "reportQueryConditon")
// .addParameter("receiveCode", "")
// .addParameter("buyerProductcode", "101123403")
.build();
CloseableHttpResponse response2 = MyMainClient.getHttpclient().execute(request);
int statusCode = response2.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY
|| statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
try {
MyLogin.loginpost(new URI(MyLogin.url));
} catch (URISyntaxException e) {
logger.info(e);
}
response2 = MyMainClient.getHttpclient().execute(request);
}
MyMainClient.mapreturn.put("InventoryModelDownLoadstatue", statusCode + "");
HttpEntity entity = response2.getEntity();
// EntityUtils.consume(entity);
InputStream in = entity.getContent();
String tempPath = PathUtill.getXMLpath();
tempPath +=
"data"
+ File.separator
+ "DownloadInventory"
+ File.separator
+ TimeUtill.getdateString()
+ File.separator
+ "SuNing";
logger.info(tempPath);
File file = new File(tempPath);
if (!file.exists()) {
file.mkdirs();
}
File file2 = new File(tempPath + File.separator + "model.csv");
file2.createNewFile();
try {
FileOutputStream fout = new FileOutputStream(file2);
int l = -1;
byte[] tmp = new byte[1024];
while ((l = in.read(tmp)) != -1) {
fout.write(tmp, 0, l);
// 注意这里如果用OutputStream.write(buff)的话，图片会失真，大家可以试试
}
fout.flush();
fout.close();
response2.close();
} finally {
// 关闭低层流。
in.close();
}
} catch (URISyntaxException e) {
// TODO Auto-generated catch block
logger.info(e);
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
logger.info(e);
} catch (IOException e) {
// TODO Auto-generated catch block
logger.info(e);
}
}
/**
* 设置请求参数<br>
* 如果想提交一段字符串<br>
* 那么需要将header中的content-type设置成非application/x-www-form-urlencoded;<br>
* 将字符串放到postdata中参数名postdata
*
* @param method
* @param postdata
* @param headers
* @return
*/
protected RequestBuilder selectRequestMethod(
String method, Map<String, String> postdata, Map<String, String> headers) {
if (method == null || method.equalsIgnoreCase("get")) {
return RequestBuilder.get();
} else if (method.equalsIgnoreCase("post")) {
RequestBuilder requestBuilder = RequestBuilder.post();
if (postdata != null) {
String contenttype = "application/x-www-form-urlencoded; charset=UTF-8";
if (headers != null && headers.size() > 0) {
for (String ksy : headers.keySet()) {
if ("Content-Type".equalsIgnoreCase(ksy)) {
contenttype = headers.get(ksy).toLowerCase().trim();
break;
}
}
}
String charset = ""; // 提交数据的传输编码
if (contenttype.contains("charset")) {
// 如果在请求的编码中，存在网页编码，那么提取改编码
String ncharset =
contenttype.substring(contenttype.lastIndexOf("=") + 1, contenttype.length());
// 如果提取到的编码合法那么使用该编码
if (checkCharset(ncharset)) {
charset = ncharset;
}
}
// 如果编码为空或不存在，在提交数据中提取编码
if (null == charset || "".equals(charset)) {
charset = postdata.get("charset"); // 提交数据的传输编码
}
if (null == charset || "".equals(charset)) {
charset = "UTF-8"; // 提交数据的传输编码
}
if ("".equals(contenttype) || contenttype.toLowerCase().contains("x-www-form-urlencoded")) {
List<NameValuePair> formParams = new ArrayList<NameValuePair>();
for (String str : postdata.keySet()) {
NameValuePair n = new BasicNameValuePair(str, postdata.get(str));
formParams.add(n);
}
HttpEntity entity = null;
try {
entity = new UrlEncodedFormEntity(formParams, charset);
} catch (UnsupportedEncodingException e) {
log.error(e.getMessage(), e);
}
requestBuilder.setEntity(entity);
} else {
log.info("post Content-Type : [ " + contenttype + " ] , pay attention to it .");
String pstdata = postdata.get("postdata"); // 提交的数据
if ("".equals(pstdata) || pstdata == null) {
pstdata = postdata.get(""); // 提交的数据
}
if (pstdata == null) {
pstdata = "";
}
StringEntity entity = new StringEntity(pstdata, charset); // 解决中文乱码问题
entity.setContentEncoding(charset);
entity.setContentType(contenttype);
entity.setChunked(true);
requestBuilder.setEntity(entity);
}
} else {
log.warn("The Method Is Post,But No Post Data .");
}
return requestBuilder;
} else if (method.equalsIgnoreCase("head")) {
return RequestBuilder.head();
} else if (method.equalsIgnoreCase("put")) {
return RequestBuilder.put();
} else if (method.equalsIgnoreCase("delete")) {
return RequestBuilder.delete();
} else if (method.equalsIgnoreCase("trace")) {
return RequestBuilder.trace();
}
throw new IllegalArgumentException("Illegal HTTP Method " + method);
}
private void addCommonHeaders(RequestBuilder requestBuilder) {
requestBuilder.addHeader("x-ms-version", "2014-02-01");
requestBuilder.addHeader("Accept", "application/json");
}
@Override
public Page download(Request request, Task task) {
Site site = null;
if (task != null) {
site = task.getSite();
}
Set<Integer> acceptStatCode;
String charset = null;
Map<String, String> headers = null;
if (site != null) {
acceptStatCode = site.getAcceptStatCode();
charset = site.getCharset();
headers = site.getHeaders();
} else {
acceptStatCode = Sets.newHashSet(200);
}
logger.info("downloading page " + request.getUrl());
RequestBuilder requestBuilder = RequestBuilder.get().setUri(request.getUrl());
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut())
.setSocketTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (site != null && site.getHttpProxy() != null) {
requestConfigBuilder.setProxy(site.getHttpProxy());
}
requestBuilder.setConfig(requestConfigBuilder.build());
CloseableHttpResponse httpResponse = null;
try {
httpResponse = getHttpClient(site).execute(requestBuilder.build());
int statusCode = httpResponse.getStatusLine().getStatusCode();
if (acceptStatCode.contains(statusCode)) {
// charset
if (charset == null) {
String value = httpResponse.getEntity().getContentType().getValue();
charset = UrlUtils.getCharset(value);
}
return handleResponse(request, charset, httpResponse, task);
} else {
logger.warn("code error " + statusCode + "\t" + request.getUrl());
return null;
}
} catch (IOException e) {
logger.warn("download page " + request.getUrl() + " error", e);
if (site.getCycleRetryTimes() > 0) {
return addToCycleRetry(request, site);
}
return null;
} finally {
try {
if (httpResponse != null) {
// ensure the connection is released back to pool
EntityUtils.consume(httpResponse.getEntity());
}
} catch (IOException e) {
logger.warn("close response fail", e);
}
}
}