/**
* Get方法传送消息
*
* @param url 连接的URL
* @param queryString 请求参数串
* @return 服务器返回的信息
* @throws Exception
*/
public String httpGet(String url, String queryString) throws Exception {
String responseData = null;
if (queryString != null && !queryString.equals("")) {
url += "?" + queryString;
}
RequestBuilder requestBuilder = RequestBuilder.get().setUri(url);
log.info("QHttpClient httpGet [1] url = " + url);
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
requestBuilder.setConfig(requestConfig);
HttpResponse response;
HttpGet httpGet = (HttpGet) requestBuilder.build();
response = httpClient.execute(httpGet);
try {
log.info("QHttpClient httpGet [2] StatusLine : " + response.getStatusLine());
responseData = EntityUtils.toString(response.getEntity());
log.info("QHttpClient httpGet [3] Response = " + responseData);
} catch (Exception e) {
e.printStackTrace();
} finally {
httpGet.abort();
}
return responseData;
}
public RequestBuilder listTemplates() {
RequestBuilder requestBuilder = RequestBuilder.get();
addCommonHeaders(requestBuilder);
requestBuilder.setUri(
String.format(
VMTEMPLATES_RESOURCES,
this.provider.getContext().getEndpoint(),
this.provider.getContext().getAccountNumber()));
return requestBuilder;
}
public RequestBuilder listVirtualDisks() {
RequestBuilder requestBuilder = RequestBuilder.get();
addCommonHeaders(requestBuilder);
requestBuilder.setUri(
String.format(
VHD_RESOURCES,
this.provider.getContext().getEndpoint(),
this.provider.getContext().getAccountNumber()));
return requestBuilder;
}
@Override
public void logout() {
final HttpUriRequest logoutGet = RequestBuilder.get().setUri(NYT_LOGOUT_URL).build();
try (final CloseableHttpResponse getResponse = this.getHttpClient().execute(logoutGet)) {
// successful NYT logout should give 200 status
final int responseStatus = getResponse.getStatusLine().getStatusCode();
if (responseStatus != 200) {
final String errorMessage =
String.format("did not detect expected 200, got %d instead", responseStatus);
throw new LogoutException(errorMessage);
}
// successful NYT logout should delete a few cookies like this:
// Set-Cookie: NYT-S=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT; path=/;
// domain=.nytimes.com
final Header[] cookies = getResponse.getHeaders("Set-Cookie");
if (cookies.length < 1) {
throw new LogoutException("no cookie deletions detected, logout might have failed");
}
final Stream<Header> cookieStream = Arrays.stream(cookies);
final Predicate<Header> deletedCheck = c -> c.getValue().contains("deleted");
if (!cookieStream.allMatch(deletedCheck)) {
final List<Header> unexpectedCookies =
cookieStream.filter(deletedCheck).collect(Collectors.toList());
LOG.error("unexpected cookies={}", unexpectedCookies);
throw new LogoutException("unexpected cookie(s) set, loguout might have failed");
}
LOG.info("successfully logged out of nyt");
} catch (IOException | LogoutException e) {
LOG.error("error while logging out of nyt, e={}", e);
}
}
@Override
public boolean authenticate() {
if (!super.authenticate()) {
LOG.error(
String.format(
"blank username or password detected, no %s xword will be downloaded",
this.getType()));
return false;
}
final HttpUriRequest loginGet = RequestBuilder.get().setUri(NYT_LOGIN_URL).build();
final String loginPage;
try (final CloseableHttpResponse getResponse = this.getHttpClient().execute(loginGet)) {
loginPage = EntityUtils.toString(getResponse.getEntity());
} catch (final IOException e) {
LOG.error("error while navigating to NYT login page", e);
return false;
}
final String token;
final String expires;
try {
final TagNode node = this.getCleaner().clean(loginPage);
final Object[] foundNodes = node.evaluateXPath("//input[@name='token']");
if (foundNodes.length != 1) {
this.throwLoginException(
"unexpected login page, found %d hidden token input elements, expected 1",
foundNodes.length);
}
final TagNode hiddenTokenInput = (TagNode) foundNodes[0];
token = hiddenTokenInput.getAttributeByName("value");
LOG.debug("found hidden input token {}", token);
final Object[] foundExpiresNodes = node.evaluateXPath("//input[@name='expires']");
if (foundExpiresNodes.length != 1) {
this.throwLoginException(
"unexpected login page, found %d hidden token expiration input elements, expected 1",
foundNodes.length);
}
final TagNode hiddenTokenExpiresInput = (TagNode) foundExpiresNodes[0];
expires = hiddenTokenExpiresInput.getAttributeByName("value");
LOG.debug("found hidden input token expiration {}", expires);
} catch (LoginException | XPatherException e) {
LOG.error("error while pulling login tokens from NYT login page", e);
return false;
}
// @formatter:off
final HttpUriRequest loginPost =
RequestBuilder.post()
.setUri("https://myaccount.nytimes.com/auth/login")
.addParameter("is_continue", Boolean.FALSE.toString())
.addParameter("token", token)
.addParameter("expires", expires)
.addParameter("userid", this.getLoginInfo().getUsername())
.addParameter("password", this.getLoginInfo().getPassword())
.addParameter("remember", Boolean.TRUE.toString())
.build();
// @formatter:on
try (CloseableHttpResponse postResponse = this.getHttpClient().execute(loginPost)) {
// successful NYT login should give 302 status
final int responseStatus = postResponse.getStatusLine().getStatusCode();
if (responseStatus != 302) {
final String errorMessage =
String.format("did not detect expected 302 redirect, got %d instead", responseStatus);
throw new LoginException(errorMessage);
}
// successful NYT login redirects to the NYT homepage
final Header location = postResponse.getFirstHeader("Location");
// have seen this redirect both with and without the final portion
final Pattern expectedRedirectLocation =
Pattern.compile("http://www.nytimes.com(\\?login=email)*");
final String actualRedirectLocation = location.getValue();
final Matcher matcher = expectedRedirectLocation.matcher(actualRedirectLocation);
if (!matcher.matches()) {
final String errorMessage =
String.format(
"redirect to unexpected URL, expected %s, found Location=%s instead",
expectedRedirectLocation, actualRedirectLocation);
throw new LoginException(errorMessage);
}
// successful NYT login should set a few cookies
final Header[] cookies = postResponse.getHeaders("Set-Cookie");
if (cookies.length < 1) {
throw new LoginException("no post login cookies set, login likely failed");
}
} catch (final IOException | LoginException e) {
LOG.error("error while logging in, e={}", e.getMessage());
return false;
}
LOG.info("successfully logged in to nyt");
return true;
}
/**
* 设置请求参数<br>
* 如果想提交一段字符串<br>
* 那么需要将header中的content-type设置成非application/x-www-form-urlencoded;<br>
* 将字符串放到postdata中参数名postdata
*
* @param method
* @param postdata
* @param headers
* @return
*/
protected RequestBuilder selectRequestMethod(
String method, Map<String, String> postdata, Map<String, String> headers) {
if (method == null || method.equalsIgnoreCase("get")) {
return RequestBuilder.get();
} else if (method.equalsIgnoreCase("post")) {
RequestBuilder requestBuilder = RequestBuilder.post();
if (postdata != null) {
String contenttype = "application/x-www-form-urlencoded; charset=UTF-8";
if (headers != null && headers.size() > 0) {
for (String ksy : headers.keySet()) {
if ("Content-Type".equalsIgnoreCase(ksy)) {
contenttype = headers.get(ksy).toLowerCase().trim();
break;
}
}
}
String charset = ""; // 提交数据的传输编码
if (contenttype.contains("charset")) {
// 如果在请求的编码中，存在网页编码，那么提取改编码
String ncharset =
contenttype.substring(contenttype.lastIndexOf("=") + 1, contenttype.length());
// 如果提取到的编码合法那么使用该编码
if (checkCharset(ncharset)) {
charset = ncharset;
}
}
// 如果编码为空或不存在，在提交数据中提取编码
if (null == charset || "".equals(charset)) {
charset = postdata.get("charset"); // 提交数据的传输编码
}
if (null == charset || "".equals(charset)) {
charset = "UTF-8"; // 提交数据的传输编码
}
if ("".equals(contenttype) || contenttype.toLowerCase().contains("x-www-form-urlencoded")) {
List<NameValuePair> formParams = new ArrayList<NameValuePair>();
for (String str : postdata.keySet()) {
NameValuePair n = new BasicNameValuePair(str, postdata.get(str));
formParams.add(n);
}
HttpEntity entity = null;
try {
entity = new UrlEncodedFormEntity(formParams, charset);
} catch (UnsupportedEncodingException e) {
log.error(e.getMessage(), e);
}
requestBuilder.setEntity(entity);
} else {
log.info("post Content-Type : [ " + contenttype + " ] , pay attention to it .");
String pstdata = postdata.get("postdata"); // 提交的数据
if ("".equals(pstdata) || pstdata == null) {
pstdata = postdata.get(""); // 提交的数据
}
if (pstdata == null) {
pstdata = "";
}
StringEntity entity = new StringEntity(pstdata, charset); // 解决中文乱码问题
entity.setContentEncoding(charset);
entity.setContentType(contenttype);
entity.setChunked(true);
requestBuilder.setEntity(entity);
}
} else {
log.warn("The Method Is Post,But No Post Data .");
}
return requestBuilder;
} else if (method.equalsIgnoreCase("head")) {
return RequestBuilder.head();
} else if (method.equalsIgnoreCase("put")) {
return RequestBuilder.put();
} else if (method.equalsIgnoreCase("delete")) {
return RequestBuilder.delete();
} else if (method.equalsIgnoreCase("trace")) {
return RequestBuilder.trace();
}
throw new IllegalArgumentException("Illegal HTTP Method " + method);
}
@Override
public Page download(Request request, Task task) {
Site site = null;
if (task != null) {
site = task.getSite();
}
Set<Integer> acceptStatCode;
String charset = null;
Map<String, String> headers = null;
if (site != null) {
acceptStatCode = site.getAcceptStatCode();
charset = site.getCharset();
headers = site.getHeaders();
} else {
acceptStatCode = Sets.newHashSet(200);
}
logger.info("downloading page " + request.getUrl());
RequestBuilder requestBuilder = RequestBuilder.get().setUri(request.getUrl());
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut())
.setSocketTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (site != null && site.getHttpProxy() != null) {
requestConfigBuilder.setProxy(site.getHttpProxy());
}
requestBuilder.setConfig(requestConfigBuilder.build());
CloseableHttpResponse httpResponse = null;
try {
httpResponse = getHttpClient(site).execute(requestBuilder.build());
int statusCode = httpResponse.getStatusLine().getStatusCode();
if (acceptStatCode.contains(statusCode)) {
// charset
if (charset == null) {
String value = httpResponse.getEntity().getContentType().getValue();
charset = UrlUtils.getCharset(value);
}
return handleResponse(request, charset, httpResponse, task);
} else {
logger.warn("code error " + statusCode + "\t" + request.getUrl());
return null;
}
} catch (IOException e) {
logger.warn("download page " + request.getUrl() + " error", e);
if (site.getCycleRetryTimes() > 0) {
return addToCycleRetry(request, site);
}
return null;
} finally {
try {
if (httpResponse != null) {
// ensure the connection is released back to pool
EntityUtils.consume(httpResponse.getEntity());
}
} catch (IOException e) {
logger.warn("close response fail", e);
}
}
}
