public static Object fetch(DocumentSource source) {
try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
URIBuilder builder = new URIBuilder(source.url);
for (Entry<String, String> it : source.parameters.entrySet()) {
builder.addParameter(it.getKey(), it.getValue());
}
URI uri = builder.build();
HttpUriRequest request = new HttpGet(uri);
request.addHeader("Accept", "application/json");
CloseableHttpResponse response = httpclient.execute(request);
String headers = response.getFirstHeader("Content-Type").getValue();
InputStream inputStream = response.getEntity().getContent();
if (headers.contains("text/html")) {
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
Document document = Jsoup.parse(inputStream, null, source.url);
return document;
} else if (headers.contains("json")) {
ObjectMapper om = new ObjectMapper();
return om.readValue(inputStream, HashMap.class);
} else {
IOUtils.copy(inputStream, System.err);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
}
protected HttpResponse getHttpResponse(final HttpUriRequest httpRequest, final String url)
throws DSSException {
final HttpClient client = getHttpClient(url);
final String host = httpRequest.getURI().getHost();
final int port = httpRequest.getURI().getPort();
final String scheme = httpRequest.getURI().getScheme();
final HttpHost targetHost = new HttpHost(host, port, scheme);
// Create AuthCache instance
AuthCache authCache = new BasicAuthCache();
// Generate BASIC scheme object and add it to the local
// auth cache
BasicScheme basicAuth = new BasicScheme();
authCache.put(targetHost, basicAuth);
// Add AuthCache to the execution context
HttpClientContext localContext = HttpClientContext.create();
localContext.setAuthCache(authCache);
try {
final HttpResponse response = client.execute(targetHost, httpRequest, localContext);
return response;
} catch (IOException e) {
throw new DSSException(e);
}
}
@Mock
public void $init(
CloudProvider provider,
HttpClientBuilder clientBuilder,
HttpUriRequest request,
ResponseHandler handler) {
String requestUri = request.getURI().toString();
if (request.getMethod().equals("GET")
&& requestUri.equals(
String.format(VM_RESOURCES, ENDPOINT, ACCOUNT_NO, DATACENTER_ID, VM_1_ID))) {
requestResourceType = 1;
}
if (request.getMethod().equals("GET")
&& requestUri.equals(
String.format(VM_NETWORK_ADAPTERS, ENDPOINT, ACCOUNT_NO, DATACENTER_ID, VM_1_ID))) {
requestResourceType = 2;
}
if (request.getMethod().equals("GET")
&& requestUri.equals(String.format(HARDWARE_PROFILES, ENDPOINT, ACCOUNT_NO))) {
requestResourceType = 3;
}
if (request.getMethod().equals("GET")
&& requestUri.equals(String.format(LIST_VM_RESOURCES, ENDPOINT, ACCOUNT_NO))) {
requestResourceType = 4;
}
responseHandler = handler;
}
@Mock
public void $init(
CloudProvider provider,
HttpClientBuilder clientBuilder,
HttpUriRequest request,
ResponseHandler handler) {
String requestUri = request.getURI().toString();
if (request.getMethod().equals("GET")
&& requestUri.equals(
String.format(VM_RESOURCES, ENDPOINT, ACCOUNT_NO, DATACENTER_ID, VM_1_ID))) {
requestResourceType = 11;
} else if (request.getMethod().equals("PUT")
&& requestUri.equals(
String.format(VM_RESOURCES, ENDPOINT, ACCOUNT_NO, DATACENTER_ID, VM_1_ID))) {
requestResourceType = 12;
WAPVirtualMachineModel wapVirtualMachineModel = createWAPVirtualMachineModel();
wapVirtualMachineModel.setOperation(operation);
assertPut(
request,
String.format(VM_RESOURCES, ENDPOINT, ACCOUNT_NO, DATACENTER_ID, VM_1_ID),
new Header[0],
wapVirtualMachineModel);
} else {
super.$init(provider, clientBuilder, request, handler);
}
}
@Override
public Result loadInBackground() {
final Fetch fetch = mFetch;
final HttpConnectionHelper helper = mHelper;
List<NameValuePair> parameters = mParameterExtractor.extract(fetch);
HttpUriRequest request =
helper.obtainHttpRequest(fetch.getFetchMethod(), fetch.getUrl(), parameters);
Utils.debug(request.getURI().toString());
if (fetch.onPreFetch(request, Collections.unmodifiableList(parameters))) {
return mResult;
}
HttpResponse response = helper.execute(request);
fetch.onPostFetch(request);
if (null == response) {
return null;
}
try {
Result result =
mResult = mParser.parse(fetch, new InputStreamReader(response.getEntity().getContent()));
Utils.debug(result);
return result;
} catch (IllegalStateException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
/** {@inheritDoc} */
public GWTLatLng geocode(final String geolocation) throws GeocoderException {
final HttpUriRequest method = new HttpGet(getUrl(geolocation));
method.addHeader("User-Agent", "OpenNMS-MapquestGeocoder/1.0");
if (m_referer != null) {
method.addHeader("Referer", m_referer);
}
try {
InputStream responseStream = m_httpClient.execute(method).getEntity().getContent();
final ElementTree tree = ElementTree.fromStream(responseStream);
if (tree == null) {
throw new GeocoderException(
"an error occurred connecting to the Nominatim geocoding service (no XML tree was found)");
}
final List<ElementTree> places = tree.findAll("//place");
if (places.size() > 1) {
LogUtils.warnf(this, "more than one location returned for query: %s", geolocation);
} else if (places.size() == 0) {
throw new GeocoderException("Nominatim returned an OK status code, but no places");
}
final ElementTree place = places.get(0);
Double latitude = Double.valueOf(place.getAttribute("lat"));
Double longitude = Double.valueOf(place.getAttribute("lon"));
return new GWTLatLng(latitude, longitude);
} catch (GeocoderException e) {
throw e;
} catch (Throwable e) {
throw new GeocoderException("unable to get lat/lng from Nominatim", e);
}
}
private void SetupHTTPConnectionParams(HttpUriRequest method) {
HttpConnectionParams.setConnectionTimeout(method.getParams(), CONNECTION_TIMEOUT_MS);
HttpConnectionParams.setSoTimeout(method.getParams(), SOCKET_TIMEOUT_MS);
// mClient.setHttpRequestRetryHandler(requestRetryHandler);
method.addHeader("Accept-Encoding", "gzip, deflate");
method.addHeader("Accept-Charset", "UTF-8,*;q=0.5");
}
@Override
public void signRequest(HttpUriRequest request) {
AWSCredentials credentials = awsCredentialsProvider.getCredentials();
if (credentials instanceof AWSSessionCredentials) {
request.addHeader(
SESSION_TOKEN_HEADER, ((AWSSessionCredentials) credentials).getSessionToken());
}
String canonicalRequest = createCanonicalRequest(request);
log.debug("canonicalRequest: " + canonicalRequest);
String[] requestParts = canonicalRequest.split("\n");
String signedHeaders = requestParts[requestParts.length - 2];
String stringToSign = createStringToSign(canonicalRequest);
log.debug("stringToSign: " + stringToSign);
String authScope = stringToSign.split("\n")[2];
String signature = createSignature(stringToSign);
String authHeader =
String.format(
AUTH_HEADER_FORMAT,
credentials.getAWSAccessKeyId(),
authScope,
signedHeaders,
signature);
request.addHeader(AUTH_HEADER_NAME, authHeader);
}
private HttpStatus commit(HttpUriRequest method) throws IOException {
HttpStatus status;
HttpClient httpclient = createHttpClient();
// reduce the TIME_WAIT
// method.addHeader("Connection", "close");
if (method.getFirstHeader("Referer") == null) {
URI uri = method.getURI();
method.setHeader("Referer", uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort());
}
;
try {
HttpResponse resp = execute(method, httpclient);
status = new HttpStatus(resp.getStatusLine());
if (resp.getEntity() != null) {
status.setMessage(EntityUtils.toString(resp.getEntity(), "UTF-8"));
}
} catch (IOException e) {
// cancel the connection when the error happen
method.abort();
throw e;
} finally {
HttpClientUtils.abortConnection(method, httpclient);
}
return status;
}
private HttpResponse executeRequest(HttpClient client, HttpUriRequest request)
throws ClientProtocolException, IOException {
if (logger != null && logger.isDebugEnabled()) {
StringBuilder builder = new StringBuilder();
Header[] allHeaders = request.getAllHeaders();
for (Header header : allHeaders) {
builder.append(header.getName());
builder.append(":");
builder.append(header.getValue());
builder.append("\n");
}
logger.debug(
"Executing request \nurl:["
+ request.getURI().toString()
+ "] \nheaders:\n"
+ builder.toString()
+ "");
}
return client.execute(request);
}
String createCanonicalRequest(HttpUriRequest request) {
StringBuilder result = new StringBuilder();
result.append(request.getMethod()).append('\n');
String path = request.getURI().getPath();
if (path.isEmpty()) {
path = "/";
}
result.append(path).append('\n');
String queryString = request.getURI().getQuery();
queryString = queryString != null ? queryString : "";
addCanonicalQueryString(queryString, result).append('\n');
addCanonicalHeaders(request, result).append('\n');
HttpEntity entity = null;
try {
if (request instanceof HttpEntityEnclosingRequestBase) {
entity = ((HttpEntityEnclosingRequestBase) request).getEntity();
} else {
entity = new StringEntity("");
}
InputStream content = entity.getContent();
addHashedPayload(content, result);
} catch (IOException e) {
throw new RuntimeException("Could not create hash for entity " + entity, e);
}
return result.toString();
}
@Test
public void GetRequest()
throws URISyntaxException, ClientProtocolException, IOException, ParserConfigurationException,
UnsupportedOperationException, SAXException {
HttpUriRequest request = new HttpGet(url);
URIBuilder uri = new URIBuilder(request.getURI()).addParameter("CountryName", country);
((HttpRequestBase) request).setURI(uri.build());
// Testing headers
HttpResponse response = client.execute(request);
Header[] headers = response.getAllHeaders();
for (Header head : headers) {
System.out.println(head.getName() + "---" + head.getValue());
}
System.out.println("================================================");
// Testing response
// parsing xml
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
Document doc = dBuilder.parse(response.getEntity().getContent());
// Printing root
doc.getDocumentElement().normalize();
NodeList nList = doc.getElementsByTagName("string");
Node aNode = nList.item(0);
Element element = (Element) aNode;
System.out.println(element.getFirstChild());
}
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
boolean retry = true;
Boolean b = (Boolean) context.getAttribute(ExecutionContext.HTTP_REQ_SENT);
boolean sent = (b != null && b.booleanValue());
if (executionCount > maxRetries) {
// 尝试次数超过用户定义的测试，默认5次
retry = false;
} else if (exceptionBlacklist.contains(exception.getClass())) {
// 线程被用户中断，则不继续尝试
retry = false;
} else if (exceptionWhitelist.contains(exception.getClass())) {
retry = true;
} else if (!sent) {
retry = true;
}
if (retry) {
HttpUriRequest currentReq =
(HttpUriRequest) context.getAttribute(ExecutionContext.HTTP_REQUEST);
retry = currentReq != null && !"POST".equals(currentReq.getMethod());
}
if (retry) {
// 休眠1秒钟后再继续尝试
SystemClock.sleep(RETRY_SLEEP_TIME_MILLIS);
} else {
exception.printStackTrace();
}
return retry;
}
public String getPageContent(String uri) {
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
HttpClient httpClient = httpClientBuilder.build();
HttpUriRequest httpReq = new HttpGet(uri);
HttpResponse httpResp = null;
HttpEntity httpEntity = null;
try {
httpResp = httpClient.execute(httpReq);
httpEntity = httpResp.getEntity();
if (httpEntity != null) {
InputStream is = httpEntity.getContent();
StringWriter writer = new StringWriter();
IOUtils.copy(is, writer, "UTF-8");
writer.close();
is.close();
return writer.toString();
}
} catch (IOException e) {
logger.error("获取网页内容时,发生异常,具体为：{}", e);
return StringUtils.EMPTY;
} finally {
httpReq.abort();
try {
EntityUtils.consume(httpEntity);
} catch (Exception e2) {
}
}
return StringUtils.EMPTY;
}
private void setHeaders(HttpUriRequest request) {
SignatureBuilder sb = new SignatureBuilder();
request.setHeader(
new BasicHeader("Date", DateUtils.formatDate(new Date()).replaceFirst("[+]00:00$", "")));
request.setHeader(
"Authorization", "VWS " + accessKey + ":" + sb.tmsSignature(request, secretKey));
}
/**
* Executes the HTTP request.
*
* <p>In case of any exception thrown by HttpClient, it will release the connection. In other
* cases it is the duty of caller to do it, or process the input stream.
*
* @param repository to execute the HTTP method for
* @param request resource store request that triggered the HTTP request
* @param httpRequest HTTP request to be executed
* @param baseUrl The BaseURL used to construct final httpRequest
* @return response of making the request
* @throws RemoteStorageException If an error occurred during execution of HTTP request
*/
@VisibleForTesting
HttpResponse executeRequest(
final ProxyRepository repository,
final ResourceStoreRequest request,
final HttpUriRequest httpRequest,
final String baseUrl,
final boolean contentRequest)
throws RemoteStorageException {
final Timer timer = timer(repository, httpRequest, baseUrl);
final TimerContext timerContext = timer.time();
Stopwatch stopwatch = null;
if (outboundRequestLog.isDebugEnabled()) {
stopwatch = new Stopwatch().start();
}
try {
return doExecuteRequest(repository, request, httpRequest, contentRequest);
} finally {
timerContext.stop();
if (stopwatch != null) {
outboundRequestLog.debug(
"[{}] {} {} - {}",
repository.getId(),
httpRequest.getMethod(),
httpRequest.getURI(),
stopwatch);
}
}
}
public PingInfo ping(String host, int restPort, boolean noProxy) throws Exception {
PingInfo myPingInfo = pingInfoProvider.createPingInfo();
RequestConfig.Builder requestConfigBuilder =
httpClientProvider.createRequestConfigBuilder(host, noProxy);
String url = String.format(URL_PATTERN, host, restPort) + PingHandler.URL;
HttpUriRequest request =
RequestBuilder.post(url)
.setConfig(requestConfigBuilder.build())
.addParameter(PingHandler.PING_INFO_INPUT_NAME, gson.toJson(myPingInfo))
.build();
CloseableHttpResponse response = httpClientProvider.executeRequest(request);
try {
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
EntityUtils.consumeQuietly(response.getEntity());
throw new Exception(statusLine.getStatusCode() + " " + statusLine.getReasonPhrase());
}
HttpEntity entity = response.getEntity();
String content = EntityUtils.toString(entity);
EntityUtils.consumeQuietly(entity);
PingInfo receivedPingInfo = gson.fromJson(content, PingInfo.class);
receivedPingInfo.getAgentId().setHost(request.getURI().getHost());
return receivedPingInfo;
} finally {
response.close();
}
}
public JestResult execute(Action clientRequest) throws IOException {
String elasticSearchRestUrl = getRequestURL(getElasticSearchServer(), clientRequest.getURI());
HttpUriRequest request =
constructHttpMethod(
clientRequest.getRestMethodName(), elasticSearchRestUrl, clientRequest.getData());
// add headers added to action
if (!clientRequest.getHeaders().isEmpty()) {
for (Entry<String, Object> header : clientRequest.getHeaders().entrySet()) {
request.addHeader(header.getKey(), header.getValue() + "");
}
}
HttpResponse response = httpClient.execute(request);
// If head method returns no content, it is added according to response code thanks to
// https://github.com/hlassiege
if (request.getMethod().equalsIgnoreCase("HEAD")) {
if (response.getEntity() == null) {
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
response.setEntity(new StringEntity("{\"ok\" : true, \"found\" : true}"));
} else if (response.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_FOUND) {
response.setEntity(new StringEntity("{\"ok\" : false, \"found\" : false}"));
}
}
}
return deserializeResponse(response, clientRequest.getName(), clientRequest.getPathToResult());
}
@Override
public Cursor query(
Uri uri, String[] projection, String selection, String[] selectionArgs, String sortOrder) {
try {
HttpUriRequest request = queryRequest(uri, projection, selection, selectionArgs, sortOrder);
if (DEBUG) Log.i(TAG, "will query: " + request.getURI());
Cursor cursor = getQueryHandler(uri).handleResponse(httpClient.execute(request));
// httpClient.getConnectionManager().shutdown();
return cursor;
} catch (ConnectException e) {
Log.w(TAG, "an error occured in query", e);
return ErrorCursor.getCursor(0, e.getMessage());
} catch (ClientProtocolException e) {
Log.w(TAG, "an error occured in query", e);
return ErrorCursor.getCursor(0, e.getMessage());
} catch (IOException e) {
Log.w(TAG, "an error occured in query", e);
return ErrorCursor.getCursor(0, e.getMessage());
} catch (IllegalArgumentException e) {
Log.w(TAG, "an error occured in query", e);
return ErrorCursor.getCursor(
0,
"Unknown URI (not in cache or not answerable by the implementator): " + uri.toString());
}
}
private HttpUriRequest newHcRequest(HttpRequest request) throws IOException {
URI uri = request.getUri().toJavaUri();
HttpUriRequest httpUriRequest = HttpMethod.valueOf(request.getMethod()).newMessage(uri);
for (Map.Entry<String, List<String>> entry : request.getHeaders().entrySet()) {
httpUriRequest.addHeader(entry.getKey(), StringUtils.join(entry.getValue(), ','));
}
return httpUriRequest;
}
private void setHeader(HttpUriRequest request) {
if (!tokenIsNull()) {
request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken.getToken());
}
request.setHeader(HttpHeaders.USER_AGENT, USER_AGENT);
request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip,deflate");
request.setHeader(HttpHeaders.ACCEPT_CHARSET, "utf-8;q=0.7,*;q=0.3");
request.setHeader(HttpHeaders.HOST, "www.yammer.com");
}
/**
* Perform a HTTP GET request and track the Android Context which initiated the request with
* customized headers
*
* @param url the URL to send the request to.
* @param headers set headers only for this request
* @param params additional GET parameters to send with the request.
* @param responseHandler the response handler instance that should handle the response.
*/
public void get(
Context context,
String url,
Header[] headers,
RequestParams params,
AsyncHttpResponseHandler responseHandler) {
HttpUriRequest request = new HttpGet(getUrlWithQueryString(url, params));
if (headers != null) request.setHeaders(headers);
sendRequest(httpClient, httpContext, request, null, responseHandler, context);
}
/**
* Perform a HTTP HEAD request and track the Android Context which initiated the request with
* customized headers
*
* @param context Context to execute request against
* @param url the URL to send the request to.
* @param headers set headers only for this request
* @param params additional HEAD parameters to send with the request.
* @param responseHandler the response handler instance that should handle the response.
*/
public RequestHandle head(
Context context,
String url,
Header[] headers,
RequestParams params,
AsyncHttpResponseHandler responseHandler) {
HttpUriRequest request = new HttpHead(getUrlWithQueryString(isUrlEncodingEnabled, url, params));
if (headers != null) request.setHeaders(headers);
return sendRequest(httpClient, httpContext, request, null, responseHandler, context);
}
private void completeRestMethodBuild(
HttpUriRequest method,
CougarMessageContentTypeEnum responseContentTypeEnum,
CougarMessageContentTypeEnum requestContentTypeEnum,
String postQuery,
Map<String, String> headerParams,
String authority,
Map<String, String> authCredentials,
String altUrl,
Map<String, String> acceptProtocols,
String ipAddress) {
String contentType = selectContent(requestContentTypeEnum);
if (!"".equals(contentType)) {
method.setHeader("Content-Type", contentType);
}
method.setHeader("User-Agent", "java/socket");
String accept = selectAccept(responseContentTypeEnum, acceptProtocols);
method.setHeader("Accept", accept);
// No need to set this header any more as it is set by the new http client version
/*if ((postQuery != null) && (!postQuery.equalsIgnoreCase(""))) {
Integer contentLength = postQuery.length();
method.setHeader("Content-Length", contentLength.toString());
}*/
if (headerParams != null) {
for (Map.Entry<String, String> entry : headerParams.entrySet()) {
String headerParamValue = entry.getValue();
String key = entry.getKey();
method.setHeader(key, headerParamValue);
// logger.LogBetfairDebugEntry("Rest request header param added: '"
// + key + ":" + headerParamValue + "'");
}
}
if (authority != null) {
method.setHeader("X-Authentication", authority);
}
if (authCredentials != null) {
if ("".equals(altUrl)) {
method.setHeader("X-Token-Username", authCredentials.get("Username"));
method.setHeader("X-Token-Password", authCredentials.get("Password"));
} else {
method.setHeader("X-AltToken-Username", authCredentials.get("Username"));
method.setHeader("X-AltToken-Password", authCredentials.get("Password"));
}
}
if (ipAddress == null) {
method.setHeader("X-Forwarded-For", null);
} else if (!ipAddress.trim().equalsIgnoreCase("DO NOT INCLUDE")) {
method.setHeader("X-Forwarded-For", ipAddress);
}
// logger.LogBetfairDebugEntry("Rest request postString: '"
// + postQuery + "'");
}
public void delete(
Context context,
String str,
Header[] headerArr,
AsyncHttpResponseHandler asyncHttpResponseHandler) {
HttpUriRequest httpDelete = new HttpDelete(str);
if (headerArr != null) {
httpDelete.setHeaders(headerArr);
}
sendRequest(
this.httpClient, this.httpContext, httpDelete, null, asyncHttpResponseHandler, context);
}
/**
* Authenticate with the chosen bridge
*
* @param ip, the ip address of the bridge you want to authenticate with
* @return the username of the hue bridge, that has been placed on the white list
*/
private String authWithBridge(String ip)
throws URISyntaxException, MalformedURLException, UnsupportedEncodingException {
URI uri = new URL("http://" + ip + "/api").toURI();
DefaultHttpClient client = new DefaultHttpClient();
HttpUriRequest request = new HttpPost(uri);
JSONObject obj = new JSONObject();
try {
obj.put("devicetype", "OpenRemoteController");
} catch (JSONException e) {
logger.error("JSONExceptionn when creating json object", e);
}
StringEntity data = new StringEntity(obj.toString(), "UTF-8");
((HttpPost) request).setEntity(data);
String resp = "";
HttpResponse response = null;
ResponseHandler<String> responseHandler = new BasicResponseHandler();
request.addHeader("User-Agent", "OpenRemoteController");
request.getParams().setBooleanParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false);
request.addHeader("Content-Type", "applicaion/json");
try {
response = client.execute(request);
resp = responseHandler.handleResponse(response);
} catch (ClientProtocolException e) {
logger.error("ClientProtocolException when executing HTTP method", e);
} catch (IOException e) {
logger.error("IOException when executing HTTP method", e);
} finally {
try {
if ((response != null) && (response.getEntity() != null)) {
response.getEntity().consumeContent();
}
} catch (IOException ignored) {
}
client.getConnectionManager().shutdown();
}
JSONObject jsonResponse = null;
String jsonUsername = "";
try {
jsonResponse = new JSONArray(resp.toString()).getJSONObject(0).getJSONObject("success");
jsonUsername = jsonResponse.getString("username");
} catch (JSONException e) {
logger.error("JSONException when reading returned json", e);
}
return jsonUsername;
}
private void runLoop() throws InterruptedException {
while (!stop) {
CrawlerTask task = crawlerResponseQueue.take();
if (task.isExitTask()) {
crawlerRequestQueue.add(CrawlerTask.createExitTask());
crawlerResponseQueue.add(task);
return;
}
HttpUriRequest req = task.getRequest();
activeRequest = req;
try {
if (task.getResponse() != null) {
task.getResponseProcessor()
.processResponse(crawler, req, task.getResponse(), task.getArgument());
}
} catch (Exception e) {
logger.log(
Level.WARNING, "Unexpected exception processing crawler request: " + req.getURI(), e);
} finally {
synchronized (requestLock) {
activeRequest = null;
}
final HttpEntity entity =
(task.getResponse() == null) ? (null) : task.getResponse().getRawResponse().getEntity();
if (entity != null)
try {
EntityUtils.consume(entity);
} catch (IOException e) {
logger.log(
Level.WARNING,
"I/O exception consuming request entity content for "
+ req.getURI()
+ " : "
+ e.getMessage());
}
}
synchronized (counter) {
counter.addCompletedTask();
crawler.updateProgress();
}
if (task.causedException()) {
crawler.notifyException(req, task.getException());
}
if (outstandingTasks.decrementAndGet() <= 0) {
crawlerRequestQueue.add(CrawlerTask.createExitTask());
crawlerResponseQueue.add(CrawlerTask.createExitTask());
return;
}
}
}
@Override
public HttpResponse performRequest(Request<?> request, Map<String, String> additionalHeaders)
throws IOException, AuthFailureError {
HttpUriRequest httpRequest = createHttpRequest(request, additionalHeaders);
addHeaders(httpRequest, additionalHeaders);
addHeaders(httpRequest, request.getHeaders());
onPrepareRequest(httpRequest);
HttpParams httpParams = httpRequest.getParams();
int timeoutMs = request.getTimeoutMs();
HttpConnectionParams.setConnectionTimeout(httpParams, 5000);
HttpConnectionParams.setSoTimeout(httpParams, timeoutMs);
return mClient.execute(httpRequest);
}
public void get(
Context context,
String str,
Header[] headerArr,
RequestParams requestParams,
AsyncHttpResponseHandler asyncHttpResponseHandler) {
HttpUriRequest httpGet = new HttpGet(getUrlWithQueryString(str, requestParams));
if (headerArr != null) {
httpGet.setHeaders(headerArr);
}
sendRequest(
this.httpClient, this.httpContext, httpGet, null, asyncHttpResponseHandler, context);
}
/**
* This is the PUT method that receives the response body.
*
* @param url Target Request URL
* @param map Hash map of Request Header
* @return DcResponse object
* @throws DaoException Exception thrown
*/
public DcResponse put(String url, HashMap<String, String> map) throws DaoException {
HttpUriRequest req =
new DcRequestBuilder()
.url(url)
.method(HttpMethods.PUT)
.token(getToken())
.defaultHeaders(this.accessor.getDefaultHeaders())
.build();
for (Map.Entry<String, String> entry : map.entrySet()) {
req.setHeader((String) entry.getKey(), (String) entry.getValue());
}
return this.request(req);
}