/**
   * 构造请求的方法,如post,get,header等<br>
   * 设置请求参数,如超时时间
   *
   * @param url 请求的URL
   * @param method 请求的方法
   * @param postdata post的数据
   * @param headers 请求头
   * @return
   */
  protected HttpUriRequest getHttpUriRequest(
      String url, String method, Map<String, String> postdata, Map<String, String> headers) {
    RequestBuilder requestBuilder = selectRequestMethod(method, postdata, headers).setUri(url);

    requestBuilder.addHeader("Accept", "*/*");
    requestBuilder.addHeader("Connection", "keep-alive");
    if (headers != null) {
      for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
        requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
      }
    }

    int timeout = 45000; // 超时时间
    RequestConfig.Builder requestConfigBuilder =
        RequestConfig.custom()
            .setConnectionRequestTimeout(timeout)
            .setSocketTimeout(timeout)
            .setConnectTimeout(timeout)
            // .setRedirectsEnabled(false)//禁止自动跳转
            .setCookieSpec(CookieSpecs.BEST_MATCH);

    if (null != host && !"".equals(host) && port > 10) {
      HttpHost proxy = new HttpHost(host, port);
      requestConfigBuilder.setProxy(proxy);
    }

    requestBuilder.setConfig(requestConfigBuilder.build());
    return requestBuilder.build();
  }
Example #2
0
  /**
   * Get方法传送消息
   *
   * @param url 连接的URL
   * @param queryString 请求参数串
   * @return 服务器返回的信息
   * @throws Exception
   */
  public String httpGet(String url, String queryString) throws Exception {

    String responseData = null;
    if (queryString != null && !queryString.equals("")) {
      url += "?" + queryString;
    }
    RequestBuilder requestBuilder = RequestBuilder.get().setUri(url);
    log.info("QHttpClient httpGet [1] url = " + url);
    RequestConfig requestConfig =
        RequestConfig.custom()
            .setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
            .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
            .build();
    requestBuilder.setConfig(requestConfig);
    HttpResponse response;
    HttpGet httpGet = (HttpGet) requestBuilder.build();
    response = httpClient.execute(httpGet);
    try {
      log.info("QHttpClient httpGet [2] StatusLine : " + response.getStatusLine());
      responseData = EntityUtils.toString(response.getEntity());
      log.info("QHttpClient httpGet [3] Response = " + responseData);
    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      httpGet.abort();
    }
    return responseData;
  }
Example #3
0
 /**
  * Called primarily from HTTPMethod to do the bulk of the execution. Assumes HTTPMethod has
  * inserted its headers into request.
  *
  * @param method
  * @param methoduri
  * @param rb
  * @return Request+Response pair
  * @throws HTTPException
  */
 ExecState execute(HTTPMethod method, URI methoduri, RequestBuilder rb) throws HTTPException {
   this.execution = new ExecState();
   this.requestURI = methoduri;
   AuthScope methodscope = HTTPAuthUtil.uriToAuthScope(methoduri);
   AuthScope target = HTTPAuthUtil.authscopeUpgrade(this.scope, methodscope);
   synchronized (this) { // keep coverity happy
     // Merge Settings;
     Settings merged = HTTPUtil.merge(globalsettings, localsettings);
     if (!this.cachevalid) {
       RequestConfig.Builder rcb = RequestConfig.custom();
       this.cachedconfig = configureRequest(rcb, merged);
       HttpClientBuilder cb = HttpClients.custom();
       configClient(cb, merged);
       setAuthenticationAndProxy(cb);
       this.cachedclient = cb.build();
       rb.setConfig(this.cachedconfig);
       this.cachevalid = true;
     }
   }
   this.execution.request = (HttpRequestBase) rb.build();
   try {
     HttpHost targethost = HTTPAuthUtil.authscopeToHost(target);
     this.execution.response =
         cachedclient.execute(targethost, this.execution.request, this.sessioncontext);
   } catch (IOException ioe) {
     throw new HTTPException(ioe);
   }
   return this.execution;
 }
  /**
   * Build a new Apache's {@link org.apache.http.client.methods.HttpUriRequest} from Jersey's {@link
   * org.glassfish.jersey.client.ClientRequest}
   *
   * <p>Convert a method, URI, body, headers and override a user-agent if necessary
   *
   * @param jerseyRequest representation of an HTTP request in Jersey
   * @return a new {@link org.apache.http.client.methods.HttpUriRequest}
   */
  private HttpUriRequest buildApacheRequest(ClientRequest jerseyRequest) {
    final RequestBuilder builder =
        RequestBuilder.create(jerseyRequest.getMethod())
            .setUri(jerseyRequest.getUri())
            .setEntity(getHttpEntity(jerseyRequest));
    for (String headerName : jerseyRequest.getHeaders().keySet()) {
      builder.addHeader(headerName, jerseyRequest.getHeaderString(headerName));
    }

    final Optional<RequestConfig> requestConfig = addJerseyRequestConfig(jerseyRequest);
    requestConfig.ifPresent(builder::setConfig);

    return builder.build();
  }
  private void assertBuild(final Charset charset) throws Exception {
    final RequestBuilder requestBuilder = RequestBuilder.create("GET").setCharset(charset);
    requestBuilder.setUri("https://somehost.com/stuff");
    requestBuilder.addParameters(createParameters());

    final String encodedData1 = URLEncoder.encode("\"1\u00aa position\"", charset.displayName());
    final String encodedData2 = URLEncoder.encode("Jos\u00e9 Abra\u00e3o", charset.displayName());

    final String uriExpected =
        String.format(
            "https://somehost.com/stuff?parameter1=value1&parameter2=%s&parameter3=%s",
            encodedData1, encodedData2);

    final HttpUriRequest request = requestBuilder.build();
    Assert.assertEquals(uriExpected, request.getURI().toString());
  }
  private <B, R> R executeRequest(
      RequestSpec requestSpec,
      PathParams pathParams,
      B requestBody,
      Class<R> responseType,
      int expectedStatus) {
    String url =
        String.format(
            ADMIN_URL_PREFIX + requestSpec.path(pathParams), scheme, host, port, urlPathPrefix);
    RequestBuilder requestBuilder =
        RequestBuilder.create(requestSpec.method().getName()).setUri(url);

    if (requestBody != null) {
      requestBuilder.setEntity(jsonStringEntity(Json.write(requestBody)));
    }

    String responseBodyString = safelyExecuteRequest(url, expectedStatus, requestBuilder.build());

    return responseType == Void.class ? null : Json.read(responseBodyString, responseType);
  }
Example #7
0
 /**
  * Post方法传送消息
  *
  * @param url 连接的URL
  * @param queryString 请求参数串
  * @return 服务器返回的信息
  * @throws Exception
  */
 public String httpPost(String url, String queryString) throws Exception {
   String responseData = null;
   URI tmpUri = new URI(url);
   URI uri =
       new URI(
           tmpUri.getScheme(),
           null,
           tmpUri.getHost(),
           tmpUri.getPort(),
           tmpUri.getPath(),
           queryString,
           null);
   log.info("QHttpClient httpPost [1] url = " + uri.toURL());
   RequestBuilder requestBuilder = RequestBuilder.post().setUri(uri);
   RequestConfig requestConfig =
       RequestConfig.custom()
           .setSocketTimeout(new Integer(CONNECTION_TIMEOUT))
           .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
           .build();
   requestBuilder.setConfig(requestConfig);
   HttpPost httpPost = (HttpPost) requestBuilder.build();
   if (queryString != null && !queryString.equals("")) {
     StringEntity reqEntity = new StringEntity(queryString);
     // 设置类型
     reqEntity.setContentType("application/x-www-form-urlencoded");
     // 设置请求的数据
     httpPost.setEntity(reqEntity);
   }
   try {
     HttpResponse response = httpClient.execute(httpPost);
     log.info("QHttpClient httpPost [2] StatusLine = " + response.getStatusLine());
     responseData = EntityUtils.toString(response.getEntity());
     log.info("QHttpClient httpPost [3] responseData = " + responseData);
   } catch (Exception e) {
     e.printStackTrace();
   } finally {
     httpPost.abort();
   }
   return responseData;
 }
 @Override
 public Page download(Request request, Task task) {
   Site site = null;
   if (task != null) {
     site = task.getSite();
   }
   Set<Integer> acceptStatCode;
   String charset = null;
   Map<String, String> headers = null;
   if (site != null) {
     acceptStatCode = site.getAcceptStatCode();
     charset = site.getCharset();
     headers = site.getHeaders();
   } else {
     acceptStatCode = Sets.newHashSet(200);
   }
   logger.info("downloading page " + request.getUrl());
   RequestBuilder requestBuilder = RequestBuilder.get().setUri(request.getUrl());
   if (headers != null) {
     for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
       requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
     }
   }
   RequestConfig.Builder requestConfigBuilder =
       RequestConfig.custom()
           .setConnectionRequestTimeout(site.getTimeOut())
           .setSocketTimeout(site.getTimeOut())
           .setConnectTimeout(site.getTimeOut())
           .setCookieSpec(CookieSpecs.BEST_MATCH);
   if (site != null && site.getHttpProxy() != null) {
     requestConfigBuilder.setProxy(site.getHttpProxy());
   }
   requestBuilder.setConfig(requestConfigBuilder.build());
   CloseableHttpResponse httpResponse = null;
   try {
     httpResponse = getHttpClient(site).execute(requestBuilder.build());
     int statusCode = httpResponse.getStatusLine().getStatusCode();
     if (acceptStatCode.contains(statusCode)) {
       // charset
       if (charset == null) {
         String value = httpResponse.getEntity().getContentType().getValue();
         charset = UrlUtils.getCharset(value);
       }
       return handleResponse(request, charset, httpResponse, task);
     } else {
       logger.warn("code error " + statusCode + "\t" + request.getUrl());
       return null;
     }
   } catch (IOException e) {
     logger.warn("download page " + request.getUrl() + " error", e);
     if (site.getCycleRetryTimes() > 0) {
       return addToCycleRetry(request, site);
     }
     return null;
   } finally {
     try {
       if (httpResponse != null) {
         // ensure the connection is released back to pool
         EntityUtils.consume(httpResponse.getEntity());
       }
     } catch (IOException e) {
       logger.warn("close response fail", e);
     }
   }
 }
  /** 传入加入参数post 参数的url */
  public CrawlResultPojo crawl4Post(UrlPojo urlPojo) {
    if (urlPojo == null) {
      return null;
    }
    CrawlResultPojo crawlResultPojo = new CrawlResultPojo();
    CloseableHttpResponse response1 = null;
    BufferedReader br = null;
    try {

      RequestBuilder rb = RequestBuilder.post().setUri(new URI(urlPojo.getUrl()));
      /* HttpUriRequest urlRequest = RequestBuilder.post()
      .setUri(new URI(urlPojo.getUrl()))
      .addParameter("IDToken1", "username")
      .addParameter("IDToken2", "password")
      .build();*/
      Map<String, Object> paramsMap = urlPojo.getParamsMap();
      if (paramsMap != null) {
        for (Entry<String, Object> entry : paramsMap.entrySet()) {
          rb.addParameter(entry.getKey(), entry.getValue().toString());
        }
      }
      HttpUriRequest httpRequst = rb.build();
      response1 = httpclient.execute(httpRequst);

      HttpEntity entity = response1.getEntity();
      InputStreamReader isr = new InputStreamReader(entity.getContent(), "utf-8");
      br = new BufferedReader(isr);

      String line = null;
      StringBuilder stringBuilder = new StringBuilder();
      while ((line = br.readLine()) != null) {
        stringBuilder.append(line + "\n");
      }
      crawlResultPojo.setSuccess(true);
      crawlResultPojo.setPageContent(stringBuilder.toString());
      return crawlResultPojo;

    } catch (Exception e) {
      e.printStackTrace();
      crawlResultPojo.setSuccess(false);
    } finally {
      if (response1 != null) {
        try {
          response1.close();
        } catch (IOException e1) {
          // TODO Auto-generated catch block
          e1.printStackTrace();
        }
      }
      if (br != null) {
        try {
          br.close();
        } catch (IOException e1) {
          // TODO Auto-generated catch block
          e1.printStackTrace();
        }
      }
    }

    return crawlResultPojo;
  }