/**
  * @param maxTotal maxTotal
  * @param maxPerRoute maxPerRoute
  * @param timeout timeout
  * @param retryExecutionCount retryExecutionCount
  * @return
  */
 public static CloseableHttpClient createHttpClient(
     int maxTotal, int maxPerRoute, int timeout, int retryExecutionCount) {
   try {
     SSLContext sslContext = SSLContexts.custom().useSSL().build();
     SSLConnectionSocketFactory sf =
         new SSLConnectionSocketFactory(
             sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
     PoolingHttpClientConnectionManager poolingHttpClientConnectionManager =
         new PoolingHttpClientConnectionManager();
     poolingHttpClientConnectionManager.setMaxTotal(maxTotal);
     poolingHttpClientConnectionManager.setDefaultMaxPerRoute(maxPerRoute);
     SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(timeout).build();
     poolingHttpClientConnectionManager.setDefaultSocketConfig(socketConfig);
     return HttpClientBuilder.create()
         .setConnectionManager(poolingHttpClientConnectionManager)
         .setSSLSocketFactory(sf)
         .setRetryHandler(new HttpRequestRetryHandlerImpl(retryExecutionCount))
         .build();
   } catch (KeyManagementException e) {
     e.printStackTrace();
   } catch (NoSuchAlgorithmException e) {
     e.printStackTrace();
   }
   return null;
 }
 /**
  * Key store 类型HttpClient
  *
  * @param keystore keystore
  * @param keyPassword keyPassword
  * @param supportedProtocols supportedProtocols
  * @param timeout timeout
  * @param retryExecutionCount retryExecutionCount
  * @return CloseableHttpClient
  */
 public static CloseableHttpClient createKeyMaterialHttpClient(
     KeyStore keystore,
     String keyPassword,
     String[] supportedProtocols,
     int timeout,
     int retryExecutionCount) {
   try {
     SSLContext sslContext =
         SSLContexts.custom()
             .useSSL()
             .loadKeyMaterial(keystore, keyPassword.toCharArray())
             .build();
     SSLConnectionSocketFactory sf =
         new SSLConnectionSocketFactory(
             sslContext,
             supportedProtocols,
             null,
             SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
     SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(timeout).build();
     return HttpClientBuilder.create()
         .setDefaultSocketConfig(socketConfig)
         .setSSLSocketFactory(sf)
         .setRetryHandler(new HttpRequestRetryHandlerImpl(retryExecutionCount))
         .build();
   } catch (KeyManagementException e) {
     e.printStackTrace();
   } catch (NoSuchAlgorithmException e) {
     e.printStackTrace();
   } catch (UnrecoverableKeyException e) {
     e.printStackTrace();
   } catch (KeyStoreException e) {
     e.printStackTrace();
   }
   return null;
 }
示例#3
0
  private CloseableHttpClient generateClient(Site site) {
    HttpClientBuilder httpClientBuilder =
        HttpClients.custom().setConnectionManager(connectionManager);
    if (site != null && site.getUserAgent() != null) {
      httpClientBuilder.setUserAgent(site.getUserAgent());
    } else {
      httpClientBuilder.setUserAgent("");
    }
    if (site == null || site.isUseGzip()) {
      httpClientBuilder.addInterceptorFirst(
          new HttpRequestInterceptor() {

            public void process(final HttpRequest request, final HttpContext context)
                throws HttpException, IOException {
              if (!request.containsHeader("Accept-Encoding")) {
                request.addHeader("Accept-Encoding", "gzip");
              }
            }
          });
    }
    SocketConfig socketConfig =
        SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
    httpClientBuilder.setDefaultSocketConfig(socketConfig);
    if (site != null) {
      httpClientBuilder.setRetryHandler(
          new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
    }
    generateCookie(httpClientBuilder, site);
    return httpClientBuilder.build();
  }
示例#4
0
  HttpClient() {
    try {
      PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
      SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();
      connManager.setDefaultSocketConfig(socketConfig);
      MessageConstraints messageConstraints =
          MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build();
      ConnectionConfig connectionConfig =
          ConnectionConfig.custom()
              .setMalformedInputAction(CodingErrorAction.IGNORE)
              .setUnmappableInputAction(CodingErrorAction.IGNORE)
              .setCharset(Consts.UTF_8)
              .setMessageConstraints(messageConstraints)
              .build();
      connManager.setDefaultConnectionConfig(connectionConfig);
      connManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
      connManager.setDefaultMaxPerRoute(MAX_ROUTE_CONNECTIONS);
      connManager.setMaxPerRoute(new HttpRoute(DEFAULT_TARGETHOST), 50);

      HttpRequestRetryHandler retryHandler =
          new HttpRequestRetryHandler() {
            @Override
            public boolean retryRequest(
                IOException exception, int executionCount, HttpContext context) {
              if (executionCount >= 5) return false;
              if (exception instanceof InterruptedIOException) return false;
              if (exception instanceof UnknownHostException) return false;
              if (exception instanceof ConnectTimeoutException) return false;
              if (exception instanceof SSLException) return false;
              HttpClientContext clientContext = HttpClientContext.adapt(context);
              HttpRequest request = clientContext.getRequest();
              boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
              if (idempotent) return true;
              return false;
            }
          };

      httpClient =
          HttpClients.custom()
              .setConnectionManager(connManager)
              .setRetryHandler(retryHandler)
              .build();
    } catch (Exception e) {
      logger.error("HttpClientUtil error : [{}].", e.getMessage(), e);
    }
  }
示例#5
0
  /**
   * 个性化配置连接管理器
   *
   * @param maxConnectionsPerHost 设置默认的连接到每个主机的最大连接数
   * @param maxTotalConnections 设置整个管理连接器的最大连接数
   * @param conTimeOutMs 连接超时
   * @param soTimeOutMs socket超时
   * @param routeCfgList 特殊路由配置列表,若无请填null
   * @param proxy 代理设置,若无请填null
   */
  public QHttpClient(
      int maxConnectionsPerHost,
      int maxTotalConnections,
      int conTimeOutMs,
      int soTimeOutMs,
      List<RouteCfg> routeCfgList,
      HttpHost proxy) {

    // 使用默认的 socket factories 注册 "http" & "https" protocol scheme
    Registry<ConnectionSocketFactory> reg =
        RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.INSTANCE)
            .register("https", SSLConnectionSocketFactory.getSocketFactory())
            .build();
    PoolingHttpClientConnectionManager connectionManager =
        new PoolingHttpClientConnectionManager(reg);

    SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(soTimeOutMs).build();
    connectionManager.setDefaultMaxPerRoute(maxConnectionsPerHost);
    connectionManager.setMaxTotal(maxTotalConnections);
    connectionManager.setDefaultSocketConfig(socketConfig);
    // 对特定路由修改最大连接数
    if (null != routeCfgList) {
      for (RouteCfg routeCfg : routeCfgList) {
        HttpHost localhost = new HttpHost(routeCfg.getHost(), routeCfg.getPort());
        connectionManager.setMaxPerRoute(new HttpRoute(localhost), routeCfg.getMaxConnetions());
      }
    }

    // 设置代理
    DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
    httpClient =
        HttpClients.custom()
            .setConnectionManager(connectionManager)
            .setRoutePlanner(routePlanner)
            .build();
  }
示例#6
0
  public static final void main(String[] args) throws Exception {

    // Use custom message parser / writer to customize the way HTTP
    // messages are parsed from and written out to the data stream.
    HttpMessageParserFactory<HttpResponse> responseParserFactory =
        new DefaultHttpResponseParserFactory() {

          @Override
          public HttpMessageParser<HttpResponse> create(
              SessionInputBuffer buffer, MessageConstraints constraints) {
            LineParser lineParser =
                new BasicLineParser() {

                  @Override
                  public Header parseHeader(final CharArrayBuffer buffer) {
                    try {
                      return super.parseHeader(buffer);
                    } catch (ParseException ex) {
                      return new BasicHeader(buffer.toString(), null);
                    }
                  }
                };
            return new DefaultHttpResponseParser(
                buffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints) {

              @Override
              protected boolean reject(final CharArrayBuffer line, int count) {
                // try to ignore all garbage preceding a status line infinitely
                return false;
              }
            };
          }
        };
    HttpMessageWriterFactory<HttpRequest> requestWriterFactory =
        new DefaultHttpRequestWriterFactory();

    // Use a custom connection factory to customize the process of
    // initialization of outgoing HTTP connections. Beside standard connection
    // configuration parameters HTTP connection factory can define message
    // parser / writer routines to be employed by individual connections.
    HttpConnectionFactory<HttpRoute, ManagedHttpClientConnection> connFactory =
        new ManagedHttpClientConnectionFactory(requestWriterFactory, responseParserFactory);

    // Client HTTP connection objects when fully initialized can be bound to
    // an arbitrary network socket. The process of network socket initialization,
    // its connection to a remote address and binding to a local one is controlled
    // by a connection socket factory.

    // SSL context for secure connections can be created either based on
    // system or application specific properties.
    SSLContext sslcontext = SSLContexts.createSystemDefault();

    // Create a registry of custom connection socket factories for supported
    // protocol schemes.
    Registry<ConnectionSocketFactory> socketFactoryRegistry =
        RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.INSTANCE)
            .register("https", new SSLConnectionSocketFactory(sslcontext))
            .build();

    // Use custom DNS resolver to override the system DNS resolution.
    DnsResolver dnsResolver =
        new SystemDefaultDnsResolver() {

          @Override
          public InetAddress[] resolve(final String host) throws UnknownHostException {
            if (host.equalsIgnoreCase("myhost")) {
              return new InetAddress[] {InetAddress.getByAddress(new byte[] {127, 0, 0, 1})};
            } else {
              return super.resolve(host);
            }
          }
        };

    // Create a connection manager with custom configuration.
    PoolingHttpClientConnectionManager connManager =
        new PoolingHttpClientConnectionManager(socketFactoryRegistry, connFactory, dnsResolver);

    // Create socket configuration
    SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();
    // Configure the connection manager to use socket configuration either
    // by default or for a specific host.
    connManager.setDefaultSocketConfig(socketConfig);
    connManager.setSocketConfig(new HttpHost("somehost", 80), socketConfig);
    // Validate connections after 1 sec of inactivity
    connManager.setValidateAfterInactivity(1000);

    // Create message constraints
    MessageConstraints messageConstraints =
        MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build();
    // Create connection configuration
    ConnectionConfig connectionConfig =
        ConnectionConfig.custom()
            .setMalformedInputAction(CodingErrorAction.IGNORE)
            .setUnmappableInputAction(CodingErrorAction.IGNORE)
            .setCharset(Consts.UTF_8)
            .setMessageConstraints(messageConstraints)
            .build();
    // Configure the connection manager to use connection configuration either
    // by default or for a specific host.
    connManager.setDefaultConnectionConfig(connectionConfig);
    connManager.setConnectionConfig(new HttpHost("somehost", 80), ConnectionConfig.DEFAULT);

    // Configure total max or per route limits for persistent connections
    // that can be kept in the pool or leased by the connection manager.
    connManager.setMaxTotal(100);
    connManager.setDefaultMaxPerRoute(10);
    connManager.setMaxPerRoute(new HttpRoute(new HttpHost("somehost", 80)), 20);

    // Use custom cookie store if necessary.
    CookieStore cookieStore = new BasicCookieStore();
    // Use custom credentials provider if necessary.
    CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    // Create global request configuration
    RequestConfig defaultRequestConfig =
        RequestConfig.custom()
            .setCookieSpec(CookieSpecs.DEFAULT)
            .setExpectContinueEnabled(true)
            .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
            .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
            .build();

    // Create an HttpClient with the given custom dependencies and configuration.
    CloseableHttpClient httpclient =
        HttpClients.custom()
            .setConnectionManager(connManager)
            .setDefaultCookieStore(cookieStore)
            .setDefaultCredentialsProvider(credentialsProvider)
            .setProxy(new HttpHost("myproxy", 8080))
            .setDefaultRequestConfig(defaultRequestConfig)
            .build();

    try {
      HttpGet httpget = new HttpGet("http://httpbin.org/get");
      // Request configuration can be overridden at the request level.
      // They will take precedence over the one set at the client level.
      RequestConfig requestConfig =
          RequestConfig.copy(defaultRequestConfig)
              .setSocketTimeout(5000)
              .setConnectTimeout(5000)
              .setConnectionRequestTimeout(5000)
              .setProxy(new HttpHost("myotherproxy", 8080))
              .build();
      httpget.setConfig(requestConfig);

      // Execution context can be customized locally.
      HttpClientContext context = HttpClientContext.create();
      // Contextual attributes set the local context level will take
      // precedence over those set at the client level.
      context.setCookieStore(cookieStore);
      context.setCredentialsProvider(credentialsProvider);

      System.out.println("executing request " + httpget.getURI());
      CloseableHttpResponse response = httpclient.execute(httpget, context);
      try {
        System.out.println("----------------------------------------");
        System.out.println(response.getStatusLine());
        System.out.println(EntityUtils.toString(response.getEntity()));
        System.out.println("----------------------------------------");

        // Once the request has been executed the local context can
        // be used to examine updated state and various objects affected
        // by the request execution.

        // Last executed request
        context.getRequest();
        // Execution route
        context.getHttpRoute();
        // Target auth state
        context.getTargetAuthState();
        // Proxy auth state
        context.getTargetAuthState();
        // Cookie origin
        context.getCookieOrigin();
        // Cookie spec used
        context.getCookieSpec();
        // User security token
        context.getUserToken();

      } finally {
        response.close();
      }
    } finally {
      httpclient.close();
    }
  }
  public EasyHttpClient() {

    CookieSpecProvider easySpecProvider =
        new CookieSpecProvider() {
          public CookieSpec create(HttpContext context) {

            return new BrowserCompatSpec() {
              public void validate(Cookie cookie, CookieOrigin origin)
                  throws MalformedCookieException {}
            };
          }
        };

    SSLContext sslContext = SSLContexts.createDefault(); // 忽略证书主机名验证
    SSLConnectionSocketFactory sslsf =
        new SSLConnectionSocketFactory(
            sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);

    Registry<ConnectionSocketFactory> reg =
        RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.INSTANCE)
            .register("https", sslsf)
            .build();
    connectionManager = new PoolingHttpClientConnectionManager(reg);
    connectionManager.setDefaultMaxPerRoute(100); // 同一个路由允许最大连接数
    // connectionManager.setMaxTotal(poolSize);
    HttpHost localhost = new HttpHost("locahost", 80); // 如果是多网卡,这里选择出口IP?
    connectionManager.setMaxPerRoute(new HttpRoute(localhost), 50);

    Registry<CookieSpecProvider> r =
        RegistryBuilder.<CookieSpecProvider>create()
            .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
            .register(CookieSpecs.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory())
            .register("easy", easySpecProvider)
            .build();

    RequestConfig requestConfig =
        RequestConfig.custom()
            .setCookieSpec("easy")
            .setSocketTimeout(10000)
            .setConnectTimeout(10000)
            .build();
    ConnectionConfig connectioncfg = ConnectionConfig.custom().setCharset(Charsets.UTF_8).build();
    SocketConfig socketConfig =
        SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();

    HttpRequestRetryHandler myRetryHandler =
        new HttpRequestRetryHandler() {
          public boolean retryRequest(
              IOException exception, int executionCount, HttpContext context) {
            if (executionCount >= 3) {
              return false;
            }
            if (exception instanceof SSLHandshakeException) {
              return false;
            } else {
              return true;
            }
          }
        };

    HttpClientBuilder builder =
        HttpClients.custom() // 默认
            .setConnectionManager(connectionManager) // 链接管理器
            .setDefaultSocketConfig(socketConfig) // socket管理器
            .setRetryHandler(myRetryHandler) // 重试3次
            .setDefaultConnectionConfig(connectioncfg) // 链接配置,如默认字符编码
            .setDefaultCookieSpecRegistry(r) // cookie策略
            .setUserAgent(Chorme_User_Agent) // 浏览器请求头
            .setDefaultRequestConfig(requestConfig) // 链接配置,超时等
            .setDefaultCookieStore(cookieStore); // cookie

    builder.addInterceptorFirst(
        new HttpRequestInterceptor() {

          public void process(final HttpRequest request, final HttpContext context)
              throws HttpException, IOException {
            if (!request.containsHeader("Accept-Encoding")) {
              request.addHeader("Accept-Encoding", "gzip");
            }
          }
        });

    httpclient = builder.build();
  }