Exemplo n.º 1
0
  @Override
  public void run() {
    HttpClient httpClient = HttpClients.createDefault();
    HttpGet httpGet = new HttpGet("http://localhost:8080/crawler/main");
    httpGet.setConfig(RequestConfig.custom().setConnectTimeout(10000).build());
    httpGet.setConfig(RequestConfig.custom().setSocketTimeout(5000).build());

    int statusCode = 0;
    try {
      HttpResponse response = httpClient.execute(httpGet);
      statusCode = response.getStatusLine().getStatusCode();
      while (statusCode != HttpStatus.SC_OK) {
        Thread.sleep(1000);
        response = httpClient.execute(httpGet);
        statusCode = response.getStatusLine().getStatusCode();
      }
    } catch (ClientProtocolException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    } catch (InterruptedException e) {
      e.printStackTrace();
    }

    ExtLogger.info("multi-thread start");
  }
  public void testMojoExecution() throws Exception {
    assertNotNull(mojo);
    mojo.execute();

    HttpClient client = HttpClientBuilder.create().build();

    HttpGet get = new HttpGet("http://localhost:" + elasticsearchNode.getHttpPort());

    final int connectionTimeout = 500; // millis
    RequestConfig requestConfig =
        RequestConfig.custom()
            .setConnectionRequestTimeout(connectionTimeout)
            .setConnectTimeout(connectionTimeout)
            .setSocketTimeout(connectionTimeout)
            .build();
    get.setConfig(requestConfig);

    try {
      client.execute(get);

      fail("The ES cluster should have been down by now");
    } catch (HttpHostConnectException | ConnectTimeoutException expected) {
    }

    assertTrue(elasticsearchNode.isClosed());
  }
  public static String get(String url, List<NameValuePair> params) {
    String body = null;
    try {
      HttpClient hc = new DefaultHttpClient();
      // Get请求
      HttpGet httpget = new HttpGet(url);
      // 设置参数
      String str = EntityUtils.toString(new UrlEncodedFormEntity(params));
      httpget.setURI(new URI(httpget.getURI().toString() + "?" + str));
      // 设置请求和传输超时时间
      RequestConfig requestConfig =
          RequestConfig.custom().setSocketTimeout(4000).setConnectTimeout(4000).build();
      httpget.setConfig(requestConfig);
      // 发送请求
      HttpResponse httpresponse = hc.execute(httpget);
      // 获取返回数据
      HttpEntity entity = httpresponse.getEntity();
      body = EntityUtils.toString(entity);

      if (entity != null) {
        entity.consumeContent();
      }
    } catch (Exception e) {
      logger.error("http的get方法" + e.toString());
      throw new RuntimeException(e);
    }
    return body;
  }
Exemplo n.º 4
0
  @Override
  public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Exception {

    LOG.debug("HTTP connection manager stats {}", CONNECTION_MANAGER.getTotalStats());

    HttpGet httpget = new HttpGet(url);
    httpget.setConfig(requestConfig);

    if (md != null) {
      String ifModifiedSince = md.getFirstValue("cachedLastModified");
      if (StringUtils.isNotBlank(ifModifiedSince)) {
        httpget.addHeader("If-Modified-Since", ifModifiedSince);
      }

      String ifNoneMatch = md.getFirstValue("cachedEtag");
      if (StringUtils.isNotBlank(ifNoneMatch)) {
        httpget.addHeader("If-None-Match", ifNoneMatch);
      }
    }

    // no need to release the connection explicitly as this is handled
    // automatically. The client itself must be closed though.
    try (CloseableHttpClient httpclient = builder.build()) {
      return httpclient.execute(httpget, this);
    }
  }
Exemplo n.º 5
0
 public InputStream getInputStream() throws Exception {
   httpClient = HttpClients.createDefault();
   HttpGet httpGet = new HttpGet(uri);
   System.out.println(uri);
   RequestConfig requestConfig =
       RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000).build();
   httpGet.setConfig(requestConfig);
   // DefaultHttpRequestRetryHandler handler = new DefaultHttpRequestRetryHandler(3, false);
   // httpClient.setHttpRequestRetryHandler(handler);
   response = httpClient.execute(httpGet);
   System.out.println(response.getStatusLine());
   if (HttpStatus.SC_OK != response.getStatusLine().getStatusCode()) {
     // System.out.println("Connect Error");
     return null;
   }
   if (key != null) {
     Header header = response.getFirstHeader(key);
     if (header == null) {
       // not valid date
       // System.out.println("Not valid date.");
       return null;
     }
   }
   entity = response.getEntity();
   return entity.getContent();
 }
Exemplo n.º 6
0
 public static IMHttpResponse get(String url) {
   IMHttpResponse response = new IMHttpResponse();
   response.setStatusCode(404);
   if (StringUtils.isEmpty(url)) {
     return response;
   }
   int statusCode = 404;
   CloseableHttpClient httpClient = null;
   HttpGet httpGet = null;
   try {
     httpClient = HttpClients.createDefault();
     httpGet = new HttpGet(url); // HTTP Get请求(POST雷同)
     RequestConfig requestConfig =
         RequestConfig.custom()
             .setSocketTimeout(2000)
             .setConnectTimeout(2000)
             .build(); // 设置请求和传输超时时间
     httpGet.setConfig(requestConfig);
     CloseableHttpResponse hresp = httpClient.execute(httpGet); // 执行请求
     String responseString = EntityUtils.toString(hresp.getEntity());
     response.setStatusCode(hresp.getStatusLine().getStatusCode());
     response.setResponseBody(responseString);
     return response;
   } catch (Exception e) {
     logger.error("error code: " + statusCode, e);
   } finally {
     if (httpGet != null) {
       httpGet.releaseConnection();
     }
   }
   return response;
 }
Exemplo n.º 7
0
  @Test
  public void example2() {
    RequestConfig globalConfig =
        RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH).build();
    CloseableHttpClient httpClient =
        HttpClients.custom().setDefaultRequestConfig(globalConfig).build();

    RequestConfig localConfig =
        RequestConfig.copy(globalConfig).setCookieSpec(CookieSpecs.DEFAULT).build();
    HttpGet httpGet = new HttpGet("http://baidu.com");
    httpGet.setConfig(localConfig);
  }
Exemplo n.º 8
0
 @Test(expected = ConnectTimeoutException.class)
 public final void givenLowTimeout_whenExecutingRequestWithTimeout_thenException()
     throws ClientProtocolException, IOException {
   final RequestConfig requestConfig =
       RequestConfig.custom()
           .setConnectionRequestTimeout(50)
           .setConnectTimeout(50)
           .setSocketTimeout(20)
           .build();
   final HttpGet request = new HttpGet(SAMPLE_URL);
   request.setConfig(requestConfig);
   response = instance.execute(request);
 }
Exemplo n.º 9
0
 public String get() throws Exception {
   HttpClient client = new DefaultHttpClient();
   this.url =
       "https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid="
           + appId
           + "&secret="
           + secret;
   HttpGet request = new HttpGet(url);
   request.setConfig(requestConfig);
   HttpResponse response = client.execute(request);
   String result = EntityUtils.toString(response.getEntity());
   return result;
 }
Exemplo n.º 10
0
 public static InputStream getInputStreamByUrl(String url, int timeout)
     throws ClientProtocolException, IOException {
   HttpClient httpClient = getHttpClientByUrl(url);
   HttpGet request = new HttpGet(url);
   RequestConfig requestConfig =
       RequestConfig.custom()
           .setSocketTimeout(timeout)
           .setConnectTimeout(timeout)
           .build(); // 设置请求和传输超时时间
   request.setConfig(requestConfig);
   HttpResponse response = httpClient.execute(request);
   if (response.getStatusLine().getStatusCode() == 200) {
     HttpEntity entity = response.getEntity();
     return entity.getContent();
   }
   return null;
 }
Exemplo n.º 11
0
 public static String openUrl(String url, HttpClient client, HttpHost httpProxy)
     throws RuntimeException {
   try {
     HttpGet httpGet = new HttpGet(url);
     if (httpProxy != null) {
       RequestConfig config = RequestConfig.custom().setProxy(httpProxy).build();
       httpGet.setConfig(config);
     }
     HttpResponse response = client.execute(httpGet);
     String responseText = new BasicResponseHandler().handleResponse(response);
     return responseText;
   } catch (ClientProtocolException e) {
     throw new RuntimeException(e);
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
Exemplo n.º 12
0
  public static String get(String url, Map<String, String> paramsMap) throws ApiException {

    if (null == httpClient) httpClient = createSSLClientDefault();

    CloseableHttpClient client = httpClient;
    String responseText = null;
    HttpEntity entity = null;

    CloseableHttpResponse response = null;
    try {
      StringBuilder sb = new StringBuilder();
      if (paramsMap != null) {
        for (Map.Entry<String, String> param : paramsMap.entrySet()) {
          sb.append("&" + param.getKey() + "=" + param.getValue());
        }
        url = url + "?" + sb.toString().substring(1);
      }

      HttpGet method = new HttpGet(url);
      RequestConfig requestConfig =
          RequestConfig.custom()
              .setConnectTimeout(CONNECTION_TIMEOUT)
              .setSocketTimeout(SOCKETCOOECTION_TIMEOUT)
              .build(); // 设置请求超时时间
      method.setConfig(requestConfig);
      response = client.execute(method);
      entity = response.getEntity();
      if (entity != null) {
        responseText = EntityUtils.toString(entity);
      }
      if (response.getStatusLine().getStatusCode() != 200) throw new ApiException(responseText);

      if (response != null) {
        response.close();
      }
    } catch (ClientProtocolException e) {
      e.printStackTrace();
      throw new ApiException(e);

    } catch (IOException e) {
      e.printStackTrace();
      throw new ApiException(e);
    }
    return responseText;
  }
Exemplo n.º 13
0
  public static HttpResponse get(
      String url, List<NameValuePair> params, RequestConfig config, CookieStore cookieStore)
      throws IOException {
    CloseableHttpClient client =
        HttpClients.custom().setUserAgent(Constants.DEFAULT_USER_AGENT).build();
    HttpContext localContext = new BasicHttpContext();
    localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);

    String reqUrl = "";
    if (params != null && params.size() > 0) {
      reqUrl = url + "?" + URLEncodedUtils.format(params, "utf-8");
    } else {
      reqUrl = url;
    }
    HttpGet get = new HttpGet(reqUrl);
    get.setConfig(config);
    HttpResponse response = client.execute(get, localContext);
    return response;
  }
Exemplo n.º 14
0
 public static HttpResponse doGetResource(String url, Map<String, String> params) {
   client = getHttpClientInstance();
   URI uri = generateURLParams(url, params);
   HttpGet get = new HttpGet(uri);
   get.setConfig(requestConfig);
   CloseableHttpResponse httpResponse = null;
   try {
     httpResponse = client.execute(get);
   } catch (IOException e) {
     e.printStackTrace();
   } finally {
     if (null != httpResponse) {
       try {
         httpResponse.close();
       } catch (IOException e) {
         e.printStackTrace();
       }
     }
   }
   return httpResponse;
 }
Exemplo n.º 15
0
  // 下载 url 指向的网页
  public String downloadFile(String url) {

    String filePath = null;
    HttpRequestRetryHandler myRetryHandler =
        new HttpRequestRetryHandler() {

          @Override
          public boolean retryRequest(IOException arg0, int executionCount, HttpContext arg2) {
            if (executionCount > 5) { // 最多重试5次
              return false;
            }
            return false;
          }
        };
    CloseableHttpClient httpClient =
        HttpClients.custom().setRetryHandler(myRetryHandler).build(); // 生成 HttpClinet
    RequestConfig requestConfig =
        RequestConfig.custom().setConnectTimeout(5000).setConnectionRequestTimeout(5000).build();
    HttpGet httpGet = new HttpGet(url);
    httpGet.setConfig(requestConfig);
    CloseableHttpResponse response;
    try {
      response = httpClient.execute(httpGet);
      StatusLine statusLine = response.getStatusLine();
      if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
        System.err.println("Method failed: " + statusLine);
        filePath = null;
      }
      HttpEntity entity = response.getEntity();
      if (entity != null) { // 处理流内容
        InputStream entityContent = entity.getContent();
        filePath = "target/html/" + getFileNameByUrl(url);
        saveToLocal(entityContent, filePath);
      }
    } catch (IOException e) {
      e.printStackTrace();
    }
    System.out.println("成功下载文件" + filePath + "到本地"); // not very bad
    return filePath;
  }
Exemplo n.º 16
0
  public static final void main(String[] args) throws Exception {

    // Use custom message parser / writer to customize the way HTTP
    // messages are parsed from and written out to the data stream.
    HttpMessageParserFactory<HttpResponse> responseParserFactory =
        new DefaultHttpResponseParserFactory() {

          @Override
          public HttpMessageParser<HttpResponse> create(
              SessionInputBuffer buffer, MessageConstraints constraints) {
            LineParser lineParser =
                new BasicLineParser() {

                  @Override
                  public Header parseHeader(final CharArrayBuffer buffer) {
                    try {
                      return super.parseHeader(buffer);
                    } catch (ParseException ex) {
                      return new BasicHeader(buffer.toString(), null);
                    }
                  }
                };
            return new DefaultHttpResponseParser(
                buffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints) {

              @Override
              protected boolean reject(final CharArrayBuffer line, int count) {
                // try to ignore all garbage preceding a status line infinitely
                return false;
              }
            };
          }
        };
    HttpMessageWriterFactory<HttpRequest> requestWriterFactory =
        new DefaultHttpRequestWriterFactory();

    // Use a custom connection factory to customize the process of
    // initialization of outgoing HTTP connections. Beside standard connection
    // configuration parameters HTTP connection factory can define message
    // parser / writer routines to be employed by individual connections.
    HttpConnectionFactory<HttpRoute, ManagedHttpClientConnection> connFactory =
        new ManagedHttpClientConnectionFactory(requestWriterFactory, responseParserFactory);

    // Client HTTP connection objects when fully initialized can be bound to
    // an arbitrary network socket. The process of network socket initialization,
    // its connection to a remote address and binding to a local one is controlled
    // by a connection socket factory.

    // SSL context for secure connections can be created either based on
    // system or application specific properties.
    SSLContext sslcontext = SSLContexts.createSystemDefault();

    // Create a registry of custom connection socket factories for supported
    // protocol schemes.
    Registry<ConnectionSocketFactory> socketFactoryRegistry =
        RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.INSTANCE)
            .register("https", new SSLConnectionSocketFactory(sslcontext))
            .build();

    // Use custom DNS resolver to override the system DNS resolution.
    DnsResolver dnsResolver =
        new SystemDefaultDnsResolver() {

          @Override
          public InetAddress[] resolve(final String host) throws UnknownHostException {
            if (host.equalsIgnoreCase("myhost")) {
              return new InetAddress[] {InetAddress.getByAddress(new byte[] {127, 0, 0, 1})};
            } else {
              return super.resolve(host);
            }
          }
        };

    // Create a connection manager with custom configuration.
    PoolingHttpClientConnectionManager connManager =
        new PoolingHttpClientConnectionManager(socketFactoryRegistry, connFactory, dnsResolver);

    // Create socket configuration
    SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();
    // Configure the connection manager to use socket configuration either
    // by default or for a specific host.
    connManager.setDefaultSocketConfig(socketConfig);
    connManager.setSocketConfig(new HttpHost("somehost", 80), socketConfig);
    // Validate connections after 1 sec of inactivity
    connManager.setValidateAfterInactivity(1000);

    // Create message constraints
    MessageConstraints messageConstraints =
        MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build();
    // Create connection configuration
    ConnectionConfig connectionConfig =
        ConnectionConfig.custom()
            .setMalformedInputAction(CodingErrorAction.IGNORE)
            .setUnmappableInputAction(CodingErrorAction.IGNORE)
            .setCharset(Consts.UTF_8)
            .setMessageConstraints(messageConstraints)
            .build();
    // Configure the connection manager to use connection configuration either
    // by default or for a specific host.
    connManager.setDefaultConnectionConfig(connectionConfig);
    connManager.setConnectionConfig(new HttpHost("somehost", 80), ConnectionConfig.DEFAULT);

    // Configure total max or per route limits for persistent connections
    // that can be kept in the pool or leased by the connection manager.
    connManager.setMaxTotal(100);
    connManager.setDefaultMaxPerRoute(10);
    connManager.setMaxPerRoute(new HttpRoute(new HttpHost("somehost", 80)), 20);

    // Use custom cookie store if necessary.
    CookieStore cookieStore = new BasicCookieStore();
    // Use custom credentials provider if necessary.
    CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    // Create global request configuration
    RequestConfig defaultRequestConfig =
        RequestConfig.custom()
            .setCookieSpec(CookieSpecs.DEFAULT)
            .setExpectContinueEnabled(true)
            .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
            .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
            .build();

    // Create an HttpClient with the given custom dependencies and configuration.
    CloseableHttpClient httpclient =
        HttpClients.custom()
            .setConnectionManager(connManager)
            .setDefaultCookieStore(cookieStore)
            .setDefaultCredentialsProvider(credentialsProvider)
            .setProxy(new HttpHost("myproxy", 8080))
            .setDefaultRequestConfig(defaultRequestConfig)
            .build();

    try {
      HttpGet httpget = new HttpGet("http://httpbin.org/get");
      // Request configuration can be overridden at the request level.
      // They will take precedence over the one set at the client level.
      RequestConfig requestConfig =
          RequestConfig.copy(defaultRequestConfig)
              .setSocketTimeout(5000)
              .setConnectTimeout(5000)
              .setConnectionRequestTimeout(5000)
              .setProxy(new HttpHost("myotherproxy", 8080))
              .build();
      httpget.setConfig(requestConfig);

      // Execution context can be customized locally.
      HttpClientContext context = HttpClientContext.create();
      // Contextual attributes set the local context level will take
      // precedence over those set at the client level.
      context.setCookieStore(cookieStore);
      context.setCredentialsProvider(credentialsProvider);

      System.out.println("executing request " + httpget.getURI());
      CloseableHttpResponse response = httpclient.execute(httpget, context);
      try {
        System.out.println("----------------------------------------");
        System.out.println(response.getStatusLine());
        System.out.println(EntityUtils.toString(response.getEntity()));
        System.out.println("----------------------------------------");

        // Once the request has been executed the local context can
        // be used to examine updated state and various objects affected
        // by the request execution.

        // Last executed request
        context.getRequest();
        // Execution route
        context.getHttpRoute();
        // Target auth state
        context.getTargetAuthState();
        // Proxy auth state
        context.getTargetAuthState();
        // Cookie origin
        context.getCookieOrigin();
        // Cookie spec used
        context.getCookieSpec();
        // User security token
        context.getUserToken();

      } finally {
        response.close();
      }
    } finally {
      httpclient.close();
    }
  }