@Override public void run() { HttpClient httpClient = HttpClients.createDefault(); HttpGet httpGet = new HttpGet("http://localhost:8080/crawler/main"); httpGet.setConfig(RequestConfig.custom().setConnectTimeout(10000).build()); httpGet.setConfig(RequestConfig.custom().setSocketTimeout(5000).build()); int statusCode = 0; try { HttpResponse response = httpClient.execute(httpGet); statusCode = response.getStatusLine().getStatusCode(); while (statusCode != HttpStatus.SC_OK) { Thread.sleep(1000); response = httpClient.execute(httpGet); statusCode = response.getStatusLine().getStatusCode(); } } catch (ClientProtocolException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } ExtLogger.info("multi-thread start"); }
public void testMojoExecution() throws Exception { assertNotNull(mojo); mojo.execute(); HttpClient client = HttpClientBuilder.create().build(); HttpGet get = new HttpGet("http://localhost:" + elasticsearchNode.getHttpPort()); final int connectionTimeout = 500; // millis RequestConfig requestConfig = RequestConfig.custom() .setConnectionRequestTimeout(connectionTimeout) .setConnectTimeout(connectionTimeout) .setSocketTimeout(connectionTimeout) .build(); get.setConfig(requestConfig); try { client.execute(get); fail("The ES cluster should have been down by now"); } catch (HttpHostConnectException | ConnectTimeoutException expected) { } assertTrue(elasticsearchNode.isClosed()); }
public static String get(String url, List<NameValuePair> params) { String body = null; try { HttpClient hc = new DefaultHttpClient(); // Get请求 HttpGet httpget = new HttpGet(url); // 设置参数 String str = EntityUtils.toString(new UrlEncodedFormEntity(params)); httpget.setURI(new URI(httpget.getURI().toString() + "?" + str)); // 设置请求和传输超时时间 RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(4000).setConnectTimeout(4000).build(); httpget.setConfig(requestConfig); // 发送请求 HttpResponse httpresponse = hc.execute(httpget); // 获取返回数据 HttpEntity entity = httpresponse.getEntity(); body = EntityUtils.toString(entity); if (entity != null) { entity.consumeContent(); } } catch (Exception e) { logger.error("http的get方法" + e.toString()); throw new RuntimeException(e); } return body; }
@Override public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Exception { LOG.debug("HTTP connection manager stats {}", CONNECTION_MANAGER.getTotalStats()); HttpGet httpget = new HttpGet(url); httpget.setConfig(requestConfig); if (md != null) { String ifModifiedSince = md.getFirstValue("cachedLastModified"); if (StringUtils.isNotBlank(ifModifiedSince)) { httpget.addHeader("If-Modified-Since", ifModifiedSince); } String ifNoneMatch = md.getFirstValue("cachedEtag"); if (StringUtils.isNotBlank(ifNoneMatch)) { httpget.addHeader("If-None-Match", ifNoneMatch); } } // no need to release the connection explicitly as this is handled // automatically. The client itself must be closed though. try (CloseableHttpClient httpclient = builder.build()) { return httpclient.execute(httpget, this); } }
public InputStream getInputStream() throws Exception { httpClient = HttpClients.createDefault(); HttpGet httpGet = new HttpGet(uri); System.out.println(uri); RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000).build(); httpGet.setConfig(requestConfig); // DefaultHttpRequestRetryHandler handler = new DefaultHttpRequestRetryHandler(3, false); // httpClient.setHttpRequestRetryHandler(handler); response = httpClient.execute(httpGet); System.out.println(response.getStatusLine()); if (HttpStatus.SC_OK != response.getStatusLine().getStatusCode()) { // System.out.println("Connect Error"); return null; } if (key != null) { Header header = response.getFirstHeader(key); if (header == null) { // not valid date // System.out.println("Not valid date."); return null; } } entity = response.getEntity(); return entity.getContent(); }
public static IMHttpResponse get(String url) { IMHttpResponse response = new IMHttpResponse(); response.setStatusCode(404); if (StringUtils.isEmpty(url)) { return response; } int statusCode = 404; CloseableHttpClient httpClient = null; HttpGet httpGet = null; try { httpClient = HttpClients.createDefault(); httpGet = new HttpGet(url); // HTTP Get请求(POST雷同) RequestConfig requestConfig = RequestConfig.custom() .setSocketTimeout(2000) .setConnectTimeout(2000) .build(); // 设置请求和传输超时时间 httpGet.setConfig(requestConfig); CloseableHttpResponse hresp = httpClient.execute(httpGet); // 执行请求 String responseString = EntityUtils.toString(hresp.getEntity()); response.setStatusCode(hresp.getStatusLine().getStatusCode()); response.setResponseBody(responseString); return response; } catch (Exception e) { logger.error("error code: " + statusCode, e); } finally { if (httpGet != null) { httpGet.releaseConnection(); } } return response; }
@Test public void example2() { RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH).build(); CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(globalConfig).build(); RequestConfig localConfig = RequestConfig.copy(globalConfig).setCookieSpec(CookieSpecs.DEFAULT).build(); HttpGet httpGet = new HttpGet("http://baidu.com"); httpGet.setConfig(localConfig); }
@Test(expected = ConnectTimeoutException.class) public final void givenLowTimeout_whenExecutingRequestWithTimeout_thenException() throws ClientProtocolException, IOException { final RequestConfig requestConfig = RequestConfig.custom() .setConnectionRequestTimeout(50) .setConnectTimeout(50) .setSocketTimeout(20) .build(); final HttpGet request = new HttpGet(SAMPLE_URL); request.setConfig(requestConfig); response = instance.execute(request); }
public String get() throws Exception { HttpClient client = new DefaultHttpClient(); this.url = "https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid=" + appId + "&secret=" + secret; HttpGet request = new HttpGet(url); request.setConfig(requestConfig); HttpResponse response = client.execute(request); String result = EntityUtils.toString(response.getEntity()); return result; }
public static InputStream getInputStreamByUrl(String url, int timeout) throws ClientProtocolException, IOException { HttpClient httpClient = getHttpClientByUrl(url); HttpGet request = new HttpGet(url); RequestConfig requestConfig = RequestConfig.custom() .setSocketTimeout(timeout) .setConnectTimeout(timeout) .build(); // 设置请求和传输超时时间 request.setConfig(requestConfig); HttpResponse response = httpClient.execute(request); if (response.getStatusLine().getStatusCode() == 200) { HttpEntity entity = response.getEntity(); return entity.getContent(); } return null; }
public static String openUrl(String url, HttpClient client, HttpHost httpProxy) throws RuntimeException { try { HttpGet httpGet = new HttpGet(url); if (httpProxy != null) { RequestConfig config = RequestConfig.custom().setProxy(httpProxy).build(); httpGet.setConfig(config); } HttpResponse response = client.execute(httpGet); String responseText = new BasicResponseHandler().handleResponse(response); return responseText; } catch (ClientProtocolException e) { throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); } }
public static String get(String url, Map<String, String> paramsMap) throws ApiException { if (null == httpClient) httpClient = createSSLClientDefault(); CloseableHttpClient client = httpClient; String responseText = null; HttpEntity entity = null; CloseableHttpResponse response = null; try { StringBuilder sb = new StringBuilder(); if (paramsMap != null) { for (Map.Entry<String, String> param : paramsMap.entrySet()) { sb.append("&" + param.getKey() + "=" + param.getValue()); } url = url + "?" + sb.toString().substring(1); } HttpGet method = new HttpGet(url); RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout(CONNECTION_TIMEOUT) .setSocketTimeout(SOCKETCOOECTION_TIMEOUT) .build(); // 设置请求超时时间 method.setConfig(requestConfig); response = client.execute(method); entity = response.getEntity(); if (entity != null) { responseText = EntityUtils.toString(entity); } if (response.getStatusLine().getStatusCode() != 200) throw new ApiException(responseText); if (response != null) { response.close(); } } catch (ClientProtocolException e) { e.printStackTrace(); throw new ApiException(e); } catch (IOException e) { e.printStackTrace(); throw new ApiException(e); } return responseText; }
public static HttpResponse get( String url, List<NameValuePair> params, RequestConfig config, CookieStore cookieStore) throws IOException { CloseableHttpClient client = HttpClients.custom().setUserAgent(Constants.DEFAULT_USER_AGENT).build(); HttpContext localContext = new BasicHttpContext(); localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore); String reqUrl = ""; if (params != null && params.size() > 0) { reqUrl = url + "?" + URLEncodedUtils.format(params, "utf-8"); } else { reqUrl = url; } HttpGet get = new HttpGet(reqUrl); get.setConfig(config); HttpResponse response = client.execute(get, localContext); return response; }
public static HttpResponse doGetResource(String url, Map<String, String> params) { client = getHttpClientInstance(); URI uri = generateURLParams(url, params); HttpGet get = new HttpGet(uri); get.setConfig(requestConfig); CloseableHttpResponse httpResponse = null; try { httpResponse = client.execute(get); } catch (IOException e) { e.printStackTrace(); } finally { if (null != httpResponse) { try { httpResponse.close(); } catch (IOException e) { e.printStackTrace(); } } } return httpResponse; }
// 下载 url 指向的网页 public String downloadFile(String url) { String filePath = null; HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() { @Override public boolean retryRequest(IOException arg0, int executionCount, HttpContext arg2) { if (executionCount > 5) { // 最多重试5次 return false; } return false; } }; CloseableHttpClient httpClient = HttpClients.custom().setRetryHandler(myRetryHandler).build(); // 生成 HttpClinet RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(5000).setConnectionRequestTimeout(5000).build(); HttpGet httpGet = new HttpGet(url); httpGet.setConfig(requestConfig); CloseableHttpResponse response; try { response = httpClient.execute(httpGet); StatusLine statusLine = response.getStatusLine(); if (statusLine.getStatusCode() != HttpStatus.SC_OK) { System.err.println("Method failed: " + statusLine); filePath = null; } HttpEntity entity = response.getEntity(); if (entity != null) { // 处理流内容 InputStream entityContent = entity.getContent(); filePath = "target/html/" + getFileNameByUrl(url); saveToLocal(entityContent, filePath); } } catch (IOException e) { e.printStackTrace(); } System.out.println("成功下载文件" + filePath + "到本地"); // not very bad return filePath; }
public static final void main(String[] args) throws Exception { // Use custom message parser / writer to customize the way HTTP // messages are parsed from and written out to the data stream. HttpMessageParserFactory<HttpResponse> responseParserFactory = new DefaultHttpResponseParserFactory() { @Override public HttpMessageParser<HttpResponse> create( SessionInputBuffer buffer, MessageConstraints constraints) { LineParser lineParser = new BasicLineParser() { @Override public Header parseHeader(final CharArrayBuffer buffer) { try { return super.parseHeader(buffer); } catch (ParseException ex) { return new BasicHeader(buffer.toString(), null); } } }; return new DefaultHttpResponseParser( buffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints) { @Override protected boolean reject(final CharArrayBuffer line, int count) { // try to ignore all garbage preceding a status line infinitely return false; } }; } }; HttpMessageWriterFactory<HttpRequest> requestWriterFactory = new DefaultHttpRequestWriterFactory(); // Use a custom connection factory to customize the process of // initialization of outgoing HTTP connections. Beside standard connection // configuration parameters HTTP connection factory can define message // parser / writer routines to be employed by individual connections. HttpConnectionFactory<HttpRoute, ManagedHttpClientConnection> connFactory = new ManagedHttpClientConnectionFactory(requestWriterFactory, responseParserFactory); // Client HTTP connection objects when fully initialized can be bound to // an arbitrary network socket. The process of network socket initialization, // its connection to a remote address and binding to a local one is controlled // by a connection socket factory. // SSL context for secure connections can be created either based on // system or application specific properties. SSLContext sslcontext = SSLContexts.createSystemDefault(); // Create a registry of custom connection socket factories for supported // protocol schemes. Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", new SSLConnectionSocketFactory(sslcontext)) .build(); // Use custom DNS resolver to override the system DNS resolution. DnsResolver dnsResolver = new SystemDefaultDnsResolver() { @Override public InetAddress[] resolve(final String host) throws UnknownHostException { if (host.equalsIgnoreCase("myhost")) { return new InetAddress[] {InetAddress.getByAddress(new byte[] {127, 0, 0, 1})}; } else { return super.resolve(host); } } }; // Create a connection manager with custom configuration. PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry, connFactory, dnsResolver); // Create socket configuration SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build(); // Configure the connection manager to use socket configuration either // by default or for a specific host. connManager.setDefaultSocketConfig(socketConfig); connManager.setSocketConfig(new HttpHost("somehost", 80), socketConfig); // Validate connections after 1 sec of inactivity connManager.setValidateAfterInactivity(1000); // Create message constraints MessageConstraints messageConstraints = MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build(); // Create connection configuration ConnectionConfig connectionConfig = ConnectionConfig.custom() .setMalformedInputAction(CodingErrorAction.IGNORE) .setUnmappableInputAction(CodingErrorAction.IGNORE) .setCharset(Consts.UTF_8) .setMessageConstraints(messageConstraints) .build(); // Configure the connection manager to use connection configuration either // by default or for a specific host. connManager.setDefaultConnectionConfig(connectionConfig); connManager.setConnectionConfig(new HttpHost("somehost", 80), ConnectionConfig.DEFAULT); // Configure total max or per route limits for persistent connections // that can be kept in the pool or leased by the connection manager. connManager.setMaxTotal(100); connManager.setDefaultMaxPerRoute(10); connManager.setMaxPerRoute(new HttpRoute(new HttpHost("somehost", 80)), 20); // Use custom cookie store if necessary. CookieStore cookieStore = new BasicCookieStore(); // Use custom credentials provider if necessary. CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); // Create global request configuration RequestConfig defaultRequestConfig = RequestConfig.custom() .setCookieSpec(CookieSpecs.DEFAULT) .setExpectContinueEnabled(true) .setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST)) .setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC)) .build(); // Create an HttpClient with the given custom dependencies and configuration. CloseableHttpClient httpclient = HttpClients.custom() .setConnectionManager(connManager) .setDefaultCookieStore(cookieStore) .setDefaultCredentialsProvider(credentialsProvider) .setProxy(new HttpHost("myproxy", 8080)) .setDefaultRequestConfig(defaultRequestConfig) .build(); try { HttpGet httpget = new HttpGet("http://httpbin.org/get"); // Request configuration can be overridden at the request level. // They will take precedence over the one set at the client level. RequestConfig requestConfig = RequestConfig.copy(defaultRequestConfig) .setSocketTimeout(5000) .setConnectTimeout(5000) .setConnectionRequestTimeout(5000) .setProxy(new HttpHost("myotherproxy", 8080)) .build(); httpget.setConfig(requestConfig); // Execution context can be customized locally. HttpClientContext context = HttpClientContext.create(); // Contextual attributes set the local context level will take // precedence over those set at the client level. context.setCookieStore(cookieStore); context.setCredentialsProvider(credentialsProvider); System.out.println("executing request " + httpget.getURI()); CloseableHttpResponse response = httpclient.execute(httpget, context); try { System.out.println("----------------------------------------"); System.out.println(response.getStatusLine()); System.out.println(EntityUtils.toString(response.getEntity())); System.out.println("----------------------------------------"); // Once the request has been executed the local context can // be used to examine updated state and various objects affected // by the request execution. // Last executed request context.getRequest(); // Execution route context.getHttpRoute(); // Target auth state context.getTargetAuthState(); // Proxy auth state context.getTargetAuthState(); // Cookie origin context.getCookieOrigin(); // Cookie spec used context.getCookieSpec(); // User security token context.getUserToken(); } finally { response.close(); } } finally { httpclient.close(); } }