@Override
public void run() {
HttpClient httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet("http://localhost:8080/crawler/main");
httpGet.setConfig(RequestConfig.custom().setConnectTimeout(10000).build());
httpGet.setConfig(RequestConfig.custom().setSocketTimeout(5000).build());
int statusCode = 0;
try {
HttpResponse response = httpClient.execute(httpGet);
statusCode = response.getStatusLine().getStatusCode();
while (statusCode != HttpStatus.SC_OK) {
Thread.sleep(1000);
response = httpClient.execute(httpGet);
statusCode = response.getStatusLine().getStatusCode();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
ExtLogger.info("multi-thread start");
}
public void testMojoExecution() throws Exception {
assertNotNull(mojo);
mojo.execute();
HttpClient client = HttpClientBuilder.create().build();
HttpGet get = new HttpGet("http://localhost:" + elasticsearchNode.getHttpPort());
final int connectionTimeout = 500; // millis
RequestConfig requestConfig =
RequestConfig.custom()
.setConnectionRequestTimeout(connectionTimeout)
.setConnectTimeout(connectionTimeout)
.setSocketTimeout(connectionTimeout)
.build();
get.setConfig(requestConfig);
try {
client.execute(get);
fail("The ES cluster should have been down by now");
} catch (HttpHostConnectException | ConnectTimeoutException expected) {
}
assertTrue(elasticsearchNode.isClosed());
}
public static String get(String url, List<NameValuePair> params) {
String body = null;
try {
HttpClient hc = new DefaultHttpClient();
// Get请求
HttpGet httpget = new HttpGet(url);
// 设置参数
String str = EntityUtils.toString(new UrlEncodedFormEntity(params));
httpget.setURI(new URI(httpget.getURI().toString() + "?" + str));
// 设置请求和传输超时时间
RequestConfig requestConfig =
RequestConfig.custom().setSocketTimeout(4000).setConnectTimeout(4000).build();
httpget.setConfig(requestConfig);
// 发送请求
HttpResponse httpresponse = hc.execute(httpget);
// 获取返回数据
HttpEntity entity = httpresponse.getEntity();
body = EntityUtils.toString(entity);
if (entity != null) {
entity.consumeContent();
}
} catch (Exception e) {
logger.error("http的get方法" + e.toString());
throw new RuntimeException(e);
}
return body;
}
@Override
public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Exception {
LOG.debug("HTTP connection manager stats {}", CONNECTION_MANAGER.getTotalStats());
HttpGet httpget = new HttpGet(url);
httpget.setConfig(requestConfig);
if (md != null) {
String ifModifiedSince = md.getFirstValue("cachedLastModified");
if (StringUtils.isNotBlank(ifModifiedSince)) {
httpget.addHeader("If-Modified-Since", ifModifiedSince);
}
String ifNoneMatch = md.getFirstValue("cachedEtag");
if (StringUtils.isNotBlank(ifNoneMatch)) {
httpget.addHeader("If-None-Match", ifNoneMatch);
}
}
// no need to release the connection explicitly as this is handled
// automatically. The client itself must be closed though.
try (CloseableHttpClient httpclient = builder.build()) {
return httpclient.execute(httpget, this);
}
}
public InputStream getInputStream() throws Exception {
httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(uri);
System.out.println(uri);
RequestConfig requestConfig =
RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000).build();
httpGet.setConfig(requestConfig);
// DefaultHttpRequestRetryHandler handler = new DefaultHttpRequestRetryHandler(3, false);
// httpClient.setHttpRequestRetryHandler(handler);
response = httpClient.execute(httpGet);
System.out.println(response.getStatusLine());
if (HttpStatus.SC_OK != response.getStatusLine().getStatusCode()) {
// System.out.println("Connect Error");
return null;
}
if (key != null) {
Header header = response.getFirstHeader(key);
if (header == null) {
// not valid date
// System.out.println("Not valid date.");
return null;
}
}
entity = response.getEntity();
return entity.getContent();
}
public static IMHttpResponse get(String url) {
IMHttpResponse response = new IMHttpResponse();
response.setStatusCode(404);
if (StringUtils.isEmpty(url)) {
return response;
}
int statusCode = 404;
CloseableHttpClient httpClient = null;
HttpGet httpGet = null;
try {
httpClient = HttpClients.createDefault();
httpGet = new HttpGet(url); // HTTP Get请求(POST雷同)
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(2000)
.setConnectTimeout(2000)
.build(); // 设置请求和传输超时时间
httpGet.setConfig(requestConfig);
CloseableHttpResponse hresp = httpClient.execute(httpGet); // 执行请求
String responseString = EntityUtils.toString(hresp.getEntity());
response.setStatusCode(hresp.getStatusLine().getStatusCode());
response.setResponseBody(responseString);
return response;
} catch (Exception e) {
logger.error("error code: " + statusCode, e);
} finally {
if (httpGet != null) {
httpGet.releaseConnection();
}
}
return response;
}
@Test
public void example2() {
RequestConfig globalConfig =
RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH).build();
CloseableHttpClient httpClient =
HttpClients.custom().setDefaultRequestConfig(globalConfig).build();
RequestConfig localConfig =
RequestConfig.copy(globalConfig).setCookieSpec(CookieSpecs.DEFAULT).build();
HttpGet httpGet = new HttpGet("http://baidu.com");
httpGet.setConfig(localConfig);
}
@Test(expected = ConnectTimeoutException.class)
public final void givenLowTimeout_whenExecutingRequestWithTimeout_thenException()
throws ClientProtocolException, IOException {
final RequestConfig requestConfig =
RequestConfig.custom()
.setConnectionRequestTimeout(50)
.setConnectTimeout(50)
.setSocketTimeout(20)
.build();
final HttpGet request = new HttpGet(SAMPLE_URL);
request.setConfig(requestConfig);
response = instance.execute(request);
}
public String get() throws Exception {
HttpClient client = new DefaultHttpClient();
this.url =
"https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid="
+ appId
+ "&secret="
+ secret;
HttpGet request = new HttpGet(url);
request.setConfig(requestConfig);
HttpResponse response = client.execute(request);
String result = EntityUtils.toString(response.getEntity());
return result;
}
public static InputStream getInputStreamByUrl(String url, int timeout)
throws ClientProtocolException, IOException {
HttpClient httpClient = getHttpClientByUrl(url);
HttpGet request = new HttpGet(url);
RequestConfig requestConfig =
RequestConfig.custom()
.setSocketTimeout(timeout)
.setConnectTimeout(timeout)
.build(); // 设置请求和传输超时时间
request.setConfig(requestConfig);
HttpResponse response = httpClient.execute(request);
if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity();
return entity.getContent();
}
return null;
}
public static String openUrl(String url, HttpClient client, HttpHost httpProxy)
throws RuntimeException {
try {
HttpGet httpGet = new HttpGet(url);
if (httpProxy != null) {
RequestConfig config = RequestConfig.custom().setProxy(httpProxy).build();
httpGet.setConfig(config);
}
HttpResponse response = client.execute(httpGet);
String responseText = new BasicResponseHandler().handleResponse(response);
return responseText;
} catch (ClientProtocolException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static String get(String url, Map<String, String> paramsMap) throws ApiException {
if (null == httpClient) httpClient = createSSLClientDefault();
CloseableHttpClient client = httpClient;
String responseText = null;
HttpEntity entity = null;
CloseableHttpResponse response = null;
try {
StringBuilder sb = new StringBuilder();
if (paramsMap != null) {
for (Map.Entry<String, String> param : paramsMap.entrySet()) {
sb.append("&" + param.getKey() + "=" + param.getValue());
}
url = url + "?" + sb.toString().substring(1);
}
HttpGet method = new HttpGet(url);
RequestConfig requestConfig =
RequestConfig.custom()
.setConnectTimeout(CONNECTION_TIMEOUT)
.setSocketTimeout(SOCKETCOOECTION_TIMEOUT)
.build(); // 设置请求超时时间
method.setConfig(requestConfig);
response = client.execute(method);
entity = response.getEntity();
if (entity != null) {
responseText = EntityUtils.toString(entity);
}
if (response.getStatusLine().getStatusCode() != 200) throw new ApiException(responseText);
if (response != null) {
response.close();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
throw new ApiException(e);
} catch (IOException e) {
e.printStackTrace();
throw new ApiException(e);
}
return responseText;
}
public static HttpResponse get(
String url, List<NameValuePair> params, RequestConfig config, CookieStore cookieStore)
throws IOException {
CloseableHttpClient client =
HttpClients.custom().setUserAgent(Constants.DEFAULT_USER_AGENT).build();
HttpContext localContext = new BasicHttpContext();
localContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore);
String reqUrl = "";
if (params != null && params.size() > 0) {
reqUrl = url + "?" + URLEncodedUtils.format(params, "utf-8");
} else {
reqUrl = url;
}
HttpGet get = new HttpGet(reqUrl);
get.setConfig(config);
HttpResponse response = client.execute(get, localContext);
return response;
}
public static HttpResponse doGetResource(String url, Map<String, String> params) {
client = getHttpClientInstance();
URI uri = generateURLParams(url, params);
HttpGet get = new HttpGet(uri);
get.setConfig(requestConfig);
CloseableHttpResponse httpResponse = null;
try {
httpResponse = client.execute(get);
} catch (IOException e) {
e.printStackTrace();
} finally {
if (null != httpResponse) {
try {
httpResponse.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return httpResponse;
}
// 下载 url 指向的网页
public String downloadFile(String url) {
String filePath = null;
HttpRequestRetryHandler myRetryHandler =
new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(IOException arg0, int executionCount, HttpContext arg2) {
if (executionCount > 5) { // 最多重试5次
return false;
}
return false;
}
};
CloseableHttpClient httpClient =
HttpClients.custom().setRetryHandler(myRetryHandler).build(); // 生成 HttpClinet
RequestConfig requestConfig =
RequestConfig.custom().setConnectTimeout(5000).setConnectionRequestTimeout(5000).build();
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(requestConfig);
CloseableHttpResponse response;
try {
response = httpClient.execute(httpGet);
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
System.err.println("Method failed: " + statusLine);
filePath = null;
}
HttpEntity entity = response.getEntity();
if (entity != null) { // 处理流内容
InputStream entityContent = entity.getContent();
filePath = "target/html/" + getFileNameByUrl(url);
saveToLocal(entityContent, filePath);
}
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("成功下载文件" + filePath + "到本地"); // not very bad
return filePath;
}
public static final void main(String[] args) throws Exception {
// Use custom message parser / writer to customize the way HTTP
// messages are parsed from and written out to the data stream.
HttpMessageParserFactory<HttpResponse> responseParserFactory =
new DefaultHttpResponseParserFactory() {
@Override
public HttpMessageParser<HttpResponse> create(
SessionInputBuffer buffer, MessageConstraints constraints) {
LineParser lineParser =
new BasicLineParser() {
@Override
public Header parseHeader(final CharArrayBuffer buffer) {
try {
return super.parseHeader(buffer);
} catch (ParseException ex) {
return new BasicHeader(buffer.toString(), null);
}
}
};
return new DefaultHttpResponseParser(
buffer, lineParser, DefaultHttpResponseFactory.INSTANCE, constraints) {
@Override
protected boolean reject(final CharArrayBuffer line, int count) {
// try to ignore all garbage preceding a status line infinitely
return false;
}
};
}
};
HttpMessageWriterFactory<HttpRequest> requestWriterFactory =
new DefaultHttpRequestWriterFactory();
// Use a custom connection factory to customize the process of
// initialization of outgoing HTTP connections. Beside standard connection
// configuration parameters HTTP connection factory can define message
// parser / writer routines to be employed by individual connections.
HttpConnectionFactory<HttpRoute, ManagedHttpClientConnection> connFactory =
new ManagedHttpClientConnectionFactory(requestWriterFactory, responseParserFactory);
// Client HTTP connection objects when fully initialized can be bound to
// an arbitrary network socket. The process of network socket initialization,
// its connection to a remote address and binding to a local one is controlled
// by a connection socket factory.
// SSL context for secure connections can be created either based on
// system or application specific properties.
SSLContext sslcontext = SSLContexts.createSystemDefault();
// Create a registry of custom connection socket factories for supported
// protocol schemes.
Registry<ConnectionSocketFactory> socketFactoryRegistry =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", new SSLConnectionSocketFactory(sslcontext))
.build();
// Use custom DNS resolver to override the system DNS resolution.
DnsResolver dnsResolver =
new SystemDefaultDnsResolver() {
@Override
public InetAddress[] resolve(final String host) throws UnknownHostException {
if (host.equalsIgnoreCase("myhost")) {
return new InetAddress[] {InetAddress.getByAddress(new byte[] {127, 0, 0, 1})};
} else {
return super.resolve(host);
}
}
};
// Create a connection manager with custom configuration.
PoolingHttpClientConnectionManager connManager =
new PoolingHttpClientConnectionManager(socketFactoryRegistry, connFactory, dnsResolver);
// Create socket configuration
SocketConfig socketConfig = SocketConfig.custom().setTcpNoDelay(true).build();
// Configure the connection manager to use socket configuration either
// by default or for a specific host.
connManager.setDefaultSocketConfig(socketConfig);
connManager.setSocketConfig(new HttpHost("somehost", 80), socketConfig);
// Validate connections after 1 sec of inactivity
connManager.setValidateAfterInactivity(1000);
// Create message constraints
MessageConstraints messageConstraints =
MessageConstraints.custom().setMaxHeaderCount(200).setMaxLineLength(2000).build();
// Create connection configuration
ConnectionConfig connectionConfig =
ConnectionConfig.custom()
.setMalformedInputAction(CodingErrorAction.IGNORE)
.setUnmappableInputAction(CodingErrorAction.IGNORE)
.setCharset(Consts.UTF_8)
.setMessageConstraints(messageConstraints)
.build();
// Configure the connection manager to use connection configuration either
// by default or for a specific host.
connManager.setDefaultConnectionConfig(connectionConfig);
connManager.setConnectionConfig(new HttpHost("somehost", 80), ConnectionConfig.DEFAULT);
// Configure total max or per route limits for persistent connections
// that can be kept in the pool or leased by the connection manager.
connManager.setMaxTotal(100);
connManager.setDefaultMaxPerRoute(10);
connManager.setMaxPerRoute(new HttpRoute(new HttpHost("somehost", 80)), 20);
// Use custom cookie store if necessary.
CookieStore cookieStore = new BasicCookieStore();
// Use custom credentials provider if necessary.
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// Create global request configuration
RequestConfig defaultRequestConfig =
RequestConfig.custom()
.setCookieSpec(CookieSpecs.DEFAULT)
.setExpectContinueEnabled(true)
.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
.build();
// Create an HttpClient with the given custom dependencies and configuration.
CloseableHttpClient httpclient =
HttpClients.custom()
.setConnectionManager(connManager)
.setDefaultCookieStore(cookieStore)
.setDefaultCredentialsProvider(credentialsProvider)
.setProxy(new HttpHost("myproxy", 8080))
.setDefaultRequestConfig(defaultRequestConfig)
.build();
try {
HttpGet httpget = new HttpGet("http://httpbin.org/get");
// Request configuration can be overridden at the request level.
// They will take precedence over the one set at the client level.
RequestConfig requestConfig =
RequestConfig.copy(defaultRequestConfig)
.setSocketTimeout(5000)
.setConnectTimeout(5000)
.setConnectionRequestTimeout(5000)
.setProxy(new HttpHost("myotherproxy", 8080))
.build();
httpget.setConfig(requestConfig);
// Execution context can be customized locally.
HttpClientContext context = HttpClientContext.create();
// Contextual attributes set the local context level will take
// precedence over those set at the client level.
context.setCookieStore(cookieStore);
context.setCredentialsProvider(credentialsProvider);
System.out.println("executing request " + httpget.getURI());
CloseableHttpResponse response = httpclient.execute(httpget, context);
try {
System.out.println("----------------------------------------");
System.out.println(response.getStatusLine());
System.out.println(EntityUtils.toString(response.getEntity()));
System.out.println("----------------------------------------");
// Once the request has been executed the local context can
// be used to examine updated state and various objects affected
// by the request execution.
// Last executed request
context.getRequest();
// Execution route
context.getHttpRoute();
// Target auth state
context.getTargetAuthState();
// Proxy auth state
context.getTargetAuthState();
// Cookie origin
context.getCookieOrigin();
// Cookie spec used
context.getCookieSpec();
// User security token
context.getUserToken();
} finally {
response.close();
}
} finally {
httpclient.close();
}
}