private HostConfiguration getHostConfiguration(HttpClient client, Map<String, Object> props) { Object proxy = props.get(ApacheHttpClientConfig.PROPERTY_PROXY_URI); if (proxy != null) { URI proxyUri = getProxyUri(proxy); String proxyHost = proxyUri.getHost(); if (proxyHost == null) { proxyHost = "localhost"; } int proxyPort = proxyUri.getPort(); if (proxyPort == -1) { proxyPort = 8080; } HostConfiguration hostConfig = new HostConfiguration(client.getHostConfiguration()); String setHost = hostConfig.getProxyHost(); int setPort = hostConfig.getProxyPort(); if ((setHost == null) || (!setHost.equals(proxyHost)) || (setPort == -1) || (setPort != proxyPort)) { hostConfig.setProxyHost(new ProxyHost(proxyHost, proxyPort)); } return hostConfig; } else { return null; } }
public String getContextByPostMethod(String url, List<KeyValue> params) { HttpClient client = getHttpClient(); client.getParams().setParameter("http.protocol.content-charset", this.codeing); PostMethod post = null; String result = ""; try { URL u = new URL(url); client .getHostConfiguration() .setHost( u.getHost(), u.getPort() == -1 ? u.getDefaultPort() : u.getPort(), u.getProtocol()); post = new PostMethod(u.getPath()); NameValuePair[] nvps = new NameValuePair[params.size()]; int i = 0; for (KeyValue kv : params) { nvps[i] = new NameValuePair(kv.getKey(), kv.getValue()); i++; } post.setRequestBody(nvps); client.executeMethod(post); result = post.getResponseBodyAsString(); } catch (Exception e) { throw new NetException("HttpClient catch!", e); } finally { if (post != null) post.releaseConnection(); } return result; }
/** Get or create the http client to be used to fetch all the map data. */ public HttpClient getHttpClient(URI uri) { MultiThreadedHttpConnectionManager connectionManager = getConnectionManager(); HttpClient httpClient = new HttpClient(connectionManager); // httpclient is a bit pesky about loading everything in memory... // disabling the warnings. Logger.getLogger(HttpMethodBase.class).setLevel(Level.ERROR); // configure proxies for URI ProxySelector selector = ProxySelector.getDefault(); List<Proxy> proxyList = selector.select(uri); Proxy proxy = proxyList.get(0); if (!proxy.equals(Proxy.NO_PROXY)) { InetSocketAddress socketAddress = (InetSocketAddress) proxy.address(); String hostName = socketAddress.getHostName(); int port = socketAddress.getPort(); httpClient.getHostConfiguration().setProxy(hostName, port); } for (SecurityStrategy sec : security) if (sec.matches(uri)) { sec.configure(uri, httpClient); break; } return httpClient; }
public static DAOLayerInterface getDAOLayer(ProxySettings proxySettings, ServerSession session) throws Exception { HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager()); if (proxySettings != null) { Credentials credentials = new UsernamePasswordCredentials(proxySettings.USERNAME, proxySettings.PASSWORD); AuthScope authScope = new AuthScope(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT); httpClient .getHostConfiguration() .setProxy(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT); httpClient.getState().setProxyCredentials(authScope, credentials); } try { db = (DAOLayerInterface) ServerRuntimeImpl.getDynamicProxy( new ServerSession(session.getCommandurl(), null), "dbinvoke", // commandurl "DAOLAYER", new Class<?>[] {DAOLayerInterface.class}, httpClient); } catch (Exception ex) { ex.printStackTrace(); } return db; }
public HttpClient(int maxConPerHost, int conTimeOutMs, int soTimeOutMs, int maxSize) { connectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = connectionManager.getParams(); params.setDefaultMaxConnectionsPerHost(maxConPerHost); params.setConnectionTimeout(conTimeOutMs); params.setSoTimeout(soTimeOutMs); HttpClientParams clientParams = new HttpClientParams(); // 忽略cookie 避免 Cookie rejected 警告 clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); client = new org.apache.commons.httpclient.HttpClient(clientParams, connectionManager); Protocol myhttps = new Protocol("https", new MySSLSocketFactory(), 443); Protocol.registerProtocol("https", myhttps); this.maxSize = maxSize; // 支持proxy if (proxyHost != null && !proxyHost.equals("")) { client.getHostConfiguration().setProxy(proxyHost, proxyPort); client.getParams().setAuthenticationPreemptive(true); if (proxyAuthUser != null && !proxyAuthUser.equals("")) { client .getState() .setProxyCredentials( AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword)); log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword); } } }
/** Create a new Client. The client will not use authentication by default. */ public Client() { client = new HttpClient(); client.getParams().setParameter("http.socket.timeout", Integer.valueOf(DEFAULT_TIMEOUT)); log.debug("proxy host: " + client.getHostConfiguration().getProxyHost()); log.debug("proxy port: " + client.getHostConfiguration().getProxyPort()); doAuthentication = false; }
@NotNull private static HttpClient getHttpClient( @Nullable final String login, @Nullable final String password) { final HttpClient client = new HttpClient(); HttpConnectionManagerParams params = client.getHttpConnectionManager().getParams(); params.setConnectionTimeout( CONNECTION_TIMEOUT); // set connection timeout (how long it takes to connect to remote host) params.setSoTimeout( CONNECTION_TIMEOUT); // set socket timeout (how long it takes to retrieve data from remote // host) client.getParams().setContentCharset("UTF-8"); // Configure proxySettings if it is required final HttpConfigurable proxySettings = HttpConfigurable.getInstance(); if (proxySettings.USE_HTTP_PROXY && !StringUtil.isEmptyOrSpaces(proxySettings.PROXY_HOST)) { client.getHostConfiguration().setProxy(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT); if (proxySettings.PROXY_AUTHENTICATION) { client .getState() .setProxyCredentials( AuthScope.ANY, new UsernamePasswordCredentials( proxySettings.PROXY_LOGIN, proxySettings.getPlainProxyPassword())); } } if (login != null && password != null) { client.getParams().setCredentialCharset("UTF-8"); client.getParams().setAuthenticationPreemptive(true); client .getState() .setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(login, password)); } return client; }
public Response multPartURL( String url, PostParameter[] params, ImageItem item, boolean authenticated) throws WeiboException { PostMethod post = new PostMethod(url); try { org.apache.commons.httpclient.HttpClient client = getHttpClient(); long t = System.currentTimeMillis(); Part[] parts = null; if (params == null) { parts = new Part[1]; } else { parts = new Part[params.length + 1]; } if (params != null) { int i = 0; for (PostParameter entry : params) { parts[i++] = new StringPart(entry.getName(), (String) entry.getValue()); } parts[parts.length - 1] = new ByteArrayPart(item.getContent(), item.getName(), item.getContentType()); } post.setRequestEntity(new MultipartRequestEntity(parts, post.getParams())); List<Header> headers = new ArrayList<Header>(); if (authenticated) { if (oauth == null) {} String authorization = null; if (null != oauth) { // use OAuth authorization = oauth.generateAuthorizationHeader("POST", url, params, oauthToken); } else { throw new IllegalStateException( "Neither user ID/password combination nor OAuth consumer key/secret combination supplied"); } headers.add(new Header("Authorization", authorization)); log("Authorization: " + authorization); } client.getHostConfiguration().getParams().setParameter("http.default-headers", headers); client.executeMethod(post); Response response = new Response(); response.setResponseAsString(post.getResponseBodyAsString()); response.setStatusCode(post.getStatusCode()); log( "multPartURL URL:" + url + ", result:" + response + ", time:" + (System.currentTimeMillis() - t)); return response; } catch (Exception ex) { throw new WeiboException(ex.getMessage(), ex, -1); } finally { post.releaseConnection(); } }
public static void setProxy(HttpClient httpClient) { String proxyHost = getValueStr(proxy_host); Logger.info("proxy_host:" + proxyHost); if (proxyHost != null) { int proxyPort = getValueInt(proxy_port); Logger.info(" proxy_port:" + proxyPort); httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort); } }
public Response httpRequest(HttpMethod method, Boolean WithTokenHeader, String token) throws WeiboException { InetAddress ipaddr; int responseCode = -1; try { ipaddr = InetAddress.getLocalHost(); List<Header> headers = new ArrayList<Header>(); if (WithTokenHeader) { if (token == null) { throw new IllegalStateException("Oauth2 token is not set!"); } headers.add(new Header("Authorization", "OAuth2 " + token)); headers.add(new Header("API-RemoteIP", ipaddr.getHostAddress())); client.getHostConfiguration().getParams().setParameter("http.default-headers", headers); for (Header hd : headers) { log(hd.getName() + ": " + hd.getValue()); } } method .getParams() .setParameter( HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); client.executeMethod(method); Header[] resHeader = method.getResponseHeaders(); responseCode = method.getStatusCode(); log("Response:"); log("https StatusCode:" + String.valueOf(responseCode)); for (Header header : resHeader) { log(header.getName() + ":" + header.getValue()); } Response response = new Response(); response.setResponseAsString(method.getResponseBodyAsString()); log(response.toString() + "\n"); if (responseCode != OK) { try { throw new WeiboException( getCause(responseCode), response.asJSONObject(), method.getStatusCode()); } catch (JSONException e) { e.printStackTrace(); } } return response; } catch (IOException ioe) { throw new WeiboException(ioe.getMessage(), ioe, responseCode); } finally { method.releaseConnection(); } }
public RestaurantRestClient(String urlStub) { httpClient.getHttpConnectionManager().getParams().setMaxTotalConnections(10); httpClient .getHttpConnectionManager() .getParams() .setMaxConnectionsPerHost(httpClient.getHostConfiguration(), 10); restClient.setErrorHandler(new MyErrorHandler()); this.urlStub = urlStub; // List<HttpMessageConverter<?>> converters = new ArrayList<>(); // converters.add(converter); // restClient.setMessageConverters(converters); // http://localhost:8888/app/backbone/restaurant/ }
@Test public void testSendSMS() { String reqUrl = "http://59.36.98.25/JSMSWebServiceNew/send.aspx"; long start = System.currentTimeMillis(); // String requestJson = // "{\"code\":\"GDCN\",\"destPhone\":\"15018784308\",\"content\":\"文字短信测试\",\"corpSignName\":\"公司签名\",\"userSignName\":\"用户签名\",\"corpAccount\":\"system\",\"userAccount\":\"admin\",\"password\":\"7236606dcece6d04ebd8a7b42eb247e0\",\"messageType\":\"1\"}"; try { HttpClient client = new HttpClient(); client.getHostConfiguration().setHost("localhost", 8080, "http"); PostMethod post = new PostMethod(reqUrl); post.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8"); post.setParameter("code", "GDCN"); post.setParameter("destPhone", "15018784308"); post.setParameter( "content", "尽力总有进步,尽心总有收获,努力总有成效,付出总有回报,前行总有风景,坚持总有希望,豁达总有快乐,拼搏总有运气。有时闲,有时累,不怨不悔,把快乐品味,愿你时时与幸福相会!"); post.setParameter("corpSignName", "中数通"); // post.setParameter("userSignName", "用户签名"); post.setParameter("corpAccount", "system"); post.setParameter("userAccount", "admin"); post.setParameter("password", "7236606dcece6d04ebd8a7b42eb247e0"); post.setParameter("messageType", "1"); /*String y=URLEncoder.encode("尽力总有进步,尽心总有收获,努力总有成效,付出总有回报,前行总有风景,坚持总有希望,豁达总有快乐,拼搏总有运气。有时闲,有时累,不怨不悔,把快乐品味,愿你时时与幸福相会!","utf-8"); System.out.println(y); String s=URLDecoder.decode("%e4%b8%8b%e8%a1%8c%e6%b6%88%e6%81%af", "utf-8"); System.out.println(s);*/ // System.out.println("请求Json: " + requestJson); // post.setRequestBody(new ByteArrayInputStream(requestJson))); /*System.out.println("请求Json的getBytes: " + EncodeUtils .base64Encode(requestJson.getBytes("UTF-8")).getBytes());*/ int result = client.executeMethod(post); if (result == 200) { String responseJson = post.getResponseBodyAsString(); System.out.println("响应JSON: " + responseJson); } } catch (Exception e) { e.printStackTrace(); } long end = System.currentTimeMillis(); System.out.println("zhixingshijian:>>>>>>>>>" + (end - start)); }
/** * Obtem a timeline publica. Nao necessita de autenticacao, basicamente é um GET na URL * http://twitter.com/statuses/public_timeline.xml */ public String getPublicTimeline() throws Exception { HttpClient client = new HttpClient(); // get para obter a timeline publica GetMethod get = new GetMethod("/statuses/public_timeline.xml"); // Criando o host, para onde sera enviada a requisicao HostConfiguration host = client.getHostConfiguration(); host.setHost(new URI("http://twitter.com")); // executando o get na url de timeline publica do twitter client.executeMethod(host, get); // obtendo a resposta return get.getResponseBodyAsString(); }
public static void main(String[] args) throws Exception { HttpClient client = new HttpClient(); client.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF8"); // 原先采用自己合并COOKIE,但是出现问题,用此方法OK DefaultHttpParams.getDefaultParams() .setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true); client .getParams() .setParameter( HttpMethodParams.USER_AGENT, "Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) Gecko/20090803 Fedora/3.5.2-2.fc11 Firefox/3.5.2"); client.getHostConfiguration().setHost(LOGON_SITE, LOGON_PORT); String ajaxLoginUrl = Login(client, preLogin(client)); String uniqueid = ajaxLogin(client, ajaxLoginUrl); getHomePage(client, homePageUrl + uniqueid); }
/** * Sends a new HTTP request to a server through a proxy. * * @param requestParameters a set of parameters that will set the content of the request and * specify the proxy it should go through */ public void callWilmaTestServer( final RequestParameters requestParameters, final TestClientParameters clientParameters) { try { HttpClient httpClient = new HttpClient(); PostMethod httpPost = new PostMethod(requestParameters.getTestServerUrl()); if (clientParameters.isUseProxy()) { httpClient .getHostConfiguration() .setProxy(requestParameters.getWilmaHost(), requestParameters.getWilmaPort()); } InputStream inputStream = requestParameters.getInputStream(); if (requestParameters.getContentType().contains("fastinfoset")) { inputStream = compress(inputStream); } if (requestParameters.getContentEncoding().contains("gzip")) { inputStream = encode(inputStream); httpPost.setRequestHeader("Content-Encoding", requestParameters.getContentEncoding()); } InputStreamRequestEntity entity = new InputStreamRequestEntity(inputStream, requestParameters.getContentType()); httpPost.setRequestEntity(entity); httpPost.setRequestHeader("Accept", requestParameters.getAcceptHeader()); httpPost.addRequestHeader("Accept-Encoding", requestParameters.getAcceptEncoding()); // httpPost.addRequestHeader("0", "WilmaBypass=true"); httpClient .getHttpConnectionManager() .getParams() .setSendBufferSize(clientParameters.getRequestBufferSize()); httpClient .getHttpConnectionManager() .getParams() .setReceiveBufferSize(clientParameters.getResponseBufferSize()); int statusCode = httpClient.executeMethod(httpPost); logger.info("status code: " + statusCode); if (clientParameters.getAllowResponseLogging()) { logger.info(getInputStreamAsString(httpPost.getResponseBodyAsStream())); } } catch (UnsupportedEncodingException e) { throw new SystemException("Unsupported encoding.", e); } catch (HttpException e) { throw new SystemException("Http exception occurred.", e); } catch (IOException e) { throw new SystemException("InputStream cannot be read.", e); } }
private org.apache.commons.httpclient.HttpClient getHttpClient() { org.apache.commons.httpclient.HttpClient client = new org.apache.commons.httpclient.HttpClient(); if (proxyHost != null && !proxyHost.equals("")) { client.getHostConfiguration().setProxy(proxyHost, proxyPort); client.getParams().setAuthenticationPreemptive(true); if (proxyAuthUser != null && !proxyAuthUser.equals("")) { client .getState() .setProxyCredentials( AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword)); log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword); } } return client; }
/** * 提交 * * @param method * @param token * @return * @throws ApiException */ private static String httpRequest(HttpMethod method, String token) throws IOException { try { List<Header> headers = new ArrayList<Header>(); if (token != null) { headers.add(new Header("Authorization", "Bearer " + token)); client.getHostConfiguration().getParams().setParameter("http.default-headers", headers); } method .getParams() .setParameter( HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); client.executeMethod(method); return method.getResponseBodyAsString(); } catch (IOException ioe) { throw ioe; } finally { method.releaseConnection(); } }
/** * 登录 * * @return * @throws IOException * @throws HttpException */ public static HttpClient login() throws IOException, HttpException { HttpClient client = new HttpClient(); client.getHostConfiguration().setHost("www.catarc.org.cn"); client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY); NameValuePair[] data = { new NameValuePair( "User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:25.0) Gecko/20100101 Firefox/25.0"), new NameValuePair("__VIEWSTATE", "/wEPDwUJNzE2NjczNjA4ZGQCaV1Dt0DOUg5r7O3ZD+sGL1Hq6Q=="), new NameValuePair( "__EVENTVALIDATION", "/wEWBQK/2uyqBgKl1bKzCQK1qbSRCwLZpaCZAwKTuvSuCTYE9T8nbPNCPEY2eqmCy5I3BwtG"), new NameValuePair("txtUserName", "asri"), new NameValuePair("txtPassword", "asri"), new NameValuePair("loginBtn", "%B5%C7%C2%BD") }; PostMethod post = new PostMethod("http://www.catarc.org.cn/standard/Default.aspx"); post.setRequestBody(data); client.executeMethod(post); return client; }
/** 初始化httpclient */ static { int connectionTimeout = 30000; int soTimeout = 30000; try { connectionTimeout = Integer.parseInt(System.getProperty("sun.net.client.defaultConnectTimeout", "30000")); } catch (Exception e) { } try { soTimeout = Integer.parseInt(System.getProperty("sun.net.client.defaultReadTimeout", "30000")); } catch (Exception e) { } MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); connectionManager.getParams().setDefaultMaxConnectionsPerHost(10); connectionManager.getParams().setMaxTotalConnections(300); connectionManager.getParams().setConnectionTimeout(connectionTimeout); connectionManager.getParams().setSoTimeout(soTimeout); client.setHttpConnectionManager(connectionManager); // 忽略cookie 避免 Cookie rejected 警告 HttpClientParams clientParams = new HttpClientParams(); clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); client.setParams(clientParams); // 支持https Protocol myhttps = new Protocol("https", new SSLSocketFactory(), 443); Protocol.registerProtocol("https", myhttps); // 设置代理 if (ProxyClient.getProxy() != null) { client.getHostConfiguration().setProxy(ProxyClient.getHost(), ProxyClient.getPort()); client.getParams().setAuthenticationPreemptive(true); if (ProxyClient.getUsername() != null && !ProxyClient.getUsername().trim().equals("")) { client .getState() .setProxyCredentials( AuthScope.ANY, new UsernamePasswordCredentials( ProxyClient.getUsername().trim(), ProxyClient.getPassword().trim())); } } }
private void configureHttpMethod( boolean skipContentCache, CacheData cacheData, long onceTimeOut, HttpMethod httpMethod) { if (skipContentCache && null != cacheData) { if (null != cacheData.getLastModifiedHeader() && Constants.NULL != cacheData.getLastModifiedHeader()) { httpMethod.addRequestHeader(Constants.IF_MODIFIED_SINCE, cacheData.getLastModifiedHeader()); } if (null != cacheData.getMd5() && Constants.NULL != cacheData.getMd5()) { httpMethod.addRequestHeader(Constants.CONTENT_MD5, cacheData.getMd5()); } } httpMethod.addRequestHeader(Constants.ACCEPT_ENCODING, "gzip,deflate"); HttpMethodParams params = new HttpMethodParams(); params.setSoTimeout((int) onceTimeOut); httpMethod.setParams(params); httpClient .getHostConfiguration() .setHost( diamondConfigure.getDomainNameList().get(this.domainNamePos.get()), diamondConfigure.getPort()); }
protected HttpClient getHttpClient() { HttpClient client = new HttpClient(); if (Jenkins.getInstance() != null) { ProxyConfiguration proxy = Jenkins.getInstance().proxy; if (proxy != null) { client.getHostConfiguration().setProxy(proxy.name, proxy.port); String username = proxy.getUserName(); String password = proxy.getPassword(); // Consider it to be passed if username specified. Sufficient? if (username != null && !"".equals(username.trim())) { logger.info("Using proxy authentication (user="******")"); // http://hc.apache.org/httpclient-3.x/authentication.html#Proxy_Authentication // and // http://svn.apache.org/viewvc/httpcomponents/oac.hc3x/trunk/src/examples/BasicAuthenticationExample.java?view=markup client .getState() .setProxyCredentials( AuthScope.ANY, new UsernamePasswordCredentials(username, password)); } } } return client; }
/** * @param sProxyUser * @param sProxyPassswd * @return an HTTP client */ private static HttpClient getHTTPClient( String sProxyUser, String sProxyPassswd, int iConTimeout, int iDataTimeout) { HttpClient client = new HttpClient(); client .getHttpConnectionManager() .getParams() .setConnectionTimeout(iConTimeout); // connection to client .getHttpConnectionManager() .getParams() .setSoTimeout(iDataTimeout); // data reception timeout if (sProxyUser != null && sProxyPassswd != null) { client .getHostConfiguration() .setProxy( ConfigurationManager.getProperty(CONF_NETWORK_PROXY_HOSTNAME), Integer.parseInt(ConfigurationManager.getProperty(CONF_NETWORK_PROXY_PORT))); client .getState() .setProxyCredentials( new AuthScope(AuthScope.ANY), new UsernamePasswordCredentials(sProxyUser, sProxyPwd)); } return client; }
public HttpTemplateDownloader( StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { this._storage = storageLayer; this.downloadUrl = downloadUrl; this.setToDir(toDir); this.status = TemplateDownloader.Status.NOT_STARTED; this.resourceType = resourceType; this.MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes; this.totalBytes = 0; this.client = new HttpClient(); myretryhandler = new HttpMethodRetryHandler() { public boolean retryMethod( final HttpMethod method, final IOException exception, int executionCount) { if (executionCount >= 2) { // Do not retry if over max retry count return false; } if (exception instanceof NoHttpResponseException) { // Retry if the server dropped connection on us return true; } if (!method.isRequestSent()) { // Retry if the request has not been sent fully or // if it's OK to retry methods that have been sent return true; } // otherwise do not retry return false; } }; try { this.request = new GetMethod(downloadUrl); this.request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); this.completionCallback = callback; // this.request.setFollowRedirects(false); File f = File.createTempFile("dnld", "tmp_", new File(toDir)); if (_storage != null) { _storage.setWorldReadableAndWriteable(f); } toFile = f.getAbsolutePath(); Pair<String, Integer> hostAndPort = validateUrl(downloadUrl); if (proxy != null) { client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort()); if (proxy.getUserName() != null) { Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword()); client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds); } } if ((user != null) && (password != null)) { client.getParams().setAuthenticationPreemptive(true); Credentials defaultcreds = new UsernamePasswordCredentials(user, password); client .getState() .setCredentials( new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); s_logger.info( "Added username="******", password="******"for host " + hostAndPort.first() + ":" + hostAndPort.second()); } else { s_logger.info( "No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second()); } } catch (IllegalArgumentException iae) { errorString = iae.getMessage(); status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; inited = false; } catch (Exception ex) { errorString = "Unable to start download -- check url? "; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; s_logger.warn("Exception in constructor -- " + ex.toString()); } catch (Throwable th) { s_logger.warn("throwable caught ", th); } }
/** Clear the proxy setting. */ public void clearProxy() { client.getHostConfiguration().setProxyHost(null); }
/** * Set a proxy that should be used by the client when trying to access the server. If this is not * set, the client will attempt to make a direct direct connection to the server. * * @param host The name of the host. * @param port The port. */ public void setProxy(String host, int port) { client.getHostConfiguration().setProxy(host, port); }
// 授权,生成access_token // 使用情形:①程序初始化;②每隔一天左右重新授权access_token public static void generate() { initAccountInfo(); accessToken.clear(); logger.info("用户授权中..."); try { // https://api.weibo.com/oauth2/authorize?client_id=750123511&redirect_uri=https://api.weibo.com/oauth2/default.html&response_type=code String url = "https://api.weibo.com/oauth2/authorize"; String redirectUri = "https://api.weibo.com/oauth2/default.html"; for (int i = 0; i < accountInfo.size(); i++) { // 获取应用的信息 clientId = WeiboConfig.getValue("client_ID"); clientSecret = WeiboConfig.getValue("client_SERCRET"); // 构造授权的url参数 PostMethod postMethod = new PostMethod(url); postMethod.addParameter("client_id", clientId); postMethod.addParameter("redirect_uri", redirectUri); postMethod.addParameter("userId", accountInfo.get(i).getUserId()); postMethod.addParameter("passwd", accountInfo.get(i).getPasswd()); postMethod.addParameter("isLoginSina", "0"); postMethod.addParameter("action", "submit"); postMethod.addParameter("response_type", "code"); HttpMethodParams param = postMethod.getParams(); param.setContentCharset("UTF-8"); // 伪造头部域信息 List<Header> headers = new ArrayList<Header>(); headers.add( new Header( "Referer", "https://api.weibo.com/oauth2/authorize?client_id=" + clientId + "&redirect_uri=" + redirectUri + "&from=sina&response_type=code")); headers.add(new Header("Host", "api.weibo.com")); headers.add( new Header( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0")); // 发送HTTP请求 HttpClient client = new HttpClient(); client.getHostConfiguration().getParams().setParameter("http.default-headers", headers); client.executeMethod(postMethod); // 获取授权响应 int status = postMethod.getStatusCode(); if (status == 302) { Header location = postMethod.getResponseHeader("location"); if (location != null) { String retUrl = location.getValue(); int begin = retUrl.indexOf("code="); int end = retUrl.length(); String code = retUrl.substring(begin + 5, end); if (code != null) { Oauth oauth = new Oauth(); String token = oauth.getAccessTokenByCode(code).getAccessToken(); accessToken.add(token); logger.info("第" + (i + 1) + "个access_token:" + token); } } } else { logger.error("第" + (i + 1) + "个用户授权失败了!"); } } } catch (Exception e) { e.printStackTrace(); logger.error("授权发生异常!"); } }
Set<String> checkUpdateDataIds(long timeout) { if (!isRun) { throw new RuntimeException("DiamondSubscriber is not running. checkUpdateDataIds return."); } if (MockServer.isTestMode()) { return testData(); } long waitTime = 0; String probeUpdateString = getProbeUpdateString(); if (StringUtils.isBlank(probeUpdateString)) { return null; } while (0 == timeout || timeout > waitTime) { long onceTimeOut = getOnceTimeOut(waitTime, timeout); waitTime += onceTimeOut; PostMethod postMethod = new PostMethod(Constants.HTTP_URI_FILE); postMethod.addParameter(Constants.PROBE_MODIFY_REQUEST, probeUpdateString); HttpMethodParams params = new HttpMethodParams(); params.setSoTimeout((int) onceTimeOut); postMethod.setParams(params); try { httpClient .getHostConfiguration() .setHost( diamondConfigure.getDomainNameList().get(this.domainNamePos.get()), this.diamondConfigure.getPort()); int httpStatus = httpClient.executeMethod(postMethod); switch (httpStatus) { case SC_OK: { Set<String> result = getUpdateDataIds(postMethod); return result; } case SC_SERVICE_UNAVAILABLE: { rotateToNextDomain(); } break; default: { log.warn("checkUpdateDataIds HTTP State: " + httpStatus); rotateToNextDomain(); } } } catch (HttpException e) { log.error("checkUpdateDataIds HttpException", e); rotateToNextDomain(); } catch (IOException e) { log.error("checkUpdateDataIds IOException", e); rotateToNextDomain(); } catch (Exception e) { log.error("checkUpdateDataIds Unknown Exception", e); rotateToNextDomain(); } finally { postMethod.releaseConnection(); } } throw new RuntimeException( "checkUpdateDataIds timeout " + diamondConfigure.getDomainNameList().get(this.domainNamePos.get()) + ", timeout=" + timeout); }
public byte[] timeStamp(byte[] data, RevocationData revocationData) throws Exception { // digest the message MessageDigest messageDigest = MessageDigest.getInstance(this.digestAlgo); byte[] digest = messageDigest.digest(data); // generate the TSP request BigInteger nonce = new BigInteger(128, new SecureRandom()); TimeStampRequestGenerator requestGenerator = new TimeStampRequestGenerator(); requestGenerator.setCertReq(true); if (null != this.requestPolicy) { requestGenerator.setReqPolicy(this.requestPolicy); } TimeStampRequest request = requestGenerator.generate(this.digestAlgoOid, digest, nonce); byte[] encodedRequest = request.getEncoded(); // create the HTTP client HttpClient httpClient = new HttpClient(); if (null != this.username) { Credentials credentials = new UsernamePasswordCredentials(this.username, this.password); httpClient.getState().setCredentials(AuthScope.ANY, credentials); } if (null != this.proxyHost) { httpClient.getHostConfiguration().setProxy(this.proxyHost, this.proxyPort); } // create the HTTP POST request PostMethod postMethod = new PostMethod(this.tspServiceUrl); RequestEntity requestEntity = new ByteArrayRequestEntity(encodedRequest, "application/timestamp-query"); postMethod.addRequestHeader("User-Agent", this.userAgent); postMethod.setRequestEntity(requestEntity); // invoke TSP service int statusCode = httpClient.executeMethod(postMethod); if (HttpStatus.SC_OK != statusCode) { LOG.error("Error contacting TSP server " + this.tspServiceUrl); throw new Exception("Error contacting TSP server " + this.tspServiceUrl); } // HTTP input validation Header responseContentTypeHeader = postMethod.getResponseHeader("Content-Type"); if (null == responseContentTypeHeader) { throw new RuntimeException("missing Content-Type header"); } String contentType = responseContentTypeHeader.getValue(); if (!contentType.startsWith("application/timestamp-reply")) { LOG.debug("response content: " + postMethod.getResponseBodyAsString()); throw new RuntimeException("invalid Content-Type: " + contentType); } if (0 == postMethod.getResponseContentLength()) { throw new RuntimeException("Content-Length is zero"); } // TSP response parsing and validation InputStream inputStream = postMethod.getResponseBodyAsStream(); TimeStampResponse timeStampResponse = new TimeStampResponse(inputStream); timeStampResponse.validate(request); if (0 != timeStampResponse.getStatus()) { LOG.debug("status: " + timeStampResponse.getStatus()); LOG.debug("status string: " + timeStampResponse.getStatusString()); PKIFailureInfo failInfo = timeStampResponse.getFailInfo(); if (null != failInfo) { LOG.debug("fail info int value: " + failInfo.intValue()); if (PKIFailureInfo.unacceptedPolicy == failInfo.intValue()) { LOG.debug("unaccepted policy"); } } throw new RuntimeException( "timestamp response status != 0: " + timeStampResponse.getStatus()); } TimeStampToken timeStampToken = timeStampResponse.getTimeStampToken(); SignerId signerId = timeStampToken.getSID(); BigInteger signerCertSerialNumber = signerId.getSerialNumber(); X500Principal signerCertIssuer = signerId.getIssuer(); LOG.debug("signer cert serial number: " + signerCertSerialNumber); LOG.debug("signer cert issuer: " + signerCertIssuer); // TSP signer certificates retrieval CertStore certStore = timeStampToken.getCertificatesAndCRLs("Collection", BouncyCastleProvider.PROVIDER_NAME); Collection<? extends Certificate> certificates = certStore.getCertificates(null); X509Certificate signerCert = null; Map<String, X509Certificate> certificateMap = new HashMap<String, X509Certificate>(); for (Certificate certificate : certificates) { X509Certificate x509Certificate = (X509Certificate) certificate; if (signerCertIssuer.equals(x509Certificate.getIssuerX500Principal()) && signerCertSerialNumber.equals(x509Certificate.getSerialNumber())) { signerCert = x509Certificate; } String ski = Hex.encodeHexString(getSubjectKeyId(x509Certificate)); certificateMap.put(ski, x509Certificate); LOG.debug( "embedded certificate: " + x509Certificate.getSubjectX500Principal() + "; SKI=" + ski); } // TSP signer cert path building if (null == signerCert) { throw new RuntimeException("TSP response token has no signer certificate"); } List<X509Certificate> tspCertificateChain = new LinkedList<X509Certificate>(); X509Certificate certificate = signerCert; do { LOG.debug("adding to certificate chain: " + certificate.getSubjectX500Principal()); tspCertificateChain.add(certificate); if (certificate.getSubjectX500Principal().equals(certificate.getIssuerX500Principal())) { break; } String aki = Hex.encodeHexString(getAuthorityKeyId(certificate)); certificate = certificateMap.get(aki); } while (null != certificate); // verify TSP signer signature timeStampToken.validate(tspCertificateChain.get(0), BouncyCastleProvider.PROVIDER_NAME); // verify TSP signer certificate this.validator.validate(tspCertificateChain, revocationData); LOG.debug("time-stamp token time: " + timeStampToken.getTimeStampInfo().getGenTime()); byte[] timestamp = timeStampToken.getEncoded(); return timestamp; }
@Override public BatchHttpResult getConfigureInformationBatch( List<String> dataIds, String group, int timeout) { if (dataIds == null) { log.error("dataId list cannot be null,group=" + group); return new BatchHttpResult(HttpStatus.SC_BAD_REQUEST); } if (group == null) { group = Constants.DEFAULT_GROUP; } StringBuilder dataIdBuilder = new StringBuilder(); for (String dataId : dataIds) { dataIdBuilder.append(dataId).append(Constants.LINE_SEPARATOR); } String dataIdStr = dataIdBuilder.toString(); PostMethod post = new PostMethod(Constants.HTTP_URI_FILE_BATCH); post.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, timeout); BatchHttpResult response = null; try { NameValuePair dataIdValue = new NameValuePair("dataIds", dataIdStr); NameValuePair groupValue = new NameValuePair("group", group); post.setRequestBody(new NameValuePair[] {dataIdValue, groupValue}); httpClient .getHostConfiguration() .setHost( diamondConfigure.getDomainNameList().get(this.domainNamePos.get()), this.diamondConfigure.getPort()); int status = httpClient.executeMethod(post); String responseMsg = post.getResponseBodyAsString(); if (status == HttpStatus.SC_OK) { String json = null; try { json = responseMsg; List<ConfigInfoEx> configInfoExList = new LinkedList<ConfigInfoEx>(); Object resultObj = JSONUtils.deserializeObject(json, new TypeReference<List<ConfigInfoEx>>() {}); if (!(resultObj instanceof List<?>)) { throw new RuntimeException( "batch query deserialize type error, not list, json=" + json); } List<ConfigInfoEx> resultList = (List<ConfigInfoEx>) resultObj; for (ConfigInfoEx configInfoEx : resultList) { configInfoExList.add(configInfoEx); } response = new BatchHttpResult(configInfoExList); log.info( "batch query success,dataIds=" + dataIdStr + ",group=" + group + ",json=" + json); } catch (Exception e) { response = new BatchHttpResult(Constants.BATCH_OP_ERROR); log.error( "batch query deserialize error,dataIdStr=" + dataIdStr + ",group=" + group + ",json=" + json, e); } } else if (status == HttpStatus.SC_REQUEST_TIMEOUT) { response = new BatchHttpResult(HttpStatus.SC_REQUEST_TIMEOUT); log.error( "batch query timeout, socket timeout(ms):" + timeout + ",dataIds=" + dataIdStr + ",group=" + group); } else { response = new BatchHttpResult(status); log.error( "batch query fail, status:" + status + ", response:" + responseMsg + ",dataIds=" + dataIdStr + ",group=" + group); } } catch (HttpException e) { response = new BatchHttpResult(Constants.BATCH_HTTP_EXCEPTION); log.error("batch query http exception,dataIds=" + dataIdStr + ",group=" + group, e); } catch (IOException e) { response = new BatchHttpResult(Constants.BATCH_IO_EXCEPTION); log.error("batch query io exception, dataIds=" + dataIdStr + ",group=" + group, e); } finally { post.releaseConnection(); } return response; }
/** * The method reads a proxy object from database, makes a GetMethod object, appends required * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the * pagesource to the caller. * * @param iCount Counter variable for passing thread group information * @param url Url to fetch the pagesource for * @param followRedirect Boolean variable to specify GetMethod followRedirect value * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value * @param region the local region of a given url * @param objProxyDao the database layer ProxyDao object variable * @param useErrsy Boolean variable to specify usage of Errsy as proxy source * @return String */ public String getPageSourceWithProxy( String url, boolean followRedirect, boolean doAuthentication, String region, Boolean useErrsy, String google) { String page = " "; String pageSource = ""; int i = 0; String exception = " "; HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(40000); clientParams.setConnectionManagerTimeout(40000); HttpClient httpclient = new HttpClient(clientParams); GetMethod getmethod = null; HttpState state = new HttpState(); // if (ProxyDao.lstProxyData.size() == 16) { ProxyData objProxyData = null; // if (!useErrsy) { try { // objProxyData = ProxyDao.lstProxyData.get(iCount); objProxyData = ProxyDao.objProxyData; if (objProxyData == null) { // objProxyDao.changeProxy(google); objProxyData = ProxyDao.objProxyData; } httpclient .getHostConfiguration() .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo()); } catch (Exception e) { pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; return pageSource; } /*} else { try { objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0); Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()); httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds); httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo()); state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword())); httpclient.setState(state); } catch (Exception e) { pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; return pageSource; } }*/ try { getmethod = new GetMethod(url); getmethod.addRequestHeader( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0"); if (url.contains("bing.com")) { if (region.equalsIgnoreCase("co.uk")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.sg")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.au")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("co.in")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("ca")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.ph")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.my")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("it")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } } getmethod.setFollowRedirects(true); getmethod.setDoAuthentication(true); httpclient.getParams().setAuthenticationPreemptive(true); httpclient.setState(state); String num100Header = ""; // if (url.contains("google")) { // int j = 0; // String url1 = "http://www.google.com/"; // try { // GetMethod objGetMethod = new GetMethod(url1); // j = httpclient.executeMethod(objGetMethod); // Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie"); // String header = responseHeader.getValue(); // String[] headerValue = header.split(";"); // // for (String head : headerValue) { // if (head.contains("PREF=ID")) { // header = head; // break; // } // } // String[] splitAll = header.split(":"); // long time = System.currentTimeMillis()+400; // String sTime = "" + time; // sTime = sTime.substring(0, 10); // //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] + // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4]; // num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] + // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4]; // Cookie ck = new Cookie("PREF", "PREF", num100Header); // httpclient.getState().clearCookies(); // httpclient.getState().addCookie(ck); // getmethod.addRequestHeader("Host", "www.google.com"); // getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; // rv:19.0) Gecko/20100101 Firefox/19.0"); // getmethod.addRequestHeader("Accept", // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); // getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5"); // getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate"); // getmethod.addRequestHeader("Referer", "https://www.google.com/"); // System.out.println(num100Header); // } catch (Exception ex) { // exception = ex.getMessage(); // l.debug(ex + " " + ex.getMessage() + "Exception occured for url" + // url); // pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1; // return pageSource; // } // } i = httpclient.executeMethod(getmethod); if (i / 100 == 4 || i / 100 == 5) { page = "<PROXY ERROR>"; } else { page = getmethod.getResponseBodyAsString(); } } catch (SocketTimeoutException ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } catch (SocketException ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } catch (Exception ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } finally { getmethod.releaseConnection(); } pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; // } return pageSource; }