private org.apache.commons.httpclient.HttpClient createHttpClient() { HttpClientParams clientParams = new HttpClientParams(); clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); org.apache.commons.httpclient.HttpClient httpClient = new org.apache.commons.httpclient.HttpClient(clientParams, connManager); return httpClient; }
public HttpClient(int maxConPerHost, int conTimeOutMs, int soTimeOutMs, int maxSize) { connectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = connectionManager.getParams(); params.setDefaultMaxConnectionsPerHost(maxConPerHost); params.setConnectionTimeout(conTimeOutMs); params.setSoTimeout(soTimeOutMs); HttpClientParams clientParams = new HttpClientParams(); // 忽略cookie 避免 Cookie rejected 警告 clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); client = new org.apache.commons.httpclient.HttpClient(clientParams, connectionManager); Protocol myhttps = new Protocol("https", new MySSLSocketFactory(), 443); Protocol.registerProtocol("https", myhttps); this.maxSize = maxSize; // 支持proxy if (proxyHost != null && !proxyHost.equals("")) { client.getHostConfiguration().setProxy(proxyHost, proxyPort); client.getParams().setAuthenticationPreemptive(true); if (proxyAuthUser != null && !proxyAuthUser.equals("")) { client .getState() .setProxyCredentials( AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword)); log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword); } } }
/** * Method to make a GET HTTP connecton to the given url and return the output * * @param urlToFetch url to be connected * @return the http get response */ public String makeRequest(String urlToFetch, String region) throws IOException { String responseBody = ""; try { HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(40000); clientParams.setConnectionManagerTimeout(40000); HttpClient httpclient = new HttpClient(clientParams); GetMethod httpget = new GetMethod(urlToFetch); if (urlToFetch.contains("linkedin.com/countserv")) { httpget.addRequestHeader("Host", "www.linkedin.com"); httpget.addRequestHeader( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:20.0) Gecko/20100101 Firefox/20.0"); httpget.addRequestHeader( "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); httpget.addRequestHeader( "Cookie", "X-LI-IDC=C1; bcookie=\"v=2&618ed56d-275f-4dcd-86cd-6a44c8421879\"; bscookie=\"v=1&201305170926410ddaf2a1-e238-4bf6-8a0a-ce8a3df25934AQFZX82SfErJuFQKMJV49JfQTvewhfzh\"; X-LI-IDC=C1"); } // httpclient.getHostConfiguration().setProxy("46.227.68.2", 3128); // Credentials cred = new UsernamePasswordCredentials("mmongoose", // "I-PHNBV9JHW6US"); // httpclient.getState().setProxyCredentials(AuthScope.ANY, cred); int i = httpclient.executeMethod(httpget); responseBody = httpget.getResponseBodyAsString(); } catch (Exception e) { l.error(e + " " + e.getMessage() + " url " + urlToFetch); } return responseBody; }
protected void construct(String legalurl) throws HTTPException { this.legalurl = legalurl; try { sessionClient = new HttpClient(connmgr); HttpClientParams clientparams = sessionClient.getParams(); // Allow (circular) redirects clientparams.setParameter(ALLOW_CIRCULAR_REDIRECTS, true); clientparams.setParameter(MAX_REDIRECTS, 25); if (globalSoTimeout > 0) setSoTimeout(globalSoTimeout); if (globalConnectionTimeout > 0) setConnectionTimeout(globalConnectionTimeout); if (globalAgent != null) setUserAgent(globalAgent); // May get overridden by setUserAgent setAuthenticationPreemptive(globalauthpreemptive); setProxy(); if (TESTING) HTTPSession.track(this); } catch (Exception e) { throw new HTTPException("url=" + legalurl, e); } }
public HttpImpl() { // Mimic behavior found in // http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html if (Validator.isNotNull(_NON_PROXY_HOSTS)) { String nonProxyHostsRegEx = _NON_PROXY_HOSTS; nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\.", "\\\\."); nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\*", ".*?"); nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\|", ")|("); nonProxyHostsRegEx = "(" + nonProxyHostsRegEx + ")"; _nonProxyHostsPattern = Pattern.compile(nonProxyHostsRegEx); } MultiThreadedHttpConnectionManager httpConnectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams httpConnectionManagerParams = httpConnectionManager.getParams(); httpConnectionManagerParams.setConnectionTimeout(_TIMEOUT); httpConnectionManagerParams.setDefaultMaxConnectionsPerHost( new Integer(_MAX_CONNECTIONS_PER_HOST)); httpConnectionManagerParams.setMaxTotalConnections(new Integer(_MAX_TOTAL_CONNECTIONS)); httpConnectionManagerParams.setSoTimeout(_TIMEOUT); _httpClient.setHttpConnectionManager(httpConnectionManager); _proxyHttpClient.setHttpConnectionManager(httpConnectionManager); if (hasProxyConfig() && Validator.isNotNull(_PROXY_USERNAME)) { List<String> authPrefs = new ArrayList<String>(); if (_PROXY_AUTH_TYPE.equals("username-password")) { _proxyCredentials = new UsernamePasswordCredentials(_PROXY_USERNAME, _PROXY_PASSWORD); authPrefs.add(AuthPolicy.BASIC); authPrefs.add(AuthPolicy.DIGEST); authPrefs.add(AuthPolicy.NTLM); } else if (_PROXY_AUTH_TYPE.equals("ntlm")) { _proxyCredentials = new NTCredentials( _PROXY_USERNAME, _PROXY_PASSWORD, _PROXY_NTLM_HOST, _PROXY_NTLM_DOMAIN); authPrefs.add(AuthPolicy.NTLM); authPrefs.add(AuthPolicy.BASIC); authPrefs.add(AuthPolicy.DIGEST); } HttpClientParams httpClientParams = _proxyHttpClient.getParams(); httpClientParams.setParameter(AuthPolicy.AUTH_SCHEME_PRIORITY, authPrefs); } }
/** * Get an instance of the HTTP client to use. * * @return HTTP client initialized with admin credentials */ protected static HttpClient getClient() { if (AbstractEscapingTest.client == null) { HttpClient adminClient = new HttpClient(); Credentials defaultcreds = new UsernamePasswordCredentials("Admin", "admin"); adminClient.getState().setCredentials(AuthScope.ANY, defaultcreds); HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(2000); HttpConnectionManagerParams connectionParams = new HttpConnectionManagerParams(); connectionParams.setConnectionTimeout(30000); adminClient.getHttpConnectionManager().setParams(connectionParams); AbstractEscapingTest.client = adminClient; } return AbstractEscapingTest.client; }
/** 初始化httpclient */ static { int connectionTimeout = 30000; int soTimeout = 30000; try { connectionTimeout = Integer.parseInt(System.getProperty("sun.net.client.defaultConnectTimeout", "30000")); } catch (Exception e) { } try { soTimeout = Integer.parseInt(System.getProperty("sun.net.client.defaultReadTimeout", "30000")); } catch (Exception e) { } MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); connectionManager.getParams().setDefaultMaxConnectionsPerHost(10); connectionManager.getParams().setMaxTotalConnections(300); connectionManager.getParams().setConnectionTimeout(connectionTimeout); connectionManager.getParams().setSoTimeout(soTimeout); client.setHttpConnectionManager(connectionManager); // 忽略cookie 避免 Cookie rejected 警告 HttpClientParams clientParams = new HttpClientParams(); clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES); client.setParams(clientParams); // 支持https Protocol myhttps = new Protocol("https", new SSLSocketFactory(), 443); Protocol.registerProtocol("https", myhttps); // 设置代理 if (ProxyClient.getProxy() != null) { client.getHostConfiguration().setProxy(ProxyClient.getHost(), ProxyClient.getPort()); client.getParams().setAuthenticationPreemptive(true); if (ProxyClient.getUsername() != null && !ProxyClient.getUsername().trim().equals("")) { client .getState() .setProxyCredentials( AuthScope.ANY, new UsernamePasswordCredentials( ProxyClient.getUsername().trim(), ProxyClient.getPassword().trim())); } } }
private HttpClient createHttpClient() { // see http://hc.apache.org/httpclient-3.x/threading.html MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams params = connectionManager.getParams(); params.setMaxConnectionsPerHost( HostConfiguration.ANY_HOST_CONFIGURATION, PreferenceHelper.getInt(ServerConstants.HTTP_MAX_CONN_HOST_CONF_KEY, 50)); params.setMaxTotalConnections( PreferenceHelper.getInt(ServerConstants.HTTP_MAX_CONN_TOTAL_CONF_KEY, 150)); params.setConnectionTimeout( PreferenceHelper.getInt(ServerConstants.HTTP_CONN_TIMEOUT_CONF_KEY, 15000)); // 15s params.setSoTimeout( PreferenceHelper.getInt(ServerConstants.HTTP_SO_TIMEOUT_CONF_KEY, 30000)); // 30s connectionManager.setParams(params); HttpClientParams clientParams = new HttpClientParams(); clientParams.setConnectionManagerTimeout( PreferenceHelper.getInt( ServerConstants.HTTP_CONN_MGR_TIMEOUT_CONF_KEY, 300000)); // 5 minutes return new HttpClient(clientParams, connectionManager); }
/** * Creates a new connection to the server. * * @param builder The HttpFileSystemConfigBuilder. * @param scheme The protocol. * @param hostname The hostname. * @param port The port number. * @param username The username. * @param password The password * @param fileSystemOptions The file system options. * @return a new HttpClient connection. * @throws FileSystemException if an error occurs. * @since 2.0 */ public static HttpClient createConnection( HttpFileSystemConfigBuilder builder, String scheme, String hostname, int port, String username, String password, FileSystemOptions fileSystemOptions) throws FileSystemException { HttpClient client; try { HttpConnectionManager mgr = new MultiThreadedHttpConnectionManager(); HttpConnectionManagerParams connectionMgrParams = mgr.getParams(); client = new HttpClient(mgr); final HostConfiguration config = new HostConfiguration(); config.setHost(hostname, port, scheme); if (fileSystemOptions != null) { String proxyHost = builder.getProxyHost(fileSystemOptions); int proxyPort = builder.getProxyPort(fileSystemOptions); if (proxyHost != null && proxyHost.length() > 0 && proxyPort > 0) { config.setProxy(proxyHost, proxyPort); } UserAuthenticator proxyAuth = builder.getProxyAuthenticator(fileSystemOptions); if (proxyAuth != null) { UserAuthenticationData authData = UserAuthenticatorUtils.authenticate( proxyAuth, new UserAuthenticationData.Type[] { UserAuthenticationData.USERNAME, UserAuthenticationData.PASSWORD }); if (authData != null) { final UsernamePasswordCredentials proxyCreds = new UsernamePasswordCredentials( UserAuthenticatorUtils.toString( UserAuthenticatorUtils.getData( authData, UserAuthenticationData.USERNAME, null)), UserAuthenticatorUtils.toString( UserAuthenticatorUtils.getData( authData, UserAuthenticationData.PASSWORD, null))); AuthScope scope = new AuthScope(proxyHost, AuthScope.ANY_PORT); client.getState().setProxyCredentials(scope, proxyCreds); } if (builder.isPreemptiveAuth(fileSystemOptions)) { HttpClientParams httpClientParams = new HttpClientParams(); httpClientParams.setAuthenticationPreemptive(true); client.setParams(httpClientParams); } } Cookie[] cookies = builder.getCookies(fileSystemOptions); if (cookies != null) { client.getState().addCookies(cookies); } } /** * ConnectionManager set methodsmust be called after the host & port and proxy host & port are * set in the HostConfiguration. They are all used as part of the key when * HttpConnectionManagerParams tries to locate the host configuration. */ connectionMgrParams.setMaxConnectionsPerHost( config, builder.getMaxConnectionsPerHost(fileSystemOptions)); connectionMgrParams.setMaxTotalConnections(builder.getMaxTotalConnections(fileSystemOptions)); client.setHostConfiguration(config); if (username != null) { final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(username, password); AuthScope scope = new AuthScope(hostname, AuthScope.ANY_PORT); client.getState().setCredentials(scope, creds); } client.executeMethod(new HeadMethod()); } catch (final Exception exc) { throw new FileSystemException( "vfs.provider.http/connect.error", new Object[] {hostname}, exc); } return client; }
/** * The method reads a proxy object from database, makes a GetMethod object, appends required * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the * pagesource to the caller. * * @param iCount Counter variable for passing thread group information * @param url Url to fetch the pagesource for * @param followRedirect Boolean variable to specify GetMethod followRedirect value * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value * @param region the local region of a given url * @param objProxyDao the database layer ProxyDao object variable * @param useErrsy Boolean variable to specify usage of Errsy as proxy source * @return String */ public String getPageSourceWithProxy( String url, boolean followRedirect, boolean doAuthentication, String region, Boolean useErrsy, String google) { String page = " "; String pageSource = ""; int i = 0; String exception = " "; HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(40000); clientParams.setConnectionManagerTimeout(40000); HttpClient httpclient = new HttpClient(clientParams); GetMethod getmethod = null; HttpState state = new HttpState(); // if (ProxyDao.lstProxyData.size() == 16) { ProxyData objProxyData = null; // if (!useErrsy) { try { // objProxyData = ProxyDao.lstProxyData.get(iCount); objProxyData = ProxyDao.objProxyData; if (objProxyData == null) { // objProxyDao.changeProxy(google); objProxyData = ProxyDao.objProxyData; } httpclient .getHostConfiguration() .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo()); } catch (Exception e) { pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; return pageSource; } /*} else { try { objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0); Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()); httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds); httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo()); state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword())); httpclient.setState(state); } catch (Exception e) { pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; return pageSource; } }*/ try { getmethod = new GetMethod(url); getmethod.addRequestHeader( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0"); if (url.contains("bing.com")) { if (region.equalsIgnoreCase("co.uk")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.sg")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.au")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("co.in")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("ca")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.ph")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.my")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("it")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } } getmethod.setFollowRedirects(true); getmethod.setDoAuthentication(true); httpclient.getParams().setAuthenticationPreemptive(true); httpclient.setState(state); String num100Header = ""; // if (url.contains("google")) { // int j = 0; // String url1 = "http://www.google.com/"; // try { // GetMethod objGetMethod = new GetMethod(url1); // j = httpclient.executeMethod(objGetMethod); // Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie"); // String header = responseHeader.getValue(); // String[] headerValue = header.split(";"); // // for (String head : headerValue) { // if (head.contains("PREF=ID")) { // header = head; // break; // } // } // String[] splitAll = header.split(":"); // long time = System.currentTimeMillis()+400; // String sTime = "" + time; // sTime = sTime.substring(0, 10); // //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] + // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4]; // num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] + // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4]; // Cookie ck = new Cookie("PREF", "PREF", num100Header); // httpclient.getState().clearCookies(); // httpclient.getState().addCookie(ck); // getmethod.addRequestHeader("Host", "www.google.com"); // getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; // rv:19.0) Gecko/20100101 Firefox/19.0"); // getmethod.addRequestHeader("Accept", // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); // getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5"); // getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate"); // getmethod.addRequestHeader("Referer", "https://www.google.com/"); // System.out.println(num100Header); // } catch (Exception ex) { // exception = ex.getMessage(); // l.debug(ex + " " + ex.getMessage() + "Exception occured for url" + // url); // pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1; // return pageSource; // } // } i = httpclient.executeMethod(getmethod); if (i / 100 == 4 || i / 100 == 5) { page = "<PROXY ERROR>"; } else { page = getmethod.getResponseBodyAsString(); } } catch (SocketTimeoutException ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } catch (SocketException ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } catch (Exception ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } finally { getmethod.releaseConnection(); } pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; // } return pageSource; }
public void setMaxRedirects(int n) { HttpClientParams clientparams = sessionClient.getParams(); clientparams.setParameter(MAX_REDIRECTS, n); }