private org.apache.commons.httpclient.HttpClient createHttpClient() {
HttpClientParams clientParams = new HttpClientParams();
clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
org.apache.commons.httpclient.HttpClient httpClient =
new org.apache.commons.httpclient.HttpClient(clientParams, connManager);
return httpClient;
}
public HttpClient(int maxConPerHost, int conTimeOutMs, int soTimeOutMs, int maxSize) {
connectionManager = new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams params = connectionManager.getParams();
params.setDefaultMaxConnectionsPerHost(maxConPerHost);
params.setConnectionTimeout(conTimeOutMs);
params.setSoTimeout(soTimeOutMs);
HttpClientParams clientParams = new HttpClientParams();
// 忽略cookie 避免 Cookie rejected 警告
clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
client = new org.apache.commons.httpclient.HttpClient(clientParams, connectionManager);
Protocol myhttps = new Protocol("https", new MySSLSocketFactory(), 443);
Protocol.registerProtocol("https", myhttps);
this.maxSize = maxSize;
// 支持proxy
if (proxyHost != null && !proxyHost.equals("")) {
client.getHostConfiguration().setProxy(proxyHost, proxyPort);
client.getParams().setAuthenticationPreemptive(true);
if (proxyAuthUser != null && !proxyAuthUser.equals("")) {
client
.getState()
.setProxyCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword));
log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword);
}
}
}
/**
* Method to make a GET HTTP connecton to the given url and return the output
*
* @param urlToFetch url to be connected
* @return the http get response
*/
public String makeRequest(String urlToFetch, String region) throws IOException {
String responseBody = "";
try {
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod httpget = new GetMethod(urlToFetch);
if (urlToFetch.contains("linkedin.com/countserv")) {
httpget.addRequestHeader("Host", "www.linkedin.com");
httpget.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:20.0) Gecko/20100101 Firefox/20.0");
httpget.addRequestHeader(
"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
httpget.addRequestHeader(
"Cookie",
"X-LI-IDC=C1; bcookie=\"v=2&618ed56d-275f-4dcd-86cd-6a44c8421879\"; bscookie=\"v=1&201305170926410ddaf2a1-e238-4bf6-8a0a-ce8a3df25934AQFZX82SfErJuFQKMJV49JfQTvewhfzh\"; X-LI-IDC=C1");
}
// httpclient.getHostConfiguration().setProxy("46.227.68.2", 3128);
// Credentials cred = new UsernamePasswordCredentials("mmongoose",
// "I-PHNBV9JHW6US");
// httpclient.getState().setProxyCredentials(AuthScope.ANY, cred);
int i = httpclient.executeMethod(httpget);
responseBody = httpget.getResponseBodyAsString();
} catch (Exception e) {
l.error(e + " " + e.getMessage() + " url " + urlToFetch);
}
return responseBody;
}
protected void construct(String legalurl) throws HTTPException {
this.legalurl = legalurl;
try {
sessionClient = new HttpClient(connmgr);
HttpClientParams clientparams = sessionClient.getParams();
// Allow (circular) redirects
clientparams.setParameter(ALLOW_CIRCULAR_REDIRECTS, true);
clientparams.setParameter(MAX_REDIRECTS, 25);
if (globalSoTimeout > 0) setSoTimeout(globalSoTimeout);
if (globalConnectionTimeout > 0) setConnectionTimeout(globalConnectionTimeout);
if (globalAgent != null) setUserAgent(globalAgent); // May get overridden by setUserAgent
setAuthenticationPreemptive(globalauthpreemptive);
setProxy();
if (TESTING) HTTPSession.track(this);
} catch (Exception e) {
throw new HTTPException("url=" + legalurl, e);
}
}
public HttpImpl() {
// Mimic behavior found in
// http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html
if (Validator.isNotNull(_NON_PROXY_HOSTS)) {
String nonProxyHostsRegEx = _NON_PROXY_HOSTS;
nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\.", "\\\\.");
nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\*", ".*?");
nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\|", ")|(");
nonProxyHostsRegEx = "(" + nonProxyHostsRegEx + ")";
_nonProxyHostsPattern = Pattern.compile(nonProxyHostsRegEx);
}
MultiThreadedHttpConnectionManager httpConnectionManager =
new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams httpConnectionManagerParams = httpConnectionManager.getParams();
httpConnectionManagerParams.setConnectionTimeout(_TIMEOUT);
httpConnectionManagerParams.setDefaultMaxConnectionsPerHost(
new Integer(_MAX_CONNECTIONS_PER_HOST));
httpConnectionManagerParams.setMaxTotalConnections(new Integer(_MAX_TOTAL_CONNECTIONS));
httpConnectionManagerParams.setSoTimeout(_TIMEOUT);
_httpClient.setHttpConnectionManager(httpConnectionManager);
_proxyHttpClient.setHttpConnectionManager(httpConnectionManager);
if (hasProxyConfig() && Validator.isNotNull(_PROXY_USERNAME)) {
List<String> authPrefs = new ArrayList<String>();
if (_PROXY_AUTH_TYPE.equals("username-password")) {
_proxyCredentials = new UsernamePasswordCredentials(_PROXY_USERNAME, _PROXY_PASSWORD);
authPrefs.add(AuthPolicy.BASIC);
authPrefs.add(AuthPolicy.DIGEST);
authPrefs.add(AuthPolicy.NTLM);
} else if (_PROXY_AUTH_TYPE.equals("ntlm")) {
_proxyCredentials =
new NTCredentials(
_PROXY_USERNAME, _PROXY_PASSWORD, _PROXY_NTLM_HOST, _PROXY_NTLM_DOMAIN);
authPrefs.add(AuthPolicy.NTLM);
authPrefs.add(AuthPolicy.BASIC);
authPrefs.add(AuthPolicy.DIGEST);
}
HttpClientParams httpClientParams = _proxyHttpClient.getParams();
httpClientParams.setParameter(AuthPolicy.AUTH_SCHEME_PRIORITY, authPrefs);
}
}
/**
* Get an instance of the HTTP client to use.
*
* @return HTTP client initialized with admin credentials
*/
protected static HttpClient getClient() {
if (AbstractEscapingTest.client == null) {
HttpClient adminClient = new HttpClient();
Credentials defaultcreds = new UsernamePasswordCredentials("Admin", "admin");
adminClient.getState().setCredentials(AuthScope.ANY, defaultcreds);
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(2000);
HttpConnectionManagerParams connectionParams = new HttpConnectionManagerParams();
connectionParams.setConnectionTimeout(30000);
adminClient.getHttpConnectionManager().setParams(connectionParams);
AbstractEscapingTest.client = adminClient;
}
return AbstractEscapingTest.client;
}
/** 初始化httpclient */
static {
int connectionTimeout = 30000;
int soTimeout = 30000;
try {
connectionTimeout =
Integer.parseInt(System.getProperty("sun.net.client.defaultConnectTimeout", "30000"));
} catch (Exception e) {
}
try {
soTimeout =
Integer.parseInt(System.getProperty("sun.net.client.defaultReadTimeout", "30000"));
} catch (Exception e) {
}
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
connectionManager.getParams().setDefaultMaxConnectionsPerHost(10);
connectionManager.getParams().setMaxTotalConnections(300);
connectionManager.getParams().setConnectionTimeout(connectionTimeout);
connectionManager.getParams().setSoTimeout(soTimeout);
client.setHttpConnectionManager(connectionManager);
// 忽略cookie 避免 Cookie rejected 警告
HttpClientParams clientParams = new HttpClientParams();
clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
client.setParams(clientParams);
// 支持https
Protocol myhttps = new Protocol("https", new SSLSocketFactory(), 443);
Protocol.registerProtocol("https", myhttps);
// 设置代理
if (ProxyClient.getProxy() != null) {
client.getHostConfiguration().setProxy(ProxyClient.getHost(), ProxyClient.getPort());
client.getParams().setAuthenticationPreemptive(true);
if (ProxyClient.getUsername() != null && !ProxyClient.getUsername().trim().equals("")) {
client
.getState()
.setProxyCredentials(
AuthScope.ANY,
new UsernamePasswordCredentials(
ProxyClient.getUsername().trim(), ProxyClient.getPassword().trim()));
}
}
}
private HttpClient createHttpClient() {
// see http://hc.apache.org/httpclient-3.x/threading.html
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams params = connectionManager.getParams();
params.setMaxConnectionsPerHost(
HostConfiguration.ANY_HOST_CONFIGURATION,
PreferenceHelper.getInt(ServerConstants.HTTP_MAX_CONN_HOST_CONF_KEY, 50));
params.setMaxTotalConnections(
PreferenceHelper.getInt(ServerConstants.HTTP_MAX_CONN_TOTAL_CONF_KEY, 150));
params.setConnectionTimeout(
PreferenceHelper.getInt(ServerConstants.HTTP_CONN_TIMEOUT_CONF_KEY, 15000)); // 15s
params.setSoTimeout(
PreferenceHelper.getInt(ServerConstants.HTTP_SO_TIMEOUT_CONF_KEY, 30000)); // 30s
connectionManager.setParams(params);
HttpClientParams clientParams = new HttpClientParams();
clientParams.setConnectionManagerTimeout(
PreferenceHelper.getInt(
ServerConstants.HTTP_CONN_MGR_TIMEOUT_CONF_KEY, 300000)); // 5 minutes
return new HttpClient(clientParams, connectionManager);
}
/**
* Creates a new connection to the server.
*
* @param builder The HttpFileSystemConfigBuilder.
* @param scheme The protocol.
* @param hostname The hostname.
* @param port The port number.
* @param username The username.
* @param password The password
* @param fileSystemOptions The file system options.
* @return a new HttpClient connection.
* @throws FileSystemException if an error occurs.
* @since 2.0
*/
public static HttpClient createConnection(
HttpFileSystemConfigBuilder builder,
String scheme,
String hostname,
int port,
String username,
String password,
FileSystemOptions fileSystemOptions)
throws FileSystemException {
HttpClient client;
try {
HttpConnectionManager mgr = new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams connectionMgrParams = mgr.getParams();
client = new HttpClient(mgr);
final HostConfiguration config = new HostConfiguration();
config.setHost(hostname, port, scheme);
if (fileSystemOptions != null) {
String proxyHost = builder.getProxyHost(fileSystemOptions);
int proxyPort = builder.getProxyPort(fileSystemOptions);
if (proxyHost != null && proxyHost.length() > 0 && proxyPort > 0) {
config.setProxy(proxyHost, proxyPort);
}
UserAuthenticator proxyAuth = builder.getProxyAuthenticator(fileSystemOptions);
if (proxyAuth != null) {
UserAuthenticationData authData =
UserAuthenticatorUtils.authenticate(
proxyAuth,
new UserAuthenticationData.Type[] {
UserAuthenticationData.USERNAME, UserAuthenticationData.PASSWORD
});
if (authData != null) {
final UsernamePasswordCredentials proxyCreds =
new UsernamePasswordCredentials(
UserAuthenticatorUtils.toString(
UserAuthenticatorUtils.getData(
authData, UserAuthenticationData.USERNAME, null)),
UserAuthenticatorUtils.toString(
UserAuthenticatorUtils.getData(
authData, UserAuthenticationData.PASSWORD, null)));
AuthScope scope = new AuthScope(proxyHost, AuthScope.ANY_PORT);
client.getState().setProxyCredentials(scope, proxyCreds);
}
if (builder.isPreemptiveAuth(fileSystemOptions)) {
HttpClientParams httpClientParams = new HttpClientParams();
httpClientParams.setAuthenticationPreemptive(true);
client.setParams(httpClientParams);
}
}
Cookie[] cookies = builder.getCookies(fileSystemOptions);
if (cookies != null) {
client.getState().addCookies(cookies);
}
}
/**
* ConnectionManager set methodsmust be called after the host & port and proxy host & port are
* set in the HostConfiguration. They are all used as part of the key when
* HttpConnectionManagerParams tries to locate the host configuration.
*/
connectionMgrParams.setMaxConnectionsPerHost(
config, builder.getMaxConnectionsPerHost(fileSystemOptions));
connectionMgrParams.setMaxTotalConnections(builder.getMaxTotalConnections(fileSystemOptions));
client.setHostConfiguration(config);
if (username != null) {
final UsernamePasswordCredentials creds =
new UsernamePasswordCredentials(username, password);
AuthScope scope = new AuthScope(hostname, AuthScope.ANY_PORT);
client.getState().setCredentials(scope, creds);
}
client.executeMethod(new HeadMethod());
} catch (final Exception exc) {
throw new FileSystemException(
"vfs.provider.http/connect.error", new Object[] {hostname}, exc);
}
return client;
}
/**
* The method reads a proxy object from database, makes a GetMethod object, appends required
* cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
* pagesource to the caller.
*
* @param iCount Counter variable for passing thread group information
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @param region the local region of a given url
* @param objProxyDao the database layer ProxyDao object variable
* @param useErrsy Boolean variable to specify usage of Errsy as proxy source
* @return String
*/
public String getPageSourceWithProxy(
String url,
boolean followRedirect,
boolean doAuthentication,
String region,
Boolean useErrsy,
String google) {
String page = " ";
String pageSource = "";
int i = 0;
String exception = " ";
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod getmethod = null;
HttpState state = new HttpState();
// if (ProxyDao.lstProxyData.size() == 16) {
ProxyData objProxyData = null;
// if (!useErrsy) {
try {
// objProxyData = ProxyDao.lstProxyData.get(iCount);
objProxyData = ProxyDao.objProxyData;
if (objProxyData == null) {
// objProxyDao.changeProxy(google);
objProxyData = ProxyDao.objProxyData;
}
httpclient
.getHostConfiguration()
.setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
/*} else {
try {
objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
httpclient.setState(state);
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
}*/
try {
getmethod = new GetMethod(url);
getmethod.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
if (url.contains("bing.com")) {
if (region.equalsIgnoreCase("co.uk")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.sg")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.au")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("co.in")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("ca")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.ph")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.my")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("it")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
}
}
getmethod.setFollowRedirects(true);
getmethod.setDoAuthentication(true);
httpclient.getParams().setAuthenticationPreemptive(true);
httpclient.setState(state);
String num100Header = "";
// if (url.contains("google")) {
// int j = 0;
// String url1 = "http://www.google.com/";
// try {
// GetMethod objGetMethod = new GetMethod(url1);
// j = httpclient.executeMethod(objGetMethod);
// Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
// String header = responseHeader.getValue();
// String[] headerValue = header.split(";");
//
// for (String head : headerValue) {
// if (head.contains("PREF=ID")) {
// header = head;
// break;
// }
// }
// String[] splitAll = header.split(":");
// long time = System.currentTimeMillis()+400;
// String sTime = "" + time;
// sTime = sTime.substring(0, 10);
// //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
// num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
// Cookie ck = new Cookie("PREF", "PREF", num100Header);
// httpclient.getState().clearCookies();
// httpclient.getState().addCookie(ck);
// getmethod.addRequestHeader("Host", "www.google.com");
// getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
// rv:19.0) Gecko/20100101 Firefox/19.0");
// getmethod.addRequestHeader("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
// getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
// getmethod.addRequestHeader("Referer", "https://www.google.com/");
// System.out.println(num100Header);
// } catch (Exception ex) {
// exception = ex.getMessage();
// l.debug(ex + " " + ex.getMessage() + "Exception occured for url" +
// url);
// pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
// return pageSource;
// }
// }
i = httpclient.executeMethod(getmethod);
if (i / 100 == 4 || i / 100 == 5) {
page = "<PROXY ERROR>";
} else {
page = getmethod.getResponseBodyAsString();
}
} catch (SocketTimeoutException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (SocketException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (Exception ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} finally {
getmethod.releaseConnection();
}
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
// }
return pageSource;
}
public void setMaxRedirects(int n) {
HttpClientParams clientparams = sessionClient.getParams();
clientparams.setParameter(MAX_REDIRECTS, n);
}