public AbstractHttpTransport() {
PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
connectionManager.setMaxTotal(1000);
connectionManager.setDefaultMaxPerRoute(500);
this.httpClient = new DefaultHttpClient(connectionManager);
}
public void init() {
/*
loadLiferayProperties();
hostname = liferayProperties.getProperty(FFPORTAL_SERVER_HOSTNAME);
port = liferayProperties.getProperty(FFPORTAL_SERVER_PORT);
companyId = liferayProperties.getProperty(FFPORTAL_REPOSITORY_COMPANYID);
loginGroupId = liferayProperties.getProperty(FFPORTAL_USER_GROUP_ID);
loginEmail = liferayProperties.getProperty(FFPORTAL_USER_EMAIL);
loginPassword = liferayProperties.getProperty(FFPORTAL_USER_PASSWORD);
*/
targetHost = new HttpHost(hostname, Integer.valueOf(port), "http");
PoolingClientConnectionManager cxMgr =
new PoolingClientConnectionManager(SchemeRegistryFactory.createDefault());
cxMgr.setMaxTotal(100);
cxMgr.setDefaultMaxPerRoute(20);
httpclient = new DefaultHttpClient(cxMgr);
httpclient
.getCredentialsProvider()
.setCredentials(
new AuthScope(targetHost.getHostName(), targetHost.getPort()),
new UsernamePasswordCredentials(loginEmail, loginPassword));
// Create AuthCache instance
this.authCache = new BasicAuthCache();
// Generate BASIC scheme object and add it to the local
// auth cache
BasicScheme basicAuth = new BasicScheme();
authCache.put(targetHost, basicAuth);
}
@Override
public void init(Map<String, String> args) {
url = (String) args.get("url");
manager = new PoolingClientConnectionManager();
manager.setMaxTotal(100);
// manager.setMaxTotal(20);
}
static {
BasicHttpParams httpParams = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(httpParams, 1000 * 1000);
HttpConnectionParams.setSoTimeout(httpParams, 1000 * 1000);
PoolingClientConnectionManager pcm = new PoolingClientConnectionManager();
pcm.setMaxTotal(20000);
pcm.setDefaultMaxPerRoute(pcm.getMaxTotal());
httpClient = new DefaultHttpClient(pcm, httpParams);
}
/**
* create PoolingClientConnectionManager
*
* @param maxCount max total connection, default max connection per route
* @param connectionTimeout milliseconds
* @param answerTimeout milliseconds
*/
public HttpManager(int maxCount, int connectionTimeout, int answerTimeout) {
PoolingClientConnectionManager manager = new PoolingClientConnectionManager();
manager.setDefaultMaxPerRoute(maxCount);
manager.setMaxTotal(maxCount);
HttpParams params = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(params, connectionTimeout);
HttpConnectionParams.setSoTimeout(params, answerTimeout);
httpClient = new DefaultHttpClient(manager, params);
}
public IntegrationMockClient(String endpoint) {
super(endpoint);
this.endpoint = endpoint;
PoolingClientConnectionManager cm = new PoolingClientConnectionManager();
// Increase max total connection to 200
cm.setMaxTotal(200);
// Increase default max connection per route to 50
cm.setDefaultMaxPerRoute(50);
httpClient = new DefaultHttpClient(cm);
httpClient = (DefaultHttpClient) WebClientWrapper.wrapClient(httpClient);
}
public void Initiallize()
throws KeyManagementException, UnrecoverableKeyException, NoSuchAlgorithmException,
KeyStoreException {
Scheme http = new Scheme("http", 80, PlainSocketFactory.getSocketFactory());
// 创建TrustManager
X509TrustManager xtm =
new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType)
throws CertificateException {}
public void checkServerTrusted(X509Certificate[] chain, String authType)
throws CertificateException {}
public X509Certificate[] getAcceptedIssuers() {
return null;
}
};
// 这个好像是HOST验证
X509HostnameVerifier hostnameVerifier =
new X509HostnameVerifier() {
public boolean verify(String arg0, SSLSession arg1) {
return true;
}
public void verify(String arg0, SSLSocket arg1) throws IOException {}
public void verify(String arg0, String[] arg1, String[] arg2) throws SSLException {}
public void verify(String arg0, X509Certificate arg1) throws SSLException {}
};
SSLContext ctx = null;
try {
ctx = SSLContext.getInstance("TLS");
ctx.init(null, new TrustManager[] {(TrustManager) xtm}, null);
// 创建SSLSocketFactory
socketFactory = new SSLSocketFactory(ctx);
socketFactory.setHostnameVerifier(hostnameVerifier);
} catch (NoSuchAlgorithmException e1) {
e1.printStackTrace();
} catch (KeyManagementException e) {
e.printStackTrace();
}
SSLContext.setDefault(ctx);
SSLSocketFactory ssf = new SSLSocketFactory(ctx);
Scheme https = new Scheme("https", 443, ssf);
sr = new SchemeRegistry();
sr.register(http);
sr.register(https);
cm = new PoolingClientConnectionManager(sr);
cm.setMaxTotal(200);
cm.setDefaultMaxPerRoute(100);
}
static {
httpParams = new BasicHttpParams();
// 设置连接超时时间
HttpConnectionParams.setConnectionTimeout(httpParams, CONNECT_TIMEOUT);
// 设置读取超时时间
HttpConnectionParams.setSoTimeout(httpParams, READ_TIMEOUT);
connectionManager = new PoolingClientConnectionManager();
// 设置最大连接数
connectionManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
// 设置每个路由最大连接数
connectionManager.setDefaultMaxPerRoute(MAX_ROUTE_CONNECTIONS);
client = new DefaultHttpClient(connectionManager, httpParams);
}
/**
* 获得连接池的实例
*
* @return
*/
public static HttpClient getCoonPoolHttpClient() {
PoolingClientConnectionManager cm = new PoolingClientConnectionManager();
cm.setMaxTotal(800);
cm.setDefaultMaxPerRoute(200);
DefaultHttpClient httpClient = new DefaultHttpClient(cm, getHttpParams());
httpClient.addRequestInterceptor(getRequestInter());
httpClient.addResponseInterceptor(getResponseInter());
httpClient.setHttpRequestRetryHandler(buildMyRetryHandler());
return httpClient;
}
public HttpClient getInstance() {
if (instance == null) {
synchronized (this) {
if (instance == null) {
PoolingClientConnectionManager manager = new PoolingClientConnectionManager();
manager.setMaxTotal(40);
instance = new DefaultHttpClient(manager);
instance
.getParams()
.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.IGNORE_COOKIES);
}
}
}
return instance;
}
/**
* Create a new instance of the HttpComponentsHttpInvokerRequestExecutor with a default {@link
* HttpClient} that uses a default {@code
* org.apache.http.impl.conn.PoolingClientConnectionManager}.
*/
public HttpComponentsHttpInvokerRequestExecutor() {
org.apache.http.conn.scheme.SchemeRegistry schemeRegistry =
new org.apache.http.conn.scheme.SchemeRegistry();
schemeRegistry.register(
new org.apache.http.conn.scheme.Scheme(
"http", 80, org.apache.http.conn.scheme.PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(
new org.apache.http.conn.scheme.Scheme(
"https", 443, org.apache.http.conn.ssl.SSLSocketFactory.getSocketFactory()));
org.apache.http.impl.conn.PoolingClientConnectionManager connectionManager =
new org.apache.http.impl.conn.PoolingClientConnectionManager(schemeRegistry);
connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);
this.httpClient = new org.apache.http.impl.client.DefaultHttpClient(connectionManager);
setReadTimeout(DEFAULT_READ_TIMEOUT_MILLISECONDS);
}
private UmssoCLCore initializeCore() {
UmssoCLCore clCore = UmssoCLCoreImpl.getInstance();
HttpParams params = clCore.getUmssoHttpClient().getParams();
HttpConnectionParams.setConnectionTimeout(params, 10000);
PoolingClientConnectionManager cm = clCore.getConnectionManager();
cm.setMaxTotal(HTTP_MAX_TOTAL_CONNECTIONS);
cm.setDefaultMaxPerRoute(HTTP_DEFAULT_MAX_CONNECTIONS_PER_ROUTE);
UmssoCLEnvironment umssoCLEnvironment =
umSsoHttpConnectionSettings.getUmSsoCLEnvironmentFromProxySettings();
if (umssoCLEnvironment != null) {
clCore.init(umssoCLEnvironment);
}
if (umSsoHttpConnectionSettings.isIgnoreCertificates()) {
ignoreCertificates(clCore.getUmssoHttpClient());
}
return clCore;
}
public void Initiallize2()
throws KeyManagementException, UnrecoverableKeyException, NoSuchAlgorithmException,
KeyStoreException {
Scheme http = new Scheme("http", 80, PlainSocketFactory.getSocketFactory());
X509TrustManager tm =
new X509TrustManager() {
public X509Certificate[] getAcceptedIssuers() {
return null;
}
@Override
public void checkClientTrusted(X509Certificate[] arg0, String arg1)
throws CertificateException {}
@Override
public void checkServerTrusted(X509Certificate[] arg0, String arg1)
throws CertificateException {}
};
SSLContext ctx = null;
try {
ctx = SSLContext.getInstance("TLS");
ctx.init(null, new TrustManager[] {(TrustManager) tm}, null);
} catch (NoSuchAlgorithmException e1) {
e1.printStackTrace();
} catch (KeyManagementException e) {
e.printStackTrace();
}
SSLContext.setDefault(ctx);
SSLSocketFactory ssf = new SSLSocketFactory(ctx);
Scheme https = new Scheme("https", 443, ssf);
sr = new SchemeRegistry();
sr.register(http);
sr.register(https);
cm = new PoolingClientConnectionManager(sr);
cm.setMaxTotal(200);
cm.setDefaultMaxPerRoute(100);
}
@Override
public void afterPropertiesSet() {
super.afterPropertiesSet();
if (getServiceInterface() == null) {
throw new IllegalArgumentException("property serviceInterface is required.");
}
org.apache.http.conn.scheme.SchemeRegistry schemeRegistry =
new org.apache.http.conn.scheme.SchemeRegistry();
schemeRegistry.register(
new org.apache.http.conn.scheme.Scheme(
"http", 80, org.apache.http.conn.scheme.PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(
new org.apache.http.conn.scheme.Scheme(
"https", 443, org.apache.http.conn.ssl.SSLSocketFactory.getSocketFactory()));
org.apache.http.impl.conn.PoolingClientConnectionManager connectionManager =
new org.apache.http.impl.conn.PoolingClientConnectionManager(schemeRegistry);
connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);
this.httpClient = new org.apache.http.impl.client.DefaultHttpClient(connectionManager);
setReadTimeout(DEFAULT_READ_TIMEOUT_MILLISECONDS);
prepare();
}
public static Client createApacheClient(
AtmosConfig config,
boolean useExpect100Continue,
List<Class<MessageBodyReader<?>>> readers,
List<Class<MessageBodyWriter<?>>> writers) {
try {
ClientConfig clientConfig = new DefaultApacheHttpClient4Config();
// make sure the apache client is thread-safe
PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
// Increase max total connection to 200
connectionManager.setMaxTotal(200);
// Increase default max connection per route to 200
connectionManager.setDefaultMaxPerRoute(200);
clientConfig
.getProperties()
.put(DefaultApacheHttpClient4Config.PROPERTY_CONNECTION_MANAGER, connectionManager);
// register an open trust manager to allow SSL connections to servers with self-signed
// certificates
if (config.isDisableSslValidation()) {
connectionManager
.getSchemeRegistry()
.register(
new Scheme(
"https",
443,
new SSLSocketFactory(
SslUtil.createGullibleSslContext(),
SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER)));
}
// set proxy uri
ProxySelector proxySelector;
// first look in config
URI proxyUri = config.getProxyUri();
if (proxyUri != null) {
proxySelector = new ConfigProxySelector(config);
} else {
// if no proxy in config, use system property sniffing selector
proxySelector = new DefaultProxySelector();
// and see if a proxy is set
String host = System.getProperty("http.proxyHost");
String portStr = System.getProperty("http.proxyPort");
int port = (portStr != null) ? Integer.parseInt(portStr) : -1;
if (host != null && host.length() > 0)
proxyUri = new URI("http", null, host, port, null, null, null);
}
// make sure any proxy credentials (below) are associated with the proxy
if (proxyUri != null)
clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_URI, proxyUri);
// set proxy auth
// first config
String proxyUser = config.getProxyUser(), proxyPassword = config.getProxyPassword();
// then system props
if (proxyUser == null) {
proxyUser = System.getProperty("http.proxyUser");
proxyPassword = System.getProperty("http.proxyPassword");
}
if (proxyUser != null && proxyUser.length() > 0) {
clientConfig
.getProperties()
.put(ApacheHttpClient4Config.PROPERTY_PROXY_USERNAME, proxyUser);
clientConfig
.getProperties()
.put(ApacheHttpClient4Config.PROPERTY_PROXY_PASSWORD, proxyPassword);
}
// specify whether to use Expect: 100-continue
HttpParams httpParams = new SyncBasicHttpParams();
DefaultHttpClient.setDefaultHttpParams(httpParams);
httpParams.setBooleanParameter(AllClientPNames.USE_EXPECT_CONTINUE, useExpect100Continue);
clientConfig
.getProperties()
.put(DefaultApacheHttpClient4Config.PROPERTY_HTTP_PARAMS, httpParams);
// pick up other configuration from system props
for (String prop : System.getProperties().stringPropertyNames()) {
if (prop.startsWith("http.")) {
// because AbstractHttpParams uses casts instead of parsing string values, we must know
// the
// parameter types ahead of time. I'm going to guess them in lieu of maintaining a map.
// however,
// we still need to define Long typed parameters so they may be differentiated from
// Integer typed
// parameters.
String value = System.getProperty(prop);
if (LONG_PARAMETERS.contains(prop))
httpParams.setLongParameter(
prop, Long.parseLong(value.substring(0, value.length() - 1)));
else if (BOOLEAN_PATTERN.matcher(value).matches())
httpParams.setBooleanParameter(prop, Boolean.parseBoolean(value));
else if (DOUBLE_PATTERN.matcher(value).matches())
httpParams.setDoubleParameter(prop, Double.parseDouble(value));
else if (INT_PATTERN.matcher(value).matches())
httpParams.setIntParameter(prop, Integer.parseInt(value));
else httpParams.setParameter(prop, System.getProperty(prop));
}
}
JerseyUtil.addHandlers(clientConfig, readers, writers);
// create the client
ApacheHttpClient4 client = ApacheHttpClient4.create(clientConfig);
AbstractHttpClient httpClient =
(AbstractHttpClient) client.getClientHandler().getHttpClient();
// do not use Apache's retry handler
httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));
// use a RoutePlanner/ProxySelector that fits our requirements
SchemeRegistry registry = httpClient.getConnectionManager().getSchemeRegistry();
httpClient.setRoutePlanner(new ProxySelectorRoutePlanner(registry, proxySelector));
JerseyUtil.addFilters(client, config);
return client;
} catch (Exception e) {
throw new AtmosException("Error configuring REST client", e);
}
}
/** @return {@link DefaultHttpClient} instance. */
private HttpClient createHttpClient(CouchDbProperties props) {
DefaultHttpClient httpclient = null;
try {
SchemeSocketFactory ssf = null;
if (props.getProtocol().equals("https")) {
TrustManager trustManager =
new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType)
throws CertificateException {}
public void checkServerTrusted(X509Certificate[] chain, String authType)
throws CertificateException {}
public X509Certificate[] getAcceptedIssuers() {
return null;
}
};
SSLContext sslcontext = SSLContext.getInstance("TLS");
sslcontext.init(null, new TrustManager[] {trustManager}, null);
ssf = new SSLSocketFactory(sslcontext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
SSLSocket socket = (SSLSocket) ssf.createSocket(null);
socket.setEnabledCipherSuites(new String[] {"SSL_RSA_WITH_RC4_128_MD5"});
} else {
ssf = PlainSocketFactory.getSocketFactory();
}
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme(props.getProtocol(), props.getPort(), ssf));
PoolingClientConnectionManager ccm = new PoolingClientConnectionManager(schemeRegistry);
httpclient = new DefaultHttpClient(ccm);
host = new HttpHost(props.getHost(), props.getPort(), props.getProtocol());
context = new BasicHttpContext();
// Http params
httpclient.getParams().setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, "UTF-8");
httpclient
.getParams()
.setParameter(CoreConnectionPNames.SO_TIMEOUT, props.getSocketTimeout());
httpclient
.getParams()
.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, props.getConnectionTimeout());
int maxConnections = props.getMaxConnections();
if (maxConnections != 0) {
ccm.setMaxTotal(maxConnections);
ccm.setDefaultMaxPerRoute(maxConnections);
}
if (props.getProxyHost() != null) {
HttpHost proxy = new HttpHost(props.getProxyHost(), props.getProxyPort());
httpclient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
}
// basic authentication
if (props.getUsername() != null && props.getPassword() != null) {
httpclient
.getCredentialsProvider()
.setCredentials(
new AuthScope(props.getHost(), props.getPort()),
new UsernamePasswordCredentials(props.getUsername(), props.getPassword()));
props.clearPassword();
AuthCache authCache = new BasicAuthCache();
BasicScheme basicAuth = new BasicScheme();
authCache.put(host, basicAuth);
context.setAttribute(ClientContext.AUTH_CACHE, authCache);
}
// request interceptor
httpclient.addRequestInterceptor(
new HttpRequestInterceptor() {
public void process(final HttpRequest request, final HttpContext context)
throws IOException {
if (log.isInfoEnabled()) {
RequestLine requestLine = request.getRequestLine();
try {
log.info(
">> "
+ (new StringBuilder())
.append(">> ")
.append(requestLine.getMethod())
.append(" ")
.append(urlCodec.decode(requestLine.getUri()))
.toString());
} catch (DecoderException e) {
log.error(e, e);
}
}
}
});
// response interceptor
httpclient.addResponseInterceptor(
new HttpResponseInterceptor() {
public void process(final HttpResponse response, final HttpContext context)
throws IOException {
validate(response);
if (log.isInfoEnabled())
log.info("<< Status: " + response.getStatusLine().getStatusCode());
}
});
} catch (Exception e) {
log.error("Error Creating HTTP client. " + e.getMessage());
throw new IllegalStateException(e);
}
return httpclient;
}
public ServiceObserver(
ServiceObserverConfiguration serviceConfig,
ExecutorService executor,
Class<D> dataClass,
Class<A> aggregateClass,
Class<S> summaryClass,
ContentParser<D> contentParser,
ObserverNamingStrategy ons) {
this.serviceConfig = serviceConfig;
this.executor = executor;
this.observers = new ConcurrentHashMap<String, Observer<D>>();
this.dataClass = dataClass;
this.aggregateClass = aggregateClass;
this.summaryClass = summaryClass;
this.contentParser = contentParser;
if (ons == null) {
this.ons = ObserverNamingStrategy.DEFAULT;
} else {
this.ons = ons;
}
this.snapshotAllTimer =
Metrics.newTimer(
new MetricName(
"com.mfizz.observer.ServiceObservers",
serviceConfig.getName(),
"snapshot-all-time"),
TimeUnit.MILLISECONDS,
TimeUnit.MILLISECONDS);
this.snapshotAllAttemptedCounter = new AtomicLong();
this.snapshotAllCompletedCounter = new AtomicLong();
this.lastSnapshotAllResult = new AtomicReference<SnapshotAllResult>();
// this.lastSnapshotAllTimestamp = new AtomicLong(0);
// this.lastSnapshotsCompletedCounter = new AtomicInteger();
// this.lastSnapshotsFailedCounter = new AtomicInteger();
this.groups = new TreeSet<String>();
this.summary = new ConcurrentHashMap<String, SummaryGroup<S>>();
this.snapshots = new ConcurrentHashMap<String, TimeSeries<ObserveAggregateSnapshot<A>>>();
// always make sure "current" is added to set of vars to track
// "current" always tracked for all
this.serviceConfig.getPeriods().add(SummaryPeriod.parse("current"));
//
// create high performance http client that can be reused by all observers
//
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
PoolingClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
// increase max total connection to 200
cm.setMaxTotal(200);
// increase default max connection per route to 20
cm.setDefaultMaxPerRoute(20);
this.httpclient = new DefaultHttpClient(cm);
this.httpclient.setHttpRequestRetryHandler(
new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(
IOException exception, int executionCount, HttpContext context) {
// always return false -- we never want a request retried
return false;
}
});
this.httpclient
.getParams()
.setIntParameter(
CoreConnectionPNames.SO_TIMEOUT, new Integer((int) serviceConfig.getSocketTimeout()))
.setIntParameter(
CoreConnectionPNames.CONNECTION_TIMEOUT,
new Integer((int) serviceConfig.getConnectionTimeout()))
.setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024)
.setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true)
.setBooleanParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false)
.setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false);
}
// 根据用户ID找该用户的粉丝对象
public Set<String> getFansUserByUid(String uid, int pageNumber) {
System.out.println("进入查找用户的粉丝用户");
PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
connectionManager.setMaxTotal(2000);
connectionManager.setDefaultMaxPerRoute(1000);
HttpClient httpClient_fans = new DefaultHttpClient(connectionManager);
// 伪装成Firefox 5,
httpClient_fans
.getParams()
.setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
httpClient_fans
.getParams()
.setParameter(
ClientPNames.COOKIE_POLICY,
CookiePolicy.BROWSER_COMPATIBILITY); // 一定要有，否则会生成多个Cookie header送给web server
httpClient_fans.getParams().setParameter(CookieSpecPNames.SINGLE_COOKIE_HEADER, true); //
httpClient_fans
.getParams()
.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, "UTF-8"); // 这个是和目标网站的编码有关；
httpClient_fans.getParams().setParameter(CoreProtocolPNames.HTTP_ELEMENT_CHARSET, "UTF-8");
List<BasicHeader> headerList_fans = new ArrayList<BasicHeader>();
headerList_fans.add(new BasicHeader("Accept", "*/*"));
headerList_fans.add(new BasicHeader("Accept-Charset", "GBK,utf-8;q=0.7,*;q=0.3"));
headerList_fans.add(new BasicHeader("Accept-Language", "zh-CN,zh;q=0.8"));
headerList_fans.add(new BasicHeader("Cache-Control", "no-cache"));
headerList_fans.add(new BasicHeader("Connection", "keep-alive"));
headerList_fans.add(
new BasicHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"));
headerList_fans.add(new BasicHeader("X-Requested-With", "XMLHttpRequest"));
headerList_fans.add(new BasicHeader("Cookie", CookieData.CookieNow));
httpClient_fans
.getParams()
.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, new Integer(300000));
httpClient_fans.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT, new Integer(300000));
httpClient_fans
.getParams()
.setParameter(ClientPNames.CONN_MANAGER_TIMEOUT, new Long(300000)); // second;
// 伪装成Firefox
httpClient_fans
.getParams()
.setParameter(
CoreProtocolPNames.USER_AGENT,
"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:17.0) Gecko/17.0 Firefox/17.0");
headerList_fans.add(new BasicHeader("Host", "weibo.com"));
headerList_fans.add(new BasicHeader("Origin", "weibo.com"));
GetFollowOrFansPageNumber page = new GetFollowOrFansPageNumber();
int number = page.getFollowOrFansPage("http://weibo.com/" + uid + "/fans");
// QueryNumberManager queryNumber = QueryNumberManager.getInstance();
Set<String> uidList = new HashSet<String>();
for (int u = pageNumber; u < number; u++) {
System.out.println("粉丝第" + (u + 1) + "页");
String url_fans = "http://weibo.com/" + uid + "/fans" + "?page=" + (u + 1);
System.out.println(url_fans);
headerList_fans.add(new BasicHeader("Referer", url_fans));
httpClient_fans.getParams().setParameter(ClientPNames.DEFAULT_HEADERS, headerList_fans);
// 设置代理对象 ip/代理名称,端口
// ProxyManager proxyManager = ProxyManager.getInstance();
// ProxyBean proxy = new ProxyBean();
// proxy = proxyManager.getOneProxy();
// System.out.println("拿到代理IP" + proxy.getIp());
// httpClient_fans.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, new
// HttpHost(proxy.getIp(), proxy.getPort()));
HttpPost httpPost_fans = new HttpPost(url_fans);
HttpResponse httpResponseFans = null;
try {
httpResponseFans = httpClient_fans.execute(httpPost_fans);
} catch (ClientProtocolException e1) {
PshLogger.logger.error(e1.getMessage());
} catch (IOException e1) {
PshLogger.logger.error(e1.getMessage());
System.out.println("重新执行找关注");
return getFansUserByUid(uid, u);
}
if (httpResponseFans == null) {
return getFansUserByUid(uid, u + 1);
}
HttpEntity httpEntityFans = httpResponseFans.getEntity();
if (httpEntityFans == null) {
return getFansUserByUid(uid, u);
}
BufferedReader in_fans = null;
try {
in_fans = new BufferedReader(new InputStreamReader(httpEntityFans.getContent(), "UTF-8"));
} catch (UnsupportedEncodingException e1) {
PshLogger.logger.error(e1.getMessage());
} catch (IllegalStateException e1) {
PshLogger.logger.error(e1.getMessage());
} catch (IOException e1) {
PshLogger.logger.error(e1.getMessage());
}
String result_fans = "";
String item_fans = "";
try {
while ((item_fans = in_fans.readLine()) != null) {
if (item_fans.trim().indexOf("cnfList") != -1) {
System.out.println("找到一行html符合");
result_fans = item_fans.trim();
}
}
} catch (IOException e1) {
PshLogger.logger.error(e1.getMessage());
} finally {
try {
in_fans.close();
} catch (IOException e) {
PshLogger.logger.error(e.getMessage());
}
}
if (result_fans.trim().equals("")) {
System.out.println("新浪微博提示行为异常follow");
// 换代理IP重新获取该页
return getFansUserByUid(uid, u);
}
System.out.println("已获取到 html_fans");
try {
result_fans =
result_fans.substring(
result_fans.indexOf("<div"), result_fans.lastIndexOf("/div>") + 5);
result_fans = result_fans.replace('\\', '`');
result_fans = result_fans.replaceAll("`n", "");
result_fans = result_fans.replaceAll("`t", "");
result_fans = result_fans.replaceAll("`r", "");
result_fans = result_fans.replaceAll("`", "");
result_fans = "<html><body>" + result_fans + "</body></html>";
} catch (Exception e) {
PshLogger.logger.error(e.getMessage());
return getFansUserByUid(uid, u + 1);
}
Document doc_fans = Jsoup.parse(result_fans);
Elements elements_fans = doc_fans.getElementsByAttributeValue("class", "W_f14 S_func1");
System.out.println(elements_fans.size());
for (int x = 0; x < elements_fans.size(); x++) {
System.out.println("遍历粉丝第" + (u + 1) + "页，第" + (x + 1) + "个用户");
String uidString = elements_fans.get(x).attr("usercard");
System.out.println("uridString------" + uidString);
String uid_fans = uidString.substring(3);
System.out.println(uid_fans);
UserModel userModel = new UserModel();
if (userModel.checkUserIsExsit(uid_fans)) {
continue;
}
// 获取该用户详细信息
UserBean user = new UserBean();
GetPersonInfo personInfo = new GetPersonInfo();
Elements element_followNum =
elements_fans
.get(x)
.parent()
.parent()
.getElementsByAttributeValue("href", "/" + uid_fans + "/follow");
Elements element_fansNum =
elements_fans
.get(x)
.parent()
.parent()
.getElementsByAttributeValue("href", "/" + uid_fans + "/fans");
String fans = "";
String follow = "";
for (int y = 0; y < element_followNum.size(); y++) {
follow = element_followNum.get(y).text();
}
for (int y = 0; y < element_fansNum.size(); y++) {
fans = element_fansNum.get(y).text();
}
// 将该用户信息加入数据库
user = personInfo.getUserInfoFromWeibo(uid_fans, fans, follow);
if (user == null) {
continue;
}
System.out.println("用户查找完" + user.getUck());
// 进行2级遍历
uidList.add(uid_fans);
}
}
// queryNumber.executeFollowAndFans(uidList, queryTaskID,level);
return uidList;
}
protected ClientHttpEngine initDefaultEngine() {
DefaultHttpClient httpClient = null;
X509HostnameVerifier verifier = null;
if (this.verifier != null) verifier = new VerifierWrapper(this.verifier);
else {
switch (policy) {
case ANY:
verifier = new AllowAllHostnameVerifier();
break;
case WILDCARD:
verifier = new BrowserCompatHostnameVerifier();
break;
case STRICT:
verifier = new StrictHostnameVerifier();
break;
}
}
try {
SSLSocketFactory sslsf = null;
SSLContext theContext = sslContext;
if (disableTrustManager) {
theContext = SSLContext.getInstance("SSL");
theContext.init(
null, new TrustManager[] {new PassthroughTrustManager()}, new SecureRandom());
verifier = new AllowAllHostnameVerifier();
sslsf = new SSLSocketFactory(theContext, verifier);
} else if (theContext != null) {
sslsf = new SSLSocketFactory(theContext, verifier);
} else if (clientKeyStore != null || truststore != null) {
sslsf =
new SSLSocketFactory(
SSLSocketFactory.TLS,
clientKeyStore,
clientPrivateKeyPassword,
truststore,
null,
verifier);
} else {
// sslsf = new SSLSocketFactory(SSLContext.getInstance(SSLSocketFactory.TLS), verifier);
final SSLContext tlsContext = SSLContext.getInstance(SSLSocketFactory.TLS);
tlsContext.init(null, null, null);
sslsf = new SSLSocketFactory(tlsContext, verifier);
}
SchemeRegistry registry = new SchemeRegistry();
registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
Scheme httpsScheme = new Scheme("https", 443, sslsf);
registry.register(httpsScheme);
ClientConnectionManager cm = null;
if (connectionPoolSize > 0) {
PoolingClientConnectionManager tcm =
new PoolingClientConnectionManager(registry, connectionTTL, connectionTTLUnit);
tcm.setMaxTotal(connectionPoolSize);
if (maxPooledPerRoute == 0) maxPooledPerRoute = connectionPoolSize;
tcm.setDefaultMaxPerRoute(maxPooledPerRoute);
cm = tcm;
} else {
cm = new BasicClientConnectionManager(registry);
}
BasicHttpParams params = new BasicHttpParams();
if (socketTimeout > -1) {
HttpConnectionParams.setSoTimeout(params, (int) socketTimeoutUnits.toMillis(socketTimeout));
}
if (establishConnectionTimeout > -1) {
HttpConnectionParams.setConnectionTimeout(
params, (int) establishConnectionTimeoutUnits.toMillis(establishConnectionTimeout));
}
httpClient = new DefaultHttpClient(cm, params);
ApacheHttpClient4Engine engine = new ApacheHttpClient4Engine(httpClient, true);
engine.setResponseBufferSize(responseBufferSize);
engine.setHostnameVerifier(verifier);
// this may be null. We can't really support this with Apache Client.
engine.setSslContext(theContext);
engine.setDefaultProxy(defaultProxy);
return engine;
} catch (Exception e) {
throw new RuntimeException(e);
}
}