private HostConfiguration getHostConfiguration(HttpClient client, Map<String, Object> props) {
Object proxy = props.get(ApacheHttpClientConfig.PROPERTY_PROXY_URI);
if (proxy != null) {
URI proxyUri = getProxyUri(proxy);
String proxyHost = proxyUri.getHost();
if (proxyHost == null) {
proxyHost = "localhost";
}
int proxyPort = proxyUri.getPort();
if (proxyPort == -1) {
proxyPort = 8080;
}
HostConfiguration hostConfig = new HostConfiguration(client.getHostConfiguration());
String setHost = hostConfig.getProxyHost();
int setPort = hostConfig.getProxyPort();
if ((setHost == null)
|| (!setHost.equals(proxyHost))
|| (setPort == -1)
|| (setPort != proxyPort)) {
hostConfig.setProxyHost(new ProxyHost(proxyHost, proxyPort));
}
return hostConfig;
} else {
return null;
}
}
public String getContextByPostMethod(String url, List<KeyValue> params) {
HttpClient client = getHttpClient();
client.getParams().setParameter("http.protocol.content-charset", this.codeing);
PostMethod post = null;
String result = "";
try {
URL u = new URL(url);
client
.getHostConfiguration()
.setHost(
u.getHost(), u.getPort() == -1 ? u.getDefaultPort() : u.getPort(), u.getProtocol());
post = new PostMethod(u.getPath());
NameValuePair[] nvps = new NameValuePair[params.size()];
int i = 0;
for (KeyValue kv : params) {
nvps[i] = new NameValuePair(kv.getKey(), kv.getValue());
i++;
}
post.setRequestBody(nvps);
client.executeMethod(post);
result = post.getResponseBodyAsString();
} catch (Exception e) {
throw new NetException("HttpClient catch!", e);
} finally {
if (post != null) post.releaseConnection();
}
return result;
}
/** Get or create the http client to be used to fetch all the map data. */
public HttpClient getHttpClient(URI uri) {
MultiThreadedHttpConnectionManager connectionManager = getConnectionManager();
HttpClient httpClient = new HttpClient(connectionManager);
// httpclient is a bit pesky about loading everything in memory...
// disabling the warnings.
Logger.getLogger(HttpMethodBase.class).setLevel(Level.ERROR);
// configure proxies for URI
ProxySelector selector = ProxySelector.getDefault();
List<Proxy> proxyList = selector.select(uri);
Proxy proxy = proxyList.get(0);
if (!proxy.equals(Proxy.NO_PROXY)) {
InetSocketAddress socketAddress = (InetSocketAddress) proxy.address();
String hostName = socketAddress.getHostName();
int port = socketAddress.getPort();
httpClient.getHostConfiguration().setProxy(hostName, port);
}
for (SecurityStrategy sec : security)
if (sec.matches(uri)) {
sec.configure(uri, httpClient);
break;
}
return httpClient;
}
public static DAOLayerInterface getDAOLayer(ProxySettings proxySettings, ServerSession session)
throws Exception {
HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager());
if (proxySettings != null) {
Credentials credentials =
new UsernamePasswordCredentials(proxySettings.USERNAME, proxySettings.PASSWORD);
AuthScope authScope = new AuthScope(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT);
httpClient
.getHostConfiguration()
.setProxy(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT);
httpClient.getState().setProxyCredentials(authScope, credentials);
}
try {
db =
(DAOLayerInterface)
ServerRuntimeImpl.getDynamicProxy(
new ServerSession(session.getCommandurl(), null),
"dbinvoke", // commandurl
"DAOLAYER",
new Class<?>[] {DAOLayerInterface.class},
httpClient);
} catch (Exception ex) {
ex.printStackTrace();
}
return db;
}
public HttpClient(int maxConPerHost, int conTimeOutMs, int soTimeOutMs, int maxSize) {
connectionManager = new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams params = connectionManager.getParams();
params.setDefaultMaxConnectionsPerHost(maxConPerHost);
params.setConnectionTimeout(conTimeOutMs);
params.setSoTimeout(soTimeOutMs);
HttpClientParams clientParams = new HttpClientParams();
// 忽略cookie 避免 Cookie rejected 警告
clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
client = new org.apache.commons.httpclient.HttpClient(clientParams, connectionManager);
Protocol myhttps = new Protocol("https", new MySSLSocketFactory(), 443);
Protocol.registerProtocol("https", myhttps);
this.maxSize = maxSize;
// 支持proxy
if (proxyHost != null && !proxyHost.equals("")) {
client.getHostConfiguration().setProxy(proxyHost, proxyPort);
client.getParams().setAuthenticationPreemptive(true);
if (proxyAuthUser != null && !proxyAuthUser.equals("")) {
client
.getState()
.setProxyCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword));
log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword);
}
}
}
/** Create a new Client. The client will not use authentication by default. */
public Client() {
client = new HttpClient();
client.getParams().setParameter("http.socket.timeout", Integer.valueOf(DEFAULT_TIMEOUT));
log.debug("proxy host: " + client.getHostConfiguration().getProxyHost());
log.debug("proxy port: " + client.getHostConfiguration().getProxyPort());
doAuthentication = false;
}
@NotNull
private static HttpClient getHttpClient(
@Nullable final String login, @Nullable final String password) {
final HttpClient client = new HttpClient();
HttpConnectionManagerParams params = client.getHttpConnectionManager().getParams();
params.setConnectionTimeout(
CONNECTION_TIMEOUT); // set connection timeout (how long it takes to connect to remote host)
params.setSoTimeout(
CONNECTION_TIMEOUT); // set socket timeout (how long it takes to retrieve data from remote
// host)
client.getParams().setContentCharset("UTF-8");
// Configure proxySettings if it is required
final HttpConfigurable proxySettings = HttpConfigurable.getInstance();
if (proxySettings.USE_HTTP_PROXY && !StringUtil.isEmptyOrSpaces(proxySettings.PROXY_HOST)) {
client.getHostConfiguration().setProxy(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT);
if (proxySettings.PROXY_AUTHENTICATION) {
client
.getState()
.setProxyCredentials(
AuthScope.ANY,
new UsernamePasswordCredentials(
proxySettings.PROXY_LOGIN, proxySettings.getPlainProxyPassword()));
}
}
if (login != null && password != null) {
client.getParams().setCredentialCharset("UTF-8");
client.getParams().setAuthenticationPreemptive(true);
client
.getState()
.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(login, password));
}
return client;
}
public Response multPartURL(
String url, PostParameter[] params, ImageItem item, boolean authenticated)
throws WeiboException {
PostMethod post = new PostMethod(url);
try {
org.apache.commons.httpclient.HttpClient client = getHttpClient();
long t = System.currentTimeMillis();
Part[] parts = null;
if (params == null) {
parts = new Part[1];
} else {
parts = new Part[params.length + 1];
}
if (params != null) {
int i = 0;
for (PostParameter entry : params) {
parts[i++] = new StringPart(entry.getName(), (String) entry.getValue());
}
parts[parts.length - 1] =
new ByteArrayPart(item.getContent(), item.getName(), item.getContentType());
}
post.setRequestEntity(new MultipartRequestEntity(parts, post.getParams()));
List<Header> headers = new ArrayList<Header>();
if (authenticated) {
if (oauth == null) {}
String authorization = null;
if (null != oauth) {
// use OAuth
authorization = oauth.generateAuthorizationHeader("POST", url, params, oauthToken);
} else {
throw new IllegalStateException(
"Neither user ID/password combination nor OAuth consumer key/secret combination supplied");
}
headers.add(new Header("Authorization", authorization));
log("Authorization: " + authorization);
}
client.getHostConfiguration().getParams().setParameter("http.default-headers", headers);
client.executeMethod(post);
Response response = new Response();
response.setResponseAsString(post.getResponseBodyAsString());
response.setStatusCode(post.getStatusCode());
log(
"multPartURL URL:"
+ url
+ ", result:"
+ response
+ ", time:"
+ (System.currentTimeMillis() - t));
return response;
} catch (Exception ex) {
throw new WeiboException(ex.getMessage(), ex, -1);
} finally {
post.releaseConnection();
}
}
public static void setProxy(HttpClient httpClient) {
String proxyHost = getValueStr(proxy_host);
Logger.info("proxy_host:" + proxyHost);
if (proxyHost != null) {
int proxyPort = getValueInt(proxy_port);
Logger.info(" proxy_port:" + proxyPort);
httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort);
}
}
public Response httpRequest(HttpMethod method, Boolean WithTokenHeader, String token)
throws WeiboException {
InetAddress ipaddr;
int responseCode = -1;
try {
ipaddr = InetAddress.getLocalHost();
List<Header> headers = new ArrayList<Header>();
if (WithTokenHeader) {
if (token == null) {
throw new IllegalStateException("Oauth2 token is not set!");
}
headers.add(new Header("Authorization", "OAuth2 " + token));
headers.add(new Header("API-RemoteIP", ipaddr.getHostAddress()));
client.getHostConfiguration().getParams().setParameter("http.default-headers", headers);
for (Header hd : headers) {
log(hd.getName() + ": " + hd.getValue());
}
}
method
.getParams()
.setParameter(
HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
client.executeMethod(method);
Header[] resHeader = method.getResponseHeaders();
responseCode = method.getStatusCode();
log("Response:");
log("https StatusCode:" + String.valueOf(responseCode));
for (Header header : resHeader) {
log(header.getName() + ":" + header.getValue());
}
Response response = new Response();
response.setResponseAsString(method.getResponseBodyAsString());
log(response.toString() + "\n");
if (responseCode != OK) {
try {
throw new WeiboException(
getCause(responseCode), response.asJSONObject(), method.getStatusCode());
} catch (JSONException e) {
e.printStackTrace();
}
}
return response;
} catch (IOException ioe) {
throw new WeiboException(ioe.getMessage(), ioe, responseCode);
} finally {
method.releaseConnection();
}
}
public RestaurantRestClient(String urlStub) {
httpClient.getHttpConnectionManager().getParams().setMaxTotalConnections(10);
httpClient
.getHttpConnectionManager()
.getParams()
.setMaxConnectionsPerHost(httpClient.getHostConfiguration(), 10);
restClient.setErrorHandler(new MyErrorHandler());
this.urlStub = urlStub;
// List<HttpMessageConverter<?>> converters = new ArrayList<>();
// converters.add(converter);
// restClient.setMessageConverters(converters);
// http://localhost:8888/app/backbone/restaurant/
}
@Test
public void testSendSMS() {
String reqUrl = "http://59.36.98.25/JSMSWebServiceNew/send.aspx";
long start = System.currentTimeMillis();
// String requestJson =
// "{\"code\":\"GDCN\",\"destPhone\":\"15018784308\",\"content\":\"文字短信测试\",\"corpSignName\":\"公司签名\",\"userSignName\":\"用户签名\",\"corpAccount\":\"system\",\"userAccount\":\"admin\",\"password\":\"7236606dcece6d04ebd8a7b42eb247e0\",\"messageType\":\"1\"}";
try {
HttpClient client = new HttpClient();
client.getHostConfiguration().setHost("localhost", 8080, "http");
PostMethod post = new PostMethod(reqUrl);
post.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8");
post.setParameter("code", "GDCN");
post.setParameter("destPhone", "15018784308");
post.setParameter(
"content",
"尽力总有进步，尽心总有收获，努力总有成效，付出总有回报，前行总有风景，坚持总有希望，豁达总有快乐，拼搏总有运气。有时闲，有时累，不怨不悔，把快乐品味，愿你时时与幸福相会！");
post.setParameter("corpSignName", "中数通");
// post.setParameter("userSignName", "用户签名");
post.setParameter("corpAccount", "system");
post.setParameter("userAccount", "admin");
post.setParameter("password", "7236606dcece6d04ebd8a7b42eb247e0");
post.setParameter("messageType", "1");
/*String y=URLEncoder.encode("尽力总有进步，尽心总有收获，努力总有成效，付出总有回报，前行总有风景，坚持总有希望，豁达总有快乐，拼搏总有运气。有时闲，有时累，不怨不悔，把快乐品味，愿你时时与幸福相会！","utf-8");
System.out.println(y);
String s=URLDecoder.decode("%e4%b8%8b%e8%a1%8c%e6%b6%88%e6%81%af", "utf-8");
System.out.println(s);*/
// System.out.println("请求Json： " + requestJson);
// post.setRequestBody(new ByteArrayInputStream(requestJson)));
/*System.out.println("请求Json的getBytes： " + EncodeUtils
.base64Encode(requestJson.getBytes("UTF-8")).getBytes());*/
int result = client.executeMethod(post);
if (result == 200) {
String responseJson = post.getResponseBodyAsString();
System.out.println("响应JSON： " + responseJson);
}
} catch (Exception e) {
e.printStackTrace();
}
long end = System.currentTimeMillis();
System.out.println("zhixingshijian:>>>>>>>>>" + (end - start));
}
/**
* Obtem a timeline publica. Nao necessita de autenticacao, basicamente é um GET na URL
* http://twitter.com/statuses/public_timeline.xml
*/
public String getPublicTimeline() throws Exception {
HttpClient client = new HttpClient();
// get para obter a timeline publica
GetMethod get = new GetMethod("/statuses/public_timeline.xml");
// Criando o host, para onde sera enviada a requisicao
HostConfiguration host = client.getHostConfiguration();
host.setHost(new URI("http://twitter.com"));
// executando o get na url de timeline publica do twitter
client.executeMethod(host, get);
// obtendo a resposta
return get.getResponseBodyAsString();
}
public static void main(String[] args) throws Exception {
HttpClient client = new HttpClient();
client.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF8");
// 原先采用自己合并COOKIE，但是出现问题，用此方法OK
DefaultHttpParams.getDefaultParams()
.setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
client
.getParams()
.setParameter(
HttpMethodParams.USER_AGENT,
"Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) Gecko/20090803 Fedora/3.5.2-2.fc11 Firefox/3.5.2");
client.getHostConfiguration().setHost(LOGON_SITE, LOGON_PORT);
String ajaxLoginUrl = Login(client, preLogin(client));
String uniqueid = ajaxLogin(client, ajaxLoginUrl);
getHomePage(client, homePageUrl + uniqueid);
}
/**
* Sends a new HTTP request to a server through a proxy.
*
* @param requestParameters a set of parameters that will set the content of the request and
* specify the proxy it should go through
*/
public void callWilmaTestServer(
final RequestParameters requestParameters, final TestClientParameters clientParameters) {
try {
HttpClient httpClient = new HttpClient();
PostMethod httpPost = new PostMethod(requestParameters.getTestServerUrl());
if (clientParameters.isUseProxy()) {
httpClient
.getHostConfiguration()
.setProxy(requestParameters.getWilmaHost(), requestParameters.getWilmaPort());
}
InputStream inputStream = requestParameters.getInputStream();
if (requestParameters.getContentType().contains("fastinfoset")) {
inputStream = compress(inputStream);
}
if (requestParameters.getContentEncoding().contains("gzip")) {
inputStream = encode(inputStream);
httpPost.setRequestHeader("Content-Encoding", requestParameters.getContentEncoding());
}
InputStreamRequestEntity entity =
new InputStreamRequestEntity(inputStream, requestParameters.getContentType());
httpPost.setRequestEntity(entity);
httpPost.setRequestHeader("Accept", requestParameters.getAcceptHeader());
httpPost.addRequestHeader("Accept-Encoding", requestParameters.getAcceptEncoding());
// httpPost.addRequestHeader("0", "WilmaBypass=true");
httpClient
.getHttpConnectionManager()
.getParams()
.setSendBufferSize(clientParameters.getRequestBufferSize());
httpClient
.getHttpConnectionManager()
.getParams()
.setReceiveBufferSize(clientParameters.getResponseBufferSize());
int statusCode = httpClient.executeMethod(httpPost);
logger.info("status code: " + statusCode);
if (clientParameters.getAllowResponseLogging()) {
logger.info(getInputStreamAsString(httpPost.getResponseBodyAsStream()));
}
} catch (UnsupportedEncodingException e) {
throw new SystemException("Unsupported encoding.", e);
} catch (HttpException e) {
throw new SystemException("Http exception occurred.", e);
} catch (IOException e) {
throw new SystemException("InputStream cannot be read.", e);
}
}
private org.apache.commons.httpclient.HttpClient getHttpClient() {
org.apache.commons.httpclient.HttpClient client =
new org.apache.commons.httpclient.HttpClient();
if (proxyHost != null && !proxyHost.equals("")) {
client.getHostConfiguration().setProxy(proxyHost, proxyPort);
client.getParams().setAuthenticationPreemptive(true);
if (proxyAuthUser != null && !proxyAuthUser.equals("")) {
client
.getState()
.setProxyCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword));
log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword);
}
}
return client;
}
/**
* 提交
*
* @param method
* @param token
* @return
* @throws ApiException
*/
private static String httpRequest(HttpMethod method, String token) throws IOException {
try {
List<Header> headers = new ArrayList<Header>();
if (token != null) {
headers.add(new Header("Authorization", "Bearer " + token));
client.getHostConfiguration().getParams().setParameter("http.default-headers", headers);
}
method
.getParams()
.setParameter(
HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
client.executeMethod(method);
return method.getResponseBodyAsString();
} catch (IOException ioe) {
throw ioe;
} finally {
method.releaseConnection();
}
}
/**
* 登录
*
* @return
* @throws IOException
* @throws HttpException
*/
public static HttpClient login() throws IOException, HttpException {
HttpClient client = new HttpClient();
client.getHostConfiguration().setHost("www.catarc.org.cn");
client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
NameValuePair[] data = {
new NameValuePair(
"User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:25.0) Gecko/20100101 Firefox/25.0"),
new NameValuePair("__VIEWSTATE", "/wEPDwUJNzE2NjczNjA4ZGQCaV1Dt0DOUg5r7O3ZD+sGL1Hq6Q=="),
new NameValuePair(
"__EVENTVALIDATION",
"/wEWBQK/2uyqBgKl1bKzCQK1qbSRCwLZpaCZAwKTuvSuCTYE9T8nbPNCPEY2eqmCy5I3BwtG"),
new NameValuePair("txtUserName", "asri"),
new NameValuePair("txtPassword", "asri"),
new NameValuePair("loginBtn", "%B5%C7%C2%BD")
};
PostMethod post = new PostMethod("http://www.catarc.org.cn/standard/Default.aspx");
post.setRequestBody(data);
client.executeMethod(post);
return client;
}
/** 初始化httpclient */
static {
int connectionTimeout = 30000;
int soTimeout = 30000;
try {
connectionTimeout =
Integer.parseInt(System.getProperty("sun.net.client.defaultConnectTimeout", "30000"));
} catch (Exception e) {
}
try {
soTimeout =
Integer.parseInt(System.getProperty("sun.net.client.defaultReadTimeout", "30000"));
} catch (Exception e) {
}
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
connectionManager.getParams().setDefaultMaxConnectionsPerHost(10);
connectionManager.getParams().setMaxTotalConnections(300);
connectionManager.getParams().setConnectionTimeout(connectionTimeout);
connectionManager.getParams().setSoTimeout(soTimeout);
client.setHttpConnectionManager(connectionManager);
// 忽略cookie 避免 Cookie rejected 警告
HttpClientParams clientParams = new HttpClientParams();
clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
client.setParams(clientParams);
// 支持https
Protocol myhttps = new Protocol("https", new SSLSocketFactory(), 443);
Protocol.registerProtocol("https", myhttps);
// 设置代理
if (ProxyClient.getProxy() != null) {
client.getHostConfiguration().setProxy(ProxyClient.getHost(), ProxyClient.getPort());
client.getParams().setAuthenticationPreemptive(true);
if (ProxyClient.getUsername() != null && !ProxyClient.getUsername().trim().equals("")) {
client
.getState()
.setProxyCredentials(
AuthScope.ANY,
new UsernamePasswordCredentials(
ProxyClient.getUsername().trim(), ProxyClient.getPassword().trim()));
}
}
}
private void configureHttpMethod(
boolean skipContentCache, CacheData cacheData, long onceTimeOut, HttpMethod httpMethod) {
if (skipContentCache && null != cacheData) {
if (null != cacheData.getLastModifiedHeader()
&& Constants.NULL != cacheData.getLastModifiedHeader()) {
httpMethod.addRequestHeader(Constants.IF_MODIFIED_SINCE, cacheData.getLastModifiedHeader());
}
if (null != cacheData.getMd5() && Constants.NULL != cacheData.getMd5()) {
httpMethod.addRequestHeader(Constants.CONTENT_MD5, cacheData.getMd5());
}
}
httpMethod.addRequestHeader(Constants.ACCEPT_ENCODING, "gzip,deflate");
HttpMethodParams params = new HttpMethodParams();
params.setSoTimeout((int) onceTimeOut);
httpMethod.setParams(params);
httpClient
.getHostConfiguration()
.setHost(
diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
diamondConfigure.getPort());
}
protected HttpClient getHttpClient() {
HttpClient client = new HttpClient();
if (Jenkins.getInstance() != null) {
ProxyConfiguration proxy = Jenkins.getInstance().proxy;
if (proxy != null) {
client.getHostConfiguration().setProxy(proxy.name, proxy.port);
String username = proxy.getUserName();
String password = proxy.getPassword();
// Consider it to be passed if username specified. Sufficient?
if (username != null && !"".equals(username.trim())) {
logger.info("Using proxy authentication (user="******")");
// http://hc.apache.org/httpclient-3.x/authentication.html#Proxy_Authentication
// and
// http://svn.apache.org/viewvc/httpcomponents/oac.hc3x/trunk/src/examples/BasicAuthenticationExample.java?view=markup
client
.getState()
.setProxyCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
}
}
}
return client;
}
/**
* @param sProxyUser
* @param sProxyPassswd
* @return an HTTP client
*/
private static HttpClient getHTTPClient(
String sProxyUser, String sProxyPassswd, int iConTimeout, int iDataTimeout) {
HttpClient client = new HttpClient();
client
.getHttpConnectionManager()
.getParams()
.setConnectionTimeout(iConTimeout); // connection to
client
.getHttpConnectionManager()
.getParams()
.setSoTimeout(iDataTimeout); // data reception timeout
if (sProxyUser != null && sProxyPassswd != null) {
client
.getHostConfiguration()
.setProxy(
ConfigurationManager.getProperty(CONF_NETWORK_PROXY_HOSTNAME),
Integer.parseInt(ConfigurationManager.getProperty(CONF_NETWORK_PROXY_PORT)));
client
.getState()
.setProxyCredentials(
new AuthScope(AuthScope.ANY), new UsernamePasswordCredentials(sProxyUser, sProxyPwd));
}
return client;
}
public HttpTemplateDownloader(
StorageLayer storageLayer,
String downloadUrl,
String toDir,
DownloadCompleteCallback callback,
long maxTemplateSizeInBytes,
String user,
String password,
Proxy proxy,
ResourceType resourceType) {
this._storage = storageLayer;
this.downloadUrl = downloadUrl;
this.setToDir(toDir);
this.status = TemplateDownloader.Status.NOT_STARTED;
this.resourceType = resourceType;
this.MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes;
this.totalBytes = 0;
this.client = new HttpClient();
myretryhandler =
new HttpMethodRetryHandler() {
public boolean retryMethod(
final HttpMethod method, final IOException exception, int executionCount) {
if (executionCount >= 2) {
// Do not retry if over max retry count
return false;
}
if (exception instanceof NoHttpResponseException) {
// Retry if the server dropped connection on us
return true;
}
if (!method.isRequestSent()) {
// Retry if the request has not been sent fully or
// if it's OK to retry methods that have been sent
return true;
}
// otherwise do not retry
return false;
}
};
try {
this.request = new GetMethod(downloadUrl);
this.request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
this.completionCallback = callback;
// this.request.setFollowRedirects(false);
File f = File.createTempFile("dnld", "tmp_", new File(toDir));
if (_storage != null) {
_storage.setWorldReadableAndWriteable(f);
}
toFile = f.getAbsolutePath();
Pair<String, Integer> hostAndPort = validateUrl(downloadUrl);
if (proxy != null) {
client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
if (proxy.getUserName() != null) {
Credentials proxyCreds =
new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword());
client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds);
}
}
if ((user != null) && (password != null)) {
client.getParams().setAuthenticationPreemptive(true);
Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
client
.getState()
.setCredentials(
new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM),
defaultcreds);
s_logger.info(
"Added username="******", password="******"for host "
+ hostAndPort.first()
+ ":"
+ hostAndPort.second());
} else {
s_logger.info(
"No credentials configured for host="
+ hostAndPort.first()
+ ":"
+ hostAndPort.second());
}
} catch (IllegalArgumentException iae) {
errorString = iae.getMessage();
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
inited = false;
} catch (Exception ex) {
errorString = "Unable to start download -- check url? ";
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
s_logger.warn("Exception in constructor -- " + ex.toString());
} catch (Throwable th) {
s_logger.warn("throwable caught ", th);
}
}
/** Clear the proxy setting. */
public void clearProxy() {
client.getHostConfiguration().setProxyHost(null);
}
/**
* Set a proxy that should be used by the client when trying to access the server. If this is not
* set, the client will attempt to make a direct direct connection to the server.
*
* @param host The name of the host.
* @param port The port.
*/
public void setProxy(String host, int port) {
client.getHostConfiguration().setProxy(host, port);
}
// 授权，生成access_token
// 使用情形：①程序初始化；②每隔一天左右重新授权access_token
public static void generate() {
initAccountInfo();
accessToken.clear();
logger.info("用户授权中...");
try {
// https://api.weibo.com/oauth2/authorize?client_id=750123511&redirect_uri=https://api.weibo.com/oauth2/default.html&response_type=code
String url = "https://api.weibo.com/oauth2/authorize";
String redirectUri = "https://api.weibo.com/oauth2/default.html";
for (int i = 0; i < accountInfo.size(); i++) {
// 获取应用的信息
clientId = WeiboConfig.getValue("client_ID");
clientSecret = WeiboConfig.getValue("client_SERCRET");
// 构造授权的url参数
PostMethod postMethod = new PostMethod(url);
postMethod.addParameter("client_id", clientId);
postMethod.addParameter("redirect_uri", redirectUri);
postMethod.addParameter("userId", accountInfo.get(i).getUserId());
postMethod.addParameter("passwd", accountInfo.get(i).getPasswd());
postMethod.addParameter("isLoginSina", "0");
postMethod.addParameter("action", "submit");
postMethod.addParameter("response_type", "code");
HttpMethodParams param = postMethod.getParams();
param.setContentCharset("UTF-8");
// 伪造头部域信息
List<Header> headers = new ArrayList<Header>();
headers.add(
new Header(
"Referer",
"https://api.weibo.com/oauth2/authorize?client_id="
+ clientId
+ "&redirect_uri="
+ redirectUri
+ "&from=sina&response_type=code"));
headers.add(new Header("Host", "api.weibo.com"));
headers.add(
new Header(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0"));
// 发送HTTP请求
HttpClient client = new HttpClient();
client.getHostConfiguration().getParams().setParameter("http.default-headers", headers);
client.executeMethod(postMethod);
// 获取授权响应
int status = postMethod.getStatusCode();
if (status == 302) {
Header location = postMethod.getResponseHeader("location");
if (location != null) {
String retUrl = location.getValue();
int begin = retUrl.indexOf("code=");
int end = retUrl.length();
String code = retUrl.substring(begin + 5, end);
if (code != null) {
Oauth oauth = new Oauth();
String token = oauth.getAccessTokenByCode(code).getAccessToken();
accessToken.add(token);
logger.info("第" + (i + 1) + "个access_token：" + token);
}
}
} else {
logger.error("第" + (i + 1) + "个用户授权失败了！");
}
}
} catch (Exception e) {
e.printStackTrace();
logger.error("授权发生异常！");
}
}
Set<String> checkUpdateDataIds(long timeout) {
if (!isRun) {
throw new RuntimeException("DiamondSubscriber is not running. checkUpdateDataIds return.");
}
if (MockServer.isTestMode()) {
return testData();
}
long waitTime = 0;
String probeUpdateString = getProbeUpdateString();
if (StringUtils.isBlank(probeUpdateString)) {
return null;
}
while (0 == timeout || timeout > waitTime) {
long onceTimeOut = getOnceTimeOut(waitTime, timeout);
waitTime += onceTimeOut;
PostMethod postMethod = new PostMethod(Constants.HTTP_URI_FILE);
postMethod.addParameter(Constants.PROBE_MODIFY_REQUEST, probeUpdateString);
HttpMethodParams params = new HttpMethodParams();
params.setSoTimeout((int) onceTimeOut);
postMethod.setParams(params);
try {
httpClient
.getHostConfiguration()
.setHost(
diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
this.diamondConfigure.getPort());
int httpStatus = httpClient.executeMethod(postMethod);
switch (httpStatus) {
case SC_OK:
{
Set<String> result = getUpdateDataIds(postMethod);
return result;
}
case SC_SERVICE_UNAVAILABLE:
{
rotateToNextDomain();
}
break;
default:
{
log.warn("checkUpdateDataIds HTTP State: " + httpStatus);
rotateToNextDomain();
}
}
} catch (HttpException e) {
log.error("checkUpdateDataIds HttpException", e);
rotateToNextDomain();
} catch (IOException e) {
log.error("checkUpdateDataIds IOException", e);
rotateToNextDomain();
} catch (Exception e) {
log.error("checkUpdateDataIds Unknown Exception", e);
rotateToNextDomain();
} finally {
postMethod.releaseConnection();
}
}
throw new RuntimeException(
"checkUpdateDataIds timeout "
+ diamondConfigure.getDomainNameList().get(this.domainNamePos.get())
+ ", timeout="
+ timeout);
}
public byte[] timeStamp(byte[] data, RevocationData revocationData) throws Exception {
// digest the message
MessageDigest messageDigest = MessageDigest.getInstance(this.digestAlgo);
byte[] digest = messageDigest.digest(data);
// generate the TSP request
BigInteger nonce = new BigInteger(128, new SecureRandom());
TimeStampRequestGenerator requestGenerator = new TimeStampRequestGenerator();
requestGenerator.setCertReq(true);
if (null != this.requestPolicy) {
requestGenerator.setReqPolicy(this.requestPolicy);
}
TimeStampRequest request = requestGenerator.generate(this.digestAlgoOid, digest, nonce);
byte[] encodedRequest = request.getEncoded();
// create the HTTP client
HttpClient httpClient = new HttpClient();
if (null != this.username) {
Credentials credentials = new UsernamePasswordCredentials(this.username, this.password);
httpClient.getState().setCredentials(AuthScope.ANY, credentials);
}
if (null != this.proxyHost) {
httpClient.getHostConfiguration().setProxy(this.proxyHost, this.proxyPort);
}
// create the HTTP POST request
PostMethod postMethod = new PostMethod(this.tspServiceUrl);
RequestEntity requestEntity =
new ByteArrayRequestEntity(encodedRequest, "application/timestamp-query");
postMethod.addRequestHeader("User-Agent", this.userAgent);
postMethod.setRequestEntity(requestEntity);
// invoke TSP service
int statusCode = httpClient.executeMethod(postMethod);
if (HttpStatus.SC_OK != statusCode) {
LOG.error("Error contacting TSP server " + this.tspServiceUrl);
throw new Exception("Error contacting TSP server " + this.tspServiceUrl);
}
// HTTP input validation
Header responseContentTypeHeader = postMethod.getResponseHeader("Content-Type");
if (null == responseContentTypeHeader) {
throw new RuntimeException("missing Content-Type header");
}
String contentType = responseContentTypeHeader.getValue();
if (!contentType.startsWith("application/timestamp-reply")) {
LOG.debug("response content: " + postMethod.getResponseBodyAsString());
throw new RuntimeException("invalid Content-Type: " + contentType);
}
if (0 == postMethod.getResponseContentLength()) {
throw new RuntimeException("Content-Length is zero");
}
// TSP response parsing and validation
InputStream inputStream = postMethod.getResponseBodyAsStream();
TimeStampResponse timeStampResponse = new TimeStampResponse(inputStream);
timeStampResponse.validate(request);
if (0 != timeStampResponse.getStatus()) {
LOG.debug("status: " + timeStampResponse.getStatus());
LOG.debug("status string: " + timeStampResponse.getStatusString());
PKIFailureInfo failInfo = timeStampResponse.getFailInfo();
if (null != failInfo) {
LOG.debug("fail info int value: " + failInfo.intValue());
if (PKIFailureInfo.unacceptedPolicy == failInfo.intValue()) {
LOG.debug("unaccepted policy");
}
}
throw new RuntimeException(
"timestamp response status != 0: " + timeStampResponse.getStatus());
}
TimeStampToken timeStampToken = timeStampResponse.getTimeStampToken();
SignerId signerId = timeStampToken.getSID();
BigInteger signerCertSerialNumber = signerId.getSerialNumber();
X500Principal signerCertIssuer = signerId.getIssuer();
LOG.debug("signer cert serial number: " + signerCertSerialNumber);
LOG.debug("signer cert issuer: " + signerCertIssuer);
// TSP signer certificates retrieval
CertStore certStore =
timeStampToken.getCertificatesAndCRLs("Collection", BouncyCastleProvider.PROVIDER_NAME);
Collection<? extends Certificate> certificates = certStore.getCertificates(null);
X509Certificate signerCert = null;
Map<String, X509Certificate> certificateMap = new HashMap<String, X509Certificate>();
for (Certificate certificate : certificates) {
X509Certificate x509Certificate = (X509Certificate) certificate;
if (signerCertIssuer.equals(x509Certificate.getIssuerX500Principal())
&& signerCertSerialNumber.equals(x509Certificate.getSerialNumber())) {
signerCert = x509Certificate;
}
String ski = Hex.encodeHexString(getSubjectKeyId(x509Certificate));
certificateMap.put(ski, x509Certificate);
LOG.debug(
"embedded certificate: " + x509Certificate.getSubjectX500Principal() + "; SKI=" + ski);
}
// TSP signer cert path building
if (null == signerCert) {
throw new RuntimeException("TSP response token has no signer certificate");
}
List<X509Certificate> tspCertificateChain = new LinkedList<X509Certificate>();
X509Certificate certificate = signerCert;
do {
LOG.debug("adding to certificate chain: " + certificate.getSubjectX500Principal());
tspCertificateChain.add(certificate);
if (certificate.getSubjectX500Principal().equals(certificate.getIssuerX500Principal())) {
break;
}
String aki = Hex.encodeHexString(getAuthorityKeyId(certificate));
certificate = certificateMap.get(aki);
} while (null != certificate);
// verify TSP signer signature
timeStampToken.validate(tspCertificateChain.get(0), BouncyCastleProvider.PROVIDER_NAME);
// verify TSP signer certificate
this.validator.validate(tspCertificateChain, revocationData);
LOG.debug("time-stamp token time: " + timeStampToken.getTimeStampInfo().getGenTime());
byte[] timestamp = timeStampToken.getEncoded();
return timestamp;
}
@Override
public BatchHttpResult getConfigureInformationBatch(
List<String> dataIds, String group, int timeout) {
if (dataIds == null) {
log.error("dataId list cannot be null,group=" + group);
return new BatchHttpResult(HttpStatus.SC_BAD_REQUEST);
}
if (group == null) {
group = Constants.DEFAULT_GROUP;
}
StringBuilder dataIdBuilder = new StringBuilder();
for (String dataId : dataIds) {
dataIdBuilder.append(dataId).append(Constants.LINE_SEPARATOR);
}
String dataIdStr = dataIdBuilder.toString();
PostMethod post = new PostMethod(Constants.HTTP_URI_FILE_BATCH);
post.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, timeout);
BatchHttpResult response = null;
try {
NameValuePair dataIdValue = new NameValuePair("dataIds", dataIdStr);
NameValuePair groupValue = new NameValuePair("group", group);
post.setRequestBody(new NameValuePair[] {dataIdValue, groupValue});
httpClient
.getHostConfiguration()
.setHost(
diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
this.diamondConfigure.getPort());
int status = httpClient.executeMethod(post);
String responseMsg = post.getResponseBodyAsString();
if (status == HttpStatus.SC_OK) {
String json = null;
try {
json = responseMsg;
List<ConfigInfoEx> configInfoExList = new LinkedList<ConfigInfoEx>();
Object resultObj =
JSONUtils.deserializeObject(json, new TypeReference<List<ConfigInfoEx>>() {});
if (!(resultObj instanceof List<?>)) {
throw new RuntimeException(
"batch query deserialize type error, not list, json=" + json);
}
List<ConfigInfoEx> resultList = (List<ConfigInfoEx>) resultObj;
for (ConfigInfoEx configInfoEx : resultList) {
configInfoExList.add(configInfoEx);
}
response = new BatchHttpResult(configInfoExList);
log.info(
"batch query success,dataIds=" + dataIdStr + ",group=" + group + ",json=" + json);
} catch (Exception e) {
response = new BatchHttpResult(Constants.BATCH_OP_ERROR);
log.error(
"batch query deserialize error,dataIdStr="
+ dataIdStr
+ ",group="
+ group
+ ",json="
+ json,
e);
}
} else if (status == HttpStatus.SC_REQUEST_TIMEOUT) {
response = new BatchHttpResult(HttpStatus.SC_REQUEST_TIMEOUT);
log.error(
"batch query timeout, socket timeout(ms):"
+ timeout
+ ",dataIds="
+ dataIdStr
+ ",group="
+ group);
} else {
response = new BatchHttpResult(status);
log.error(
"batch query fail, status:"
+ status
+ ", response:"
+ responseMsg
+ ",dataIds="
+ dataIdStr
+ ",group="
+ group);
}
} catch (HttpException e) {
response = new BatchHttpResult(Constants.BATCH_HTTP_EXCEPTION);
log.error("batch query http exception,dataIds=" + dataIdStr + ",group=" + group, e);
} catch (IOException e) {
response = new BatchHttpResult(Constants.BATCH_IO_EXCEPTION);
log.error("batch query io exception, dataIds=" + dataIdStr + ",group=" + group, e);
} finally {
post.releaseConnection();
}
return response;
}
/**
* The method reads a proxy object from database, makes a GetMethod object, appends required
* cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
* pagesource to the caller.
*
* @param iCount Counter variable for passing thread group information
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @param region the local region of a given url
* @param objProxyDao the database layer ProxyDao object variable
* @param useErrsy Boolean variable to specify usage of Errsy as proxy source
* @return String
*/
public String getPageSourceWithProxy(
String url,
boolean followRedirect,
boolean doAuthentication,
String region,
Boolean useErrsy,
String google) {
String page = " ";
String pageSource = "";
int i = 0;
String exception = " ";
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod getmethod = null;
HttpState state = new HttpState();
// if (ProxyDao.lstProxyData.size() == 16) {
ProxyData objProxyData = null;
// if (!useErrsy) {
try {
// objProxyData = ProxyDao.lstProxyData.get(iCount);
objProxyData = ProxyDao.objProxyData;
if (objProxyData == null) {
// objProxyDao.changeProxy(google);
objProxyData = ProxyDao.objProxyData;
}
httpclient
.getHostConfiguration()
.setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
/*} else {
try {
objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
httpclient.setState(state);
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
}*/
try {
getmethod = new GetMethod(url);
getmethod.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
if (url.contains("bing.com")) {
if (region.equalsIgnoreCase("co.uk")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.sg")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.au")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("co.in")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("ca")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.ph")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.my")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("it")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
}
}
getmethod.setFollowRedirects(true);
getmethod.setDoAuthentication(true);
httpclient.getParams().setAuthenticationPreemptive(true);
httpclient.setState(state);
String num100Header = "";
// if (url.contains("google")) {
// int j = 0;
// String url1 = "http://www.google.com/";
// try {
// GetMethod objGetMethod = new GetMethod(url1);
// j = httpclient.executeMethod(objGetMethod);
// Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
// String header = responseHeader.getValue();
// String[] headerValue = header.split(";");
//
// for (String head : headerValue) {
// if (head.contains("PREF=ID")) {
// header = head;
// break;
// }
// }
// String[] splitAll = header.split(":");
// long time = System.currentTimeMillis()+400;
// String sTime = "" + time;
// sTime = sTime.substring(0, 10);
// //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
// num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
// Cookie ck = new Cookie("PREF", "PREF", num100Header);
// httpclient.getState().clearCookies();
// httpclient.getState().addCookie(ck);
// getmethod.addRequestHeader("Host", "www.google.com");
// getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
// rv:19.0) Gecko/20100101 Firefox/19.0");
// getmethod.addRequestHeader("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
// getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
// getmethod.addRequestHeader("Referer", "https://www.google.com/");
// System.out.println(num100Header);
// } catch (Exception ex) {
// exception = ex.getMessage();
// l.debug(ex + " " + ex.getMessage() + "Exception occured for url" +
// url);
// pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
// return pageSource;
// }
// }
i = httpclient.executeMethod(getmethod);
if (i / 100 == 4 || i / 100 == 5) {
page = "<PROXY ERROR>";
} else {
page = getmethod.getResponseBodyAsString();
}
} catch (SocketTimeoutException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (SocketException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (Exception ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} finally {
getmethod.releaseConnection();
}
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
// }
return pageSource;
}