private DataClient(Builder builder) {
/** * REPOSITORY HOST:PORT ** */
repoHostname = builder.hostname;
repoHostPort = builder.hostPort;
/** * STORE INFO ** */
storeKey = builder.storeKey;
storeType = builder.storeType;
/** * PRIVATE CREDENTIONALS - DEFAULT TO NONE ** */
memberId = builder.memberId;
memberPassword = builder.memberPassword;
/**
* * THE "path" IS THE METADATA INFO FOR EXAMPLE: Let say you want to create a finance dev
* environment database data structure. The path should be "finance/dev/env/db" To add a value
* for example db named fndb1 Submit a PUT request to /data/finance/dev/env {db:"fndb1"}
*
* <p>*
*/
client = new HttpClient();
List authPrefs = new ArrayList(2);
authPrefs.add(AuthPolicy.BASIC);
client.getParams().setParameter(AuthPolicy.AUTH_SCHEME_PRIORITY, authPrefs);
}
protected void construct(String legalurl) throws HTTPException {
this.legalurl = legalurl;
try {
sessionClient = new HttpClient(connmgr);
HttpClientParams clientparams = sessionClient.getParams();
// Allow (circular) redirects
clientparams.setParameter(ALLOW_CIRCULAR_REDIRECTS, true);
clientparams.setParameter(MAX_REDIRECTS, 25);
if (globalSoTimeout > 0) setSoTimeout(globalSoTimeout);
if (globalConnectionTimeout > 0) setConnectionTimeout(globalConnectionTimeout);
if (globalAgent != null) setUserAgent(globalAgent); // May get overridden by setUserAgent
setAuthenticationPreemptive(globalauthpreemptive);
setProxy();
if (TESTING) HTTPSession.track(this);
} catch (Exception e) {
throw new HTTPException("url=" + legalurl, e);
}
}
private String getRequest(String path) {
logger.log(Level.FINEST, "PR-GET-REQUEST:" + path);
HttpClient client = getHttpClient();
client.getState().setCredentials(AuthScope.ANY, credentials);
GetMethod httpget = new GetMethod(path);
client.getParams().setAuthenticationPreemptive(true);
String response = null;
int responseCode;
try {
responseCode = client.executeMethod(httpget);
InputStream responseBodyAsStream = httpget.getResponseBodyAsStream();
StringWriter stringWriter = new StringWriter();
IOUtils.copy(responseBodyAsStream, stringWriter, "UTF-8");
response = stringWriter.toString();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Failed to process PR get request; " + path, e);
}
logger.log(Level.FINEST, "PR-GET-RESPONSE:" + response);
if (!validResponseCode(responseCode)) {
logger.log(Level.SEVERE, "Failing to get response from Stash PR GET" + path);
throw new RuntimeException(
"Didn't get a 200 response from Stash PR GET! Response; '"
+ HttpStatus.getStatusText(responseCode)
+ "' with message; "
+ response);
}
return response;
}
public void deleteRequest(String path) {
HttpClient client = getHttpClient();
client.getState().setCredentials(AuthScope.ANY, credentials);
DeleteMethod httppost = new DeleteMethod(path);
client.getParams().setAuthenticationPreemptive(true);
int res = -1;
try {
res = client.executeMethod(httppost);
} catch (IOException e) {
e.printStackTrace();
}
logger.log(Level.FINE, "Delete comment {" + path + "} returned result code; " + res);
}
/**
* Run the first test.
*
* @param string a part of URL to connect to.
* @return ManagerClient object
* @throws IOException for any failures.
*/
public ManagerClient(String string) throws Exception {
URL = "http://" + string + "/mod_cluster_manager/";
GetMethod gm = null;
HttpMethodBase bm = null;
if (httpClient == null) {
httpClient = new HttpClient();
gm = new GetMethod(URL);
bm = gm;
}
System.out.println("Connecting to " + URL);
Integer connectionTimeout = 40000;
bm.getParams().setParameter("http.socket.timeout", connectionTimeout);
bm.getParams().setParameter("http.connection.timeout", connectionTimeout);
httpClient.getParams().setParameter("http.socket.timeout", connectionTimeout);
httpClient.getParams().setParameter("http.connection.timeout", connectionTimeout);
try {
httpResponseCode = httpClient.executeMethod(gm);
if (httpResponseCode == 200) {
// Read the nonce.
String result = gm.getResponseBodyAsString();
String[] records = result.split("\n");
for (int i = 0; i < records.length; i++) {
int j = records[i].indexOf("?nonce=");
if (j < 0) continue;
j = j + 7;
String nnonce = records[i].substring(j);
int k = nnonce.indexOf('&');
if (k > 0) {
nonce = nnonce.substring(0, k);
break;
}
}
} else {
System.out.println("response: " + httpResponseCode);
System.out.println("response: " + bm.getStatusLine());
throw (new Exception("Reponse notok"));
}
// System.out.println("response:\n" + bm.getResponseBodyAsString(len));
} catch (HttpException e) {
System.out.println("error: " + e);
throw (e);
}
bm.releaseConnection();
}
/** 根据模版及参数产生静态页面 */
public boolean createHtmlPage(String url, String htmlFileName) {
boolean status = false;
int statusCode = 0;
try {
// 创建一个HttpClient实例充当模拟浏览器
httpClient = new HttpClient();
// 设置httpclient读取内容时使用的字符集
httpClient.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "gbk");
// 创建GET方法的实例
getMethod = new GetMethod(url);
// 使用系统提供的默认的恢复策略，在发生异常时候将自动重试3次
getMethod
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
// 设置Get方法提交参数时使用的字符集,以支持中文参数的正常传递
getMethod.addRequestHeader("Content-Type", "text/html;charset=gbk");
// 执行Get方法并取得返回状态码，200表示正常，其它代码为异常
statusCode = httpClient.executeMethod(getMethod);
if (statusCode != 200) {
logger.fatal("静态页面引擎在解析" + url + "产生静态页面" + htmlFileName + "时出错!");
} else {
// 读取解析结果
sb = new StringBuffer();
in = getMethod.getResponseBodyAsStream();
br = new BufferedReader(new InputStreamReader(in));
while ((line = br.readLine()) != null) {
sb.append(line + "\n");
}
if (br != null) br.close();
page = sb.toString();
// 将页面中的相对路径替换成绝对路径，以确保页面资源正常访问
page = formatPage(page);
// 将解析结果写入指定的静态HTML文件中，实现静态HTML生成
writeHtml(htmlFileName, page);
status = true;
}
} catch (Exception ex) {
logger.fatal("静态页面引擎在解析" + url + "产生静态页面" + htmlFileName + "时出错:" + ex.getMessage());
} finally {
// 释放http连接
getMethod.releaseConnection();
}
return status;
}
private String postRequest(String path, String comment) throws UnsupportedEncodingException {
logger.log(Level.FINEST, "PR-POST-REQUEST:" + path + " with: " + comment);
HttpClient client = getHttpClient();
client.getState().setCredentials(AuthScope.ANY, credentials);
PostMethod httppost = new PostMethod(path);
ObjectNode node = mapper.getNodeFactory().objectNode();
node.put("text", comment);
StringRequestEntity requestEntity = null;
try {
requestEntity =
new StringRequestEntity(mapper.writeValueAsString(node), "application/json", "UTF-8");
} catch (IOException e) {
e.printStackTrace();
}
httppost.setRequestEntity(requestEntity);
client.getParams().setAuthenticationPreemptive(true);
String response = "";
int responseCode;
try {
responseCode = client.executeMethod(httppost);
InputStream responseBodyAsStream = httppost.getResponseBodyAsStream();
StringWriter stringWriter = new StringWriter();
IOUtils.copy(responseBodyAsStream, stringWriter, "UTF-8");
response = stringWriter.toString();
logger.log(Level.FINEST, "API Request Response: " + response);
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Failed to process PR get request; " + path, e);
}
logger.log(Level.FINEST, "PR-POST-RESPONSE:" + response);
if (!validResponseCode(responseCode)) {
logger.log(Level.SEVERE, "Failing to get response from Stash PR POST" + path);
throw new RuntimeException(
"Didn't get a 200 response from Stash PR POST! Response; '"
+ HttpStatus.getStatusText(responseCode)
+ "' with message; "
+ response);
}
return response;
}
/**
* Makes a rest request of any type at the specified urlSuffix. The urlSuffix should be of the
* form /userService/users. If CS throws an exception it handled and transalated to a Openfire
* exception if possible. This is done using the check fault method that tries to throw the best
* maching exception.
*
* @param type Must be GET or DELETE
* @param urlSuffix The url suffix of the rest request
* @param xmlParams The xml with the request params, must be null if type is GET or DELETE only
* @return The response as a xml doc.
* @throws ConnectionException Thrown if there are issues perfoming the request.
* @throws Exception Thrown if the response from Clearspace contains an exception.
*/
public Element executeRequest(HttpType type, String urlSuffix, String xmlParams)
throws ConnectionException, Exception {
if (Log.isDebugEnabled()) {
Log.debug("Outgoing REST call [" + type + "] to " + urlSuffix + ": " + xmlParams);
}
String wsUrl = getConnectionURI() + WEBSERVICES_PATH + urlSuffix;
String secret = getSharedSecret();
HttpClient client = new HttpClient();
HttpMethod method;
// Configures the authentication
client.getParams().setAuthenticationPreemptive(true);
Credentials credentials = new UsernamePasswordCredentials(OPENFIRE_USERNAME, secret);
AuthScope scope = new AuthScope(host, port, AuthScope.ANY_REALM);
client.getState().setCredentials(scope, credentials);
// Creates the method
switch (type) {
case GET:
method = new GetMethod(wsUrl);
break;
case POST:
PostMethod pm = new PostMethod(wsUrl);
StringRequestEntity requestEntity = new StringRequestEntity(xmlParams);
pm.setRequestEntity(requestEntity);
method = pm;
break;
case PUT:
PutMethod pm1 = new PutMethod(wsUrl);
StringRequestEntity requestEntity1 = new StringRequestEntity(xmlParams);
pm1.setRequestEntity(requestEntity1);
method = pm1;
break;
case DELETE:
method = new DeleteMethod(wsUrl);
break;
default:
throw new IllegalArgumentException();
}
method.setRequestHeader("Accept", "text/xml");
method.setDoAuthentication(true);
try {
// Executes the request
client.executeMethod(method);
// Parses the result
String body = method.getResponseBodyAsString();
if (Log.isDebugEnabled()) {
Log.debug("Outgoing REST call results: " + body);
}
// Checks the http status
if (method.getStatusCode() != 200) {
if (method.getStatusCode() == 401) {
throw new ConnectionException(
"Invalid password to connect to Clearspace.",
ConnectionException.ErrorType.AUTHENTICATION);
} else if (method.getStatusCode() == 404) {
throw new ConnectionException(
"Web service not found in Clearspace.", ConnectionException.ErrorType.PAGE_NOT_FOUND);
} else if (method.getStatusCode() == 503) {
throw new ConnectionException(
"Web service not avaible in Clearspace.",
ConnectionException.ErrorType.SERVICE_NOT_AVAIBLE);
} else {
throw new ConnectionException(
"Error connecting to Clearspace, http status code: " + method.getStatusCode(),
new HTTPConnectionException(method.getStatusCode()),
ConnectionException.ErrorType.OTHER);
}
} else if (body.contains("Clearspace Upgrade Console")) {
// TODO Change CS to send a more standard error message
throw new ConnectionException(
"Clearspace is in an update state.", ConnectionException.ErrorType.UPDATE_STATE);
}
Element response = localParser.get().parseDocument(body).getRootElement();
// Check for exceptions
checkFault(response);
// Since there is no exception, returns the response
return response;
} catch (DocumentException e) {
throw new ConnectionException(
"Error parsing the response of Clearspace.", e, ConnectionException.ErrorType.OTHER);
} catch (HttpException e) {
throw new ConnectionException(
"Error performing http request to Clearspace", e, ConnectionException.ErrorType.OTHER);
} catch (UnknownHostException e) {
throw new ConnectionException(
"Unknown Host " + getConnectionURI() + " trying to connect to Clearspace",
e,
ConnectionException.ErrorType.UNKNOWN_HOST);
} catch (IOException e) {
throw new ConnectionException(
"Error peforming http request to Clearspace.", e, ConnectionException.ErrorType.OTHER);
} finally {
method.releaseConnection();
}
}
/**
* The method reads a proxy object from database, makes a GetMethod object, appends required
* cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
* pagesource to the caller.
*
* @param iCount Counter variable for passing thread group information
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @param region the local region of a given url
* @param objProxyDao the database layer ProxyDao object variable
* @param useErrsy Boolean variable to specify usage of Errsy as proxy source
* @return String
*/
public String getPageSourceWithProxy(
String url,
boolean followRedirect,
boolean doAuthentication,
String region,
Boolean useErrsy,
String google) {
String page = " ";
String pageSource = "";
int i = 0;
String exception = " ";
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod getmethod = null;
HttpState state = new HttpState();
// if (ProxyDao.lstProxyData.size() == 16) {
ProxyData objProxyData = null;
// if (!useErrsy) {
try {
// objProxyData = ProxyDao.lstProxyData.get(iCount);
objProxyData = ProxyDao.objProxyData;
if (objProxyData == null) {
// objProxyDao.changeProxy(google);
objProxyData = ProxyDao.objProxyData;
}
httpclient
.getHostConfiguration()
.setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
/*} else {
try {
objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
httpclient.setState(state);
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
}*/
try {
getmethod = new GetMethod(url);
getmethod.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
if (url.contains("bing.com")) {
if (region.equalsIgnoreCase("co.uk")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.sg")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.au")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("co.in")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("ca")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.ph")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.my")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("it")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
}
}
getmethod.setFollowRedirects(true);
getmethod.setDoAuthentication(true);
httpclient.getParams().setAuthenticationPreemptive(true);
httpclient.setState(state);
String num100Header = "";
// if (url.contains("google")) {
// int j = 0;
// String url1 = "http://www.google.com/";
// try {
// GetMethod objGetMethod = new GetMethod(url1);
// j = httpclient.executeMethod(objGetMethod);
// Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
// String header = responseHeader.getValue();
// String[] headerValue = header.split(";");
//
// for (String head : headerValue) {
// if (head.contains("PREF=ID")) {
// header = head;
// break;
// }
// }
// String[] splitAll = header.split(":");
// long time = System.currentTimeMillis()+400;
// String sTime = "" + time;
// sTime = sTime.substring(0, 10);
// //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
// num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
// Cookie ck = new Cookie("PREF", "PREF", num100Header);
// httpclient.getState().clearCookies();
// httpclient.getState().addCookie(ck);
// getmethod.addRequestHeader("Host", "www.google.com");
// getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
// rv:19.0) Gecko/20100101 Firefox/19.0");
// getmethod.addRequestHeader("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
// getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
// getmethod.addRequestHeader("Referer", "https://www.google.com/");
// System.out.println(num100Header);
// } catch (Exception ex) {
// exception = ex.getMessage();
// l.debug(ex + " " + ex.getMessage() + "Exception occured for url" +
// url);
// pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
// return pageSource;
// }
// }
i = httpclient.executeMethod(getmethod);
if (i / 100 == 4 || i / 100 == 5) {
page = "<PROXY ERROR>";
} else {
page = getmethod.getResponseBodyAsString();
}
} catch (SocketTimeoutException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (SocketException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (Exception ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} finally {
getmethod.releaseConnection();
}
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
// }
return pageSource;
}
public void setMaxRedirects(int n) {
HttpClientParams clientparams = sessionClient.getParams();
clientparams.setParameter(MAX_REDIRECTS, n);
}
public String getCookiePolicy() {
return sessionClient == null ? null : sessionClient.getParams().getCookiePolicy();
}
public void setSoTimeout(int timeout) {
sessionClient.getParams().setSoTimeout(timeout);
}
public void setAuthenticationPreemptive(boolean tf) {
if (sessionClient != null) sessionClient.getParams().setAuthenticationPreemptive(tf);
}
public void setUserAgent(String agent) {
useragent = agent;
if (useragent != null && sessionClient != null)
sessionClient.getParams().setParameter(USER_AGENT, useragent);
}