/**
* Obtem a timeline publica. Nao necessita de autenticacao, basicamente é um GET na URL
* http://twitter.com/statuses/public_timeline.xml
*/
public String getPublicTimeline() throws Exception {
HttpClient client = new HttpClient();
// get para obter a timeline publica
GetMethod get = new GetMethod("/statuses/public_timeline.xml");
// Criando o host, para onde sera enviada a requisicao
HostConfiguration host = client.getHostConfiguration();
host.setHost(new URI("http://twitter.com"));
// executando o get na url de timeline publica do twitter
client.executeMethod(host, get);
// obtendo a resposta
return get.getResponseBodyAsString();
}
private void configureHttpMethod(
boolean skipContentCache, CacheData cacheData, long onceTimeOut, HttpMethod httpMethod) {
if (skipContentCache && null != cacheData) {
if (null != cacheData.getLastModifiedHeader()
&& Constants.NULL != cacheData.getLastModifiedHeader()) {
httpMethod.addRequestHeader(Constants.IF_MODIFIED_SINCE, cacheData.getLastModifiedHeader());
}
if (null != cacheData.getMd5() && Constants.NULL != cacheData.getMd5()) {
httpMethod.addRequestHeader(Constants.CONTENT_MD5, cacheData.getMd5());
}
}
httpMethod.addRequestHeader(Constants.ACCEPT_ENCODING, "gzip,deflate");
HttpMethodParams params = new HttpMethodParams();
params.setSoTimeout((int) onceTimeOut);
httpMethod.setParams(params);
httpClient
.getHostConfiguration()
.setHost(
diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
diamondConfigure.getPort());
}
Set<String> checkUpdateDataIds(long timeout) {
if (!isRun) {
throw new RuntimeException("DiamondSubscriber is not running. checkUpdateDataIds return.");
}
if (MockServer.isTestMode()) {
return testData();
}
long waitTime = 0;
String probeUpdateString = getProbeUpdateString();
if (StringUtils.isBlank(probeUpdateString)) {
return null;
}
while (0 == timeout || timeout > waitTime) {
long onceTimeOut = getOnceTimeOut(waitTime, timeout);
waitTime += onceTimeOut;
PostMethod postMethod = new PostMethod(Constants.HTTP_URI_FILE);
postMethod.addParameter(Constants.PROBE_MODIFY_REQUEST, probeUpdateString);
HttpMethodParams params = new HttpMethodParams();
params.setSoTimeout((int) onceTimeOut);
postMethod.setParams(params);
try {
httpClient
.getHostConfiguration()
.setHost(
diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
this.diamondConfigure.getPort());
int httpStatus = httpClient.executeMethod(postMethod);
switch (httpStatus) {
case SC_OK:
{
Set<String> result = getUpdateDataIds(postMethod);
return result;
}
case SC_SERVICE_UNAVAILABLE:
{
rotateToNextDomain();
}
break;
default:
{
log.warn("checkUpdateDataIds HTTP State: " + httpStatus);
rotateToNextDomain();
}
}
} catch (HttpException e) {
log.error("checkUpdateDataIds HttpException", e);
rotateToNextDomain();
} catch (IOException e) {
log.error("checkUpdateDataIds IOException", e);
rotateToNextDomain();
} catch (Exception e) {
log.error("checkUpdateDataIds Unknown Exception", e);
rotateToNextDomain();
} finally {
postMethod.releaseConnection();
}
}
throw new RuntimeException(
"checkUpdateDataIds timeout "
+ diamondConfigure.getDomainNameList().get(this.domainNamePos.get())
+ ", timeout="
+ timeout);
}
@Override
public BatchHttpResult getConfigureInformationBatch(
List<String> dataIds, String group, int timeout) {
if (dataIds == null) {
log.error("dataId list cannot be null,group=" + group);
return new BatchHttpResult(HttpStatus.SC_BAD_REQUEST);
}
if (group == null) {
group = Constants.DEFAULT_GROUP;
}
StringBuilder dataIdBuilder = new StringBuilder();
for (String dataId : dataIds) {
dataIdBuilder.append(dataId).append(Constants.LINE_SEPARATOR);
}
String dataIdStr = dataIdBuilder.toString();
PostMethod post = new PostMethod(Constants.HTTP_URI_FILE_BATCH);
post.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, timeout);
BatchHttpResult response = null;
try {
NameValuePair dataIdValue = new NameValuePair("dataIds", dataIdStr);
NameValuePair groupValue = new NameValuePair("group", group);
post.setRequestBody(new NameValuePair[] {dataIdValue, groupValue});
httpClient
.getHostConfiguration()
.setHost(
diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
this.diamondConfigure.getPort());
int status = httpClient.executeMethod(post);
String responseMsg = post.getResponseBodyAsString();
if (status == HttpStatus.SC_OK) {
String json = null;
try {
json = responseMsg;
List<ConfigInfoEx> configInfoExList = new LinkedList<ConfigInfoEx>();
Object resultObj =
JSONUtils.deserializeObject(json, new TypeReference<List<ConfigInfoEx>>() {});
if (!(resultObj instanceof List<?>)) {
throw new RuntimeException(
"batch query deserialize type error, not list, json=" + json);
}
List<ConfigInfoEx> resultList = (List<ConfigInfoEx>) resultObj;
for (ConfigInfoEx configInfoEx : resultList) {
configInfoExList.add(configInfoEx);
}
response = new BatchHttpResult(configInfoExList);
log.info(
"batch query success,dataIds=" + dataIdStr + ",group=" + group + ",json=" + json);
} catch (Exception e) {
response = new BatchHttpResult(Constants.BATCH_OP_ERROR);
log.error(
"batch query deserialize error,dataIdStr="
+ dataIdStr
+ ",group="
+ group
+ ",json="
+ json,
e);
}
} else if (status == HttpStatus.SC_REQUEST_TIMEOUT) {
response = new BatchHttpResult(HttpStatus.SC_REQUEST_TIMEOUT);
log.error(
"batch query timeout, socket timeout(ms):"
+ timeout
+ ",dataIds="
+ dataIdStr
+ ",group="
+ group);
} else {
response = new BatchHttpResult(status);
log.error(
"batch query fail, status:"
+ status
+ ", response:"
+ responseMsg
+ ",dataIds="
+ dataIdStr
+ ",group="
+ group);
}
} catch (HttpException e) {
response = new BatchHttpResult(Constants.BATCH_HTTP_EXCEPTION);
log.error("batch query http exception,dataIds=" + dataIdStr + ",group=" + group, e);
} catch (IOException e) {
response = new BatchHttpResult(Constants.BATCH_IO_EXCEPTION);
log.error("batch query io exception, dataIds=" + dataIdStr + ",group=" + group, e);
} finally {
post.releaseConnection();
}
return response;
}
/**
* The method reads a proxy object from database, makes a GetMethod object, appends required
* cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
* pagesource to the caller.
*
* @param iCount Counter variable for passing thread group information
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @param region the local region of a given url
* @param objProxyDao the database layer ProxyDao object variable
* @param useErrsy Boolean variable to specify usage of Errsy as proxy source
* @return String
*/
public String getPageSourceWithProxy(
String url,
boolean followRedirect,
boolean doAuthentication,
String region,
Boolean useErrsy,
String google) {
String page = " ";
String pageSource = "";
int i = 0;
String exception = " ";
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod getmethod = null;
HttpState state = new HttpState();
// if (ProxyDao.lstProxyData.size() == 16) {
ProxyData objProxyData = null;
// if (!useErrsy) {
try {
// objProxyData = ProxyDao.lstProxyData.get(iCount);
objProxyData = ProxyDao.objProxyData;
if (objProxyData == null) {
// objProxyDao.changeProxy(google);
objProxyData = ProxyDao.objProxyData;
}
httpclient
.getHostConfiguration()
.setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
/*} else {
try {
objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
httpclient.setState(state);
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
}*/
try {
getmethod = new GetMethod(url);
getmethod.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
if (url.contains("bing.com")) {
if (region.equalsIgnoreCase("co.uk")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.sg")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.au")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("co.in")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("ca")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.ph")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.my")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("it")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
}
}
getmethod.setFollowRedirects(true);
getmethod.setDoAuthentication(true);
httpclient.getParams().setAuthenticationPreemptive(true);
httpclient.setState(state);
String num100Header = "";
// if (url.contains("google")) {
// int j = 0;
// String url1 = "http://www.google.com/";
// try {
// GetMethod objGetMethod = new GetMethod(url1);
// j = httpclient.executeMethod(objGetMethod);
// Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
// String header = responseHeader.getValue();
// String[] headerValue = header.split(";");
//
// for (String head : headerValue) {
// if (head.contains("PREF=ID")) {
// header = head;
// break;
// }
// }
// String[] splitAll = header.split(":");
// long time = System.currentTimeMillis()+400;
// String sTime = "" + time;
// sTime = sTime.substring(0, 10);
// //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
// num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
// Cookie ck = new Cookie("PREF", "PREF", num100Header);
// httpclient.getState().clearCookies();
// httpclient.getState().addCookie(ck);
// getmethod.addRequestHeader("Host", "www.google.com");
// getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
// rv:19.0) Gecko/20100101 Firefox/19.0");
// getmethod.addRequestHeader("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
// getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
// getmethod.addRequestHeader("Referer", "https://www.google.com/");
// System.out.println(num100Header);
// } catch (Exception ex) {
// exception = ex.getMessage();
// l.debug(ex + " " + ex.getMessage() + "Exception occured for url" +
// url);
// pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
// return pageSource;
// }
// }
i = httpclient.executeMethod(getmethod);
if (i / 100 == 4 || i / 100 == 5) {
page = "<PROXY ERROR>";
} else {
page = getmethod.getResponseBodyAsString();
}
} catch (SocketTimeoutException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (SocketException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (Exception ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} finally {
getmethod.releaseConnection();
}
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
// }
return pageSource;
}
// All proxy activity goes thru here
void setProxy(Proxy proxy) {
if (sessionClient == null) return;
if (proxy != null && proxy.host != null)
sessionClient.getHostConfiguration().setProxy(proxy.host, proxy.port);
}