public static void main(String args[]) {
HTTPDataSource ds = new HTTPDataSource();
try {
if (args.length != 1 && args.length != 2) {
throw new Exception(
/* (non-Javadoc)
* @i18n.test
* @org-mes="Need an url"
*/
org.openlaszlo.i18n.LaszloMessages.getMessage(
HTTPDataSource.class.getName(), "051018-828"));
}
String surl = args[0];
FileOutputStream out = null;
if (args.length == 2) {
out = new FileOutputStream(args[1]);
}
System.out.println("url is " + surl);
HttpData data = ds.getDataOnce(null, null, -1, surl, 0, 0);
System.out.println("Response code: " + data.code);
if (out != null) {
FileUtils.send(data.getInputStream(), out);
}
data.release();
} catch (Exception e) {
e.printStackTrace();
}
}
public void setErrResponse(Exception ex, Response response) {
if (ex instanceof java.sql.SQLException
|| ex instanceof com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException
|| ex instanceof org.springframework.jdbc.BadSqlGrammarException
|| ex instanceof org.springframework.dao.InvalidDataAccessApiUsageException
|| ex instanceof org.springframework.dao.DataAccessException
|| ex instanceof org.springframework.web.util.NestedServletException
|| ex
instanceof com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException) {
response.setResponseMsg(ex.getMessage());
response.setResponseStatus(ConstException.ERR_CODE_DB_ERROR);
} else if (ex instanceof java.lang.NullPointerException) {
response.setResponseMsg(ex.getMessage());
response.setResponseStatus(ConstException.ERR_CODE_UNKNOWN);
} else if (ex instanceof ConstException) {
ConstException constException = (ConstException) ex;
response.setResponseMsg(constException.getMessage());
response.setResponseStatus(constException.getCode());
} else if (ex instanceof java.lang.IllegalThreadStateException) {
response.setResponseMsg(ex.getMessage());
response.setResponseStatus(ConstException.ERR_CODE_INVALID_THREAD_STOP);
} else {
setErrResponse(ex, response, ConstException.ERR_CODE_UNKNOWN);
}
}
/**
* Method to make a GET HTTP connecton to the given url and return the output
*
* @param urlToFetch url to be connected
* @return the http get response
*/
public String makeRequest(String urlToFetch, String region) throws IOException {
String responseBody = "";
try {
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod httpget = new GetMethod(urlToFetch);
if (urlToFetch.contains("linkedin.com/countserv")) {
httpget.addRequestHeader("Host", "www.linkedin.com");
httpget.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:20.0) Gecko/20100101 Firefox/20.0");
httpget.addRequestHeader(
"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
httpget.addRequestHeader(
"Cookie",
"X-LI-IDC=C1; bcookie=\"v=2&618ed56d-275f-4dcd-86cd-6a44c8421879\"; bscookie=\"v=1&201305170926410ddaf2a1-e238-4bf6-8a0a-ce8a3df25934AQFZX82SfErJuFQKMJV49JfQTvewhfzh\"; X-LI-IDC=C1");
}
// httpclient.getHostConfiguration().setProxy("46.227.68.2", 3128);
// Credentials cred = new UsernamePasswordCredentials("mmongoose",
// "I-PHNBV9JHW6US");
// httpclient.getState().setProxyCredentials(AuthScope.ANY, cred);
int i = httpclient.executeMethod(httpget);
responseBody = httpget.getResponseBodyAsString();
} catch (Exception e) {
l.error(e + " " + e.getMessage() + " url " + urlToFetch);
}
return responseBody;
}
private String getStoreKey() {
try {
System.out.println("CURRENT STORE KEY VALUE: " + storeKey);
if (storeKey == null) {
Store store = KeyStorage.INSTANCE.getByName("jsonhub-store-key");
if (store != null) {
this.storeKey = store.getKey();
System.out.println("OBJDB STORE KEY VALUE: " + storeKey);
} else {
JSONObject jshubkey = register();
this.storeKey = jshubkey.getString("storeKey");
System.out.println("REGISTRATION STORE KEY VALUE: " + storeKey);
store = new Store();
store.setName("jsonhub-store-key");
store.setKey(storeKey);
KeyStorage.INSTANCE.add(store);
}
}
return storeKey;
} catch (Exception e) {
throw new RuntimeException("could not get jsonhub store key! \n Reason: \n" + e.getMessage());
}
}
private String getRequest(String path) {
logger.log(Level.FINEST, "PR-GET-REQUEST:" + path);
HttpClient client = getHttpClient();
client.getState().setCredentials(AuthScope.ANY, credentials);
GetMethod httpget = new GetMethod(path);
client.getParams().setAuthenticationPreemptive(true);
String response = null;
int responseCode;
try {
responseCode = client.executeMethod(httpget);
InputStream responseBodyAsStream = httpget.getResponseBodyAsStream();
StringWriter stringWriter = new StringWriter();
IOUtils.copy(responseBodyAsStream, stringWriter, "UTF-8");
response = stringWriter.toString();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Failed to process PR get request; " + path, e);
}
logger.log(Level.FINEST, "PR-GET-RESPONSE:" + response);
if (!validResponseCode(responseCode)) {
logger.log(Level.SEVERE, "Failing to get response from Stash PR GET" + path);
throw new RuntimeException(
"Didn't get a 200 response from Stash PR GET! Response; '"
+ HttpStatus.getStatusText(responseCode)
+ "' with message; "
+ response);
}
return response;
}
/**
* The method makes a GetMethod object and an HttpClient object. The HttpClient then executes the
* Getmethod and returns the pagesource to the caller.
*
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @return String
*/
public String getPageSourceWithoutProxy(
String url, boolean followRedirects, boolean doAuthentication) {
GetMethod getMethod = null;
String pageSouce = null;
HttpClient httpClient = new HttpClient();
try {
getMethod = new GetMethod(url);
getMethod.setFollowRedirects(followRedirects);
getMethod.setDoAuthentication(doAuthentication);
httpClient.executeMethod(getMethod);
pageSouce = getMethod.getResponseBodyAsString();
} catch (Exception e) {
l.error(e + " " + e.getMessage() + "exception occured for url" + url);
try {
getMethod = new GetMethod(url);
getMethod.setFollowRedirects(followRedirects);
getMethod.setDoAuthentication(doAuthentication);
httpClient.executeMethod(getMethod);
pageSouce = getMethod.getResponseBodyAsString();
getMethod.releaseConnection();
} catch (Exception ex) {
l.error(ex + " " + ex.getMessage() + "exception occured for url " + url);
getMethod.releaseConnection();
}
} finally {
getMethod.releaseConnection();
}
return pageSouce;
}
public static void main(String[] args) {
try {
// ProxyPageSource.makeRequest("http://www.rmccurdy.com/scripts/proxy/good.txt", "");
ProxyPageSource.geSourceWithoutProxy(
"http://www.rmccurdy.com/scripts/proxy/good.txt", true, true);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* This method makes a Json post to google plus weburl to read the google plus likes for a given
* url. The json post includes a registered google account developer key and 13 request headers to
* simulate browser behaviour. The json response is returned to caller class.
*
* @param url to fetch the data for
* @param objProxyDao the database layer ProxyDao object variable
* @return String
*/
public String getPostSourceGoogle(String url, ProxyDao objProxyDao) {
String source = "";
org.apache.http.client.HttpClient httpClient = new DefaultHttpClient();
try {
HttpPost request =
new HttpPost(
"https://clients6.google.com/rpc?key=AIzaSyCKSbrvQasunBoV16zDH9R33D88CeLr9gQ");
request.addHeader("Host", "clients6.google.com");
request.addHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:18.0) Gecko/20100101 Firefox/18.0");
request.addHeader(
"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
request.addHeader("Accept-Language", "en-US,en;q=0.5");
request.addHeader("X-Goog-AuthUser", "0");
request.addHeader("Authorization", "SAPISIDHASH 8b80e6cb4f438e869e8f6cc227fca5cf8856c6ab");
request.addHeader(
"X-ClientDetails",
"appVersion=5.0%20(Windows)&platform=Win32&userAgent=Mozilla%2F5.0%20(Windows%20NT%206.1%3B%20rv%3A18.0)%20Gecko%2F20100101%20Firefox%2F18.0");
request.addHeader("Content-Type", "application/json; charset=UTF-8");
request.addHeader("X-JavaScript-User-Agent", "google-api-javascript-client/1.0.0-alpha");
request.addHeader("X-Origin", "https://plusone.google.com");
request.addHeader("X-Referer", "https://plusone.google.com");
request.addHeader("X-Goog-Encode-Response-If-Executable", "base64");
request.addHeader(
"Cookie",
"PREF=ID=a426d8fb1e2750ee:U=6af15e4f74c74ed0:FF=0:LD=en:TM=1361159234:LM=1361169098:S=IFSJimDkqaNmvAhe; NID=67=shTVymtcFTEolwk0opYlWFkokW8qk9CqIGlSKZNTb6yJd684LQ5aJoSk5bMrQdp29SaMHo5sDCH48dxea7HgWXoPvBIdKd3xT7f3Pm9QsY3tgvscweUNp6FVTCGMJoypsV5OniwmQun0eVPUeFtauZbS; SID=DQAAAMIAAADKWgTA8SrvfpQcPBYOvtZMfeRs8gskEeTPpe4tGoK58rkzZNsIW0d0EawqI-0PmrxWF4Je7CbcWOdh0IaJtspvsAl2IaTftD28of_srBD-19rOobp9BRDsFNgro5V6NSKozVNTOl8cmFJUdBlUZ5iU5miI9X8lixN9kvjbydirkOKsb_ptkLELKHv0ZruwADrDNQm4-80T0accTOvBTT5MI_JP3kT9bjrvP3EOlE1dd71Va2VhpjPze_J5akbj3iQQA6ENor-1r9dN2G48UVGe; HSID=AJ3vWp905JFQ4Fsm4; SSID=ALbvnRrBXqNVAquz1; APISID=hJVIUcaXfkppC-OW/A6YIudTulrIg4f-HT; SAPISID=3EZqx7bM9IesGpqO/ARcqGVQeQoWyxxxnL");
StringEntity params =
new StringEntity(
"[{\"method\":\"pos.plusones.get\",\"id\":\"p\",\"params\":{\"nolog\":true,\"id\":\""
+ url
+ "\",\"source\":\"widget\",\"userId\":\"@viewer\",\"groupId\":\"@self\"},\"jsonrpc\":\"2.0\",\"key\":\"p\",\"apiVersion\":\"v1\"}]");
request.setEntity(params);
HttpResponse response = httpClient.execute(request);
org.apache.http.StatusLine statusLine = response.getStatusLine();
BufferedReader rd =
new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
String line = "";
while ((line = rd.readLine()) != null) {
source = source + line;
}
} catch (Exception ex) {
l.error(ex + " " + ex.getMessage() + " Exception occured for url " + url);
} finally {
httpClient.getConnectionManager().shutdown();
}
return source;
}
/** 根据模版及参数产生静态页面 */
public boolean createHtmlPage(String url, String htmlFileName) {
boolean status = false;
int statusCode = 0;
try {
// 创建一个HttpClient实例充当模拟浏览器
httpClient = new HttpClient();
// 设置httpclient读取内容时使用的字符集
httpClient.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "gbk");
// 创建GET方法的实例
getMethod = new GetMethod(url);
// 使用系统提供的默认的恢复策略，在发生异常时候将自动重试3次
getMethod
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
// 设置Get方法提交参数时使用的字符集,以支持中文参数的正常传递
getMethod.addRequestHeader("Content-Type", "text/html;charset=gbk");
// 执行Get方法并取得返回状态码，200表示正常，其它代码为异常
statusCode = httpClient.executeMethod(getMethod);
if (statusCode != 200) {
logger.fatal("静态页面引擎在解析" + url + "产生静态页面" + htmlFileName + "时出错!");
} else {
// 读取解析结果
sb = new StringBuffer();
in = getMethod.getResponseBodyAsStream();
br = new BufferedReader(new InputStreamReader(in));
while ((line = br.readLine()) != null) {
sb.append(line + "\n");
}
if (br != null) br.close();
page = sb.toString();
// 将页面中的相对路径替换成绝对路径，以确保页面资源正常访问
page = formatPage(page);
// 将解析结果写入指定的静态HTML文件中，实现静态HTML生成
writeHtml(htmlFileName, page);
status = true;
}
} catch (Exception ex) {
logger.fatal("静态页面引擎在解析" + url + "产生静态页面" + htmlFileName + "时出错:" + ex.getMessage());
} finally {
// 释放http连接
getMethod.releaseConnection();
}
return status;
}
private String postRequest(String path, String comment) throws UnsupportedEncodingException {
logger.log(Level.FINEST, "PR-POST-REQUEST:" + path + " with: " + comment);
HttpClient client = getHttpClient();
client.getState().setCredentials(AuthScope.ANY, credentials);
PostMethod httppost = new PostMethod(path);
ObjectNode node = mapper.getNodeFactory().objectNode();
node.put("text", comment);
StringRequestEntity requestEntity = null;
try {
requestEntity =
new StringRequestEntity(mapper.writeValueAsString(node), "application/json", "UTF-8");
} catch (IOException e) {
e.printStackTrace();
}
httppost.setRequestEntity(requestEntity);
client.getParams().setAuthenticationPreemptive(true);
String response = "";
int responseCode;
try {
responseCode = client.executeMethod(httppost);
InputStream responseBodyAsStream = httppost.getResponseBodyAsStream();
StringWriter stringWriter = new StringWriter();
IOUtils.copy(responseBodyAsStream, stringWriter, "UTF-8");
response = stringWriter.toString();
logger.log(Level.FINEST, "API Request Response: " + response);
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("Failed to process PR get request; " + path, e);
}
logger.log(Level.FINEST, "PR-POST-RESPONSE:" + response);
if (!validResponseCode(responseCode)) {
logger.log(Level.SEVERE, "Failing to get response from Stash PR POST" + path);
throw new RuntimeException(
"Didn't get a 200 response from Stash PR POST! Response; '"
+ HttpStatus.getStatusText(responseCode)
+ "' with message; "
+ response);
}
return response;
}
private static SSLContext createEasySSLContext() {
try {
TrustManager[] trustAllCerts =
new TrustManager[] {
new X509TrustManager() {
public X509Certificate[] getAcceptedIssuers() {
return null;
}
public void checkClientTrusted(X509Certificate[] certs, String authType) {}
public void checkServerTrusted(X509Certificate[] certs, String authType) {}
}
};
SSLContext context = SSLContext.getInstance("SSL");
context.init(null, trustAllCerts, null);
return context;
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new HttpClientError(e.toString());
}
}
/**
* The method reads a proxy object from database, makes a GetMethod object, appends required
* cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
* pagesource to the caller.
*
* @param iCount Counter variable for passing thread group information
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @param region the local region of a given url
* @param objProxyDao the database layer ProxyDao object variable
* @param useErrsy Boolean variable to specify usage of Errsy as proxy source
* @return String
*/
public String getPageSourceWithProxy(
String url,
boolean followRedirect,
boolean doAuthentication,
String region,
Boolean useErrsy,
String google) {
String page = " ";
String pageSource = "";
int i = 0;
String exception = " ";
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod getmethod = null;
HttpState state = new HttpState();
// if (ProxyDao.lstProxyData.size() == 16) {
ProxyData objProxyData = null;
// if (!useErrsy) {
try {
// objProxyData = ProxyDao.lstProxyData.get(iCount);
objProxyData = ProxyDao.objProxyData;
if (objProxyData == null) {
// objProxyDao.changeProxy(google);
objProxyData = ProxyDao.objProxyData;
}
httpclient
.getHostConfiguration()
.setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
/*} else {
try {
objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
httpclient.setState(state);
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
}*/
try {
getmethod = new GetMethod(url);
getmethod.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
if (url.contains("bing.com")) {
if (region.equalsIgnoreCase("co.uk")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.sg")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.au")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("co.in")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("ca")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.ph")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.my")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("it")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
}
}
getmethod.setFollowRedirects(true);
getmethod.setDoAuthentication(true);
httpclient.getParams().setAuthenticationPreemptive(true);
httpclient.setState(state);
String num100Header = "";
// if (url.contains("google")) {
// int j = 0;
// String url1 = "http://www.google.com/";
// try {
// GetMethod objGetMethod = new GetMethod(url1);
// j = httpclient.executeMethod(objGetMethod);
// Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
// String header = responseHeader.getValue();
// String[] headerValue = header.split(";");
//
// for (String head : headerValue) {
// if (head.contains("PREF=ID")) {
// header = head;
// break;
// }
// }
// String[] splitAll = header.split(":");
// long time = System.currentTimeMillis()+400;
// String sTime = "" + time;
// sTime = sTime.substring(0, 10);
// //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
// num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
// Cookie ck = new Cookie("PREF", "PREF", num100Header);
// httpclient.getState().clearCookies();
// httpclient.getState().addCookie(ck);
// getmethod.addRequestHeader("Host", "www.google.com");
// getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
// rv:19.0) Gecko/20100101 Firefox/19.0");
// getmethod.addRequestHeader("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
// getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
// getmethod.addRequestHeader("Referer", "https://www.google.com/");
// System.out.println(num100Header);
// } catch (Exception ex) {
// exception = ex.getMessage();
// l.debug(ex + " " + ex.getMessage() + "Exception occured for url" +
// url);
// pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
// return pageSource;
// }
// }
i = httpclient.executeMethod(getmethod);
if (i / 100 == 4 || i / 100 == 5) {
page = "<PROXY ERROR>";
} else {
page = getmethod.getResponseBodyAsString();
}
} catch (SocketTimeoutException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (SocketException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (Exception ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} finally {
getmethod.releaseConnection();
}
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
// }
return pageSource;
}
/**
* The method delegates call to
*
* @method getPageSourceWithProxy(
* @params), reads and parses the pagesource to verify the correctness of pagesource. If a
* restricted phrase is found in pagesource, it recursively make calls to
* @method getPageSourceWithProxy(
* @params) unless correct pagesource is obtained.
* @param iCount Counter variable for passing thread group information
* @param url Url to fetch the pagesource for
* @param objProxyDao the database layer ProxyDao object variable
* @param region the local region of a given url
* @return String
*/
public String getSource(String url, String region, String google) {
String page = "";
String exception = "";
url = ReplaceAllHtmlTags.replaceUrlsFrenchCanCode(url);
String pageSource = getPageSourceWithProxy(url, true, true, region, false, google);
int c = 0;
while (true) {
c++;
// if (c == 10) {
// try {
// c = 0;
// pageSource = ProxyPageSource.makeRequest(url, region);
//
// } catch (Exception e) {
// l.error(e + "pagesource not come by makeRequest()");
// }
// }
String[] proxyLog = pageSource.split("@@@@");
try {
page = proxyLog[2];
exception = proxyLog[1];
if (!exception.equals(" ")) {
page = "";
}
} catch (Exception e) {
page = "";
exception = "";
}
if (page == null || page.equals("")) {
// objProxyDao.changeProxy(google);
pageSource = getPageSourceWithProxy(url, true, true, region, false, google);
} else if (page.contains("407 Proxy Authentication Required")
|| page.contains(
"<span class=\"uiButtonText\">Rejestracja</span></a><span class=\"signup_box_content\">")
|| page.contains("<title>ERROR: The requested URL could not be retrieved</title>")
|| page.contains("404 Not Found")
|| page.contains("The page cannot be found")
|| page.contains("<p>HTTP Error 404. The requested resource is not found.</p>")
|| page.contains("Error 404: Not Found")
|| page.contains("HTTP Status 404 -")
|| page.contains("403 Access Denied")
|| page.contains("<h1>ERROR</h1>")
|| page.contains("400 Bad request")
|| page.contains("403 Forbidden")
|| page.contains("<title>Site not allowed - PacketIP</title>")
|| page.contains("502 Bad Gateway")
|| page.contains("but your computer or network may be sending automated queries")
|| (page.contains("To continue, please type the characters below:<br>")
&& page.contains("id=\"captcha\""))
|| page.contains("m.baidu.com")
|| page.contains("<PROXY ERROR>")
|| page.contains("Pardon the interruption")
|| page.equals(" ")) {
// l.error("Using Errsy" + "url:" + url + "page:");
// objProxyDao.changeProxy(google);
pageSource = getPageSourceWithProxy(url, true, true, region, false, google);
} else {
try {
// objProxyDao.updateUseCount(ProxyDao.lstProxyData.get(iCount));
// objProxyDao.updateUseCount(ProxyDao.objProxyData);
} catch (Exception e) {
l.error(e + " " + e.getMessage() + "Here is the problem" + url);
}
break;
}
}
return page;
}
/**
* Open a connection to the DODS server.
*
* @param urlString the URL to open.
* @param command execute this command on the input stream
* @throws IOException if an IO exception occurred.
* @throws DAP2Exception if the DODS server returned an error.
* @throws ParseException is cant parse the return
*/
private void openConnection(String urlString, Command command)
throws IOException, DAP2Exception, ParseException {
HTTPMethod method = null;
InputStream is = null;
initSession();
try {
method = _session.newMethodGet(urlString);
if (acceptCompress) method.setRequestHeader("Accept-Encoding", "deflate,gzip");
// enable sessions
if (allowSessions) method.setRequestHeader("X-Accept-Session", "true");
// Execute the method.
int statusCode = method.execute();
// debug
// if (debugHeaders) ucar.nc2.util.net.HttpClientManager.showHttpRequestInfo(f, method);
if (statusCode == HTTPSession.SC_NOT_FOUND) {
throw new DAP2Exception(DAP2Exception.NO_SUCH_FILE, method.getStatusText());
}
if (statusCode == HTTPSession.SC_UNAUTHORIZED) {
throw new InvalidCredentialsException(method.getStatusText());
}
if (statusCode != HTTPSession.SC_OK) {
throw new DAP2Exception(
"Method failed:" + method.getStatusText() + " on URL= " + urlString);
}
// Get the response body.
is = method.getResponseAsStream();
// check if its an error
Header header = method.getResponseHeader("Content-Description");
if (header != null
&& (header.getValue().equals("dods-error") || header.getValue().equals("dods_error"))) {
// create server exception object
DAP2Exception ds = new DAP2Exception();
is = dumpStream(is);
// parse the Error object from stream and throw it
ds.parse(is);
throw ds;
}
ver = new ServerVersion(method);
checkHeaders(method);
// check for deflator
Header h = method.getResponseHeader("content-encoding");
String encoding = (h == null) ? null : h.getValue();
// if (encoding != null) System.out.println("encoding= " + encoding);
if (encoding != null && encoding.equals("deflate")) {
is = new BufferedInputStream(new InflaterInputStream(is), 1000);
} else if (encoding != null && encoding.equals("gzip")) {
is = new BufferedInputStream(new GZIPInputStream(is), 1000);
}
command.process(is);
} catch (Exception e) {
e.printStackTrace();
throw new DAP2Exception(e);
} finally {
// Release the connection.
if (method != null) method.close();
}
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
address = PAConfiguration.getAddress();
port = PAConfiguration.getPort(instantiation);
poslAddress = PAConfiguration.getPOSL(instantiation, topic);
rdfAddress = PAConfiguration.getRDFTaxonomy(instantiation);
messageEndpoint = PAConfiguration.getEndpointName(instantiation, topic);
} catch (BadConfigurationException e) {
System.out.println(e.getMessage());
e.printStackTrace();
System.exit(0);
}
response.setContentType("text/html; charset=UTF-8");
PrintWriter out = response.getWriter();
try {
System.out.println("5 Publicty Chair Servlet");
System.out.println(response.toString());
BufferedReader brd = request.getReader();
String input = "";
String message = "";
while (!input.equals("</RuleML>")) {
input = brd.readLine();
message = message + input;
}
String[] varOrder = getVariableOrder(message);
System.out.println("Received Message: " + message);
// BackwardReasoner br = new BackwardReasoner();
// Iterator solit =null;
// DefiniteClause dc = null;
// SymbolTable.reset();
POSLParser pp = new POSLParser();
// String contents = "c(a).\nc(b).\nc(c).";
Date t1 = new GregorianCalendar().getTime();
System.out.println(t1.getHours() + ":" + t1.getMinutes());
// append time to contents
System.out.println("day: " + t1.getDay());
System.out.println("day: " + t1.getYear());
System.out.println("day: " + t1.getMonth());
// time
String time = "time(" + t1.getHours() + ":integer).";
System.out.println(time);
String url = poslAddress;
// String url = "http://www.jdrew.org/oojdrew/test.posl";
String contents = "";
// day of the week
int day = t1.getDay();
boolean weekday = true;
if (day == 0 || day == 6) {
weekday = false;
}
String dayOfWeek;
if (weekday) {
dayOfWeek = "day(weekday).";
} else {
dayOfWeek = "day(weekend).";
}
// full date
Calendar cal = new GregorianCalendar();
int year = cal.get(Calendar.YEAR);
int month = cal.get(Calendar.MONTH) + 1;
int day2 = cal.get(Calendar.DAY_OF_MONTH);
String date;
String day3 = "" + day2;
if (day2 == 1 || day2 == 2 || day2 == 3 || day2 == 4 || day2 == 5 || day2 == 6 || day2 == 7
|| day2 == 8 || day2 == 9) {
day3 = "0" + day2;
}
if (month == 10 || month == 11 || month == 12) date = "" + year + month + day3;
else date = "" + year + "0" + month + day3;
date = "date(" + date + ":integer).";
System.out.println(date);
String url2 = rdfAddress;
HttpClient client2 = new HttpClient();
GetMethod method2 = new GetMethod(url2);
method2.setFollowRedirects(true);
String typestr = "";
// Execute the GET method
int statusCode2 = client2.executeMethod(method2);
if (statusCode2 != -1) {
typestr = method2.getResponseBodyAsString();
}
System.out.println("Types: " + typestr);
Types.reset();
RDFSParser.parseRDFSString(typestr);
try {
HttpClient client = new HttpClient();
GetMethod method = new GetMethod(url);
method.setFollowRedirects(true);
// Execute the GET method
int statusCode = client.executeMethod(method);
if (statusCode != -1) {
contents = method.getResponseBodyAsString();
}
} catch (Exception e) {
e.printStackTrace();
}
contents = contents + "\n" + time;
contents = contents + "\n" + dayOfWeek;
contents = contents + "\n" + date;
BackwardReasoner br = new BackwardReasoner();
Iterator solit = null;
DefiniteClause dc = null;
SymbolTable.reset();
pp.parseDefiniteClauses(contents);
br.loadClauses(pp.iterator());
System.out.println("TEST");
Iterator it = pp.iterator();
while (it.hasNext()) {
DefiniteClause d = (DefiniteClause) it.next();
System.out.println("Loaded clause: " + d.toPOSLString());
}
br = new BackwardReasoner(br.clauses, br.oids);
MessageParser m = new MessageParser(message);
Element atom = null;
try {
atom = m.parseForContent();
} catch (Exception e) {
System.out.println("Invalid Message");
// out.flush();
}
QueryBuilder q = new QueryBuilder(atom);
String query = q.generateDoc();
System.out.println("ABOUT TO INPUT THIS QUERY:" + query);
RuleMLParser qp = new RuleMLParser();
try {
dc = qp.parseRuleMLQuery(query);
} catch (Exception e) {
System.out.println("Invalid Query");
// out.flush();
}
// solit = br.iterativeDepthFirstSolutionIterator(dc);
solit = br.iterativeDepthFirstSolutionIterator(dc);
int varSize = 0;
while (solit.hasNext()) {
Vector data = new Vector();
BackwardReasoner.GoalList gl = (BackwardReasoner.GoalList) solit.next();
Hashtable varbind = gl.varBindings;
javax.swing.tree.DefaultMutableTreeNode root = br.toTree();
root.setAllowsChildren(true);
javax.swing.tree.DefaultTreeModel dtm = new DefaultTreeModel(root);
int i = 0;
Object[][] rowdata = new Object[varbind.size()][2];
varSize = varbind.size();
Enumeration e = varbind.keys();
while (e.hasMoreElements()) {
Object k = e.nextElement();
Object val = varbind.get(k);
String ks = (String) k;
rowdata[i][0] = ks;
rowdata[i][1] = val;
i++;
}
data.addElement(rowdata);
String[] messages = new String[data.size()];
MessageGenerator g =
new MessageGenerator(
data, varSize, messageEndpoint, m.getId(), m.getProtocol(), m.getRel(), varOrder);
messages = g.Messages2();
String appender = "";
URL sender = new URL(address + ":" + port);
HttpMessage msg = new HttpMessage(sender);
Properties props = new Properties();
for (int i1 = 0; i1 < data.size(); i1++) {
System.out.println(i1 + ")" + messages[i1].toString());
props.put("text", messages[i1].toString());
InputStream in = msg.sendGetMessage(props);
}
System.out.println("NEXT MESSAGE");
}
MessageGenerator g =
new MessageGenerator(
null, varSize, messageEndpoint, m.getId(), m.getProtocol(), m.getRel());
URL sender = new URL(address + ":" + port);
HttpMessage msg = new HttpMessage(sender);
Properties props = new Properties();
String finalMessage = g.finalMessage(query);
System.out.println(finalMessage);
props.put("text", finalMessage);
InputStream in = msg.sendGetMessage(props);
System.out.println("Stop_Communication");
} catch (Exception e) {
System.out.println("ERROR has occured : " + e.toString());
}
out.close();
}