/**
* The method makes a GetMethod object and an HttpClient object. The HttpClient then executes the
* Getmethod and returns the pagesource to the caller.
*
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @return String
*/
public String getPageSourceWithoutProxy(
String url, boolean followRedirects, boolean doAuthentication) {
GetMethod getMethod = null;
String pageSouce = null;
HttpClient httpClient = new HttpClient();
try {
getMethod = new GetMethod(url);
getMethod.setFollowRedirects(followRedirects);
getMethod.setDoAuthentication(doAuthentication);
httpClient.executeMethod(getMethod);
pageSouce = getMethod.getResponseBodyAsString();
} catch (Exception e) {
l.error(e + " " + e.getMessage() + "exception occured for url" + url);
try {
getMethod = new GetMethod(url);
getMethod.setFollowRedirects(followRedirects);
getMethod.setDoAuthentication(doAuthentication);
httpClient.executeMethod(getMethod);
pageSouce = getMethod.getResponseBodyAsString();
getMethod.releaseConnection();
} catch (Exception ex) {
l.error(ex + " " + ex.getMessage() + "exception occured for url " + url);
getMethod.releaseConnection();
}
} finally {
getMethod.releaseConnection();
}
return pageSouce;
}
public void testGetFromMultipleThreads() {
this.server.setHttpService(new EchoService());
client.setHttpConnectionManager(new MultiThreadedHttpConnectionManager());
ExecuteMethodThread[] threads = new ExecuteMethodThread[10];
for (int i = 0; i < threads.length; i++) {
GetMethod method = new GetMethod("/");
method.setFollowRedirects(true);
threads[i] = new ExecuteMethodThread(method, client);
threads[i].start();
}
for (int i = 0; i < threads.length; i++) {
try {
// wait until this thread finishes. we'll give it 10 seconds,
// but it shouldn't take that long
threads[i].join(10000);
} catch (InterruptedException e) {
}
// make sure an exception did not occur
Exception e = threads[i].getException();
if (e != null) {
fail("An error occured in the get: " + e);
}
// we should have a 200 status
assertEquals(threads[i].getMethod().getStatusCode(), HttpStatus.SC_OK);
}
}
/** Checks all the servers marked as being online if they still are online. */
private synchronized void checkOnlineServers() {
Iterator itr;
itr = online.listIterator();
while (itr.hasNext()) {
Server server = (Server) itr.next();
String url = getServerURL(server);
GetMethod get = new GetMethod(url);
get.setFollowRedirects(false);
try {
httpClient.executeMethod(get);
if (!okServerResponse(get.getStatusCode())) {
offline.add(server);
itr.remove();
log.debug("Server going OFFLINE! " + getServerURL(server));
listener.serverOffline(server);
}
} catch (Exception e) {
offline.add(server);
itr.remove();
log.debug("Server going OFFLINE! " + getServerURL(server));
listener.serverOffline(server);
} finally {
get.releaseConnection();
}
}
}
private String getHistoryFile(Configuration conf, String jobId) throws IOException {
String jtAddress = "scheme://" + conf.get("mapred.job.tracker");
String jtHttpAddr = "scheme://" + conf.get("mapred.job.tracker.http.address");
try {
String host = new URI(jtAddress).getHost();
int port = new URI(jtHttpAddr).getPort();
HttpClient client = new HttpClient();
String jobUrl = "http://" + host + ":" + port + "/jobdetails.jsp";
GetMethod get = new GetMethod(jobUrl);
get.setQueryString("jobid=" + jobId);
get.setFollowRedirects(false);
int status = client.executeMethod(get);
String file = null;
if (status == HttpStatus.SC_MOVED_PERMANENTLY || status == HttpStatus.SC_MOVED_TEMPORARILY) {
file = get.getResponseHeader("Location").toString();
file = file.substring(file.lastIndexOf('=') + 1);
file = JobHistory.JobInfo.decodeJobHistoryFileName(file);
} else {
LOG.warn("JobURL {} for id: {} returned {}", jobUrl, jobId, status);
}
return file;
} catch (URISyntaxException e) {
throw new IOException("JT Address: " + jtAddress + ", http Address: " + jtHttpAddr, e);
}
}
/**
* Download a page from the server and return its content. Throws a {@link RuntimeException} on
* connection problems etc.
*
* @param url URL of the page
* @return content of the page
*/
protected static InputStream getUrlContent(String url) {
GetMethod get = new GetMethod(url);
get.setFollowRedirects(true);
if (isLoggedIn()) {
get.setDoAuthentication(true);
get.addRequestHeader(
"Authorization", "Basic " + new String(Base64.encodeBase64("Admin:admin".getBytes())));
}
try {
int statusCode = AbstractEscapingTest.getClient().executeMethod(get);
switch (statusCode) {
case HttpStatus.SC_OK:
// everything is fine
break;
case HttpStatus.SC_UNAUTHORIZED:
// do not fail on 401 (unauthorized), used in some tests
System.out.println("WARNING, Ignoring status 401 (unauthorized) for URL: " + url);
break;
case HttpStatus.SC_CONFLICT:
// do not fail on 409 (conflict), used in some templates
System.out.println("WARNING, Ignoring status 409 (conflict) for URL: " + url);
break;
case HttpStatus.SC_NOT_FOUND:
// ignore 404 (the page is still rendered)
break;
default:
throw new RuntimeException(
"HTTP GET request returned status "
+ statusCode
+ " ("
+ get.getStatusText()
+ ") for URL: "
+ url);
}
// get the data, converting to utf-8
String str = get.getResponseBodyAsString();
if (str == null) {
return null;
}
return new ByteArrayInputStream(str.getBytes("utf-8"));
} catch (IOException exception) {
throw new RuntimeException("Error retrieving URL: " + url + "\n" + exception);
} finally {
get.releaseConnection();
}
}
/** Run method of the thread */
public void run() {
queue = manager.workQueue;
while (manager.hasWorkLeft()) {
working = false;
// code to make the worker pause, if the pause button has been presed
// if the stop signal has been given stop the thread
if (stop) {
return;
}
// this pasuses the thread
synchronized (this) {
while (pleaseWait) {
try {
wait();
} catch (InterruptedException e) {
return;
} catch (Exception e) {
e.printStackTrace();
}
}
}
GetMethod httpget = null;
HeadMethod httphead = null;
try {
work = (WorkUnit) queue.take();
working = true;
url = work.getWork();
int code = 0;
String responce = "";
String rawResponce = "";
// if the work is a head request
if (work.getMethod().equalsIgnoreCase("HEAD")) {
if (Config.debug) {
System.out.println("DEBUG Worker[" + threadId + "]: HEAD " + url.toString());
}
httphead = new HeadMethod(url.toString());
// set the custom HTTP headers
Vector HTTPheaders = manager.getHTTPHeaders();
for (int a = 0; a < HTTPheaders.size(); a++) {
HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
/*
* Host header has to be set in a different way!
*/
if (httpHeader.getHeader().startsWith("Host")) {
httphead.getParams().setVirtualHost(httpHeader.getValue());
} else {
httphead.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
}
}
httphead.setFollowRedirects(Config.followRedirects);
/*
* this code is used to limit the number of request/sec
*/
if (manager.isLimitRequests()) {
while (manager.getTotalDone()
/ ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
> manager.getLimitRequestsTo()) {
Thread.sleep(100);
}
}
/*
* Send the head request
*/
code = httpclient.executeMethod(httphead);
if (Config.debug) {
System.out.println("DEBUG Worker[" + threadId + "]: " + code + " " + url.toString());
}
httphead.releaseConnection();
}
// if we are doing a get request
else if (work.getMethod().equalsIgnoreCase("GET")) {
// make the request;
if (Config.debug) {
System.out.println("DEBUG Worker[" + threadId + "]: GET " + url.toString());
}
httpget = new GetMethod(url.toString());
// set the custom HTTP headers
Vector HTTPheaders = manager.getHTTPHeaders();
for (int a = 0; a < HTTPheaders.size(); a++) {
HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
/*
* Host header has to be set in a different way!
*/
if (httpHeader.getHeader().startsWith("Host")) {
httpget.getParams().setVirtualHost(httpHeader.getValue());
} else {
httpget.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
}
}
httpget.setFollowRedirects(Config.followRedirects);
/*
* this code is used to limit the number of request/sec
*/
if (manager.isLimitRequests()) {
while (manager.getTotalDone()
/ ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
> manager.getLimitRequestsTo()) {
Thread.sleep(100);
}
}
code = httpclient.executeMethod(httpget);
if (Config.debug) {
System.out.println("DEBUG Worker[" + threadId + "]: " + code + " " + url.toString());
}
// set up the input stream
BufferedReader input =
new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));
// save the headers into a string, used in viewing raw responce
String rawHeader;
rawHeader = httpget.getStatusLine() + "\r\n";
Header headers[] = httpget.getResponseHeaders();
StringBuffer buf = new StringBuffer();
for (int a = 0; a < headers.length; a++) {
buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
}
rawHeader = rawHeader + buf.toString();
buf = new StringBuffer();
// read in the responce body
String line;
while ((line = input.readLine()) != null) {
buf.append("\r\n" + line);
}
responce = buf.toString();
input.close();
rawResponce = rawHeader + responce;
// clean the responce
// parse the html of what we have found
if (Config.parseHTML && !work.getBaseCaseObj().isUseRegexInstead()) {
Header contentType = httpget.getResponseHeader("Content-Type");
if (contentType != null) {
if (contentType.getValue().startsWith("text")) {
manager.addHTMLToParseQueue(new HTMLparseWorkUnit(responce, work));
}
}
}
responce = FilterResponce.CleanResponce(responce, work);
Thread.sleep(10);
httpget.releaseConnection();
} else {
// There is no need to deal with requests other than HEAD or GET
}
// if we need to check the against the base case
if (work.getMethod().equalsIgnoreCase("GET")
&& work.getBaseCaseObj().useContentAnalysisMode()) {
if (code == 200) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Base Case Check " + url.toString());
}
// TODO move this option to the Adv options
// if the responce does not match the base case
Pattern regexFindFile = Pattern.compile(".*file not found.*", Pattern.CASE_INSENSITIVE);
Matcher m = regexFindFile.matcher(responce);
// need to clean the base case of the item we are looking for
String basecase =
FilterResponce.removeItemCheckedFor(
work.getBaseCaseObj().getBaseCase(), work.getItemToCheck());
if (m.find()) {
// do nothing as we have a 404
} else if (!responce.equalsIgnoreCase(basecase)) {
if (work.isDir()) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found Dir (base case)" + url.toString());
}
// we found a dir
manager.foundDir(url, code, responce, basecase, rawResponce, work.getBaseCaseObj());
} else {
// found a file
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found File (base case)" + url.toString());
}
manager.foundFile(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
}
}
} else if (code == 404 || code == 400) {
// again do nothing as it is not there
} else {
if (work.isDir()) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found Dir (base case)" + url.toString());
}
// we found a dir
manager.foundDir(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
} else {
// found a file
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found File (base case)" + url.toString());
}
manager.foundFile(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
}
// manager.foundError(url, "Base Case Mode Error - Responce code came back as " + code +
// " it should have been 200");
// manager.workDone();
}
}
/*
* use the custom regex check instead
*/
else if (work.getBaseCaseObj().isUseRegexInstead()) {
Pattern regexFindFile = Pattern.compile(work.getBaseCaseObj().getRegex());
Matcher m = regexFindFile.matcher(rawResponce);
/*
System.out.println("======Trying to find======");
System.out.println(work.getBaseCaseObj().getRegex());
System.out.println("======In======");
System.out.println(responce);
System.out.println("======/In======");
*/
if (m.find()) {
// do nothing as we have a 404
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Regex matched so it's a 404, " + url.toString());
}
} else {
if (Config.parseHTML) {
Header contentType = httpget.getResponseHeader("Content-Type");
if (contentType != null) {
if (contentType.getValue().startsWith("text")) {
manager.addHTMLToParseQueue(new HTMLparseWorkUnit(rawResponce, work));
}
}
}
if (work.isDir()) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found Dir (regex) " + url.toString());
}
// we found a dir
manager.foundDir(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
} else {
// found a file
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found File (regex) " + url.toString());
}
manager.foundFile(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
}
// manager.foundError(url, "Base Case Mode Error - Responce code came back as " + code +
// " it should have been 200");
// manager.workDone();
}
}
// just check the responce code
else {
// if is not the fail code, a 404 or a 400 then we have a possible
if (code != work.getBaseCaseObj().getFailCode()
&& code != 404
&& code != 0
&& code != 400) {
if (work.getMethod().equalsIgnoreCase("HEAD")) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Getting responce via GET " + url.toString());
}
rawResponce = "";
httpget = new GetMethod(url.toString());
Vector HTTPheaders = manager.getHTTPHeaders();
for (int a = 0; a < HTTPheaders.size(); a++) {
HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
httpget.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
}
httpget.setFollowRedirects(Config.followRedirects);
/*
* this code is used to limit the number of request/sec
*/
if (manager.isLimitRequests()) {
while (manager.getTotalDone()
/ ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
> manager.getLimitRequestsTo()) {
Thread.sleep(100);
}
}
int newCode = httpclient.executeMethod(httpget);
// in some cases the second get can return a different result, than the first head
// request!
if (newCode != code) {
manager.foundError(
url,
"Return code for first HEAD, is different to the second GET: "
+ code
+ " - "
+ newCode);
}
rawResponce = "";
// build a string version of the headers
rawResponce = httpget.getStatusLine() + "\r\n";
Header headers[] = httpget.getResponseHeaders();
StringBuffer buf = new StringBuffer();
for (int a = 0; a < headers.length; a++) {
buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
}
buf.append("\r\n");
rawResponce = rawResponce + buf.toString();
if (httpget.getResponseContentLength() > 0) {
// get the http body
BufferedReader input =
new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));
String line;
String tempResponce = "";
buf = new StringBuffer();
while ((line = input.readLine()) != null) {
buf.append("\r\n" + line);
}
tempResponce = buf.toString();
input.close();
rawResponce = rawResponce + tempResponce;
Header contentType = httpget.getResponseHeader("Content-Type");
if (Config.parseHTML) {
contentType = httpget.getResponseHeader("Content-Type");
if (contentType != null) {
if (contentType.getValue().startsWith("text")) {
manager.addHTMLToParseQueue(new HTMLparseWorkUnit(tempResponce, work));
}
}
}
}
httpget.releaseConnection();
}
if (work.isDir()) {
manager.foundDir(url, code, rawResponce, work.getBaseCaseObj());
} else {
manager.foundFile(url, code, rawResponce, work.getBaseCaseObj());
}
}
}
manager.workDone();
Thread.sleep(20);
} catch (NoHttpResponseException e) {
manager.foundError(url, "NoHttpResponseException " + e.getMessage());
manager.workDone();
} catch (ConnectTimeoutException e) {
manager.foundError(url, "ConnectTimeoutException " + e.getMessage());
manager.workDone();
} catch (URIException e) {
manager.foundError(url, "URIException " + e.getMessage());
manager.workDone();
} catch (IOException e) {
manager.foundError(url, "IOException " + e.getMessage());
manager.workDone();
} catch (InterruptedException e) {
// manager.foundError(url, "InterruptedException " + e.getMessage());
manager.workDone();
return;
} catch (IllegalArgumentException e) {
e.printStackTrace();
manager.foundError(url, "IllegalArgumentException " + e.getMessage());
manager.workDone();
} finally {
if (httpget != null) {
httpget.releaseConnection();
}
if (httphead != null) {
httphead.releaseConnection();
}
}
}
}
/**
* The method reads a proxy object from database, makes a GetMethod object, appends required
* cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
* pagesource to the caller.
*
* @param iCount Counter variable for passing thread group information
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @param region the local region of a given url
* @param objProxyDao the database layer ProxyDao object variable
* @param useErrsy Boolean variable to specify usage of Errsy as proxy source
* @return String
*/
public String getPageSourceWithProxy(
String url,
boolean followRedirect,
boolean doAuthentication,
String region,
Boolean useErrsy,
String google) {
String page = " ";
String pageSource = "";
int i = 0;
String exception = " ";
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod getmethod = null;
HttpState state = new HttpState();
// if (ProxyDao.lstProxyData.size() == 16) {
ProxyData objProxyData = null;
// if (!useErrsy) {
try {
// objProxyData = ProxyDao.lstProxyData.get(iCount);
objProxyData = ProxyDao.objProxyData;
if (objProxyData == null) {
// objProxyDao.changeProxy(google);
objProxyData = ProxyDao.objProxyData;
}
httpclient
.getHostConfiguration()
.setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
/*} else {
try {
objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
httpclient.setState(state);
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
}*/
try {
getmethod = new GetMethod(url);
getmethod.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
if (url.contains("bing.com")) {
if (region.equalsIgnoreCase("co.uk")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.sg")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.au")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("co.in")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("ca")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.ph")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.my")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("it")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
}
}
getmethod.setFollowRedirects(true);
getmethod.setDoAuthentication(true);
httpclient.getParams().setAuthenticationPreemptive(true);
httpclient.setState(state);
String num100Header = "";
// if (url.contains("google")) {
// int j = 0;
// String url1 = "http://www.google.com/";
// try {
// GetMethod objGetMethod = new GetMethod(url1);
// j = httpclient.executeMethod(objGetMethod);
// Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
// String header = responseHeader.getValue();
// String[] headerValue = header.split(";");
//
// for (String head : headerValue) {
// if (head.contains("PREF=ID")) {
// header = head;
// break;
// }
// }
// String[] splitAll = header.split(":");
// long time = System.currentTimeMillis()+400;
// String sTime = "" + time;
// sTime = sTime.substring(0, 10);
// //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
// num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
// Cookie ck = new Cookie("PREF", "PREF", num100Header);
// httpclient.getState().clearCookies();
// httpclient.getState().addCookie(ck);
// getmethod.addRequestHeader("Host", "www.google.com");
// getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
// rv:19.0) Gecko/20100101 Firefox/19.0");
// getmethod.addRequestHeader("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
// getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
// getmethod.addRequestHeader("Referer", "https://www.google.com/");
// System.out.println(num100Header);
// } catch (Exception ex) {
// exception = ex.getMessage();
// l.debug(ex + " " + ex.getMessage() + "Exception occured for url" +
// url);
// pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
// return pageSource;
// }
// }
i = httpclient.executeMethod(getmethod);
if (i / 100 == 4 || i / 100 == 5) {
page = "<PROXY ERROR>";
} else {
page = getmethod.getResponseBodyAsString();
}
} catch (SocketTimeoutException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (SocketException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (Exception ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} finally {
getmethod.releaseConnection();
}
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
// }
return pageSource;
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
try {
address = PAConfiguration.getAddress();
port = PAConfiguration.getPort(instantiation);
poslAddress = PAConfiguration.getPOSL(instantiation, topic);
rdfAddress = PAConfiguration.getRDFTaxonomy(instantiation);
messageEndpoint = PAConfiguration.getEndpointName(instantiation, topic);
} catch (BadConfigurationException e) {
System.out.println(e.getMessage());
e.printStackTrace();
System.exit(0);
}
response.setContentType("text/html; charset=UTF-8");
PrintWriter out = response.getWriter();
try {
System.out.println("5 Publicty Chair Servlet");
System.out.println(response.toString());
BufferedReader brd = request.getReader();
String input = "";
String message = "";
while (!input.equals("</RuleML>")) {
input = brd.readLine();
message = message + input;
}
String[] varOrder = getVariableOrder(message);
System.out.println("Received Message: " + message);
// BackwardReasoner br = new BackwardReasoner();
// Iterator solit =null;
// DefiniteClause dc = null;
// SymbolTable.reset();
POSLParser pp = new POSLParser();
// String contents = "c(a).\nc(b).\nc(c).";
Date t1 = new GregorianCalendar().getTime();
System.out.println(t1.getHours() + ":" + t1.getMinutes());
// append time to contents
System.out.println("day: " + t1.getDay());
System.out.println("day: " + t1.getYear());
System.out.println("day: " + t1.getMonth());
// time
String time = "time(" + t1.getHours() + ":integer).";
System.out.println(time);
String url = poslAddress;
// String url = "http://www.jdrew.org/oojdrew/test.posl";
String contents = "";
// day of the week
int day = t1.getDay();
boolean weekday = true;
if (day == 0 || day == 6) {
weekday = false;
}
String dayOfWeek;
if (weekday) {
dayOfWeek = "day(weekday).";
} else {
dayOfWeek = "day(weekend).";
}
// full date
Calendar cal = new GregorianCalendar();
int year = cal.get(Calendar.YEAR);
int month = cal.get(Calendar.MONTH) + 1;
int day2 = cal.get(Calendar.DAY_OF_MONTH);
String date;
String day3 = "" + day2;
if (day2 == 1 || day2 == 2 || day2 == 3 || day2 == 4 || day2 == 5 || day2 == 6 || day2 == 7
|| day2 == 8 || day2 == 9) {
day3 = "0" + day2;
}
if (month == 10 || month == 11 || month == 12) date = "" + year + month + day3;
else date = "" + year + "0" + month + day3;
date = "date(" + date + ":integer).";
System.out.println(date);
String url2 = rdfAddress;
HttpClient client2 = new HttpClient();
GetMethod method2 = new GetMethod(url2);
method2.setFollowRedirects(true);
String typestr = "";
// Execute the GET method
int statusCode2 = client2.executeMethod(method2);
if (statusCode2 != -1) {
typestr = method2.getResponseBodyAsString();
}
System.out.println("Types: " + typestr);
Types.reset();
RDFSParser.parseRDFSString(typestr);
try {
HttpClient client = new HttpClient();
GetMethod method = new GetMethod(url);
method.setFollowRedirects(true);
// Execute the GET method
int statusCode = client.executeMethod(method);
if (statusCode != -1) {
contents = method.getResponseBodyAsString();
}
} catch (Exception e) {
e.printStackTrace();
}
contents = contents + "\n" + time;
contents = contents + "\n" + dayOfWeek;
contents = contents + "\n" + date;
BackwardReasoner br = new BackwardReasoner();
Iterator solit = null;
DefiniteClause dc = null;
SymbolTable.reset();
pp.parseDefiniteClauses(contents);
br.loadClauses(pp.iterator());
System.out.println("TEST");
Iterator it = pp.iterator();
while (it.hasNext()) {
DefiniteClause d = (DefiniteClause) it.next();
System.out.println("Loaded clause: " + d.toPOSLString());
}
br = new BackwardReasoner(br.clauses, br.oids);
MessageParser m = new MessageParser(message);
Element atom = null;
try {
atom = m.parseForContent();
} catch (Exception e) {
System.out.println("Invalid Message");
// out.flush();
}
QueryBuilder q = new QueryBuilder(atom);
String query = q.generateDoc();
System.out.println("ABOUT TO INPUT THIS QUERY:" + query);
RuleMLParser qp = new RuleMLParser();
try {
dc = qp.parseRuleMLQuery(query);
} catch (Exception e) {
System.out.println("Invalid Query");
// out.flush();
}
// solit = br.iterativeDepthFirstSolutionIterator(dc);
solit = br.iterativeDepthFirstSolutionIterator(dc);
int varSize = 0;
while (solit.hasNext()) {
Vector data = new Vector();
BackwardReasoner.GoalList gl = (BackwardReasoner.GoalList) solit.next();
Hashtable varbind = gl.varBindings;
javax.swing.tree.DefaultMutableTreeNode root = br.toTree();
root.setAllowsChildren(true);
javax.swing.tree.DefaultTreeModel dtm = new DefaultTreeModel(root);
int i = 0;
Object[][] rowdata = new Object[varbind.size()][2];
varSize = varbind.size();
Enumeration e = varbind.keys();
while (e.hasMoreElements()) {
Object k = e.nextElement();
Object val = varbind.get(k);
String ks = (String) k;
rowdata[i][0] = ks;
rowdata[i][1] = val;
i++;
}
data.addElement(rowdata);
String[] messages = new String[data.size()];
MessageGenerator g =
new MessageGenerator(
data, varSize, messageEndpoint, m.getId(), m.getProtocol(), m.getRel(), varOrder);
messages = g.Messages2();
String appender = "";
URL sender = new URL(address + ":" + port);
HttpMessage msg = new HttpMessage(sender);
Properties props = new Properties();
for (int i1 = 0; i1 < data.size(); i1++) {
System.out.println(i1 + ")" + messages[i1].toString());
props.put("text", messages[i1].toString());
InputStream in = msg.sendGetMessage(props);
}
System.out.println("NEXT MESSAGE");
}
MessageGenerator g =
new MessageGenerator(
null, varSize, messageEndpoint, m.getId(), m.getProtocol(), m.getRel());
URL sender = new URL(address + ":" + port);
HttpMessage msg = new HttpMessage(sender);
Properties props = new Properties();
String finalMessage = g.finalMessage(query);
System.out.println(finalMessage);
props.put("text", finalMessage);
InputStream in = msg.sendGetMessage(props);
System.out.println("Stop_Communication");
} catch (Exception e) {
System.out.println("ERROR has occured : " + e.toString());
}
out.close();
}