/**
   * The method makes a GetMethod object and an HttpClient object. The HttpClient then executes the
   * Getmethod and returns the pagesource to the caller.
   *
   * @param url Url to fetch the pagesource for
   * @param followRedirect Boolean variable to specify GetMethod followRedirect value
   * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
   * @return String
   */
  public String getPageSourceWithoutProxy(
      String url, boolean followRedirects, boolean doAuthentication) {

    GetMethod getMethod = null;
    String pageSouce = null;
    HttpClient httpClient = new HttpClient();
    try {
      getMethod = new GetMethod(url);
      getMethod.setFollowRedirects(followRedirects);
      getMethod.setDoAuthentication(doAuthentication);
      httpClient.executeMethod(getMethod);
      pageSouce = getMethod.getResponseBodyAsString();
    } catch (Exception e) {
      l.error(e + "  " + e.getMessage() + "exception occured for url" + url);
      try {
        getMethod = new GetMethod(url);
        getMethod.setFollowRedirects(followRedirects);
        getMethod.setDoAuthentication(doAuthentication);
        httpClient.executeMethod(getMethod);
        pageSouce = getMethod.getResponseBodyAsString();
        getMethod.releaseConnection();
      } catch (Exception ex) {
        l.error(ex + "  " + ex.getMessage() + "exception occured for url " + url);
        getMethod.releaseConnection();
      }
    } finally {
      getMethod.releaseConnection();
    }
    return pageSouce;
  }
  public void testGetFromMultipleThreads() {

    this.server.setHttpService(new EchoService());

    client.setHttpConnectionManager(new MultiThreadedHttpConnectionManager());
    ExecuteMethodThread[] threads = new ExecuteMethodThread[10];

    for (int i = 0; i < threads.length; i++) {
      GetMethod method = new GetMethod("/");
      method.setFollowRedirects(true);

      threads[i] = new ExecuteMethodThread(method, client);
      threads[i].start();
    }

    for (int i = 0; i < threads.length; i++) {
      try {
        // wait until this thread finishes. we'll give it 10 seconds,
        // but it shouldn't take that long
        threads[i].join(10000);
      } catch (InterruptedException e) {
      }
      // make sure an exception did not occur
      Exception e = threads[i].getException();
      if (e != null) {
        fail("An error occured in the get: " + e);
      }
      // we should have a 200 status
      assertEquals(threads[i].getMethod().getStatusCode(), HttpStatus.SC_OK);
    }
  }
  /** Checks all the servers marked as being online if they still are online. */
  private synchronized void checkOnlineServers() {
    Iterator itr;
    itr = online.listIterator();
    while (itr.hasNext()) {
      Server server = (Server) itr.next();
      String url = getServerURL(server);
      GetMethod get = new GetMethod(url);
      get.setFollowRedirects(false);

      try {
        httpClient.executeMethod(get);
        if (!okServerResponse(get.getStatusCode())) {
          offline.add(server);
          itr.remove();
          log.debug("Server going OFFLINE! " + getServerURL(server));
          listener.serverOffline(server);
        }
      } catch (Exception e) {
        offline.add(server);
        itr.remove();
        log.debug("Server going OFFLINE! " + getServerURL(server));
        listener.serverOffline(server);
      } finally {
        get.releaseConnection();
      }
    }
  }
 private String getHistoryFile(Configuration conf, String jobId) throws IOException {
   String jtAddress = "scheme://" + conf.get("mapred.job.tracker");
   String jtHttpAddr = "scheme://" + conf.get("mapred.job.tracker.http.address");
   try {
     String host = new URI(jtAddress).getHost();
     int port = new URI(jtHttpAddr).getPort();
     HttpClient client = new HttpClient();
     String jobUrl = "http://" + host + ":" + port + "/jobdetails.jsp";
     GetMethod get = new GetMethod(jobUrl);
     get.setQueryString("jobid=" + jobId);
     get.setFollowRedirects(false);
     int status = client.executeMethod(get);
     String file = null;
     if (status == HttpStatus.SC_MOVED_PERMANENTLY || status == HttpStatus.SC_MOVED_TEMPORARILY) {
       file = get.getResponseHeader("Location").toString();
       file = file.substring(file.lastIndexOf('=') + 1);
       file = JobHistory.JobInfo.decodeJobHistoryFileName(file);
     } else {
       LOG.warn("JobURL {} for id: {} returned {}", jobUrl, jobId, status);
     }
     return file;
   } catch (URISyntaxException e) {
     throw new IOException("JT Address: " + jtAddress + ", http Address: " + jtHttpAddr, e);
   }
 }
  /**
   * Download a page from the server and return its content. Throws a {@link RuntimeException} on
   * connection problems etc.
   *
   * @param url URL of the page
   * @return content of the page
   */
  protected static InputStream getUrlContent(String url) {
    GetMethod get = new GetMethod(url);
    get.setFollowRedirects(true);
    if (isLoggedIn()) {
      get.setDoAuthentication(true);
      get.addRequestHeader(
          "Authorization", "Basic " + new String(Base64.encodeBase64("Admin:admin".getBytes())));
    }

    try {
      int statusCode = AbstractEscapingTest.getClient().executeMethod(get);
      switch (statusCode) {
        case HttpStatus.SC_OK:
          // everything is fine
          break;
        case HttpStatus.SC_UNAUTHORIZED:
          // do not fail on 401 (unauthorized), used in some tests
          System.out.println("WARNING, Ignoring status 401 (unauthorized) for URL: " + url);
          break;
        case HttpStatus.SC_CONFLICT:
          // do not fail on 409 (conflict), used in some templates
          System.out.println("WARNING, Ignoring status 409 (conflict) for URL: " + url);
          break;
        case HttpStatus.SC_NOT_FOUND:
          // ignore 404 (the page is still rendered)
          break;
        default:
          throw new RuntimeException(
              "HTTP GET request returned status "
                  + statusCode
                  + " ("
                  + get.getStatusText()
                  + ") for URL: "
                  + url);
      }

      // get the data, converting to utf-8
      String str = get.getResponseBodyAsString();
      if (str == null) {
        return null;
      }
      return new ByteArrayInputStream(str.getBytes("utf-8"));
    } catch (IOException exception) {
      throw new RuntimeException("Error retrieving URL: " + url + "\n" + exception);
    } finally {
      get.releaseConnection();
    }
  }
Example #6
0
  /** Run method of the thread */
  public void run() {

    queue = manager.workQueue;
    while (manager.hasWorkLeft()) {

      working = false;
      // code to make the worker pause, if the pause button has been presed

      // if the stop signal has been given stop the thread
      if (stop) {
        return;
      }

      // this pasuses the thread
      synchronized (this) {
        while (pleaseWait) {
          try {
            wait();
          } catch (InterruptedException e) {
            return;
          } catch (Exception e) {
            e.printStackTrace();
          }
        }
      }

      GetMethod httpget = null;
      HeadMethod httphead = null;

      try {

        work = (WorkUnit) queue.take();
        working = true;
        url = work.getWork();
        int code = 0;

        String responce = "";
        String rawResponce = "";

        // if the work is a head request
        if (work.getMethod().equalsIgnoreCase("HEAD")) {
          if (Config.debug) {
            System.out.println("DEBUG Worker[" + threadId + "]: HEAD " + url.toString());
          }

          httphead = new HeadMethod(url.toString());

          // set the custom HTTP headers
          Vector HTTPheaders = manager.getHTTPHeaders();
          for (int a = 0; a < HTTPheaders.size(); a++) {
            HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
            /*
             * Host header has to be set in a different way!
             */
            if (httpHeader.getHeader().startsWith("Host")) {
              httphead.getParams().setVirtualHost(httpHeader.getValue());
            } else {
              httphead.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
            }
          }
          httphead.setFollowRedirects(Config.followRedirects);

          /*
           * this code is used to limit the number of request/sec
           */
          if (manager.isLimitRequests()) {
            while (manager.getTotalDone()
                    / ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
                > manager.getLimitRequestsTo()) {
              Thread.sleep(100);
            }
          }
          /*
           * Send the head request
           */
          code = httpclient.executeMethod(httphead);
          if (Config.debug) {
            System.out.println("DEBUG Worker[" + threadId + "]: " + code + " " + url.toString());
          }
          httphead.releaseConnection();

        }
        // if we are doing a get request
        else if (work.getMethod().equalsIgnoreCase("GET")) {
          // make the request;
          if (Config.debug) {
            System.out.println("DEBUG Worker[" + threadId + "]: GET " + url.toString());
          }
          httpget = new GetMethod(url.toString());

          // set the custom HTTP headers
          Vector HTTPheaders = manager.getHTTPHeaders();
          for (int a = 0; a < HTTPheaders.size(); a++) {

            HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
            /*
             * Host header has to be set in a different way!
             */
            if (httpHeader.getHeader().startsWith("Host")) {
              httpget.getParams().setVirtualHost(httpHeader.getValue());
            } else {
              httpget.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
            }
          }
          httpget.setFollowRedirects(Config.followRedirects);

          /*
           * this code is used to limit the number of request/sec
           */
          if (manager.isLimitRequests()) {
            while (manager.getTotalDone()
                    / ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
                > manager.getLimitRequestsTo()) {
              Thread.sleep(100);
            }
          }

          code = httpclient.executeMethod(httpget);

          if (Config.debug) {
            System.out.println("DEBUG Worker[" + threadId + "]: " + code + " " + url.toString());
          }

          // set up the input stream
          BufferedReader input =
              new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));

          // save the headers into a string, used in viewing raw responce
          String rawHeader;
          rawHeader = httpget.getStatusLine() + "\r\n";
          Header headers[] = httpget.getResponseHeaders();

          StringBuffer buf = new StringBuffer();
          for (int a = 0; a < headers.length; a++) {
            buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
          }

          rawHeader = rawHeader + buf.toString();

          buf = new StringBuffer();
          // read in the responce body
          String line;
          while ((line = input.readLine()) != null) {
            buf.append("\r\n" + line);
          }
          responce = buf.toString();
          input.close();

          rawResponce = rawHeader + responce;
          // clean the responce

          // parse the html of what we have found

          if (Config.parseHTML && !work.getBaseCaseObj().isUseRegexInstead()) {
            Header contentType = httpget.getResponseHeader("Content-Type");

            if (contentType != null) {
              if (contentType.getValue().startsWith("text")) {
                manager.addHTMLToParseQueue(new HTMLparseWorkUnit(responce, work));
              }
            }
          }

          responce = FilterResponce.CleanResponce(responce, work);

          Thread.sleep(10);
          httpget.releaseConnection();
        } else {
          // There is no need to deal with requests other than HEAD or GET
        }

        // if we need to check the against the base case
        if (work.getMethod().equalsIgnoreCase("GET")
            && work.getBaseCaseObj().useContentAnalysisMode()) {
          if (code == 200) {
            if (Config.debug) {
              System.out.println(
                  "DEBUG Worker[" + threadId + "]: Base Case Check " + url.toString());
            }

            // TODO move this option to the Adv options
            // if the responce does not match the base case
            Pattern regexFindFile = Pattern.compile(".*file not found.*", Pattern.CASE_INSENSITIVE);

            Matcher m = regexFindFile.matcher(responce);

            // need to clean the base case of the item we are looking for
            String basecase =
                FilterResponce.removeItemCheckedFor(
                    work.getBaseCaseObj().getBaseCase(), work.getItemToCheck());

            if (m.find()) {
              // do nothing as we have a 404
            } else if (!responce.equalsIgnoreCase(basecase)) {
              if (work.isDir()) {
                if (Config.debug) {
                  System.out.println(
                      "DEBUG Worker[" + threadId + "]: Found Dir (base case)" + url.toString());
                }
                // we found a dir
                manager.foundDir(url, code, responce, basecase, rawResponce, work.getBaseCaseObj());
              } else {
                // found a file
                if (Config.debug) {
                  System.out.println(
                      "DEBUG Worker[" + threadId + "]: Found File (base case)" + url.toString());
                }
                manager.foundFile(
                    url,
                    code,
                    responce,
                    work.getBaseCaseObj().getBaseCase(),
                    rawResponce,
                    work.getBaseCaseObj());
              }
            }
          } else if (code == 404 || code == 400) {
            // again do nothing as it is not there
          } else {
            if (work.isDir()) {
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Found Dir (base case)" + url.toString());
              }
              // we found a dir
              manager.foundDir(
                  url,
                  code,
                  responce,
                  work.getBaseCaseObj().getBaseCase(),
                  rawResponce,
                  work.getBaseCaseObj());
            } else {
              // found a file
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Found File (base case)" + url.toString());
              }
              manager.foundFile(
                  url,
                  code,
                  responce,
                  work.getBaseCaseObj().getBaseCase(),
                  rawResponce,
                  work.getBaseCaseObj());
            }
            // manager.foundError(url, "Base Case Mode Error - Responce code came back as " + code +
            // " it should have been 200");
            // manager.workDone();
          }
        }
        /*
         * use the custom regex check instead
         */
        else if (work.getBaseCaseObj().isUseRegexInstead()) {
          Pattern regexFindFile = Pattern.compile(work.getBaseCaseObj().getRegex());

          Matcher m = regexFindFile.matcher(rawResponce);
          /*
          System.out.println("======Trying to find======");
          System.out.println(work.getBaseCaseObj().getRegex());
          System.out.println("======In======");
          System.out.println(responce);
          System.out.println("======/In======");
           */
          if (m.find()) {
            // do nothing as we have a 404
            if (Config.debug) {

              System.out.println(
                  "DEBUG Worker[" + threadId + "]: Regex matched so it's a 404, " + url.toString());
            }

          } else {
            if (Config.parseHTML) {
              Header contentType = httpget.getResponseHeader("Content-Type");

              if (contentType != null) {
                if (contentType.getValue().startsWith("text")) {
                  manager.addHTMLToParseQueue(new HTMLparseWorkUnit(rawResponce, work));
                }
              }
            }
            if (work.isDir()) {
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Found Dir (regex) " + url.toString());
              }
              // we found a dir
              manager.foundDir(
                  url,
                  code,
                  responce,
                  work.getBaseCaseObj().getBaseCase(),
                  rawResponce,
                  work.getBaseCaseObj());
            } else {
              // found a file
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Found File (regex) " + url.toString());
              }
              manager.foundFile(
                  url,
                  code,
                  responce,
                  work.getBaseCaseObj().getBaseCase(),
                  rawResponce,
                  work.getBaseCaseObj());
            }
            // manager.foundError(url, "Base Case Mode Error - Responce code came back as " + code +
            // " it should have been 200");
            // manager.workDone();
          }

        }
        // just check the responce code
        else {
          // if is not the fail code, a 404 or a 400 then we have a possible
          if (code != work.getBaseCaseObj().getFailCode()
              && code != 404
              && code != 0
              && code != 400) {
            if (work.getMethod().equalsIgnoreCase("HEAD")) {
              if (Config.debug) {
                System.out.println(
                    "DEBUG Worker[" + threadId + "]: Getting responce via GET " + url.toString());
              }
              rawResponce = "";

              httpget = new GetMethod(url.toString());
              Vector HTTPheaders = manager.getHTTPHeaders();
              for (int a = 0; a < HTTPheaders.size(); a++) {
                HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
                httpget.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
              }
              httpget.setFollowRedirects(Config.followRedirects);

              /*
               * this code is used to limit the number of request/sec
               */
              if (manager.isLimitRequests()) {
                while (manager.getTotalDone()
                        / ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
                    > manager.getLimitRequestsTo()) {
                  Thread.sleep(100);
                }
              }

              int newCode = httpclient.executeMethod(httpget);

              // in some cases the second get can return a different result, than the first head
              // request!
              if (newCode != code) {
                manager.foundError(
                    url,
                    "Return code for first HEAD, is different to the second GET: "
                        + code
                        + " - "
                        + newCode);
              }

              rawResponce = "";
              // build a string version of the headers
              rawResponce = httpget.getStatusLine() + "\r\n";
              Header headers[] = httpget.getResponseHeaders();

              StringBuffer buf = new StringBuffer();
              for (int a = 0; a < headers.length; a++) {
                buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
              }

              buf.append("\r\n");

              rawResponce = rawResponce + buf.toString();

              if (httpget.getResponseContentLength() > 0) {

                // get the http body
                BufferedReader input =
                    new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));

                String line;

                String tempResponce = "";

                buf = new StringBuffer();
                while ((line = input.readLine()) != null) {
                  buf.append("\r\n" + line);
                }
                tempResponce = buf.toString();
                input.close();

                rawResponce = rawResponce + tempResponce;

                Header contentType = httpget.getResponseHeader("Content-Type");

                if (Config.parseHTML) {
                  contentType = httpget.getResponseHeader("Content-Type");

                  if (contentType != null) {
                    if (contentType.getValue().startsWith("text")) {
                      manager.addHTMLToParseQueue(new HTMLparseWorkUnit(tempResponce, work));
                    }
                  }
                }
              }

              httpget.releaseConnection();
            }

            if (work.isDir()) {
              manager.foundDir(url, code, rawResponce, work.getBaseCaseObj());
            } else {
              manager.foundFile(url, code, rawResponce, work.getBaseCaseObj());
            }
          }
        }

        manager.workDone();
        Thread.sleep(20);

      } catch (NoHttpResponseException e) {
        manager.foundError(url, "NoHttpResponseException " + e.getMessage());
        manager.workDone();
      } catch (ConnectTimeoutException e) {
        manager.foundError(url, "ConnectTimeoutException " + e.getMessage());
        manager.workDone();
      } catch (URIException e) {
        manager.foundError(url, "URIException " + e.getMessage());
        manager.workDone();
      } catch (IOException e) {

        manager.foundError(url, "IOException " + e.getMessage());
        manager.workDone();
      } catch (InterruptedException e) {
        // manager.foundError(url, "InterruptedException " + e.getMessage());
        manager.workDone();
        return;
      } catch (IllegalArgumentException e) {

        e.printStackTrace();
        manager.foundError(url, "IllegalArgumentException " + e.getMessage());
        manager.workDone();
      } finally {
        if (httpget != null) {
          httpget.releaseConnection();
        }

        if (httphead != null) {
          httphead.releaseConnection();
        }
      }
    }
  }
  /**
   * The method reads a proxy object from database, makes a GetMethod object, appends required
   * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
   * pagesource to the caller.
   *
   * @param iCount Counter variable for passing thread group information
   * @param url Url to fetch the pagesource for
   * @param followRedirect Boolean variable to specify GetMethod followRedirect value
   * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
   * @param region the local region of a given url
   * @param objProxyDao the database layer ProxyDao object variable
   * @param useErrsy Boolean variable to specify usage of Errsy as proxy source
   * @return String
   */
  public String getPageSourceWithProxy(
      String url,
      boolean followRedirect,
      boolean doAuthentication,
      String region,
      Boolean useErrsy,
      String google) {

    String page = " ";
    String pageSource = "";
    int i = 0;
    String exception = " ";
    HttpClientParams clientParams = new HttpClientParams();
    clientParams.setSoTimeout(40000);
    clientParams.setConnectionManagerTimeout(40000);
    HttpClient httpclient = new HttpClient(clientParams);
    GetMethod getmethod = null;

    HttpState state = new HttpState();

    //  if (ProxyDao.lstProxyData.size() == 16) {
    ProxyData objProxyData = null;
    // if (!useErrsy) {
    try {
      // objProxyData = ProxyDao.lstProxyData.get(iCount);
      objProxyData = ProxyDao.objProxyData;
      if (objProxyData == null) {
        //                objProxyDao.changeProxy(google);
        objProxyData = ProxyDao.objProxyData;
      }
      httpclient
          .getHostConfiguration()
          .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
    } catch (Exception e) {
      pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
      return pageSource;
    }
    /*} else {
    try {
    objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
    Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
    httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
    httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
    state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
    httpclient.setState(state);
    } catch (Exception e) {
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
    return pageSource;
    }
    }*/
    try {
      getmethod = new GetMethod(url);
      getmethod.addRequestHeader(
          "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
      if (url.contains("bing.com")) {

        if (region.equalsIgnoreCase("co.uk")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.sg")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.au")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("co.in")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("ca")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.ph")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.my")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("it")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        }
      }
      getmethod.setFollowRedirects(true);
      getmethod.setDoAuthentication(true);
      httpclient.getParams().setAuthenticationPreemptive(true);
      httpclient.setState(state);
      String num100Header = "";
      //            if (url.contains("google")) {
      //                int j = 0;
      //                String url1 = "http://www.google.com/";
      //                try {
      //                    GetMethod objGetMethod = new GetMethod(url1);
      //                    j = httpclient.executeMethod(objGetMethod);
      //                    Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
      //                    String header = responseHeader.getValue();
      //                    String[] headerValue = header.split(";");
      //
      //                    for (String head : headerValue) {
      //                        if (head.contains("PREF=ID")) {
      //                            header = head;
      //                            break;
      //                        }
      //                    }
      //                    String[] splitAll = header.split(":");
      //                    long time = System.currentTimeMillis()+400;
      //                    String sTime = "" + time;
      //                    sTime = sTime.substring(0, 10);
      //                    //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1]  +
      // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
      //                    num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
      // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
      //                    Cookie ck = new Cookie("PREF", "PREF", num100Header);
      //                    httpclient.getState().clearCookies();
      //                    httpclient.getState().addCookie(ck);
      //                    getmethod.addRequestHeader("Host", "www.google.com");
      //                    getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
      // rv:19.0) Gecko/20100101 Firefox/19.0");
      //                    getmethod.addRequestHeader("Accept",
      // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
      //                    getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
      //                    getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
      //                    getmethod.addRequestHeader("Referer", "https://www.google.com/");
      //                    System.out.println(num100Header);
      //                } catch (Exception ex) {
      //                    exception = ex.getMessage();
      //                    l.debug(ex + "  " + ex.getMessage() + "Exception occured for url" +
      // url);
      //                    pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
      //                    return pageSource;
      //                }
      //            }
      i = httpclient.executeMethod(getmethod);
      if (i / 100 == 4 || i / 100 == 5) {
        page = "<PROXY ERROR>";

      } else {

        page = getmethod.getResponseBodyAsString();
      }
    } catch (SocketTimeoutException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (SocketException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (Exception ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);

    } finally {
      getmethod.releaseConnection();
    }
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;

    // }
    return pageSource;
  }
  public void doPost(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    try {
      address = PAConfiguration.getAddress();
      port = PAConfiguration.getPort(instantiation);
      poslAddress = PAConfiguration.getPOSL(instantiation, topic);
      rdfAddress = PAConfiguration.getRDFTaxonomy(instantiation);
      messageEndpoint = PAConfiguration.getEndpointName(instantiation, topic);
    } catch (BadConfigurationException e) {
      System.out.println(e.getMessage());
      e.printStackTrace();
      System.exit(0);
    }
    response.setContentType("text/html; charset=UTF-8");
    PrintWriter out = response.getWriter();

    try {
      System.out.println("5 Publicty Chair Servlet");
      System.out.println(response.toString());

      BufferedReader brd = request.getReader();

      String input = "";
      String message = "";

      while (!input.equals("</RuleML>")) {

        input = brd.readLine();

        message = message + input;
      }
      String[] varOrder = getVariableOrder(message);
      System.out.println("Received Message: " + message);

      //	BackwardReasoner br = new BackwardReasoner();
      //   Iterator solit =null;
      //   DefiniteClause dc = null;
      //   SymbolTable.reset();

      POSLParser pp = new POSLParser();
      // String contents = "c(a).\nc(b).\nc(c).";

      Date t1 = new GregorianCalendar().getTime();
      System.out.println(t1.getHours() + ":" + t1.getMinutes());
      // append time to contents

      System.out.println("day: " + t1.getDay());
      System.out.println("day: " + t1.getYear());
      System.out.println("day: " + t1.getMonth());

      // time
      String time = "time(" + t1.getHours() + ":integer).";
      System.out.println(time);

      String url = poslAddress;

      // String url = "http://www.jdrew.org/oojdrew/test.posl";
      String contents = "";

      // day of the week
      int day = t1.getDay();
      boolean weekday = true;

      if (day == 0 || day == 6) {
        weekday = false;
      }

      String dayOfWeek;

      if (weekday) {
        dayOfWeek = "day(weekday).";
      } else {
        dayOfWeek = "day(weekend).";
      }
      // full date
      Calendar cal = new GregorianCalendar();

      int year = cal.get(Calendar.YEAR);
      int month = cal.get(Calendar.MONTH) + 1;
      int day2 = cal.get(Calendar.DAY_OF_MONTH);

      String date;

      String day3 = "" + day2;

      if (day2 == 1 || day2 == 2 || day2 == 3 || day2 == 4 || day2 == 5 || day2 == 6 || day2 == 7
          || day2 == 8 || day2 == 9) {

        day3 = "0" + day2;
      }

      if (month == 10 || month == 11 || month == 12) date = "" + year + month + day3;
      else date = "" + year + "0" + month + day3;

      date = "date(" + date + ":integer).";

      System.out.println(date);

      String url2 = rdfAddress;
      HttpClient client2 = new HttpClient();
      GetMethod method2 = new GetMethod(url2);
      method2.setFollowRedirects(true);
      String typestr = "";
      // Execute the GET method
      int statusCode2 = client2.executeMethod(method2);
      if (statusCode2 != -1) {
        typestr = method2.getResponseBodyAsString();
      }
      System.out.println("Types: " + typestr);
      Types.reset();
      RDFSParser.parseRDFSString(typestr);

      try {
        HttpClient client = new HttpClient();
        GetMethod method = new GetMethod(url);
        method.setFollowRedirects(true);

        // Execute the GET method
        int statusCode = client.executeMethod(method);
        if (statusCode != -1) {
          contents = method.getResponseBodyAsString();
        }
      } catch (Exception e) {
        e.printStackTrace();
      }
      contents = contents + "\n" + time;
      contents = contents + "\n" + dayOfWeek;
      contents = contents + "\n" + date;

      BackwardReasoner br = new BackwardReasoner();
      Iterator solit = null;
      DefiniteClause dc = null;
      SymbolTable.reset();

      pp.parseDefiniteClauses(contents);

      br.loadClauses(pp.iterator());
      System.out.println("TEST");
      Iterator it = pp.iterator();
      while (it.hasNext()) {
        DefiniteClause d = (DefiniteClause) it.next();
        System.out.println("Loaded clause: " + d.toPOSLString());
      }

      br = new BackwardReasoner(br.clauses, br.oids);

      MessageParser m = new MessageParser(message);
      Element atom = null;

      try {

        atom = m.parseForContent();

      } catch (Exception e) {

        System.out.println("Invalid Message");
        // out.flush();

      }

      QueryBuilder q = new QueryBuilder(atom);
      String query = q.generateDoc();
      System.out.println("ABOUT TO INPUT THIS QUERY:" + query);
      RuleMLParser qp = new RuleMLParser();

      try {

        dc = qp.parseRuleMLQuery(query);

      } catch (Exception e) {
        System.out.println("Invalid Query");
        // out.flush();
      }

      // solit = br.iterativeDepthFirstSolutionIterator(dc);

      solit = br.iterativeDepthFirstSolutionIterator(dc);

      int varSize = 0;

      while (solit.hasNext()) {

        Vector data = new Vector();

        BackwardReasoner.GoalList gl = (BackwardReasoner.GoalList) solit.next();

        Hashtable varbind = gl.varBindings;
        javax.swing.tree.DefaultMutableTreeNode root = br.toTree();
        root.setAllowsChildren(true);

        javax.swing.tree.DefaultTreeModel dtm = new DefaultTreeModel(root);

        int i = 0;
        Object[][] rowdata = new Object[varbind.size()][2];
        varSize = varbind.size();

        Enumeration e = varbind.keys();
        while (e.hasMoreElements()) {
          Object k = e.nextElement();
          Object val = varbind.get(k);
          String ks = (String) k;
          rowdata[i][0] = ks;
          rowdata[i][1] = val;
          i++;
        }

        data.addElement(rowdata);
        String[] messages = new String[data.size()];
        MessageGenerator g =
            new MessageGenerator(
                data, varSize, messageEndpoint, m.getId(), m.getProtocol(), m.getRel(), varOrder);
        messages = g.Messages2();

        String appender = "";

        URL sender = new URL(address + ":" + port);
        HttpMessage msg = new HttpMessage(sender);
        Properties props = new Properties();

        for (int i1 = 0; i1 < data.size(); i1++) {
          System.out.println(i1 + ")" + messages[i1].toString());
          props.put("text", messages[i1].toString());
          InputStream in = msg.sendGetMessage(props);
        }
        System.out.println("NEXT MESSAGE");
      }

      MessageGenerator g =
          new MessageGenerator(
              null, varSize, messageEndpoint, m.getId(), m.getProtocol(), m.getRel());

      URL sender = new URL(address + ":" + port);
      HttpMessage msg = new HttpMessage(sender);
      Properties props = new Properties();

      String finalMessage = g.finalMessage(query);

      System.out.println(finalMessage);

      props.put("text", finalMessage);
      InputStream in = msg.sendGetMessage(props);

      System.out.println("Stop_Communication");

    } catch (Exception e) {
      System.out.println("ERROR has occured : " + e.toString());
    }
    out.close();
  }