Esempio n. 1
0
  public static void main(String args[]) {

    HTTPDataSource ds = new HTTPDataSource();

    try {
      if (args.length != 1 && args.length != 2) {
        throw new Exception(
            /* (non-Javadoc)
             * @i18n.test
             * @org-mes="Need an url"
             */
            org.openlaszlo.i18n.LaszloMessages.getMessage(
                HTTPDataSource.class.getName(), "051018-828"));
      }
      String surl = args[0];
      FileOutputStream out = null;
      if (args.length == 2) {
        out = new FileOutputStream(args[1]);
      }
      System.out.println("url is " + surl);

      HttpData data = ds.getDataOnce(null, null, -1, surl, 0, 0);

      System.out.println("Response code: " + data.code);

      if (out != null) {
        FileUtils.send(data.getInputStream(), out);
      }

      data.release();

    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Esempio n. 2
0
 public void setErrResponse(Exception ex, Response response) {
   if (ex instanceof java.sql.SQLException
       || ex instanceof com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException
       || ex instanceof org.springframework.jdbc.BadSqlGrammarException
       || ex instanceof org.springframework.dao.InvalidDataAccessApiUsageException
       || ex instanceof org.springframework.dao.DataAccessException
       || ex instanceof org.springframework.web.util.NestedServletException
       || ex
           instanceof com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException) {
     response.setResponseMsg(ex.getMessage());
     response.setResponseStatus(ConstException.ERR_CODE_DB_ERROR);
   } else if (ex instanceof java.lang.NullPointerException) {
     response.setResponseMsg(ex.getMessage());
     response.setResponseStatus(ConstException.ERR_CODE_UNKNOWN);
   } else if (ex instanceof ConstException) {
     ConstException constException = (ConstException) ex;
     response.setResponseMsg(constException.getMessage());
     response.setResponseStatus(constException.getCode());
   } else if (ex instanceof java.lang.IllegalThreadStateException) {
     response.setResponseMsg(ex.getMessage());
     response.setResponseStatus(ConstException.ERR_CODE_INVALID_THREAD_STOP);
   } else {
     setErrResponse(ex, response, ConstException.ERR_CODE_UNKNOWN);
   }
 }
 /**
  * Method to make a GET HTTP connecton to the given url and return the output
  *
  * @param urlToFetch url to be connected
  * @return the http get response
  */
 public String makeRequest(String urlToFetch, String region) throws IOException {
   String responseBody = "";
   try {
     HttpClientParams clientParams = new HttpClientParams();
     clientParams.setSoTimeout(40000);
     clientParams.setConnectionManagerTimeout(40000);
     HttpClient httpclient = new HttpClient(clientParams);
     GetMethod httpget = new GetMethod(urlToFetch);
     if (urlToFetch.contains("linkedin.com/countserv")) {
       httpget.addRequestHeader("Host", "www.linkedin.com");
       httpget.addRequestHeader(
           "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:20.0) Gecko/20100101 Firefox/20.0");
       httpget.addRequestHeader(
           "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
       httpget.addRequestHeader(
           "Cookie",
           "X-LI-IDC=C1; bcookie=\"v=2&618ed56d-275f-4dcd-86cd-6a44c8421879\"; bscookie=\"v=1&201305170926410ddaf2a1-e238-4bf6-8a0a-ce8a3df25934AQFZX82SfErJuFQKMJV49JfQTvewhfzh\"; X-LI-IDC=C1");
     }
     //            httpclient.getHostConfiguration().setProxy("46.227.68.2", 3128);
     //            Credentials cred = new UsernamePasswordCredentials("mmongoose",
     // "I-PHNBV9JHW6US");
     //            httpclient.getState().setProxyCredentials(AuthScope.ANY, cred);
     int i = httpclient.executeMethod(httpget);
     responseBody = httpget.getResponseBodyAsString();
   } catch (Exception e) {
     l.error(e + "  " + e.getMessage() + " url " + urlToFetch);
   }
   return responseBody;
 }
  private String getStoreKey() {
    try {
      System.out.println("CURRENT STORE KEY VALUE: " + storeKey);

      if (storeKey == null) {
        Store store = KeyStorage.INSTANCE.getByName("jsonhub-store-key");

        if (store != null) {
          this.storeKey = store.getKey();

          System.out.println("OBJDB STORE KEY VALUE: " + storeKey);

        } else {
          JSONObject jshubkey = register();

          this.storeKey = jshubkey.getString("storeKey");

          System.out.println("REGISTRATION STORE KEY VALUE: " + storeKey);

          store = new Store();
          store.setName("jsonhub-store-key");
          store.setKey(storeKey);

          KeyStorage.INSTANCE.add(store);
        }
      }

      return storeKey;
    } catch (Exception e) {
      throw new RuntimeException("could not get jsonhub store key! \n Reason: \n" + e.getMessage());
    }
  }
 private String getRequest(String path) {
   logger.log(Level.FINEST, "PR-GET-REQUEST:" + path);
   HttpClient client = getHttpClient();
   client.getState().setCredentials(AuthScope.ANY, credentials);
   GetMethod httpget = new GetMethod(path);
   client.getParams().setAuthenticationPreemptive(true);
   String response = null;
   int responseCode;
   try {
     responseCode = client.executeMethod(httpget);
     InputStream responseBodyAsStream = httpget.getResponseBodyAsStream();
     StringWriter stringWriter = new StringWriter();
     IOUtils.copy(responseBodyAsStream, stringWriter, "UTF-8");
     response = stringWriter.toString();
   } catch (Exception e) {
     e.printStackTrace();
     throw new RuntimeException("Failed to process PR get request; " + path, e);
   }
   logger.log(Level.FINEST, "PR-GET-RESPONSE:" + response);
   if (!validResponseCode(responseCode)) {
     logger.log(Level.SEVERE, "Failing to get response from Stash PR GET" + path);
     throw new RuntimeException(
         "Didn't get a 200 response from Stash PR GET! Response; '"
             + HttpStatus.getStatusText(responseCode)
             + "' with message; "
             + response);
   }
   return response;
 }
  /**
   * The method makes a GetMethod object and an HttpClient object. The HttpClient then executes the
   * Getmethod and returns the pagesource to the caller.
   *
   * @param url Url to fetch the pagesource for
   * @param followRedirect Boolean variable to specify GetMethod followRedirect value
   * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
   * @return String
   */
  public String getPageSourceWithoutProxy(
      String url, boolean followRedirects, boolean doAuthentication) {

    GetMethod getMethod = null;
    String pageSouce = null;
    HttpClient httpClient = new HttpClient();
    try {
      getMethod = new GetMethod(url);
      getMethod.setFollowRedirects(followRedirects);
      getMethod.setDoAuthentication(doAuthentication);
      httpClient.executeMethod(getMethod);
      pageSouce = getMethod.getResponseBodyAsString();
    } catch (Exception e) {
      l.error(e + "  " + e.getMessage() + "exception occured for url" + url);
      try {
        getMethod = new GetMethod(url);
        getMethod.setFollowRedirects(followRedirects);
        getMethod.setDoAuthentication(doAuthentication);
        httpClient.executeMethod(getMethod);
        pageSouce = getMethod.getResponseBodyAsString();
        getMethod.releaseConnection();
      } catch (Exception ex) {
        l.error(ex + "  " + ex.getMessage() + "exception occured for url " + url);
        getMethod.releaseConnection();
      }
    } finally {
      getMethod.releaseConnection();
    }
    return pageSouce;
  }
 public static void main(String[] args) {
   try {
     // ProxyPageSource.makeRequest("http://www.rmccurdy.com/scripts/proxy/good.txt", "");
     ProxyPageSource.geSourceWithoutProxy(
         "http://www.rmccurdy.com/scripts/proxy/good.txt", true, true);
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
 /**
  * This method makes a Json post to google plus weburl to read the google plus likes for a given
  * url. The json post includes a registered google account developer key and 13 request headers to
  * simulate browser behaviour. The json response is returned to caller class.
  *
  * @param url to fetch the data for
  * @param objProxyDao the database layer ProxyDao object variable
  * @return String
  */
 public String getPostSourceGoogle(String url, ProxyDao objProxyDao) {
   String source = "";
   org.apache.http.client.HttpClient httpClient = new DefaultHttpClient();
   try {
     HttpPost request =
         new HttpPost(
             "https://clients6.google.com/rpc?key=AIzaSyCKSbrvQasunBoV16zDH9R33D88CeLr9gQ");
     request.addHeader("Host", "clients6.google.com");
     request.addHeader(
         "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:18.0) Gecko/20100101 Firefox/18.0");
     request.addHeader(
         "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
     request.addHeader("Accept-Language", "en-US,en;q=0.5");
     request.addHeader("X-Goog-AuthUser", "0");
     request.addHeader("Authorization", "SAPISIDHASH 8b80e6cb4f438e869e8f6cc227fca5cf8856c6ab");
     request.addHeader(
         "X-ClientDetails",
         "appVersion=5.0%20(Windows)&platform=Win32&userAgent=Mozilla%2F5.0%20(Windows%20NT%206.1%3B%20rv%3A18.0)%20Gecko%2F20100101%20Firefox%2F18.0");
     request.addHeader("Content-Type", "application/json; charset=UTF-8");
     request.addHeader("X-JavaScript-User-Agent", "google-api-javascript-client/1.0.0-alpha");
     request.addHeader("X-Origin", "https://plusone.google.com");
     request.addHeader("X-Referer", "https://plusone.google.com");
     request.addHeader("X-Goog-Encode-Response-If-Executable", "base64");
     request.addHeader(
         "Cookie",
         "PREF=ID=a426d8fb1e2750ee:U=6af15e4f74c74ed0:FF=0:LD=en:TM=1361159234:LM=1361169098:S=IFSJimDkqaNmvAhe; NID=67=shTVymtcFTEolwk0opYlWFkokW8qk9CqIGlSKZNTb6yJd684LQ5aJoSk5bMrQdp29SaMHo5sDCH48dxea7HgWXoPvBIdKd3xT7f3Pm9QsY3tgvscweUNp6FVTCGMJoypsV5OniwmQun0eVPUeFtauZbS; SID=DQAAAMIAAADKWgTA8SrvfpQcPBYOvtZMfeRs8gskEeTPpe4tGoK58rkzZNsIW0d0EawqI-0PmrxWF4Je7CbcWOdh0IaJtspvsAl2IaTftD28of_srBD-19rOobp9BRDsFNgro5V6NSKozVNTOl8cmFJUdBlUZ5iU5miI9X8lixN9kvjbydirkOKsb_ptkLELKHv0ZruwADrDNQm4-80T0accTOvBTT5MI_JP3kT9bjrvP3EOlE1dd71Va2VhpjPze_J5akbj3iQQA6ENor-1r9dN2G48UVGe; HSID=AJ3vWp905JFQ4Fsm4; SSID=ALbvnRrBXqNVAquz1; APISID=hJVIUcaXfkppC-OW/A6YIudTulrIg4f-HT; SAPISID=3EZqx7bM9IesGpqO/ARcqGVQeQoWyxxxnL");
     StringEntity params =
         new StringEntity(
             "[{\"method\":\"pos.plusones.get\",\"id\":\"p\",\"params\":{\"nolog\":true,\"id\":\""
                 + url
                 + "\",\"source\":\"widget\",\"userId\":\"@viewer\",\"groupId\":\"@self\"},\"jsonrpc\":\"2.0\",\"key\":\"p\",\"apiVersion\":\"v1\"}]");
     request.setEntity(params);
     HttpResponse response = httpClient.execute(request);
     org.apache.http.StatusLine statusLine = response.getStatusLine();
     BufferedReader rd =
         new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
     String line = "";
     while ((line = rd.readLine()) != null) {
       source = source + line;
     }
   } catch (Exception ex) {
     l.error(ex + "  " + ex.getMessage() + " Exception occured for url " + url);
   } finally {
     httpClient.getConnectionManager().shutdown();
   }
   return source;
 }
Esempio n. 9
0
 /** 根据模版及参数产生静态页面 */
 public boolean createHtmlPage(String url, String htmlFileName) {
   boolean status = false;
   int statusCode = 0;
   try {
     // 创建一个HttpClient实例充当模拟浏览器
     httpClient = new HttpClient();
     // 设置httpclient读取内容时使用的字符集
     httpClient.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "gbk");
     // 创建GET方法的实例
     getMethod = new GetMethod(url);
     // 使用系统提供的默认的恢复策略,在发生异常时候将自动重试3次
     getMethod
         .getParams()
         .setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
     // 设置Get方法提交参数时使用的字符集,以支持中文参数的正常传递
     getMethod.addRequestHeader("Content-Type", "text/html;charset=gbk");
     // 执行Get方法并取得返回状态码,200表示正常,其它代码为异常
     statusCode = httpClient.executeMethod(getMethod);
     if (statusCode != 200) {
       logger.fatal("静态页面引擎在解析" + url + "产生静态页面" + htmlFileName + "时出错!");
     } else {
       // 读取解析结果
       sb = new StringBuffer();
       in = getMethod.getResponseBodyAsStream();
       br = new BufferedReader(new InputStreamReader(in));
       while ((line = br.readLine()) != null) {
         sb.append(line + "\n");
       }
       if (br != null) br.close();
       page = sb.toString();
       // 将页面中的相对路径替换成绝对路径,以确保页面资源正常访问
       page = formatPage(page);
       // 将解析结果写入指定的静态HTML文件中,实现静态HTML生成
       writeHtml(htmlFileName, page);
       status = true;
     }
   } catch (Exception ex) {
     logger.fatal("静态页面引擎在解析" + url + "产生静态页面" + htmlFileName + "时出错:" + ex.getMessage());
   } finally {
     // 释放http连接
     getMethod.releaseConnection();
   }
   return status;
 }
  private String postRequest(String path, String comment) throws UnsupportedEncodingException {
    logger.log(Level.FINEST, "PR-POST-REQUEST:" + path + " with: " + comment);
    HttpClient client = getHttpClient();
    client.getState().setCredentials(AuthScope.ANY, credentials);
    PostMethod httppost = new PostMethod(path);

    ObjectNode node = mapper.getNodeFactory().objectNode();
    node.put("text", comment);

    StringRequestEntity requestEntity = null;
    try {
      requestEntity =
          new StringRequestEntity(mapper.writeValueAsString(node), "application/json", "UTF-8");
    } catch (IOException e) {
      e.printStackTrace();
    }

    httppost.setRequestEntity(requestEntity);
    client.getParams().setAuthenticationPreemptive(true);
    String response = "";
    int responseCode;
    try {
      responseCode = client.executeMethod(httppost);
      InputStream responseBodyAsStream = httppost.getResponseBodyAsStream();
      StringWriter stringWriter = new StringWriter();
      IOUtils.copy(responseBodyAsStream, stringWriter, "UTF-8");
      response = stringWriter.toString();
      logger.log(Level.FINEST, "API Request Response: " + response);
    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException("Failed to process PR get request; " + path, e);
    }
    logger.log(Level.FINEST, "PR-POST-RESPONSE:" + response);
    if (!validResponseCode(responseCode)) {
      logger.log(Level.SEVERE, "Failing to get response from Stash PR POST" + path);
      throw new RuntimeException(
          "Didn't get a 200 response from Stash PR POST! Response; '"
              + HttpStatus.getStatusText(responseCode)
              + "' with message; "
              + response);
    }
    return response;
  }
    private static SSLContext createEasySSLContext() {
      try {
        TrustManager[] trustAllCerts =
            new TrustManager[] {
              new X509TrustManager() {
                public X509Certificate[] getAcceptedIssuers() {
                  return null;
                }

                public void checkClientTrusted(X509Certificate[] certs, String authType) {}

                public void checkServerTrusted(X509Certificate[] certs, String authType) {}
              }
            };

        SSLContext context = SSLContext.getInstance("SSL");
        context.init(null, trustAllCerts, null);
        return context;
      } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new HttpClientError(e.toString());
      }
    }
  /**
   * The method reads a proxy object from database, makes a GetMethod object, appends required
   * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
   * pagesource to the caller.
   *
   * @param iCount Counter variable for passing thread group information
   * @param url Url to fetch the pagesource for
   * @param followRedirect Boolean variable to specify GetMethod followRedirect value
   * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
   * @param region the local region of a given url
   * @param objProxyDao the database layer ProxyDao object variable
   * @param useErrsy Boolean variable to specify usage of Errsy as proxy source
   * @return String
   */
  public String getPageSourceWithProxy(
      String url,
      boolean followRedirect,
      boolean doAuthentication,
      String region,
      Boolean useErrsy,
      String google) {

    String page = " ";
    String pageSource = "";
    int i = 0;
    String exception = " ";
    HttpClientParams clientParams = new HttpClientParams();
    clientParams.setSoTimeout(40000);
    clientParams.setConnectionManagerTimeout(40000);
    HttpClient httpclient = new HttpClient(clientParams);
    GetMethod getmethod = null;

    HttpState state = new HttpState();

    //  if (ProxyDao.lstProxyData.size() == 16) {
    ProxyData objProxyData = null;
    // if (!useErrsy) {
    try {
      // objProxyData = ProxyDao.lstProxyData.get(iCount);
      objProxyData = ProxyDao.objProxyData;
      if (objProxyData == null) {
        //                objProxyDao.changeProxy(google);
        objProxyData = ProxyDao.objProxyData;
      }
      httpclient
          .getHostConfiguration()
          .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
    } catch (Exception e) {
      pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
      return pageSource;
    }
    /*} else {
    try {
    objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
    Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
    httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
    httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
    state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
    httpclient.setState(state);
    } catch (Exception e) {
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
    return pageSource;
    }
    }*/
    try {
      getmethod = new GetMethod(url);
      getmethod.addRequestHeader(
          "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
      if (url.contains("bing.com")) {

        if (region.equalsIgnoreCase("co.uk")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.sg")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.au")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("co.in")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("ca")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.ph")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.my")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("it")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        }
      }
      getmethod.setFollowRedirects(true);
      getmethod.setDoAuthentication(true);
      httpclient.getParams().setAuthenticationPreemptive(true);
      httpclient.setState(state);
      String num100Header = "";
      //            if (url.contains("google")) {
      //                int j = 0;
      //                String url1 = "http://www.google.com/";
      //                try {
      //                    GetMethod objGetMethod = new GetMethod(url1);
      //                    j = httpclient.executeMethod(objGetMethod);
      //                    Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
      //                    String header = responseHeader.getValue();
      //                    String[] headerValue = header.split(";");
      //
      //                    for (String head : headerValue) {
      //                        if (head.contains("PREF=ID")) {
      //                            header = head;
      //                            break;
      //                        }
      //                    }
      //                    String[] splitAll = header.split(":");
      //                    long time = System.currentTimeMillis()+400;
      //                    String sTime = "" + time;
      //                    sTime = sTime.substring(0, 10);
      //                    //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1]  +
      // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
      //                    num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
      // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
      //                    Cookie ck = new Cookie("PREF", "PREF", num100Header);
      //                    httpclient.getState().clearCookies();
      //                    httpclient.getState().addCookie(ck);
      //                    getmethod.addRequestHeader("Host", "www.google.com");
      //                    getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
      // rv:19.0) Gecko/20100101 Firefox/19.0");
      //                    getmethod.addRequestHeader("Accept",
      // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
      //                    getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
      //                    getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
      //                    getmethod.addRequestHeader("Referer", "https://www.google.com/");
      //                    System.out.println(num100Header);
      //                } catch (Exception ex) {
      //                    exception = ex.getMessage();
      //                    l.debug(ex + "  " + ex.getMessage() + "Exception occured for url" +
      // url);
      //                    pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
      //                    return pageSource;
      //                }
      //            }
      i = httpclient.executeMethod(getmethod);
      if (i / 100 == 4 || i / 100 == 5) {
        page = "<PROXY ERROR>";

      } else {

        page = getmethod.getResponseBodyAsString();
      }
    } catch (SocketTimeoutException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (SocketException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (Exception ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);

    } finally {
      getmethod.releaseConnection();
    }
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;

    // }
    return pageSource;
  }
  /**
   * The method delegates call to
   *
   * @method getPageSourceWithProxy(
   * @params), reads and parses the pagesource to verify the correctness of pagesource. If a
   *     restricted phrase is found in pagesource, it recursively make calls to
   * @method getPageSourceWithProxy(
   * @params) unless correct pagesource is obtained.
   * @param iCount Counter variable for passing thread group information
   * @param url Url to fetch the pagesource for
   * @param objProxyDao the database layer ProxyDao object variable
   * @param region the local region of a given url
   * @return String
   */
  public String getSource(String url, String region, String google) {

    String page = "";
    String exception = "";
    url = ReplaceAllHtmlTags.replaceUrlsFrenchCanCode(url);
    String pageSource = getPageSourceWithProxy(url, true, true, region, false, google);
    int c = 0;
    while (true) {
      c++;
      //            if (c == 10) {
      //                try {
      //                    c = 0;
      //                    pageSource = ProxyPageSource.makeRequest(url, region);
      //
      //                } catch (Exception e) {
      //                    l.error(e + "pagesource not come by makeRequest()");
      //                }
      //            }
      String[] proxyLog = pageSource.split("@@@@");
      try {
        page = proxyLog[2];
        exception = proxyLog[1];
        if (!exception.equals(" ")) {
          page = "";
        }
      } catch (Exception e) {
        page = "";
        exception = "";
      }
      if (page == null || page.equals("")) {
        //                objProxyDao.changeProxy(google);
        pageSource = getPageSourceWithProxy(url, true, true, region, false, google);
      } else if (page.contains("407 Proxy Authentication Required")
          || page.contains(
              "<span class=\"uiButtonText\">Rejestracja</span></a><span class=\"signup_box_content\">")
          || page.contains("<title>ERROR: The requested URL could not be retrieved</title>")
          || page.contains("404 Not Found")
          || page.contains("The page cannot be found")
          || page.contains("<p>HTTP Error 404. The requested resource is not found.</p>")
          || page.contains("Error 404: Not Found")
          || page.contains("HTTP Status 404 -")
          || page.contains("403 Access Denied")
          || page.contains("<h1>ERROR</h1>")
          || page.contains("400 Bad request")
          || page.contains("403 Forbidden")
          || page.contains("<title>Site not allowed - PacketIP</title>")
          || page.contains("502 Bad Gateway")
          || page.contains("but your computer or network may be sending automated queries")
          || (page.contains("To continue, please type the characters below:<br>")
              && page.contains("id=\"captcha\""))
          || page.contains("m.baidu.com")
          || page.contains("<PROXY ERROR>")
          || page.contains("Pardon the interruption")
          || page.equals(" ")) {
        // l.error("Using Errsy" + "url:" + url + "page:");
        //                objProxyDao.changeProxy(google);
        pageSource = getPageSourceWithProxy(url, true, true, region, false, google);
      } else {
        try {
          // objProxyDao.updateUseCount(ProxyDao.lstProxyData.get(iCount));
          // objProxyDao.updateUseCount(ProxyDao.objProxyData);
        } catch (Exception e) {
          l.error(e + "  " + e.getMessage() + "Here is the problem" + url);
        }
        break;
      }
    }
    return page;
  }
Esempio n. 14
0
  /**
   * Open a connection to the DODS server.
   *
   * @param urlString the URL to open.
   * @param command execute this command on the input stream
   * @throws IOException if an IO exception occurred.
   * @throws DAP2Exception if the DODS server returned an error.
   * @throws ParseException is cant parse the return
   */
  private void openConnection(String urlString, Command command)
      throws IOException, DAP2Exception, ParseException {
    HTTPMethod method = null;
    InputStream is = null;

    initSession();

    try {

      method = _session.newMethodGet(urlString);

      if (acceptCompress) method.setRequestHeader("Accept-Encoding", "deflate,gzip");

      // enable sessions
      if (allowSessions) method.setRequestHeader("X-Accept-Session", "true");

      // Execute the method.
      int statusCode = method.execute();

      // debug
      // if (debugHeaders) ucar.nc2.util.net.HttpClientManager.showHttpRequestInfo(f, method);

      if (statusCode == HTTPSession.SC_NOT_FOUND) {
        throw new DAP2Exception(DAP2Exception.NO_SUCH_FILE, method.getStatusText());
      }

      if (statusCode == HTTPSession.SC_UNAUTHORIZED) {
        throw new InvalidCredentialsException(method.getStatusText());
      }

      if (statusCode != HTTPSession.SC_OK) {
        throw new DAP2Exception(
            "Method failed:" + method.getStatusText() + " on URL= " + urlString);
      }

      // Get the response body.
      is = method.getResponseAsStream();

      // check if its an error
      Header header = method.getResponseHeader("Content-Description");
      if (header != null
          && (header.getValue().equals("dods-error") || header.getValue().equals("dods_error"))) {
        // create server exception object
        DAP2Exception ds = new DAP2Exception();
        is = dumpStream(is);
        // parse the Error object from stream and throw it
        ds.parse(is);
        throw ds;
      }

      ver = new ServerVersion(method);

      checkHeaders(method);

      // check for deflator
      Header h = method.getResponseHeader("content-encoding");
      String encoding = (h == null) ? null : h.getValue();
      // if (encoding != null) System.out.println("encoding= " + encoding);

      if (encoding != null && encoding.equals("deflate")) {
        is = new BufferedInputStream(new InflaterInputStream(is), 1000);

      } else if (encoding != null && encoding.equals("gzip")) {
        is = new BufferedInputStream(new GZIPInputStream(is), 1000);
      }

      command.process(is);

    } catch (Exception e) {
      e.printStackTrace();
      throw new DAP2Exception(e);

    } finally {
      // Release the connection.
      if (method != null) method.close();
    }
  }
  public void doPost(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    try {
      address = PAConfiguration.getAddress();
      port = PAConfiguration.getPort(instantiation);
      poslAddress = PAConfiguration.getPOSL(instantiation, topic);
      rdfAddress = PAConfiguration.getRDFTaxonomy(instantiation);
      messageEndpoint = PAConfiguration.getEndpointName(instantiation, topic);
    } catch (BadConfigurationException e) {
      System.out.println(e.getMessage());
      e.printStackTrace();
      System.exit(0);
    }
    response.setContentType("text/html; charset=UTF-8");
    PrintWriter out = response.getWriter();

    try {
      System.out.println("5 Publicty Chair Servlet");
      System.out.println(response.toString());

      BufferedReader brd = request.getReader();

      String input = "";
      String message = "";

      while (!input.equals("</RuleML>")) {

        input = brd.readLine();

        message = message + input;
      }
      String[] varOrder = getVariableOrder(message);
      System.out.println("Received Message: " + message);

      //	BackwardReasoner br = new BackwardReasoner();
      //   Iterator solit =null;
      //   DefiniteClause dc = null;
      //   SymbolTable.reset();

      POSLParser pp = new POSLParser();
      // String contents = "c(a).\nc(b).\nc(c).";

      Date t1 = new GregorianCalendar().getTime();
      System.out.println(t1.getHours() + ":" + t1.getMinutes());
      // append time to contents

      System.out.println("day: " + t1.getDay());
      System.out.println("day: " + t1.getYear());
      System.out.println("day: " + t1.getMonth());

      // time
      String time = "time(" + t1.getHours() + ":integer).";
      System.out.println(time);

      String url = poslAddress;

      // String url = "http://www.jdrew.org/oojdrew/test.posl";
      String contents = "";

      // day of the week
      int day = t1.getDay();
      boolean weekday = true;

      if (day == 0 || day == 6) {
        weekday = false;
      }

      String dayOfWeek;

      if (weekday) {
        dayOfWeek = "day(weekday).";
      } else {
        dayOfWeek = "day(weekend).";
      }
      // full date
      Calendar cal = new GregorianCalendar();

      int year = cal.get(Calendar.YEAR);
      int month = cal.get(Calendar.MONTH) + 1;
      int day2 = cal.get(Calendar.DAY_OF_MONTH);

      String date;

      String day3 = "" + day2;

      if (day2 == 1 || day2 == 2 || day2 == 3 || day2 == 4 || day2 == 5 || day2 == 6 || day2 == 7
          || day2 == 8 || day2 == 9) {

        day3 = "0" + day2;
      }

      if (month == 10 || month == 11 || month == 12) date = "" + year + month + day3;
      else date = "" + year + "0" + month + day3;

      date = "date(" + date + ":integer).";

      System.out.println(date);

      String url2 = rdfAddress;
      HttpClient client2 = new HttpClient();
      GetMethod method2 = new GetMethod(url2);
      method2.setFollowRedirects(true);
      String typestr = "";
      // Execute the GET method
      int statusCode2 = client2.executeMethod(method2);
      if (statusCode2 != -1) {
        typestr = method2.getResponseBodyAsString();
      }
      System.out.println("Types: " + typestr);
      Types.reset();
      RDFSParser.parseRDFSString(typestr);

      try {
        HttpClient client = new HttpClient();
        GetMethod method = new GetMethod(url);
        method.setFollowRedirects(true);

        // Execute the GET method
        int statusCode = client.executeMethod(method);
        if (statusCode != -1) {
          contents = method.getResponseBodyAsString();
        }
      } catch (Exception e) {
        e.printStackTrace();
      }
      contents = contents + "\n" + time;
      contents = contents + "\n" + dayOfWeek;
      contents = contents + "\n" + date;

      BackwardReasoner br = new BackwardReasoner();
      Iterator solit = null;
      DefiniteClause dc = null;
      SymbolTable.reset();

      pp.parseDefiniteClauses(contents);

      br.loadClauses(pp.iterator());
      System.out.println("TEST");
      Iterator it = pp.iterator();
      while (it.hasNext()) {
        DefiniteClause d = (DefiniteClause) it.next();
        System.out.println("Loaded clause: " + d.toPOSLString());
      }

      br = new BackwardReasoner(br.clauses, br.oids);

      MessageParser m = new MessageParser(message);
      Element atom = null;

      try {

        atom = m.parseForContent();

      } catch (Exception e) {

        System.out.println("Invalid Message");
        // out.flush();

      }

      QueryBuilder q = new QueryBuilder(atom);
      String query = q.generateDoc();
      System.out.println("ABOUT TO INPUT THIS QUERY:" + query);
      RuleMLParser qp = new RuleMLParser();

      try {

        dc = qp.parseRuleMLQuery(query);

      } catch (Exception e) {
        System.out.println("Invalid Query");
        // out.flush();
      }

      // solit = br.iterativeDepthFirstSolutionIterator(dc);

      solit = br.iterativeDepthFirstSolutionIterator(dc);

      int varSize = 0;

      while (solit.hasNext()) {

        Vector data = new Vector();

        BackwardReasoner.GoalList gl = (BackwardReasoner.GoalList) solit.next();

        Hashtable varbind = gl.varBindings;
        javax.swing.tree.DefaultMutableTreeNode root = br.toTree();
        root.setAllowsChildren(true);

        javax.swing.tree.DefaultTreeModel dtm = new DefaultTreeModel(root);

        int i = 0;
        Object[][] rowdata = new Object[varbind.size()][2];
        varSize = varbind.size();

        Enumeration e = varbind.keys();
        while (e.hasMoreElements()) {
          Object k = e.nextElement();
          Object val = varbind.get(k);
          String ks = (String) k;
          rowdata[i][0] = ks;
          rowdata[i][1] = val;
          i++;
        }

        data.addElement(rowdata);
        String[] messages = new String[data.size()];
        MessageGenerator g =
            new MessageGenerator(
                data, varSize, messageEndpoint, m.getId(), m.getProtocol(), m.getRel(), varOrder);
        messages = g.Messages2();

        String appender = "";

        URL sender = new URL(address + ":" + port);
        HttpMessage msg = new HttpMessage(sender);
        Properties props = new Properties();

        for (int i1 = 0; i1 < data.size(); i1++) {
          System.out.println(i1 + ")" + messages[i1].toString());
          props.put("text", messages[i1].toString());
          InputStream in = msg.sendGetMessage(props);
        }
        System.out.println("NEXT MESSAGE");
      }

      MessageGenerator g =
          new MessageGenerator(
              null, varSize, messageEndpoint, m.getId(), m.getProtocol(), m.getRel());

      URL sender = new URL(address + ":" + port);
      HttpMessage msg = new HttpMessage(sender);
      Properties props = new Properties();

      String finalMessage = g.finalMessage(query);

      System.out.println(finalMessage);

      props.put("text", finalMessage);
      InputStream in = msg.sendGetMessage(props);

      System.out.println("Stop_Communication");

    } catch (Exception e) {
      System.out.println("ERROR has occured : " + e.toString());
    }
    out.close();
  }