Beispiel #1
0
 /** Setup before testing. */
 @Before
 public void setUp() {
   webUrl = System.getProperty("integration.url");
   webClient = new WebClient();
   webClient.setJavaScriptEnabled(true);
   webClient.setJavaScriptTimeout(60000);
 }
Beispiel #2
0
  public static void main(String[] args) throws Exception {

    //    	WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24, "54.186.230.121", 3128);
    WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);

    webClient.getOptions().setThrowExceptionOnScriptError(false);
    webClient.setJavaScriptTimeout(10000);
    webClient.getOptions().setJavaScriptEnabled(true);
    webClient.setAjaxController(new NicelyResynchronizingAjaxController());
    webClient.getOptions().setTimeout(10000);

    //    	webClient.getOptions().setJavaScriptEnabled(false);
    //		webClient.getOptions().setAppletEnabled(false);
    //		webClient.getOptions().setCssEnabled(false);

    //		webClient.getOptions().setThrowExceptionOnScriptError(false);
    //		webClient.setJavaScriptTimeout(10000);
    //		webClient.getOptions().setJavaScriptEnabled(true);
    //		webClient.setAjaxController(new NicelyResynchronizingAjaxController());
    //		webClient.getOptions().setTimeout(10000);

    //		webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    //		webClient.getOptions().setThrowExceptionOnScriptError(false);

    HtmlPage currentPage = webClient.getPage("http://www.yandex.ru/");
    //    	HtmlPage currentPage = webClient.getPage("http://www.google.ru");

    //    	HtmlDivision div = currentPage.getHtmlElementById("del_competitors-1_42");

    //    	HtmlElement clickable = (HtmlElement)
    // currentPage.getHtmlElementById("del_competitors-1_42");
    //    	currentPage = (HtmlPage) clickable.click();

    HtmlAnchor advancedSearchAn = currentPage.getAnchorByText("Завести ящик");
    currentPage = advancedSearchAn.click();

    HtmlImage image = currentPage.<HtmlImage>getFirstByXPath("//img[@src='images/ash2008.jpg']");
    currentPage = (HtmlPage) image.click();

    System.out.println(currentPage.asXml());

    //    	HtmlImage image =
    // currentPage.<HtmlImage>getFirstByXPath("//img[@src='images/ash2008.jpg']");
    //    	currentPage = (HtmlPage) image.click();

    //    	HtmlImage imagetosave =
    // currentPage.<HtmlImage>getFirstByXPath("//img[@src='//yastatic.net/www/1.977/yaru/i/logo.png']");

    //    	HtmlImage image = currentPage.<HtmlImage>getHtmlElementById("add_competitors-1_3");
    //    	currentPage = (HtmlPage) image.click();

    //    	File imageFile = new File("test_new.jpg");
    //    	image.saveAs(imageFile);

    //    	System.out.println(currentPage.asXml());

    System.out.println("It is done.");

    webClient.closeAllWindows();
  }
Beispiel #3
0
 @Test
 public void testLabelAjaxBehavior() throws Exception {
   webClient.setJavaScriptEnabled(true);
   webClient.setJavaScriptTimeout(60000);
   HtmlPage page = webClient.getPage(webUrl + "/faces/label.xhtml");
   HtmlElement label = page.getHtmlElementById("label2");
   page = (HtmlPage) label.mouseOver();
   webClient.waitForBackgroundJavaScript(60000);
   assertTrue(page.asXml().contains("label2 Event: begin"));
   assertTrue(page.asXml().contains("label2 Event: complete"));
   assertTrue(page.asXml().contains("label2 Event: success"));
 }
Beispiel #4
0
 @Test
 public void testFieldSetAjaxBehavior() throws Exception {
   webClient.setJavaScriptEnabled(true);
   webClient.setJavaScriptTimeout(60000);
   HtmlPage page = webClient.getPage(webUrl + "/faces/fieldset.xhtml");
   HtmlFieldSet fieldset = (HtmlFieldSet) page.getHtmlElementById("fieldset4");
   page = fieldset.click();
   webClient.waitForBackgroundJavaScript(60000);
   assertTrue(page.asXml().contains("fieldset4 Event: begin"));
   assertTrue(page.asXml().contains("fieldset4 Event: complete"));
   assertTrue(page.asXml().contains("fieldset4 Event: success"));
 }
Beispiel #5
0
  public static void main(String[] args) throws IOException {

    WebClient client = new WebClient(BrowserVersion.CHROME);
    CookieManager cookie = new CookieManager();
    client.setCookieManager(cookie);
    client.getOptions().setJavaScriptEnabled(true);
    client.getOptions().setActiveXNative(false);
    client.getOptions().setCssEnabled(false);
    client.getOptions().setThrowExceptionOnScriptError(false);
    client.getOptions().setThrowExceptionOnFailingStatusCode(false);
    client.getOptions().setDoNotTrackEnabled(true);
    client.getOptions().setPrintContentOnFailingStatusCode(false);
    client.setAjaxController(new NicelyResynchronizingAjaxController());
    client.setJavaScriptTimeout(Long.MAX_VALUE);

    List<Integer> TopCategory = new LinkedList<Integer>(TopCategoryUrl.keySet());
    Collections.sort(TopCategory);

    long today = new Date().getTime();
    long ymd = today - 1000 * 60 * 60 * 24;
    String date = sdf.format(ymd);
    FileWriter fw = new FileWriter("ShuTaobaoTop/search-" + date, false);
    BufferedWriter bw = new BufferedWriter(fw);

    while (TopCategory.size() > 0) {
      int cid = TopCategory.get(0);
      //			if(cid<50002766)
      //			{
      //				TopCategory.remove(0);
      //				continue;
      //			}
      String cname = TopCategoryUrl.get(cid);
      String pid = "0", pname = "";
      String topUrl = "http://shu.taobao.com/top/" + cid + "/search";

      boolean success = false;
      ArrayList<String> result = new ArrayList<String>();
      try {
        //				client.getOptions().setJavaScriptEnabled(true);
        do {
          System.out.println("剩余类目数=" + TopCategory.size());
          // ProxyUnit.configProxy(client, ProxyUnit.CHECKURL);
        } while (false == AccountLogin.loginTaoBao(client, AccountLogin.TAOBAOLOGINURL, true));
        //				client.getOptions().setJavaScriptEnabled(false);

        HtmlPage queryTrade = client.getPage(topUrl);
        Thread.sleep(1000L);
        System.out.println(queryTrade.getTitleText());

        /*
         * 解析
         */
        Document doc = Jsoup.parse(queryTrade.asXml());
        Element time = doc.getElementsByAttributeValue("class", "time").get(0);
        String startTime =
            time.text().substring(6, 16).replace("-", ""); // 统计时间: 2014-05-17 - 2014-05-23
        String endTime = time.text().substring(19).replace("-", "");
        Elements elements =
            doc.getElementsByAttributeValueStarting(
                "class", "mod "); // <div class="mod odd mod-10 sm-ua"> <h3 class="title">
        for (Element element : elements) {
          String title = element.getElementsByTag("h3").get(0).text();
          System.out.println(title);
          Elements lis =
              element.getElementsByTag("ol").get(0).getElementsByTag("li"); // <li class="up ">
          for (Element li : lis) {
            Elements spans = li.getElementsByTag("span");
            String rank = spans.get(0).text();
            String key = spans.get(1).text();
            String rise = spans.get(2).text();
            rise = rise.substring(0, rise.length() - 1);

            URL href =
                new URL(
                    URLDecoder.decode(
                        "http://shu.taobao.com"
                            + spans.get(1).getElementsByTag("a").get(0).attr("href"),
                        "utf-8"));
            String hrefQuery = href.getQuery();
            if (hrefQuery.contains("cid=")) {
              if ("0".equals(pid)) {
                pid = String.valueOf(cid);
                pname = cname;
              }
              cid = Integer.parseInt(hrefQuery.split("cid=")[1].split("&")[0]);
              cname = title;
            }

            System.out.println(
                startTime + "\001" + endTime + "\001" + cid + "\001" + cname + "\001" + pid + "\001"
                    + pname + "\001" + rank + "\001" + key + "\001" + rise);
            result.add(
                startTime + "\001" + endTime + "\001" + cid + "\001" + cname + "\001" + pid + "\001"
                    + pname + "\001" + rank + "\001" + key + "\001" + rise);
          }
        }
        success = true;
      } catch (Exception e) {
        // TODO Auto-generated catch block
        // e.printStackTrace();
      }
      if (success) {
        TopCategory.remove(0);
        System.out.println("剩余类目数=" + TopCategory.size());
        for (String r : result) {
          bw.write(r + "\n");
        }
      }
    }
    bw.close();
    fw.close();
  }