Example #1
0
 /**
  * <span style="color:red">POTENIAL PERFORMANCE KILLER - DOWNLOADS THE RESOURCE - USE AT YOUR OWN
  * RISK.</span><br>
  * If the linked content is not already downloaded it triggers a download. Then it stores the
  * response for later use.<br>
  *
  * @param downloadIfNeeded indicates if a request should be performed this hasn't been done
  *     previously
  * @return <code>null</code> if no download should be performed and when this wasn't already done;
  *     the response received when performing a request for the content referenced by this tag
  *     otherwise
  * @throws IOException if an error occurs while downloading the content
  */
 public WebResponse getWebResponse(final boolean downloadIfNeeded) throws IOException {
   if (downloadIfNeeded && cachedWebResponse_ == null) {
     final WebClient webclient = getPage().getWebClient();
     cachedWebResponse_ = webclient.loadWebResponse(getWebRequest());
   }
   return cachedWebResponse_;
 }
Example #2
0
  public void start() throws Exception {

    // TODO 不下载非本站的静态资源与链接
    // TODO 不继续抓取已经爬过的页面
    // TODO 9skb链接写法的下载

    if (!siteSaveFolder.endsWith(File.separator)) {
      siteSaveFolder += File.separator;
    }

    File saveFolder = new File(siteSaveFolder);
    if (!saveFolder.exists()) {
      saveFolder.mkdirs();
    }

    staticModels.clear();

    client = new WebClient();
    // 设置css activex applet 失效
    client.setActiveXNative(false);
    client.setAppletEnabled(false);
    client.setCssEnabled(false);
    client.setJavaScriptEnabled(false);
    // 10秒超时
    client.setTimeout(10 * 1000);

    fetchPage(siteUrl, "index.html");

    System.out.println("----------------------------------------------------------------------");

    System.out.println("[ 正在下载静态资源文件... ]");
    //		fetchStaticResources();
  }
Example #3
0
  public static final WebClient newWebClient(CookieManager cookieManager) {
    // Create and initialize WebClient object
    WebClient webClient = newWebClient();
    webClient.setCookieManager(cookieManager);

    return webClient;
  }
Example #4
0
  @Test
  @Ignore
  public void test3() throws Exception {
    System.out.println("-------------------------------");

    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setCssEnabled(false);
    webClient.getOptions().setJavaScriptEnabled(false);
    HtmlPage page =
        webClient.getPage("http://www.zjnu.edu.cn/news/common/article_show.aspx?article_id=19285");

    System.out.println("---------------标题----------------");
    HtmlSpan span1 = (HtmlSpan) page.getElementById("mytitle");
    System.out.println(span1.asText());
    System.out.println("-------------------------------");

    System.out.println("---------------正文----------------");
    HtmlSpan span2 = (HtmlSpan) page.getElementById("mycontent");
    System.out.println(span2.asText());
    System.out.println("-------------------------------");

    System.out.println("---------------图片----------------");
    DomNodeList<HtmlElement> elements = span2.getElementsByTagName("img");
    for (HtmlElement element : elements) {
      System.out.println(element.getAttribute("src"));
    }
    // log.debug("{}", elements);
    System.out.println("-------------------------------");

    webClient.close();
    System.out.println("-------------------------------");
  }
Example #5
0
 @Test
 public void testResponse() throws Exception {
   WebClient webClient = new WebClient();
   TextPage page = webClient.getPage(url);
   assertEquals("Pong", page.getContent());
   webClient.closeAllWindows();
 }
  @Test
  public void testExceptionPhaseListener() throws Exception {
    WebClient client = new WebClient();
    client.setThrowExceptionOnFailingStatusCode(false);

    // First, try a transient conversation

    // Access a page that throws an exception
    client.getPage(getPath("/thunderstorm.jsf"));

    // Then access another page that doesn't and check the contexts are ok
    HtmlPage cloud = client.getPage(getPath("/cloud.jsf"));
    String cloudName = getFirstMatchingElement(cloud, HtmlSpan.class, "cloudName").getTextContent();
    assertEquals(Cloud.NAME, cloudName);

    // Now start a conversation and access the page that throws an exception again
    HtmlPage thunderstorm =
        getFirstMatchingElement(cloud, HtmlSubmitInput.class, "beginConversation").click();
    // This page will error

    String cid = getCid(thunderstorm);

    cloud = client.getPage(getPath("/cloud.jsf", cid));

    // And navigate to another page, checking the conversation exists by verifying that state is
    // maintained
    cloudName = getFirstMatchingElement(cloud, HtmlSpan.class, "cloudName").getTextContent();
    assertEquals("gavin", cloudName);
  }
Example #7
0
  public boolean athenticated() {

    web = new WebClient();
    try {
      page1 = web.getPage(url);
    } catch (FailingHttpStatusCodeException | IOException e) {
      e.printStackTrace();
    }
    HtmlForm form = page1.getFormByName("form2");

    form.getInputByName("txtUserName").setAttribute("value", username);
    form.getInputByName("txtPassword").setAttribute("value", password);
    try {
      page2 = form.getInputByName("LoginButton").click();
    } catch (ElementNotFoundException e) {
      e.printStackTrace();
    } catch (IOException e) {
      e.printStackTrace();
    }

    if (page2.getUrl().toString().equals("http://78.39.200.210/Default.aspx")) {
      return true;
    } else {
      web.closeAllWindows();
      return false;
    }
  }
Example #8
0
 /** Setup before testing. */
 @Before
 public void setUp() {
   webUrl = System.getProperty("integration.url");
   webClient = new WebClient();
   webClient.setJavaScriptEnabled(true);
   webClient.setJavaScriptTimeout(60000);
 }
Example #9
0
 public static HtmlPage getPage(String url)
     throws FailingHttpStatusCodeException, MalformedURLException, IOException {
   WebClient wc = new WebClient(BrowserVersion.CHROME);
   wc.getOptions().setCssEnabled(false);
   HtmlPage page = wc.getPage(url);
   return page;
 }
Example #10
0
  public static ArrayList<String> trans(String url)
      throws FailingHttpStatusCodeException, MalformedURLException, IOException {

    ArrayList<String> hrefList = new ArrayList<String>();
    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setJavaScriptEnabled(false);
    webClient.getOptions().setCssEnabled(false);
    try {
      HtmlPage page = null;
      try {
        page = (HtmlPage) webClient.getPage(url);
      } catch (ConnectException e) {
        System.out.println("Connect fails here:" + e.getMessage());
      }
      InputStream temp = new ByteArrayInputStream(page.asText().getBytes());
      InputStreamReader isr = new InputStreamReader(temp);
      BufferedReader br = new BufferedReader(isr);
      String str = null, rs = null;
      while ((str = br.readLine()) != null) {
        rs = str;
        // System.out.println(rs);
        if (rs != null) hrefList.add(rs);
      }
      System.out.println("从该网址查找的可能相关文本如下:");
      for (int i = 0; i < hrefList.size(); i++) {
        String string = hrefList.get(i);
        string = getTextFromHtml(string);
        if (string.length() >= 30) System.out.println("------" + i + ":" + string);
      }
    } catch (IOException e) {
    }
    return hrefList;
  }
  public static WebClient buildWebClient(CookieManager manager) {
    WebClient webClient = buildWebClient();

    webClient.setCookieManager(manager);

    return webClient;
  }
Example #12
0
  public static void main(String[] args) {
    WebClient client = new WebClient(BrowserVersion.CHROME);
    HtmlPage page = null;

    try {
      page = client.getPage("http://www.facebook.com");

      System.out.println(page.getTitleText());

      HtmlForm form = (HtmlForm) page.getElementById("login_form");
      form.getInputByName("email").setValueAttribute("*****@*****.**");
      form.getInputByName("pass").setValueAttribute("saburtalo16");
      page = form.getInputByValue("Log In").click();
      System.out.println(page.getTitleText());

      HtmlTextArea statusText = (HtmlTextArea) page.getElementByName("xhpc_message_text");
      statusText.click();
      statusText.setText("I'm a robot");
      HtmlButton post =
          (HtmlButton)
              page.getFirstByXPath("//button[@id=\"u_jsonp_3_4\"]/div/div[4]/div/ul/li[2]/button");
      post.click();

    } catch (Exception e) {
      e
          .printStackTrace(); // To change body of catch statement use File | Settings | File
                              // Templates.
    }
  }
 @Override
 protected SkypeGuestTempSession run(WebClient webClient) throws Exception {
   webClient.getPage(joinUrl);
   String csrf = webClient.getCookieManager().getCookie("csrf_token").getValue();
   String sessionId = webClient.getCookieManager().getCookie("launcher_session_id").getValue();
   return new SkypeGuestTempSession(csrf, sessionId);
 }
Example #14
0
  /**
   * Tests asynchronous use of XMLHttpRequest, using Mozilla style object creation.
   *
   * @throws Exception if the test fails
   */
  @Test
  public void testAsyncUse() throws Exception {
    final URL secondUrl = new URL(FIRST_URL, "/second/");

    final String html =
        "<html>\n"
            + "  <head>\n"
            + "    <title>XMLHttpRequest Test</title>\n"
            + "    <script>\n"
            + "      var request;\n"
            + "      function testAsync() {\n"
            + "        if (window.XMLHttpRequest)\n"
            + "          request = new XMLHttpRequest();\n"
            + "        else if (window.ActiveXObject)\n"
            + "          request = new ActiveXObject('Microsoft.XMLHTTP');\n"
            + "        request.onreadystatechange = onReadyStateChange;\n"
            + "        alert(request.readyState);\n"
            + "        request.open('GET', '/second/', true);\n"
            + "        request.send('');\n"
            + "      }\n"
            + "      function onReadyStateChange() {\n"
            + "        alert(request.readyState);\n"
            + "        if (request.readyState == 4)\n"
            + "          alert(request.responseText);\n"
            + "      }\n"
            + "    </script>\n"
            + "  </head>\n"
            + "  <body onload='testAsync()'>\n"
            + "  </body>\n"
            + "</html>";

    final String xml =
        "<xml2>\n"
            + "<content2>sdgxsdgx</content2>\n"
            + "<content2>sdgxsdgx2</content2>\n"
            + "</xml2>";

    final WebClient client = new WebClient();
    final List<String> collectedAlerts = new ArrayList<String>();
    client.setAlertHandler(new CollectingAlertHandler(collectedAlerts));
    final MockWebConnection conn = new MockWebConnection();
    conn.setResponse(FIRST_URL, html);
    conn.setResponse(secondUrl, xml, "text/xml");
    client.setWebConnection(conn);

    client.getPage(FIRST_URL);

    final int executedJobs = client.getJavaScriptEngine().pumpEventLoop(1000);
    final String[] alerts = {
      String.valueOf(XMLHttpRequest.STATE_UNINITIALIZED),
      String.valueOf(XMLHttpRequest.STATE_LOADING),
      String.valueOf(XMLHttpRequest.STATE_LOADING),
      String.valueOf(XMLHttpRequest.STATE_LOADED),
      String.valueOf(XMLHttpRequest.STATE_INTERACTIVE),
      String.valueOf(XMLHttpRequest.STATE_COMPLETED),
      xml
    };
    assertEquals(Arrays.asList(alerts).toString(), collectedAlerts.toString());
    assertEquals(1, executedJobs);
  }
Example #15
0
 private HtmlPage getPage(String url) throws IOException {
   final WebClient webClient =
       new WebClient(
           getBrowserVersionFromName(
               searchEngine.getDefaultBrowser())); // BrowserVersion.FIREFOX_24);
   webClient.getOptions().setJavaScriptEnabled(false);
   return webClient.getPage(url);
 }
 private static List<String> getSimulatedProperties(final BrowserVersion browserVersion)
     throws Exception {
   final URL url = PropertiesTest.class.getClassLoader().getResource("objects/properties.html");
   final WebClient webClient = new WebClient(browserVersion);
   final HtmlPage page = webClient.getPage(url);
   final HtmlTextArea textarea = page.getHtmlElementById("myTextarea");
   return Arrays.asList(textarea.getText().split("\r\n|\n"));
 }
Example #17
0
 /**
  * Test method.
  *
  * @throws IOException exception
  */
 public void testLogin() throws IOException {
   final WebClient webClient = new WebClient();
   // XXX: issues with jquery
   webClient.setJavaScriptEnabled(false);
   final HtmlPage page = webClient.getPage(HTTP_PROCC_LOGIN_USER);
   assertEquals("Taskflow-Logowanie", page.getTitleText());
   webClient.closeAllWindows();
 }
 protected WebContext loadPage(String content) throws Exception {
   server = getServer();
   String path = getName() + ".html";
   server.addPage(path, content);
   WebClient webClient = new WebClient();
   String url = server.getUrl(path);
   return new WebContext(webClient.getPage(url), webClient, project);
 }
Example #19
0
 @Test
 public void testInvalidatedSession() throws Exception {
   HtmlPage page = webClient.getPage(webUrl + "faces/invalidatedSession.xhtml");
   assertTrue(page.asText().indexOf("This is from the @PostConstruct") != -1);
   webClient.getPage(webUrl + "faces/invalidatedPerform.xhtml");
   page = webClient.getPage(webUrl + "faces/invalidatedVerify.xhtml");
   assertTrue(page.asText().indexOf("true") != -1);
 }
 private void assertComponentInstancesNotEqual(String mode) throws Exception {
   WebClient webClient = new WebClient();
   webClient.setThrowExceptionOnFailingStatusCode(true);
   TextPage page = webClient.getPage(contextPath + "test?mode=" + mode);
   String[] ids = page.getContent().split(":");
   System.out.println(Arrays.toString(ids));
   assertEquals(ids.length, 2);
   assertNotEquals(ids[0], ids[1]);
 }
Example #21
0
  /**
   * geOutput
   *
   * @param pageName
   * @return String
   * @throws Exception
   */
  private String getOutput(String pageName) throws Exception {
    URL url = this.getURL(pageName);
    WebClient webClient = this.getWebClient();
    HtmlPage page = (HtmlPage) webClient.getPage(url);

    HtmlElement output = page.getHtmlElementById("output");

    return output.asText();
  }
Example #22
0
 @Test(groups = {"contexts"})
 public void testConversationPropagationToNonExistentConversationLeadsException()
     throws Exception {
   WebClient client = new WebClient();
   client.setThrowExceptionOnFailingStatusCode(false);
   Page page = client.getPage(getPath("/index.jsp"));
   assert page.getWebResponse().getStatusCode() == 200;
   assert page.getWebResponse().getRequestUrl().toString().contains("home.jsf");
 }
Example #23
0
  @Test
  @Ignore
  public void test01() throws Exception {
    System.out.println("-------------------------------");
    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    try {
      WebClientOptions options = webClient.getOptions();
      options.setThrowExceptionOnFailingStatusCode(false);
      options.setThrowExceptionOnScriptError(false);
      options.setCssEnabled(false);
      options.setJavaScriptEnabled(true);
      options.setTimeout(50000);
      // webClient.setAjaxController(new NicelyResynchronizingAjaxController());
      HtmlPage pageOrgin = webClient.getPage("http://lvyou.baidu.com/jinhua/jingdian");
      Thread.sleep(5000);
      DomNodeList<DomNode> pageNodes = pageOrgin.querySelectorAll(".pagination");
      HtmlDivision pageDiv = (HtmlDivision) pageNodes.get(0);
      DomNodeList<HtmlElement> liElements = pageDiv.getElementsByTagName("li");
      Integer pageSize = liElements.size() - 1;
      for (int pageNow = 0; pageNow < pageSize; pageNow++) {
        DomNodeList<HtmlElement> pageAnchors = liElements.get(pageNow).getElementsByTagName("a");
        HtmlAnchor pageAnchor = (HtmlAnchor) pageAnchors.get(0);
        HtmlPage page = pageAnchor.click();
        Thread.sleep(10000);
        DomElement jViewDom = page.getElementById("J-view-list-container");
        DomNodeList<HtmlElement> lis = jViewDom.getElementsByTagName("li");
        for (HtmlElement li : lis) {
          DomNodeList<DomNode> titleNodes = li.querySelectorAll(".title");
          HtmlAnchor titleAnchor = (HtmlAnchor) titleNodes.get(0);
          System.out.println("---------------标题----------------");
          log.debug("{}", titleAnchor.asText());

          DomNodeList<DomNode> picNodes = li.querySelectorAll(".pic");
          HtmlAnchor picAnchor = (HtmlAnchor) picNodes.get(0);
          System.out.println("---------------详情URL----------------");
          String detailUrl = "http://lvyou.baidu.com" + picAnchor.getAttribute("href");
          log.debug("{}", detailUrl);
          DomNodeList<HtmlElement> imgEelements = picAnchor.getElementsByTagName("img");
          for (HtmlElement imgEelement : imgEelements) {
            System.out.println("---------------图片----------------");
            log.debug("{}", imgEelement.getAttribute("src"));
          }

          DomNodeList<DomNode> sumNodes = li.querySelectorAll(".view-userSays");
          HtmlDivision sumDiv = (HtmlDivision) sumNodes.get(0);
          DomNodeList<HtmlElement> sumElements = sumDiv.getElementsByTagName("p");
          HtmlParagraph sumPara = (HtmlParagraph) sumElements.get(0);
          System.out.println("---------------摘要----------------");
          log.debug("{}", sumPara.asText());
        }
      }
    } finally {
      webClient.close();
    }
    System.out.println("-------------------------------");
  }
Example #24
0
 @JsfTest(JsfVersion.JSF_2_2_5)
 @Test
 public void testSetForEach() throws Exception {
   webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
   HtmlPage page = webClient.getPage(webUrl + "faces/setForEach.xhtml");
   assertEquals(200, page.getWebResponse().getStatusCode());
   assertTrue(page.asXml().indexOf("foo") == -1);
   assertTrue(page.asXml().indexOf("bar") != -1);
   assertTrue(page.asXml().indexOf("baz") != -1);
 }
Example #25
0
 /**
  * The request scope is active during the doFilter() method of any Filter in the web application.
  */
 @Test
 @SpecAssertions({
   @SpecAssertion(section = REQUEST_CONTEXT_EE, id = "ab"),
   @SpecAssertion(section = REQUEST_CONTEXT_EE, id = "ac")
 })
 public void testRequestScopeActiveDuringServletFilter() throws Exception {
   WebClient webClient = new WebClient();
   webClient.setThrowExceptionOnFailingStatusCode(true);
   webClient.getPage(contextPath + "SimplePage.html");
 }
Example #26
0
 /**
  * Test to validate empty fields.
  *
  * <p>Note: this test is excluded on Tomcat because the included EL parser requires a System
  * property for this test to work, which would cause problems with other tests. See
  * http://tomcat.apache.org/tomcat-7.0-doc/config/systemprops.html and look for COERCE_TO_ZERO
  */
 @Test
 public void testValidateEmptyFields() throws Exception {
   HtmlPage page = webClient.getPage(webUrl + "faces/index.xhtml");
   if (page.getWebResponse().getResponseHeaderValue("Server") == null
       || !page.getWebResponse().getResponseHeaderValue("Server").startsWith("Apache-Coyote")) {
     page = webClient.getPage(webUrl + "faces/validateEmptyFields.xhtml");
     HtmlSubmitInput button = (HtmlSubmitInput) page.getElementById("form:submitButton");
     page = button.click();
     assertTrue(page.asXml().contains("We got called!"));
   }
 }
Example #27
0
  @Test
  @Ignore
  public void test() throws Exception {
    System.out.println("-------------------------------");

    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setCssEnabled(false);
    webClient.getOptions().setJavaScriptEnabled(false);
    HtmlPage page = webClient.getPage("http://news.163.com/domestic/");
    // DomNodeList<HtmlElement> elements = page.getElementBy

    System.out.println("---------------标题----------------");
    DomNodeList<DomNode> domNodes = page.querySelectorAll(".item-top");
    // log.debug("{}", domNodes);
    for (DomNode domNode : domNodes) {
      HtmlDivision htmlDivision = (HtmlDivision) domNode;
      DomNodeList<HtmlElement> aElements = htmlDivision.getElementsByTagName("a");
      HtmlAnchor htmlAnchor = (HtmlAnchor) aElements.get(0);
      // HTMLHeadingElement htmlHeading2 = (HTMLHeadingElement)
      // htmlDivision.getElementsByTagName("h2");
      // HtmlAnchor htmlAnchor = (HtmlAnchor) htmlDivision.getElementsByTagName("a");
      log.debug("{}", htmlAnchor.asText());
      log.debug("{}", htmlAnchor.getAttribute("href"));

      DomNodeList<HtmlElement> pElements = htmlDivision.getElementsByTagName("p");
      HtmlParagraph htmlParagraph = (HtmlParagraph) pElements.get(0);
      log.debug("{}", htmlParagraph.asText());

      DomNodeList<HtmlElement> iEelements = htmlDivision.getElementsByTagName("img");
      for (HtmlElement iEelement : iEelements) {
        log.debug("{}", iEelement.getAttribute("src"));
      }

      String detailUrl = htmlAnchor.getAttribute("href");
      if (detailUrl.equals("http://news.163.com/15/1215/17/BAT2L8RB00014JB6.html#f=dlist")) {
        HtmlPage detailPage = webClient.getPage(detailUrl);
        System.out.println("---------------正文----------------");
        DomElement endTextElement = detailPage.getElementById("endText");
        log.debug("{}", endTextElement.asText());

        System.out.println("---------------图片----------------");
        DomNodeList<DomNode> imgNodes = endTextElement.querySelectorAll(".f_center");
        for (DomNode imgNode : imgNodes) {
          HtmlParagraph imgpara = (HtmlParagraph) imgNode;
          DomNodeList<HtmlElement> endImgs = imgpara.getElementsByTagName("img");
          for (HtmlElement endImg : endImgs) {
            log.debug("{}", endImg.getAttribute("src"));
          }
        }
      }
    }
    webClient.close();
    System.out.println("-------------------------------");
  }
Example #28
0
 /**
  * This test verifies that an attribute named 'readonly' can be successfully updated from a
  * partial response (over Ajax).
  */
 @Test
 public void testUpdateAttributeNamedReadonly() throws Exception {
   HtmlPage page = webClient.getPage(webUrl + "faces/attributeNameIsReadonly.xhtml");
   HtmlTextInput input = (HtmlTextInput) page.getElementById("form1:foo");
   assertTrue(input.isReadOnly() == false);
   HtmlSubmitInput button = (HtmlSubmitInput) page.getElementById("form1:button");
   page = button.click();
   webClient.waitForBackgroundJavaScript(60000);
   input = (HtmlTextInput) page.getElementById("form1:foo");
   assertTrue(input.isReadOnly() == true);
 }
Example #29
0
 @Test
 public void testLabelAjaxBehavior() throws Exception {
   webClient.setJavaScriptEnabled(true);
   webClient.setJavaScriptTimeout(60000);
   HtmlPage page = webClient.getPage(webUrl + "/faces/label.xhtml");
   HtmlElement label = page.getHtmlElementById("label2");
   page = (HtmlPage) label.mouseOver();
   webClient.waitForBackgroundJavaScript(60000);
   assertTrue(page.asXml().contains("label2 Event: begin"));
   assertTrue(page.asXml().contains("label2 Event: complete"));
   assertTrue(page.asXml().contains("label2 Event: success"));
 }
Example #30
0
 @Test
 public void testFieldSetAjaxBehavior() throws Exception {
   webClient.setJavaScriptEnabled(true);
   webClient.setJavaScriptTimeout(60000);
   HtmlPage page = webClient.getPage(webUrl + "/faces/fieldset.xhtml");
   HtmlFieldSet fieldset = (HtmlFieldSet) page.getHtmlElementById("fieldset4");
   page = fieldset.click();
   webClient.waitForBackgroundJavaScript(60000);
   assertTrue(page.asXml().contains("fieldset4 Event: begin"));
   assertTrue(page.asXml().contains("fieldset4 Event: complete"));
   assertTrue(page.asXml().contains("fieldset4 Event: success"));
 }