/**
  * @param ticker The ticker of the stock to look up.
  * @throws ConnectionException if it has problems connecting to Yahoo.
  * @return The value of the given stock, as %CompanyName: %Value.
  */
 public String getStockValue(String ticker) throws ConnectionException {
   try {
     final HtmlPage lookup =
         client.getPage(
             new StringBuilder("http://www.google.com/finance?q=").append(ticker).toString());
     final DomElement pricediv =
         lookup.getFirstByXPath("//div[@id = 'price-panel']/div/span/span");
     final DomElement compname =
         lookup.getFirstByXPath("//div[@class = 'appbar-snippet-primary']/span");
     return new StringBuilder(compname.getTextContent())
         .append(": ")
         .append(pricediv.getTextContent())
         .toString();
   } catch (IOException | FailingHttpStatusCodeException e) {
     System.out.println(
         new StringBuilder("Exception in retrieval.")
             .append("InternetConnection.getStockValue(")
             .append(ticker)
             .append(")")
             .toString());
     System.out.println(e.toString());
   }
   throw new ConnectionException(
       new StringBuilder("Unable to find data for ").append(ticker).toString());
 }
  /**
   * Test evaluation of paths after they're changed through JavaScript.
   *
   * @throws Exception if test fails
   */
  @Test
  public void whenJSChangesPage() throws Exception {
    final String content =
        "<html><head><title>foo</title><script>\n"
            + "function addOption() {\n"
            + "    var options = document.form1.select1.options;\n"
            + "    var index = options.length;\n"
            + "    options[index] = new Option('Four','value4');\n"
            + "}</script>\n"
            + "</head>\n"
            + "<body>\n"
            + "<p>hello world</p>\n"
            + "<form name='form1'>\n"
            + "    <select name='select1'>\n"
            + "        <option name='option1' value='value1'>One</option>\n"
            + "        <option name='option2' value='value2' selected>Two</option>\n"
            + "        <option name='option3' value='value3'>Three</option>\n"
            + "    </select>\n"
            + "</form>\n"
            + "<a href='javascript:addOption()'>add option</a>\n"
            + "</body></html>";

    final HtmlPage page = loadPage(content);
    assertEquals("foo", page.getTitleText());

    assertEquals(
        3, page.<Number>getFirstByXPath("count(//select[@name='select1']/option)").intValue());

    page.getAnchors().get(0).click();
    assertEquals(
        4, page.<Number>getFirstByXPath("count(//select[@name='select1']/option)").intValue());
  }
  @Test
  public void testConfigurationAuthentication() throws Exception {
    prepareSecurity();

    FreeStyleProject p = j.createFreeStyleProject();

    WebClient wc = j.createWebClient();
    wc.login("test1");

    // Reauthentication is not required if No need for re-authentication is checked
    p.addProperty(
        new AuthorizeProjectProperty(new SpecificUsersAuthorizationStrategy("admin", true)));
    j.submit(wc.getPage(p, "configure").getFormByName("config"));

    // Reauthentication is required if No need for re-authentication is checked
    p.removeProperty(AuthorizeProjectProperty.class);
    p.addProperty(
        new AuthorizeProjectProperty(new SpecificUsersAuthorizationStrategy("admin", false)));
    try {
      j.submit(wc.getPage(p, "configure").getFormByName("config"));
      fail();
    } catch (FailingHttpStatusCodeException e) {
      assertEquals(400, e.getStatusCode());
    }

    // No authentication is required if oneself.
    {
      HtmlPage page = wc.getPage(p, "configure");
      HtmlTextInput userid =
          page.<HtmlTextInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'userid') and @type='text']");
      userid.setValueAttribute("test1");
      j.submit(page.getFormByName("config"));

      assertEquals(
          "test1",
          ((SpecificUsersAuthorizationStrategy)
                  p.getProperty(AuthorizeProjectProperty.class).getStrategy())
              .getUserid());
    }

    // Reauthentication is required to change userid even if No need for re-authentication is
    // checked
    p.addProperty(
        new AuthorizeProjectProperty(new SpecificUsersAuthorizationStrategy("admin", true)));
    {
      HtmlPage page = wc.getPage(p, "configure");
      HtmlTextInput userid =
          page.<HtmlTextInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'userid') and @type='text']");
      userid.setValueAttribute("test2");
      try {
        j.submit(page.getFormByName("config"));
        fail();
      } catch (FailingHttpStatusCodeException e) {
        assertEquals(400, e.getStatusCode());
      }
    }
  }
  /**
   * Test evaluation of some simple paths.
   *
   * @throws Exception if test fails
   */
  @Test
  public void simplePath() throws Exception {
    final String content =
        "<html><head><title>Test page</title></head>\n"
            + "<body><a href='foo.html' id='myLink'>foo</a></body>\n"
            + "</html>";

    final HtmlPage page = loadPage(content);
    assertEquals(page.getDocumentElement(), page.getFirstByXPath("/html"));
    assertEquals(page.getDocumentElement().getFirstChild(), page.getFirstByXPath("/html/head"));
    assertEquals(page.getHtmlElementById("myLink"), page.getFirstByXPath("/html/body/a"));
    assertEquals(
        "Test page", ((DomText) page.getFirstByXPath("/html/head/title/text()")).getNodeValue());
  }
  /** @throws Exception if test fails */
  @Test
  public void id() throws Exception {
    final String content =
        "<html><head><title>foo</title></head>\n"
            + "<body>\n"
            + "<div><a href='link.html' id='test'></div>\n"
            + "</body></html>";

    final HtmlPage page = loadPage(content);

    assertNull(page.getFirstByXPath("//div[@id='doesNotExist']"));

    assertNull(page.getFirstByXPath("id('doesNotExist')"));
  }
示例#6
0
  public static void main(String[] args) {
    WebClient client = new WebClient(BrowserVersion.CHROME);
    HtmlPage page = null;

    try {
      page = client.getPage("http://www.facebook.com");

      System.out.println(page.getTitleText());

      HtmlForm form = (HtmlForm) page.getElementById("login_form");
      form.getInputByName("email").setValueAttribute("*****@*****.**");
      form.getInputByName("pass").setValueAttribute("saburtalo16");
      page = form.getInputByValue("Log In").click();
      System.out.println(page.getTitleText());

      HtmlTextArea statusText = (HtmlTextArea) page.getElementByName("xhpc_message_text");
      statusText.click();
      statusText.setText("I'm a robot");
      HtmlButton post =
          (HtmlButton)
              page.getFirstByXPath("//button[@id=\"u_jsonp_3_4\"]/div/div[4]/div/ul/li[2]/button");
      post.click();

    } catch (Exception e) {
      e
          .printStackTrace(); // To change body of catch statement use File | Settings | File
                              // Templates.
    }
  }
示例#7
0
    @Test public void privateView() throws Exception {
        j.createFreeStyleProject("project1");
        User user = User.get("me", true); // create user

        WebClient wc = j.createWebClient();
        HtmlPage userPage = wc.goTo("user/me");
        HtmlAnchor privateViewsLink = userPage.getFirstAnchorByText("My Views");
        assertNotNull("My Views link not available", privateViewsLink);

        HtmlPage privateViewsPage = (HtmlPage) privateViewsLink.click();

        Text viewLabel = (Text) privateViewsPage.getFirstByXPath("//div[@class='tabBar']//div[@class='tab active']/a/text()");
        assertTrue("'All' view should be selected", viewLabel.getTextContent().contains(Hudson_ViewName()));

        View listView = listView("listView");

        HtmlPage newViewPage = wc.goTo("user/me/my-views/newView");
        HtmlForm form = newViewPage.getFormByName("createItem");
        form.getInputByName("name").setValueAttribute("proxy-view");
        ((HtmlRadioButtonInput) form.getInputByValue("hudson.model.ProxyView")).setChecked(true);
        HtmlPage proxyViewConfigurePage = j.submit(form);
        View proxyView = user.getProperty(MyViewsProperty.class).getView("proxy-view");
        assertNotNull(proxyView);
        form = proxyViewConfigurePage.getFormByName("viewConfig");
        form.getSelectByName("proxiedViewName").setSelectedAttribute("listView", true);
        j.submit(form);

        assertTrue(proxyView instanceof ProxyView);
        assertEquals(((ProxyView) proxyView).getProxiedViewName(), "listView");
        assertEquals(((ProxyView) proxyView).getProxiedView(), listView);
    }
示例#8
0
  public static void main(String[] args) throws Exception {

    //    	WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24, "54.186.230.121", 3128);
    WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);

    webClient.getOptions().setThrowExceptionOnScriptError(false);
    webClient.setJavaScriptTimeout(10000);
    webClient.getOptions().setJavaScriptEnabled(true);
    webClient.setAjaxController(new NicelyResynchronizingAjaxController());
    webClient.getOptions().setTimeout(10000);

    //    	webClient.getOptions().setJavaScriptEnabled(false);
    //		webClient.getOptions().setAppletEnabled(false);
    //		webClient.getOptions().setCssEnabled(false);

    //		webClient.getOptions().setThrowExceptionOnScriptError(false);
    //		webClient.setJavaScriptTimeout(10000);
    //		webClient.getOptions().setJavaScriptEnabled(true);
    //		webClient.setAjaxController(new NicelyResynchronizingAjaxController());
    //		webClient.getOptions().setTimeout(10000);

    //		webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    //		webClient.getOptions().setThrowExceptionOnScriptError(false);

    HtmlPage currentPage = webClient.getPage("http://www.yandex.ru/");
    //    	HtmlPage currentPage = webClient.getPage("http://www.google.ru");

    //    	HtmlDivision div = currentPage.getHtmlElementById("del_competitors-1_42");

    //    	HtmlElement clickable = (HtmlElement)
    // currentPage.getHtmlElementById("del_competitors-1_42");
    //    	currentPage = (HtmlPage) clickable.click();

    HtmlAnchor advancedSearchAn = currentPage.getAnchorByText("Завести ящик");
    currentPage = advancedSearchAn.click();

    HtmlImage image = currentPage.<HtmlImage>getFirstByXPath("//img[@src='images/ash2008.jpg']");
    currentPage = (HtmlPage) image.click();

    System.out.println(currentPage.asXml());

    //    	HtmlImage image =
    // currentPage.<HtmlImage>getFirstByXPath("//img[@src='images/ash2008.jpg']");
    //    	currentPage = (HtmlPage) image.click();

    //    	HtmlImage imagetosave =
    // currentPage.<HtmlImage>getFirstByXPath("//img[@src='//yastatic.net/www/1.977/yaru/i/logo.png']");

    //    	HtmlImage image = currentPage.<HtmlImage>getHtmlElementById("add_competitors-1_3");
    //    	currentPage = (HtmlPage) image.click();

    //    	File imageFile = new File("test_new.jpg");
    //    	image.saveAs(imageFile);

    //    	System.out.println(currentPage.asXml());

    System.out.println("It is done.");

    webClient.closeAllWindows();
  }
  /** @throws Exception if test fails */
  @Test
  public void changingAttributes() throws Exception {
    final String content =
        "<html><head><title>foo</title></head>\n"
            + "<body>\n"
            + "<div id='testDiv' title='foo'></div>\n"
            + "</body></html>";

    final HtmlPage page = loadPage(content);
    final HtmlDivision div = page.getHtmlElementById("testDiv");

    Assert.assertSame(div, page.getFirstByXPath("//*[@title = 'foo']"));
    assertNull(page.getFirstByXPath("//*[@class = 'design']"));

    div.setAttribute("class", "design");
    Assert.assertSame(div, page.getFirstByXPath("//*[@class = 'design']"));
  }
示例#10
0
 public ClickStreamType getLoginClickStream(HtmlPage page) {
   List<RedirectType> redirectedUrls = tasksDef.getLoginInfo().getRedirectedURL();
   for (RedirectType rt : redirectedUrls) {
     if (rt.getLanding().getFromType() == VarType.XPATH) {
       if (page.getFirstByXPath(rt.getLanding().getValue()) != null) {
         return clickStreamMap.get(rt.getClickstream());
       }
     }
   }
   return null;
 }
  /**
   * Test evaluation relative from elements other than the whole page.
   *
   * @throws Exception if test fails
   */
  @Test
  public void xpathFromElement() throws Exception {
    final String content =
        "<html><head><title>Test page</title></head>\n"
            + "<body><a href='foo.html' id='myLink'>foo</a></body>\n"
            + "</html>";

    final HtmlPage page = loadPage(content);
    final HtmlBody body = page.getFirstByXPath("/html/body");

    assertEquals(page.getHtmlElementById("myLink"), body.getFirstByXPath("./a"));
  }
示例#12
0
  /**
   * Static method for <code>Login</code>. Uses dependencies <b>HttpUnit</b> in connecting to Keats.
   *
   * @see com.gargoylesoftware.htmlunit
   * @param parent The parent window. instanceof<code>Scrape</code>, to set relative locations to
   * @param link The url of which to retrieve information from.
   * @param username The username to log in KEATS with.
   * @param password The password to log in KEATS with.
   * @return results.asText() Returns the content of the url if login successful. Returns null
   *     otherwise.
   */
  public static String login(Scrape parent, String link, String username, String password) {
    try {
      WebClient client = new WebClient();

      // Settings
      client.getOptions().setThrowExceptionOnScriptError(false);
      client.getOptions().setThrowExceptionOnScriptError(false);
      client.getOptions().setThrowExceptionOnFailingStatusCode(false);
      client.getOptions().setJavaScriptEnabled(false);
      client.getOptions().setCssEnabled(false);
      client.getOptions().setRedirectEnabled(true);
      client.getOptions().setUseInsecureSSL(true);
      client.getCookieManager().setCookiesEnabled(true);

      HtmlPage page = client.getPage("https://login-keats.kcl.ac.uk/");

      HtmlForm form =
          page.getFirstByXPath("//form[@action='https://keats.kcl.ac.uk/login/index.php']");

      HtmlInput usernameInput = form.getInputByName("username");
      usernameInput.setValueAttribute(username);
      HtmlInput passwordInput = form.getInputByName("password");
      passwordInput.setValueAttribute(password);

      page = form.getInputByValue("Log in").click();

      HtmlPage results = client.getPage(link);

      client.closeAllWindows();
      return results.asText();
    } catch (MalformedURLException e) {
      JOptionPane.showMessageDialog(
          parent,
          "The URL you have provided is not recognised. Please double check your "
              + "input and try again.",
          "MalformedURLException found.",
          JOptionPane.ERROR_MESSAGE);
      e.printStackTrace();
    } catch (IOException e) {
      JOptionPane.showMessageDialog(
          parent,
          "An error has occurred when attempting to read information from the "
              + "server. The input could be interrupted by external processes.",
          "IOException found.",
          JOptionPane.ERROR_MESSAGE);
      e.printStackTrace();
    }

    return null;
  }
  /**
   * Test if option/text() is cleaned like other text().
   *
   * @throws Exception if test fails
   */
  @Test
  public void optionText_getFirstByXPath() throws Exception {
    final String content =
        "<html><head><title>Test page</title></head>\n"
            + "<body><form name='foo'>\n"
            + "<select name='test'><option value='1'>foo&nbsp;and&nbsp;foo</option></select>\n"
            + "</form></body></html>";

    final HtmlPage page = loadPage(content);
    final String value = page.getFirstByXPath("string(//option)");
    final int[] expectedValues = {102, 111, 111, 160, 97, 110, 100, 160, 102, 111, 111};
    int index = 0;
    for (final int v : expectedValues) {
      if (value.codePointAt(index++) != v) {
        fail();
      }
    }
  }
示例#14
0
 @Override
 public boolean verifySuccess(HtmlPage page, Object param) {
   // no success defined, means always pass
   if (xpathsSuccess == null) {
     return true;
   }
   for (int i = 0; i < xpathsSuccess.length; i++) {
     if (xpathsSuccess[i] != null) {
       Object result = page.getFirstByXPath(xpathsSuccess[i]);
       if (result == null) {
         logger.warn(
             String.format(
                 "xpath:%s not found on page %s",
                 xpathsSuccess[i], page.getUrl().toExternalForm()));
         ;
         return false;
       } else {
         String strResult = CrawlTaskEval.getStringValue(result);
         if (expectedValues != null) {
           if (strResult != null) {
             if (!expectedValues[i].equals(strResult)) {
               logger.warn(
                   String.format(
                       "result get from xpath %s on page %s is [%s] different then expected [%s]",
                       xpathsSuccess[i],
                       page.getUrl().toExternalForm(),
                       strResult,
                       expectedValues[i]));
               return false;
             }
           } else {
             logger.warn(String.format("xpath result:%s can't be transformed to string.", result));
             return false;
           }
         }
       }
     }
   }
   return true;
 }
  @Test
  public void testConfigureApitoken() throws Exception {
    prepareSecurity();

    FreeStyleProject p = j.createFreeStyleProject();
    p.addProperty(
        new AuthorizeProjectProperty(new SpecificUsersAuthorizationStrategy("test2", false)));

    WebClient wc = j.createWebClient();
    wc.login("test1");

    String apitokenForTest2 = User.get("test2").getProperty(ApiTokenProperty.class).getApiToken();
    assertNotNull(apitokenForTest2);
    assertNotEquals("", apitokenForTest2);

    // authentication fails without apitoken
    {
      HtmlPage page = wc.getPage(p, "configure");
      HtmlCheckBoxInput useApitoken =
          page.<HtmlCheckBoxInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'useApitoken') and @type='checkbox']");
      useApitoken.setChecked(true);
      try {
        j.submit(page.getFormByName("config"));
        fail();
      } catch (FailingHttpStatusCodeException e) {
        assertEquals(400, e.getStatusCode());
      }
    }

    // authentication succeeds with the good apitoken
    {
      HtmlPage page = wc.getPage(p, "configure");
      HtmlCheckBoxInput useApitoken =
          page.<HtmlCheckBoxInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'useApitoken') and @type='checkbox']");
      useApitoken.setChecked(true);
      HtmlTextInput apitoken =
          page.<HtmlTextInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'apitoken') and @type='text']");
      apitoken.setValueAttribute(apitokenForTest2);
      j.submit(page.getFormByName("config"));

      assertEquals(
          "test2",
          ((SpecificUsersAuthorizationStrategy)
                  p.getProperty(AuthorizeProjectProperty.class).getStrategy())
              .getUserid());
    }

    // authentication fails with a bad apitoken
    {
      HtmlPage page = wc.getPage(p, "configure");
      HtmlCheckBoxInput useApitoken =
          page.<HtmlCheckBoxInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'useApitoken') and @type='checkbox']");
      useApitoken.setChecked(true);
      HtmlTextInput apitoken =
          page.<HtmlTextInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'apitoken') and @type='text']");
      apitoken.setValueAttribute(apitokenForTest2 + "xxx");
      try {
        j.submit(page.getFormByName("config"));
        fail();
      } catch (FailingHttpStatusCodeException e) {
        assertEquals(400, e.getStatusCode());
      }
    }

    // authentication fails if the apitoken is used for password
    {
      HtmlPage page = wc.getPage(p, "configure");
      HtmlCheckBoxInput useApitoken =
          page.<HtmlCheckBoxInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'useApitoken') and @type='checkbox']");
      useApitoken.setChecked(false);
      HtmlPasswordInput password =
          page.<HtmlPasswordInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'password') and @type='password']");
      password.setValueAttribute(apitokenForTest2);
      HtmlTextInput apitoken =
          page.<HtmlTextInput>getFirstByXPath(
              "//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'apitoken') and @type='text']");
      apitoken.setValueAttribute(apitokenForTest2);
      try {
        j.submit(page.getFormByName("config"));
        fail();
      } catch (FailingHttpStatusCodeException e) {
        assertEquals(400, e.getStatusCode());
      }
    }
  }