/**
* @param ticker The ticker of the stock to look up.
* @throws ConnectionException if it has problems connecting to Yahoo.
* @return The value of the given stock, as %CompanyName: %Value.
*/
public String getStockValue(String ticker) throws ConnectionException {
try {
final HtmlPage lookup =
client.getPage(
new StringBuilder("http://www.google.com/finance?q=").append(ticker).toString());
final DomElement pricediv =
lookup.getFirstByXPath("//div[@id = 'price-panel']/div/span/span");
final DomElement compname =
lookup.getFirstByXPath("//div[@class = 'appbar-snippet-primary']/span");
return new StringBuilder(compname.getTextContent())
.append(": ")
.append(pricediv.getTextContent())
.toString();
} catch (IOException | FailingHttpStatusCodeException e) {
System.out.println(
new StringBuilder("Exception in retrieval.")
.append("InternetConnection.getStockValue(")
.append(ticker)
.append(")")
.toString());
System.out.println(e.toString());
}
throw new ConnectionException(
new StringBuilder("Unable to find data for ").append(ticker).toString());
}
/**
* Test evaluation of paths after they're changed through JavaScript.
*
* @throws Exception if test fails
*/
@Test
public void whenJSChangesPage() throws Exception {
final String content =
"<html><head><title>foo</title><script>\n"
+ "function addOption() {\n"
+ " var options = document.form1.select1.options;\n"
+ " var index = options.length;\n"
+ " options[index] = new Option('Four','value4');\n"
+ "}</script>\n"
+ "</head>\n"
+ "<body>\n"
+ "<p>hello world</p>\n"
+ "<form name='form1'>\n"
+ " <select name='select1'>\n"
+ " <option name='option1' value='value1'>One</option>\n"
+ " <option name='option2' value='value2' selected>Two</option>\n"
+ " <option name='option3' value='value3'>Three</option>\n"
+ " </select>\n"
+ "</form>\n"
+ "<a href='javascript:addOption()'>add option</a>\n"
+ "</body></html>";
final HtmlPage page = loadPage(content);
assertEquals("foo", page.getTitleText());
assertEquals(
3, page.<Number>getFirstByXPath("count(//select[@name='select1']/option)").intValue());
page.getAnchors().get(0).click();
assertEquals(
4, page.<Number>getFirstByXPath("count(//select[@name='select1']/option)").intValue());
}
@Test
public void testConfigurationAuthentication() throws Exception {
prepareSecurity();
FreeStyleProject p = j.createFreeStyleProject();
WebClient wc = j.createWebClient();
wc.login("test1");
// Reauthentication is not required if No need for re-authentication is checked
p.addProperty(
new AuthorizeProjectProperty(new SpecificUsersAuthorizationStrategy("admin", true)));
j.submit(wc.getPage(p, "configure").getFormByName("config"));
// Reauthentication is required if No need for re-authentication is checked
p.removeProperty(AuthorizeProjectProperty.class);
p.addProperty(
new AuthorizeProjectProperty(new SpecificUsersAuthorizationStrategy("admin", false)));
try {
j.submit(wc.getPage(p, "configure").getFormByName("config"));
fail();
} catch (FailingHttpStatusCodeException e) {
assertEquals(400, e.getStatusCode());
}
// No authentication is required if oneself.
{
HtmlPage page = wc.getPage(p, "configure");
HtmlTextInput userid =
page.<HtmlTextInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'userid') and @type='text']");
userid.setValueAttribute("test1");
j.submit(page.getFormByName("config"));
assertEquals(
"test1",
((SpecificUsersAuthorizationStrategy)
p.getProperty(AuthorizeProjectProperty.class).getStrategy())
.getUserid());
}
// Reauthentication is required to change userid even if No need for re-authentication is
// checked
p.addProperty(
new AuthorizeProjectProperty(new SpecificUsersAuthorizationStrategy("admin", true)));
{
HtmlPage page = wc.getPage(p, "configure");
HtmlTextInput userid =
page.<HtmlTextInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'userid') and @type='text']");
userid.setValueAttribute("test2");
try {
j.submit(page.getFormByName("config"));
fail();
} catch (FailingHttpStatusCodeException e) {
assertEquals(400, e.getStatusCode());
}
}
}
/**
* Test evaluation of some simple paths.
*
* @throws Exception if test fails
*/
@Test
public void simplePath() throws Exception {
final String content =
"<html><head><title>Test page</title></head>\n"
+ "<body><a href='foo.html' id='myLink'>foo</a></body>\n"
+ "</html>";
final HtmlPage page = loadPage(content);
assertEquals(page.getDocumentElement(), page.getFirstByXPath("/html"));
assertEquals(page.getDocumentElement().getFirstChild(), page.getFirstByXPath("/html/head"));
assertEquals(page.getHtmlElementById("myLink"), page.getFirstByXPath("/html/body/a"));
assertEquals(
"Test page", ((DomText) page.getFirstByXPath("/html/head/title/text()")).getNodeValue());
}
/** @throws Exception if test fails */
@Test
public void id() throws Exception {
final String content =
"<html><head><title>foo</title></head>\n"
+ "<body>\n"
+ "<div><a href='link.html' id='test'></div>\n"
+ "</body></html>";
final HtmlPage page = loadPage(content);
assertNull(page.getFirstByXPath("//div[@id='doesNotExist']"));
assertNull(page.getFirstByXPath("id('doesNotExist')"));
}
public static void main(String[] args) {
WebClient client = new WebClient(BrowserVersion.CHROME);
HtmlPage page = null;
try {
page = client.getPage("http://www.facebook.com");
System.out.println(page.getTitleText());
HtmlForm form = (HtmlForm) page.getElementById("login_form");
form.getInputByName("email").setValueAttribute("*****@*****.**");
form.getInputByName("pass").setValueAttribute("saburtalo16");
page = form.getInputByValue("Log In").click();
System.out.println(page.getTitleText());
HtmlTextArea statusText = (HtmlTextArea) page.getElementByName("xhpc_message_text");
statusText.click();
statusText.setText("I'm a robot");
HtmlButton post =
(HtmlButton)
page.getFirstByXPath("//button[@id=\"u_jsonp_3_4\"]/div/div[4]/div/ul/li[2]/button");
post.click();
} catch (Exception e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
}
}
@Test public void privateView() throws Exception {
j.createFreeStyleProject("project1");
User user = User.get("me", true); // create user
WebClient wc = j.createWebClient();
HtmlPage userPage = wc.goTo("user/me");
HtmlAnchor privateViewsLink = userPage.getFirstAnchorByText("My Views");
assertNotNull("My Views link not available", privateViewsLink);
HtmlPage privateViewsPage = (HtmlPage) privateViewsLink.click();
Text viewLabel = (Text) privateViewsPage.getFirstByXPath("//div[@class='tabBar']//div[@class='tab active']/a/text()");
assertTrue("'All' view should be selected", viewLabel.getTextContent().contains(Hudson_ViewName()));
View listView = listView("listView");
HtmlPage newViewPage = wc.goTo("user/me/my-views/newView");
HtmlForm form = newViewPage.getFormByName("createItem");
form.getInputByName("name").setValueAttribute("proxy-view");
((HtmlRadioButtonInput) form.getInputByValue("hudson.model.ProxyView")).setChecked(true);
HtmlPage proxyViewConfigurePage = j.submit(form);
View proxyView = user.getProperty(MyViewsProperty.class).getView("proxy-view");
assertNotNull(proxyView);
form = proxyViewConfigurePage.getFormByName("viewConfig");
form.getSelectByName("proxiedViewName").setSelectedAttribute("listView", true);
j.submit(form);
assertTrue(proxyView instanceof ProxyView);
assertEquals(((ProxyView) proxyView).getProxiedViewName(), "listView");
assertEquals(((ProxyView) proxyView).getProxiedView(), listView);
}
public static void main(String[] args) throws Exception {
// WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24, "54.186.230.121", 3128);
WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);
webClient.getOptions().setThrowExceptionOnScriptError(false);
webClient.setJavaScriptTimeout(10000);
webClient.getOptions().setJavaScriptEnabled(true);
webClient.setAjaxController(new NicelyResynchronizingAjaxController());
webClient.getOptions().setTimeout(10000);
// webClient.getOptions().setJavaScriptEnabled(false);
// webClient.getOptions().setAppletEnabled(false);
// webClient.getOptions().setCssEnabled(false);
// webClient.getOptions().setThrowExceptionOnScriptError(false);
// webClient.setJavaScriptTimeout(10000);
// webClient.getOptions().setJavaScriptEnabled(true);
// webClient.setAjaxController(new NicelyResynchronizingAjaxController());
// webClient.getOptions().setTimeout(10000);
// webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
// webClient.getOptions().setThrowExceptionOnScriptError(false);
HtmlPage currentPage = webClient.getPage("http://www.yandex.ru/");
// HtmlPage currentPage = webClient.getPage("http://www.google.ru");
// HtmlDivision div = currentPage.getHtmlElementById("del_competitors-1_42");
// HtmlElement clickable = (HtmlElement)
// currentPage.getHtmlElementById("del_competitors-1_42");
// currentPage = (HtmlPage) clickable.click();
HtmlAnchor advancedSearchAn = currentPage.getAnchorByText("Завести ящик");
currentPage = advancedSearchAn.click();
HtmlImage image = currentPage.<HtmlImage>getFirstByXPath("//img[@src='images/ash2008.jpg']");
currentPage = (HtmlPage) image.click();
System.out.println(currentPage.asXml());
// HtmlImage image =
// currentPage.<HtmlImage>getFirstByXPath("//img[@src='images/ash2008.jpg']");
// currentPage = (HtmlPage) image.click();
// HtmlImage imagetosave =
// currentPage.<HtmlImage>getFirstByXPath("//img[@src='//yastatic.net/www/1.977/yaru/i/logo.png']");
// HtmlImage image = currentPage.<HtmlImage>getHtmlElementById("add_competitors-1_3");
// currentPage = (HtmlPage) image.click();
// File imageFile = new File("test_new.jpg");
// image.saveAs(imageFile);
// System.out.println(currentPage.asXml());
System.out.println("It is done.");
webClient.closeAllWindows();
}
/** @throws Exception if test fails */
@Test
public void changingAttributes() throws Exception {
final String content =
"<html><head><title>foo</title></head>\n"
+ "<body>\n"
+ "<div id='testDiv' title='foo'></div>\n"
+ "</body></html>";
final HtmlPage page = loadPage(content);
final HtmlDivision div = page.getHtmlElementById("testDiv");
Assert.assertSame(div, page.getFirstByXPath("//*[@title = 'foo']"));
assertNull(page.getFirstByXPath("//*[@class = 'design']"));
div.setAttribute("class", "design");
Assert.assertSame(div, page.getFirstByXPath("//*[@class = 'design']"));
}
public ClickStreamType getLoginClickStream(HtmlPage page) {
List<RedirectType> redirectedUrls = tasksDef.getLoginInfo().getRedirectedURL();
for (RedirectType rt : redirectedUrls) {
if (rt.getLanding().getFromType() == VarType.XPATH) {
if (page.getFirstByXPath(rt.getLanding().getValue()) != null) {
return clickStreamMap.get(rt.getClickstream());
}
}
}
return null;
}
/**
* Test evaluation relative from elements other than the whole page.
*
* @throws Exception if test fails
*/
@Test
public void xpathFromElement() throws Exception {
final String content =
"<html><head><title>Test page</title></head>\n"
+ "<body><a href='foo.html' id='myLink'>foo</a></body>\n"
+ "</html>";
final HtmlPage page = loadPage(content);
final HtmlBody body = page.getFirstByXPath("/html/body");
assertEquals(page.getHtmlElementById("myLink"), body.getFirstByXPath("./a"));
}
/**
* Static method for <code>Login</code>. Uses dependencies <b>HttpUnit</b> in connecting to Keats.
*
* @see com.gargoylesoftware.htmlunit
* @param parent The parent window. instanceof<code>Scrape</code>, to set relative locations to
* @param link The url of which to retrieve information from.
* @param username The username to log in KEATS with.
* @param password The password to log in KEATS with.
* @return results.asText() Returns the content of the url if login successful. Returns null
* otherwise.
*/
public static String login(Scrape parent, String link, String username, String password) {
try {
WebClient client = new WebClient();
// Settings
client.getOptions().setThrowExceptionOnScriptError(false);
client.getOptions().setThrowExceptionOnScriptError(false);
client.getOptions().setThrowExceptionOnFailingStatusCode(false);
client.getOptions().setJavaScriptEnabled(false);
client.getOptions().setCssEnabled(false);
client.getOptions().setRedirectEnabled(true);
client.getOptions().setUseInsecureSSL(true);
client.getCookieManager().setCookiesEnabled(true);
HtmlPage page = client.getPage("https://login-keats.kcl.ac.uk/");
HtmlForm form =
page.getFirstByXPath("//form[@action='https://keats.kcl.ac.uk/login/index.php']");
HtmlInput usernameInput = form.getInputByName("username");
usernameInput.setValueAttribute(username);
HtmlInput passwordInput = form.getInputByName("password");
passwordInput.setValueAttribute(password);
page = form.getInputByValue("Log in").click();
HtmlPage results = client.getPage(link);
client.closeAllWindows();
return results.asText();
} catch (MalformedURLException e) {
JOptionPane.showMessageDialog(
parent,
"The URL you have provided is not recognised. Please double check your "
+ "input and try again.",
"MalformedURLException found.",
JOptionPane.ERROR_MESSAGE);
e.printStackTrace();
} catch (IOException e) {
JOptionPane.showMessageDialog(
parent,
"An error has occurred when attempting to read information from the "
+ "server. The input could be interrupted by external processes.",
"IOException found.",
JOptionPane.ERROR_MESSAGE);
e.printStackTrace();
}
return null;
}
/**
* Test if option/text() is cleaned like other text().
*
* @throws Exception if test fails
*/
@Test
public void optionText_getFirstByXPath() throws Exception {
final String content =
"<html><head><title>Test page</title></head>\n"
+ "<body><form name='foo'>\n"
+ "<select name='test'><option value='1'>foo and foo</option></select>\n"
+ "</form></body></html>";
final HtmlPage page = loadPage(content);
final String value = page.getFirstByXPath("string(//option)");
final int[] expectedValues = {102, 111, 111, 160, 97, 110, 100, 160, 102, 111, 111};
int index = 0;
for (final int v : expectedValues) {
if (value.codePointAt(index++) != v) {
fail();
}
}
}
@Override
public boolean verifySuccess(HtmlPage page, Object param) {
// no success defined, means always pass
if (xpathsSuccess == null) {
return true;
}
for (int i = 0; i < xpathsSuccess.length; i++) {
if (xpathsSuccess[i] != null) {
Object result = page.getFirstByXPath(xpathsSuccess[i]);
if (result == null) {
logger.warn(
String.format(
"xpath:%s not found on page %s",
xpathsSuccess[i], page.getUrl().toExternalForm()));
;
return false;
} else {
String strResult = CrawlTaskEval.getStringValue(result);
if (expectedValues != null) {
if (strResult != null) {
if (!expectedValues[i].equals(strResult)) {
logger.warn(
String.format(
"result get from xpath %s on page %s is [%s] different then expected [%s]",
xpathsSuccess[i],
page.getUrl().toExternalForm(),
strResult,
expectedValues[i]));
return false;
}
} else {
logger.warn(String.format("xpath result:%s can't be transformed to string.", result));
return false;
}
}
}
}
}
return true;
}
@Test
public void testConfigureApitoken() throws Exception {
prepareSecurity();
FreeStyleProject p = j.createFreeStyleProject();
p.addProperty(
new AuthorizeProjectProperty(new SpecificUsersAuthorizationStrategy("test2", false)));
WebClient wc = j.createWebClient();
wc.login("test1");
String apitokenForTest2 = User.get("test2").getProperty(ApiTokenProperty.class).getApiToken();
assertNotNull(apitokenForTest2);
assertNotEquals("", apitokenForTest2);
// authentication fails without apitoken
{
HtmlPage page = wc.getPage(p, "configure");
HtmlCheckBoxInput useApitoken =
page.<HtmlCheckBoxInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'useApitoken') and @type='checkbox']");
useApitoken.setChecked(true);
try {
j.submit(page.getFormByName("config"));
fail();
} catch (FailingHttpStatusCodeException e) {
assertEquals(400, e.getStatusCode());
}
}
// authentication succeeds with the good apitoken
{
HtmlPage page = wc.getPage(p, "configure");
HtmlCheckBoxInput useApitoken =
page.<HtmlCheckBoxInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'useApitoken') and @type='checkbox']");
useApitoken.setChecked(true);
HtmlTextInput apitoken =
page.<HtmlTextInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'apitoken') and @type='text']");
apitoken.setValueAttribute(apitokenForTest2);
j.submit(page.getFormByName("config"));
assertEquals(
"test2",
((SpecificUsersAuthorizationStrategy)
p.getProperty(AuthorizeProjectProperty.class).getStrategy())
.getUserid());
}
// authentication fails with a bad apitoken
{
HtmlPage page = wc.getPage(p, "configure");
HtmlCheckBoxInput useApitoken =
page.<HtmlCheckBoxInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'useApitoken') and @type='checkbox']");
useApitoken.setChecked(true);
HtmlTextInput apitoken =
page.<HtmlTextInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'apitoken') and @type='text']");
apitoken.setValueAttribute(apitokenForTest2 + "xxx");
try {
j.submit(page.getFormByName("config"));
fail();
} catch (FailingHttpStatusCodeException e) {
assertEquals(400, e.getStatusCode());
}
}
// authentication fails if the apitoken is used for password
{
HtmlPage page = wc.getPage(p, "configure");
HtmlCheckBoxInput useApitoken =
page.<HtmlCheckBoxInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'useApitoken') and @type='checkbox']");
useApitoken.setChecked(false);
HtmlPasswordInput password =
page.<HtmlPasswordInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'password') and @type='password']");
password.setValueAttribute(apitokenForTest2);
HtmlTextInput apitoken =
page.<HtmlTextInput>getFirstByXPath(
"//*[contains(@class, 'specific-user-authorization')]//input[contains(@name, 'apitoken') and @type='text']");
apitoken.setValueAttribute(apitokenForTest2);
try {
j.submit(page.getFormByName("config"));
fail();
} catch (FailingHttpStatusCodeException e) {
assertEquals(400, e.getStatusCode());
}
}
}