public static void main(String[] args) {
WebClient client = new WebClient(BrowserVersion.CHROME);
HtmlPage page = null;
try {
page = client.getPage("http://www.facebook.com");
System.out.println(page.getTitleText());
HtmlForm form = (HtmlForm) page.getElementById("login_form");
form.getInputByName("email").setValueAttribute("*****@*****.**");
form.getInputByName("pass").setValueAttribute("saburtalo16");
page = form.getInputByValue("Log In").click();
System.out.println(page.getTitleText());
HtmlTextArea statusText = (HtmlTextArea) page.getElementByName("xhpc_message_text");
statusText.click();
statusText.setText("I'm a robot");
HtmlButton post =
(HtmlButton)
page.getFirstByXPath("//button[@id=\"u_jsonp_3_4\"]/div/div[4]/div/ul/li[2]/button");
post.click();
} catch (Exception e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
}
}
/**
* Tests that the details of cargo with tracking Id DEF789 can be viewed through the admin
* interface.
*/
@Test
@RunAsClient
public void testViewDetailsId4() {
log.log(
Level.INFO,
"Starting automated test to view details for Id \""
+ trackingId4
+ "\" through admin interface.");
try {
System.out.println(landingPageResponse.getUrl());
// Stores the adminDashboard as a HtmlPage object.
HtmlPage adminDashboard = landingPageResponse.getElementById("adminLandingLink").click();
Assert.assertThat(
"Page title was not as expected for the admin dashboard. Expected \"Cargo Dashboard\" but actual was \""
+ adminDashboard.getTitleText()
+ "\".",
adminDashboard.getTitleText(),
is("Cargo Dashboard"));
// Stores the details page as a HtmlPage object.
HtmlPage detailsPage = adminDashboard.getAnchorByText(trackingId4).click();
Assert.assertTrue(
"Expected \"Not routed\" message was not found.",
detailsPage.asText().contains("Not routed"));
} catch (IOException ex) {
Assert.fail(
"An IOException was thrown during the test for class \""
+ ViewDetailTest.class.getSimpleName()
+ "\" at method \""
+ testName.getMethodName()
+ "\" with message: "
+ ex.getMessage());
}
}
/**
* Test evaluation of paths after they're changed through JavaScript.
*
* @throws Exception if test fails
*/
@Test
public void whenJSChangesPage() throws Exception {
final String content =
"<html><head><title>foo</title><script>\n"
+ "function addOption() {\n"
+ " var options = document.form1.select1.options;\n"
+ " var index = options.length;\n"
+ " options[index] = new Option('Four','value4');\n"
+ "}</script>\n"
+ "</head>\n"
+ "<body>\n"
+ "<p>hello world</p>\n"
+ "<form name='form1'>\n"
+ " <select name='select1'>\n"
+ " <option name='option1' value='value1'>One</option>\n"
+ " <option name='option2' value='value2' selected>Two</option>\n"
+ " <option name='option3' value='value3'>Three</option>\n"
+ " </select>\n"
+ "</form>\n"
+ "<a href='javascript:addOption()'>add option</a>\n"
+ "</body></html>";
final HtmlPage page = loadPage(content);
assertEquals("foo", page.getTitleText());
assertEquals(
3, page.<Number>getFirstByXPath("count(//select[@name='select1']/option)").intValue());
page.getAnchors().get(0).click();
assertEquals(
4, page.<Number>getFirstByXPath("count(//select[@name='select1']/option)").intValue());
}
// Check the actual conditional page with a "true" flag
private void checkJstlIf04a() throws Exception {
HtmlPage page = getPage("/faces/jsp/jstl-if-04.jsp?cond=true");
assertEquals("Correct page title", "jstl-if-04", page.getTitleText());
assertEquals(
"Correct body element", "[1] [2] [3] { [4a] [4b] [4c] } [5] [6] [7]", getBodyText(page));
}
/**
* get the ajax url from the click button
*
* @param clickOfXpath:页面待点击按钮的xpath表达式
* @param index
* @return List<String>:链表的第一个信息是页面的title，以后的信息是所有的ajax的url
*/
public static List<String> getAjaxUrl(String targetUrl, String clickOfXpath, int index)
throws FailingHttpStatusCodeException, MalformedURLException, IOException {
// TARGET_URL =
// "http://app.flyme.cn/apps/public/detail?package_name=com.myzaker.zaker_phone_smartbar";
List<String> urls = new LinkedList<String>();
// 每次ajax请求时都会创建一个AjaxController对象，在该对象中可以查看ajax请求的地址
MyNicelyResynchronizingAjaxController ajaxController =
new MyNicelyResynchronizingAjaxController();
List alertHandler = new LinkedList();
// 模拟一个浏览器
WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);
// HtmlUnitDriver
// 设置webClient的相关参数
webClient.getOptions().setJavaScriptEnabled(true);
webClient.getOptions().setCssEnabled(false);
webClient.setAjaxController(ajaxController);
webClient.getOptions().setTimeout(35000);
webClient.getOptions().setThrowExceptionOnScriptError(false);
webClient.setAlertHandler(
new CollectingAlertHandler(alertHandler)); // 将JavaScript中alert标签产生的数据保存在一个链表中
// 模拟浏览器打开一个目标网址
HtmlPage rootPage = webClient.getPage(targetUrl);
urls.add(rootPage.getTitleText());
urls.add(ajaxController.getVisitUrl());
// System.out.println("url1:" + url);
HtmlElement elementA = (HtmlElement) rootPage.getByXPath(clickOfXpath).get(index);
Page page = elementA.click();
urls.add(ajaxController.getVisitUrl());
return urls;
}
@Override
public String[] getRowCSV(HtmlPage page) {
String title = page.getTitleText();
String video = null;
List<HtmlElement> anchorsBgcover =
page.getHtmlElementById("bgcover").getHtmlElementsByTagName("a");
if (!anchorsBgcover.isEmpty()) {
video = anchorsBgcover.get(0).getAttribute("href");
}
String rating = null;
List<HtmlElement> spansBgcover =
page.getHtmlElementById("bgcover").getHtmlElementsByTagName("span");
if (!spansBgcover.isEmpty()) {
rating = spansBgcover.get(0).asText();
if (rating.equals("MYMO NET RO?")) {
rating = null;
}
}
String[] idArray = page.getUrl().toString().split("/");
String id = idArray[idArray.length - 1];
String[] elementRow = {id, title, video, rating};
System.out.println(title);
return elementRow;
}
@Test(
groups = {"loggedIn"},
dependsOnMethods = {"testLogin"})
public void testContributedFilterChainDefinition() throws Exception {
clickOnBasePage("contributed");
assertEquals("contribution success", page.getTitleText());
}
// Check the actual facet page with true/true/true flags
private void checkJstlIf02a() throws Exception {
HtmlPage page = getPage("/faces/jsp/jstl-if-02.jsp?component=true&header=true&footer=true");
assertEquals("Correct page title", "jstl-if-02", page.getTitleText());
assertEquals(
"Correct body element", "[First] [Header] [Second] [Footer] [Third]", getBodyText(page));
}
public void testNesting03() throws Exception {
HtmlPage page = getPage("/faces/composite/nesting03.xhtml");
HtmlSubmitInput input = (HtmlSubmitInput) getInputContainingGivenId(page, "commandButton");
assertNotNull(input);
page = input.click();
assertEquals("Navigation Result", page.getTitleText());
}
/**
* Test method.
*
* @throws IOException exception
*/
public void testLogin() throws IOException {
final WebClient webClient = new WebClient();
// XXX: issues with jquery
webClient.setJavaScriptEnabled(false);
final HtmlPage page = webClient.getPage(HTTP_PROCC_LOGIN_USER);
assertEquals("Taskflow-Logowanie", page.getTitleText());
webClient.closeAllWindows();
}
private void doHttpTest(final WebClient client)
throws Exception, IOException, MalformedURLException {
final Map<String, Class<? extends Servlet>> servlets = new HashMap<>();
servlets.put("/test", SocksProxyTestServlet.class);
startWebServer("./", new String[0], servlets);
final HtmlPage page = client.getPage("http://localhost:" + PORT + "/test");
assertEquals("hello", page.getTitleText());
}
private void doHttpsTest(final WebClient webClient) throws Exception {
localServer_ = new InsecureHttpsServer(SocksProxyTestServlet.HTML);
localServer_.start();
webClient.getOptions().setUseInsecureSSL(true);
final String url = "https://" + localServer_.getHostName() + ":" + localServer_.getPort();
final HtmlPage page = webClient.getPage(url);
assertEquals("hello", page.getTitleText());
}
// Check the values of the input fields against the specified list
private void checkJstlForEach01(HtmlPage page, String expected[]) {
assertEquals("Correct page title", "jstl-foreach-01", page.getTitleText());
HtmlForm form = getFormById(page, jstlForEach01_name);
assertNotNull("form exists", form);
for (int i = 0; i < expected.length; i++) {
HtmlTextInput input =
(HtmlTextInput)
form.getInputByName(
jstlForEach01_name + NamingContainer.SEPARATOR_CHAR + jstlForEach01_names[i]);
assertNotNull("field '" + jstlForEach01_names[i] + "' exists", input);
assertEquals(
"field '" + jstlForEach01_names[i] + "' value", expected[i], input.getValueAttribute());
}
}
/**
* Test method.
*
* @throws IOException exception
*/
public void testLoginValid() throws IOException {
final WebClient webClient = new WebClient();
// XXX: issues with jquery
webClient.setJavaScriptEnabled(false);
final HtmlPage page = webClient.getPage(HTTP_PROCC_LOGIN_USER);
final HtmlForm form = page.getFormByName(LOGIN_USER_FORM_NAME);
final HtmlTextInput textEmail = form.getInputByName(EMAIL_INPUT_NAME);
final HtmlPasswordInput textPassword = form.getInputByName(PASSWD_INPUT_NAME);
final HtmlSubmitInput btnLogin = form.getInputByName(LOGIN_SUBMIT_NAME);
textEmail.setValueAttribute("*****@*****.**");
textPassword.setValueAttribute("malczyk123");
final HtmlPage nextPage = btnLogin.click();
assertTrue(nextPage.getTitleText().equals("Taskflow-Main"));
webClient.closeAllWindows();
}
public void run() {
if (this.classifier == null) return;
WebClient client = WebClientFactory.borrowClient();
String[] urls = extractor.getUrls();
final ArrayList<Object[]> valuesToInsert = new ArrayList<Object[]>();
final Date timeNow = new Date();
final int revisionId;
final Annotation[] annotations = extractor.getAnnotations();
try {
revisionId = ScrapeHelper.createRevision(extractor);
HtmlPage page;
for (int i = 0; i < urls.length; i++) {
try {
page = ScrapeHelper.downloadPage(urls[0], client, null);
System.out.println(page.getTitleText());
classifier.classifyPageElements(
page,
new ClassifiedTask() {
public void performTask(int label, DomNode element) {
if (label < annotations.length
&& !element.getNodeName().equalsIgnoreCase("body")) {
valuesToInsert.add(
new Object[] {
annotations[label].getId(), element.asXml(), timeNow, timeNow, revisionId
});
}
}
});
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
Object[][] values = new Object[valuesToInsert.size()][valuesToInsert.get(0).length];
for (int j = 0; j < values.length; j++) values[j] = valuesToInsert.get(j);
queryRunner.batch(
"INSERT INTO scraped_values (annotation_id,value,created_at,updated_at,revision_id) VALUES (?,?,?,?,?)",
values);
} catch (SQLException e1) {
e1.printStackTrace();
}
WebClientFactory.returnClient(client);
}
/**
* Test storefront.
*
* @throws FailingHttpStatusCodeException the failing http status code exception
* @throws MalformedURLException the malformed url exception
* @throws IOException Signals that an I/O exception has occurred.
*/
@Test
public void testStorefront()
throws FailingHttpStatusCodeException, MalformedURLException, IOException {
final Properties settings = new Properties();
settings.load(ApplicationStartupTest.class.getResourceAsStream("/test.properties"));
final WebClient webClient = new WebClient();
// use the store code to get to the storefront
final Page page =
webClient.getPage(
"http://localhost:"
+ settings.getProperty("cargo.port")
+ settings.getProperty("storefront.context")
+ "?storeCode="
+ settings.getProperty("store.code"));
assertEquals(HTTP_STATUS_OK, page.getWebResponse().getStatusCode());
assertTrue(page instanceof HtmlPage);
HtmlPage htmlPage = (HtmlPage) page;
assertTrue(htmlPage.getTitleText().length() != 0);
webClient.closeAllWindows();
}
/**
* Set up method run before each test to create a new Web client and load the application's
* landing page.
*/
@Before
@RunAsClient
public void setUp() {
try {
browser = new WebClient();
browser.getOptions().setThrowExceptionOnScriptError(false);
landingPageResponse = browser.getPage(deploymentUrl.toString() + "index.xhtml");
Assert.assertEquals(
"Could not load the application landing page.",
"Cargo Tracker",
landingPageResponse.getTitleText());
} catch (IOException ex) {
Assert.fail(
"An IOException was thrown during the test setup for class \""
+ ViewDetailTest.class.getSimpleName()
+ "\" at method \""
+ testName.getMethodName()
+ "\" with message: "
+ ex.getMessage());
}
}
/** {@inheritDoc} */
@Override
public boolean isAt() {
return "Album Page".equals(page.getTitleText());
}
// Check selective imports with simple components (naming container)
private void checkJstlImport04c() throws Exception {
HtmlPage page = getPage("/faces/jsp/jstl-import-04.jsp?choose=c");
assertEquals("Correct page title", "jstl-import-04", page.getTitleText());
assertEquals("Correct body element", "[1] [2c][2x] [3]", getBodyText(page));
}
// Check the actual conditional page with a "false" flag
private void checkJstlIf03b() throws Exception {
HtmlPage page = getPage("/faces/jsp/jstl-if-03.jsp?cond=false");
assertEquals("Correct page title", "jstl-if-03", page.getTitleText());
assertEquals("Correct body element", "[1] [5]", getBodyText(page));
}
// Check the actual conditional page with a "true" flag
private void checkJstlIf01a() throws Exception {
HtmlPage page = getPage("/faces/jsp/jstl-if-01.jsp?cond=true");
assertEquals("Correct page title", "jstl-if-01", page.getTitleText());
assertEquals("Correct body element", "[First] [Second] [Third]", getBodyText(page));
}
// Check chosen components with naming containers
private void checkJstlChoose02b() throws Exception {
HtmlPage page = getPage("/faces/jsp/jstl-choose-02.jsp?choose=b");
assertEquals("Correct page title", "jstl-choose-02", page.getTitleText());
assertEquals("Correct body element", "[1] [2b] [2y] [3]", getBodyText(page));
}
// Check the success page was received
private void checkFormInput01(HtmlPage page) {
assertEquals("Correct page title", "form-input-01", page.getTitleText());
}
private String getTitle() {
return page.getTitleText();
}
// Check imports with simple components
private void checkJstlImport02() throws Exception {
HtmlPage page = getPage("/faces/jsp/jstl-import-02.jsp");
assertEquals("Correct page title", "jstl-import-02", page.getTitleText());
assertEquals("Correct body element", "[A] [B] [C] [D] [E]", getBodyText(page));
}
private void checkFormInput00() throws Exception {
HtmlPage page = getPage("/faces/jsp/form-input-00.jsp");
assertEquals("Correct page title", "form-input-00", page.getTitleText());
}
public static void main(String[] args) throws IOException {
WebClient client = new WebClient(BrowserVersion.CHROME);
CookieManager cookie = new CookieManager();
client.setCookieManager(cookie);
client.getOptions().setJavaScriptEnabled(true);
client.getOptions().setActiveXNative(false);
client.getOptions().setCssEnabled(false);
client.getOptions().setThrowExceptionOnScriptError(false);
client.getOptions().setThrowExceptionOnFailingStatusCode(false);
client.getOptions().setDoNotTrackEnabled(true);
client.getOptions().setPrintContentOnFailingStatusCode(false);
client.setAjaxController(new NicelyResynchronizingAjaxController());
client.setJavaScriptTimeout(Long.MAX_VALUE);
List<Integer> TopCategory = new LinkedList<Integer>(TopCategoryUrl.keySet());
Collections.sort(TopCategory);
long today = new Date().getTime();
long ymd = today - 1000 * 60 * 60 * 24;
String date = sdf.format(ymd);
FileWriter fw = new FileWriter("ShuTaobaoTop/search-" + date, false);
BufferedWriter bw = new BufferedWriter(fw);
while (TopCategory.size() > 0) {
int cid = TopCategory.get(0);
// if(cid<50002766)
// {
// TopCategory.remove(0);
// continue;
// }
String cname = TopCategoryUrl.get(cid);
String pid = "0", pname = "";
String topUrl = "http://shu.taobao.com/top/" + cid + "/search";
boolean success = false;
ArrayList<String> result = new ArrayList<String>();
try {
// client.getOptions().setJavaScriptEnabled(true);
do {
System.out.println("剩余类目数=" + TopCategory.size());
// ProxyUnit.configProxy(client, ProxyUnit.CHECKURL);
} while (false == AccountLogin.loginTaoBao(client, AccountLogin.TAOBAOLOGINURL, true));
// client.getOptions().setJavaScriptEnabled(false);
HtmlPage queryTrade = client.getPage(topUrl);
Thread.sleep(1000L);
System.out.println(queryTrade.getTitleText());
/*
* 解析
*/
Document doc = Jsoup.parse(queryTrade.asXml());
Element time = doc.getElementsByAttributeValue("class", "time").get(0);
String startTime =
time.text().substring(6, 16).replace("-", ""); // 统计时间: 2014-05-17 - 2014-05-23
String endTime = time.text().substring(19).replace("-", "");
Elements elements =
doc.getElementsByAttributeValueStarting(
"class", "mod "); // <div class="mod odd mod-10 sm-ua"> <h3 class="title">
for (Element element : elements) {
String title = element.getElementsByTag("h3").get(0).text();
System.out.println(title);
Elements lis =
element.getElementsByTag("ol").get(0).getElementsByTag("li"); // <li class="up ">
for (Element li : lis) {
Elements spans = li.getElementsByTag("span");
String rank = spans.get(0).text();
String key = spans.get(1).text();
String rise = spans.get(2).text();
rise = rise.substring(0, rise.length() - 1);
URL href =
new URL(
URLDecoder.decode(
"http://shu.taobao.com"
+ spans.get(1).getElementsByTag("a").get(0).attr("href"),
"utf-8"));
String hrefQuery = href.getQuery();
if (hrefQuery.contains("cid=")) {
if ("0".equals(pid)) {
pid = String.valueOf(cid);
pname = cname;
}
cid = Integer.parseInt(hrefQuery.split("cid=")[1].split("&")[0]);
cname = title;
}
System.out.println(
startTime + "\001" + endTime + "\001" + cid + "\001" + cname + "\001" + pid + "\001"
+ pname + "\001" + rank + "\001" + key + "\001" + rise);
result.add(
startTime + "\001" + endTime + "\001" + cid + "\001" + cname + "\001" + pid + "\001"
+ pname + "\001" + rank + "\001" + key + "\001" + rise);
}
}
success = true;
} catch (Exception e) {
// TODO Auto-generated catch block
// e.printStackTrace();
}
if (success) {
TopCategory.remove(0);
System.out.println("剩余类目数=" + TopCategory.size());
for (String r : result) {
bw.write(r + "\n");
}
}
}
bw.close();
fw.close();
}
// Check the reset page to force a new component tree
private void checkJstlImport00() throws Exception {
HtmlPage page = getPage("/faces/jsp/jstl-import-00.jsp");
assertEquals("Correct page title", "jstl-import-00", page.getTitleText());
}