/** Setup before testing. */
@Before
public void setUp() {
webUrl = System.getProperty("integration.url");
webClient = new WebClient();
webClient.setJavaScriptEnabled(true);
webClient.setJavaScriptTimeout(60000);
}
public static void main(String[] args) throws Exception {
// WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24, "54.186.230.121", 3128);
WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);
webClient.getOptions().setThrowExceptionOnScriptError(false);
webClient.setJavaScriptTimeout(10000);
webClient.getOptions().setJavaScriptEnabled(true);
webClient.setAjaxController(new NicelyResynchronizingAjaxController());
webClient.getOptions().setTimeout(10000);
// webClient.getOptions().setJavaScriptEnabled(false);
// webClient.getOptions().setAppletEnabled(false);
// webClient.getOptions().setCssEnabled(false);
// webClient.getOptions().setThrowExceptionOnScriptError(false);
// webClient.setJavaScriptTimeout(10000);
// webClient.getOptions().setJavaScriptEnabled(true);
// webClient.setAjaxController(new NicelyResynchronizingAjaxController());
// webClient.getOptions().setTimeout(10000);
// webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
// webClient.getOptions().setThrowExceptionOnScriptError(false);
HtmlPage currentPage = webClient.getPage("http://www.yandex.ru/");
// HtmlPage currentPage = webClient.getPage("http://www.google.ru");
// HtmlDivision div = currentPage.getHtmlElementById("del_competitors-1_42");
// HtmlElement clickable = (HtmlElement)
// currentPage.getHtmlElementById("del_competitors-1_42");
// currentPage = (HtmlPage) clickable.click();
HtmlAnchor advancedSearchAn = currentPage.getAnchorByText("Завести ящик");
currentPage = advancedSearchAn.click();
HtmlImage image = currentPage.<HtmlImage>getFirstByXPath("//img[@src='images/ash2008.jpg']");
currentPage = (HtmlPage) image.click();
System.out.println(currentPage.asXml());
// HtmlImage image =
// currentPage.<HtmlImage>getFirstByXPath("//img[@src='images/ash2008.jpg']");
// currentPage = (HtmlPage) image.click();
// HtmlImage imagetosave =
// currentPage.<HtmlImage>getFirstByXPath("//img[@src='//yastatic.net/www/1.977/yaru/i/logo.png']");
// HtmlImage image = currentPage.<HtmlImage>getHtmlElementById("add_competitors-1_3");
// currentPage = (HtmlPage) image.click();
// File imageFile = new File("test_new.jpg");
// image.saveAs(imageFile);
// System.out.println(currentPage.asXml());
System.out.println("It is done.");
webClient.closeAllWindows();
}
@Test
public void testLabelAjaxBehavior() throws Exception {
webClient.setJavaScriptEnabled(true);
webClient.setJavaScriptTimeout(60000);
HtmlPage page = webClient.getPage(webUrl + "/faces/label.xhtml");
HtmlElement label = page.getHtmlElementById("label2");
page = (HtmlPage) label.mouseOver();
webClient.waitForBackgroundJavaScript(60000);
assertTrue(page.asXml().contains("label2 Event: begin"));
assertTrue(page.asXml().contains("label2 Event: complete"));
assertTrue(page.asXml().contains("label2 Event: success"));
}
@Test
public void testFieldSetAjaxBehavior() throws Exception {
webClient.setJavaScriptEnabled(true);
webClient.setJavaScriptTimeout(60000);
HtmlPage page = webClient.getPage(webUrl + "/faces/fieldset.xhtml");
HtmlFieldSet fieldset = (HtmlFieldSet) page.getHtmlElementById("fieldset4");
page = fieldset.click();
webClient.waitForBackgroundJavaScript(60000);
assertTrue(page.asXml().contains("fieldset4 Event: begin"));
assertTrue(page.asXml().contains("fieldset4 Event: complete"));
assertTrue(page.asXml().contains("fieldset4 Event: success"));
}
public static void main(String[] args) throws IOException {
WebClient client = new WebClient(BrowserVersion.CHROME);
CookieManager cookie = new CookieManager();
client.setCookieManager(cookie);
client.getOptions().setJavaScriptEnabled(true);
client.getOptions().setActiveXNative(false);
client.getOptions().setCssEnabled(false);
client.getOptions().setThrowExceptionOnScriptError(false);
client.getOptions().setThrowExceptionOnFailingStatusCode(false);
client.getOptions().setDoNotTrackEnabled(true);
client.getOptions().setPrintContentOnFailingStatusCode(false);
client.setAjaxController(new NicelyResynchronizingAjaxController());
client.setJavaScriptTimeout(Long.MAX_VALUE);
List<Integer> TopCategory = new LinkedList<Integer>(TopCategoryUrl.keySet());
Collections.sort(TopCategory);
long today = new Date().getTime();
long ymd = today - 1000 * 60 * 60 * 24;
String date = sdf.format(ymd);
FileWriter fw = new FileWriter("ShuTaobaoTop/search-" + date, false);
BufferedWriter bw = new BufferedWriter(fw);
while (TopCategory.size() > 0) {
int cid = TopCategory.get(0);
// if(cid<50002766)
// {
// TopCategory.remove(0);
// continue;
// }
String cname = TopCategoryUrl.get(cid);
String pid = "0", pname = "";
String topUrl = "http://shu.taobao.com/top/" + cid + "/search";
boolean success = false;
ArrayList<String> result = new ArrayList<String>();
try {
// client.getOptions().setJavaScriptEnabled(true);
do {
System.out.println("剩余类目数=" + TopCategory.size());
// ProxyUnit.configProxy(client, ProxyUnit.CHECKURL);
} while (false == AccountLogin.loginTaoBao(client, AccountLogin.TAOBAOLOGINURL, true));
// client.getOptions().setJavaScriptEnabled(false);
HtmlPage queryTrade = client.getPage(topUrl);
Thread.sleep(1000L);
System.out.println(queryTrade.getTitleText());
/*
* 解析
*/
Document doc = Jsoup.parse(queryTrade.asXml());
Element time = doc.getElementsByAttributeValue("class", "time").get(0);
String startTime =
time.text().substring(6, 16).replace("-", ""); // 统计时间: 2014-05-17 - 2014-05-23
String endTime = time.text().substring(19).replace("-", "");
Elements elements =
doc.getElementsByAttributeValueStarting(
"class", "mod "); // <div class="mod odd mod-10 sm-ua"> <h3 class="title">
for (Element element : elements) {
String title = element.getElementsByTag("h3").get(0).text();
System.out.println(title);
Elements lis =
element.getElementsByTag("ol").get(0).getElementsByTag("li"); // <li class="up ">
for (Element li : lis) {
Elements spans = li.getElementsByTag("span");
String rank = spans.get(0).text();
String key = spans.get(1).text();
String rise = spans.get(2).text();
rise = rise.substring(0, rise.length() - 1);
URL href =
new URL(
URLDecoder.decode(
"http://shu.taobao.com"
+ spans.get(1).getElementsByTag("a").get(0).attr("href"),
"utf-8"));
String hrefQuery = href.getQuery();
if (hrefQuery.contains("cid=")) {
if ("0".equals(pid)) {
pid = String.valueOf(cid);
pname = cname;
}
cid = Integer.parseInt(hrefQuery.split("cid=")[1].split("&")[0]);
cname = title;
}
System.out.println(
startTime + "\001" + endTime + "\001" + cid + "\001" + cname + "\001" + pid + "\001"
+ pname + "\001" + rank + "\001" + key + "\001" + rise);
result.add(
startTime + "\001" + endTime + "\001" + cid + "\001" + cname + "\001" + pid + "\001"
+ pname + "\001" + rank + "\001" + key + "\001" + rise);
}
}
success = true;
} catch (Exception e) {
// TODO Auto-generated catch block
// e.printStackTrace();
}
if (success) {
TopCategory.remove(0);
System.out.println("剩余类目数=" + TopCategory.size());
for (String r : result) {
bw.write(r + "\n");
}
}
}
bw.close();
fw.close();
}
