/**
* <span style="color:red">POTENIAL PERFORMANCE KILLER - DOWNLOADS THE RESOURCE - USE AT YOUR OWN
* RISK.</span><br>
* If the linked content is not already downloaded it triggers a download. Then it stores the
* response for later use.<br>
*
* @param downloadIfNeeded indicates if a request should be performed this hasn't been done
* previously
* @return <code>null</code> if no download should be performed and when this wasn't already done;
* the response received when performing a request for the content referenced by this tag
* otherwise
* @throws IOException if an error occurs while downloading the content
*/
public WebResponse getWebResponse(final boolean downloadIfNeeded) throws IOException {
if (downloadIfNeeded && cachedWebResponse_ == null) {
final WebClient webclient = getPage().getWebClient();
cachedWebResponse_ = webclient.loadWebResponse(getWebRequest());
}
return cachedWebResponse_;
}
public void start() throws Exception {
// TODO 不下载非本站的静态资源与链接
// TODO 不继续抓取已经爬过的页面
// TODO 9skb链接写法的下载
if (!siteSaveFolder.endsWith(File.separator)) {
siteSaveFolder += File.separator;
}
File saveFolder = new File(siteSaveFolder);
if (!saveFolder.exists()) {
saveFolder.mkdirs();
}
staticModels.clear();
client = new WebClient();
// 设置css activex applet 失效
client.setActiveXNative(false);
client.setAppletEnabled(false);
client.setCssEnabled(false);
client.setJavaScriptEnabled(false);
// 10秒超时
client.setTimeout(10 * 1000);
fetchPage(siteUrl, "index.html");
System.out.println("----------------------------------------------------------------------");
System.out.println("[ 正在下载静态资源文件... ]");
// fetchStaticResources();
}
public static final WebClient newWebClient(CookieManager cookieManager) {
// Create and initialize WebClient object
WebClient webClient = newWebClient();
webClient.setCookieManager(cookieManager);
return webClient;
}
@Test
@Ignore
public void test3() throws Exception {
System.out.println("-------------------------------");
WebClient webClient = new WebClient(BrowserVersion.CHROME);
webClient.getOptions().setCssEnabled(false);
webClient.getOptions().setJavaScriptEnabled(false);
HtmlPage page =
webClient.getPage("http://www.zjnu.edu.cn/news/common/article_show.aspx?article_id=19285");
System.out.println("---------------标题----------------");
HtmlSpan span1 = (HtmlSpan) page.getElementById("mytitle");
System.out.println(span1.asText());
System.out.println("-------------------------------");
System.out.println("---------------正文----------------");
HtmlSpan span2 = (HtmlSpan) page.getElementById("mycontent");
System.out.println(span2.asText());
System.out.println("-------------------------------");
System.out.println("---------------图片----------------");
DomNodeList<HtmlElement> elements = span2.getElementsByTagName("img");
for (HtmlElement element : elements) {
System.out.println(element.getAttribute("src"));
}
// log.debug("{}", elements);
System.out.println("-------------------------------");
webClient.close();
System.out.println("-------------------------------");
}
@Test
public void testResponse() throws Exception {
WebClient webClient = new WebClient();
TextPage page = webClient.getPage(url);
assertEquals("Pong", page.getContent());
webClient.closeAllWindows();
}
@Test
public void testExceptionPhaseListener() throws Exception {
WebClient client = new WebClient();
client.setThrowExceptionOnFailingStatusCode(false);
// First, try a transient conversation
// Access a page that throws an exception
client.getPage(getPath("/thunderstorm.jsf"));
// Then access another page that doesn't and check the contexts are ok
HtmlPage cloud = client.getPage(getPath("/cloud.jsf"));
String cloudName = getFirstMatchingElement(cloud, HtmlSpan.class, "cloudName").getTextContent();
assertEquals(Cloud.NAME, cloudName);
// Now start a conversation and access the page that throws an exception again
HtmlPage thunderstorm =
getFirstMatchingElement(cloud, HtmlSubmitInput.class, "beginConversation").click();
// This page will error
String cid = getCid(thunderstorm);
cloud = client.getPage(getPath("/cloud.jsf", cid));
// And navigate to another page, checking the conversation exists by verifying that state is
// maintained
cloudName = getFirstMatchingElement(cloud, HtmlSpan.class, "cloudName").getTextContent();
assertEquals("gavin", cloudName);
}
public boolean athenticated() {
web = new WebClient();
try {
page1 = web.getPage(url);
} catch (FailingHttpStatusCodeException | IOException e) {
e.printStackTrace();
}
HtmlForm form = page1.getFormByName("form2");
form.getInputByName("txtUserName").setAttribute("value", username);
form.getInputByName("txtPassword").setAttribute("value", password);
try {
page2 = form.getInputByName("LoginButton").click();
} catch (ElementNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
if (page2.getUrl().toString().equals("http://78.39.200.210/Default.aspx")) {
return true;
} else {
web.closeAllWindows();
return false;
}
}
/** Setup before testing. */
@Before
public void setUp() {
webUrl = System.getProperty("integration.url");
webClient = new WebClient();
webClient.setJavaScriptEnabled(true);
webClient.setJavaScriptTimeout(60000);
}
public static HtmlPage getPage(String url)
throws FailingHttpStatusCodeException, MalformedURLException, IOException {
WebClient wc = new WebClient(BrowserVersion.CHROME);
wc.getOptions().setCssEnabled(false);
HtmlPage page = wc.getPage(url);
return page;
}
public static ArrayList<String> trans(String url)
throws FailingHttpStatusCodeException, MalformedURLException, IOException {
ArrayList<String> hrefList = new ArrayList<String>();
WebClient webClient = new WebClient(BrowserVersion.CHROME);
webClient.getOptions().setJavaScriptEnabled(false);
webClient.getOptions().setCssEnabled(false);
try {
HtmlPage page = null;
try {
page = (HtmlPage) webClient.getPage(url);
} catch (ConnectException e) {
System.out.println("Connect fails here:" + e.getMessage());
}
InputStream temp = new ByteArrayInputStream(page.asText().getBytes());
InputStreamReader isr = new InputStreamReader(temp);
BufferedReader br = new BufferedReader(isr);
String str = null, rs = null;
while ((str = br.readLine()) != null) {
rs = str;
// System.out.println(rs);
if (rs != null) hrefList.add(rs);
}
System.out.println("从该网址查找的可能相关文本如下：");
for (int i = 0; i < hrefList.size(); i++) {
String string = hrefList.get(i);
string = getTextFromHtml(string);
if (string.length() >= 30) System.out.println("------" + i + ":" + string);
}
} catch (IOException e) {
}
return hrefList;
}
public static WebClient buildWebClient(CookieManager manager) {
WebClient webClient = buildWebClient();
webClient.setCookieManager(manager);
return webClient;
}
public static void main(String[] args) {
WebClient client = new WebClient(BrowserVersion.CHROME);
HtmlPage page = null;
try {
page = client.getPage("http://www.facebook.com");
System.out.println(page.getTitleText());
HtmlForm form = (HtmlForm) page.getElementById("login_form");
form.getInputByName("email").setValueAttribute("*****@*****.**");
form.getInputByName("pass").setValueAttribute("saburtalo16");
page = form.getInputByValue("Log In").click();
System.out.println(page.getTitleText());
HtmlTextArea statusText = (HtmlTextArea) page.getElementByName("xhpc_message_text");
statusText.click();
statusText.setText("I'm a robot");
HtmlButton post =
(HtmlButton)
page.getFirstByXPath("//button[@id=\"u_jsonp_3_4\"]/div/div[4]/div/ul/li[2]/button");
post.click();
} catch (Exception e) {
e
.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
}
}
@Override
protected SkypeGuestTempSession run(WebClient webClient) throws Exception {
webClient.getPage(joinUrl);
String csrf = webClient.getCookieManager().getCookie("csrf_token").getValue();
String sessionId = webClient.getCookieManager().getCookie("launcher_session_id").getValue();
return new SkypeGuestTempSession(csrf, sessionId);
}
/**
* Tests asynchronous use of XMLHttpRequest, using Mozilla style object creation.
*
* @throws Exception if the test fails
*/
@Test
public void testAsyncUse() throws Exception {
final URL secondUrl = new URL(FIRST_URL, "/second/");
final String html =
"<html>\n"
+ " <head>\n"
+ " <title>XMLHttpRequest Test</title>\n"
+ " <script>\n"
+ " var request;\n"
+ " function testAsync() {\n"
+ " if (window.XMLHttpRequest)\n"
+ " request = new XMLHttpRequest();\n"
+ " else if (window.ActiveXObject)\n"
+ " request = new ActiveXObject('Microsoft.XMLHTTP');\n"
+ " request.onreadystatechange = onReadyStateChange;\n"
+ " alert(request.readyState);\n"
+ " request.open('GET', '/second/', true);\n"
+ " request.send('');\n"
+ " }\n"
+ " function onReadyStateChange() {\n"
+ " alert(request.readyState);\n"
+ " if (request.readyState == 4)\n"
+ " alert(request.responseText);\n"
+ " }\n"
+ " </script>\n"
+ " </head>\n"
+ " <body onload='testAsync()'>\n"
+ " </body>\n"
+ "</html>";
final String xml =
"<xml2>\n"
+ "<content2>sdgxsdgx</content2>\n"
+ "<content2>sdgxsdgx2</content2>\n"
+ "</xml2>";
final WebClient client = new WebClient();
final List<String> collectedAlerts = new ArrayList<String>();
client.setAlertHandler(new CollectingAlertHandler(collectedAlerts));
final MockWebConnection conn = new MockWebConnection();
conn.setResponse(FIRST_URL, html);
conn.setResponse(secondUrl, xml, "text/xml");
client.setWebConnection(conn);
client.getPage(FIRST_URL);
final int executedJobs = client.getJavaScriptEngine().pumpEventLoop(1000);
final String[] alerts = {
String.valueOf(XMLHttpRequest.STATE_UNINITIALIZED),
String.valueOf(XMLHttpRequest.STATE_LOADING),
String.valueOf(XMLHttpRequest.STATE_LOADING),
String.valueOf(XMLHttpRequest.STATE_LOADED),
String.valueOf(XMLHttpRequest.STATE_INTERACTIVE),
String.valueOf(XMLHttpRequest.STATE_COMPLETED),
xml
};
assertEquals(Arrays.asList(alerts).toString(), collectedAlerts.toString());
assertEquals(1, executedJobs);
}
private HtmlPage getPage(String url) throws IOException {
final WebClient webClient =
new WebClient(
getBrowserVersionFromName(
searchEngine.getDefaultBrowser())); // BrowserVersion.FIREFOX_24);
webClient.getOptions().setJavaScriptEnabled(false);
return webClient.getPage(url);
}
private static List<String> getSimulatedProperties(final BrowserVersion browserVersion)
throws Exception {
final URL url = PropertiesTest.class.getClassLoader().getResource("objects/properties.html");
final WebClient webClient = new WebClient(browserVersion);
final HtmlPage page = webClient.getPage(url);
final HtmlTextArea textarea = page.getHtmlElementById("myTextarea");
return Arrays.asList(textarea.getText().split("\r\n|\n"));
}
/**
* Test method.
*
* @throws IOException exception
*/
public void testLogin() throws IOException {
final WebClient webClient = new WebClient();
// XXX: issues with jquery
webClient.setJavaScriptEnabled(false);
final HtmlPage page = webClient.getPage(HTTP_PROCC_LOGIN_USER);
assertEquals("Taskflow-Logowanie", page.getTitleText());
webClient.closeAllWindows();
}
protected WebContext loadPage(String content) throws Exception {
server = getServer();
String path = getName() + ".html";
server.addPage(path, content);
WebClient webClient = new WebClient();
String url = server.getUrl(path);
return new WebContext(webClient.getPage(url), webClient, project);
}
@Test
public void testInvalidatedSession() throws Exception {
HtmlPage page = webClient.getPage(webUrl + "faces/invalidatedSession.xhtml");
assertTrue(page.asText().indexOf("This is from the @PostConstruct") != -1);
webClient.getPage(webUrl + "faces/invalidatedPerform.xhtml");
page = webClient.getPage(webUrl + "faces/invalidatedVerify.xhtml");
assertTrue(page.asText().indexOf("true") != -1);
}
private void assertComponentInstancesNotEqual(String mode) throws Exception {
WebClient webClient = new WebClient();
webClient.setThrowExceptionOnFailingStatusCode(true);
TextPage page = webClient.getPage(contextPath + "test?mode=" + mode);
String[] ids = page.getContent().split(":");
System.out.println(Arrays.toString(ids));
assertEquals(ids.length, 2);
assertNotEquals(ids[0], ids[1]);
}
/**
* geOutput
*
* @param pageName
* @return String
* @throws Exception
*/
private String getOutput(String pageName) throws Exception {
URL url = this.getURL(pageName);
WebClient webClient = this.getWebClient();
HtmlPage page = (HtmlPage) webClient.getPage(url);
HtmlElement output = page.getHtmlElementById("output");
return output.asText();
}
@Test(groups = {"contexts"})
public void testConversationPropagationToNonExistentConversationLeadsException()
throws Exception {
WebClient client = new WebClient();
client.setThrowExceptionOnFailingStatusCode(false);
Page page = client.getPage(getPath("/index.jsp"));
assert page.getWebResponse().getStatusCode() == 200;
assert page.getWebResponse().getRequestUrl().toString().contains("home.jsf");
}
@Test
@Ignore
public void test01() throws Exception {
System.out.println("-------------------------------");
WebClient webClient = new WebClient(BrowserVersion.CHROME);
try {
WebClientOptions options = webClient.getOptions();
options.setThrowExceptionOnFailingStatusCode(false);
options.setThrowExceptionOnScriptError(false);
options.setCssEnabled(false);
options.setJavaScriptEnabled(true);
options.setTimeout(50000);
// webClient.setAjaxController(new NicelyResynchronizingAjaxController());
HtmlPage pageOrgin = webClient.getPage("http://lvyou.baidu.com/jinhua/jingdian");
Thread.sleep(5000);
DomNodeList<DomNode> pageNodes = pageOrgin.querySelectorAll(".pagination");
HtmlDivision pageDiv = (HtmlDivision) pageNodes.get(0);
DomNodeList<HtmlElement> liElements = pageDiv.getElementsByTagName("li");
Integer pageSize = liElements.size() - 1;
for (int pageNow = 0; pageNow < pageSize; pageNow++) {
DomNodeList<HtmlElement> pageAnchors = liElements.get(pageNow).getElementsByTagName("a");
HtmlAnchor pageAnchor = (HtmlAnchor) pageAnchors.get(0);
HtmlPage page = pageAnchor.click();
Thread.sleep(10000);
DomElement jViewDom = page.getElementById("J-view-list-container");
DomNodeList<HtmlElement> lis = jViewDom.getElementsByTagName("li");
for (HtmlElement li : lis) {
DomNodeList<DomNode> titleNodes = li.querySelectorAll(".title");
HtmlAnchor titleAnchor = (HtmlAnchor) titleNodes.get(0);
System.out.println("---------------标题----------------");
log.debug("{}", titleAnchor.asText());
DomNodeList<DomNode> picNodes = li.querySelectorAll(".pic");
HtmlAnchor picAnchor = (HtmlAnchor) picNodes.get(0);
System.out.println("---------------详情URL----------------");
String detailUrl = "http://lvyou.baidu.com" + picAnchor.getAttribute("href");
log.debug("{}", detailUrl);
DomNodeList<HtmlElement> imgEelements = picAnchor.getElementsByTagName("img");
for (HtmlElement imgEelement : imgEelements) {
System.out.println("---------------图片----------------");
log.debug("{}", imgEelement.getAttribute("src"));
}
DomNodeList<DomNode> sumNodes = li.querySelectorAll(".view-userSays");
HtmlDivision sumDiv = (HtmlDivision) sumNodes.get(0);
DomNodeList<HtmlElement> sumElements = sumDiv.getElementsByTagName("p");
HtmlParagraph sumPara = (HtmlParagraph) sumElements.get(0);
System.out.println("---------------摘要----------------");
log.debug("{}", sumPara.asText());
}
}
} finally {
webClient.close();
}
System.out.println("-------------------------------");
}
@JsfTest(JsfVersion.JSF_2_2_5)
@Test
public void testSetForEach() throws Exception {
webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
HtmlPage page = webClient.getPage(webUrl + "faces/setForEach.xhtml");
assertEquals(200, page.getWebResponse().getStatusCode());
assertTrue(page.asXml().indexOf("foo") == -1);
assertTrue(page.asXml().indexOf("bar") != -1);
assertTrue(page.asXml().indexOf("baz") != -1);
}
/**
* The request scope is active during the doFilter() method of any Filter in the web application.
*/
@Test
@SpecAssertions({
@SpecAssertion(section = REQUEST_CONTEXT_EE, id = "ab"),
@SpecAssertion(section = REQUEST_CONTEXT_EE, id = "ac")
})
public void testRequestScopeActiveDuringServletFilter() throws Exception {
WebClient webClient = new WebClient();
webClient.setThrowExceptionOnFailingStatusCode(true);
webClient.getPage(contextPath + "SimplePage.html");
}
/**
* Test to validate empty fields.
*
* <p>Note: this test is excluded on Tomcat because the included EL parser requires a System
* property for this test to work, which would cause problems with other tests. See
* http://tomcat.apache.org/tomcat-7.0-doc/config/systemprops.html and look for COERCE_TO_ZERO
*/
@Test
public void testValidateEmptyFields() throws Exception {
HtmlPage page = webClient.getPage(webUrl + "faces/index.xhtml");
if (page.getWebResponse().getResponseHeaderValue("Server") == null
|| !page.getWebResponse().getResponseHeaderValue("Server").startsWith("Apache-Coyote")) {
page = webClient.getPage(webUrl + "faces/validateEmptyFields.xhtml");
HtmlSubmitInput button = (HtmlSubmitInput) page.getElementById("form:submitButton");
page = button.click();
assertTrue(page.asXml().contains("We got called!"));
}
}
@Test
@Ignore
public void test() throws Exception {
System.out.println("-------------------------------");
WebClient webClient = new WebClient(BrowserVersion.CHROME);
webClient.getOptions().setCssEnabled(false);
webClient.getOptions().setJavaScriptEnabled(false);
HtmlPage page = webClient.getPage("http://news.163.com/domestic/");
// DomNodeList<HtmlElement> elements = page.getElementBy
System.out.println("---------------标题----------------");
DomNodeList<DomNode> domNodes = page.querySelectorAll(".item-top");
// log.debug("{}", domNodes);
for (DomNode domNode : domNodes) {
HtmlDivision htmlDivision = (HtmlDivision) domNode;
DomNodeList<HtmlElement> aElements = htmlDivision.getElementsByTagName("a");
HtmlAnchor htmlAnchor = (HtmlAnchor) aElements.get(0);
// HTMLHeadingElement htmlHeading2 = (HTMLHeadingElement)
// htmlDivision.getElementsByTagName("h2");
// HtmlAnchor htmlAnchor = (HtmlAnchor) htmlDivision.getElementsByTagName("a");
log.debug("{}", htmlAnchor.asText());
log.debug("{}", htmlAnchor.getAttribute("href"));
DomNodeList<HtmlElement> pElements = htmlDivision.getElementsByTagName("p");
HtmlParagraph htmlParagraph = (HtmlParagraph) pElements.get(0);
log.debug("{}", htmlParagraph.asText());
DomNodeList<HtmlElement> iEelements = htmlDivision.getElementsByTagName("img");
for (HtmlElement iEelement : iEelements) {
log.debug("{}", iEelement.getAttribute("src"));
}
String detailUrl = htmlAnchor.getAttribute("href");
if (detailUrl.equals("http://news.163.com/15/1215/17/BAT2L8RB00014JB6.html#f=dlist")) {
HtmlPage detailPage = webClient.getPage(detailUrl);
System.out.println("---------------正文----------------");
DomElement endTextElement = detailPage.getElementById("endText");
log.debug("{}", endTextElement.asText());
System.out.println("---------------图片----------------");
DomNodeList<DomNode> imgNodes = endTextElement.querySelectorAll(".f_center");
for (DomNode imgNode : imgNodes) {
HtmlParagraph imgpara = (HtmlParagraph) imgNode;
DomNodeList<HtmlElement> endImgs = imgpara.getElementsByTagName("img");
for (HtmlElement endImg : endImgs) {
log.debug("{}", endImg.getAttribute("src"));
}
}
}
}
webClient.close();
System.out.println("-------------------------------");
}
/**
* This test verifies that an attribute named 'readonly' can be successfully updated from a
* partial response (over Ajax).
*/
@Test
public void testUpdateAttributeNamedReadonly() throws Exception {
HtmlPage page = webClient.getPage(webUrl + "faces/attributeNameIsReadonly.xhtml");
HtmlTextInput input = (HtmlTextInput) page.getElementById("form1:foo");
assertTrue(input.isReadOnly() == false);
HtmlSubmitInput button = (HtmlSubmitInput) page.getElementById("form1:button");
page = button.click();
webClient.waitForBackgroundJavaScript(60000);
input = (HtmlTextInput) page.getElementById("form1:foo");
assertTrue(input.isReadOnly() == true);
}
@Test
public void testLabelAjaxBehavior() throws Exception {
webClient.setJavaScriptEnabled(true);
webClient.setJavaScriptTimeout(60000);
HtmlPage page = webClient.getPage(webUrl + "/faces/label.xhtml");
HtmlElement label = page.getHtmlElementById("label2");
page = (HtmlPage) label.mouseOver();
webClient.waitForBackgroundJavaScript(60000);
assertTrue(page.asXml().contains("label2 Event: begin"));
assertTrue(page.asXml().contains("label2 Event: complete"));
assertTrue(page.asXml().contains("label2 Event: success"));
}
@Test
public void testFieldSetAjaxBehavior() throws Exception {
webClient.setJavaScriptEnabled(true);
webClient.setJavaScriptTimeout(60000);
HtmlPage page = webClient.getPage(webUrl + "/faces/fieldset.xhtml");
HtmlFieldSet fieldset = (HtmlFieldSet) page.getHtmlElementById("fieldset4");
page = fieldset.click();
webClient.waitForBackgroundJavaScript(60000);
assertTrue(page.asXml().contains("fieldset4 Event: begin"));
assertTrue(page.asXml().contains("fieldset4 Event: complete"));
assertTrue(page.asXml().contains("fieldset4 Event: success"));
}