/**
* Returns the owner element.
*
* @return the owner element
*/
public Object jsxGet_ownerElement() {
final DomElement parent = getDomNodeOrDie().getOwnerElement();
if (parent != null) {
return parent.getScriptObject();
}
return null;
}
/**
* @param ticker The ticker of the stock to look up.
* @throws ConnectionException if it has problems connecting to Yahoo.
* @return The value of the given stock, as %CompanyName: %Value.
*/
public String getStockValue(String ticker) throws ConnectionException {
try {
final HtmlPage lookup =
client.getPage(
new StringBuilder("http://www.google.com/finance?q=").append(ticker).toString());
final DomElement pricediv =
lookup.getFirstByXPath("//div[@id = 'price-panel']/div/span/span");
final DomElement compname =
lookup.getFirstByXPath("//div[@class = 'appbar-snippet-primary']/span");
return new StringBuilder(compname.getTextContent())
.append(": ")
.append(pricediv.getTextContent())
.toString();
} catch (IOException | FailingHttpStatusCodeException e) {
System.out.println(
new StringBuilder("Exception in retrieval.")
.append("InternetConnection.getStockValue(")
.append(ticker)
.append(")")
.toString());
System.out.println(e.toString());
}
throw new ConnectionException(
new StringBuilder("Unable to find data for ").append(ticker).toString());
}
@Issue("JENKINS-18641")
@Test
public void display() throws Exception {
User.get("bob");
JenkinsRule.WebClient wc = j.createWebClient();
HtmlPage page;
try {
page = wc.goTo("asynchPeople");
} catch (FailingHttpStatusCodeException x) {
System.err.println(x.getResponse().getResponseHeaders());
System.err.println(x.getResponse().getContentAsString());
throw x;
}
assertEquals(0, wc.waitForBackgroundJavaScript(120000));
boolean found = false;
for (DomElement table : page.getElementsByTagName("table")) {
if (table.getAttribute("class").contains("progress-bar")) {
found = true;
assertEquals("display: none;", table.getAttribute("style"));
break;
}
}
assertTrue(found);
/* TODO this still fails occasionally, for reasons TBD (I think because User.getAll sometimes is empty):
assertNotNull(page.getElementById("person-bob"));
*/
}
/** Detaches this attribute from the parent HTML element after caching the attribute value. */
public void detachFromParent() {
final DomAttr domNode = getDomNodeOrDie();
final DomElement parent = (DomElement) domNode.getParentNode();
if (parent != null) {
domNode.setValue(parent.getAttribute(jsxGet_name()));
}
domNode.remove();
}
/**
* Returns the previous element sibling.
*
* @return the previous element sibling
*/
@JsxGetter(@WebBrowser(FF))
public Element getPreviousElementSibling() {
final DomElement child = getDomNodeOrDie().getPreviousElementSibling();
if (child != null) {
return (Element) child.getScriptObject();
}
return null;
}
@Test
@Ignore
public void test01() throws Exception {
System.out.println("-------------------------------");
WebClient webClient = new WebClient(BrowserVersion.CHROME);
try {
WebClientOptions options = webClient.getOptions();
options.setThrowExceptionOnFailingStatusCode(false);
options.setThrowExceptionOnScriptError(false);
options.setCssEnabled(false);
options.setJavaScriptEnabled(true);
options.setTimeout(50000);
// webClient.setAjaxController(new NicelyResynchronizingAjaxController());
HtmlPage pageOrgin = webClient.getPage("http://lvyou.baidu.com/jinhua/jingdian");
Thread.sleep(5000);
DomNodeList<DomNode> pageNodes = pageOrgin.querySelectorAll(".pagination");
HtmlDivision pageDiv = (HtmlDivision) pageNodes.get(0);
DomNodeList<HtmlElement> liElements = pageDiv.getElementsByTagName("li");
Integer pageSize = liElements.size() - 1;
for (int pageNow = 0; pageNow < pageSize; pageNow++) {
DomNodeList<HtmlElement> pageAnchors = liElements.get(pageNow).getElementsByTagName("a");
HtmlAnchor pageAnchor = (HtmlAnchor) pageAnchors.get(0);
HtmlPage page = pageAnchor.click();
Thread.sleep(10000);
DomElement jViewDom = page.getElementById("J-view-list-container");
DomNodeList<HtmlElement> lis = jViewDom.getElementsByTagName("li");
for (HtmlElement li : lis) {
DomNodeList<DomNode> titleNodes = li.querySelectorAll(".title");
HtmlAnchor titleAnchor = (HtmlAnchor) titleNodes.get(0);
System.out.println("---------------标题----------------");
log.debug("{}", titleAnchor.asText());
DomNodeList<DomNode> picNodes = li.querySelectorAll(".pic");
HtmlAnchor picAnchor = (HtmlAnchor) picNodes.get(0);
System.out.println("---------------详情URL----------------");
String detailUrl = "http://lvyou.baidu.com" + picAnchor.getAttribute("href");
log.debug("{}", detailUrl);
DomNodeList<HtmlElement> imgEelements = picAnchor.getElementsByTagName("img");
for (HtmlElement imgEelement : imgEelements) {
System.out.println("---------------图片----------------");
log.debug("{}", imgEelement.getAttribute("src"));
}
DomNodeList<DomNode> sumNodes = li.querySelectorAll(".view-userSays");
HtmlDivision sumDiv = (HtmlDivision) sumNodes.get(0);
DomNodeList<HtmlElement> sumElements = sumDiv.getElementsByTagName("p");
HtmlParagraph sumPara = (HtmlParagraph) sumElements.get(0);
System.out.println("---------------摘要----------------");
log.debug("{}", sumPara.asText());
}
}
} finally {
webClient.close();
}
System.out.println("-------------------------------");
}
@Test
@Ignore
public void test() throws Exception {
System.out.println("-------------------------------");
WebClient webClient = new WebClient(BrowserVersion.CHROME);
webClient.getOptions().setCssEnabled(false);
webClient.getOptions().setJavaScriptEnabled(false);
HtmlPage page = webClient.getPage("http://news.163.com/domestic/");
// DomNodeList<HtmlElement> elements = page.getElementBy
System.out.println("---------------标题----------------");
DomNodeList<DomNode> domNodes = page.querySelectorAll(".item-top");
// log.debug("{}", domNodes);
for (DomNode domNode : domNodes) {
HtmlDivision htmlDivision = (HtmlDivision) domNode;
DomNodeList<HtmlElement> aElements = htmlDivision.getElementsByTagName("a");
HtmlAnchor htmlAnchor = (HtmlAnchor) aElements.get(0);
// HTMLHeadingElement htmlHeading2 = (HTMLHeadingElement)
// htmlDivision.getElementsByTagName("h2");
// HtmlAnchor htmlAnchor = (HtmlAnchor) htmlDivision.getElementsByTagName("a");
log.debug("{}", htmlAnchor.asText());
log.debug("{}", htmlAnchor.getAttribute("href"));
DomNodeList<HtmlElement> pElements = htmlDivision.getElementsByTagName("p");
HtmlParagraph htmlParagraph = (HtmlParagraph) pElements.get(0);
log.debug("{}", htmlParagraph.asText());
DomNodeList<HtmlElement> iEelements = htmlDivision.getElementsByTagName("img");
for (HtmlElement iEelement : iEelements) {
log.debug("{}", iEelement.getAttribute("src"));
}
String detailUrl = htmlAnchor.getAttribute("href");
if (detailUrl.equals("http://news.163.com/15/1215/17/BAT2L8RB00014JB6.html#f=dlist")) {
HtmlPage detailPage = webClient.getPage(detailUrl);
System.out.println("---------------正文----------------");
DomElement endTextElement = detailPage.getElementById("endText");
log.debug("{}", endTextElement.asText());
System.out.println("---------------图片----------------");
DomNodeList<DomNode> imgNodes = endTextElement.querySelectorAll(".f_center");
for (DomNode imgNode : imgNodes) {
HtmlParagraph imgpara = (HtmlParagraph) imgNode;
DomNodeList<HtmlElement> endImgs = imgpara.getElementsByTagName("img");
for (HtmlElement endImg : endImgs) {
log.debug("{}", endImg.getAttribute("src"));
}
}
}
}
webClient.close();
System.out.println("-------------------------------");
}
/* 19: */ public Page createPage(WebResponse webResponse, WebWindow webWindow)
/* 20: */ throws IOException
/* 21: */ {
/* 22:100 */ String contentType =
determineContentType(
webResponse.getContentType().toLowerCase(), webResponse.getContentAsStream());
/* 23: */
/* 24: */
/* 25: */
/* 26:104 */ String pageType = determinePageType(contentType);
/* 27: */ Page newPage;
/* 28: */ Page newPage;
/* 29:105 */ if ("html".equals(pageType))
/* 30: */ {
/* 31:106 */ newPage = createHtmlPage(webResponse, webWindow);
/* 32: */ }
/* 33: */ else
/* 34: */ {
/* 35: */ Page newPage;
/* 36:108 */ if ("javascript".equals(pageType))
/* 37: */ {
/* 38:109 */ newPage = createJavaScriptPage(webResponse, webWindow);
/* 39: */ }
/* 40: */ else
/* 41: */ {
/* 42: */ Page newPage;
/* 43:111 */ if ("xml".equals(pageType))
/* 44: */ {
/* 45:112 */ XmlPage xml = createXmlPage(webResponse, webWindow);
/* 46:113 */ DomElement doc = xml.getDocumentElement();
/* 47: */ Page newPage;
/* 48:114 */ if ((doc != null)
&& ("http://www.w3.org/1999/xhtml".equals(doc.getNamespaceURI()))) {
/* 49:115 */ newPage = createXHtmlPage(webResponse, webWindow);
/* 50: */ } else {
/* 51:118 */ newPage = xml;
/* 52: */ }
/* 53: */ }
/* 54: */ else
/* 55: */ {
/* 56: */ Page newPage;
/* 57:121 */ if ("text".equals(pageType)) {
/* 58:122 */ newPage = createTextPage(webResponse, webWindow);
/* 59: */ } else {
/* 60:125 */ newPage = createUnexpectedPage(webResponse, webWindow);
/* 61: */ }
/* 62: */ }
/* 63: */ }
/* 64: */ }
/* 65:127 */ return newPage;
/* 66: */ }
public static HtmlTable getTableFromDomElement(DomElement element) throws Exception {
if (element instanceof com.gargoylesoftware.htmlunit.html.HtmlTable) {
System.out.println("----YES!!!!-----");
HtmlTable table = (HtmlTable) element;
return table;
}
for (DomElement e : element.getChildElements()) {
getTableFromDomElement(e);
}
throw new Exception("No HTML table found");
}
public WebForm(DomElement form) {
this.form = form;
this.inputs = form.getElementsByTagName("input");
this.submitField = null;
this.usernameField = null;
this.passwordField = null;
for (HtmlElement input : inputs) {
String inputType = input.getAttribute("type");
if (inputType == null) {
continue;
} else if (inputType.equals("password")) {
passwordField = (HtmlPasswordInput) input;
} else if (inputType.equals("submit")) {
submitField = (HtmlSubmitInput) input;
} else if ((input.getId() != null && input.getId().contains("user"))
|| (input.getAttribute("name") != null && input.getAttribute("name").contains("user"))) {
usernameField = (HtmlInput) input;
}
}
}
@Override
public HtmlPage getData() {
// TODO Auto-generated method stub
MainDataBean mainDataBean =
(MainDataBean) UserBeanFactory.getBeanFactoryObject().getBean("mainDataBean");
String URL = mainDataBean.getRedBusUrl();
URL = "https://www.redbus.in/";
ClientFactory cf =
(ClientFactory) UserBeanFactory.getBeanFactoryObject().getBean("clientFactory");
WebClient client = cf.getWebClient();
// try {
// client.getOptions().setJavaScriptEnabled(true);
// client.getOptions().setThrowExceptionOnScriptError(false);
// new InterceptWebConnection(client);
// HtmlPage page = client.getPage(URL);
// client.waitForBackgroundJavaScript(10000);
// System.out.print(page.asXml());
// } catch (Exception e) {
// // TODO Auto-generated catch block
// e.printStackTrace();
// }
String html;
try {
// client.getOptions().setUseInsecureSSL(true);
// client.se
// client.getOptions().setJavaScriptEnabled(true);
// client.getOptions().setThrowExceptionOnScriptError(false);
new InterceptWebConnection(client);
// client.addRequestHeader("Host", "www.redbus.in");
// client.addRequestHeader("Referer", "https://www.redbus.in/");
// Booking/SelectBus.aspx?fromCityId=248&toCityId=134&doj=12-Dec-2015&busType=Any&opId=0
HtmlPage page = client.getPage("https://www.redbus.in/");
client.waitForBackgroundJavaScript(1000);
DomElement source = page.getElementById("txtSource");
DomElement destination = page.getElementById("txtDestination");
DomElement date = page.getElementById("txtOnwardCalendar");
DomElement search = page.getElementById("searchBtn");
// List<DomElement> inputs = (List<DomElement>)
// page.getByXPath("//input[@class='XXinput']");
// HtmlTextInput source=null,destination=null,date1=null;
// for (DomElement domElement : inputs) {
// System.out.println("Coming into sourrce");
// if(domElement.getAttribute("id").equals("txtSource")){
// System.out.println("Coming into sourrce");
// source = (HtmlTextInput) domElement;
// }else if (domElement.getAttribute("id").equals("txtDestination")){
// System.out.println("Coming into destination");
// destination =(HtmlTextInput) domElement;
// }
// }
// List<DomElement> Cal_inputs = (List<DomElement>) page.getByXPath("//input[@class='XXinput
// calendar']");
// HtmlElement searchButton = (HtmlElement) page.getFirstByXPath("//button[@class='RB
// Xbutton']");
// for (DomElement domElement : Cal_inputs) {
// if(domElement.getAttribute("id").equals("txtOnwardCalendar")){
// System.out.println("Coming into calandar");
// date1 =(HtmlTextInput) domElement;
// }
// }
source.setAttribute("value", "Visakapatnam");
// source.setValueAttribute("Visakapatnam");
destination.setAttribute("value", "Vijayawada");
// destination.setValueAttribute("Vijayawada");
date.setAttribute("value", "12-12-2015");
// date.setValueAttribute("27-10-2015");
// client.getOptions().setThrowExceptionOnScriptError(false);
// client.getOptions().setThrowExceptionOnFailingStatusCode(false);
//
HtmlPage page1 = search.click();
client.waitForBackgroundJavaScript(200);
html = page1.asXml();
System.out.println(html);
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
*
* @param request servlet request
* @param response servlet response
* @throws ServletException if a servlet-specific error occurs
* @throws IOException if an I/O error occurs
*/
protected void processRequest(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
response.setContentType("text/html;charset=UTF-8");
PrintWriter out = response.getWriter();
int no = 0;
try {
no = Integer.parseInt(request.getParameter("no"));
} catch (NumberFormatException exception) {
no = 0;
}
try {
/* TODO output your page here. You may use following sample code. */
out.println("<!DOCTYPE html>");
out.println("<html>");
out.println("<head>");
out.println("<title>Servlet NewServlet</title>");
out.println("</head>");
out.println("<body>");
final WebClient webClient = new WebClient();
webClient.getCookieManager().clearCookies();
webClient.getOptions().setUseInsecureSSL(true);
webClient.setJavaScriptEnabled(false);
final HtmlPage page = webClient.getPage("https://filestream.me/");
// out.println(page.getTitleText());
System.out.println("no of forms : " + page.getForms().size());
HtmlForm form = page.getForms().get(1);
final HtmlTextInput textField = form.getInputByName("login");
textField.setAttribute("value", "*****@*****.**");
final HtmlPasswordInput passField = form.getInputByName("password");
passField.setAttribute("value", "ramkrishnan18");
HtmlSubmitInput htmlSubmitInput = form.getInputByValue("login");
HtmlPage page1 = htmlSubmitInput.click();
HtmlTable htmlTable = page1.getHtmlElementById("fileCatTable");
List<HtmlTableRow> listOfHtmlTableRow = htmlTable.getBodies().get(0).getRows();
List<String> allLinks = new ArrayList<String>();
for (HtmlTableRow htmlTableRow : listOfHtmlTableRow) {
HtmlTableCell cell = htmlTableRow.getCells().get(htmlTableRow.getCells().size() - 1);
for (DomElement domElement : cell.getChildElements()) {
// out.println(domElement.getTagName());
if (domElement.getTagName().equals("div")) {
boolean flag = true;
for (DomElement celldomElement : domElement.getChildElements()) {
if (flag) {
// out.println(celldomElement.getTagName());
String title = celldomElement.getAttribute("title");
if ("Download".equals(title) || "Downloads".equals(title)) {
String link = celldomElement.getAttribute("onclick");
String http =
link.substring(
link.indexOf('\'') + 1, link.indexOf('\'', link.indexOf('\'') + 1));
// out.println(celldomElement.getAttribute("onclick") + "<br/>");
// out.println(http + "<br/>");
allLinks.add(http);
flag = false;
}
}
}
}
}
}
// InputStream is =anchorAttachment.click().getWebResponse().getContentAsStream();
out.println("************ Start *************");
// HtmlAnchor anchorElement=HTMLAnchorElement.;
// anchorElement.set
// anchorElement.setHref(allLinks.get(0));
// anchorElement.cl
final String u = allLinks.get(no);
Thread t =
new Thread(
new Runnable() {
@Override
public void run() {
try {
URL url = new URL(u);
System.out.println("Link : " + u);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
// System.out.println("Response Code : " + con.getInputStream().available());
con.setRequestProperty("Accept-Encoding", "gzip,deflate");
con.setRequestProperty(
"User-Agent",
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36");
Map<String, List<String>> map = con.getHeaderFields();
for (String object : map.keySet()) {
System.out.println(object + " --> " + map.get(object));
}
System.out.println("Response Code : " + con.getResponseCode());
System.out.println("length :" + con.getContentLength());
System.out.println("Message " + con.getResponseMessage());
InputStream inputStream = con.getInputStream();
String path = System.getenv("OPENSHIFT_JBOSSAS_DIR");
FileOutputStream fileOutputStream = new FileOutputStream(path + "test.zip");
byte[] buffer = new byte[1024];
System.out.println(inputStream.available());
int bytesRead = 10;
while (true) {
// System.out.print("--");
bytesRead = inputStream.read(buffer);
// System.out.println(bytesRead);
if (bytesRead == -1) {
break;
}
fileOutputStream.write(buffer, 0, bytesRead);
// System.out.print(">");
}
fileOutputStream.close();
System.out.println("File Completed");
} catch (IOException ex) {
Logger.getLogger(MakeData.class.getName()).log(Level.SEVERE, null, ex);
}
}
});
t.start();
// System.out.println(allLinks.get(0));
// System.out.println(webClient.getCookieManager().getCookies(new
// URL(allLinks.get(0))).size());
// // System.out.println(httpPage.asText());
// HtmlPage httpPage = webClient.getPage(allLinks.get(0));
//
// // final String pageAsXml = page1.asXml();
// // InputStream inputStream=httpPage.getWebResponse().getContentAsStream();
// // inputStream.available();
// System.out.println("Code : "+httpPage.getWebResponse().getStatusMessage());
// System.out.println("Code : "+httpPage.getWebResponse().getContentType());
// System.out.println("netCode :
// "+httpPage.getWebResponse().getContentCharset());
//
// out.println("<br/><br/>Code : "+httpPage.getWebResponse().getStatusCode());
// out.println("<br/><br/>Code : "+httpPage.getWebResponse().getContentType());
// out.println("<br/><br/>Code : "+httpPage.getWebResponse().getContentCharset());
// final String pageAsText = page1.asText();
// out.println(pageAsXml);
webClient.closeAllWindows();
out.println("<h1>Servlet NewServlet at " + request.getContextPath() + "</h1>");
out.println("</body>");
out.println("</html>");
} finally {
out.close();
}
}