Пример #1
0
 /**
  * Returns the owner element.
  *
  * @return the owner element
  */
 public Object jsxGet_ownerElement() {
   final DomElement parent = getDomNodeOrDie().getOwnerElement();
   if (parent != null) {
     return parent.getScriptObject();
   }
   return null;
 }
 /**
  * @param ticker The ticker of the stock to look up.
  * @throws ConnectionException if it has problems connecting to Yahoo.
  * @return The value of the given stock, as %CompanyName: %Value.
  */
 public String getStockValue(String ticker) throws ConnectionException {
   try {
     final HtmlPage lookup =
         client.getPage(
             new StringBuilder("http://www.google.com/finance?q=").append(ticker).toString());
     final DomElement pricediv =
         lookup.getFirstByXPath("//div[@id = 'price-panel']/div/span/span");
     final DomElement compname =
         lookup.getFirstByXPath("//div[@class = 'appbar-snippet-primary']/span");
     return new StringBuilder(compname.getTextContent())
         .append(": ")
         .append(pricediv.getTextContent())
         .toString();
   } catch (IOException | FailingHttpStatusCodeException e) {
     System.out.println(
         new StringBuilder("Exception in retrieval.")
             .append("InternetConnection.getStockValue(")
             .append(ticker)
             .append(")")
             .toString());
     System.out.println(e.toString());
   }
   throw new ConnectionException(
       new StringBuilder("Unable to find data for ").append(ticker).toString());
 }
Пример #3
0
 @Issue("JENKINS-18641")
 @Test
 public void display() throws Exception {
   User.get("bob");
   JenkinsRule.WebClient wc = j.createWebClient();
   HtmlPage page;
   try {
     page = wc.goTo("asynchPeople");
   } catch (FailingHttpStatusCodeException x) {
     System.err.println(x.getResponse().getResponseHeaders());
     System.err.println(x.getResponse().getContentAsString());
     throw x;
   }
   assertEquals(0, wc.waitForBackgroundJavaScript(120000));
   boolean found = false;
   for (DomElement table : page.getElementsByTagName("table")) {
     if (table.getAttribute("class").contains("progress-bar")) {
       found = true;
       assertEquals("display: none;", table.getAttribute("style"));
       break;
     }
   }
   assertTrue(found);
   /* TODO this still fails occasionally, for reasons TBD (I think because User.getAll sometimes is empty):
   assertNotNull(page.getElementById("person-bob"));
   */
 }
Пример #4
0
 /** Detaches this attribute from the parent HTML element after caching the attribute value. */
 public void detachFromParent() {
   final DomAttr domNode = getDomNodeOrDie();
   final DomElement parent = (DomElement) domNode.getParentNode();
   if (parent != null) {
     domNode.setValue(parent.getAttribute(jsxGet_name()));
   }
   domNode.remove();
 }
Пример #5
0
 /**
  * Returns the previous element sibling.
  *
  * @return the previous element sibling
  */
 @JsxGetter(@WebBrowser(FF))
 public Element getPreviousElementSibling() {
   final DomElement child = getDomNodeOrDie().getPreviousElementSibling();
   if (child != null) {
     return (Element) child.getScriptObject();
   }
   return null;
 }
Пример #6
0
  @Test
  @Ignore
  public void test01() throws Exception {
    System.out.println("-------------------------------");
    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    try {
      WebClientOptions options = webClient.getOptions();
      options.setThrowExceptionOnFailingStatusCode(false);
      options.setThrowExceptionOnScriptError(false);
      options.setCssEnabled(false);
      options.setJavaScriptEnabled(true);
      options.setTimeout(50000);
      // webClient.setAjaxController(new NicelyResynchronizingAjaxController());
      HtmlPage pageOrgin = webClient.getPage("http://lvyou.baidu.com/jinhua/jingdian");
      Thread.sleep(5000);
      DomNodeList<DomNode> pageNodes = pageOrgin.querySelectorAll(".pagination");
      HtmlDivision pageDiv = (HtmlDivision) pageNodes.get(0);
      DomNodeList<HtmlElement> liElements = pageDiv.getElementsByTagName("li");
      Integer pageSize = liElements.size() - 1;
      for (int pageNow = 0; pageNow < pageSize; pageNow++) {
        DomNodeList<HtmlElement> pageAnchors = liElements.get(pageNow).getElementsByTagName("a");
        HtmlAnchor pageAnchor = (HtmlAnchor) pageAnchors.get(0);
        HtmlPage page = pageAnchor.click();
        Thread.sleep(10000);
        DomElement jViewDom = page.getElementById("J-view-list-container");
        DomNodeList<HtmlElement> lis = jViewDom.getElementsByTagName("li");
        for (HtmlElement li : lis) {
          DomNodeList<DomNode> titleNodes = li.querySelectorAll(".title");
          HtmlAnchor titleAnchor = (HtmlAnchor) titleNodes.get(0);
          System.out.println("---------------标题----------------");
          log.debug("{}", titleAnchor.asText());

          DomNodeList<DomNode> picNodes = li.querySelectorAll(".pic");
          HtmlAnchor picAnchor = (HtmlAnchor) picNodes.get(0);
          System.out.println("---------------详情URL----------------");
          String detailUrl = "http://lvyou.baidu.com" + picAnchor.getAttribute("href");
          log.debug("{}", detailUrl);
          DomNodeList<HtmlElement> imgEelements = picAnchor.getElementsByTagName("img");
          for (HtmlElement imgEelement : imgEelements) {
            System.out.println("---------------图片----------------");
            log.debug("{}", imgEelement.getAttribute("src"));
          }

          DomNodeList<DomNode> sumNodes = li.querySelectorAll(".view-userSays");
          HtmlDivision sumDiv = (HtmlDivision) sumNodes.get(0);
          DomNodeList<HtmlElement> sumElements = sumDiv.getElementsByTagName("p");
          HtmlParagraph sumPara = (HtmlParagraph) sumElements.get(0);
          System.out.println("---------------摘要----------------");
          log.debug("{}", sumPara.asText());
        }
      }
    } finally {
      webClient.close();
    }
    System.out.println("-------------------------------");
  }
Пример #7
0
  @Test
  @Ignore
  public void test() throws Exception {
    System.out.println("-------------------------------");

    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setCssEnabled(false);
    webClient.getOptions().setJavaScriptEnabled(false);
    HtmlPage page = webClient.getPage("http://news.163.com/domestic/");
    // DomNodeList<HtmlElement> elements = page.getElementBy

    System.out.println("---------------标题----------------");
    DomNodeList<DomNode> domNodes = page.querySelectorAll(".item-top");
    // log.debug("{}", domNodes);
    for (DomNode domNode : domNodes) {
      HtmlDivision htmlDivision = (HtmlDivision) domNode;
      DomNodeList<HtmlElement> aElements = htmlDivision.getElementsByTagName("a");
      HtmlAnchor htmlAnchor = (HtmlAnchor) aElements.get(0);
      // HTMLHeadingElement htmlHeading2 = (HTMLHeadingElement)
      // htmlDivision.getElementsByTagName("h2");
      // HtmlAnchor htmlAnchor = (HtmlAnchor) htmlDivision.getElementsByTagName("a");
      log.debug("{}", htmlAnchor.asText());
      log.debug("{}", htmlAnchor.getAttribute("href"));

      DomNodeList<HtmlElement> pElements = htmlDivision.getElementsByTagName("p");
      HtmlParagraph htmlParagraph = (HtmlParagraph) pElements.get(0);
      log.debug("{}", htmlParagraph.asText());

      DomNodeList<HtmlElement> iEelements = htmlDivision.getElementsByTagName("img");
      for (HtmlElement iEelement : iEelements) {
        log.debug("{}", iEelement.getAttribute("src"));
      }

      String detailUrl = htmlAnchor.getAttribute("href");
      if (detailUrl.equals("http://news.163.com/15/1215/17/BAT2L8RB00014JB6.html#f=dlist")) {
        HtmlPage detailPage = webClient.getPage(detailUrl);
        System.out.println("---------------正文----------------");
        DomElement endTextElement = detailPage.getElementById("endText");
        log.debug("{}", endTextElement.asText());

        System.out.println("---------------图片----------------");
        DomNodeList<DomNode> imgNodes = endTextElement.querySelectorAll(".f_center");
        for (DomNode imgNode : imgNodes) {
          HtmlParagraph imgpara = (HtmlParagraph) imgNode;
          DomNodeList<HtmlElement> endImgs = imgpara.getElementsByTagName("img");
          for (HtmlElement endImg : endImgs) {
            log.debug("{}", endImg.getAttribute("src"));
          }
        }
      }
    }
    webClient.close();
    System.out.println("-------------------------------");
  }
Пример #8
0
 /*  19:    */ public Page createPage(WebResponse webResponse, WebWindow webWindow)
     /*  20:    */ throws IOException
       /*  21:    */ {
   /*  22:100 */ String contentType =
       determineContentType(
           webResponse.getContentType().toLowerCase(), webResponse.getContentAsStream());
   /*  23:    */
   /*  24:    */
   /*  25:    */
   /*  26:104 */ String pageType = determinePageType(contentType);
   /*  27:    */ Page newPage;
   /*  28:    */ Page newPage;
   /*  29:105 */ if ("html".equals(pageType))
   /*  30:    */ {
     /*  31:106 */ newPage = createHtmlPage(webResponse, webWindow);
     /*  32:    */ }
   /*  33:    */ else
   /*  34:    */ {
     /*  35:    */ Page newPage;
     /*  36:108 */ if ("javascript".equals(pageType))
     /*  37:    */ {
       /*  38:109 */ newPage = createJavaScriptPage(webResponse, webWindow);
       /*  39:    */ }
     /*  40:    */ else
     /*  41:    */ {
       /*  42:    */ Page newPage;
       /*  43:111 */ if ("xml".equals(pageType))
       /*  44:    */ {
         /*  45:112 */ XmlPage xml = createXmlPage(webResponse, webWindow);
         /*  46:113 */ DomElement doc = xml.getDocumentElement();
         /*  47:    */ Page newPage;
         /*  48:114 */ if ((doc != null)
             && ("http://www.w3.org/1999/xhtml".equals(doc.getNamespaceURI()))) {
           /*  49:115 */ newPage = createXHtmlPage(webResponse, webWindow);
           /*  50:    */ } else {
           /*  51:118 */ newPage = xml;
           /*  52:    */ }
         /*  53:    */ }
       /*  54:    */ else
       /*  55:    */ {
         /*  56:    */ Page newPage;
         /*  57:121 */ if ("text".equals(pageType)) {
           /*  58:122 */ newPage = createTextPage(webResponse, webWindow);
           /*  59:    */ } else {
           /*  60:125 */ newPage = createUnexpectedPage(webResponse, webWindow);
           /*  61:    */ }
         /*  62:    */ }
       /*  63:    */ }
     /*  64:    */ }
   /*  65:127 */ return newPage;
   /*  66:    */ }
Пример #9
0
  public static HtmlTable getTableFromDomElement(DomElement element) throws Exception {

    if (element instanceof com.gargoylesoftware.htmlunit.html.HtmlTable) {
      System.out.println("----YES!!!!-----");
      HtmlTable table = (HtmlTable) element;
      return table;
    }

    for (DomElement e : element.getChildElements()) {
      getTableFromDomElement(e);
    }

    throw new Exception("No HTML table found");
  }
Пример #10
0
  public WebForm(DomElement form) {
    this.form = form;
    this.inputs = form.getElementsByTagName("input");
    this.submitField = null;
    this.usernameField = null;
    this.passwordField = null;

    for (HtmlElement input : inputs) {
      String inputType = input.getAttribute("type");

      if (inputType == null) {
        continue;
      } else if (inputType.equals("password")) {
        passwordField = (HtmlPasswordInput) input;
      } else if (inputType.equals("submit")) {
        submitField = (HtmlSubmitInput) input;
      } else if ((input.getId() != null && input.getId().contains("user"))
          || (input.getAttribute("name") != null && input.getAttribute("name").contains("user"))) {
        usernameField = (HtmlInput) input;
      }
    }
  }
Пример #11
0
  @Override
  public HtmlPage getData() {
    // TODO Auto-generated method stub
    MainDataBean mainDataBean =
        (MainDataBean) UserBeanFactory.getBeanFactoryObject().getBean("mainDataBean");
    String URL = mainDataBean.getRedBusUrl();
    URL = "https://www.redbus.in/";
    ClientFactory cf =
        (ClientFactory) UserBeanFactory.getBeanFactoryObject().getBean("clientFactory");
    WebClient client = cf.getWebClient();
    //		try {
    //			client.getOptions().setJavaScriptEnabled(true);
    //			client.getOptions().setThrowExceptionOnScriptError(false);
    //			new InterceptWebConnection(client);
    //			HtmlPage page = client.getPage(URL);
    //			client.waitForBackgroundJavaScript(10000);
    //			System.out.print(page.asXml());
    //		} catch (Exception e) {
    //			// TODO Auto-generated catch block
    //			e.printStackTrace();
    //		}
    String html;
    try {
      //			client.getOptions().setUseInsecureSSL(true);
      //			client.se
      // client.getOptions().setJavaScriptEnabled(true);
      // client.getOptions().setThrowExceptionOnScriptError(false);
      new InterceptWebConnection(client);
      // client.addRequestHeader("Host", "www.redbus.in");
      // client.addRequestHeader("Referer", "https://www.redbus.in/");
      // Booking/SelectBus.aspx?fromCityId=248&toCityId=134&doj=12-Dec-2015&busType=Any&opId=0
      HtmlPage page = client.getPage("https://www.redbus.in/");
      client.waitForBackgroundJavaScript(1000);

      DomElement source = page.getElementById("txtSource");
      DomElement destination = page.getElementById("txtDestination");
      DomElement date = page.getElementById("txtOnwardCalendar");
      DomElement search = page.getElementById("searchBtn");
      //			List<DomElement> inputs = (List<DomElement>)
      // page.getByXPath("//input[@class='XXinput']");
      //			HtmlTextInput source=null,destination=null,date1=null;
      //			for (DomElement domElement : inputs) {
      //				System.out.println("Coming  into sourrce");
      //				if(domElement.getAttribute("id").equals("txtSource")){
      //					System.out.println("Coming  into sourrce");
      //					source = (HtmlTextInput) domElement;
      //				}else if (domElement.getAttribute("id").equals("txtDestination")){
      //					System.out.println("Coming  into destination");
      //					destination =(HtmlTextInput) domElement;
      //				}
      //			}
      //			List<DomElement> Cal_inputs = (List<DomElement>) page.getByXPath("//input[@class='XXinput
      // calendar']");
      //			HtmlElement searchButton = (HtmlElement) page.getFirstByXPath("//button[@class='RB
      // Xbutton']");
      //			for (DomElement domElement : Cal_inputs) {
      //				if(domElement.getAttribute("id").equals("txtOnwardCalendar")){
      //					System.out.println("Coming  into calandar");
      //					date1 =(HtmlTextInput) domElement;
      //				}
      //			}
      source.setAttribute("value", "Visakapatnam");
      // source.setValueAttribute("Visakapatnam");
      destination.setAttribute("value", "Vijayawada");
      // destination.setValueAttribute("Vijayawada");
      date.setAttribute("value", "12-12-2015");

      // date.setValueAttribute("27-10-2015");
      //			client.getOptions().setThrowExceptionOnScriptError(false);
      //			client.getOptions().setThrowExceptionOnFailingStatusCode(false);
      //
      HtmlPage page1 = search.click();
      client.waitForBackgroundJavaScript(200);
      html = page1.asXml();
      System.out.println(html);
    } catch (Exception e) {
      e.printStackTrace();
    }

    return null;
  }
Пример #12
0
  /**
   * Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
   *
   * @param request servlet request
   * @param response servlet response
   * @throws ServletException if a servlet-specific error occurs
   * @throws IOException if an I/O error occurs
   */
  protected void processRequest(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {
    response.setContentType("text/html;charset=UTF-8");
    PrintWriter out = response.getWriter();

    int no = 0;
    try {
      no = Integer.parseInt(request.getParameter("no"));

    } catch (NumberFormatException exception) {
      no = 0;
    }

    try {
      /* TODO output your page here. You may use following sample code. */
      out.println("<!DOCTYPE html>");
      out.println("<html>");
      out.println("<head>");
      out.println("<title>Servlet NewServlet</title>");
      out.println("</head>");
      out.println("<body>");

      final WebClient webClient = new WebClient();
      webClient.getCookieManager().clearCookies();
      webClient.getOptions().setUseInsecureSSL(true);
      webClient.setJavaScriptEnabled(false);
      final HtmlPage page = webClient.getPage("https://filestream.me/");
      // out.println(page.getTitleText());
      System.out.println("no of forms : " + page.getForms().size());
      HtmlForm form = page.getForms().get(1);
      final HtmlTextInput textField = form.getInputByName("login");
      textField.setAttribute("value", "*****@*****.**");

      final HtmlPasswordInput passField = form.getInputByName("password");
      passField.setAttribute("value", "ramkrishnan18");
      HtmlSubmitInput htmlSubmitInput = form.getInputByValue("login");
      HtmlPage page1 = htmlSubmitInput.click();

      HtmlTable htmlTable = page1.getHtmlElementById("fileCatTable");
      List<HtmlTableRow> listOfHtmlTableRow = htmlTable.getBodies().get(0).getRows();
      List<String> allLinks = new ArrayList<String>();
      for (HtmlTableRow htmlTableRow : listOfHtmlTableRow) {
        HtmlTableCell cell = htmlTableRow.getCells().get(htmlTableRow.getCells().size() - 1);
        for (DomElement domElement : cell.getChildElements()) {
          // out.println(domElement.getTagName());
          if (domElement.getTagName().equals("div")) {
            boolean flag = true;

            for (DomElement celldomElement : domElement.getChildElements()) {
              if (flag) {
                // out.println(celldomElement.getTagName());

                String title = celldomElement.getAttribute("title");
                if ("Download".equals(title) || "Downloads".equals(title)) {

                  String link = celldomElement.getAttribute("onclick");
                  String http =
                      link.substring(
                          link.indexOf('\'') + 1, link.indexOf('\'', link.indexOf('\'') + 1));

                  //  out.println(celldomElement.getAttribute("onclick") + "<br/>");
                  // out.println(http + "<br/>");
                  allLinks.add(http);
                  flag = false;
                }
              }
            }
          }
        }
      }

      //  InputStream is =anchorAttachment.click().getWebResponse().getContentAsStream();

      out.println("************ Start *************");

      //  HtmlAnchor anchorElement=HTMLAnchorElement.;
      //  anchorElement.set
      // anchorElement.setHref(allLinks.get(0));

      //  anchorElement.cl
      final String u = allLinks.get(no);
      Thread t =
          new Thread(
              new Runnable() {
                @Override
                public void run() {
                  try {
                    URL url = new URL(u);
                    System.out.println("Link : " + u);
                    HttpURLConnection con = (HttpURLConnection) url.openConnection();

                    // System.out.println("Response Code : " + con.getInputStream().available());
                    con.setRequestProperty("Accept-Encoding", "gzip,deflate");
                    con.setRequestProperty(
                        "User-Agent",
                        "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36");

                    Map<String, List<String>> map = con.getHeaderFields();

                    for (String object : map.keySet()) {
                      System.out.println(object + " --> " + map.get(object));
                    }

                    System.out.println("Response Code : " + con.getResponseCode());
                    System.out.println("length :" + con.getContentLength());
                    System.out.println("Message " + con.getResponseMessage());

                    InputStream inputStream = con.getInputStream();
                    String path = System.getenv("OPENSHIFT_JBOSSAS_DIR");
                    FileOutputStream fileOutputStream = new FileOutputStream(path + "test.zip");
                    byte[] buffer = new byte[1024];
                    System.out.println(inputStream.available());
                    int bytesRead = 10;
                    while (true) {
                      //   System.out.print("--");
                      bytesRead = inputStream.read(buffer);
                      //   System.out.println(bytesRead);
                      if (bytesRead == -1) {
                        break;
                      }
                      fileOutputStream.write(buffer, 0, bytesRead);
                      // System.out.print(">");
                    }
                    fileOutputStream.close();

                    System.out.println("File Completed");
                  } catch (IOException ex) {
                    Logger.getLogger(MakeData.class.getName()).log(Level.SEVERE, null, ex);
                  }
                }
              });
      t.start();

      //            System.out.println(allLinks.get(0));
      //            System.out.println(webClient.getCookieManager().getCookies(new
      // URL(allLinks.get(0))).size());
      //            // System.out.println(httpPage.asText());
      //             HtmlPage httpPage = webClient.getPage(allLinks.get(0));
      //
      //           // final String pageAsXml = page1.asXml();
      //            // InputStream inputStream=httpPage.getWebResponse().getContentAsStream();
      //           //  inputStream.available();
      //              System.out.println("Code : "+httpPage.getWebResponse().getStatusMessage());
      //               System.out.println("Code : "+httpPage.getWebResponse().getContentType());
      //                System.out.println("netCode :
      // "+httpPage.getWebResponse().getContentCharset());
      //
      // out.println("<br/><br/>Code : "+httpPage.getWebResponse().getStatusCode());
      // out.println("<br/><br/>Code : "+httpPage.getWebResponse().getContentType());
      // out.println("<br/><br/>Code : "+httpPage.getWebResponse().getContentCharset());

      // final String pageAsText = page1.asText();

      // out.println(pageAsXml);
      webClient.closeAllWindows();
      out.println("<h1>Servlet NewServlet at " + request.getContextPath() + "</h1>");
      out.println("</body>");
      out.println("</html>");
    } finally {
      out.close();
    }
  }