@Override
protected SkypeGuestTempSession run(WebClient webClient) throws Exception {
webClient.getPage(joinUrl);
String csrf = webClient.getCookieManager().getCookie("csrf_token").getValue();
String sessionId = webClient.getCookieManager().getCookie("launcher_session_id").getValue();
return new SkypeGuestTempSession(csrf, sessionId);
}
@JsfTest(JsfVersion.JSF_2_2_5)
@Test
public void testDropFlashCookie() throws Exception {
HtmlPage page = webClient.getPage(webUrl + "faces/flashDropCookie.xhtml");
webClient.getOptions().setRedirectEnabled(true);
HtmlTextInput textInput = (HtmlTextInput) page.getHtmlElementById("input");
textInput.setValueAttribute("test");
HtmlSubmitInput button = (HtmlSubmitInput) page.getHtmlElementById("submit");
int currentSize = webClient.getCookieManager().getCookies().size();
page = button.click();
HtmlElement element = page.getHtmlElementById("link");
page = element.click();
int newSize = webClient.getCookieManager().getCookies().size();
assertTrue(newSize < currentSize);
}
/**
* Static method for <code>Login</code>. Uses dependencies <b>HttpUnit</b> in connecting to Keats.
*
* @see com.gargoylesoftware.htmlunit
* @param parent The parent window. instanceof<code>Scrape</code>, to set relative locations to
* @param link The url of which to retrieve information from.
* @param username The username to log in KEATS with.
* @param password The password to log in KEATS with.
* @return results.asText() Returns the content of the url if login successful. Returns null
* otherwise.
*/
public static String login(Scrape parent, String link, String username, String password) {
try {
WebClient client = new WebClient();
// Settings
client.getOptions().setThrowExceptionOnScriptError(false);
client.getOptions().setThrowExceptionOnScriptError(false);
client.getOptions().setThrowExceptionOnFailingStatusCode(false);
client.getOptions().setJavaScriptEnabled(false);
client.getOptions().setCssEnabled(false);
client.getOptions().setRedirectEnabled(true);
client.getOptions().setUseInsecureSSL(true);
client.getCookieManager().setCookiesEnabled(true);
HtmlPage page = client.getPage("https://login-keats.kcl.ac.uk/");
HtmlForm form =
page.getFirstByXPath("//form[@action='https://keats.kcl.ac.uk/login/index.php']");
HtmlInput usernameInput = form.getInputByName("username");
usernameInput.setValueAttribute(username);
HtmlInput passwordInput = form.getInputByName("password");
passwordInput.setValueAttribute(password);
page = form.getInputByValue("Log in").click();
HtmlPage results = client.getPage(link);
client.closeAllWindows();
return results.asText();
} catch (MalformedURLException e) {
JOptionPane.showMessageDialog(
parent,
"The URL you have provided is not recognised. Please double check your "
+ "input and try again.",
"MalformedURLException found.",
JOptionPane.ERROR_MESSAGE);
e.printStackTrace();
} catch (IOException e) {
JOptionPane.showMessageDialog(
parent,
"An error has occurred when attempting to read information from the "
+ "server. The input could be interrupted by external processes.",
"IOException found.",
JOptionPane.ERROR_MESSAGE);
e.printStackTrace();
}
return null;
}
public void discoverCookies(WebClient client, String url) {
HashSet<Cookie> foundCookies = new HashSet<Cookie>();
Set<Cookie> cookies = client.getCookieManager().getCookies();
if (cookies != null && cookies.size() > 0) {
foundCookies.addAll(cookies);
}
System.out.println("--------------------------------");
System.out.println("Cookies Found");
for (Cookie cookie : foundCookies) {
System.out.println(cookie.toString());
}
}
public AppAnnieCrawlerJava(String application, String store, String rankType, AuthObject auth)
throws ParsingException {
url =
"http://www.appannie.com/app/"
+ (store.equals("appstore") ? "ios" : store)
+ "/"
+ application.replaceAll(" ", "-").toLowerCase()
+ "/ranking/#view="
+ rankType
+ "&date=";
WebClient webClient = buildWebClient();
HtmlPage page = null;
try {
page = webClient.getPage("https://www.appannie.com/account/login/");
} catch (IOException e) {
e.printStackTrace(); // todo: throw exception
}
HtmlForm form = page.getFormByName("");
// authentication
try {
form.getInputByName("username").setValueAttribute(auth.username());
} catch (Throwable t) {
throw new ParsingException(
"Login form incorrect! Please check " + getClass().getName() + " for errors.");
}
try {
form.getInputByName("password").setValueAttribute(auth.password());
} catch (Throwable t) {
throw new ParsingException(
"Login form incorrect! Please check " + getClass().getName() + " for errors.");
}
try {
form.getButtonByName("").click();
} catch (IOException e) {
throw new ParsingException(
"Login form incorrect! Please check " + getClass().getName() + " for errors.");
}
cookieManager = webClient.getCookieManager();
}
/** Validates case sensitivity for resource name. */
@Test(
groups = {
"ldapv3",
"ldapv3_sec",
"s1ds",
"s1ds_sec",
"ad",
"ad_sec",
"amsdk",
"amsdk_sec",
"jdbc",
"jdbc_sec"
})
public void evaluateCaseSensitive() throws Exception {
entering("evaluateCaseSensitive", null);
webClient = new WebClient();
webClient.getCookieManager().setCookiesEnabled(true);
URL urlLoc = new URL(resourceCase);
try {
page = consoleLogin(webClient, resourceProtected, "generalagenttests", "generalagenttests");
iIdx = -1;
iIdx = getHtmlPageStringIndex(page, "Allow Page");
assert (iIdx != -1);
log(Level.SEVERE, "evaluateCaseSensitive", "resourceCase:" + resourceCase);
page = (HtmlPage) webClient.getPage(urlLoc);
iIdx = -1;
iIdx = getHtmlPageStringIndex(page, "Access Denied");
assert (iIdx != -1);
} catch (com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException ee) {
} catch (Exception e) {
log(Level.SEVERE, "evaluateCaseSensitive", e.getMessage());
e.printStackTrace();
throw e;
} finally {
consoleLogout(webClient, logoutURL);
}
exiting("evaluateCaseSensitive");
}
/**
* Processes requests for both HTTP <code>GET</code> and <code>POST</code> methods.
*
* @param request servlet request
* @param response servlet response
* @throws ServletException if a servlet-specific error occurs
* @throws IOException if an I/O error occurs
*/
protected void processRequest(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
response.setContentType("text/html;charset=UTF-8");
PrintWriter out = response.getWriter();
int no = 0;
try {
no = Integer.parseInt(request.getParameter("no"));
} catch (NumberFormatException exception) {
no = 0;
}
try {
/* TODO output your page here. You may use following sample code. */
out.println("<!DOCTYPE html>");
out.println("<html>");
out.println("<head>");
out.println("<title>Servlet NewServlet</title>");
out.println("</head>");
out.println("<body>");
final WebClient webClient = new WebClient();
webClient.getCookieManager().clearCookies();
webClient.getOptions().setUseInsecureSSL(true);
webClient.setJavaScriptEnabled(false);
final HtmlPage page = webClient.getPage("https://filestream.me/");
// out.println(page.getTitleText());
System.out.println("no of forms : " + page.getForms().size());
HtmlForm form = page.getForms().get(1);
final HtmlTextInput textField = form.getInputByName("login");
textField.setAttribute("value", "*****@*****.**");
final HtmlPasswordInput passField = form.getInputByName("password");
passField.setAttribute("value", "ramkrishnan18");
HtmlSubmitInput htmlSubmitInput = form.getInputByValue("login");
HtmlPage page1 = htmlSubmitInput.click();
HtmlTable htmlTable = page1.getHtmlElementById("fileCatTable");
List<HtmlTableRow> listOfHtmlTableRow = htmlTable.getBodies().get(0).getRows();
List<String> allLinks = new ArrayList<String>();
for (HtmlTableRow htmlTableRow : listOfHtmlTableRow) {
HtmlTableCell cell = htmlTableRow.getCells().get(htmlTableRow.getCells().size() - 1);
for (DomElement domElement : cell.getChildElements()) {
// out.println(domElement.getTagName());
if (domElement.getTagName().equals("div")) {
boolean flag = true;
for (DomElement celldomElement : domElement.getChildElements()) {
if (flag) {
// out.println(celldomElement.getTagName());
String title = celldomElement.getAttribute("title");
if ("Download".equals(title) || "Downloads".equals(title)) {
String link = celldomElement.getAttribute("onclick");
String http =
link.substring(
link.indexOf('\'') + 1, link.indexOf('\'', link.indexOf('\'') + 1));
// out.println(celldomElement.getAttribute("onclick") + "<br/>");
// out.println(http + "<br/>");
allLinks.add(http);
flag = false;
}
}
}
}
}
}
// InputStream is =anchorAttachment.click().getWebResponse().getContentAsStream();
out.println("************ Start *************");
// HtmlAnchor anchorElement=HTMLAnchorElement.;
// anchorElement.set
// anchorElement.setHref(allLinks.get(0));
// anchorElement.cl
final String u = allLinks.get(no);
Thread t =
new Thread(
new Runnable() {
@Override
public void run() {
try {
URL url = new URL(u);
System.out.println("Link : " + u);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
// System.out.println("Response Code : " + con.getInputStream().available());
con.setRequestProperty("Accept-Encoding", "gzip,deflate");
con.setRequestProperty(
"User-Agent",
"Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36");
Map<String, List<String>> map = con.getHeaderFields();
for (String object : map.keySet()) {
System.out.println(object + " --> " + map.get(object));
}
System.out.println("Response Code : " + con.getResponseCode());
System.out.println("length :" + con.getContentLength());
System.out.println("Message " + con.getResponseMessage());
InputStream inputStream = con.getInputStream();
String path = System.getenv("OPENSHIFT_JBOSSAS_DIR");
FileOutputStream fileOutputStream = new FileOutputStream(path + "test.zip");
byte[] buffer = new byte[1024];
System.out.println(inputStream.available());
int bytesRead = 10;
while (true) {
// System.out.print("--");
bytesRead = inputStream.read(buffer);
// System.out.println(bytesRead);
if (bytesRead == -1) {
break;
}
fileOutputStream.write(buffer, 0, bytesRead);
// System.out.print(">");
}
fileOutputStream.close();
System.out.println("File Completed");
} catch (IOException ex) {
Logger.getLogger(MakeData.class.getName()).log(Level.SEVERE, null, ex);
}
}
});
t.start();
// System.out.println(allLinks.get(0));
// System.out.println(webClient.getCookieManager().getCookies(new
// URL(allLinks.get(0))).size());
// // System.out.println(httpPage.asText());
// HtmlPage httpPage = webClient.getPage(allLinks.get(0));
//
// // final String pageAsXml = page1.asXml();
// // InputStream inputStream=httpPage.getWebResponse().getContentAsStream();
// // inputStream.available();
// System.out.println("Code : "+httpPage.getWebResponse().getStatusMessage());
// System.out.println("Code : "+httpPage.getWebResponse().getContentType());
// System.out.println("netCode :
// "+httpPage.getWebResponse().getContentCharset());
//
// out.println("<br/><br/>Code : "+httpPage.getWebResponse().getStatusCode());
// out.println("<br/><br/>Code : "+httpPage.getWebResponse().getContentType());
// out.println("<br/><br/>Code : "+httpPage.getWebResponse().getContentCharset());
// final String pageAsText = page1.asText();
// out.println(pageAsXml);
webClient.closeAllWindows();
out.println("<h1>Servlet NewServlet at " + request.getContextPath() + "</h1>");
out.println("</body>");
out.println("</html>");
} finally {
out.close();
}
}