/** * The method makes a GetMethod object and an HttpClient object. The HttpClient then executes the * Getmethod and returns the pagesource to the caller. * * @param url Url to fetch the pagesource for * @param followRedirect Boolean variable to specify GetMethod followRedirect value * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value * @return String */ public String getPageSourceWithoutProxy( String url, boolean followRedirects, boolean doAuthentication) { GetMethod getMethod = null; String pageSouce = null; HttpClient httpClient = new HttpClient(); try { getMethod = new GetMethod(url); getMethod.setFollowRedirects(followRedirects); getMethod.setDoAuthentication(doAuthentication); httpClient.executeMethod(getMethod); pageSouce = getMethod.getResponseBodyAsString(); } catch (Exception e) { l.error(e + " " + e.getMessage() + "exception occured for url" + url); try { getMethod = new GetMethod(url); getMethod.setFollowRedirects(followRedirects); getMethod.setDoAuthentication(doAuthentication); httpClient.executeMethod(getMethod); pageSouce = getMethod.getResponseBodyAsString(); getMethod.releaseConnection(); } catch (Exception ex) { l.error(ex + " " + ex.getMessage() + "exception occured for url " + url); getMethod.releaseConnection(); } } finally { getMethod.releaseConnection(); } return pageSouce; }
private void doGet( String realm, String host, String user, String pass, String url, boolean handshake, boolean preemtive, int result) throws Exception { HttpClient client = new HttpClient(); client.getParams().setAuthenticationPreemptive(preemtive); client .getState() .setCredentials( new AuthScope(host, -1, realm), new UsernamePasswordCredentials(user, pass)); GetMethod get = new GetMethod(url); get.setDoAuthentication(handshake); try { int status = client.executeMethod(get); assertEquals(result, status); } finally { get.releaseConnection(); } }
/** * Download a page from the server and return its content. Throws a {@link RuntimeException} on * connection problems etc. * * @param url URL of the page * @return content of the page */ protected static InputStream getUrlContent(String url) { GetMethod get = new GetMethod(url); get.setFollowRedirects(true); if (isLoggedIn()) { get.setDoAuthentication(true); get.addRequestHeader( "Authorization", "Basic " + new String(Base64.encodeBase64("Admin:admin".getBytes()))); } try { int statusCode = AbstractEscapingTest.getClient().executeMethod(get); switch (statusCode) { case HttpStatus.SC_OK: // everything is fine break; case HttpStatus.SC_UNAUTHORIZED: // do not fail on 401 (unauthorized), used in some tests System.out.println("WARNING, Ignoring status 401 (unauthorized) for URL: " + url); break; case HttpStatus.SC_CONFLICT: // do not fail on 409 (conflict), used in some templates System.out.println("WARNING, Ignoring status 409 (conflict) for URL: " + url); break; case HttpStatus.SC_NOT_FOUND: // ignore 404 (the page is still rendered) break; default: throw new RuntimeException( "HTTP GET request returned status " + statusCode + " (" + get.getStatusText() + ") for URL: " + url); } // get the data, converting to utf-8 String str = get.getResponseBodyAsString(); if (str == null) { return null; } return new ByteArrayInputStream(str.getBytes("utf-8")); } catch (IOException exception) { throw new RuntimeException("Error retrieving URL: " + url + "\n" + exception); } finally { get.releaseConnection(); } }
private Crumb getCsrfCrumb(HttpClient client, Candidate target) throws IOException { GetMethod httpGet = new GetMethod( target.url + "crumbIssuer/api/xml?xpath=" + URLEncoder.encode("concat(//crumbRequestField,\":\",//crumb)", "UTF-8")); httpGet.setDoAuthentication(true); int responseCode = client.executeMethod(httpGet); if (responseCode != HttpStatus.SC_OK) { System.out.println("Could not obtain CSRF crumb. Response code: " + responseCode); return null; } String[] crumbResponse = httpGet.getResponseBodyAsString().split(":"); if (crumbResponse.length != 2) { System.out.println("Unexpected CSRF crumb response: " + httpGet.getResponseBodyAsString()); return null; } return new Crumb(crumbResponse[0], crumbResponse[1]); }
public void testAuthenticationFailureNoContextGet() throws Exception { HttpClient client = new HttpClient(); client.getParams().setAuthenticationPreemptive(true); GetMethod get = new GetMethod("http://localhost:60443/services/Echo"); get.setDoAuthentication(false); try { int status = client.executeMethod(get); assertEquals(HttpConstants.SC_UNAUTHORIZED, status); assertEquals( "Registered authentication is set to org.mule.module.acegi.filters.http.HttpBasicAuthenticationFilter " + "but there was no security context on the session. Authentication denied on " + "endpoint http://localhost:60443/services/Echo. Message payload is of type: " + "String", get.getResponseBodyAsString()); } finally { get.releaseConnection(); } }
public static void sendGetTransStatusRequest(String urlString, String authentication) throws Exception { GetMethod method = new GetMethod(urlString); method.setDoAuthentication(true); // adding authorization token if (authentication != null) { method.addRequestHeader(new Header("Authorization", authentication)); } // executing method HttpClient client = new HttpClient(); int code = client.executeMethod(method); String response = method.getResponseBodyAsString(); method.releaseConnection(); if (code >= 400) { System.out.println("Error occurred during getting job status."); } System.out.println("Server response:"); System.out.println(response); }
public GeoRSSReader createReader(final URL url, final String username, final String password) throws IOException { if (log.isDebugEnabled()) { log.debug( "Creating GeoRSS reader for URL " + url.toExternalForm() + " with user " + username); } HttpClientBuilder builder = new HttpClientBuilder(); builder.setHttpCredentials(username, password, url); builder.setBackendTimeout(120); HttpClient httpClient = builder.buildClient(); GetMethod getMethod = new GetMethod(url.toString()); getMethod.setRequestHeader("Connection", "close"); if (builder.isDoAuthentication()) { getMethod.setDoAuthentication(true); httpClient.getParams().setAuthenticationPreemptive(true); } if (log.isDebugEnabled()) { log.debug("Executing HTTP GET requesr for feed URL " + url.toExternalForm()); } httpClient.executeMethod(getMethod); if (log.isDebugEnabled()) { log.debug("Building GeoRSS reader out of URL response"); } String contentEncoding = getMethod.getResponseCharSet(); if (contentEncoding == null) { contentEncoding = "UTF-8"; } InputStream in = getMethod.getResponseBodyAsStream(); Reader reader = new BufferedReader(new InputStreamReader(in, contentEncoding)); if (log.isDebugEnabled()) { log.debug("GeoRSS reader created, returning."); } return createReader(reader); }
public Candidate discoverFromMasterUrl() throws IOException, ParserConfigurationException, RetryException { if (!options.master.endsWith("/")) { options.master += "/"; } URL masterURL; try { masterURL = new URL(options.master); } catch (MalformedURLException e) { throw new RuntimeException( MessageFormat.format("The master URL \"{0}\" is invalid", options.master), e); } HttpClient client = createHttpClient(masterURL); String url = masterURL.toExternalForm() + "plugin/swarm/slaveInfo"; GetMethod get = new GetMethod(url); get.setDoAuthentication(true); get.addRequestHeader("Connection", "close"); int responseCode = client.executeMethod(get); if (responseCode != 200) { throw new RetryException("Failed to fetch slave info from Jenkins CODE: " + responseCode); } Document xml; try { xml = DocumentBuilderFactory.newInstance() .newDocumentBuilder() .parse(get.getResponseBodyAsStream()); } catch (SAXException e) { throw new RetryException("Invalid XML received from " + url); } String swarmSecret = getChildElementString(xml.getDocumentElement(), "swarmSecret"); return new Candidate(masterURL.toExternalForm(), swarmSecret); }
protected GetMethod createGetMethod(final String url, final boolean authenticate) { GetMethod get = new GetMethod(url); get.setDoAuthentication(authenticate); return get; }
/** * Retrieve the service document. The service document is located at the specified URL. This calls * getServiceDocument(url,onBehalfOf). * * @param url The location of the service document. * @return The ServiceDocument, or <code>null</code> if there was a problem accessing the * document. e.g. invalid access. * @throws SWORDClientException If there is an error accessing the resource. */ public ServiceDocument getServiceDocument(String url, String onBehalfOf) throws SWORDClientException { URL serviceDocURL = null; try { serviceDocURL = new URL(url); } catch (MalformedURLException e) { // Try relative URL URL baseURL = null; try { baseURL = new URL("http", server, Integer.valueOf(port), "/"); serviceDocURL = new URL(baseURL, (url == null) ? "" : url); } catch (MalformedURLException e1) { // No dice, can't even form base URL... throw new SWORDClientException( url + " is not a valid URL (" + e1.getMessage() + "), and could not form a relative one from: " + baseURL + " / " + url, e1); } } GetMethod httpget = new GetMethod(serviceDocURL.toExternalForm()); if (doAuthentication) { // this does not perform any check on the username password. It // relies on the server to determine if the values are correct. setBasicCredentials(username, password); httpget.setDoAuthentication(true); } Properties properties = new Properties(); if (containsValue(onBehalfOf)) { log.debug("Setting on-behalf-of: " + onBehalfOf); httpget.addRequestHeader(new Header(HttpHeaders.X_ON_BEHALF_OF, onBehalfOf)); properties.put(HttpHeaders.X_ON_BEHALF_OF, onBehalfOf); } if (containsValue(userAgent)) { log.debug("Setting userAgent: " + userAgent); httpget.addRequestHeader(new Header(HttpHeaders.USER_AGENT, userAgent)); properties.put(HttpHeaders.USER_AGENT, userAgent); } ServiceDocument doc = null; try { client.executeMethod(httpget); // store the status code status = new Status(httpget.getStatusCode(), httpget.getStatusText()); if (status.getCode() == HttpStatus.SC_OK) { String message = readResponse(httpget.getResponseBodyAsStream()); log.debug("returned message is: " + message); doc = new ServiceDocument(); lastUnmarshallInfo = doc.unmarshall(message, properties); } else { throw new SWORDClientException("Received error from service document request: " + status); } } catch (HttpException ex) { throw new SWORDClientException(ex.getMessage(), ex); } catch (IOException ioex) { throw new SWORDClientException(ioex.getMessage(), ioex); } catch (UnmarshallException uex) { throw new SWORDClientException(uex.getMessage(), uex); } finally { httpget.releaseConnection(); } return doc; }
/** * The method reads a proxy object from database, makes a GetMethod object, appends required * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the * pagesource to the caller. * * @param iCount Counter variable for passing thread group information * @param url Url to fetch the pagesource for * @param followRedirect Boolean variable to specify GetMethod followRedirect value * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value * @param region the local region of a given url * @param objProxyDao the database layer ProxyDao object variable * @param useErrsy Boolean variable to specify usage of Errsy as proxy source * @return String */ public String getPageSourceWithProxy( String url, boolean followRedirect, boolean doAuthentication, String region, Boolean useErrsy, String google) { String page = " "; String pageSource = ""; int i = 0; String exception = " "; HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(40000); clientParams.setConnectionManagerTimeout(40000); HttpClient httpclient = new HttpClient(clientParams); GetMethod getmethod = null; HttpState state = new HttpState(); // if (ProxyDao.lstProxyData.size() == 16) { ProxyData objProxyData = null; // if (!useErrsy) { try { // objProxyData = ProxyDao.lstProxyData.get(iCount); objProxyData = ProxyDao.objProxyData; if (objProxyData == null) { // objProxyDao.changeProxy(google); objProxyData = ProxyDao.objProxyData; } httpclient .getHostConfiguration() .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo()); } catch (Exception e) { pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; return pageSource; } /*} else { try { objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0); Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()); httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds); httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo()); state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword())); httpclient.setState(state); } catch (Exception e) { pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; return pageSource; } }*/ try { getmethod = new GetMethod(url); getmethod.addRequestHeader( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0"); if (url.contains("bing.com")) { if (region.equalsIgnoreCase("co.uk")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.sg")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.au")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("co.in")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("ca")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.ph")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.my")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("it")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } } getmethod.setFollowRedirects(true); getmethod.setDoAuthentication(true); httpclient.getParams().setAuthenticationPreemptive(true); httpclient.setState(state); String num100Header = ""; // if (url.contains("google")) { // int j = 0; // String url1 = "http://www.google.com/"; // try { // GetMethod objGetMethod = new GetMethod(url1); // j = httpclient.executeMethod(objGetMethod); // Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie"); // String header = responseHeader.getValue(); // String[] headerValue = header.split(";"); // // for (String head : headerValue) { // if (head.contains("PREF=ID")) { // header = head; // break; // } // } // String[] splitAll = header.split(":"); // long time = System.currentTimeMillis()+400; // String sTime = "" + time; // sTime = sTime.substring(0, 10); // //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] + // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4]; // num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] + // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4]; // Cookie ck = new Cookie("PREF", "PREF", num100Header); // httpclient.getState().clearCookies(); // httpclient.getState().addCookie(ck); // getmethod.addRequestHeader("Host", "www.google.com"); // getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; // rv:19.0) Gecko/20100101 Firefox/19.0"); // getmethod.addRequestHeader("Accept", // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); // getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5"); // getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate"); // getmethod.addRequestHeader("Referer", "https://www.google.com/"); // System.out.println(num100Header); // } catch (Exception ex) { // exception = ex.getMessage(); // l.debug(ex + " " + ex.getMessage() + "Exception occured for url" + // url); // pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1; // return pageSource; // } // } i = httpclient.executeMethod(getmethod); if (i / 100 == 4 || i / 100 == 5) { page = "<PROXY ERROR>"; } else { page = getmethod.getResponseBodyAsString(); } } catch (SocketTimeoutException ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } catch (SocketException ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } catch (Exception ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } finally { getmethod.releaseConnection(); } pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; // } return pageSource; }