/**
* The method makes a GetMethod object and an HttpClient object. The HttpClient then executes the
* Getmethod and returns the pagesource to the caller.
*
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @return String
*/
public String getPageSourceWithoutProxy(
String url, boolean followRedirects, boolean doAuthentication) {
GetMethod getMethod = null;
String pageSouce = null;
HttpClient httpClient = new HttpClient();
try {
getMethod = new GetMethod(url);
getMethod.setFollowRedirects(followRedirects);
getMethod.setDoAuthentication(doAuthentication);
httpClient.executeMethod(getMethod);
pageSouce = getMethod.getResponseBodyAsString();
} catch (Exception e) {
l.error(e + " " + e.getMessage() + "exception occured for url" + url);
try {
getMethod = new GetMethod(url);
getMethod.setFollowRedirects(followRedirects);
getMethod.setDoAuthentication(doAuthentication);
httpClient.executeMethod(getMethod);
pageSouce = getMethod.getResponseBodyAsString();
getMethod.releaseConnection();
} catch (Exception ex) {
l.error(ex + " " + ex.getMessage() + "exception occured for url " + url);
getMethod.releaseConnection();
}
} finally {
getMethod.releaseConnection();
}
return pageSouce;
}
private void doGet(
String realm,
String host,
String user,
String pass,
String url,
boolean handshake,
boolean preemtive,
int result)
throws Exception {
HttpClient client = new HttpClient();
client.getParams().setAuthenticationPreemptive(preemtive);
client
.getState()
.setCredentials(
new AuthScope(host, -1, realm), new UsernamePasswordCredentials(user, pass));
GetMethod get = new GetMethod(url);
get.setDoAuthentication(handshake);
try {
int status = client.executeMethod(get);
assertEquals(result, status);
} finally {
get.releaseConnection();
}
}
/**
* Download a page from the server and return its content. Throws a {@link RuntimeException} on
* connection problems etc.
*
* @param url URL of the page
* @return content of the page
*/
protected static InputStream getUrlContent(String url) {
GetMethod get = new GetMethod(url);
get.setFollowRedirects(true);
if (isLoggedIn()) {
get.setDoAuthentication(true);
get.addRequestHeader(
"Authorization", "Basic " + new String(Base64.encodeBase64("Admin:admin".getBytes())));
}
try {
int statusCode = AbstractEscapingTest.getClient().executeMethod(get);
switch (statusCode) {
case HttpStatus.SC_OK:
// everything is fine
break;
case HttpStatus.SC_UNAUTHORIZED:
// do not fail on 401 (unauthorized), used in some tests
System.out.println("WARNING, Ignoring status 401 (unauthorized) for URL: " + url);
break;
case HttpStatus.SC_CONFLICT:
// do not fail on 409 (conflict), used in some templates
System.out.println("WARNING, Ignoring status 409 (conflict) for URL: " + url);
break;
case HttpStatus.SC_NOT_FOUND:
// ignore 404 (the page is still rendered)
break;
default:
throw new RuntimeException(
"HTTP GET request returned status "
+ statusCode
+ " ("
+ get.getStatusText()
+ ") for URL: "
+ url);
}
// get the data, converting to utf-8
String str = get.getResponseBodyAsString();
if (str == null) {
return null;
}
return new ByteArrayInputStream(str.getBytes("utf-8"));
} catch (IOException exception) {
throw new RuntimeException("Error retrieving URL: " + url + "\n" + exception);
} finally {
get.releaseConnection();
}
}
private Crumb getCsrfCrumb(HttpClient client, Candidate target) throws IOException {
GetMethod httpGet =
new GetMethod(
target.url
+ "crumbIssuer/api/xml?xpath="
+ URLEncoder.encode("concat(//crumbRequestField,\":\",//crumb)", "UTF-8"));
httpGet.setDoAuthentication(true);
int responseCode = client.executeMethod(httpGet);
if (responseCode != HttpStatus.SC_OK) {
System.out.println("Could not obtain CSRF crumb. Response code: " + responseCode);
return null;
}
String[] crumbResponse = httpGet.getResponseBodyAsString().split(":");
if (crumbResponse.length != 2) {
System.out.println("Unexpected CSRF crumb response: " + httpGet.getResponseBodyAsString());
return null;
}
return new Crumb(crumbResponse[0], crumbResponse[1]);
}
public void testAuthenticationFailureNoContextGet() throws Exception {
HttpClient client = new HttpClient();
client.getParams().setAuthenticationPreemptive(true);
GetMethod get = new GetMethod("http://localhost:60443/services/Echo");
get.setDoAuthentication(false);
try {
int status = client.executeMethod(get);
assertEquals(HttpConstants.SC_UNAUTHORIZED, status);
assertEquals(
"Registered authentication is set to org.mule.module.acegi.filters.http.HttpBasicAuthenticationFilter "
+ "but there was no security context on the session. Authentication denied on "
+ "endpoint http://localhost:60443/services/Echo. Message payload is of type: "
+ "String",
get.getResponseBodyAsString());
} finally {
get.releaseConnection();
}
}
public static void sendGetTransStatusRequest(String urlString, String authentication)
throws Exception {
GetMethod method = new GetMethod(urlString);
method.setDoAuthentication(true);
// adding authorization token
if (authentication != null) {
method.addRequestHeader(new Header("Authorization", authentication));
}
// executing method
HttpClient client = new HttpClient();
int code = client.executeMethod(method);
String response = method.getResponseBodyAsString();
method.releaseConnection();
if (code >= 400) {
System.out.println("Error occurred during getting job status.");
}
System.out.println("Server response:");
System.out.println(response);
}
public GeoRSSReader createReader(final URL url, final String username, final String password)
throws IOException {
if (log.isDebugEnabled()) {
log.debug(
"Creating GeoRSS reader for URL " + url.toExternalForm() + " with user " + username);
}
HttpClientBuilder builder = new HttpClientBuilder();
builder.setHttpCredentials(username, password, url);
builder.setBackendTimeout(120);
HttpClient httpClient = builder.buildClient();
GetMethod getMethod = new GetMethod(url.toString());
getMethod.setRequestHeader("Connection", "close");
if (builder.isDoAuthentication()) {
getMethod.setDoAuthentication(true);
httpClient.getParams().setAuthenticationPreemptive(true);
}
if (log.isDebugEnabled()) {
log.debug("Executing HTTP GET requesr for feed URL " + url.toExternalForm());
}
httpClient.executeMethod(getMethod);
if (log.isDebugEnabled()) {
log.debug("Building GeoRSS reader out of URL response");
}
String contentEncoding = getMethod.getResponseCharSet();
if (contentEncoding == null) {
contentEncoding = "UTF-8";
}
InputStream in = getMethod.getResponseBodyAsStream();
Reader reader = new BufferedReader(new InputStreamReader(in, contentEncoding));
if (log.isDebugEnabled()) {
log.debug("GeoRSS reader created, returning.");
}
return createReader(reader);
}
public Candidate discoverFromMasterUrl()
throws IOException, ParserConfigurationException, RetryException {
if (!options.master.endsWith("/")) {
options.master += "/";
}
URL masterURL;
try {
masterURL = new URL(options.master);
} catch (MalformedURLException e) {
throw new RuntimeException(
MessageFormat.format("The master URL \"{0}\" is invalid", options.master), e);
}
HttpClient client = createHttpClient(masterURL);
String url = masterURL.toExternalForm() + "plugin/swarm/slaveInfo";
GetMethod get = new GetMethod(url);
get.setDoAuthentication(true);
get.addRequestHeader("Connection", "close");
int responseCode = client.executeMethod(get);
if (responseCode != 200) {
throw new RetryException("Failed to fetch slave info from Jenkins CODE: " + responseCode);
}
Document xml;
try {
xml =
DocumentBuilderFactory.newInstance()
.newDocumentBuilder()
.parse(get.getResponseBodyAsStream());
} catch (SAXException e) {
throw new RetryException("Invalid XML received from " + url);
}
String swarmSecret = getChildElementString(xml.getDocumentElement(), "swarmSecret");
return new Candidate(masterURL.toExternalForm(), swarmSecret);
}
protected GetMethod createGetMethod(final String url, final boolean authenticate) {
GetMethod get = new GetMethod(url);
get.setDoAuthentication(authenticate);
return get;
}
/**
* Retrieve the service document. The service document is located at the specified URL. This calls
* getServiceDocument(url,onBehalfOf).
*
* @param url The location of the service document.
* @return The ServiceDocument, or <code>null</code> if there was a problem accessing the
* document. e.g. invalid access.
* @throws SWORDClientException If there is an error accessing the resource.
*/
public ServiceDocument getServiceDocument(String url, String onBehalfOf)
throws SWORDClientException {
URL serviceDocURL = null;
try {
serviceDocURL = new URL(url);
} catch (MalformedURLException e) {
// Try relative URL
URL baseURL = null;
try {
baseURL = new URL("http", server, Integer.valueOf(port), "/");
serviceDocURL = new URL(baseURL, (url == null) ? "" : url);
} catch (MalformedURLException e1) {
// No dice, can't even form base URL...
throw new SWORDClientException(
url
+ " is not a valid URL ("
+ e1.getMessage()
+ "), and could not form a relative one from: "
+ baseURL
+ " / "
+ url,
e1);
}
}
GetMethod httpget = new GetMethod(serviceDocURL.toExternalForm());
if (doAuthentication) {
// this does not perform any check on the username password. It
// relies on the server to determine if the values are correct.
setBasicCredentials(username, password);
httpget.setDoAuthentication(true);
}
Properties properties = new Properties();
if (containsValue(onBehalfOf)) {
log.debug("Setting on-behalf-of: " + onBehalfOf);
httpget.addRequestHeader(new Header(HttpHeaders.X_ON_BEHALF_OF, onBehalfOf));
properties.put(HttpHeaders.X_ON_BEHALF_OF, onBehalfOf);
}
if (containsValue(userAgent)) {
log.debug("Setting userAgent: " + userAgent);
httpget.addRequestHeader(new Header(HttpHeaders.USER_AGENT, userAgent));
properties.put(HttpHeaders.USER_AGENT, userAgent);
}
ServiceDocument doc = null;
try {
client.executeMethod(httpget);
// store the status code
status = new Status(httpget.getStatusCode(), httpget.getStatusText());
if (status.getCode() == HttpStatus.SC_OK) {
String message = readResponse(httpget.getResponseBodyAsStream());
log.debug("returned message is: " + message);
doc = new ServiceDocument();
lastUnmarshallInfo = doc.unmarshall(message, properties);
} else {
throw new SWORDClientException("Received error from service document request: " + status);
}
} catch (HttpException ex) {
throw new SWORDClientException(ex.getMessage(), ex);
} catch (IOException ioex) {
throw new SWORDClientException(ioex.getMessage(), ioex);
} catch (UnmarshallException uex) {
throw new SWORDClientException(uex.getMessage(), uex);
} finally {
httpget.releaseConnection();
}
return doc;
}
/**
* The method reads a proxy object from database, makes a GetMethod object, appends required
* cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
* pagesource to the caller.
*
* @param iCount Counter variable for passing thread group information
* @param url Url to fetch the pagesource for
* @param followRedirect Boolean variable to specify GetMethod followRedirect value
* @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
* @param region the local region of a given url
* @param objProxyDao the database layer ProxyDao object variable
* @param useErrsy Boolean variable to specify usage of Errsy as proxy source
* @return String
*/
public String getPageSourceWithProxy(
String url,
boolean followRedirect,
boolean doAuthentication,
String region,
Boolean useErrsy,
String google) {
String page = " ";
String pageSource = "";
int i = 0;
String exception = " ";
HttpClientParams clientParams = new HttpClientParams();
clientParams.setSoTimeout(40000);
clientParams.setConnectionManagerTimeout(40000);
HttpClient httpclient = new HttpClient(clientParams);
GetMethod getmethod = null;
HttpState state = new HttpState();
// if (ProxyDao.lstProxyData.size() == 16) {
ProxyData objProxyData = null;
// if (!useErrsy) {
try {
// objProxyData = ProxyDao.lstProxyData.get(iCount);
objProxyData = ProxyDao.objProxyData;
if (objProxyData == null) {
// objProxyDao.changeProxy(google);
objProxyData = ProxyDao.objProxyData;
}
httpclient
.getHostConfiguration()
.setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
/*} else {
try {
objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
httpclient.setState(state);
} catch (Exception e) {
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
return pageSource;
}
}*/
try {
getmethod = new GetMethod(url);
getmethod.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
if (url.contains("bing.com")) {
if (region.equalsIgnoreCase("co.uk")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.sg")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.au")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("co.in")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("ca")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.ph")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("com.my")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else if (region.equalsIgnoreCase("it")) {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
} else {
getmethod.addRequestHeader(
"Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
}
}
getmethod.setFollowRedirects(true);
getmethod.setDoAuthentication(true);
httpclient.getParams().setAuthenticationPreemptive(true);
httpclient.setState(state);
String num100Header = "";
// if (url.contains("google")) {
// int j = 0;
// String url1 = "http://www.google.com/";
// try {
// GetMethod objGetMethod = new GetMethod(url1);
// j = httpclient.executeMethod(objGetMethod);
// Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
// String header = responseHeader.getValue();
// String[] headerValue = header.split(";");
//
// for (String head : headerValue) {
// if (head.contains("PREF=ID")) {
// header = head;
// break;
// }
// }
// String[] splitAll = header.split(":");
// long time = System.currentTimeMillis()+400;
// String sTime = "" + time;
// sTime = sTime.substring(0, 10);
// //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
// num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
// ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
// Cookie ck = new Cookie("PREF", "PREF", num100Header);
// httpclient.getState().clearCookies();
// httpclient.getState().addCookie(ck);
// getmethod.addRequestHeader("Host", "www.google.com");
// getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
// rv:19.0) Gecko/20100101 Firefox/19.0");
// getmethod.addRequestHeader("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
// getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
// getmethod.addRequestHeader("Referer", "https://www.google.com/");
// System.out.println(num100Header);
// } catch (Exception ex) {
// exception = ex.getMessage();
// l.debug(ex + " " + ex.getMessage() + "Exception occured for url" +
// url);
// pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
// return pageSource;
// }
// }
i = httpclient.executeMethod(getmethod);
if (i / 100 == 4 || i / 100 == 5) {
page = "<PROXY ERROR>";
} else {
page = getmethod.getResponseBodyAsString();
}
} catch (SocketTimeoutException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (SocketException ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} catch (Exception ex) {
exception = ex.getMessage();
l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url);
} finally {
getmethod.releaseConnection();
}
pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
// }
return pageSource;
}