/** * Method to make a GET HTTP connecton to the given url and return the output * * @param urlToFetch url to be connected * @return the http get response */ public String makeRequest(String urlToFetch, String region) throws IOException { String responseBody = ""; try { HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(40000); clientParams.setConnectionManagerTimeout(40000); HttpClient httpclient = new HttpClient(clientParams); GetMethod httpget = new GetMethod(urlToFetch); if (urlToFetch.contains("linkedin.com/countserv")) { httpget.addRequestHeader("Host", "www.linkedin.com"); httpget.addRequestHeader( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:20.0) Gecko/20100101 Firefox/20.0"); httpget.addRequestHeader( "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); httpget.addRequestHeader( "Cookie", "X-LI-IDC=C1; bcookie=\"v=2&618ed56d-275f-4dcd-86cd-6a44c8421879\"; bscookie=\"v=1&201305170926410ddaf2a1-e238-4bf6-8a0a-ce8a3df25934AQFZX82SfErJuFQKMJV49JfQTvewhfzh\"; X-LI-IDC=C1"); } // httpclient.getHostConfiguration().setProxy("46.227.68.2", 3128); // Credentials cred = new UsernamePasswordCredentials("mmongoose", // "I-PHNBV9JHW6US"); // httpclient.getState().setProxyCredentials(AuthScope.ANY, cred); int i = httpclient.executeMethod(httpget); responseBody = httpget.getResponseBodyAsString(); } catch (Exception e) { l.error(e + " " + e.getMessage() + " url " + urlToFetch); } return responseBody; }
/** * Get an instance of the HTTP client to use. * * @return HTTP client initialized with admin credentials */ protected static HttpClient getClient() { if (AbstractEscapingTest.client == null) { HttpClient adminClient = new HttpClient(); Credentials defaultcreds = new UsernamePasswordCredentials("Admin", "admin"); adminClient.getState().setCredentials(AuthScope.ANY, defaultcreds); HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(2000); HttpConnectionManagerParams connectionParams = new HttpConnectionManagerParams(); connectionParams.setConnectionTimeout(30000); adminClient.getHttpConnectionManager().setParams(connectionParams); AbstractEscapingTest.client = adminClient; } return AbstractEscapingTest.client; }
/** * The method reads a proxy object from database, makes a GetMethod object, appends required * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the * pagesource to the caller. * * @param iCount Counter variable for passing thread group information * @param url Url to fetch the pagesource for * @param followRedirect Boolean variable to specify GetMethod followRedirect value * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value * @param region the local region of a given url * @param objProxyDao the database layer ProxyDao object variable * @param useErrsy Boolean variable to specify usage of Errsy as proxy source * @return String */ public String getPageSourceWithProxy( String url, boolean followRedirect, boolean doAuthentication, String region, Boolean useErrsy, String google) { String page = " "; String pageSource = ""; int i = 0; String exception = " "; HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(40000); clientParams.setConnectionManagerTimeout(40000); HttpClient httpclient = new HttpClient(clientParams); GetMethod getmethod = null; HttpState state = new HttpState(); // if (ProxyDao.lstProxyData.size() == 16) { ProxyData objProxyData = null; // if (!useErrsy) { try { // objProxyData = ProxyDao.lstProxyData.get(iCount); objProxyData = ProxyDao.objProxyData; if (objProxyData == null) { // objProxyDao.changeProxy(google); objProxyData = ProxyDao.objProxyData; } httpclient .getHostConfiguration() .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo()); } catch (Exception e) { pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; return pageSource; } /*} else { try { objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0); Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()); httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds); httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo()); state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword())); httpclient.setState(state); } catch (Exception e) { pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; return pageSource; } }*/ try { getmethod = new GetMethod(url); getmethod.addRequestHeader( "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0"); if (url.contains("bing.com")) { if (region.equalsIgnoreCase("co.uk")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.sg")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.au")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("co.in")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("ca")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.ph")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("com.my")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else if (region.equalsIgnoreCase("it")) { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } else { getmethod.addRequestHeader( "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;"); } } getmethod.setFollowRedirects(true); getmethod.setDoAuthentication(true); httpclient.getParams().setAuthenticationPreemptive(true); httpclient.setState(state); String num100Header = ""; // if (url.contains("google")) { // int j = 0; // String url1 = "http://www.google.com/"; // try { // GetMethod objGetMethod = new GetMethod(url1); // j = httpclient.executeMethod(objGetMethod); // Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie"); // String header = responseHeader.getValue(); // String[] headerValue = header.split(";"); // // for (String head : headerValue) { // if (head.contains("PREF=ID")) { // header = head; // break; // } // } // String[] splitAll = header.split(":"); // long time = System.currentTimeMillis()+400; // String sTime = "" + time; // sTime = sTime.substring(0, 10); // //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] + // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4]; // num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] + // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4]; // Cookie ck = new Cookie("PREF", "PREF", num100Header); // httpclient.getState().clearCookies(); // httpclient.getState().addCookie(ck); // getmethod.addRequestHeader("Host", "www.google.com"); // getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; // rv:19.0) Gecko/20100101 Firefox/19.0"); // getmethod.addRequestHeader("Accept", // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); // getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5"); // getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate"); // getmethod.addRequestHeader("Referer", "https://www.google.com/"); // System.out.println(num100Header); // } catch (Exception ex) { // exception = ex.getMessage(); // l.debug(ex + " " + ex.getMessage() + "Exception occured for url" + // url); // pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1; // return pageSource; // } // } i = httpclient.executeMethod(getmethod); if (i / 100 == 4 || i / 100 == 5) { page = "<PROXY ERROR>"; } else { page = getmethod.getResponseBodyAsString(); } } catch (SocketTimeoutException ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } catch (SocketException ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } catch (Exception ex) { exception = ex.getMessage(); l.error(ex + " " + ex.getMessage() + "Exception occured for url" + url); } finally { getmethod.releaseConnection(); } pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url; // } return pageSource; }