/**
   * The method makes a GetMethod object and an HttpClient object. The HttpClient then executes the
   * Getmethod and returns the pagesource to the caller.
   *
   * @param url Url to fetch the pagesource for
   * @param followRedirect Boolean variable to specify GetMethod followRedirect value
   * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
   * @return String
   */
  public String getPageSourceWithoutProxy(
      String url, boolean followRedirects, boolean doAuthentication) {

    GetMethod getMethod = null;
    String pageSouce = null;
    HttpClient httpClient = new HttpClient();
    try {
      getMethod = new GetMethod(url);
      getMethod.setFollowRedirects(followRedirects);
      getMethod.setDoAuthentication(doAuthentication);
      httpClient.executeMethod(getMethod);
      pageSouce = getMethod.getResponseBodyAsString();
    } catch (Exception e) {
      l.error(e + "  " + e.getMessage() + "exception occured for url" + url);
      try {
        getMethod = new GetMethod(url);
        getMethod.setFollowRedirects(followRedirects);
        getMethod.setDoAuthentication(doAuthentication);
        httpClient.executeMethod(getMethod);
        pageSouce = getMethod.getResponseBodyAsString();
        getMethod.releaseConnection();
      } catch (Exception ex) {
        l.error(ex + "  " + ex.getMessage() + "exception occured for url " + url);
        getMethod.releaseConnection();
      }
    } finally {
      getMethod.releaseConnection();
    }
    return pageSouce;
  }
Example #2
0
  private void doGet(
      String realm,
      String host,
      String user,
      String pass,
      String url,
      boolean handshake,
      boolean preemtive,
      int result)
      throws Exception {
    HttpClient client = new HttpClient();
    client.getParams().setAuthenticationPreemptive(preemtive);
    client
        .getState()
        .setCredentials(
            new AuthScope(host, -1, realm), new UsernamePasswordCredentials(user, pass));
    GetMethod get = new GetMethod(url);
    get.setDoAuthentication(handshake);

    try {
      int status = client.executeMethod(get);
      assertEquals(result, status);
    } finally {
      get.releaseConnection();
    }
  }
  /**
   * Download a page from the server and return its content. Throws a {@link RuntimeException} on
   * connection problems etc.
   *
   * @param url URL of the page
   * @return content of the page
   */
  protected static InputStream getUrlContent(String url) {
    GetMethod get = new GetMethod(url);
    get.setFollowRedirects(true);
    if (isLoggedIn()) {
      get.setDoAuthentication(true);
      get.addRequestHeader(
          "Authorization", "Basic " + new String(Base64.encodeBase64("Admin:admin".getBytes())));
    }

    try {
      int statusCode = AbstractEscapingTest.getClient().executeMethod(get);
      switch (statusCode) {
        case HttpStatus.SC_OK:
          // everything is fine
          break;
        case HttpStatus.SC_UNAUTHORIZED:
          // do not fail on 401 (unauthorized), used in some tests
          System.out.println("WARNING, Ignoring status 401 (unauthorized) for URL: " + url);
          break;
        case HttpStatus.SC_CONFLICT:
          // do not fail on 409 (conflict), used in some templates
          System.out.println("WARNING, Ignoring status 409 (conflict) for URL: " + url);
          break;
        case HttpStatus.SC_NOT_FOUND:
          // ignore 404 (the page is still rendered)
          break;
        default:
          throw new RuntimeException(
              "HTTP GET request returned status "
                  + statusCode
                  + " ("
                  + get.getStatusText()
                  + ") for URL: "
                  + url);
      }

      // get the data, converting to utf-8
      String str = get.getResponseBodyAsString();
      if (str == null) {
        return null;
      }
      return new ByteArrayInputStream(str.getBytes("utf-8"));
    } catch (IOException exception) {
      throw new RuntimeException("Error retrieving URL: " + url + "\n" + exception);
    } finally {
      get.releaseConnection();
    }
  }
 private Crumb getCsrfCrumb(HttpClient client, Candidate target) throws IOException {
   GetMethod httpGet =
       new GetMethod(
           target.url
               + "crumbIssuer/api/xml?xpath="
               + URLEncoder.encode("concat(//crumbRequestField,\":\",//crumb)", "UTF-8"));
   httpGet.setDoAuthentication(true);
   int responseCode = client.executeMethod(httpGet);
   if (responseCode != HttpStatus.SC_OK) {
     System.out.println("Could not obtain CSRF crumb. Response code: " + responseCode);
     return null;
   }
   String[] crumbResponse = httpGet.getResponseBodyAsString().split(":");
   if (crumbResponse.length != 2) {
     System.out.println("Unexpected CSRF crumb response: " + httpGet.getResponseBodyAsString());
     return null;
   }
   return new Crumb(crumbResponse[0], crumbResponse[1]);
 }
Example #5
0
  public void testAuthenticationFailureNoContextGet() throws Exception {
    HttpClient client = new HttpClient();
    client.getParams().setAuthenticationPreemptive(true);
    GetMethod get = new GetMethod("http://localhost:60443/services/Echo");

    get.setDoAuthentication(false);

    try {
      int status = client.executeMethod(get);
      assertEquals(HttpConstants.SC_UNAUTHORIZED, status);
      assertEquals(
          "Registered authentication is set to org.mule.module.acegi.filters.http.HttpBasicAuthenticationFilter "
              + "but there was no security context on the session. Authentication denied on "
              + "endpoint http://localhost:60443/services/Echo. Message payload is of type: "
              + "String",
          get.getResponseBodyAsString());
    } finally {
      get.releaseConnection();
    }
  }
  public static void sendGetTransStatusRequest(String urlString, String authentication)
      throws Exception {
    GetMethod method = new GetMethod(urlString);
    method.setDoAuthentication(true);
    // adding authorization token
    if (authentication != null) {
      method.addRequestHeader(new Header("Authorization", authentication));
    }

    // executing method
    HttpClient client = new HttpClient();
    int code = client.executeMethod(method);
    String response = method.getResponseBodyAsString();
    method.releaseConnection();
    if (code >= 400) {
      System.out.println("Error occurred during getting job status.");
    }
    System.out.println("Server response:");
    System.out.println(response);
  }
  public GeoRSSReader createReader(final URL url, final String username, final String password)
      throws IOException {

    if (log.isDebugEnabled()) {
      log.debug(
          "Creating GeoRSS reader for URL " + url.toExternalForm() + " with user " + username);
    }

    HttpClientBuilder builder = new HttpClientBuilder();
    builder.setHttpCredentials(username, password, url);
    builder.setBackendTimeout(120);

    HttpClient httpClient = builder.buildClient();

    GetMethod getMethod = new GetMethod(url.toString());
    getMethod.setRequestHeader("Connection", "close");
    if (builder.isDoAuthentication()) {
      getMethod.setDoAuthentication(true);
      httpClient.getParams().setAuthenticationPreemptive(true);
    }

    if (log.isDebugEnabled()) {
      log.debug("Executing HTTP GET requesr for feed URL " + url.toExternalForm());
    }
    httpClient.executeMethod(getMethod);

    if (log.isDebugEnabled()) {
      log.debug("Building GeoRSS reader out of URL response");
    }
    String contentEncoding = getMethod.getResponseCharSet();
    if (contentEncoding == null) {
      contentEncoding = "UTF-8";
    }

    InputStream in = getMethod.getResponseBodyAsStream();
    Reader reader = new BufferedReader(new InputStreamReader(in, contentEncoding));
    if (log.isDebugEnabled()) {
      log.debug("GeoRSS reader created, returning.");
    }
    return createReader(reader);
  }
  public Candidate discoverFromMasterUrl()
      throws IOException, ParserConfigurationException, RetryException {
    if (!options.master.endsWith("/")) {
      options.master += "/";
    }
    URL masterURL;
    try {
      masterURL = new URL(options.master);
    } catch (MalformedURLException e) {
      throw new RuntimeException(
          MessageFormat.format("The master URL \"{0}\" is invalid", options.master), e);
    }

    HttpClient client = createHttpClient(masterURL);

    String url = masterURL.toExternalForm() + "plugin/swarm/slaveInfo";
    GetMethod get = new GetMethod(url);
    get.setDoAuthentication(true);
    get.addRequestHeader("Connection", "close");

    int responseCode = client.executeMethod(get);
    if (responseCode != 200) {
      throw new RetryException("Failed to fetch slave info from Jenkins CODE: " + responseCode);
    }

    Document xml;
    try {
      xml =
          DocumentBuilderFactory.newInstance()
              .newDocumentBuilder()
              .parse(get.getResponseBodyAsStream());
    } catch (SAXException e) {
      throw new RetryException("Invalid XML received from " + url);
    }
    String swarmSecret = getChildElementString(xml.getDocumentElement(), "swarmSecret");
    return new Candidate(masterURL.toExternalForm(), swarmSecret);
  }
 protected GetMethod createGetMethod(final String url, final boolean authenticate) {
   GetMethod get = new GetMethod(url);
   get.setDoAuthentication(authenticate);
   return get;
 }
Example #10
0
  /**
   * Retrieve the service document. The service document is located at the specified URL. This calls
   * getServiceDocument(url,onBehalfOf).
   *
   * @param url The location of the service document.
   * @return The ServiceDocument, or <code>null</code> if there was a problem accessing the
   *     document. e.g. invalid access.
   * @throws SWORDClientException If there is an error accessing the resource.
   */
  public ServiceDocument getServiceDocument(String url, String onBehalfOf)
      throws SWORDClientException {
    URL serviceDocURL = null;
    try {
      serviceDocURL = new URL(url);
    } catch (MalformedURLException e) {
      // Try relative URL
      URL baseURL = null;
      try {
        baseURL = new URL("http", server, Integer.valueOf(port), "/");
        serviceDocURL = new URL(baseURL, (url == null) ? "" : url);
      } catch (MalformedURLException e1) {
        // No dice, can't even form base URL...
        throw new SWORDClientException(
            url
                + " is not a valid URL ("
                + e1.getMessage()
                + "), and could not form a relative one from: "
                + baseURL
                + " / "
                + url,
            e1);
      }
    }

    GetMethod httpget = new GetMethod(serviceDocURL.toExternalForm());
    if (doAuthentication) {
      // this does not perform any check on the username password. It
      // relies on the server to determine if the values are correct.
      setBasicCredentials(username, password);
      httpget.setDoAuthentication(true);
    }

    Properties properties = new Properties();

    if (containsValue(onBehalfOf)) {
      log.debug("Setting on-behalf-of: " + onBehalfOf);
      httpget.addRequestHeader(new Header(HttpHeaders.X_ON_BEHALF_OF, onBehalfOf));
      properties.put(HttpHeaders.X_ON_BEHALF_OF, onBehalfOf);
    }

    if (containsValue(userAgent)) {
      log.debug("Setting userAgent: " + userAgent);
      httpget.addRequestHeader(new Header(HttpHeaders.USER_AGENT, userAgent));
      properties.put(HttpHeaders.USER_AGENT, userAgent);
    }

    ServiceDocument doc = null;

    try {
      client.executeMethod(httpget);
      // store the status code
      status = new Status(httpget.getStatusCode(), httpget.getStatusText());

      if (status.getCode() == HttpStatus.SC_OK) {
        String message = readResponse(httpget.getResponseBodyAsStream());
        log.debug("returned message is: " + message);
        doc = new ServiceDocument();
        lastUnmarshallInfo = doc.unmarshall(message, properties);
      } else {
        throw new SWORDClientException("Received error from service document request: " + status);
      }
    } catch (HttpException ex) {
      throw new SWORDClientException(ex.getMessage(), ex);
    } catch (IOException ioex) {
      throw new SWORDClientException(ioex.getMessage(), ioex);
    } catch (UnmarshallException uex) {
      throw new SWORDClientException(uex.getMessage(), uex);
    } finally {
      httpget.releaseConnection();
    }

    return doc;
  }
  /**
   * The method reads a proxy object from database, makes a GetMethod object, appends required
   * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
   * pagesource to the caller.
   *
   * @param iCount Counter variable for passing thread group information
   * @param url Url to fetch the pagesource for
   * @param followRedirect Boolean variable to specify GetMethod followRedirect value
   * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
   * @param region the local region of a given url
   * @param objProxyDao the database layer ProxyDao object variable
   * @param useErrsy Boolean variable to specify usage of Errsy as proxy source
   * @return String
   */
  public String getPageSourceWithProxy(
      String url,
      boolean followRedirect,
      boolean doAuthentication,
      String region,
      Boolean useErrsy,
      String google) {

    String page = " ";
    String pageSource = "";
    int i = 0;
    String exception = " ";
    HttpClientParams clientParams = new HttpClientParams();
    clientParams.setSoTimeout(40000);
    clientParams.setConnectionManagerTimeout(40000);
    HttpClient httpclient = new HttpClient(clientParams);
    GetMethod getmethod = null;

    HttpState state = new HttpState();

    //  if (ProxyDao.lstProxyData.size() == 16) {
    ProxyData objProxyData = null;
    // if (!useErrsy) {
    try {
      // objProxyData = ProxyDao.lstProxyData.get(iCount);
      objProxyData = ProxyDao.objProxyData;
      if (objProxyData == null) {
        //                objProxyDao.changeProxy(google);
        objProxyData = ProxyDao.objProxyData;
      }
      httpclient
          .getHostConfiguration()
          .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
    } catch (Exception e) {
      pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
      return pageSource;
    }
    /*} else {
    try {
    objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
    Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
    httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
    httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
    state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
    httpclient.setState(state);
    } catch (Exception e) {
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
    return pageSource;
    }
    }*/
    try {
      getmethod = new GetMethod(url);
      getmethod.addRequestHeader(
          "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
      if (url.contains("bing.com")) {

        if (region.equalsIgnoreCase("co.uk")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.sg")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.au")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("co.in")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("ca")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.ph")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.my")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("it")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        }
      }
      getmethod.setFollowRedirects(true);
      getmethod.setDoAuthentication(true);
      httpclient.getParams().setAuthenticationPreemptive(true);
      httpclient.setState(state);
      String num100Header = "";
      //            if (url.contains("google")) {
      //                int j = 0;
      //                String url1 = "http://www.google.com/";
      //                try {
      //                    GetMethod objGetMethod = new GetMethod(url1);
      //                    j = httpclient.executeMethod(objGetMethod);
      //                    Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
      //                    String header = responseHeader.getValue();
      //                    String[] headerValue = header.split(";");
      //
      //                    for (String head : headerValue) {
      //                        if (head.contains("PREF=ID")) {
      //                            header = head;
      //                            break;
      //                        }
      //                    }
      //                    String[] splitAll = header.split(":");
      //                    long time = System.currentTimeMillis()+400;
      //                    String sTime = "" + time;
      //                    sTime = sTime.substring(0, 10);
      //                    //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1]  +
      // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
      //                    num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
      // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
      //                    Cookie ck = new Cookie("PREF", "PREF", num100Header);
      //                    httpclient.getState().clearCookies();
      //                    httpclient.getState().addCookie(ck);
      //                    getmethod.addRequestHeader("Host", "www.google.com");
      //                    getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
      // rv:19.0) Gecko/20100101 Firefox/19.0");
      //                    getmethod.addRequestHeader("Accept",
      // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
      //                    getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
      //                    getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
      //                    getmethod.addRequestHeader("Referer", "https://www.google.com/");
      //                    System.out.println(num100Header);
      //                } catch (Exception ex) {
      //                    exception = ex.getMessage();
      //                    l.debug(ex + "  " + ex.getMessage() + "Exception occured for url" +
      // url);
      //                    pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
      //                    return pageSource;
      //                }
      //            }
      i = httpclient.executeMethod(getmethod);
      if (i / 100 == 4 || i / 100 == 5) {
        page = "<PROXY ERROR>";

      } else {

        page = getmethod.getResponseBodyAsString();
      }
    } catch (SocketTimeoutException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (SocketException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (Exception ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);

    } finally {
      getmethod.releaseConnection();
    }
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;

    // }
    return pageSource;
  }