コード例 #1
0
ファイル: SimpleHttpClient.java プロジェクト: chenjw/spider
 private org.apache.commons.httpclient.HttpClient createHttpClient() {
   HttpClientParams clientParams = new HttpClientParams();
   clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
   org.apache.commons.httpclient.HttpClient httpClient =
       new org.apache.commons.httpclient.HttpClient(clientParams, connManager);
   return httpClient;
 }
コード例 #2
0
ファイル: HttpClient.java プロジェクト: Araell/prototype
  public HttpClient(int maxConPerHost, int conTimeOutMs, int soTimeOutMs, int maxSize) {
    connectionManager = new MultiThreadedHttpConnectionManager();
    HttpConnectionManagerParams params = connectionManager.getParams();
    params.setDefaultMaxConnectionsPerHost(maxConPerHost);
    params.setConnectionTimeout(conTimeOutMs);
    params.setSoTimeout(soTimeOutMs);

    HttpClientParams clientParams = new HttpClientParams();
    // 忽略cookie 避免 Cookie rejected 警告
    clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
    client = new org.apache.commons.httpclient.HttpClient(clientParams, connectionManager);
    Protocol myhttps = new Protocol("https", new MySSLSocketFactory(), 443);
    Protocol.registerProtocol("https", myhttps);
    this.maxSize = maxSize;
    // 支持proxy
    if (proxyHost != null && !proxyHost.equals("")) {
      client.getHostConfiguration().setProxy(proxyHost, proxyPort);
      client.getParams().setAuthenticationPreemptive(true);
      if (proxyAuthUser != null && !proxyAuthUser.equals("")) {
        client
            .getState()
            .setProxyCredentials(
                AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword));
        log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword);
      }
    }
  }
コード例 #3
0
 /**
  * Method to make a GET HTTP connecton to the given url and return the output
  *
  * @param urlToFetch url to be connected
  * @return the http get response
  */
 public String makeRequest(String urlToFetch, String region) throws IOException {
   String responseBody = "";
   try {
     HttpClientParams clientParams = new HttpClientParams();
     clientParams.setSoTimeout(40000);
     clientParams.setConnectionManagerTimeout(40000);
     HttpClient httpclient = new HttpClient(clientParams);
     GetMethod httpget = new GetMethod(urlToFetch);
     if (urlToFetch.contains("linkedin.com/countserv")) {
       httpget.addRequestHeader("Host", "www.linkedin.com");
       httpget.addRequestHeader(
           "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:20.0) Gecko/20100101 Firefox/20.0");
       httpget.addRequestHeader(
           "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
       httpget.addRequestHeader(
           "Cookie",
           "X-LI-IDC=C1; bcookie=\"v=2&618ed56d-275f-4dcd-86cd-6a44c8421879\"; bscookie=\"v=1&201305170926410ddaf2a1-e238-4bf6-8a0a-ce8a3df25934AQFZX82SfErJuFQKMJV49JfQTvewhfzh\"; X-LI-IDC=C1");
     }
     //            httpclient.getHostConfiguration().setProxy("46.227.68.2", 3128);
     //            Credentials cred = new UsernamePasswordCredentials("mmongoose",
     // "I-PHNBV9JHW6US");
     //            httpclient.getState().setProxyCredentials(AuthScope.ANY, cred);
     int i = httpclient.executeMethod(httpget);
     responseBody = httpget.getResponseBodyAsString();
   } catch (Exception e) {
     l.error(e + "  " + e.getMessage() + " url " + urlToFetch);
   }
   return responseBody;
 }
コード例 #4
0
ファイル: HTTPSession.java プロジェクト: feihugis/NetCDF
  protected void construct(String legalurl) throws HTTPException {
    this.legalurl = legalurl;
    try {
      sessionClient = new HttpClient(connmgr);
      HttpClientParams clientparams = sessionClient.getParams();

      // Allow (circular) redirects
      clientparams.setParameter(ALLOW_CIRCULAR_REDIRECTS, true);
      clientparams.setParameter(MAX_REDIRECTS, 25);

      if (globalSoTimeout > 0) setSoTimeout(globalSoTimeout);

      if (globalConnectionTimeout > 0) setConnectionTimeout(globalConnectionTimeout);

      if (globalAgent != null) setUserAgent(globalAgent); // May get overridden by setUserAgent

      setAuthenticationPreemptive(globalauthpreemptive);

      setProxy();

      if (TESTING) HTTPSession.track(this);

    } catch (Exception e) {
      throw new HTTPException("url=" + legalurl, e);
    }
  }
コード例 #5
0
  public HttpImpl() {

    // Mimic behavior found in
    // http://java.sun.com/j2se/1.5.0/docs/guide/net/properties.html

    if (Validator.isNotNull(_NON_PROXY_HOSTS)) {
      String nonProxyHostsRegEx = _NON_PROXY_HOSTS;

      nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\.", "\\\\.");
      nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\*", ".*?");
      nonProxyHostsRegEx = nonProxyHostsRegEx.replaceAll("\\|", ")|(");

      nonProxyHostsRegEx = "(" + nonProxyHostsRegEx + ")";

      _nonProxyHostsPattern = Pattern.compile(nonProxyHostsRegEx);
    }

    MultiThreadedHttpConnectionManager httpConnectionManager =
        new MultiThreadedHttpConnectionManager();

    HttpConnectionManagerParams httpConnectionManagerParams = httpConnectionManager.getParams();

    httpConnectionManagerParams.setConnectionTimeout(_TIMEOUT);
    httpConnectionManagerParams.setDefaultMaxConnectionsPerHost(
        new Integer(_MAX_CONNECTIONS_PER_HOST));
    httpConnectionManagerParams.setMaxTotalConnections(new Integer(_MAX_TOTAL_CONNECTIONS));
    httpConnectionManagerParams.setSoTimeout(_TIMEOUT);

    _httpClient.setHttpConnectionManager(httpConnectionManager);
    _proxyHttpClient.setHttpConnectionManager(httpConnectionManager);

    if (hasProxyConfig() && Validator.isNotNull(_PROXY_USERNAME)) {
      List<String> authPrefs = new ArrayList<String>();

      if (_PROXY_AUTH_TYPE.equals("username-password")) {
        _proxyCredentials = new UsernamePasswordCredentials(_PROXY_USERNAME, _PROXY_PASSWORD);

        authPrefs.add(AuthPolicy.BASIC);
        authPrefs.add(AuthPolicy.DIGEST);
        authPrefs.add(AuthPolicy.NTLM);
      } else if (_PROXY_AUTH_TYPE.equals("ntlm")) {
        _proxyCredentials =
            new NTCredentials(
                _PROXY_USERNAME, _PROXY_PASSWORD, _PROXY_NTLM_HOST, _PROXY_NTLM_DOMAIN);

        authPrefs.add(AuthPolicy.NTLM);
        authPrefs.add(AuthPolicy.BASIC);
        authPrefs.add(AuthPolicy.DIGEST);
      }

      HttpClientParams httpClientParams = _proxyHttpClient.getParams();

      httpClientParams.setParameter(AuthPolicy.AUTH_SCHEME_PRIORITY, authPrefs);
    }
  }
コード例 #6
0
 /**
  * Get an instance of the HTTP client to use.
  *
  * @return HTTP client initialized with admin credentials
  */
 protected static HttpClient getClient() {
   if (AbstractEscapingTest.client == null) {
     HttpClient adminClient = new HttpClient();
     Credentials defaultcreds = new UsernamePasswordCredentials("Admin", "admin");
     adminClient.getState().setCredentials(AuthScope.ANY, defaultcreds);
     HttpClientParams clientParams = new HttpClientParams();
     clientParams.setSoTimeout(2000);
     HttpConnectionManagerParams connectionParams = new HttpConnectionManagerParams();
     connectionParams.setConnectionTimeout(30000);
     adminClient.getHttpConnectionManager().setParams(connectionParams);
     AbstractEscapingTest.client = adminClient;
   }
   return AbstractEscapingTest.client;
 }
コード例 #7
0
ファイル: HttpUtils.java プロジェクト: tiansiyuan/misc
 /** 初始化httpclient */
 static {
   int connectionTimeout = 30000;
   int soTimeout = 30000;
   try {
     connectionTimeout =
         Integer.parseInt(System.getProperty("sun.net.client.defaultConnectTimeout", "30000"));
   } catch (Exception e) {
   }
   try {
     soTimeout =
         Integer.parseInt(System.getProperty("sun.net.client.defaultReadTimeout", "30000"));
   } catch (Exception e) {
   }
   MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
   connectionManager.getParams().setDefaultMaxConnectionsPerHost(10);
   connectionManager.getParams().setMaxTotalConnections(300);
   connectionManager.getParams().setConnectionTimeout(connectionTimeout);
   connectionManager.getParams().setSoTimeout(soTimeout);
   client.setHttpConnectionManager(connectionManager);
   // 忽略cookie 避免 Cookie rejected 警告
   HttpClientParams clientParams = new HttpClientParams();
   clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
   client.setParams(clientParams);
   // 支持https
   Protocol myhttps = new Protocol("https", new SSLSocketFactory(), 443);
   Protocol.registerProtocol("https", myhttps);
   // 设置代理
   if (ProxyClient.getProxy() != null) {
     client.getHostConfiguration().setProxy(ProxyClient.getHost(), ProxyClient.getPort());
     client.getParams().setAuthenticationPreemptive(true);
     if (ProxyClient.getUsername() != null && !ProxyClient.getUsername().trim().equals("")) {
       client
           .getState()
           .setProxyCredentials(
               AuthScope.ANY,
               new UsernamePasswordCredentials(
                   ProxyClient.getUsername().trim(), ProxyClient.getPassword().trim()));
     }
   }
 }
コード例 #8
0
ファイル: CFActivator.java プロジェクト: eclipse/orion.server
  private HttpClient createHttpClient() {
    // see http://hc.apache.org/httpclient-3.x/threading.html
    MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
    HttpConnectionManagerParams params = connectionManager.getParams();
    params.setMaxConnectionsPerHost(
        HostConfiguration.ANY_HOST_CONFIGURATION,
        PreferenceHelper.getInt(ServerConstants.HTTP_MAX_CONN_HOST_CONF_KEY, 50));
    params.setMaxTotalConnections(
        PreferenceHelper.getInt(ServerConstants.HTTP_MAX_CONN_TOTAL_CONF_KEY, 150));
    params.setConnectionTimeout(
        PreferenceHelper.getInt(ServerConstants.HTTP_CONN_TIMEOUT_CONF_KEY, 15000)); // 15s
    params.setSoTimeout(
        PreferenceHelper.getInt(ServerConstants.HTTP_SO_TIMEOUT_CONF_KEY, 30000)); // 30s
    connectionManager.setParams(params);

    HttpClientParams clientParams = new HttpClientParams();
    clientParams.setConnectionManagerTimeout(
        PreferenceHelper.getInt(
            ServerConstants.HTTP_CONN_MGR_TIMEOUT_CONF_KEY, 300000)); // 5 minutes

    return new HttpClient(clientParams, connectionManager);
  }
コード例 #9
0
  /**
   * Creates a new connection to the server.
   *
   * @param builder The HttpFileSystemConfigBuilder.
   * @param scheme The protocol.
   * @param hostname The hostname.
   * @param port The port number.
   * @param username The username.
   * @param password The password
   * @param fileSystemOptions The file system options.
   * @return a new HttpClient connection.
   * @throws FileSystemException if an error occurs.
   * @since 2.0
   */
  public static HttpClient createConnection(
      HttpFileSystemConfigBuilder builder,
      String scheme,
      String hostname,
      int port,
      String username,
      String password,
      FileSystemOptions fileSystemOptions)
      throws FileSystemException {
    HttpClient client;
    try {
      HttpConnectionManager mgr = new MultiThreadedHttpConnectionManager();
      HttpConnectionManagerParams connectionMgrParams = mgr.getParams();

      client = new HttpClient(mgr);

      final HostConfiguration config = new HostConfiguration();
      config.setHost(hostname, port, scheme);

      if (fileSystemOptions != null) {
        String proxyHost = builder.getProxyHost(fileSystemOptions);
        int proxyPort = builder.getProxyPort(fileSystemOptions);

        if (proxyHost != null && proxyHost.length() > 0 && proxyPort > 0) {
          config.setProxy(proxyHost, proxyPort);
        }

        UserAuthenticator proxyAuth = builder.getProxyAuthenticator(fileSystemOptions);
        if (proxyAuth != null) {
          UserAuthenticationData authData =
              UserAuthenticatorUtils.authenticate(
                  proxyAuth,
                  new UserAuthenticationData.Type[] {
                    UserAuthenticationData.USERNAME, UserAuthenticationData.PASSWORD
                  });

          if (authData != null) {
            final UsernamePasswordCredentials proxyCreds =
                new UsernamePasswordCredentials(
                    UserAuthenticatorUtils.toString(
                        UserAuthenticatorUtils.getData(
                            authData, UserAuthenticationData.USERNAME, null)),
                    UserAuthenticatorUtils.toString(
                        UserAuthenticatorUtils.getData(
                            authData, UserAuthenticationData.PASSWORD, null)));

            AuthScope scope = new AuthScope(proxyHost, AuthScope.ANY_PORT);
            client.getState().setProxyCredentials(scope, proxyCreds);
          }

          if (builder.isPreemptiveAuth(fileSystemOptions)) {
            HttpClientParams httpClientParams = new HttpClientParams();
            httpClientParams.setAuthenticationPreemptive(true);
            client.setParams(httpClientParams);
          }
        }

        Cookie[] cookies = builder.getCookies(fileSystemOptions);
        if (cookies != null) {
          client.getState().addCookies(cookies);
        }
      }
      /**
       * ConnectionManager set methodsmust be called after the host & port and proxy host & port are
       * set in the HostConfiguration. They are all used as part of the key when
       * HttpConnectionManagerParams tries to locate the host configuration.
       */
      connectionMgrParams.setMaxConnectionsPerHost(
          config, builder.getMaxConnectionsPerHost(fileSystemOptions));
      connectionMgrParams.setMaxTotalConnections(builder.getMaxTotalConnections(fileSystemOptions));

      client.setHostConfiguration(config);

      if (username != null) {
        final UsernamePasswordCredentials creds =
            new UsernamePasswordCredentials(username, password);
        AuthScope scope = new AuthScope(hostname, AuthScope.ANY_PORT);
        client.getState().setCredentials(scope, creds);
      }

      client.executeMethod(new HeadMethod());
    } catch (final Exception exc) {
      throw new FileSystemException(
          "vfs.provider.http/connect.error", new Object[] {hostname}, exc);
    }

    return client;
  }
コード例 #10
0
  /**
   * The method reads a proxy object from database, makes a GetMethod object, appends required
   * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
   * pagesource to the caller.
   *
   * @param iCount Counter variable for passing thread group information
   * @param url Url to fetch the pagesource for
   * @param followRedirect Boolean variable to specify GetMethod followRedirect value
   * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
   * @param region the local region of a given url
   * @param objProxyDao the database layer ProxyDao object variable
   * @param useErrsy Boolean variable to specify usage of Errsy as proxy source
   * @return String
   */
  public String getPageSourceWithProxy(
      String url,
      boolean followRedirect,
      boolean doAuthentication,
      String region,
      Boolean useErrsy,
      String google) {

    String page = " ";
    String pageSource = "";
    int i = 0;
    String exception = " ";
    HttpClientParams clientParams = new HttpClientParams();
    clientParams.setSoTimeout(40000);
    clientParams.setConnectionManagerTimeout(40000);
    HttpClient httpclient = new HttpClient(clientParams);
    GetMethod getmethod = null;

    HttpState state = new HttpState();

    //  if (ProxyDao.lstProxyData.size() == 16) {
    ProxyData objProxyData = null;
    // if (!useErrsy) {
    try {
      // objProxyData = ProxyDao.lstProxyData.get(iCount);
      objProxyData = ProxyDao.objProxyData;
      if (objProxyData == null) {
        //                objProxyDao.changeProxy(google);
        objProxyData = ProxyDao.objProxyData;
      }
      httpclient
          .getHostConfiguration()
          .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
    } catch (Exception e) {
      pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
      return pageSource;
    }
    /*} else {
    try {
    objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
    Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
    httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
    httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
    state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
    httpclient.setState(state);
    } catch (Exception e) {
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
    return pageSource;
    }
    }*/
    try {
      getmethod = new GetMethod(url);
      getmethod.addRequestHeader(
          "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
      if (url.contains("bing.com")) {

        if (region.equalsIgnoreCase("co.uk")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.sg")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.au")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("co.in")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("ca")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.ph")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.my")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("it")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        }
      }
      getmethod.setFollowRedirects(true);
      getmethod.setDoAuthentication(true);
      httpclient.getParams().setAuthenticationPreemptive(true);
      httpclient.setState(state);
      String num100Header = "";
      //            if (url.contains("google")) {
      //                int j = 0;
      //                String url1 = "http://www.google.com/";
      //                try {
      //                    GetMethod objGetMethod = new GetMethod(url1);
      //                    j = httpclient.executeMethod(objGetMethod);
      //                    Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
      //                    String header = responseHeader.getValue();
      //                    String[] headerValue = header.split(";");
      //
      //                    for (String head : headerValue) {
      //                        if (head.contains("PREF=ID")) {
      //                            header = head;
      //                            break;
      //                        }
      //                    }
      //                    String[] splitAll = header.split(":");
      //                    long time = System.currentTimeMillis()+400;
      //                    String sTime = "" + time;
      //                    sTime = sTime.substring(0, 10);
      //                    //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1]  +
      // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
      //                    num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
      // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
      //                    Cookie ck = new Cookie("PREF", "PREF", num100Header);
      //                    httpclient.getState().clearCookies();
      //                    httpclient.getState().addCookie(ck);
      //                    getmethod.addRequestHeader("Host", "www.google.com");
      //                    getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
      // rv:19.0) Gecko/20100101 Firefox/19.0");
      //                    getmethod.addRequestHeader("Accept",
      // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
      //                    getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
      //                    getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
      //                    getmethod.addRequestHeader("Referer", "https://www.google.com/");
      //                    System.out.println(num100Header);
      //                } catch (Exception ex) {
      //                    exception = ex.getMessage();
      //                    l.debug(ex + "  " + ex.getMessage() + "Exception occured for url" +
      // url);
      //                    pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
      //                    return pageSource;
      //                }
      //            }
      i = httpclient.executeMethod(getmethod);
      if (i / 100 == 4 || i / 100 == 5) {
        page = "<PROXY ERROR>";

      } else {

        page = getmethod.getResponseBodyAsString();
      }
    } catch (SocketTimeoutException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (SocketException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (Exception ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);

    } finally {
      getmethod.releaseConnection();
    }
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;

    // }
    return pageSource;
  }
コード例 #11
0
ファイル: HTTPSession.java プロジェクト: feihugis/NetCDF
 public void setMaxRedirects(int n) {
   HttpClientParams clientparams = sessionClient.getParams();
   clientparams.setParameter(MAX_REDIRECTS, n);
 }