private HostConfiguration getHostConfiguration(HttpClient client, Map<String, Object> props) {
    Object proxy = props.get(ApacheHttpClientConfig.PROPERTY_PROXY_URI);
    if (proxy != null) {
      URI proxyUri = getProxyUri(proxy);

      String proxyHost = proxyUri.getHost();
      if (proxyHost == null) {
        proxyHost = "localhost";
      }

      int proxyPort = proxyUri.getPort();
      if (proxyPort == -1) {
        proxyPort = 8080;
      }

      HostConfiguration hostConfig = new HostConfiguration(client.getHostConfiguration());
      String setHost = hostConfig.getProxyHost();
      int setPort = hostConfig.getProxyPort();

      if ((setHost == null)
          || (!setHost.equals(proxyHost))
          || (setPort == -1)
          || (setPort != proxyPort)) {
        hostConfig.setProxyHost(new ProxyHost(proxyHost, proxyPort));
      }
      return hostConfig;
    } else {
      return null;
    }
  }
  public String getContextByPostMethod(String url, List<KeyValue> params) {
    HttpClient client = getHttpClient();

    client.getParams().setParameter("http.protocol.content-charset", this.codeing);

    PostMethod post = null;
    String result = "";
    try {
      URL u = new URL(url);
      client
          .getHostConfiguration()
          .setHost(
              u.getHost(), u.getPort() == -1 ? u.getDefaultPort() : u.getPort(), u.getProtocol());

      post = new PostMethod(u.getPath());

      NameValuePair[] nvps = new NameValuePair[params.size()];
      int i = 0;
      for (KeyValue kv : params) {
        nvps[i] = new NameValuePair(kv.getKey(), kv.getValue());
        i++;
      }

      post.setRequestBody(nvps);

      client.executeMethod(post);
      result = post.getResponseBodyAsString();
    } catch (Exception e) {
      throw new NetException("HttpClient catch!", e);
    } finally {
      if (post != null) post.releaseConnection();
    }
    return result;
  }
Esempio n. 3
0
  /** Get or create the http client to be used to fetch all the map data. */
  public HttpClient getHttpClient(URI uri) {
    MultiThreadedHttpConnectionManager connectionManager = getConnectionManager();
    HttpClient httpClient = new HttpClient(connectionManager);

    // httpclient is a bit pesky about loading everything in memory...
    // disabling the warnings.
    Logger.getLogger(HttpMethodBase.class).setLevel(Level.ERROR);

    // configure proxies for URI
    ProxySelector selector = ProxySelector.getDefault();

    List<Proxy> proxyList = selector.select(uri);
    Proxy proxy = proxyList.get(0);

    if (!proxy.equals(Proxy.NO_PROXY)) {
      InetSocketAddress socketAddress = (InetSocketAddress) proxy.address();
      String hostName = socketAddress.getHostName();
      int port = socketAddress.getPort();

      httpClient.getHostConfiguration().setProxy(hostName, port);
    }

    for (SecurityStrategy sec : security)
      if (sec.matches(uri)) {
        sec.configure(uri, httpClient);
        break;
      }
    return httpClient;
  }
Esempio n. 4
0
  public static DAOLayerInterface getDAOLayer(ProxySettings proxySettings, ServerSession session)
      throws Exception {

    HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager());

    if (proxySettings != null) {
      Credentials credentials =
          new UsernamePasswordCredentials(proxySettings.USERNAME, proxySettings.PASSWORD);

      AuthScope authScope = new AuthScope(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT);

      httpClient
          .getHostConfiguration()
          .setProxy(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT);

      httpClient.getState().setProxyCredentials(authScope, credentials);
    }

    try {
      db =
          (DAOLayerInterface)
              ServerRuntimeImpl.getDynamicProxy(
                  new ServerSession(session.getCommandurl(), null),
                  "dbinvoke", // commandurl
                  "DAOLAYER",
                  new Class<?>[] {DAOLayerInterface.class},
                  httpClient);

    } catch (Exception ex) {
      ex.printStackTrace();
    }

    return db;
  }
Esempio n. 5
0
  public HttpClient(int maxConPerHost, int conTimeOutMs, int soTimeOutMs, int maxSize) {
    connectionManager = new MultiThreadedHttpConnectionManager();
    HttpConnectionManagerParams params = connectionManager.getParams();
    params.setDefaultMaxConnectionsPerHost(maxConPerHost);
    params.setConnectionTimeout(conTimeOutMs);
    params.setSoTimeout(soTimeOutMs);

    HttpClientParams clientParams = new HttpClientParams();
    // 忽略cookie 避免 Cookie rejected 警告
    clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
    client = new org.apache.commons.httpclient.HttpClient(clientParams, connectionManager);
    Protocol myhttps = new Protocol("https", new MySSLSocketFactory(), 443);
    Protocol.registerProtocol("https", myhttps);
    this.maxSize = maxSize;
    // 支持proxy
    if (proxyHost != null && !proxyHost.equals("")) {
      client.getHostConfiguration().setProxy(proxyHost, proxyPort);
      client.getParams().setAuthenticationPreemptive(true);
      if (proxyAuthUser != null && !proxyAuthUser.equals("")) {
        client
            .getState()
            .setProxyCredentials(
                AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword));
        log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword);
      }
    }
  }
Esempio n. 6
0
 /** Create a new Client. The client will not use authentication by default. */
 public Client() {
   client = new HttpClient();
   client.getParams().setParameter("http.socket.timeout", Integer.valueOf(DEFAULT_TIMEOUT));
   log.debug("proxy host: " + client.getHostConfiguration().getProxyHost());
   log.debug("proxy port: " + client.getHostConfiguration().getProxyPort());
   doAuthentication = false;
 }
  @NotNull
  private static HttpClient getHttpClient(
      @Nullable final String login, @Nullable final String password) {
    final HttpClient client = new HttpClient();
    HttpConnectionManagerParams params = client.getHttpConnectionManager().getParams();
    params.setConnectionTimeout(
        CONNECTION_TIMEOUT); // set connection timeout (how long it takes to connect to remote host)
    params.setSoTimeout(
        CONNECTION_TIMEOUT); // set socket timeout (how long it takes to retrieve data from remote
                             // host)

    client.getParams().setContentCharset("UTF-8");
    // Configure proxySettings if it is required
    final HttpConfigurable proxySettings = HttpConfigurable.getInstance();
    if (proxySettings.USE_HTTP_PROXY && !StringUtil.isEmptyOrSpaces(proxySettings.PROXY_HOST)) {
      client.getHostConfiguration().setProxy(proxySettings.PROXY_HOST, proxySettings.PROXY_PORT);
      if (proxySettings.PROXY_AUTHENTICATION) {
        client
            .getState()
            .setProxyCredentials(
                AuthScope.ANY,
                new UsernamePasswordCredentials(
                    proxySettings.PROXY_LOGIN, proxySettings.getPlainProxyPassword()));
      }
    }
    if (login != null && password != null) {
      client.getParams().setCredentialCharset("UTF-8");
      client.getParams().setAuthenticationPreemptive(true);
      client
          .getState()
          .setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(login, password));
    }
    return client;
  }
Esempio n. 8
0
  public Response multPartURL(
      String url, PostParameter[] params, ImageItem item, boolean authenticated)
      throws WeiboException {
    PostMethod post = new PostMethod(url);
    try {
      org.apache.commons.httpclient.HttpClient client = getHttpClient();
      long t = System.currentTimeMillis();
      Part[] parts = null;
      if (params == null) {
        parts = new Part[1];
      } else {
        parts = new Part[params.length + 1];
      }
      if (params != null) {
        int i = 0;
        for (PostParameter entry : params) {
          parts[i++] = new StringPart(entry.getName(), (String) entry.getValue());
        }
        parts[parts.length - 1] =
            new ByteArrayPart(item.getContent(), item.getName(), item.getContentType());
      }
      post.setRequestEntity(new MultipartRequestEntity(parts, post.getParams()));
      List<Header> headers = new ArrayList<Header>();

      if (authenticated) {
        if (oauth == null) {}
        String authorization = null;
        if (null != oauth) {
          // use OAuth
          authorization = oauth.generateAuthorizationHeader("POST", url, params, oauthToken);
        } else {
          throw new IllegalStateException(
              "Neither user ID/password combination nor OAuth consumer key/secret combination supplied");
        }
        headers.add(new Header("Authorization", authorization));
        log("Authorization: " + authorization);
      }
      client.getHostConfiguration().getParams().setParameter("http.default-headers", headers);
      client.executeMethod(post);

      Response response = new Response();
      response.setResponseAsString(post.getResponseBodyAsString());
      response.setStatusCode(post.getStatusCode());

      log(
          "multPartURL URL:"
              + url
              + ", result:"
              + response
              + ", time:"
              + (System.currentTimeMillis() - t));
      return response;
    } catch (Exception ex) {
      throw new WeiboException(ex.getMessage(), ex, -1);
    } finally {
      post.releaseConnection();
    }
  }
Esempio n. 9
0
 public static void setProxy(HttpClient httpClient) {
   String proxyHost = getValueStr(proxy_host);
   Logger.info("proxy_host:" + proxyHost);
   if (proxyHost != null) {
     int proxyPort = getValueInt(proxy_port);
     Logger.info(" proxy_port:" + proxyPort);
     httpClient.getHostConfiguration().setProxy(proxyHost, proxyPort);
   }
 }
Esempio n. 10
0
  public Response httpRequest(HttpMethod method, Boolean WithTokenHeader, String token)
      throws WeiboException {
    InetAddress ipaddr;
    int responseCode = -1;
    try {
      ipaddr = InetAddress.getLocalHost();
      List<Header> headers = new ArrayList<Header>();
      if (WithTokenHeader) {
        if (token == null) {
          throw new IllegalStateException("Oauth2 token is not set!");
        }
        headers.add(new Header("Authorization", "OAuth2 " + token));
        headers.add(new Header("API-RemoteIP", ipaddr.getHostAddress()));
        client.getHostConfiguration().getParams().setParameter("http.default-headers", headers);
        for (Header hd : headers) {
          log(hd.getName() + ": " + hd.getValue());
        }
      }

      method
          .getParams()
          .setParameter(
              HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
      client.executeMethod(method);
      Header[] resHeader = method.getResponseHeaders();
      responseCode = method.getStatusCode();
      log("Response:");
      log("https StatusCode:" + String.valueOf(responseCode));

      for (Header header : resHeader) {
        log(header.getName() + ":" + header.getValue());
      }
      Response response = new Response();
      response.setResponseAsString(method.getResponseBodyAsString());
      log(response.toString() + "\n");

      if (responseCode != OK) {
        try {
          throw new WeiboException(
              getCause(responseCode), response.asJSONObject(), method.getStatusCode());
        } catch (JSONException e) {
          e.printStackTrace();
        }
      }
      return response;

    } catch (IOException ioe) {
      throw new WeiboException(ioe.getMessage(), ioe, responseCode);
    } finally {
      method.releaseConnection();
    }
  }
 public RestaurantRestClient(String urlStub) {
   httpClient.getHttpConnectionManager().getParams().setMaxTotalConnections(10);
   httpClient
       .getHttpConnectionManager()
       .getParams()
       .setMaxConnectionsPerHost(httpClient.getHostConfiguration(), 10);
   restClient.setErrorHandler(new MyErrorHandler());
   this.urlStub = urlStub;
   //     List<HttpMessageConverter<?>> converters = new ArrayList<>();
   //    converters.add(converter);
   //     restClient.setMessageConverters(converters);
   // http://localhost:8888/app/backbone/restaurant/
 }
Esempio n. 12
0
  @Test
  public void testSendSMS() {
    String reqUrl = "http://59.36.98.25/JSMSWebServiceNew/send.aspx";
    long start = System.currentTimeMillis();

    // String requestJson =
    // "{\"code\":\"GDCN\",\"destPhone\":\"15018784308\",\"content\":\"文字短信测试\",\"corpSignName\":\"公司签名\",\"userSignName\":\"用户签名\",\"corpAccount\":\"system\",\"userAccount\":\"admin\",\"password\":\"7236606dcece6d04ebd8a7b42eb247e0\",\"messageType\":\"1\"}";
    try {
      HttpClient client = new HttpClient();
      client.getHostConfiguration().setHost("localhost", 8080, "http");
      PostMethod post = new PostMethod(reqUrl);

      post.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8");

      post.setParameter("code", "GDCN");
      post.setParameter("destPhone", "15018784308");
      post.setParameter(
          "content",
          "尽力总有进步,尽心总有收获,努力总有成效,付出总有回报,前行总有风景,坚持总有希望,豁达总有快乐,拼搏总有运气。有时闲,有时累,不怨不悔,把快乐品味,愿你时时与幸福相会!");
      post.setParameter("corpSignName", "中数通");
      // post.setParameter("userSignName", "用户签名");
      post.setParameter("corpAccount", "system");
      post.setParameter("userAccount", "admin");
      post.setParameter("password", "7236606dcece6d04ebd8a7b42eb247e0");
      post.setParameter("messageType", "1");

      /*String y=URLEncoder.encode("尽力总有进步,尽心总有收获,努力总有成效,付出总有回报,前行总有风景,坚持总有希望,豁达总有快乐,拼搏总有运气。有时闲,有时累,不怨不悔,把快乐品味,愿你时时与幸福相会!","utf-8");
      System.out.println(y);
      String s=URLDecoder.decode("%e4%b8%8b%e8%a1%8c%e6%b6%88%e6%81%af", "utf-8");
      System.out.println(s);*/

      // System.out.println("请求Json: " + requestJson);

      // post.setRequestBody(new ByteArrayInputStream(requestJson)));

      /*System.out.println("请求Json的getBytes: " + EncodeUtils
      .base64Encode(requestJson.getBytes("UTF-8")).getBytes());*/

      int result = client.executeMethod(post);

      if (result == 200) {
        String responseJson = post.getResponseBodyAsString();
        System.out.println("响应JSON: " + responseJson);
      }
    } catch (Exception e) {
      e.printStackTrace();
    }
    long end = System.currentTimeMillis();
    System.out.println("zhixingshijian:>>>>>>>>>" + (end - start));
  }
Esempio n. 13
0
  /**
   * Obtem a timeline publica. Nao necessita de autenticacao, basicamente é um GET na URL
   * http://twitter.com/statuses/public_timeline.xml
   */
  public String getPublicTimeline() throws Exception {
    HttpClient client = new HttpClient();

    // get para obter a timeline publica
    GetMethod get = new GetMethod("/statuses/public_timeline.xml");

    // Criando o host, para onde sera enviada a requisicao
    HostConfiguration host = client.getHostConfiguration();
    host.setHost(new URI("http://twitter.com"));

    // executando o get na url de timeline publica do twitter
    client.executeMethod(host, get);

    // obtendo a resposta
    return get.getResponseBodyAsString();
  }
Esempio n. 14
0
 public static void main(String[] args) throws Exception {
   HttpClient client = new HttpClient();
   client.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF8");
   // 原先采用自己合并COOKIE,但是出现问题,用此方法OK
   DefaultHttpParams.getDefaultParams()
       .setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
   client
       .getParams()
       .setParameter(
           HttpMethodParams.USER_AGENT,
           "Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) Gecko/20090803 Fedora/3.5.2-2.fc11 Firefox/3.5.2");
   client.getHostConfiguration().setHost(LOGON_SITE, LOGON_PORT);
   String ajaxLoginUrl = Login(client, preLogin(client));
   String uniqueid = ajaxLogin(client, ajaxLoginUrl);
   getHomePage(client, homePageUrl + uniqueid);
 }
Esempio n. 15
0
  /**
   * Sends a new HTTP request to a server through a proxy.
   *
   * @param requestParameters a set of parameters that will set the content of the request and
   *     specify the proxy it should go through
   */
  public void callWilmaTestServer(
      final RequestParameters requestParameters, final TestClientParameters clientParameters) {
    try {
      HttpClient httpClient = new HttpClient();
      PostMethod httpPost = new PostMethod(requestParameters.getTestServerUrl());
      if (clientParameters.isUseProxy()) {
        httpClient
            .getHostConfiguration()
            .setProxy(requestParameters.getWilmaHost(), requestParameters.getWilmaPort());
      }
      InputStream inputStream = requestParameters.getInputStream();
      if (requestParameters.getContentType().contains("fastinfoset")) {
        inputStream = compress(inputStream);
      }
      if (requestParameters.getContentEncoding().contains("gzip")) {
        inputStream = encode(inputStream);
        httpPost.setRequestHeader("Content-Encoding", requestParameters.getContentEncoding());
      }
      InputStreamRequestEntity entity =
          new InputStreamRequestEntity(inputStream, requestParameters.getContentType());
      httpPost.setRequestEntity(entity);
      httpPost.setRequestHeader("Accept", requestParameters.getAcceptHeader());
      httpPost.addRequestHeader("Accept-Encoding", requestParameters.getAcceptEncoding());
      // httpPost.addRequestHeader("0", "WilmaBypass=true");

      httpClient
          .getHttpConnectionManager()
          .getParams()
          .setSendBufferSize(clientParameters.getRequestBufferSize());
      httpClient
          .getHttpConnectionManager()
          .getParams()
          .setReceiveBufferSize(clientParameters.getResponseBufferSize());

      int statusCode = httpClient.executeMethod(httpPost);
      logger.info("status code: " + statusCode);
      if (clientParameters.getAllowResponseLogging()) {
        logger.info(getInputStreamAsString(httpPost.getResponseBodyAsStream()));
      }
    } catch (UnsupportedEncodingException e) {
      throw new SystemException("Unsupported encoding.", e);
    } catch (HttpException e) {
      throw new SystemException("Http exception occurred.", e);
    } catch (IOException e) {
      throw new SystemException("InputStream cannot be read.", e);
    }
  }
Esempio n. 16
0
 private org.apache.commons.httpclient.HttpClient getHttpClient() {
   org.apache.commons.httpclient.HttpClient client =
       new org.apache.commons.httpclient.HttpClient();
   if (proxyHost != null && !proxyHost.equals("")) {
     client.getHostConfiguration().setProxy(proxyHost, proxyPort);
     client.getParams().setAuthenticationPreemptive(true);
     if (proxyAuthUser != null && !proxyAuthUser.equals("")) {
       client
           .getState()
           .setProxyCredentials(
               AuthScope.ANY, new UsernamePasswordCredentials(proxyAuthUser, proxyAuthPassword));
       log("Proxy AuthUser: "******"Proxy AuthPassword: " + proxyAuthPassword);
     }
   }
   return client;
 }
Esempio n. 17
0
 /**
  * 提交
  *
  * @param method
  * @param token
  * @return
  * @throws ApiException
  */
 private static String httpRequest(HttpMethod method, String token) throws IOException {
   try {
     List<Header> headers = new ArrayList<Header>();
     if (token != null) {
       headers.add(new Header("Authorization", "Bearer " + token));
       client.getHostConfiguration().getParams().setParameter("http.default-headers", headers);
     }
     method
         .getParams()
         .setParameter(
             HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
     client.executeMethod(method);
     return method.getResponseBodyAsString();
   } catch (IOException ioe) {
     throw ioe;
   } finally {
     method.releaseConnection();
   }
 }
Esempio n. 18
0
  /**
   * 登录
   *
   * @return
   * @throws IOException
   * @throws HttpException
   */
  public static HttpClient login() throws IOException, HttpException {
    HttpClient client = new HttpClient();
    client.getHostConfiguration().setHost("www.catarc.org.cn");
    client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
    NameValuePair[] data = {
      new NameValuePair(
          "User-Agent", "Mozilla/5.0 (Windows NT 5.1; rv:25.0) Gecko/20100101 Firefox/25.0"),
      new NameValuePair("__VIEWSTATE", "/wEPDwUJNzE2NjczNjA4ZGQCaV1Dt0DOUg5r7O3ZD+sGL1Hq6Q=="),
      new NameValuePair(
          "__EVENTVALIDATION",
          "/wEWBQK/2uyqBgKl1bKzCQK1qbSRCwLZpaCZAwKTuvSuCTYE9T8nbPNCPEY2eqmCy5I3BwtG"),
      new NameValuePair("txtUserName", "asri"),
      new NameValuePair("txtPassword", "asri"),
      new NameValuePair("loginBtn", "%B5%C7%C2%BD")
    };
    PostMethod post = new PostMethod("http://www.catarc.org.cn/standard/Default.aspx");
    post.setRequestBody(data);
    client.executeMethod(post);

    return client;
  }
Esempio n. 19
0
 /** 初始化httpclient */
 static {
   int connectionTimeout = 30000;
   int soTimeout = 30000;
   try {
     connectionTimeout =
         Integer.parseInt(System.getProperty("sun.net.client.defaultConnectTimeout", "30000"));
   } catch (Exception e) {
   }
   try {
     soTimeout =
         Integer.parseInt(System.getProperty("sun.net.client.defaultReadTimeout", "30000"));
   } catch (Exception e) {
   }
   MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
   connectionManager.getParams().setDefaultMaxConnectionsPerHost(10);
   connectionManager.getParams().setMaxTotalConnections(300);
   connectionManager.getParams().setConnectionTimeout(connectionTimeout);
   connectionManager.getParams().setSoTimeout(soTimeout);
   client.setHttpConnectionManager(connectionManager);
   // 忽略cookie 避免 Cookie rejected 警告
   HttpClientParams clientParams = new HttpClientParams();
   clientParams.setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
   client.setParams(clientParams);
   // 支持https
   Protocol myhttps = new Protocol("https", new SSLSocketFactory(), 443);
   Protocol.registerProtocol("https", myhttps);
   // 设置代理
   if (ProxyClient.getProxy() != null) {
     client.getHostConfiguration().setProxy(ProxyClient.getHost(), ProxyClient.getPort());
     client.getParams().setAuthenticationPreemptive(true);
     if (ProxyClient.getUsername() != null && !ProxyClient.getUsername().trim().equals("")) {
       client
           .getState()
           .setProxyCredentials(
               AuthScope.ANY,
               new UsernamePasswordCredentials(
                   ProxyClient.getUsername().trim(), ProxyClient.getPassword().trim()));
     }
   }
 }
  private void configureHttpMethod(
      boolean skipContentCache, CacheData cacheData, long onceTimeOut, HttpMethod httpMethod) {
    if (skipContentCache && null != cacheData) {
      if (null != cacheData.getLastModifiedHeader()
          && Constants.NULL != cacheData.getLastModifiedHeader()) {
        httpMethod.addRequestHeader(Constants.IF_MODIFIED_SINCE, cacheData.getLastModifiedHeader());
      }
      if (null != cacheData.getMd5() && Constants.NULL != cacheData.getMd5()) {
        httpMethod.addRequestHeader(Constants.CONTENT_MD5, cacheData.getMd5());
      }
    }

    httpMethod.addRequestHeader(Constants.ACCEPT_ENCODING, "gzip,deflate");

    HttpMethodParams params = new HttpMethodParams();
    params.setSoTimeout((int) onceTimeOut);
    httpMethod.setParams(params);
    httpClient
        .getHostConfiguration()
        .setHost(
            diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
            diamondConfigure.getPort());
  }
 protected HttpClient getHttpClient() {
   HttpClient client = new HttpClient();
   if (Jenkins.getInstance() != null) {
     ProxyConfiguration proxy = Jenkins.getInstance().proxy;
     if (proxy != null) {
       client.getHostConfiguration().setProxy(proxy.name, proxy.port);
       String username = proxy.getUserName();
       String password = proxy.getPassword();
       // Consider it to be passed if username specified. Sufficient?
       if (username != null && !"".equals(username.trim())) {
         logger.info("Using proxy authentication (user="******")");
         // http://hc.apache.org/httpclient-3.x/authentication.html#Proxy_Authentication
         // and
         // http://svn.apache.org/viewvc/httpcomponents/oac.hc3x/trunk/src/examples/BasicAuthenticationExample.java?view=markup
         client
             .getState()
             .setProxyCredentials(
                 AuthScope.ANY, new UsernamePasswordCredentials(username, password));
       }
     }
   }
   return client;
 }
Esempio n. 22
0
 /**
  * @param sProxyUser
  * @param sProxyPassswd
  * @return an HTTP client
  */
 private static HttpClient getHTTPClient(
     String sProxyUser, String sProxyPassswd, int iConTimeout, int iDataTimeout) {
   HttpClient client = new HttpClient();
   client
       .getHttpConnectionManager()
       .getParams()
       .setConnectionTimeout(iConTimeout); // connection to
   client
       .getHttpConnectionManager()
       .getParams()
       .setSoTimeout(iDataTimeout); // data reception timeout
   if (sProxyUser != null && sProxyPassswd != null) {
     client
         .getHostConfiguration()
         .setProxy(
             ConfigurationManager.getProperty(CONF_NETWORK_PROXY_HOSTNAME),
             Integer.parseInt(ConfigurationManager.getProperty(CONF_NETWORK_PROXY_PORT)));
     client
         .getState()
         .setProxyCredentials(
             new AuthScope(AuthScope.ANY), new UsernamePasswordCredentials(sProxyUser, sProxyPwd));
   }
   return client;
 }
  public HttpTemplateDownloader(
      StorageLayer storageLayer,
      String downloadUrl,
      String toDir,
      DownloadCompleteCallback callback,
      long maxTemplateSizeInBytes,
      String user,
      String password,
      Proxy proxy,
      ResourceType resourceType) {
    this._storage = storageLayer;
    this.downloadUrl = downloadUrl;
    this.setToDir(toDir);
    this.status = TemplateDownloader.Status.NOT_STARTED;
    this.resourceType = resourceType;
    this.MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes;

    this.totalBytes = 0;
    this.client = new HttpClient();

    myretryhandler =
        new HttpMethodRetryHandler() {
          public boolean retryMethod(
              final HttpMethod method, final IOException exception, int executionCount) {
            if (executionCount >= 2) {
              // Do not retry if over max retry count
              return false;
            }
            if (exception instanceof NoHttpResponseException) {
              // Retry if the server dropped connection on us
              return true;
            }
            if (!method.isRequestSent()) {
              // Retry if the request has not been sent fully or
              // if it's OK to retry methods that have been sent
              return true;
            }
            // otherwise do not retry
            return false;
          }
        };

    try {
      this.request = new GetMethod(downloadUrl);
      this.request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
      this.completionCallback = callback;
      // this.request.setFollowRedirects(false);

      File f = File.createTempFile("dnld", "tmp_", new File(toDir));

      if (_storage != null) {
        _storage.setWorldReadableAndWriteable(f);
      }

      toFile = f.getAbsolutePath();
      Pair<String, Integer> hostAndPort = validateUrl(downloadUrl);

      if (proxy != null) {
        client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
        if (proxy.getUserName() != null) {
          Credentials proxyCreds =
              new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword());
          client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds);
        }
      }
      if ((user != null) && (password != null)) {
        client.getParams().setAuthenticationPreemptive(true);
        Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
        client
            .getState()
            .setCredentials(
                new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM),
                defaultcreds);
        s_logger.info(
            "Added username="******", password="******"for host "
                + hostAndPort.first()
                + ":"
                + hostAndPort.second());
      } else {
        s_logger.info(
            "No credentials configured for host="
                + hostAndPort.first()
                + ":"
                + hostAndPort.second());
      }
    } catch (IllegalArgumentException iae) {
      errorString = iae.getMessage();
      status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
      inited = false;
    } catch (Exception ex) {
      errorString = "Unable to start download -- check url? ";
      status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
      s_logger.warn("Exception in constructor -- " + ex.toString());
    } catch (Throwable th) {
      s_logger.warn("throwable caught ", th);
    }
  }
Esempio n. 24
0
 /** Clear the proxy setting. */
 public void clearProxy() {
   client.getHostConfiguration().setProxyHost(null);
 }
Esempio n. 25
0
 /**
  * Set a proxy that should be used by the client when trying to access the server. If this is not
  * set, the client will attempt to make a direct direct connection to the server.
  *
  * @param host The name of the host.
  * @param port The port.
  */
 public void setProxy(String host, int port) {
   client.getHostConfiguration().setProxy(host, port);
 }
  // 授权,生成access_token
  // 使用情形:①程序初始化;②每隔一天左右重新授权access_token
  public static void generate() {
    initAccountInfo();
    accessToken.clear();

    logger.info("用户授权中...");
    try {
      // https://api.weibo.com/oauth2/authorize?client_id=750123511&redirect_uri=https://api.weibo.com/oauth2/default.html&response_type=code
      String url = "https://api.weibo.com/oauth2/authorize";
      String redirectUri = "https://api.weibo.com/oauth2/default.html";

      for (int i = 0; i < accountInfo.size(); i++) {
        // 获取应用的信息
        clientId = WeiboConfig.getValue("client_ID");
        clientSecret = WeiboConfig.getValue("client_SERCRET");

        // 构造授权的url参数
        PostMethod postMethod = new PostMethod(url);
        postMethod.addParameter("client_id", clientId);
        postMethod.addParameter("redirect_uri", redirectUri);
        postMethod.addParameter("userId", accountInfo.get(i).getUserId());
        postMethod.addParameter("passwd", accountInfo.get(i).getPasswd());
        postMethod.addParameter("isLoginSina", "0");
        postMethod.addParameter("action", "submit");
        postMethod.addParameter("response_type", "code");
        HttpMethodParams param = postMethod.getParams();
        param.setContentCharset("UTF-8");

        // 伪造头部域信息
        List<Header> headers = new ArrayList<Header>();
        headers.add(
            new Header(
                "Referer",
                "https://api.weibo.com/oauth2/authorize?client_id="
                    + clientId
                    + "&redirect_uri="
                    + redirectUri
                    + "&from=sina&response_type=code"));
        headers.add(new Header("Host", "api.weibo.com"));
        headers.add(
            new Header(
                "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:11.0) Gecko/20100101 Firefox/11.0"));

        // 发送HTTP请求
        HttpClient client = new HttpClient();
        client.getHostConfiguration().getParams().setParameter("http.default-headers", headers);
        client.executeMethod(postMethod);

        // 获取授权响应
        int status = postMethod.getStatusCode();
        if (status == 302) {
          Header location = postMethod.getResponseHeader("location");
          if (location != null) {
            String retUrl = location.getValue();
            int begin = retUrl.indexOf("code=");
            int end = retUrl.length();
            String code = retUrl.substring(begin + 5, end);
            if (code != null) {
              Oauth oauth = new Oauth();
              String token = oauth.getAccessTokenByCode(code).getAccessToken();
              accessToken.add(token);
              logger.info("第" + (i + 1) + "个access_token:" + token);
            }
          }
        } else {
          logger.error("第" + (i + 1) + "个用户授权失败了!");
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
      logger.error("授权发生异常!");
    }
  }
  Set<String> checkUpdateDataIds(long timeout) {
    if (!isRun) {
      throw new RuntimeException("DiamondSubscriber is not running. checkUpdateDataIds return.");
    }
    if (MockServer.isTestMode()) {
      return testData();
    }
    long waitTime = 0;

    String probeUpdateString = getProbeUpdateString();
    if (StringUtils.isBlank(probeUpdateString)) {
      return null;
    }

    while (0 == timeout || timeout > waitTime) {
      long onceTimeOut = getOnceTimeOut(waitTime, timeout);
      waitTime += onceTimeOut;

      PostMethod postMethod = new PostMethod(Constants.HTTP_URI_FILE);

      postMethod.addParameter(Constants.PROBE_MODIFY_REQUEST, probeUpdateString);

      HttpMethodParams params = new HttpMethodParams();
      params.setSoTimeout((int) onceTimeOut);
      postMethod.setParams(params);

      try {
        httpClient
            .getHostConfiguration()
            .setHost(
                diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
                this.diamondConfigure.getPort());

        int httpStatus = httpClient.executeMethod(postMethod);

        switch (httpStatus) {
          case SC_OK:
            {
              Set<String> result = getUpdateDataIds(postMethod);
              return result;
            }

          case SC_SERVICE_UNAVAILABLE:
            {
              rotateToNextDomain();
            }
            break;

          default:
            {
              log.warn("checkUpdateDataIds HTTP State: " + httpStatus);
              rotateToNextDomain();
            }
        }
      } catch (HttpException e) {
        log.error("checkUpdateDataIds HttpException", e);
        rotateToNextDomain();
      } catch (IOException e) {
        log.error("checkUpdateDataIds IOException", e);
        rotateToNextDomain();
      } catch (Exception e) {
        log.error("checkUpdateDataIds Unknown Exception", e);
        rotateToNextDomain();
      } finally {
        postMethod.releaseConnection();
      }
    }
    throw new RuntimeException(
        "checkUpdateDataIds timeout "
            + diamondConfigure.getDomainNameList().get(this.domainNamePos.get())
            + ", timeout="
            + timeout);
  }
  public byte[] timeStamp(byte[] data, RevocationData revocationData) throws Exception {
    // digest the message
    MessageDigest messageDigest = MessageDigest.getInstance(this.digestAlgo);
    byte[] digest = messageDigest.digest(data);

    // generate the TSP request
    BigInteger nonce = new BigInteger(128, new SecureRandom());
    TimeStampRequestGenerator requestGenerator = new TimeStampRequestGenerator();
    requestGenerator.setCertReq(true);
    if (null != this.requestPolicy) {
      requestGenerator.setReqPolicy(this.requestPolicy);
    }
    TimeStampRequest request = requestGenerator.generate(this.digestAlgoOid, digest, nonce);
    byte[] encodedRequest = request.getEncoded();

    // create the HTTP client
    HttpClient httpClient = new HttpClient();
    if (null != this.username) {
      Credentials credentials = new UsernamePasswordCredentials(this.username, this.password);
      httpClient.getState().setCredentials(AuthScope.ANY, credentials);
    }
    if (null != this.proxyHost) {
      httpClient.getHostConfiguration().setProxy(this.proxyHost, this.proxyPort);
    }

    // create the HTTP POST request
    PostMethod postMethod = new PostMethod(this.tspServiceUrl);
    RequestEntity requestEntity =
        new ByteArrayRequestEntity(encodedRequest, "application/timestamp-query");
    postMethod.addRequestHeader("User-Agent", this.userAgent);
    postMethod.setRequestEntity(requestEntity);

    // invoke TSP service
    int statusCode = httpClient.executeMethod(postMethod);
    if (HttpStatus.SC_OK != statusCode) {
      LOG.error("Error contacting TSP server " + this.tspServiceUrl);
      throw new Exception("Error contacting TSP server " + this.tspServiceUrl);
    }

    // HTTP input validation
    Header responseContentTypeHeader = postMethod.getResponseHeader("Content-Type");
    if (null == responseContentTypeHeader) {
      throw new RuntimeException("missing Content-Type header");
    }
    String contentType = responseContentTypeHeader.getValue();
    if (!contentType.startsWith("application/timestamp-reply")) {
      LOG.debug("response content: " + postMethod.getResponseBodyAsString());
      throw new RuntimeException("invalid Content-Type: " + contentType);
    }
    if (0 == postMethod.getResponseContentLength()) {
      throw new RuntimeException("Content-Length is zero");
    }

    // TSP response parsing and validation
    InputStream inputStream = postMethod.getResponseBodyAsStream();
    TimeStampResponse timeStampResponse = new TimeStampResponse(inputStream);
    timeStampResponse.validate(request);

    if (0 != timeStampResponse.getStatus()) {
      LOG.debug("status: " + timeStampResponse.getStatus());
      LOG.debug("status string: " + timeStampResponse.getStatusString());
      PKIFailureInfo failInfo = timeStampResponse.getFailInfo();
      if (null != failInfo) {
        LOG.debug("fail info int value: " + failInfo.intValue());
        if (PKIFailureInfo.unacceptedPolicy == failInfo.intValue()) {
          LOG.debug("unaccepted policy");
        }
      }
      throw new RuntimeException(
          "timestamp response status != 0: " + timeStampResponse.getStatus());
    }
    TimeStampToken timeStampToken = timeStampResponse.getTimeStampToken();
    SignerId signerId = timeStampToken.getSID();
    BigInteger signerCertSerialNumber = signerId.getSerialNumber();
    X500Principal signerCertIssuer = signerId.getIssuer();
    LOG.debug("signer cert serial number: " + signerCertSerialNumber);
    LOG.debug("signer cert issuer: " + signerCertIssuer);

    // TSP signer certificates retrieval
    CertStore certStore =
        timeStampToken.getCertificatesAndCRLs("Collection", BouncyCastleProvider.PROVIDER_NAME);
    Collection<? extends Certificate> certificates = certStore.getCertificates(null);
    X509Certificate signerCert = null;
    Map<String, X509Certificate> certificateMap = new HashMap<String, X509Certificate>();
    for (Certificate certificate : certificates) {
      X509Certificate x509Certificate = (X509Certificate) certificate;
      if (signerCertIssuer.equals(x509Certificate.getIssuerX500Principal())
          && signerCertSerialNumber.equals(x509Certificate.getSerialNumber())) {
        signerCert = x509Certificate;
      }
      String ski = Hex.encodeHexString(getSubjectKeyId(x509Certificate));
      certificateMap.put(ski, x509Certificate);
      LOG.debug(
          "embedded certificate: " + x509Certificate.getSubjectX500Principal() + "; SKI=" + ski);
    }

    // TSP signer cert path building
    if (null == signerCert) {
      throw new RuntimeException("TSP response token has no signer certificate");
    }
    List<X509Certificate> tspCertificateChain = new LinkedList<X509Certificate>();
    X509Certificate certificate = signerCert;
    do {
      LOG.debug("adding to certificate chain: " + certificate.getSubjectX500Principal());
      tspCertificateChain.add(certificate);
      if (certificate.getSubjectX500Principal().equals(certificate.getIssuerX500Principal())) {
        break;
      }
      String aki = Hex.encodeHexString(getAuthorityKeyId(certificate));
      certificate = certificateMap.get(aki);
    } while (null != certificate);

    // verify TSP signer signature
    timeStampToken.validate(tspCertificateChain.get(0), BouncyCastleProvider.PROVIDER_NAME);

    // verify TSP signer certificate
    this.validator.validate(tspCertificateChain, revocationData);

    LOG.debug("time-stamp token time: " + timeStampToken.getTimeStampInfo().getGenTime());

    byte[] timestamp = timeStampToken.getEncoded();
    return timestamp;
  }
  @Override
  public BatchHttpResult getConfigureInformationBatch(
      List<String> dataIds, String group, int timeout) {
    if (dataIds == null) {
      log.error("dataId list cannot be null,group=" + group);
      return new BatchHttpResult(HttpStatus.SC_BAD_REQUEST);
    }
    if (group == null) {
      group = Constants.DEFAULT_GROUP;
    }

    StringBuilder dataIdBuilder = new StringBuilder();
    for (String dataId : dataIds) {
      dataIdBuilder.append(dataId).append(Constants.LINE_SEPARATOR);
    }
    String dataIdStr = dataIdBuilder.toString();

    PostMethod post = new PostMethod(Constants.HTTP_URI_FILE_BATCH);
    post.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, timeout);

    BatchHttpResult response = null;
    try {
      NameValuePair dataIdValue = new NameValuePair("dataIds", dataIdStr);
      NameValuePair groupValue = new NameValuePair("group", group);

      post.setRequestBody(new NameValuePair[] {dataIdValue, groupValue});

      httpClient
          .getHostConfiguration()
          .setHost(
              diamondConfigure.getDomainNameList().get(this.domainNamePos.get()),
              this.diamondConfigure.getPort());
      int status = httpClient.executeMethod(post);
      String responseMsg = post.getResponseBodyAsString();

      if (status == HttpStatus.SC_OK) {
        String json = null;
        try {
          json = responseMsg;

          List<ConfigInfoEx> configInfoExList = new LinkedList<ConfigInfoEx>();
          Object resultObj =
              JSONUtils.deserializeObject(json, new TypeReference<List<ConfigInfoEx>>() {});
          if (!(resultObj instanceof List<?>)) {
            throw new RuntimeException(
                "batch query deserialize type error, not list, json=" + json);
          }
          List<ConfigInfoEx> resultList = (List<ConfigInfoEx>) resultObj;
          for (ConfigInfoEx configInfoEx : resultList) {
            configInfoExList.add(configInfoEx);
          }

          response = new BatchHttpResult(configInfoExList);
          log.info(
              "batch query success,dataIds=" + dataIdStr + ",group=" + group + ",json=" + json);
        } catch (Exception e) {
          response = new BatchHttpResult(Constants.BATCH_OP_ERROR);
          log.error(
              "batch query deserialize error,dataIdStr="
                  + dataIdStr
                  + ",group="
                  + group
                  + ",json="
                  + json,
              e);
        }

      } else if (status == HttpStatus.SC_REQUEST_TIMEOUT) {
        response = new BatchHttpResult(HttpStatus.SC_REQUEST_TIMEOUT);
        log.error(
            "batch query timeout, socket timeout(ms):"
                + timeout
                + ",dataIds="
                + dataIdStr
                + ",group="
                + group);
      } else {
        response = new BatchHttpResult(status);
        log.error(
            "batch query fail, status:"
                + status
                + ", response:"
                + responseMsg
                + ",dataIds="
                + dataIdStr
                + ",group="
                + group);
      }
    } catch (HttpException e) {
      response = new BatchHttpResult(Constants.BATCH_HTTP_EXCEPTION);
      log.error("batch query http exception,dataIds=" + dataIdStr + ",group=" + group, e);
    } catch (IOException e) {
      response = new BatchHttpResult(Constants.BATCH_IO_EXCEPTION);
      log.error("batch query io exception, dataIds=" + dataIdStr + ",group=" + group, e);
    } finally {
      post.releaseConnection();
    }

    return response;
  }
  /**
   * The method reads a proxy object from database, makes a GetMethod object, appends required
   * cookies to the HttpClient object. The HttpClient then executes the Getmethod and returns the
   * pagesource to the caller.
   *
   * @param iCount Counter variable for passing thread group information
   * @param url Url to fetch the pagesource for
   * @param followRedirect Boolean variable to specify GetMethod followRedirect value
   * @param doAuthentication Boolean variable to specify GetMethod doAuthentication value
   * @param region the local region of a given url
   * @param objProxyDao the database layer ProxyDao object variable
   * @param useErrsy Boolean variable to specify usage of Errsy as proxy source
   * @return String
   */
  public String getPageSourceWithProxy(
      String url,
      boolean followRedirect,
      boolean doAuthentication,
      String region,
      Boolean useErrsy,
      String google) {

    String page = " ";
    String pageSource = "";
    int i = 0;
    String exception = " ";
    HttpClientParams clientParams = new HttpClientParams();
    clientParams.setSoTimeout(40000);
    clientParams.setConnectionManagerTimeout(40000);
    HttpClient httpclient = new HttpClient(clientParams);
    GetMethod getmethod = null;

    HttpState state = new HttpState();

    //  if (ProxyDao.lstProxyData.size() == 16) {
    ProxyData objProxyData = null;
    // if (!useErrsy) {
    try {
      // objProxyData = ProxyDao.lstProxyData.get(iCount);
      objProxyData = ProxyDao.objProxyData;
      if (objProxyData == null) {
        //                objProxyDao.changeProxy(google);
        objProxyData = ProxyDao.objProxyData;
      }
      httpclient
          .getHostConfiguration()
          .setProxy(objProxyData.getIPAddress(), objProxyData.getPortNo());
    } catch (Exception e) {
      pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
      return pageSource;
    }
    /*} else {
    try {
    objProxyData = new ProxyData(0, "46.227.68.2", 3128, "Mongoose", "I-C5GS0FTAL61L", 0, 0);
    Credentials defaultcreds = new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword());
    httpclient.getState().setCredentials(AuthScope.ANY, defaultcreds);
    httpclient.getHostConfiguration().setProxy(objProxyData.getIpaddress(), objProxyData.getPortNo());
    state.setProxyCredentials(null, null, new UsernamePasswordCredentials(objProxyData.getProxyUser(), objProxyData.getProxyPassword()));
    httpclient.setState(state);
    } catch (Exception e) {
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;
    return pageSource;
    }
    }*/
    try {
      getmethod = new GetMethod(url);
      getmethod.addRequestHeader(
          "User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0");
      if (url.contains("bing.com")) {

        if (region.equalsIgnoreCase("co.uk")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-GB;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.sg")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-SG;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.au")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-AU;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("co.in")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IN;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("ca")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-CA;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.ph")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-PH;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("com.my")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-WW;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else if (region.equalsIgnoreCase("it")) {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-IT;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        } else {
          getmethod.addRequestHeader(
              "Cookie", "_FP=mkt=en-US;SRCHHPGUSR=NEWWND=0&NRSLT=50&SRCHLANG=&AS=1;");
        }
      }
      getmethod.setFollowRedirects(true);
      getmethod.setDoAuthentication(true);
      httpclient.getParams().setAuthenticationPreemptive(true);
      httpclient.setState(state);
      String num100Header = "";
      //            if (url.contains("google")) {
      //                int j = 0;
      //                String url1 = "http://www.google.com/";
      //                try {
      //                    GetMethod objGetMethod = new GetMethod(url1);
      //                    j = httpclient.executeMethod(objGetMethod);
      //                    Header responseHeader = objGetMethod.getResponseHeader("Set-Cookie");
      //                    String header = responseHeader.getValue();
      //                    String[] headerValue = header.split(";");
      //
      //                    for (String head : headerValue) {
      //                        if (head.contains("PREF=ID")) {
      //                            header = head;
      //                            break;
      //                        }
      //                    }
      //                    String[] splitAll = header.split(":");
      //                    long time = System.currentTimeMillis()+400;
      //                    String sTime = "" + time;
      //                    sTime = sTime.substring(0, 10);
      //                    //num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1]  +
      // ":LD=en:NR=100:" + splitAll[2] + ":" + splitAll[3] + ":" + splitAll[4];
      //                    num100Header = splitAll[0].replace("PREF=", "") + ":" + splitAll[1] +
      // ":LD=en:NR=100:" + "TM=" + sTime + ":LM=" + sTime + ":SG=2:" + splitAll[4];
      //                    Cookie ck = new Cookie("PREF", "PREF", num100Header);
      //                    httpclient.getState().clearCookies();
      //                    httpclient.getState().addCookie(ck);
      //                    getmethod.addRequestHeader("Host", "www.google.com");
      //                    getmethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1;
      // rv:19.0) Gecko/20100101 Firefox/19.0");
      //                    getmethod.addRequestHeader("Accept",
      // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
      //                    getmethod.addRequestHeader("Accept-Language", "en-US,en;q=0.5");
      //                    getmethod.addRequestHeader("Accept-Encoding", "gzip, deflate");
      //                    getmethod.addRequestHeader("Referer", "https://www.google.com/");
      //                    System.out.println(num100Header);
      //                } catch (Exception ex) {
      //                    exception = ex.getMessage();
      //                    l.debug(ex + "  " + ex.getMessage() + "Exception occured for url" +
      // url);
      //                    pageSource = j + "@@@@" + exception + "@@@@" + page + "@@@@" + url1;
      //                    return pageSource;
      //                }
      //            }
      i = httpclient.executeMethod(getmethod);
      if (i / 100 == 4 || i / 100 == 5) {
        page = "<PROXY ERROR>";

      } else {

        page = getmethod.getResponseBodyAsString();
      }
    } catch (SocketTimeoutException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (SocketException ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);
    } catch (Exception ex) {
      exception = ex.getMessage();
      l.error(ex + "  " + ex.getMessage() + "Exception occured for url" + url);

    } finally {
      getmethod.releaseConnection();
    }
    pageSource = i + "@@@@" + exception + "@@@@" + page + "@@@@" + url;

    // }
    return pageSource;
  }