示例#1
0
  /**
   * Explicitly construct a TwilioClient with the given API credentials and endpoint.
   *
   * @param username Username for authentication. For most cases, it is the 34 character Account
   *     identifier (starting with 'AC'). This can be found on your Twilio dashboard page.
   * @param password Password for authentication. For most cases, it is the 32 character AuthToken.
   *     This can be found on your Twilio dashboard page.
   * @param endpoint the url of API endpoint you wish to use. (e.g. - 'https://api.twilio.com')
   */
  public TwilioClient(final String username, final String password, final String endpoint) {
    if (username == null) {
      throw new IllegalArgumentException(
          "username is required. Please provide username or signing key.");
    }
    if (password == null) {
      throw new IllegalArgumentException(
          "password is required. Please provide authToken or signing key secret.");
    }

    this.username = username;
    this.accountSid = username;
    this.password = password;

    if ((endpoint != null) && (!endpoint.equals(""))) {
      this.endpoint = endpoint;
    }

    // Grab the proper connection manager, based on runtime environment
    ClientConnectionManager mgr;
    try {
      Class.forName("com.google.appengine.api.urlfetch.HTTPRequest");
      mgr = new AppEngineClientConnectionManager();
    } catch (final ClassNotFoundException e) {
      // Not GAE
      mgr = new PoolingClientConnectionManager();
      ((PoolingClientConnectionManager) mgr).setDefaultMaxPerRoute(10);
    }

    setHttpClient(new DefaultHttpClient(mgr));
    httpclient.getParams().setParameter("http.protocol.version", HttpVersion.HTTP_1_1);
    httpclient.getParams().setParameter("http.socket.timeout", new Integer(READ_TIMEOUT));
    httpclient.getParams().setParameter("http.connection.timeout", new Integer(CONNECTION_TIMEOUT));
    httpclient.getParams().setParameter("http.protocol.content-charset", "UTF-8");
  }
  public AbstractHttpTransport() {
    PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
    connectionManager.setMaxTotal(1000);
    connectionManager.setDefaultMaxPerRoute(500);

    this.httpClient = new DefaultHttpClient(connectionManager);
  }
  public void init() {
    /*
    loadLiferayProperties();

    hostname = liferayProperties.getProperty(FFPORTAL_SERVER_HOSTNAME);
    port = liferayProperties.getProperty(FFPORTAL_SERVER_PORT);
    companyId = liferayProperties.getProperty(FFPORTAL_REPOSITORY_COMPANYID);
    loginGroupId = liferayProperties.getProperty(FFPORTAL_USER_GROUP_ID);
    loginEmail = liferayProperties.getProperty(FFPORTAL_USER_EMAIL);
    loginPassword = liferayProperties.getProperty(FFPORTAL_USER_PASSWORD);
    */
    targetHost = new HttpHost(hostname, Integer.valueOf(port), "http");
    PoolingClientConnectionManager cxMgr =
        new PoolingClientConnectionManager(SchemeRegistryFactory.createDefault());
    cxMgr.setMaxTotal(100);
    cxMgr.setDefaultMaxPerRoute(20);
    httpclient = new DefaultHttpClient(cxMgr);

    httpclient
        .getCredentialsProvider()
        .setCredentials(
            new AuthScope(targetHost.getHostName(), targetHost.getPort()),
            new UsernamePasswordCredentials(loginEmail, loginPassword));

    // Create AuthCache instance
    this.authCache = new BasicAuthCache();
    // Generate BASIC scheme object and add it to the local
    // auth cache
    BasicScheme basicAuth = new BasicScheme();
    authCache.put(targetHost, basicAuth);
  }
示例#4
0
  static {
    BasicHttpParams httpParams = new BasicHttpParams();
    HttpConnectionParams.setConnectionTimeout(httpParams, 1000 * 1000);
    HttpConnectionParams.setSoTimeout(httpParams, 1000 * 1000);

    PoolingClientConnectionManager pcm = new PoolingClientConnectionManager();
    pcm.setMaxTotal(20000);
    pcm.setDefaultMaxPerRoute(pcm.getMaxTotal());
    httpClient = new DefaultHttpClient(pcm, httpParams);
  }
示例#5
0
  /**
   * create PoolingClientConnectionManager
   *
   * @param maxCount max total connection, default max connection per route
   * @param connectionTimeout milliseconds
   * @param answerTimeout milliseconds
   */
  public HttpManager(int maxCount, int connectionTimeout, int answerTimeout) {
    PoolingClientConnectionManager manager = new PoolingClientConnectionManager();
    manager.setDefaultMaxPerRoute(maxCount);
    manager.setMaxTotal(maxCount);
    HttpParams params = new BasicHttpParams();
    HttpConnectionParams.setConnectionTimeout(params, connectionTimeout);
    HttpConnectionParams.setSoTimeout(params, answerTimeout);

    httpClient = new DefaultHttpClient(manager, params);
  }
  public IntegrationMockClient(String endpoint) {
    super(endpoint);
    this.endpoint = endpoint;
    PoolingClientConnectionManager cm = new PoolingClientConnectionManager();
    // Increase max total connection to 200
    cm.setMaxTotal(200);
    // Increase default max connection per route to 50
    cm.setDefaultMaxPerRoute(50);

    httpClient = new DefaultHttpClient(cm);
    httpClient = (DefaultHttpClient) WebClientWrapper.wrapClient(httpClient);
  }
  public void Initiallize()
      throws KeyManagementException, UnrecoverableKeyException, NoSuchAlgorithmException,
          KeyStoreException {
    Scheme http = new Scheme("http", 80, PlainSocketFactory.getSocketFactory());
    // 创建TrustManager
    X509TrustManager xtm =
        new X509TrustManager() {
          public void checkClientTrusted(X509Certificate[] chain, String authType)
              throws CertificateException {}

          public void checkServerTrusted(X509Certificate[] chain, String authType)
              throws CertificateException {}

          public X509Certificate[] getAcceptedIssuers() {
            return null;
          }
        };
    // 这个好像是HOST验证
    X509HostnameVerifier hostnameVerifier =
        new X509HostnameVerifier() {
          public boolean verify(String arg0, SSLSession arg1) {
            return true;
          }

          public void verify(String arg0, SSLSocket arg1) throws IOException {}

          public void verify(String arg0, String[] arg1, String[] arg2) throws SSLException {}

          public void verify(String arg0, X509Certificate arg1) throws SSLException {}
        };
    SSLContext ctx = null;
    try {
      ctx = SSLContext.getInstance("TLS");
      ctx.init(null, new TrustManager[] {(TrustManager) xtm}, null);
      // 创建SSLSocketFactory
      socketFactory = new SSLSocketFactory(ctx);
      socketFactory.setHostnameVerifier(hostnameVerifier);
    } catch (NoSuchAlgorithmException e1) {
      e1.printStackTrace();
    } catch (KeyManagementException e) {
      e.printStackTrace();
    }
    SSLContext.setDefault(ctx);
    SSLSocketFactory ssf = new SSLSocketFactory(ctx);
    Scheme https = new Scheme("https", 443, ssf);
    sr = new SchemeRegistry();
    sr.register(http);
    sr.register(https);
    cm = new PoolingClientConnectionManager(sr);
    cm.setMaxTotal(200);
    cm.setDefaultMaxPerRoute(100);
  }
 static {
   httpParams = new BasicHttpParams();
   // 设置连接超时时间
   HttpConnectionParams.setConnectionTimeout(httpParams, CONNECT_TIMEOUT);
   // 设置读取超时时间
   HttpConnectionParams.setSoTimeout(httpParams, READ_TIMEOUT);
   connectionManager = new PoolingClientConnectionManager();
   // 设置最大连接数
   connectionManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
   // 设置每个路由最大连接数
   connectionManager.setDefaultMaxPerRoute(MAX_ROUTE_CONNECTIONS);
   client = new DefaultHttpClient(connectionManager, httpParams);
 }
示例#9
0
  /**
   * 获得连接池的实例
   *
   * @return
   */
  public static HttpClient getCoonPoolHttpClient() {

    PoolingClientConnectionManager cm = new PoolingClientConnectionManager();
    cm.setMaxTotal(800);
    cm.setDefaultMaxPerRoute(200);

    DefaultHttpClient httpClient = new DefaultHttpClient(cm, getHttpParams());

    httpClient.addRequestInterceptor(getRequestInter());
    httpClient.addResponseInterceptor(getResponseInter());
    httpClient.setHttpRequestRetryHandler(buildMyRetryHandler());

    return httpClient;
  }
  /**
   * Create a new instance of the HttpComponentsHttpInvokerRequestExecutor with a default {@link
   * HttpClient} that uses a default {@code
   * org.apache.http.impl.conn.PoolingClientConnectionManager}.
   */
  public HttpComponentsHttpInvokerRequestExecutor() {
    org.apache.http.conn.scheme.SchemeRegistry schemeRegistry =
        new org.apache.http.conn.scheme.SchemeRegistry();
    schemeRegistry.register(
        new org.apache.http.conn.scheme.Scheme(
            "http", 80, org.apache.http.conn.scheme.PlainSocketFactory.getSocketFactory()));
    schemeRegistry.register(
        new org.apache.http.conn.scheme.Scheme(
            "https", 443, org.apache.http.conn.ssl.SSLSocketFactory.getSocketFactory()));

    org.apache.http.impl.conn.PoolingClientConnectionManager connectionManager =
        new org.apache.http.impl.conn.PoolingClientConnectionManager(schemeRegistry);
    connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL_CONNECTIONS);
    connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);

    this.httpClient = new org.apache.http.impl.client.DefaultHttpClient(connectionManager);
    setReadTimeout(DEFAULT_READ_TIMEOUT_MILLISECONDS);
  }
  private UmssoCLCore initializeCore() {
    UmssoCLCore clCore = UmssoCLCoreImpl.getInstance();
    HttpParams params = clCore.getUmssoHttpClient().getParams();
    HttpConnectionParams.setConnectionTimeout(params, 10000);
    PoolingClientConnectionManager cm = clCore.getConnectionManager();
    cm.setMaxTotal(HTTP_MAX_TOTAL_CONNECTIONS);
    cm.setDefaultMaxPerRoute(HTTP_DEFAULT_MAX_CONNECTIONS_PER_ROUTE);

    UmssoCLEnvironment umssoCLEnvironment =
        umSsoHttpConnectionSettings.getUmSsoCLEnvironmentFromProxySettings();
    if (umssoCLEnvironment != null) {
      clCore.init(umssoCLEnvironment);
    }

    if (umSsoHttpConnectionSettings.isIgnoreCertificates()) {
      ignoreCertificates(clCore.getUmssoHttpClient());
    }
    return clCore;
  }
  public void Initiallize2()
      throws KeyManagementException, UnrecoverableKeyException, NoSuchAlgorithmException,
          KeyStoreException {
    Scheme http = new Scheme("http", 80, PlainSocketFactory.getSocketFactory());
    X509TrustManager tm =
        new X509TrustManager() {
          public X509Certificate[] getAcceptedIssuers() {
            return null;
          }

          @Override
          public void checkClientTrusted(X509Certificate[] arg0, String arg1)
              throws CertificateException {}

          @Override
          public void checkServerTrusted(X509Certificate[] arg0, String arg1)
              throws CertificateException {}
        };
    SSLContext ctx = null;
    try {
      ctx = SSLContext.getInstance("TLS");
      ctx.init(null, new TrustManager[] {(TrustManager) tm}, null);
    } catch (NoSuchAlgorithmException e1) {
      e1.printStackTrace();
    } catch (KeyManagementException e) {
      e.printStackTrace();
    }
    SSLContext.setDefault(ctx);
    SSLSocketFactory ssf = new SSLSocketFactory(ctx);
    Scheme https = new Scheme("https", 443, ssf);
    sr = new SchemeRegistry();
    sr.register(http);
    sr.register(https);
    cm = new PoolingClientConnectionManager(sr);
    cm.setMaxTotal(200);
    cm.setDefaultMaxPerRoute(100);
  }
  @Override
  public void afterPropertiesSet() {
    super.afterPropertiesSet();
    if (getServiceInterface() == null) {
      throw new IllegalArgumentException("property serviceInterface is required.");
    }
    org.apache.http.conn.scheme.SchemeRegistry schemeRegistry =
        new org.apache.http.conn.scheme.SchemeRegistry();
    schemeRegistry.register(
        new org.apache.http.conn.scheme.Scheme(
            "http", 80, org.apache.http.conn.scheme.PlainSocketFactory.getSocketFactory()));
    schemeRegistry.register(
        new org.apache.http.conn.scheme.Scheme(
            "https", 443, org.apache.http.conn.ssl.SSLSocketFactory.getSocketFactory()));

    org.apache.http.impl.conn.PoolingClientConnectionManager connectionManager =
        new org.apache.http.impl.conn.PoolingClientConnectionManager(schemeRegistry);
    connectionManager.setMaxTotal(DEFAULT_MAX_TOTAL_CONNECTIONS);
    connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);

    this.httpClient = new org.apache.http.impl.client.DefaultHttpClient(connectionManager);
    setReadTimeout(DEFAULT_READ_TIMEOUT_MILLISECONDS);
    prepare();
  }
示例#14
0
  protected ClientHttpEngine initDefaultEngine() {
    DefaultHttpClient httpClient = null;

    X509HostnameVerifier verifier = null;
    if (this.verifier != null) verifier = new VerifierWrapper(this.verifier);
    else {
      switch (policy) {
        case ANY:
          verifier = new AllowAllHostnameVerifier();
          break;
        case WILDCARD:
          verifier = new BrowserCompatHostnameVerifier();
          break;
        case STRICT:
          verifier = new StrictHostnameVerifier();
          break;
      }
    }
    try {
      SSLSocketFactory sslsf = null;
      SSLContext theContext = sslContext;
      if (disableTrustManager) {
        theContext = SSLContext.getInstance("SSL");
        theContext.init(
            null, new TrustManager[] {new PassthroughTrustManager()}, new SecureRandom());
        verifier = new AllowAllHostnameVerifier();
        sslsf = new SSLSocketFactory(theContext, verifier);
      } else if (theContext != null) {
        sslsf = new SSLSocketFactory(theContext, verifier);
      } else if (clientKeyStore != null || truststore != null) {
        sslsf =
            new SSLSocketFactory(
                SSLSocketFactory.TLS,
                clientKeyStore,
                clientPrivateKeyPassword,
                truststore,
                null,
                verifier);
      } else {
        // sslsf = new SSLSocketFactory(SSLContext.getInstance(SSLSocketFactory.TLS), verifier);
        final SSLContext tlsContext = SSLContext.getInstance(SSLSocketFactory.TLS);
        tlsContext.init(null, null, null);
        sslsf = new SSLSocketFactory(tlsContext, verifier);
      }
      SchemeRegistry registry = new SchemeRegistry();
      registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
      Scheme httpsScheme = new Scheme("https", 443, sslsf);
      registry.register(httpsScheme);
      ClientConnectionManager cm = null;
      if (connectionPoolSize > 0) {
        PoolingClientConnectionManager tcm =
            new PoolingClientConnectionManager(registry, connectionTTL, connectionTTLUnit);
        tcm.setMaxTotal(connectionPoolSize);
        if (maxPooledPerRoute == 0) maxPooledPerRoute = connectionPoolSize;
        tcm.setDefaultMaxPerRoute(maxPooledPerRoute);
        cm = tcm;

      } else {
        cm = new BasicClientConnectionManager(registry);
      }
      BasicHttpParams params = new BasicHttpParams();
      if (socketTimeout > -1) {
        HttpConnectionParams.setSoTimeout(params, (int) socketTimeoutUnits.toMillis(socketTimeout));
      }
      if (establishConnectionTimeout > -1) {
        HttpConnectionParams.setConnectionTimeout(
            params, (int) establishConnectionTimeoutUnits.toMillis(establishConnectionTimeout));
      }
      httpClient = new DefaultHttpClient(cm, params);
      ApacheHttpClient4Engine engine = new ApacheHttpClient4Engine(httpClient, true);
      engine.setResponseBufferSize(responseBufferSize);
      engine.setHostnameVerifier(verifier);
      // this may be null.  We can't really support this with Apache Client.
      engine.setSslContext(theContext);
      engine.setDefaultProxy(defaultProxy);
      return engine;
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  public static Client createApacheClient(
      AtmosConfig config,
      boolean useExpect100Continue,
      List<Class<MessageBodyReader<?>>> readers,
      List<Class<MessageBodyWriter<?>>> writers) {
    try {
      ClientConfig clientConfig = new DefaultApacheHttpClient4Config();

      // make sure the apache client is thread-safe
      PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
      // Increase max total connection to 200
      connectionManager.setMaxTotal(200);
      // Increase default max connection per route to 200
      connectionManager.setDefaultMaxPerRoute(200);
      clientConfig
          .getProperties()
          .put(DefaultApacheHttpClient4Config.PROPERTY_CONNECTION_MANAGER, connectionManager);

      // register an open trust manager to allow SSL connections to servers with self-signed
      // certificates
      if (config.isDisableSslValidation()) {
        connectionManager
            .getSchemeRegistry()
            .register(
                new Scheme(
                    "https",
                    443,
                    new SSLSocketFactory(
                        SslUtil.createGullibleSslContext(),
                        SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER)));
      }

      // set proxy uri
      ProxySelector proxySelector;
      // first look in config
      URI proxyUri = config.getProxyUri();
      if (proxyUri != null) {
        proxySelector = new ConfigProxySelector(config);
      } else {
        // if no proxy in config, use system property sniffing selector
        proxySelector = new DefaultProxySelector();

        // and see if a proxy is set
        String host = System.getProperty("http.proxyHost");
        String portStr = System.getProperty("http.proxyPort");
        int port = (portStr != null) ? Integer.parseInt(portStr) : -1;
        if (host != null && host.length() > 0)
          proxyUri = new URI("http", null, host, port, null, null, null);
      }

      // make sure any proxy credentials (below) are associated with the proxy
      if (proxyUri != null)
        clientConfig.getProperties().put(ApacheHttpClient4Config.PROPERTY_PROXY_URI, proxyUri);

      // set proxy auth
      // first config
      String proxyUser = config.getProxyUser(), proxyPassword = config.getProxyPassword();
      // then system props
      if (proxyUser == null) {
        proxyUser = System.getProperty("http.proxyUser");
        proxyPassword = System.getProperty("http.proxyPassword");
      }
      if (proxyUser != null && proxyUser.length() > 0) {
        clientConfig
            .getProperties()
            .put(ApacheHttpClient4Config.PROPERTY_PROXY_USERNAME, proxyUser);
        clientConfig
            .getProperties()
            .put(ApacheHttpClient4Config.PROPERTY_PROXY_PASSWORD, proxyPassword);
      }

      // specify whether to use Expect: 100-continue
      HttpParams httpParams = new SyncBasicHttpParams();
      DefaultHttpClient.setDefaultHttpParams(httpParams);
      httpParams.setBooleanParameter(AllClientPNames.USE_EXPECT_CONTINUE, useExpect100Continue);
      clientConfig
          .getProperties()
          .put(DefaultApacheHttpClient4Config.PROPERTY_HTTP_PARAMS, httpParams);

      // pick up other configuration from system props
      for (String prop : System.getProperties().stringPropertyNames()) {
        if (prop.startsWith("http.")) {
          // because AbstractHttpParams uses casts instead of parsing string values, we must know
          // the
          // parameter types ahead of time. I'm going to guess them in lieu of maintaining a map.
          // however,
          // we still need to define Long typed parameters so they may be differentiated from
          // Integer typed
          // parameters.
          String value = System.getProperty(prop);
          if (LONG_PARAMETERS.contains(prop))
            httpParams.setLongParameter(
                prop, Long.parseLong(value.substring(0, value.length() - 1)));
          else if (BOOLEAN_PATTERN.matcher(value).matches())
            httpParams.setBooleanParameter(prop, Boolean.parseBoolean(value));
          else if (DOUBLE_PATTERN.matcher(value).matches())
            httpParams.setDoubleParameter(prop, Double.parseDouble(value));
          else if (INT_PATTERN.matcher(value).matches())
            httpParams.setIntParameter(prop, Integer.parseInt(value));
          else httpParams.setParameter(prop, System.getProperty(prop));
        }
      }

      JerseyUtil.addHandlers(clientConfig, readers, writers);

      // create the client
      ApacheHttpClient4 client = ApacheHttpClient4.create(clientConfig);
      AbstractHttpClient httpClient =
          (AbstractHttpClient) client.getClientHandler().getHttpClient();

      // do not use Apache's retry handler
      httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(0, false));

      // use a RoutePlanner/ProxySelector that fits our requirements
      SchemeRegistry registry = httpClient.getConnectionManager().getSchemeRegistry();
      httpClient.setRoutePlanner(new ProxySelectorRoutePlanner(registry, proxySelector));

      JerseyUtil.addFilters(client, config);

      return client;
    } catch (Exception e) {
      throw new AtmosException("Error configuring REST client", e);
    }
  }
示例#16
0
  /** @return {@link DefaultHttpClient} instance. */
  private HttpClient createHttpClient(CouchDbProperties props) {
    DefaultHttpClient httpclient = null;
    try {
      SchemeSocketFactory ssf = null;
      if (props.getProtocol().equals("https")) {
        TrustManager trustManager =
            new X509TrustManager() {
              public void checkClientTrusted(X509Certificate[] chain, String authType)
                  throws CertificateException {}

              public void checkServerTrusted(X509Certificate[] chain, String authType)
                  throws CertificateException {}

              public X509Certificate[] getAcceptedIssuers() {
                return null;
              }
            };
        SSLContext sslcontext = SSLContext.getInstance("TLS");
        sslcontext.init(null, new TrustManager[] {trustManager}, null);
        ssf = new SSLSocketFactory(sslcontext, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
        SSLSocket socket = (SSLSocket) ssf.createSocket(null);
        socket.setEnabledCipherSuites(new String[] {"SSL_RSA_WITH_RC4_128_MD5"});
      } else {
        ssf = PlainSocketFactory.getSocketFactory();
      }
      SchemeRegistry schemeRegistry = new SchemeRegistry();
      schemeRegistry.register(new Scheme(props.getProtocol(), props.getPort(), ssf));
      PoolingClientConnectionManager ccm = new PoolingClientConnectionManager(schemeRegistry);
      httpclient = new DefaultHttpClient(ccm);
      host = new HttpHost(props.getHost(), props.getPort(), props.getProtocol());
      context = new BasicHttpContext();
      // Http params
      httpclient.getParams().setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, "UTF-8");
      httpclient
          .getParams()
          .setParameter(CoreConnectionPNames.SO_TIMEOUT, props.getSocketTimeout());
      httpclient
          .getParams()
          .setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, props.getConnectionTimeout());
      int maxConnections = props.getMaxConnections();
      if (maxConnections != 0) {
        ccm.setMaxTotal(maxConnections);
        ccm.setDefaultMaxPerRoute(maxConnections);
      }
      if (props.getProxyHost() != null) {
        HttpHost proxy = new HttpHost(props.getProxyHost(), props.getProxyPort());
        httpclient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
      }
      // basic authentication
      if (props.getUsername() != null && props.getPassword() != null) {
        httpclient
            .getCredentialsProvider()
            .setCredentials(
                new AuthScope(props.getHost(), props.getPort()),
                new UsernamePasswordCredentials(props.getUsername(), props.getPassword()));
        props.clearPassword();
        AuthCache authCache = new BasicAuthCache();
        BasicScheme basicAuth = new BasicScheme();
        authCache.put(host, basicAuth);
        context.setAttribute(ClientContext.AUTH_CACHE, authCache);
      }
      // request interceptor
      httpclient.addRequestInterceptor(
          new HttpRequestInterceptor() {
            public void process(final HttpRequest request, final HttpContext context)
                throws IOException {
              if (log.isInfoEnabled()) {
                RequestLine requestLine = request.getRequestLine();
                try {
                  log.info(
                      ">> "
                          + (new StringBuilder())
                              .append(">> ")
                              .append(requestLine.getMethod())
                              .append(" ")
                              .append(urlCodec.decode(requestLine.getUri()))
                              .toString());
                } catch (DecoderException e) {
                  log.error(e, e);
                }
              }
            }
          });
      // response interceptor
      httpclient.addResponseInterceptor(
          new HttpResponseInterceptor() {
            public void process(final HttpResponse response, final HttpContext context)
                throws IOException {
              validate(response);
              if (log.isInfoEnabled())
                log.info("<< Status: " + response.getStatusLine().getStatusCode());
            }
          });
    } catch (Exception e) {
      log.error("Error Creating HTTP client. " + e.getMessage());
      throw new IllegalStateException(e);
    }
    return httpclient;
  }
示例#17
0
  public ServiceObserver(
      ServiceObserverConfiguration serviceConfig,
      ExecutorService executor,
      Class<D> dataClass,
      Class<A> aggregateClass,
      Class<S> summaryClass,
      ContentParser<D> contentParser,
      ObserverNamingStrategy ons) {
    this.serviceConfig = serviceConfig;
    this.executor = executor;
    this.observers = new ConcurrentHashMap<String, Observer<D>>();
    this.dataClass = dataClass;
    this.aggregateClass = aggregateClass;
    this.summaryClass = summaryClass;
    this.contentParser = contentParser;
    if (ons == null) {
      this.ons = ObserverNamingStrategy.DEFAULT;
    } else {
      this.ons = ons;
    }
    this.snapshotAllTimer =
        Metrics.newTimer(
            new MetricName(
                "com.mfizz.observer.ServiceObservers",
                serviceConfig.getName(),
                "snapshot-all-time"),
            TimeUnit.MILLISECONDS,
            TimeUnit.MILLISECONDS);
    this.snapshotAllAttemptedCounter = new AtomicLong();
    this.snapshotAllCompletedCounter = new AtomicLong();
    this.lastSnapshotAllResult = new AtomicReference<SnapshotAllResult>();

    // this.lastSnapshotAllTimestamp = new AtomicLong(0);
    // this.lastSnapshotsCompletedCounter = new AtomicInteger();
    // this.lastSnapshotsFailedCounter = new AtomicInteger();

    this.groups = new TreeSet<String>();
    this.summary = new ConcurrentHashMap<String, SummaryGroup<S>>();
    this.snapshots = new ConcurrentHashMap<String, TimeSeries<ObserveAggregateSnapshot<A>>>();

    // always make sure "current" is added to set of vars to track
    // "current" always tracked for all
    this.serviceConfig.getPeriods().add(SummaryPeriod.parse("current"));

    //
    // create high performance http client that can be reused by all observers
    //
    SchemeRegistry schemeRegistry = new SchemeRegistry();
    schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
    schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

    PoolingClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
    // increase max total connection to 200
    cm.setMaxTotal(200);
    // increase default max connection per route to 20
    cm.setDefaultMaxPerRoute(20);

    this.httpclient = new DefaultHttpClient(cm);
    this.httpclient.setHttpRequestRetryHandler(
        new HttpRequestRetryHandler() {
          @Override
          public boolean retryRequest(
              IOException exception, int executionCount, HttpContext context) {
            // always return false -- we never want a request retried
            return false;
          }
        });
    this.httpclient
        .getParams()
        .setIntParameter(
            CoreConnectionPNames.SO_TIMEOUT, new Integer((int) serviceConfig.getSocketTimeout()))
        .setIntParameter(
            CoreConnectionPNames.CONNECTION_TIMEOUT,
            new Integer((int) serviceConfig.getConnectionTimeout()))
        .setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024)
        .setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true)
        .setBooleanParameter(CoreProtocolPNames.USE_EXPECT_CONTINUE, false)
        .setBooleanParameter(CoreConnectionPNames.STALE_CONNECTION_CHECK, false);
  }
示例#18
0
  // 根据用户ID找该用户的粉丝对象
  public Set<String> getFansUserByUid(String uid, int pageNumber) {

    System.out.println("进入查找用户的粉丝用户");

    PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
    connectionManager.setMaxTotal(2000);
    connectionManager.setDefaultMaxPerRoute(1000);

    HttpClient httpClient_fans = new DefaultHttpClient(connectionManager);

    // 伪装成Firefox 5,
    httpClient_fans
        .getParams()
        .setParameter(CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1);
    httpClient_fans
        .getParams()
        .setParameter(
            ClientPNames.COOKIE_POLICY,
            CookiePolicy.BROWSER_COMPATIBILITY); // 一定要有,否则会生成多个Cookie header送给web server
    httpClient_fans.getParams().setParameter(CookieSpecPNames.SINGLE_COOKIE_HEADER, true); //
    httpClient_fans
        .getParams()
        .setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, "UTF-8"); // 这个是和目标网站的编码有关;
    httpClient_fans.getParams().setParameter(CoreProtocolPNames.HTTP_ELEMENT_CHARSET, "UTF-8");
    List<BasicHeader> headerList_fans = new ArrayList<BasicHeader>();

    headerList_fans.add(new BasicHeader("Accept", "*/*"));
    headerList_fans.add(new BasicHeader("Accept-Charset", "GBK,utf-8;q=0.7,*;q=0.3"));
    headerList_fans.add(new BasicHeader("Accept-Language", "zh-CN,zh;q=0.8"));
    headerList_fans.add(new BasicHeader("Cache-Control", "no-cache"));
    headerList_fans.add(new BasicHeader("Connection", "keep-alive"));
    headerList_fans.add(
        new BasicHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"));
    headerList_fans.add(new BasicHeader("X-Requested-With", "XMLHttpRequest"));
    headerList_fans.add(new BasicHeader("Cookie", CookieData.CookieNow));

    httpClient_fans
        .getParams()
        .setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, new Integer(300000));
    httpClient_fans.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT, new Integer(300000));
    httpClient_fans
        .getParams()
        .setParameter(ClientPNames.CONN_MANAGER_TIMEOUT, new Long(300000)); // second;
    // 伪装成Firefox
    httpClient_fans
        .getParams()
        .setParameter(
            CoreProtocolPNames.USER_AGENT,
            "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:17.0) Gecko/17.0 Firefox/17.0");

    headerList_fans.add(new BasicHeader("Host", "weibo.com"));
    headerList_fans.add(new BasicHeader("Origin", "weibo.com"));

    GetFollowOrFansPageNumber page = new GetFollowOrFansPageNumber();
    int number = page.getFollowOrFansPage("http://weibo.com/" + uid + "/fans");

    //		QueryNumberManager queryNumber = QueryNumberManager.getInstance();

    Set<String> uidList = new HashSet<String>();

    for (int u = pageNumber; u < number; u++) {

      System.out.println("粉丝第" + (u + 1) + "页");
      String url_fans = "http://weibo.com/" + uid + "/fans" + "?page=" + (u + 1);
      System.out.println(url_fans);

      headerList_fans.add(new BasicHeader("Referer", url_fans));
      httpClient_fans.getParams().setParameter(ClientPNames.DEFAULT_HEADERS, headerList_fans);

      // 设置代理对象 ip/代理名称,端口
      //			ProxyManager proxyManager = ProxyManager.getInstance();
      //			ProxyBean proxy = new ProxyBean();
      //			proxy = proxyManager.getOneProxy();
      //			System.out.println("拿到代理IP" + proxy.getIp());
      //			httpClient_fans.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, new
      // HttpHost(proxy.getIp(), proxy.getPort()));

      HttpPost httpPost_fans = new HttpPost(url_fans);

      HttpResponse httpResponseFans = null;
      try {
        httpResponseFans = httpClient_fans.execute(httpPost_fans);
      } catch (ClientProtocolException e1) {
        PshLogger.logger.error(e1.getMessage());
      } catch (IOException e1) {
        PshLogger.logger.error(e1.getMessage());
        System.out.println("重新执行找关注");
        return getFansUserByUid(uid, u);
      }

      if (httpResponseFans == null) {

        return getFansUserByUid(uid, u + 1);
      }

      HttpEntity httpEntityFans = httpResponseFans.getEntity();

      if (httpEntityFans == null) {

        return getFansUserByUid(uid, u);
      }

      BufferedReader in_fans = null;
      try {
        in_fans = new BufferedReader(new InputStreamReader(httpEntityFans.getContent(), "UTF-8"));
      } catch (UnsupportedEncodingException e1) {
        PshLogger.logger.error(e1.getMessage());
      } catch (IllegalStateException e1) {
        PshLogger.logger.error(e1.getMessage());
      } catch (IOException e1) {
        PshLogger.logger.error(e1.getMessage());
      }

      String result_fans = "";
      String item_fans = "";

      try {
        while ((item_fans = in_fans.readLine()) != null) {
          if (item_fans.trim().indexOf("cnfList") != -1) {
            System.out.println("找到一行html符合");
            result_fans = item_fans.trim();
          }
        }
      } catch (IOException e1) {
        PshLogger.logger.error(e1.getMessage());
      } finally {
        try {
          in_fans.close();
        } catch (IOException e) {
          PshLogger.logger.error(e.getMessage());
        }
      }

      if (result_fans.trim().equals("")) {

        System.out.println("新浪微博提示行为异常follow");
        // 换代理IP重新获取该页
        return getFansUserByUid(uid, u);
      }

      System.out.println("已获取到 html_fans");

      try {

        result_fans =
            result_fans.substring(
                result_fans.indexOf("<div"), result_fans.lastIndexOf("/div>") + 5);
        result_fans = result_fans.replace('\\', '`');
        result_fans = result_fans.replaceAll("`n", "");
        result_fans = result_fans.replaceAll("`t", "");
        result_fans = result_fans.replaceAll("`r", "");
        result_fans = result_fans.replaceAll("`", "");
        result_fans = "<html><body>" + result_fans + "</body></html>";
      } catch (Exception e) {
        PshLogger.logger.error(e.getMessage());
        return getFansUserByUid(uid, u + 1);
      }

      Document doc_fans = Jsoup.parse(result_fans);

      Elements elements_fans = doc_fans.getElementsByAttributeValue("class", "W_f14 S_func1");
      System.out.println(elements_fans.size());

      for (int x = 0; x < elements_fans.size(); x++) {
        System.out.println("遍历粉丝第" + (u + 1) + "页,第" + (x + 1) + "个用户");
        String uidString = elements_fans.get(x).attr("usercard");
        System.out.println("uridString------" + uidString);
        String uid_fans = uidString.substring(3);
        System.out.println(uid_fans);

        UserModel userModel = new UserModel();

        if (userModel.checkUserIsExsit(uid_fans)) {
          continue;
        }

        // 获取该用户详细信息
        UserBean user = new UserBean();
        GetPersonInfo personInfo = new GetPersonInfo();
        Elements element_followNum =
            elements_fans
                .get(x)
                .parent()
                .parent()
                .getElementsByAttributeValue("href", "/" + uid_fans + "/follow");
        Elements element_fansNum =
            elements_fans
                .get(x)
                .parent()
                .parent()
                .getElementsByAttributeValue("href", "/" + uid_fans + "/fans");
        String fans = "";
        String follow = "";
        for (int y = 0; y < element_followNum.size(); y++) {
          follow = element_followNum.get(y).text();
        }
        for (int y = 0; y < element_fansNum.size(); y++) {
          fans = element_fansNum.get(y).text();
        }
        // 将该用户信息加入数据库
        user = personInfo.getUserInfoFromWeibo(uid_fans, fans, follow);
        if (user == null) {
          continue;
        }
        System.out.println("用户查找完" + user.getUck());

        // 进行2级遍历
        uidList.add(uid_fans);
      }
    }

    //		queryNumber.executeFollowAndFans(uidList, queryTaskID,level);
    return uidList;
  }
示例#19
0
 private ClientConnectionManager createConnectionManager() {
   PoolingClientConnectionManager manager = new PoolingClientConnectionManager();
   manager.setDefaultMaxPerRoute(MAX_CONNECTIONS_PER_ROUTE);
   return manager;
 }