コード例 #1
0
  private CloseableHttpClient generateClient(Site site) {
    HttpClientBuilder httpClientBuilder =
        HttpClients.custom().setConnectionManager(connectionManager);
    if (site != null && site.getUserAgent() != null) {
      httpClientBuilder.setUserAgent(site.getUserAgent());
    } else {
      httpClientBuilder.setUserAgent("");
    }
    if (site == null || site.isUseGzip()) {
      httpClientBuilder.addInterceptorFirst(
          new HttpRequestInterceptor() {

            public void process(final HttpRequest request, final HttpContext context)
                throws HttpException, IOException {
              if (!request.containsHeader("Accept-Encoding")) {
                request.addHeader("Accept-Encoding", "gzip");
              }
            }
          });
    }
    SocketConfig socketConfig =
        SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
    httpClientBuilder.setDefaultSocketConfig(socketConfig);
    if (site != null) {
      httpClientBuilder.setRetryHandler(
          new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
    }
    generateCookie(httpClientBuilder, site);
    return httpClientBuilder.build();
  }
コード例 #2
0
ファイル: HTTPSession.java プロジェクト: msdsoftware/thredds
 protected static void setInterceptors(HttpClientBuilder cb) {
   for (HttpRequestInterceptor hrq : reqintercepts) {
     cb.addInterceptorLast(hrq);
   }
   for (HttpResponseInterceptor hrs : rspintercepts) {
     cb.addInterceptorLast(hrs);
   }
   // Add debug interceptors
   for (HttpRequestInterceptor hrq : dbgreq) {
     cb.addInterceptorFirst(hrq);
   }
   for (HttpResponseInterceptor hrs : dbgrsp) {
     cb.addInterceptorFirst(hrs);
   }
   // Hack: add Content-Encoding suppressor
   cb.addInterceptorFirst(CEKILL);
 }
コード例 #3
0
  public FusionPipelineClient(
      String endpointUrl, String fusionUser, String fusionPass, String fusionRealm)
      throws MalformedURLException {

    globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH).build();
    cookieStore = new BasicCookieStore();

    this.fusionUser = fusionUser;
    this.fusionPass = fusionPass;
    this.fusionRealm = fusionRealm;

    // build the HttpClient to be used for all requests
    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    httpClientBuilder.setDefaultRequestConfig(globalConfig).setDefaultCookieStore(cookieStore);

    if (fusionUser != null && fusionRealm == null)
      httpClientBuilder.addInterceptorFirst(
          new PreEmptiveBasicAuthenticator(fusionUser, fusionPass));

    httpClient = httpClientBuilder.build();
    HttpClientUtil.setMaxConnections(httpClient, 500);
    HttpClientUtil.setMaxConnectionsPerHost(httpClient, 100);

    originalEndpoints = Arrays.asList(endpointUrl.split(","));
    try {
      sessions = establishSessions(originalEndpoints, fusionUser, fusionPass, fusionRealm);
    } catch (Exception exc) {
      if (exc instanceof RuntimeException) {
        throw (RuntimeException) exc;
      } else {
        throw new RuntimeException(exc);
      }
    }

    random = new Random();
    jsonObjectMapper = new ObjectMapper();

    requestCounter = new AtomicInteger(0);
  }
コード例 #4
0
ファイル: Config.java プロジェクト: slavianp/webscrap
  public CloseableHttpClient makeHttpClient() {
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.addInterceptorFirst(
        new HttpRequestInterceptor() {
          public void process(final HttpRequest request, final HttpContext context)
              throws HttpException, IOException {
            if (!request.containsHeader("Accept-Encoding")) {
              request.addHeader("Accept-Encoding", "gzip");
            }
            if (!request.containsHeader("User-Agent")) {
              request.addHeader(
                  "User-Agent",
                  "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:25.0) Gecko/20100101 Firefox/25.0");
            }
            if (!request.containsHeader("Accept")) {
              request.addHeader(
                  "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            }
          }
        });

    // builder.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(socketTimeout).build());
    return builder.build();
  }
コード例 #5
0
ファイル: HiveConnection.java プロジェクト: JkSelf/hive
  private CloseableHttpClient getHttpClient(Boolean useSsl) throws SQLException {
    boolean isCookieEnabled =
        sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH) == null
            || (!JdbcConnectionParams.COOKIE_AUTH_FALSE.equalsIgnoreCase(
                sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH)));
    String cookieName =
        sessConfMap.get(JdbcConnectionParams.COOKIE_NAME) == null
            ? JdbcConnectionParams.DEFAULT_COOKIE_NAMES_HS2
            : sessConfMap.get(JdbcConnectionParams.COOKIE_NAME);
    CookieStore cookieStore = isCookieEnabled ? new BasicCookieStore() : null;
    HttpClientBuilder httpClientBuilder;
    // Request interceptor for any request pre-processing logic
    HttpRequestInterceptor requestInterceptor;
    Map<String, String> additionalHttpHeaders = new HashMap<String, String>();

    // Retrieve the additional HttpHeaders
    for (Map.Entry<String, String> entry : sessConfMap.entrySet()) {
      String key = entry.getKey();

      if (key.startsWith(JdbcConnectionParams.HTTP_HEADER_PREFIX)) {
        additionalHttpHeaders.put(
            key.substring(JdbcConnectionParams.HTTP_HEADER_PREFIX.length()), entry.getValue());
      }
    }
    // Configure http client for kerberos/password based authentication
    if (isKerberosAuthMode()) {
      /**
       * Add an interceptor which sets the appropriate header in the request. It does the kerberos
       * authentication and get the final service ticket, for sending to the server before every
       * request. In https mode, the entire information is encrypted
       */
      requestInterceptor =
          new HttpKerberosRequestInterceptor(
              sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL),
              host,
              getServerHttpUrl(useSsl),
              assumeSubject,
              cookieStore,
              cookieName,
              useSsl,
              additionalHttpHeaders);
    } else {
      /**
       * Add an interceptor to pass username/password in the header. In https mode, the entire
       * information is encrypted
       */
      requestInterceptor =
          new HttpBasicAuthInterceptor(
              getUserName(), getPassword(), cookieStore, cookieName, useSsl, additionalHttpHeaders);
    }
    // Configure http client for cookie based authentication
    if (isCookieEnabled) {
      // Create a http client with a retry mechanism when the server returns a status code of 401.
      httpClientBuilder =
          HttpClients.custom()
              .setServiceUnavailableRetryStrategy(
                  new ServiceUnavailableRetryStrategy() {

                    @Override
                    public boolean retryRequest(
                        final HttpResponse response,
                        final int executionCount,
                        final HttpContext context) {
                      int statusCode = response.getStatusLine().getStatusCode();
                      boolean ret = statusCode == 401 && executionCount <= 1;

                      // Set the context attribute to true which will be interpreted by the request
                      // interceptor
                      if (ret) {
                        context.setAttribute(
                            Utils.HIVE_SERVER2_RETRY_KEY, Utils.HIVE_SERVER2_RETRY_TRUE);
                      }
                      return ret;
                    }

                    @Override
                    public long getRetryInterval() {
                      // Immediate retry
                      return 0;
                    }
                  });
    } else {
      httpClientBuilder = HttpClientBuilder.create();
    }
    // Add the request interceptor to the client builder
    httpClientBuilder.addInterceptorFirst(requestInterceptor);
    // Configure http client for SSL
    if (useSsl) {
      String useTwoWaySSL = sessConfMap.get(JdbcConnectionParams.USE_TWO_WAY_SSL);
      String sslTrustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE);
      String sslTrustStorePassword = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD);
      KeyStore sslTrustStore;
      SSLSocketFactory socketFactory;

      /**
       * The code within the try block throws: 1. SSLInitializationException 2. KeyStoreException 3.
       * IOException 4. NoSuchAlgorithmException 5. CertificateException 6. KeyManagementException
       * 7. UnrecoverableKeyException We don't want the client to retry on any of these, hence we
       * catch all and throw a SQLException.
       */
      try {
        if (useTwoWaySSL != null && useTwoWaySSL.equalsIgnoreCase(JdbcConnectionParams.TRUE)) {
          socketFactory = getTwoWaySSLSocketFactory();
        } else if (sslTrustStorePath == null || sslTrustStorePath.isEmpty()) {
          // Create a default socket factory based on standard JSSE trust material
          socketFactory = SSLSocketFactory.getSocketFactory();
        } else {
          // Pick trust store config from the given path
          sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
          try (FileInputStream fis = new FileInputStream(sslTrustStorePath)) {
            sslTrustStore.load(fis, sslTrustStorePassword.toCharArray());
          }
          socketFactory = new SSLSocketFactory(sslTrustStore);
        }
        socketFactory.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);

        final Registry<ConnectionSocketFactory> registry =
            RegistryBuilder.<ConnectionSocketFactory>create()
                .register("https", socketFactory)
                .build();

        httpClientBuilder.setConnectionManager(new BasicHttpClientConnectionManager(registry));
      } catch (Exception e) {
        String msg =
            "Could not create an https connection to " + jdbcUriString + ". " + e.getMessage();
        throw new SQLException(msg, " 08S01", e);
      }
    }
    return httpClientBuilder.build();
  }
コード例 #6
0
ファイル: PostHTTP.java プロジェクト: agent001/incubator-nifi
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
    final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
    final String userAgent = context.getProperty(USER_AGENT).getValue();

    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(
        context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(
        context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder.setSocketTimeout(
        context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();

    final StreamThrottler throttler = throttlerRef.get();
    final ProcessorLog logger = getLogger();

    final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
    String lastUrl = null;
    long bytesToSend = 0L;

    final List<FlowFile> toSend = new ArrayList<>();
    DestinationAccepts destinationAccepts = null;
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();

    final ObjectHolder<String> dnHolder = new ObjectHolder<>("none");
    while (true) {
      FlowFile flowFile = session.get();
      if (flowFile == null) {
        break;
      }

      final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
      try {
        new java.net.URL(url);
      } catch (final MalformedURLException e) {
        logger.error(
            "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
            new Object[] {flowFile, url});
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        continue;
      }

      // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing
      // FlowFiles
      if (lastUrl != null && !lastUrl.equals(url)) {
        session.transfer(flowFile);
        break;
      }

      lastUrl = url;
      toSend.add(flowFile);

      if (client == null || destinationAccepts == null) {
        final Config config = getConfig(url, context);
        final HttpClientConnectionManager conMan = config.getConnectionManager();

        final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
        clientBuilder.setConnectionManager(conMan);
        clientBuilder.setUserAgent(userAgent);
        clientBuilder.addInterceptorFirst(
            new HttpResponseInterceptor() {
              @Override
              public void process(final HttpResponse response, final HttpContext httpContext)
                  throws HttpException, IOException {
                HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
                ManagedHttpClientConnection conn =
                    coreContext.getConnection(ManagedHttpClientConnection.class);
                if (!conn.isOpen()) {
                  return;
                }

                SSLSession sslSession = conn.getSSLSession();

                if (sslSession != null) {
                  final X509Certificate[] certChain = sslSession.getPeerCertificateChain();
                  if (certChain == null || certChain.length == 0) {
                    throw new SSLPeerUnverifiedException("No certificates found");
                  }

                  final X509Certificate cert = certChain[0];
                  dnHolder.set(cert.getSubjectDN().getName().trim());
                }
              }
            });

        clientBuilder.disableAutomaticRetries();
        clientBuilder.disableContentCompression();

        final String username = context.getProperty(USERNAME).getValue();
        final String password = context.getProperty(PASSWORD).getValue();
        // set the credentials if appropriate
        if (username != null) {
          final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
          if (password == null) {
            credentialsProvider.setCredentials(
                AuthScope.ANY, new UsernamePasswordCredentials(username));
          } else {
            credentialsProvider.setCredentials(
                AuthScope.ANY, new UsernamePasswordCredentials(username, password));
          }
          clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }
        client = clientBuilder.build();

        // determine whether or not destination accepts flowfile/gzip
        destinationAccepts = config.getDestinationAccepts();
        if (destinationAccepts == null) {
          try {
            if (sendAsFlowFile) {
              destinationAccepts =
                  getDestinationAcceptance(client, url, getLogger(), transactionId);
            } else {
              destinationAccepts = new DestinationAccepts(false, false, false, false, null);
            }

            config.setDestinationAccepts(destinationAccepts);
          } catch (IOException e) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            logger.error(
                "Unable to communicate with destination {} to determine whether or not it can accept "
                    + "flowfiles/gzip; routing {} to failure due to {}",
                new Object[] {url, flowFile, e});
            context.yield();
            return;
          }
        }
      }

      // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2
      // (streaming) format,
      // then only use a single FlowFile
      if (!sendAsFlowFile
          || (!destinationAccepts.isFlowFileV3Accepted()
              && !destinationAccepts.isFlowFileV2Accepted())) {
        break;
      }

      bytesToSend += flowFile.getSize();
      if (bytesToSend > maxBatchBytes.longValue()) {
        break;
      }
    }

    if (toSend.isEmpty()) {
      return;
    }

    final String url = lastUrl;
    final HttpPost post = new HttpPost(url);
    final List<FlowFile> flowFileList = toSend;
    final DestinationAccepts accepts = destinationAccepts;
    final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;

    final EntityTemplate entity =
        new EntityTemplate(
            new ContentProducer() {
              @Override
              public void writeTo(final OutputStream rawOut) throws IOException {
                final OutputStream throttled =
                    (throttler == null) ? rawOut : throttler.newThrottledOutputStream(rawOut);
                OutputStream wrappedOut = new BufferedOutputStream(throttled);
                if (compressionLevel > 0 && accepts.isGzipAccepted()) {
                  wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
                }

                try (final OutputStream out = wrappedOut) {
                  for (final FlowFile flowFile : flowFileList) {
                    session.read(
                        flowFile,
                        new InputStreamCallback() {
                          @Override
                          public void process(final InputStream rawIn) throws IOException {
                            try (final InputStream in = new BufferedInputStream(rawIn)) {

                              FlowFilePackager packager = null;
                              if (!sendAsFlowFile) {
                                packager = null;
                              } else if (accepts.isFlowFileV3Accepted()) {
                                packager = new FlowFilePackagerV3();
                              } else if (accepts.isFlowFileV2Accepted()) {
                                packager = new FlowFilePackagerV2();
                              } else if (accepts.isFlowFileV1Accepted()) {
                                packager = new FlowFilePackagerV1();
                              }

                              // if none of the above conditions is met, we should never get here,
                              // because
                              // we will have already verified that at least 1 of the FlowFile
                              // packaging
                              // formats is acceptable if sending as FlowFile.
                              if (packager == null) {
                                StreamUtils.copy(in, out);
                              } else {
                                final Map<String, String> flowFileAttributes;
                                if (isDestinationLegacyNiFi) {
                                  // Old versions of NiFi expect nf.file.name and nf.file.path to
                                  // indicate filename & path;
                                  // in order to maintain backward compatibility, we copy the
                                  // filename & path to those attribute keys.
                                  flowFileAttributes = new HashMap<>(flowFile.getAttributes());
                                  flowFileAttributes.put(
                                      "nf.file.name",
                                      flowFile.getAttribute(CoreAttributes.FILENAME.key()));
                                  flowFileAttributes.put(
                                      "nf.file.path",
                                      flowFile.getAttribute(CoreAttributes.PATH.key()));
                                } else {
                                  flowFileAttributes = flowFile.getAttributes();
                                }

                                packager.packageFlowFile(
                                    in, out, flowFileAttributes, flowFile.getSize());
                              }
                            }
                          }
                        });
                  }

                  out.flush();
                }
              }
            });

    entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
    post.setEntity(entity);
    post.setConfig(requestConfig);

    final String contentType;
    if (sendAsFlowFile) {
      if (accepts.isFlowFileV3Accepted()) {
        contentType = APPLICATION_FLOW_FILE_V3;
      } else if (accepts.isFlowFileV2Accepted()) {
        contentType = APPLICATION_FLOW_FILE_V2;
      } else if (accepts.isFlowFileV1Accepted()) {
        contentType = APPLICATION_FLOW_FILE_V1;
      } else {
        logger.error(
            "Cannot send data to {} because the destination does not accept FlowFiles and this processor is "
                + "configured to deliver FlowFiles; rolling back session",
            new Object[] {url});
        session.rollback();
        context.yield();
        return;
      }
    } else {
      final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key());
      contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue;
    }

    final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
    if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) {
      final Pattern pattern = Pattern.compile(attributeHeaderRegex);

      final Map<String, String> attributes = flowFileList.get(0).getAttributes();
      for (final Map.Entry<String, String> entry : attributes.entrySet()) {
        final String key = entry.getKey();
        if (pattern.matcher(key).matches()) {
          post.setHeader(entry.getKey(), entry.getValue());
        }
      }
    }

    post.setHeader(CONTENT_TYPE, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);
    if (compressionLevel > 0 && accepts.isGzipAccepted()) {
      post.setHeader(GZIPPED_HEADER, "true");
    }

    // Do the actual POST
    final String flowFileDescription =
        toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";

    final String uploadDataRate;
    final long uploadMillis;
    CloseableHttpResponse response = null;
    try {
      final StopWatch stopWatch = new StopWatch(true);
      response = client.execute(post);

      // consume input stream entirely, ignoring its contents. If we
      // don't do this, the Connection will not be returned to the pool
      EntityUtils.consume(response.getEntity());
      stopWatch.stop();
      uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
      uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
      logger.error(
          "Failed to Post {} due to {}; transferring to failure",
          new Object[] {flowFileDescription, e});
      context.yield();
      for (FlowFile flowFile : toSend) {
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
      }
      return;
    } finally {
      if (response != null) {
        try {
          response.close();
        } catch (IOException e) {
          getLogger().warn("Failed to close HTTP Response due to {}", new Object[] {e});
        }
      }
    }

    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
      final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
      if (locationUriHeader != null) {
        if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
          final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
          if (holdUriHeader != null) {
            holdUri = holdUriHeader.getValue();
          }
        }
      }

      if (holdUri == null) {
        for (FlowFile flowFile : toSend) {
          flowFile = session.penalize(flowFile);
          logger.error(
              "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
              new Object[] {flowFile, url, responseCode, responseReason});
          session.transfer(flowFile, REL_FAILURE);
        }
        return;
      }
    }

    if (holdUri == null) {
      if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
        for (FlowFile flowFile : toSend) {
          flowFile = session.penalize(flowFile);
          logger.error(
              "Failed to Post {} to {}: response code was {}:{}; will yield processing, "
                  + "since the destination is temporarily unavailable",
              new Object[] {flowFile, url, responseCode, responseReason});
          session.transfer(flowFile, REL_FAILURE);
        }
        context.yield();
        return;
      }

      if (responseCode >= 300) {
        for (FlowFile flowFile : toSend) {
          flowFile = session.penalize(flowFile);
          logger.error(
              "Failed to Post {} to {}: response code was {}:{}",
              new Object[] {flowFile, url, responseCode, responseReason});
          session.transfer(flowFile, REL_FAILURE);
        }
        return;
      }

      logger.info(
          "Successfully Posted {} to {} in {} at a rate of {}",
          new Object[] {
            flowFileDescription,
            url,
            FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS),
            uploadDataRate
          });

      for (final FlowFile flowFile : toSend) {
        session
            .getProvenanceReporter()
            .send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true);
        session.transfer(flowFile, REL_SUCCESS);
      }
      return;
    }

    //
    // the response indicated a Hold URI; delete the Hold.
    //
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are
    // returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
      // If the Hold URI that we get starts with /contentListener, it may not really be
      // /contentListener,
      // as this really indicates that it should be whatever we posted to -- if posting directly to
      // the
      // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer,
      // we may
      // be posting to some other URL.
      fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
      // URL indicates the full path but not hostname or port; use the same hostname & port that we
      // posted
      // to but use the full path indicated by the response.
      int firstSlash = url.indexOf("/", 8);
      if (firstSlash < 0) {
        firstSlash = url.length();
      }
      final String beforeSlash = url.substring(0, firstSlash);
      fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
      // Absolute URL
      fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }

    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);

    while (true) {
      try {
        final HttpResponse holdResponse = client.execute(delete);
        EntityUtils.consume(holdResponse.getEntity());
        final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
        final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
        if (holdStatusCode >= 300) {
          logger.error(
              "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
              new Object[] {flowFileDescription, holdStatusCode, holdReason});

          for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
          }
          return;
        }

        logger.info(
            "Successfully Posted {} to {} in {} milliseconds at a rate of {}",
            new Object[] {flowFileDescription, url, uploadMillis, uploadDataRate});

        for (FlowFile flowFile : toSend) {
          session.getProvenanceReporter().send(flowFile, url);
          session.transfer(flowFile, REL_SUCCESS);
        }
        return;
      } catch (final IOException e) {
        logger.warn(
            "Failed to delete Hold that destination placed on {} due to {}",
            new Object[] {flowFileDescription, e});
      }

      if (!isScheduled()) {
        context.yield();
        logger.warn(
            "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
            new Object[] {flowFileDescription});
        for (FlowFile flowFile : toSend) {
          flowFile = session.penalize(flowFile);
          session.transfer(flowFile, REL_FAILURE);
        }
        return;
      }
    }
  }
コード例 #7
0
  public EasyHttpClient() {

    CookieSpecProvider easySpecProvider =
        new CookieSpecProvider() {
          public CookieSpec create(HttpContext context) {

            return new BrowserCompatSpec() {
              public void validate(Cookie cookie, CookieOrigin origin)
                  throws MalformedCookieException {}
            };
          }
        };

    SSLContext sslContext = SSLContexts.createDefault(); // 忽略证书主机名验证
    SSLConnectionSocketFactory sslsf =
        new SSLConnectionSocketFactory(
            sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);

    Registry<ConnectionSocketFactory> reg =
        RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.INSTANCE)
            .register("https", sslsf)
            .build();
    connectionManager = new PoolingHttpClientConnectionManager(reg);
    connectionManager.setDefaultMaxPerRoute(100); // 同一个路由允许最大连接数
    // connectionManager.setMaxTotal(poolSize);
    HttpHost localhost = new HttpHost("locahost", 80); // 如果是多网卡,这里选择出口IP?
    connectionManager.setMaxPerRoute(new HttpRoute(localhost), 50);

    Registry<CookieSpecProvider> r =
        RegistryBuilder.<CookieSpecProvider>create()
            .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
            .register(CookieSpecs.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory())
            .register("easy", easySpecProvider)
            .build();

    RequestConfig requestConfig =
        RequestConfig.custom()
            .setCookieSpec("easy")
            .setSocketTimeout(10000)
            .setConnectTimeout(10000)
            .build();
    ConnectionConfig connectioncfg = ConnectionConfig.custom().setCharset(Charsets.UTF_8).build();
    SocketConfig socketConfig =
        SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();

    HttpRequestRetryHandler myRetryHandler =
        new HttpRequestRetryHandler() {
          public boolean retryRequest(
              IOException exception, int executionCount, HttpContext context) {
            if (executionCount >= 3) {
              return false;
            }
            if (exception instanceof SSLHandshakeException) {
              return false;
            } else {
              return true;
            }
          }
        };

    HttpClientBuilder builder =
        HttpClients.custom() // 默认
            .setConnectionManager(connectionManager) // 链接管理器
            .setDefaultSocketConfig(socketConfig) // socket管理器
            .setRetryHandler(myRetryHandler) // 重试3次
            .setDefaultConnectionConfig(connectioncfg) // 链接配置,如默认字符编码
            .setDefaultCookieSpecRegistry(r) // cookie策略
            .setUserAgent(Chorme_User_Agent) // 浏览器请求头
            .setDefaultRequestConfig(requestConfig) // 链接配置,超时等
            .setDefaultCookieStore(cookieStore); // cookie

    builder.addInterceptorFirst(
        new HttpRequestInterceptor() {

          public void process(final HttpRequest request, final HttpContext context)
              throws HttpException, IOException {
            if (!request.containsHeader("Accept-Encoding")) {
              request.addHeader("Accept-Encoding", "gzip");
            }
          }
        });

    httpclient = builder.build();
  }
コード例 #8
0
  @Override
  @Guarded(by = STARTED)
  public HttpClientBuilder prepare(final @Nullable Customizer customizer) {
    final HttpClientPlan plan = new HttpClientPlan();

    // attach connection manager early, so customizer has chance to replace it if needed
    plan.getClient().setConnectionManager(sharedConnectionManager);

    // apply defaults
    defaultsCustomizer.customize(plan);

    // apply globals
    new ConfigurationCustomizer(getConfigurationInternal()).customize(plan);

    // apply instance customization
    if (customizer != null) {
      customizer.customize(plan);
    }

    // apply plan to builder
    HttpClientBuilder builder = plan.getClient();
    builder.setDefaultConnectionConfig(plan.getConnection().build());
    builder.setDefaultSocketConfig(plan.getSocket().build());
    builder.setDefaultRequestConfig(plan.getRequest().build());
    builder.setDefaultCredentialsProvider(plan.getCredentials());

    builder.addInterceptorFirst(
        (HttpRequest request, HttpContext context) -> {
          // add custom http-context attributes
          for (Entry<String, Object> entry : plan.getAttributes().entrySet()) {
            // only set context attribute if not already set, to allow per request overrides
            if (context.getAttribute(entry.getKey()) == null) {
              context.setAttribute(entry.getKey(), entry.getValue());
            }
          }

          // add custom http-request headers
          for (Entry<String, String> entry : plan.getHeaders().entrySet()) {
            request.addHeader(entry.getKey(), entry.getValue());
          }
        });
    builder.addInterceptorLast(
        (HttpRequest httpRequest, HttpContext httpContext) -> {
          if (outboundLog.isDebugEnabled()) {
            httpContext.setAttribute(CTX_REQ_STOPWATCH, Stopwatch.createStarted());
            httpContext.setAttribute(CTX_REQ_URI, getRequestURI(httpContext));
            outboundLog.debug(
                "{} > {}", httpContext.getAttribute(CTX_REQ_URI), httpRequest.getRequestLine());
          }
        });
    builder.addInterceptorLast(
        (HttpResponse httpResponse, HttpContext httpContext) -> {
          Stopwatch stopwatch = (Stopwatch) httpContext.getAttribute(CTX_REQ_STOPWATCH);
          if (stopwatch != null) {
            outboundLog.debug(
                "{} < {} @ {}",
                httpContext.getAttribute(CTX_REQ_URI),
                httpResponse.getStatusLine(),
                stopwatch);
          }
        });

    return builder;
  }