private CloseableHttpClient generateClient(Site site) {
HttpClientBuilder httpClientBuilder =
HttpClients.custom().setConnectionManager(connectionManager);
if (site != null && site.getUserAgent() != null) {
httpClientBuilder.setUserAgent(site.getUserAgent());
} else {
httpClientBuilder.setUserAgent("");
}
if (site == null || site.isUseGzip()) {
httpClientBuilder.addInterceptorFirst(
new HttpRequestInterceptor() {
public void process(final HttpRequest request, final HttpContext context)
throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
}
SocketConfig socketConfig =
SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
if (site != null) {
httpClientBuilder.setRetryHandler(
new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
}
generateCookie(httpClientBuilder, site);
return httpClientBuilder.build();
}
protected static void setInterceptors(HttpClientBuilder cb) {
for (HttpRequestInterceptor hrq : reqintercepts) {
cb.addInterceptorLast(hrq);
}
for (HttpResponseInterceptor hrs : rspintercepts) {
cb.addInterceptorLast(hrs);
}
// Add debug interceptors
for (HttpRequestInterceptor hrq : dbgreq) {
cb.addInterceptorFirst(hrq);
}
for (HttpResponseInterceptor hrs : dbgrsp) {
cb.addInterceptorFirst(hrs);
}
// Hack: add Content-Encoding suppressor
cb.addInterceptorFirst(CEKILL);
}
public FusionPipelineClient(
String endpointUrl, String fusionUser, String fusionPass, String fusionRealm)
throws MalformedURLException {
globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH).build();
cookieStore = new BasicCookieStore();
this.fusionUser = fusionUser;
this.fusionPass = fusionPass;
this.fusionRealm = fusionRealm;
// build the HttpClient to be used for all requests
HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
httpClientBuilder.setDefaultRequestConfig(globalConfig).setDefaultCookieStore(cookieStore);
if (fusionUser != null && fusionRealm == null)
httpClientBuilder.addInterceptorFirst(
new PreEmptiveBasicAuthenticator(fusionUser, fusionPass));
httpClient = httpClientBuilder.build();
HttpClientUtil.setMaxConnections(httpClient, 500);
HttpClientUtil.setMaxConnectionsPerHost(httpClient, 100);
originalEndpoints = Arrays.asList(endpointUrl.split(","));
try {
sessions = establishSessions(originalEndpoints, fusionUser, fusionPass, fusionRealm);
} catch (Exception exc) {
if (exc instanceof RuntimeException) {
throw (RuntimeException) exc;
} else {
throw new RuntimeException(exc);
}
}
random = new Random();
jsonObjectMapper = new ObjectMapper();
requestCounter = new AtomicInteger(0);
}
public CloseableHttpClient makeHttpClient() {
HttpClientBuilder builder = HttpClientBuilder.create();
builder.addInterceptorFirst(
new HttpRequestInterceptor() {
public void process(final HttpRequest request, final HttpContext context)
throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
if (!request.containsHeader("User-Agent")) {
request.addHeader(
"User-Agent",
"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:25.0) Gecko/20100101 Firefox/25.0");
}
if (!request.containsHeader("Accept")) {
request.addHeader(
"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
}
}
});
// builder.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(socketTimeout).build());
return builder.build();
}
private CloseableHttpClient getHttpClient(Boolean useSsl) throws SQLException {
boolean isCookieEnabled =
sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH) == null
|| (!JdbcConnectionParams.COOKIE_AUTH_FALSE.equalsIgnoreCase(
sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH)));
String cookieName =
sessConfMap.get(JdbcConnectionParams.COOKIE_NAME) == null
? JdbcConnectionParams.DEFAULT_COOKIE_NAMES_HS2
: sessConfMap.get(JdbcConnectionParams.COOKIE_NAME);
CookieStore cookieStore = isCookieEnabled ? new BasicCookieStore() : null;
HttpClientBuilder httpClientBuilder;
// Request interceptor for any request pre-processing logic
HttpRequestInterceptor requestInterceptor;
Map<String, String> additionalHttpHeaders = new HashMap<String, String>();
// Retrieve the additional HttpHeaders
for (Map.Entry<String, String> entry : sessConfMap.entrySet()) {
String key = entry.getKey();
if (key.startsWith(JdbcConnectionParams.HTTP_HEADER_PREFIX)) {
additionalHttpHeaders.put(
key.substring(JdbcConnectionParams.HTTP_HEADER_PREFIX.length()), entry.getValue());
}
}
// Configure http client for kerberos/password based authentication
if (isKerberosAuthMode()) {
/**
* Add an interceptor which sets the appropriate header in the request. It does the kerberos
* authentication and get the final service ticket, for sending to the server before every
* request. In https mode, the entire information is encrypted
*/
requestInterceptor =
new HttpKerberosRequestInterceptor(
sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL),
host,
getServerHttpUrl(useSsl),
assumeSubject,
cookieStore,
cookieName,
useSsl,
additionalHttpHeaders);
} else {
/**
* Add an interceptor to pass username/password in the header. In https mode, the entire
* information is encrypted
*/
requestInterceptor =
new HttpBasicAuthInterceptor(
getUserName(), getPassword(), cookieStore, cookieName, useSsl, additionalHttpHeaders);
}
// Configure http client for cookie based authentication
if (isCookieEnabled) {
// Create a http client with a retry mechanism when the server returns a status code of 401.
httpClientBuilder =
HttpClients.custom()
.setServiceUnavailableRetryStrategy(
new ServiceUnavailableRetryStrategy() {
@Override
public boolean retryRequest(
final HttpResponse response,
final int executionCount,
final HttpContext context) {
int statusCode = response.getStatusLine().getStatusCode();
boolean ret = statusCode == 401 && executionCount <= 1;
// Set the context attribute to true which will be interpreted by the request
// interceptor
if (ret) {
context.setAttribute(
Utils.HIVE_SERVER2_RETRY_KEY, Utils.HIVE_SERVER2_RETRY_TRUE);
}
return ret;
}
@Override
public long getRetryInterval() {
// Immediate retry
return 0;
}
});
} else {
httpClientBuilder = HttpClientBuilder.create();
}
// Add the request interceptor to the client builder
httpClientBuilder.addInterceptorFirst(requestInterceptor);
// Configure http client for SSL
if (useSsl) {
String useTwoWaySSL = sessConfMap.get(JdbcConnectionParams.USE_TWO_WAY_SSL);
String sslTrustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE);
String sslTrustStorePassword = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD);
KeyStore sslTrustStore;
SSLSocketFactory socketFactory;
/**
* The code within the try block throws: 1. SSLInitializationException 2. KeyStoreException 3.
* IOException 4. NoSuchAlgorithmException 5. CertificateException 6. KeyManagementException
* 7. UnrecoverableKeyException We don't want the client to retry on any of these, hence we
* catch all and throw a SQLException.
*/
try {
if (useTwoWaySSL != null && useTwoWaySSL.equalsIgnoreCase(JdbcConnectionParams.TRUE)) {
socketFactory = getTwoWaySSLSocketFactory();
} else if (sslTrustStorePath == null || sslTrustStorePath.isEmpty()) {
// Create a default socket factory based on standard JSSE trust material
socketFactory = SSLSocketFactory.getSocketFactory();
} else {
// Pick trust store config from the given path
sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
try (FileInputStream fis = new FileInputStream(sslTrustStorePath)) {
sslTrustStore.load(fis, sslTrustStorePassword.toCharArray());
}
socketFactory = new SSLSocketFactory(sslTrustStore);
}
socketFactory.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
final Registry<ConnectionSocketFactory> registry =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("https", socketFactory)
.build();
httpClientBuilder.setConnectionManager(new BasicHttpClientConnectionManager(registry));
} catch (Exception e) {
String msg =
"Could not create an https connection to " + jdbcUriString + ". " + e.getMessage();
throw new SQLException(msg, " 08S01", e);
}
}
return httpClientBuilder.build();
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
final String userAgent = context.getProperty(USER_AGENT).getValue();
final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
requestConfigBuilder.setConnectionRequestTimeout(
context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
requestConfigBuilder.setConnectTimeout(
context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
requestConfigBuilder.setRedirectsEnabled(false);
requestConfigBuilder.setSocketTimeout(
context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
final RequestConfig requestConfig = requestConfigBuilder.build();
final StreamThrottler throttler = throttlerRef.get();
final ProcessorLog logger = getLogger();
final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
String lastUrl = null;
long bytesToSend = 0L;
final List<FlowFile> toSend = new ArrayList<>();
DestinationAccepts destinationAccepts = null;
CloseableHttpClient client = null;
final String transactionId = UUID.randomUUID().toString();
final ObjectHolder<String> dnHolder = new ObjectHolder<>("none");
while (true) {
FlowFile flowFile = session.get();
if (flowFile == null) {
break;
}
final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
try {
new java.net.URL(url);
} catch (final MalformedURLException e) {
logger.error(
"After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
new Object[] {flowFile, url});
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
continue;
}
// If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing
// FlowFiles
if (lastUrl != null && !lastUrl.equals(url)) {
session.transfer(flowFile);
break;
}
lastUrl = url;
toSend.add(flowFile);
if (client == null || destinationAccepts == null) {
final Config config = getConfig(url, context);
final HttpClientConnectionManager conMan = config.getConnectionManager();
final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
clientBuilder.setConnectionManager(conMan);
clientBuilder.setUserAgent(userAgent);
clientBuilder.addInterceptorFirst(
new HttpResponseInterceptor() {
@Override
public void process(final HttpResponse response, final HttpContext httpContext)
throws HttpException, IOException {
HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
ManagedHttpClientConnection conn =
coreContext.getConnection(ManagedHttpClientConnection.class);
if (!conn.isOpen()) {
return;
}
SSLSession sslSession = conn.getSSLSession();
if (sslSession != null) {
final X509Certificate[] certChain = sslSession.getPeerCertificateChain();
if (certChain == null || certChain.length == 0) {
throw new SSLPeerUnverifiedException("No certificates found");
}
final X509Certificate cert = certChain[0];
dnHolder.set(cert.getSubjectDN().getName().trim());
}
}
});
clientBuilder.disableAutomaticRetries();
clientBuilder.disableContentCompression();
final String username = context.getProperty(USERNAME).getValue();
final String password = context.getProperty(PASSWORD).getValue();
// set the credentials if appropriate
if (username != null) {
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
if (password == null) {
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username));
} else {
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
}
clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
client = clientBuilder.build();
// determine whether or not destination accepts flowfile/gzip
destinationAccepts = config.getDestinationAccepts();
if (destinationAccepts == null) {
try {
if (sendAsFlowFile) {
destinationAccepts =
getDestinationAcceptance(client, url, getLogger(), transactionId);
} else {
destinationAccepts = new DestinationAccepts(false, false, false, false, null);
}
config.setDestinationAccepts(destinationAccepts);
} catch (IOException e) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
logger.error(
"Unable to communicate with destination {} to determine whether or not it can accept "
+ "flowfiles/gzip; routing {} to failure due to {}",
new Object[] {url, flowFile, e});
context.yield();
return;
}
}
}
// if we are not sending as flowfile, or if the destination doesn't accept V3 or V2
// (streaming) format,
// then only use a single FlowFile
if (!sendAsFlowFile
|| (!destinationAccepts.isFlowFileV3Accepted()
&& !destinationAccepts.isFlowFileV2Accepted())) {
break;
}
bytesToSend += flowFile.getSize();
if (bytesToSend > maxBatchBytes.longValue()) {
break;
}
}
if (toSend.isEmpty()) {
return;
}
final String url = lastUrl;
final HttpPost post = new HttpPost(url);
final List<FlowFile> flowFileList = toSend;
final DestinationAccepts accepts = destinationAccepts;
final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;
final EntityTemplate entity =
new EntityTemplate(
new ContentProducer() {
@Override
public void writeTo(final OutputStream rawOut) throws IOException {
final OutputStream throttled =
(throttler == null) ? rawOut : throttler.newThrottledOutputStream(rawOut);
OutputStream wrappedOut = new BufferedOutputStream(throttled);
if (compressionLevel > 0 && accepts.isGzipAccepted()) {
wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
}
try (final OutputStream out = wrappedOut) {
for (final FlowFile flowFile : flowFileList) {
session.read(
flowFile,
new InputStreamCallback() {
@Override
public void process(final InputStream rawIn) throws IOException {
try (final InputStream in = new BufferedInputStream(rawIn)) {
FlowFilePackager packager = null;
if (!sendAsFlowFile) {
packager = null;
} else if (accepts.isFlowFileV3Accepted()) {
packager = new FlowFilePackagerV3();
} else if (accepts.isFlowFileV2Accepted()) {
packager = new FlowFilePackagerV2();
} else if (accepts.isFlowFileV1Accepted()) {
packager = new FlowFilePackagerV1();
}
// if none of the above conditions is met, we should never get here,
// because
// we will have already verified that at least 1 of the FlowFile
// packaging
// formats is acceptable if sending as FlowFile.
if (packager == null) {
StreamUtils.copy(in, out);
} else {
final Map<String, String> flowFileAttributes;
if (isDestinationLegacyNiFi) {
// Old versions of NiFi expect nf.file.name and nf.file.path to
// indicate filename & path;
// in order to maintain backward compatibility, we copy the
// filename & path to those attribute keys.
flowFileAttributes = new HashMap<>(flowFile.getAttributes());
flowFileAttributes.put(
"nf.file.name",
flowFile.getAttribute(CoreAttributes.FILENAME.key()));
flowFileAttributes.put(
"nf.file.path",
flowFile.getAttribute(CoreAttributes.PATH.key()));
} else {
flowFileAttributes = flowFile.getAttributes();
}
packager.packageFlowFile(
in, out, flowFileAttributes, flowFile.getSize());
}
}
}
});
}
out.flush();
}
}
});
entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
post.setEntity(entity);
post.setConfig(requestConfig);
final String contentType;
if (sendAsFlowFile) {
if (accepts.isFlowFileV3Accepted()) {
contentType = APPLICATION_FLOW_FILE_V3;
} else if (accepts.isFlowFileV2Accepted()) {
contentType = APPLICATION_FLOW_FILE_V2;
} else if (accepts.isFlowFileV1Accepted()) {
contentType = APPLICATION_FLOW_FILE_V1;
} else {
logger.error(
"Cannot send data to {} because the destination does not accept FlowFiles and this processor is "
+ "configured to deliver FlowFiles; rolling back session",
new Object[] {url});
session.rollback();
context.yield();
return;
}
} else {
final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key());
contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue;
}
final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) {
final Pattern pattern = Pattern.compile(attributeHeaderRegex);
final Map<String, String> attributes = flowFileList.get(0).getAttributes();
for (final Map.Entry<String, String> entry : attributes.entrySet()) {
final String key = entry.getKey();
if (pattern.matcher(key).matches()) {
post.setHeader(entry.getKey(), entry.getValue());
}
}
}
post.setHeader(CONTENT_TYPE, contentType);
post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
post.setHeader(TRANSACTION_ID_HEADER, transactionId);
if (compressionLevel > 0 && accepts.isGzipAccepted()) {
post.setHeader(GZIPPED_HEADER, "true");
}
// Do the actual POST
final String flowFileDescription =
toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";
final String uploadDataRate;
final long uploadMillis;
CloseableHttpResponse response = null;
try {
final StopWatch stopWatch = new StopWatch(true);
response = client.execute(post);
// consume input stream entirely, ignoring its contents. If we
// don't do this, the Connection will not be returned to the pool
EntityUtils.consume(response.getEntity());
stopWatch.stop();
uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
} catch (final IOException e) {
logger.error(
"Failed to Post {} due to {}; transferring to failure",
new Object[] {flowFileDescription, e});
context.yield();
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
}
return;
} finally {
if (response != null) {
try {
response.close();
} catch (IOException e) {
getLogger().warn("Failed to close HTTP Response due to {}", new Object[] {e});
}
}
}
// If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
// of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
// from posting to some other webservice and then attempting to delete some resource to which
// we are redirected
final int responseCode = response.getStatusLine().getStatusCode();
final String responseReason = response.getStatusLine().getReasonPhrase();
String holdUri = null;
if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
if (locationUriHeader != null) {
if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
if (holdUriHeader != null) {
holdUri = holdUriHeader.getValue();
}
}
}
if (holdUri == null) {
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
logger.error(
"Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
new Object[] {flowFile, url, responseCode, responseReason});
session.transfer(flowFile, REL_FAILURE);
}
return;
}
}
if (holdUri == null) {
if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
logger.error(
"Failed to Post {} to {}: response code was {}:{}; will yield processing, "
+ "since the destination is temporarily unavailable",
new Object[] {flowFile, url, responseCode, responseReason});
session.transfer(flowFile, REL_FAILURE);
}
context.yield();
return;
}
if (responseCode >= 300) {
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
logger.error(
"Failed to Post {} to {}: response code was {}:{}",
new Object[] {flowFile, url, responseCode, responseReason});
session.transfer(flowFile, REL_FAILURE);
}
return;
}
logger.info(
"Successfully Posted {} to {} in {} at a rate of {}",
new Object[] {
flowFileDescription,
url,
FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS),
uploadDataRate
});
for (final FlowFile flowFile : toSend) {
session
.getProvenanceReporter()
.send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true);
session.transfer(flowFile, REL_SUCCESS);
}
return;
}
//
// the response indicated a Hold URI; delete the Hold.
//
// determine the full URI of the Flow File's Hold; Unfortunately, the responses that are
// returned have
// changed over the past, so we have to take into account a few different possibilities.
String fullHoldUri = holdUri;
if (holdUri.startsWith("/contentListener")) {
// If the Hold URI that we get starts with /contentListener, it may not really be
// /contentListener,
// as this really indicates that it should be whatever we posted to -- if posting directly to
// the
// ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer,
// we may
// be posting to some other URL.
fullHoldUri = url + holdUri.substring(16);
} else if (holdUri.startsWith("/")) {
// URL indicates the full path but not hostname or port; use the same hostname & port that we
// posted
// to but use the full path indicated by the response.
int firstSlash = url.indexOf("/", 8);
if (firstSlash < 0) {
firstSlash = url.length();
}
final String beforeSlash = url.substring(0, firstSlash);
fullHoldUri = beforeSlash + holdUri;
} else if (!holdUri.startsWith("http")) {
// Absolute URL
fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
}
final HttpDelete delete = new HttpDelete(fullHoldUri);
delete.setHeader(TRANSACTION_ID_HEADER, transactionId);
while (true) {
try {
final HttpResponse holdResponse = client.execute(delete);
EntityUtils.consume(holdResponse.getEntity());
final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
if (holdStatusCode >= 300) {
logger.error(
"Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
new Object[] {flowFileDescription, holdStatusCode, holdReason});
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
}
return;
}
logger.info(
"Successfully Posted {} to {} in {} milliseconds at a rate of {}",
new Object[] {flowFileDescription, url, uploadMillis, uploadDataRate});
for (FlowFile flowFile : toSend) {
session.getProvenanceReporter().send(flowFile, url);
session.transfer(flowFile, REL_SUCCESS);
}
return;
} catch (final IOException e) {
logger.warn(
"Failed to delete Hold that destination placed on {} due to {}",
new Object[] {flowFileDescription, e});
}
if (!isScheduled()) {
context.yield();
logger.warn(
"Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
new Object[] {flowFileDescription});
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
}
return;
}
}
}
public EasyHttpClient() {
CookieSpecProvider easySpecProvider =
new CookieSpecProvider() {
public CookieSpec create(HttpContext context) {
return new BrowserCompatSpec() {
public void validate(Cookie cookie, CookieOrigin origin)
throws MalformedCookieException {}
};
}
};
SSLContext sslContext = SSLContexts.createDefault(); // 忽略证书主机名验证
SSLConnectionSocketFactory sslsf =
new SSLConnectionSocketFactory(
sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
Registry<ConnectionSocketFactory> reg =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https", sslsf)
.build();
connectionManager = new PoolingHttpClientConnectionManager(reg);
connectionManager.setDefaultMaxPerRoute(100); // 同一个路由允许最大连接数
// connectionManager.setMaxTotal(poolSize);
HttpHost localhost = new HttpHost("locahost", 80); // 如果是多网卡，这里选择出口IP？
connectionManager.setMaxPerRoute(new HttpRoute(localhost), 50);
Registry<CookieSpecProvider> r =
RegistryBuilder.<CookieSpecProvider>create()
.register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())
.register(CookieSpecs.BROWSER_COMPATIBILITY, new BrowserCompatSpecFactory())
.register("easy", easySpecProvider)
.build();
RequestConfig requestConfig =
RequestConfig.custom()
.setCookieSpec("easy")
.setSocketTimeout(10000)
.setConnectTimeout(10000)
.build();
ConnectionConfig connectioncfg = ConnectionConfig.custom().setCharset(Charsets.UTF_8).build();
SocketConfig socketConfig =
SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
HttpRequestRetryHandler myRetryHandler =
new HttpRequestRetryHandler() {
public boolean retryRequest(
IOException exception, int executionCount, HttpContext context) {
if (executionCount >= 3) {
return false;
}
if (exception instanceof SSLHandshakeException) {
return false;
} else {
return true;
}
}
};
HttpClientBuilder builder =
HttpClients.custom() // 默认
.setConnectionManager(connectionManager) // 链接管理器
.setDefaultSocketConfig(socketConfig) // socket管理器
.setRetryHandler(myRetryHandler) // 重试3次
.setDefaultConnectionConfig(connectioncfg) // 链接配置，如默认字符编码
.setDefaultCookieSpecRegistry(r) // cookie策略
.setUserAgent(Chorme_User_Agent) // 浏览器请求头
.setDefaultRequestConfig(requestConfig) // 链接配置，超时等
.setDefaultCookieStore(cookieStore); // cookie
builder.addInterceptorFirst(
new HttpRequestInterceptor() {
public void process(final HttpRequest request, final HttpContext context)
throws HttpException, IOException {
if (!request.containsHeader("Accept-Encoding")) {
request.addHeader("Accept-Encoding", "gzip");
}
}
});
httpclient = builder.build();
}
@Override
@Guarded(by = STARTED)
public HttpClientBuilder prepare(final @Nullable Customizer customizer) {
final HttpClientPlan plan = new HttpClientPlan();
// attach connection manager early, so customizer has chance to replace it if needed
plan.getClient().setConnectionManager(sharedConnectionManager);
// apply defaults
defaultsCustomizer.customize(plan);
// apply globals
new ConfigurationCustomizer(getConfigurationInternal()).customize(plan);
// apply instance customization
if (customizer != null) {
customizer.customize(plan);
}
// apply plan to builder
HttpClientBuilder builder = plan.getClient();
builder.setDefaultConnectionConfig(plan.getConnection().build());
builder.setDefaultSocketConfig(plan.getSocket().build());
builder.setDefaultRequestConfig(plan.getRequest().build());
builder.setDefaultCredentialsProvider(plan.getCredentials());
builder.addInterceptorFirst(
(HttpRequest request, HttpContext context) -> {
// add custom http-context attributes
for (Entry<String, Object> entry : plan.getAttributes().entrySet()) {
// only set context attribute if not already set, to allow per request overrides
if (context.getAttribute(entry.getKey()) == null) {
context.setAttribute(entry.getKey(), entry.getValue());
}
}
// add custom http-request headers
for (Entry<String, String> entry : plan.getHeaders().entrySet()) {
request.addHeader(entry.getKey(), entry.getValue());
}
});
builder.addInterceptorLast(
(HttpRequest httpRequest, HttpContext httpContext) -> {
if (outboundLog.isDebugEnabled()) {
httpContext.setAttribute(CTX_REQ_STOPWATCH, Stopwatch.createStarted());
httpContext.setAttribute(CTX_REQ_URI, getRequestURI(httpContext));
outboundLog.debug(
"{} > {}", httpContext.getAttribute(CTX_REQ_URI), httpRequest.getRequestLine());
}
});
builder.addInterceptorLast(
(HttpResponse httpResponse, HttpContext httpContext) -> {
Stopwatch stopwatch = (Stopwatch) httpContext.getAttribute(CTX_REQ_STOPWATCH);
if (stopwatch != null) {
outboundLog.debug(
"{} < {} @ {}",
httpContext.getAttribute(CTX_REQ_URI),
httpResponse.getStatusLine(),
stopwatch);
}
});
return builder;
}