private HttpClient getClient() {
if (client == null) {
HttpClientBuilder clientBuilder = HttpClients.custom();
RequestConfig.Builder configBuilder = RequestConfig.custom();
if (proxy != null && !Proxy.NO_PROXY.equals(proxy)) {
isUsingProxy = true;
InetSocketAddress adr = (InetSocketAddress) proxy.address();
clientBuilder.setProxy(new HttpHost(adr.getHostName(), adr.getPort()));
}
if (timeout != null) {
configBuilder.setConnectTimeout(timeout.intValue());
}
if (readTimeout != null) {
configBuilder.setSocketTimeout(readTimeout.intValue());
}
if (followRedirects != null) {
configBuilder.setRedirectsEnabled(followRedirects.booleanValue());
}
if (hostnameverifier != null) {
SSLConnectionSocketFactory sslConnectionFactory =
new SSLConnectionSocketFactory(getSSLContext(), hostnameverifier);
clientBuilder.setSSLSocketFactory(sslConnectionFactory);
Registry<ConnectionSocketFactory> registry =
RegistryBuilder.<ConnectionSocketFactory>create()
.register("https", sslConnectionFactory)
.register("http", PlainConnectionSocketFactory.INSTANCE)
.build();
clientBuilder.setConnectionManager(new BasicHttpClientConnectionManager(registry));
}
clientBuilder.setDefaultRequestConfig(configBuilder.build());
client = clientBuilder.build();
}
return client;
}
private Optional<RequestConfig> addJerseyRequestConfig(ClientRequest clientRequest) {
final Integer timeout =
clientRequest.resolveProperty(ClientProperties.READ_TIMEOUT, Integer.class);
final Integer connectTimeout =
clientRequest.resolveProperty(ClientProperties.CONNECT_TIMEOUT, Integer.class);
final Boolean followRedirects =
clientRequest.resolveProperty(ClientProperties.FOLLOW_REDIRECTS, Boolean.class);
if (timeout != null || connectTimeout != null || followRedirects != null) {
final RequestConfig.Builder requestConfig = RequestConfig.copy(defaultRequestConfig);
if (timeout != null) {
requestConfig.setSocketTimeout(timeout);
}
if (connectTimeout != null) {
requestConfig.setConnectTimeout(connectTimeout);
}
if (followRedirects != null) {
requestConfig.setRedirectsEnabled(followRedirects);
}
return Optional.of(requestConfig.build());
}
return Optional.empty();
}
/**
* 构造请求的方法，如post，get，header等<br>
* 设置请求参数，如超时时间
*
* @param url 请求的URL
* @param method 请求的方法
* @param postdata post的数据
* @param headers 请求头
* @return
*/
protected HttpUriRequest getHttpUriRequest(
String url, String method, Map<String, String> postdata, Map<String, String> headers) {
RequestBuilder requestBuilder = selectRequestMethod(method, postdata, headers).setUri(url);
requestBuilder.addHeader("Accept", "*/*");
requestBuilder.addHeader("Connection", "keep-alive");
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
int timeout = 45000; // 超时时间
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(timeout)
.setSocketTimeout(timeout)
.setConnectTimeout(timeout)
// .setRedirectsEnabled(false)//禁止自动跳转
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (null != host && !"".equals(host) && port > 10) {
HttpHost proxy = new HttpHost(host, port);
requestConfigBuilder.setProxy(proxy);
}
requestBuilder.setConfig(requestConfigBuilder.build());
return requestBuilder.build();
}
public PingInfo ping(String host, int restPort, boolean noProxy) throws Exception {
PingInfo myPingInfo = pingInfoProvider.createPingInfo();
RequestConfig.Builder requestConfigBuilder =
httpClientProvider.createRequestConfigBuilder(host, noProxy);
String url = String.format(URL_PATTERN, host, restPort) + PingHandler.URL;
HttpUriRequest request =
RequestBuilder.post(url)
.setConfig(requestConfigBuilder.build())
.addParameter(PingHandler.PING_INFO_INPUT_NAME, gson.toJson(myPingInfo))
.build();
CloseableHttpResponse response = httpClientProvider.executeRequest(request);
try {
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
EntityUtils.consumeQuietly(response.getEntity());
throw new Exception(statusLine.getStatusCode() + " " + statusLine.getReasonPhrase());
}
HttpEntity entity = response.getEntity();
String content = EntityUtils.toString(entity);
EntityUtils.consumeQuietly(entity);
PingInfo receivedPingInfo = gson.fromJson(content, PingInfo.class);
receivedPingInfo.getAgentId().setHost(request.getURI().getHost());
return receivedPingInfo;
} finally {
response.close();
}
}
private static CloseableHttpClient createHTTPClient(
PrivateKey privateKey, X509Certificate certificate, boolean validateCertificate) {
// HttpClientBuilder clientBuilder = HttpClientBuilder.create();
HttpClientBuilder clientBuilder = HttpClients.custom();
try {
SSLContext sslContext =
TrustUtils.getCustomSSLContext(privateKey, certificate, validateCertificate);
clientBuilder.setSslcontext(sslContext);
} catch (KeyStoreException
| NoSuchAlgorithmException
| CertificateException
| IOException
| KeyManagementException
| UnrecoverableKeyException
| NoSuchProviderException ex) {
LOGGER.log(Level.WARNING, "unable to set SSL context", ex);
return null;
}
RequestConfig.Builder rcBuilder = RequestConfig.custom();
// handle redirects :)
rcBuilder.setRedirectsEnabled(true);
// HttpClient bug caused by Lighttpd
rcBuilder.setExpectContinueEnabled(false);
clientBuilder.setDefaultRequestConfig(rcBuilder.build());
// create connection manager
// ClientConnectionManager connMgr = new SingleClientConnManager(params, registry);
// return new DefaultHttpClient(connMgr, params);
return clientBuilder.build();
}
public void init() {
if (httpClient == null) {
RequestConfig.Builder requestBuilder = RequestConfig.custom();
requestBuilder = requestBuilder.setConnectTimeout(bookmakerConfig.getTimeout());
requestBuilder = requestBuilder.setConnectionRequestTimeout(bookmakerConfig.getTimeout());
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setDefaultRequestConfig(requestBuilder.build());
httpClient = builder.build();
}
}
protected RequestConfig configureRequest(RequestConfig.Builder rcb, Settings settings)
throws HTTPException {
// Configure the RequestConfig
for (Prop key : settings.getKeys()) {
Object value = settings.getParameter(key);
boolean tf = (value instanceof Boolean ? (Boolean) value : false);
if (key == Prop.ALLOW_CIRCULAR_REDIRECTS) {
rcb.setCircularRedirectsAllowed(tf);
} else if (key == Prop.HANDLE_REDIRECTS) {
rcb.setRedirectsEnabled(tf);
rcb.setRelativeRedirectsAllowed(tf);
} else if (key == Prop.MAX_REDIRECTS) {
rcb.setMaxRedirects((Integer) value);
} else if (key == Prop.SO_TIMEOUT) {
rcb.setSocketTimeout((Integer) value);
} else if (key == Prop.CONN_TIMEOUT) {
rcb.setConnectTimeout((Integer) value);
} else if (key == Prop.CONN_REQ_TIMEOUT) {
rcb.setConnectionRequestTimeout((Integer) value);
} else if (key == Prop.MAX_THREADS) {
connmgr.setMaxTotal((Integer) value);
connmgr.setDefaultMaxPerRoute((Integer) value);
} /* else ignore */
}
RequestConfig cfg = rcb.build();
return cfg;
}
private ActivityClient(String host, int port, int socketTimeout) {
RequestConfig.Builder requestBuilder = RequestConfig.custom();
requestBuilder = requestBuilder.setConnectTimeout(socketTimeout);
requestBuilder = requestBuilder.setConnectionRequestTimeout(socketTimeout);
requestBuilder = requestBuilder.setSocketTimeout(socketTimeout);
HttpClientBuilder builder = HttpClientBuilder.create();
builder.setDefaultRequestConfig(requestBuilder.build());
httpClient = builder.build();
this.connectionUrl = "http://" + host + ":" + port + "/connection";
this.queryUrl = "http://" + host + ":" + port + "/query";
this.preparedStatementExecutionUrl =
"http://" + host + ":" + port + "/prepared-statement-execution";
this.batchUrl = "http://" + host + ":" + port + "/batch-execution";
this.preparedStatementPreparationUrl =
"http://" + host + ":" + port + "/prepared-statement-preparation";
}
/**
* Sets the timeouts of the HTTP client.
*
* @param connectionTimeout timeout until connection established in milliseconds. Zero means no
* timeout.
* @param socketTimeout timeout for waiting for data in milliseconds. Zero means no timeout.
* @param maxRequests maximum number of connections to a particuar host
*/
public static void setParams(int connectionTimeout, int socketTimeout, int maxRequests) {
PrudentHttpEntityResolver.maxRequests = maxRequests;
PoolingHttpClientConnectionManager phcConnMgr;
Registry<ConnectionSocketFactory> registry = //
RegistryBuilder.<ConnectionSocketFactory>create() //
.register("http", PlainConnectionSocketFactory.getSocketFactory()) //
.register("https", SSLConnectionSocketFactory.getSocketFactory()) //
.build();
HttpClientBuilder builder = HttpClients.custom();
builder.setRedirectStrategy(new LaxRedirectStrategy());
builder.setMaxConnPerRoute(maxRequests);
builder.setMaxConnTotal(200);
if ("true".equals(System.getProperty("nu.validator.xml.promiscuous-ssl", "false"))) { //
try {
SSLContext promiscuousSSLContext =
new SSLContextBuilder() //
.loadTrustMaterial(
null,
new TrustStrategy() {
@Override
public boolean isTrusted(X509Certificate[] arg0, String arg1)
throws CertificateException {
return true;
}
})
.build();
builder.setSslcontext(promiscuousSSLContext);
HostnameVerifier verifier = //
SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;
SSLConnectionSocketFactory promiscuousSSLConnSocketFactory = //
new SSLConnectionSocketFactory(promiscuousSSLContext, verifier);
registry =
RegistryBuilder.<ConnectionSocketFactory>create() //
.register("https", promiscuousSSLConnSocketFactory) //
.register("http", PlainConnectionSocketFactory.getSocketFactory()) //
.build();
} catch (KeyManagementException
| KeyStoreException
| NoSuchAlgorithmException
| NumberFormatException e) {
e.printStackTrace();
}
}
phcConnMgr = new PoolingHttpClientConnectionManager(registry);
phcConnMgr.setDefaultMaxPerRoute(maxRequests);
phcConnMgr.setMaxTotal(200);
builder.setConnectionManager(phcConnMgr);
RequestConfig.Builder config = RequestConfig.custom();
config.setCircularRedirectsAllowed(true);
config.setMaxRedirects(20); // Gecko default
config.setConnectTimeout(connectionTimeout);
config.setCookieSpec(CookieSpecs.BEST_MATCH);
config.setSocketTimeout(socketTimeout);
client = builder.setDefaultRequestConfig(config.build()).build();
}
@Override
public OutgoingRequest createOutgoingRequest(
DriverRequest originalRequest, String uri, boolean proxy) {
// Extract the host in the URI. This is the host we have to send the
// request to physically.
HttpHost physicalHost = UriUtils.extractHost(uri);
if (!originalRequest.isExternal()) {
if (preserveHost) {
// Preserve host if required
HttpHost virtualHost = HttpRequestHelper.getHost(originalRequest.getOriginalRequest());
// Rewrite the uri with the virtualHost
uri = UriUtils.rewriteURI(uri, virtualHost);
} else {
uri = UriUtils.rewriteURI(uri, firstBaseUrlHost);
}
}
RequestConfig.Builder builder = RequestConfig.custom();
builder.setConnectTimeout(connectTimeout);
builder.setSocketTimeout(socketTimeout);
// Use browser compatibility cookie policy. This policy is the closest
// to the behavior of a real browser.
builder.setCookieSpec(CustomBrowserCompatSpecFactory.CUSTOM_BROWSER_COMPATIBILITY);
builder.setRedirectsEnabled(false);
RequestConfig config = builder.build();
OutgoingRequestContext context = new OutgoingRequestContext();
String method = "GET";
if (proxy) {
method = originalRequest.getOriginalRequest().getRequestLine().getMethod().toUpperCase();
}
OutgoingRequest outgoingRequest =
new OutgoingRequest(
method,
uri,
originalRequest.getOriginalRequest().getProtocolVersion(),
originalRequest,
config,
context);
if (ENTITY_METHODS.contains(method)) {
outgoingRequest.setEntity(originalRequest.getOriginalRequest().getEntity());
} else if (!SIMPLE_METHODS.contains(method)) {
throw new UnsupportedHttpMethodException(method + " " + uri);
}
context.setPhysicalHost(physicalHost);
context.setOutgoingRequest(outgoingRequest);
context.setProxy(proxy);
return outgoingRequest;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
final String userAgent = context.getProperty(USER_AGENT).getValue();
final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
requestConfigBuilder.setConnectionRequestTimeout(
context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
requestConfigBuilder.setConnectTimeout(
context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
requestConfigBuilder.setRedirectsEnabled(false);
requestConfigBuilder.setSocketTimeout(
context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
final RequestConfig requestConfig = requestConfigBuilder.build();
final StreamThrottler throttler = throttlerRef.get();
final ProcessorLog logger = getLogger();
final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
String lastUrl = null;
long bytesToSend = 0L;
final List<FlowFile> toSend = new ArrayList<>();
DestinationAccepts destinationAccepts = null;
CloseableHttpClient client = null;
final String transactionId = UUID.randomUUID().toString();
final ObjectHolder<String> dnHolder = new ObjectHolder<>("none");
while (true) {
FlowFile flowFile = session.get();
if (flowFile == null) {
break;
}
final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
try {
new java.net.URL(url);
} catch (final MalformedURLException e) {
logger.error(
"After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
new Object[] {flowFile, url});
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
continue;
}
// If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing
// FlowFiles
if (lastUrl != null && !lastUrl.equals(url)) {
session.transfer(flowFile);
break;
}
lastUrl = url;
toSend.add(flowFile);
if (client == null || destinationAccepts == null) {
final Config config = getConfig(url, context);
final HttpClientConnectionManager conMan = config.getConnectionManager();
final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
clientBuilder.setConnectionManager(conMan);
clientBuilder.setUserAgent(userAgent);
clientBuilder.addInterceptorFirst(
new HttpResponseInterceptor() {
@Override
public void process(final HttpResponse response, final HttpContext httpContext)
throws HttpException, IOException {
HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
ManagedHttpClientConnection conn =
coreContext.getConnection(ManagedHttpClientConnection.class);
if (!conn.isOpen()) {
return;
}
SSLSession sslSession = conn.getSSLSession();
if (sslSession != null) {
final X509Certificate[] certChain = sslSession.getPeerCertificateChain();
if (certChain == null || certChain.length == 0) {
throw new SSLPeerUnverifiedException("No certificates found");
}
final X509Certificate cert = certChain[0];
dnHolder.set(cert.getSubjectDN().getName().trim());
}
}
});
clientBuilder.disableAutomaticRetries();
clientBuilder.disableContentCompression();
final String username = context.getProperty(USERNAME).getValue();
final String password = context.getProperty(PASSWORD).getValue();
// set the credentials if appropriate
if (username != null) {
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
if (password == null) {
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username));
} else {
credentialsProvider.setCredentials(
AuthScope.ANY, new UsernamePasswordCredentials(username, password));
}
clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}
client = clientBuilder.build();
// determine whether or not destination accepts flowfile/gzip
destinationAccepts = config.getDestinationAccepts();
if (destinationAccepts == null) {
try {
if (sendAsFlowFile) {
destinationAccepts =
getDestinationAcceptance(client, url, getLogger(), transactionId);
} else {
destinationAccepts = new DestinationAccepts(false, false, false, false, null);
}
config.setDestinationAccepts(destinationAccepts);
} catch (IOException e) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
logger.error(
"Unable to communicate with destination {} to determine whether or not it can accept "
+ "flowfiles/gzip; routing {} to failure due to {}",
new Object[] {url, flowFile, e});
context.yield();
return;
}
}
}
// if we are not sending as flowfile, or if the destination doesn't accept V3 or V2
// (streaming) format,
// then only use a single FlowFile
if (!sendAsFlowFile
|| (!destinationAccepts.isFlowFileV3Accepted()
&& !destinationAccepts.isFlowFileV2Accepted())) {
break;
}
bytesToSend += flowFile.getSize();
if (bytesToSend > maxBatchBytes.longValue()) {
break;
}
}
if (toSend.isEmpty()) {
return;
}
final String url = lastUrl;
final HttpPost post = new HttpPost(url);
final List<FlowFile> flowFileList = toSend;
final DestinationAccepts accepts = destinationAccepts;
final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;
final EntityTemplate entity =
new EntityTemplate(
new ContentProducer() {
@Override
public void writeTo(final OutputStream rawOut) throws IOException {
final OutputStream throttled =
(throttler == null) ? rawOut : throttler.newThrottledOutputStream(rawOut);
OutputStream wrappedOut = new BufferedOutputStream(throttled);
if (compressionLevel > 0 && accepts.isGzipAccepted()) {
wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
}
try (final OutputStream out = wrappedOut) {
for (final FlowFile flowFile : flowFileList) {
session.read(
flowFile,
new InputStreamCallback() {
@Override
public void process(final InputStream rawIn) throws IOException {
try (final InputStream in = new BufferedInputStream(rawIn)) {
FlowFilePackager packager = null;
if (!sendAsFlowFile) {
packager = null;
} else if (accepts.isFlowFileV3Accepted()) {
packager = new FlowFilePackagerV3();
} else if (accepts.isFlowFileV2Accepted()) {
packager = new FlowFilePackagerV2();
} else if (accepts.isFlowFileV1Accepted()) {
packager = new FlowFilePackagerV1();
}
// if none of the above conditions is met, we should never get here,
// because
// we will have already verified that at least 1 of the FlowFile
// packaging
// formats is acceptable if sending as FlowFile.
if (packager == null) {
StreamUtils.copy(in, out);
} else {
final Map<String, String> flowFileAttributes;
if (isDestinationLegacyNiFi) {
// Old versions of NiFi expect nf.file.name and nf.file.path to
// indicate filename & path;
// in order to maintain backward compatibility, we copy the
// filename & path to those attribute keys.
flowFileAttributes = new HashMap<>(flowFile.getAttributes());
flowFileAttributes.put(
"nf.file.name",
flowFile.getAttribute(CoreAttributes.FILENAME.key()));
flowFileAttributes.put(
"nf.file.path",
flowFile.getAttribute(CoreAttributes.PATH.key()));
} else {
flowFileAttributes = flowFile.getAttributes();
}
packager.packageFlowFile(
in, out, flowFileAttributes, flowFile.getSize());
}
}
}
});
}
out.flush();
}
}
});
entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
post.setEntity(entity);
post.setConfig(requestConfig);
final String contentType;
if (sendAsFlowFile) {
if (accepts.isFlowFileV3Accepted()) {
contentType = APPLICATION_FLOW_FILE_V3;
} else if (accepts.isFlowFileV2Accepted()) {
contentType = APPLICATION_FLOW_FILE_V2;
} else if (accepts.isFlowFileV1Accepted()) {
contentType = APPLICATION_FLOW_FILE_V1;
} else {
logger.error(
"Cannot send data to {} because the destination does not accept FlowFiles and this processor is "
+ "configured to deliver FlowFiles; rolling back session",
new Object[] {url});
session.rollback();
context.yield();
return;
}
} else {
final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key());
contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue;
}
final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) {
final Pattern pattern = Pattern.compile(attributeHeaderRegex);
final Map<String, String> attributes = flowFileList.get(0).getAttributes();
for (final Map.Entry<String, String> entry : attributes.entrySet()) {
final String key = entry.getKey();
if (pattern.matcher(key).matches()) {
post.setHeader(entry.getKey(), entry.getValue());
}
}
}
post.setHeader(CONTENT_TYPE, contentType);
post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
post.setHeader(TRANSACTION_ID_HEADER, transactionId);
if (compressionLevel > 0 && accepts.isGzipAccepted()) {
post.setHeader(GZIPPED_HEADER, "true");
}
// Do the actual POST
final String flowFileDescription =
toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";
final String uploadDataRate;
final long uploadMillis;
CloseableHttpResponse response = null;
try {
final StopWatch stopWatch = new StopWatch(true);
response = client.execute(post);
// consume input stream entirely, ignoring its contents. If we
// don't do this, the Connection will not be returned to the pool
EntityUtils.consume(response.getEntity());
stopWatch.stop();
uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
} catch (final IOException e) {
logger.error(
"Failed to Post {} due to {}; transferring to failure",
new Object[] {flowFileDescription, e});
context.yield();
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
}
return;
} finally {
if (response != null) {
try {
response.close();
} catch (IOException e) {
getLogger().warn("Failed to close HTTP Response due to {}", new Object[] {e});
}
}
}
// If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
// of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
// from posting to some other webservice and then attempting to delete some resource to which
// we are redirected
final int responseCode = response.getStatusLine().getStatusCode();
final String responseReason = response.getStatusLine().getReasonPhrase();
String holdUri = null;
if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
if (locationUriHeader != null) {
if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
if (holdUriHeader != null) {
holdUri = holdUriHeader.getValue();
}
}
}
if (holdUri == null) {
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
logger.error(
"Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
new Object[] {flowFile, url, responseCode, responseReason});
session.transfer(flowFile, REL_FAILURE);
}
return;
}
}
if (holdUri == null) {
if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
logger.error(
"Failed to Post {} to {}: response code was {}:{}; will yield processing, "
+ "since the destination is temporarily unavailable",
new Object[] {flowFile, url, responseCode, responseReason});
session.transfer(flowFile, REL_FAILURE);
}
context.yield();
return;
}
if (responseCode >= 300) {
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
logger.error(
"Failed to Post {} to {}: response code was {}:{}",
new Object[] {flowFile, url, responseCode, responseReason});
session.transfer(flowFile, REL_FAILURE);
}
return;
}
logger.info(
"Successfully Posted {} to {} in {} at a rate of {}",
new Object[] {
flowFileDescription,
url,
FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS),
uploadDataRate
});
for (final FlowFile flowFile : toSend) {
session
.getProvenanceReporter()
.send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true);
session.transfer(flowFile, REL_SUCCESS);
}
return;
}
//
// the response indicated a Hold URI; delete the Hold.
//
// determine the full URI of the Flow File's Hold; Unfortunately, the responses that are
// returned have
// changed over the past, so we have to take into account a few different possibilities.
String fullHoldUri = holdUri;
if (holdUri.startsWith("/contentListener")) {
// If the Hold URI that we get starts with /contentListener, it may not really be
// /contentListener,
// as this really indicates that it should be whatever we posted to -- if posting directly to
// the
// ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer,
// we may
// be posting to some other URL.
fullHoldUri = url + holdUri.substring(16);
} else if (holdUri.startsWith("/")) {
// URL indicates the full path but not hostname or port; use the same hostname & port that we
// posted
// to but use the full path indicated by the response.
int firstSlash = url.indexOf("/", 8);
if (firstSlash < 0) {
firstSlash = url.length();
}
final String beforeSlash = url.substring(0, firstSlash);
fullHoldUri = beforeSlash + holdUri;
} else if (!holdUri.startsWith("http")) {
// Absolute URL
fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
}
final HttpDelete delete = new HttpDelete(fullHoldUri);
delete.setHeader(TRANSACTION_ID_HEADER, transactionId);
while (true) {
try {
final HttpResponse holdResponse = client.execute(delete);
EntityUtils.consume(holdResponse.getEntity());
final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
if (holdStatusCode >= 300) {
logger.error(
"Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
new Object[] {flowFileDescription, holdStatusCode, holdReason});
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
}
return;
}
logger.info(
"Successfully Posted {} to {} in {} milliseconds at a rate of {}",
new Object[] {flowFileDescription, url, uploadMillis, uploadDataRate});
for (FlowFile flowFile : toSend) {
session.getProvenanceReporter().send(flowFile, url);
session.transfer(flowFile, REL_SUCCESS);
}
return;
} catch (final IOException e) {
logger.warn(
"Failed to delete Hold that destination placed on {} due to {}",
new Object[] {flowFileDescription, e});
}
if (!isScheduled()) {
context.yield();
logger.warn(
"Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
new Object[] {flowFileDescription});
for (FlowFile flowFile : toSend) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
}
return;
}
}
}
@Override
public Page download(Request request, Task task) {
Site site = null;
if (task != null) {
site = task.getSite();
}
Set<Integer> acceptStatCode;
String charset = null;
Map<String, String> headers = null;
if (site != null) {
acceptStatCode = site.getAcceptStatCode();
charset = site.getCharset();
headers = site.getHeaders();
} else {
acceptStatCode = Sets.newHashSet(200);
}
logger.info("downloading page " + request.getUrl());
RequestBuilder requestBuilder = RequestBuilder.get().setUri(request.getUrl());
if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
}
}
RequestConfig.Builder requestConfigBuilder =
RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut())
.setSocketTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (site != null && site.getHttpProxy() != null) {
requestConfigBuilder.setProxy(site.getHttpProxy());
}
requestBuilder.setConfig(requestConfigBuilder.build());
CloseableHttpResponse httpResponse = null;
try {
httpResponse = getHttpClient(site).execute(requestBuilder.build());
int statusCode = httpResponse.getStatusLine().getStatusCode();
if (acceptStatCode.contains(statusCode)) {
// charset
if (charset == null) {
String value = httpResponse.getEntity().getContentType().getValue();
charset = UrlUtils.getCharset(value);
}
return handleResponse(request, charset, httpResponse, task);
} else {
logger.warn("code error " + statusCode + "\t" + request.getUrl());
return null;
}
} catch (IOException e) {
logger.warn("download page " + request.getUrl() + " error", e);
if (site.getCycleRetryTimes() > 0) {
return addToCycleRetry(request, site);
}
return null;
} finally {
try {
if (httpResponse != null) {
// ensure the connection is released back to pool
EntityUtils.consume(httpResponse.getEntity());
}
} catch (IOException e) {
logger.warn("close response fail", e);
}
}
}
