private void updateDataDogTransport(ReportingContext context) throws IOException {
String dataDogTransport = context.getProperty(DATADOG_TRANSPORT).getValue();
if (dataDogTransport.equalsIgnoreCase(DATADOG_AGENT.getValue())) {
ddMetricRegistryBuilder.build("agent");
} else if (dataDogTransport.equalsIgnoreCase(DATADOG_HTTP.getValue())
&& context.getProperty(API_KEY).isSet()) {
ddMetricRegistryBuilder.build(context.getProperty(API_KEY).getValue());
}
}
@Override
public ValidationResult setProperty(
final ControllerService service,
final PropertyDescriptor property,
final AllowableValue value) {
return setProperty(service, property, value.getValue());
}
@Override
public ValidationResult setProperty(
final PropertyDescriptor descriptor, final AllowableValue value) {
return context.setProperty(descriptor, value.getValue());
}
@EventDriven
@SupportsBatching
@Tags({"map", "cache", "put", "distributed"})
@InputRequirement(Requirement.INPUT_REQUIRED)
@CapabilityDescription(
"Gets the content of a FlowFile and puts it to a distributed map cache, using a cache key "
+ "computed from FlowFile attributes. If the cache already contains the entry and the cache update strategy is "
+ "'keep original' the entry is not replaced.'")
@WritesAttribute(
attribute = "cached",
description =
"All FlowFiles will have an attribute 'cached'. The value of this "
+ "attribute is true, is the FlowFile is cached, otherwise false.")
@SeeAlso(
classNames = {
"org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService",
"org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer",
"org.apache.nifi.processors.standard.FetchDistributedMapCache"
})
public class PutDistributedMapCache extends AbstractProcessor {
public static final String CACHED_ATTRIBUTE_NAME = "cached";
// Identifies the distributed map cache client
public static final PropertyDescriptor DISTRIBUTED_CACHE_SERVICE =
new PropertyDescriptor.Builder()
.name("Distributed Cache Service")
.description("The Controller Service that is used to cache flow files")
.required(true)
.identifiesControllerService(DistributedMapCacheClient.class)
.build();
// Selects the FlowFile attribute, whose value is used as cache key
public static final PropertyDescriptor CACHE_ENTRY_IDENTIFIER =
new PropertyDescriptor.Builder()
.name("Cache Entry Identifier")
.description(
"A FlowFile attribute, or the results of an Attribute Expression Language statement, which will "
+ "be evaluated against a FlowFile in order to determine the cache key")
.required(true)
.addValidator(
StandardValidators.createAttributeExpressionLanguageValidator(
ResultType.STRING, true))
.expressionLanguageSupported(true)
.build();
public static final AllowableValue CACHE_UPDATE_REPLACE =
new AllowableValue(
"replace",
"Replace if present",
"Adds the specified entry to the cache, replacing any value that is currently set.");
public static final AllowableValue CACHE_UPDATE_KEEP_ORIGINAL =
new AllowableValue(
"keeporiginal",
"Keep original",
"Adds the specified entry to the cache, if the key does not exist.");
public static final PropertyDescriptor CACHE_UPDATE_STRATEGY =
new PropertyDescriptor.Builder()
.name("Cache update strategy")
.description(
"Determines how the cache is updated if the cache already contains the entry")
.required(true)
.allowableValues(CACHE_UPDATE_REPLACE, CACHE_UPDATE_KEEP_ORIGINAL)
.defaultValue(CACHE_UPDATE_REPLACE.getValue())
.build();
public static final PropertyDescriptor CACHE_ENTRY_MAX_BYTES =
new PropertyDescriptor.Builder()
.name("Max cache entry size")
.description("The maximum amount of data to put into cache")
.required(false)
.addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
.defaultValue("1 MB")
.expressionLanguageSupported(false)
.build();
public static final Relationship REL_SUCCESS =
new Relationship.Builder()
.name("success")
.description(
"Any FlowFile that is successfully inserted into cache will be routed to this relationship")
.build();
public static final Relationship REL_FAILURE =
new Relationship.Builder()
.name("failure")
.description(
"Any FlowFile that cannot be inserted into the cache will be routed to this relationship")
.build();
private final Set<Relationship> relationships;
private final Serializer<String> keySerializer = new StringSerializer();
private final Serializer<byte[]> valueSerializer = new CacheValueSerializer();
private final Deserializer<byte[]> valueDeserializer = new CacheValueDeserializer();
public PutDistributedMapCache() {
final Set<Relationship> rels = new HashSet<>();
rels.add(REL_SUCCESS);
rels.add(REL_FAILURE);
relationships = Collections.unmodifiableSet(rels);
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
final List<PropertyDescriptor> descriptors = new ArrayList<>();
descriptors.add(CACHE_ENTRY_IDENTIFIER);
descriptors.add(DISTRIBUTED_CACHE_SERVICE);
descriptors.add(CACHE_UPDATE_STRATEGY);
descriptors.add(CACHE_ENTRY_MAX_BYTES);
return descriptors;
}
@Override
public Set<Relationship> getRelationships() {
return relationships;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session)
throws ProcessException {
FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
final ProcessorLog logger = getLogger();
// cache key is computed from attribute 'CACHE_ENTRY_IDENTIFIER' with expression language
// support
final String cacheKey =
context
.getProperty(CACHE_ENTRY_IDENTIFIER)
.evaluateAttributeExpressions(flowFile)
.getValue();
// if the computed value is null, or empty, we transfer the flow file to failure relationship
if (StringUtils.isBlank(cacheKey)) {
logger.error(
"FlowFile {} has no attribute for given Cache Entry Identifier", new Object[] {flowFile});
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
return;
}
// the cache client used to interact with the distributed cache
final DistributedMapCacheClient cache =
context
.getProperty(DISTRIBUTED_CACHE_SERVICE)
.asControllerService(DistributedMapCacheClient.class);
try {
final long maxCacheEntrySize =
context.getProperty(CACHE_ENTRY_MAX_BYTES).asDataSize(DataUnit.B).longValue();
long flowFileSize = flowFile.getSize();
// too big flow file
if (flowFileSize > maxCacheEntrySize) {
logger.warn(
"Flow file {} size {} exceeds the max cache entry size ({} B).",
new Object[] {flowFile, flowFileSize, maxCacheEntrySize});
session.transfer(flowFile, REL_FAILURE);
return;
}
if (flowFileSize == 0) {
logger.warn("Flow file {} is empty, there is nothing to cache.", new Object[] {flowFile});
session.transfer(flowFile, REL_FAILURE);
return;
}
// get flow file content
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
session.exportTo(flowFile, byteStream);
byte[] cacheValue = byteStream.toByteArray();
final String updateStrategy = context.getProperty(CACHE_UPDATE_STRATEGY).getValue();
boolean cached = false;
if (updateStrategy.equals(CACHE_UPDATE_REPLACE.getValue())) {
cache.put(cacheKey, cacheValue, keySerializer, valueSerializer);
cached = true;
} else if (updateStrategy.equals(CACHE_UPDATE_KEEP_ORIGINAL.getValue())) {
final byte[] oldValue =
cache.getAndPutIfAbsent(
cacheKey, cacheValue, keySerializer, valueSerializer, valueDeserializer);
if (oldValue == null) {
cached = true;
}
}
// set 'cached' attribute
flowFile = session.putAttribute(flowFile, CACHED_ATTRIBUTE_NAME, String.valueOf(cached));
if (cached) {
session.transfer(flowFile, REL_SUCCESS);
} else {
session.transfer(flowFile, REL_FAILURE);
}
} catch (final IOException e) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
logger.error(
"Unable to communicate with cache when processing {} due to {}",
new Object[] {flowFile, e});
}
}
public static class CacheValueSerializer implements Serializer<byte[]> {
@Override
public void serialize(final byte[] bytes, final OutputStream out)
throws SerializationException, IOException {
out.write(bytes);
}
}
public static class CacheValueDeserializer implements Deserializer<byte[]> {
@Override
public byte[] deserialize(final byte[] input) throws DeserializationException, IOException {
if (input == null || input.length == 0) {
return null;
}
return input;
}
}
/** Simple string serializer, used for serializing the cache key */
public static class StringSerializer implements Serializer<String> {
@Override
public void serialize(final String value, final OutputStream out)
throws SerializationException, IOException {
out.write(value.getBytes(StandardCharsets.UTF_8));
}
}
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session)
throws ProcessException {
FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
final ProcessorLog logger = getLogger();
// cache key is computed from attribute 'CACHE_ENTRY_IDENTIFIER' with expression language
// support
final String cacheKey =
context
.getProperty(CACHE_ENTRY_IDENTIFIER)
.evaluateAttributeExpressions(flowFile)
.getValue();
// if the computed value is null, or empty, we transfer the flow file to failure relationship
if (StringUtils.isBlank(cacheKey)) {
logger.error(
"FlowFile {} has no attribute for given Cache Entry Identifier", new Object[] {flowFile});
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
return;
}
// the cache client used to interact with the distributed cache
final DistributedMapCacheClient cache =
context
.getProperty(DISTRIBUTED_CACHE_SERVICE)
.asControllerService(DistributedMapCacheClient.class);
try {
final long maxCacheEntrySize =
context.getProperty(CACHE_ENTRY_MAX_BYTES).asDataSize(DataUnit.B).longValue();
long flowFileSize = flowFile.getSize();
// too big flow file
if (flowFileSize > maxCacheEntrySize) {
logger.warn(
"Flow file {} size {} exceeds the max cache entry size ({} B).",
new Object[] {flowFile, flowFileSize, maxCacheEntrySize});
session.transfer(flowFile, REL_FAILURE);
return;
}
if (flowFileSize == 0) {
logger.warn("Flow file {} is empty, there is nothing to cache.", new Object[] {flowFile});
session.transfer(flowFile, REL_FAILURE);
return;
}
// get flow file content
final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
session.exportTo(flowFile, byteStream);
byte[] cacheValue = byteStream.toByteArray();
final String updateStrategy = context.getProperty(CACHE_UPDATE_STRATEGY).getValue();
boolean cached = false;
if (updateStrategy.equals(CACHE_UPDATE_REPLACE.getValue())) {
cache.put(cacheKey, cacheValue, keySerializer, valueSerializer);
cached = true;
} else if (updateStrategy.equals(CACHE_UPDATE_KEEP_ORIGINAL.getValue())) {
final byte[] oldValue =
cache.getAndPutIfAbsent(
cacheKey, cacheValue, keySerializer, valueSerializer, valueDeserializer);
if (oldValue == null) {
cached = true;
}
}
// set 'cached' attribute
flowFile = session.putAttribute(flowFile, CACHED_ATTRIBUTE_NAME, String.valueOf(cached));
if (cached) {
session.transfer(flowFile, REL_SUCCESS);
} else {
session.transfer(flowFile, REL_FAILURE);
}
} catch (final IOException e) {
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
logger.error(
"Unable to communicate with cache when processing {} due to {}",
new Object[] {flowFile, e});
}
}
@Tags({"reporting", "datadog", "metrics"})
@CapabilityDescription(
"Publishes metrics from NiFi to datadog. For accurate and informative reporting, components should have unique names.")
public class DataDogReportingTask extends AbstractReportingTask {
static final AllowableValue DATADOG_AGENT =
new AllowableValue(
"Datadog Agent",
"Datadog Agent",
"Metrics will be sent via locally installed Datadog agent. "
+ "Datadog agent needs to be installed manually before using this option");
static final AllowableValue DATADOG_HTTP =
new AllowableValue(
"Datadog HTTP",
"Datadog HTTP",
"Metrics will be sent via HTTP transport with no need of Agent installed. "
+ "Datadog API key needs to be set");
static final PropertyDescriptor DATADOG_TRANSPORT =
new PropertyDescriptor.Builder()
.name("Datadog transport")
.description("Transport through which metrics will be sent to Datadog")
.required(true)
.allowableValues(DATADOG_AGENT, DATADOG_HTTP)
.defaultValue(DATADOG_HTTP.getValue())
.build();
static final PropertyDescriptor API_KEY =
new PropertyDescriptor.Builder()
.name("API key")
.description(
"Datadog API key. If specified value is 'agent', local Datadog agent will be used.")
.expressionLanguageSupported(false)
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
static final PropertyDescriptor METRICS_PREFIX =
new PropertyDescriptor.Builder()
.name("Metrics prefix")
.description("Prefix to be added before every metric")
.required(true)
.expressionLanguageSupported(true)
.defaultValue("nifi")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
static final PropertyDescriptor ENVIRONMENT =
new PropertyDescriptor.Builder()
.name("Environment")
.description(
"Environment, dataflow is running in. "
+ "This property will be included as metrics tag.")
.required(true)
.expressionLanguageSupported(true)
.defaultValue("dev")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
private MetricsService metricsService;
private DDMetricRegistryBuilder ddMetricRegistryBuilder;
private MetricRegistry metricRegistry;
private String metricsPrefix;
private String environment;
private String statusId;
private ConcurrentHashMap<String, AtomicDouble> metricsMap;
private Map<String, String> defaultTags;
private volatile VirtualMachineMetrics virtualMachineMetrics;
private Logger logger = LoggerFactory.getLogger(getClass().getName());
@OnScheduled
public void setup(final ConfigurationContext context) {
metricsService = getMetricsService();
ddMetricRegistryBuilder = getMetricRegistryBuilder();
metricRegistry = getMetricRegistry();
metricsMap = getMetricsMap();
metricsPrefix = METRICS_PREFIX.getDefaultValue();
environment = ENVIRONMENT.getDefaultValue();
virtualMachineMetrics = VirtualMachineMetrics.getInstance();
ddMetricRegistryBuilder
.setMetricRegistry(metricRegistry)
.setTags(metricsService.getAllTagsList());
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(METRICS_PREFIX);
properties.add(ENVIRONMENT);
properties.add(API_KEY);
properties.add(DATADOG_TRANSPORT);
return properties;
}
@Override
public void onTrigger(ReportingContext context) {
final ProcessGroupStatus status = context.getEventAccess().getControllerStatus();
metricsPrefix = context.getProperty(METRICS_PREFIX).evaluateAttributeExpressions().getValue();
environment = context.getProperty(ENVIRONMENT).evaluateAttributeExpressions().getValue();
statusId = status.getId();
defaultTags = ImmutableMap.of("env", environment, "dataflow_id", statusId);
try {
updateDataDogTransport(context);
} catch (IOException e) {
e.printStackTrace();
}
updateAllMetricGroups(status);
ddMetricRegistryBuilder.getDatadogReporter().report();
}
protected void updateMetrics(
Map<String, Double> metrics, Optional<String> processorName, Map<String, String> tags) {
for (Map.Entry<String, Double> entry : metrics.entrySet()) {
final String metricName = buildMetricName(processorName, entry.getKey());
logger.debug(metricName + ": " + entry.getValue());
// if metric is not registered yet - register it
if (!metricsMap.containsKey(metricName)) {
metricsMap.put(metricName, new AtomicDouble(entry.getValue()));
metricRegistry.register(metricName, new MetricGauge(metricName, tags));
}
// set real time value to metrics map
metricsMap.get(metricName).set(entry.getValue());
}
}
private void updateAllMetricGroups(ProcessGroupStatus processGroupStatus) {
final List<ProcessorStatus> processorStatuses = new ArrayList<>();
populateProcessorStatuses(processGroupStatus, processorStatuses);
for (final ProcessorStatus processorStatus : processorStatuses) {
updateMetrics(
metricsService.getProcessorMetrics(processorStatus),
Optional.of(processorStatus.getName()),
defaultTags);
}
final List<ConnectionStatus> connectionStatuses = new ArrayList<>();
populateConnectionStatuses(processGroupStatus, connectionStatuses);
for (ConnectionStatus connectionStatus : connectionStatuses) {
Map<String, String> connectionStatusTags = new HashMap<>(defaultTags);
connectionStatusTags.putAll(metricsService.getConnectionStatusTags(connectionStatus));
updateMetrics(
metricsService.getConnectionStatusMetrics(connectionStatus),
Optional.<String>absent(),
connectionStatusTags);
}
final List<PortStatus> inputPortStatuses = new ArrayList<>();
populateInputPortStatuses(processGroupStatus, inputPortStatuses);
for (PortStatus portStatus : inputPortStatuses) {
Map<String, String> portTags = new HashMap<>(defaultTags);
portTags.putAll(metricsService.getPortStatusTags(portStatus));
updateMetrics(
metricsService.getPortStatusMetrics(portStatus), Optional.<String>absent(), portTags);
}
final List<PortStatus> outputPortStatuses = new ArrayList<>();
populateOutputPortStatuses(processGroupStatus, outputPortStatuses);
for (PortStatus portStatus : outputPortStatuses) {
Map<String, String> portTags = new HashMap<>(defaultTags);
portTags.putAll(metricsService.getPortStatusTags(portStatus));
updateMetrics(
metricsService.getPortStatusMetrics(portStatus), Optional.<String>absent(), portTags);
}
updateMetrics(
metricsService.getJVMMetrics(virtualMachineMetrics),
Optional.<String>absent(),
defaultTags);
updateMetrics(
metricsService.getDataFlowMetrics(processGroupStatus),
Optional.<String>absent(),
defaultTags);
}
private class MetricGauge implements Gauge, DynamicTagsCallback {
private Map<String, String> tags;
private String metricName;
public MetricGauge(String metricName, Map<String, String> tagsMap) {
this.tags = tagsMap;
this.metricName = metricName;
}
@Override
public Object getValue() {
return metricsMap.get(metricName).get();
}
@Override
public List<String> getTags() {
List<String> tagsList = Lists.newArrayList();
for (Map.Entry<String, String> entry : tags.entrySet()) {
tagsList.add(entry.getKey() + ":" + entry.getValue());
}
return tagsList;
}
}
private void updateDataDogTransport(ReportingContext context) throws IOException {
String dataDogTransport = context.getProperty(DATADOG_TRANSPORT).getValue();
if (dataDogTransport.equalsIgnoreCase(DATADOG_AGENT.getValue())) {
ddMetricRegistryBuilder.build("agent");
} else if (dataDogTransport.equalsIgnoreCase(DATADOG_HTTP.getValue())
&& context.getProperty(API_KEY).isSet()) {
ddMetricRegistryBuilder.build(context.getProperty(API_KEY).getValue());
}
}
private void populateProcessorStatuses(
final ProcessGroupStatus groupStatus, final List<ProcessorStatus> statuses) {
statuses.addAll(groupStatus.getProcessorStatus());
for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) {
populateProcessorStatuses(childGroupStatus, statuses);
}
}
private void populateConnectionStatuses(
final ProcessGroupStatus groupStatus, final List<ConnectionStatus> statuses) {
statuses.addAll(groupStatus.getConnectionStatus());
for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) {
populateConnectionStatuses(childGroupStatus, statuses);
}
}
private void populateInputPortStatuses(
final ProcessGroupStatus groupStatus, final List<PortStatus> statuses) {
statuses.addAll(groupStatus.getInputPortStatus());
for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) {
populateInputPortStatuses(childGroupStatus, statuses);
}
}
private void populateOutputPortStatuses(
final ProcessGroupStatus groupStatus, final List<PortStatus> statuses) {
statuses.addAll(groupStatus.getOutputPortStatus());
for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) {
populateOutputPortStatuses(childGroupStatus, statuses);
}
}
private String buildMetricName(Optional<String> processorName, String metricName) {
return metricsPrefix + "." + processorName.or("flow") + "." + metricName;
}
protected MetricsService getMetricsService() {
return new MetricsService();
}
protected DDMetricRegistryBuilder getMetricRegistryBuilder() {
return new DDMetricRegistryBuilder();
}
protected MetricRegistry getMetricRegistry() {
return new MetricRegistry();
}
protected ConcurrentHashMap<String, AtomicDouble> getMetricsMap() {
return new ConcurrentHashMap<>();
}
}
@InputRequirement(Requirement.INPUT_FORBIDDEN)
@Tags({"http", "https", "request", "listen", "ingress", "web service"})
@CapabilityDescription(
"Starts an HTTP Server and listens for HTTP Requests. For each request, creates a FlowFile and transfers to 'success'. "
+ "This Processor is designed to be used in conjunction with the HandleHttpResponse Processor in order to create a Web Service")
@WritesAttributes({
@WritesAttribute(
attribute = HTTPUtils.HTTP_CONTEXT_ID,
description =
"An identifier that allows the HandleHttpRequest and HandleHttpResponse "
+ "to coordinate which FlowFile belongs to which HTTP Request/Response."),
@WritesAttribute(
attribute = "mime.type",
description = "The MIME Type of the data, according to the HTTP Header \"Content-Type\""),
@WritesAttribute(
attribute = "http.servlet.path",
description = "The part of the request URL that is considered the Servlet Path"),
@WritesAttribute(
attribute = "http.context.path",
description = "The part of the request URL that is considered to be the Context Path"),
@WritesAttribute(
attribute = "http.method",
description = "The HTTP Method that was used for the request, such as GET or POST"),
@WritesAttribute(
attribute = HTTPUtils.HTTP_LOCAL_NAME,
description = "IP address/hostname of the server"),
@WritesAttribute(attribute = HTTPUtils.HTTP_PORT, description = "Listening port of the server"),
@WritesAttribute(
attribute = "http.query.string",
description = "The query string portion of hte Request URL"),
@WritesAttribute(
attribute = HTTPUtils.HTTP_REMOTE_HOST,
description = "The hostname of the requestor"),
@WritesAttribute(
attribute = "http.remote.addr",
description = "The hostname:port combination of the requestor"),
@WritesAttribute(attribute = "http.remote.user", description = "The username of the requestor"),
@WritesAttribute(attribute = HTTPUtils.HTTP_REQUEST_URI, description = "The full Request URL"),
@WritesAttribute(
attribute = "http.auth.type",
description = "The type of HTTP Authorization used"),
@WritesAttribute(
attribute = "http.principal.name",
description = "The name of the authenticated user making the request"),
@WritesAttribute(
attribute = HTTPUtils.HTTP_SSL_CERT,
description =
"The Distinguished Name of the requestor. This value will not be populated "
+ "unless the Processor is configured to use an SSLContext Service"),
@WritesAttribute(
attribute = "http.issuer.dn",
description =
"The Distinguished Name of the entity that issued the Subject's certificate. "
+ "This value will not be populated unless the Processor is configured to use an SSLContext Service"),
@WritesAttribute(
attribute = "http.headers.XXX",
description =
"Each of the HTTP Headers that is received in the request will be added as an "
+ "attribute, prefixed with \"http.headers.\" For example, if the request contains an HTTP Header named \"x-my-header\", then the value "
+ "will be added to an attribute named \"http.headers.x-my-header\"")
})
@SeeAlso(
value = {HandleHttpResponse.class},
classNames = {
"org.apache.nifi.http.StandardHttpContextMap",
"org.apache.nifi.ssl.StandardSSLContextService"
})
public class HandleHttpRequest extends AbstractProcessor {
private static final Pattern URL_QUERY_PARAM_DELIMITER = Pattern.compile("&");
// Allowable values for client auth
public static final AllowableValue CLIENT_NONE =
new AllowableValue(
"No Authentication",
"No Authentication",
"Processor will not authenticate clients. Anyone can communicate with this Processor anonymously");
public static final AllowableValue CLIENT_WANT =
new AllowableValue(
"Want Authentication",
"Want Authentication",
"Processor will try to verify the client but if unable to verify will allow the client to communicate anonymously");
public static final AllowableValue CLIENT_NEED =
new AllowableValue(
"Need Authentication",
"Need Authentication",
"Processor will reject communications from any client unless the client provides a certificate that is trusted by the TrustStore"
+ "specified in the SSL Context Service");
public static final PropertyDescriptor PORT =
new PropertyDescriptor.Builder()
.name("Listening Port")
.description("The Port to listen on for incoming HTTP requests")
.required(true)
.addValidator(StandardValidators.createLongValidator(0L, 65535L, true))
.expressionLanguageSupported(false)
.defaultValue("80")
.build();
public static final PropertyDescriptor HOSTNAME =
new PropertyDescriptor.Builder()
.name("Hostname")
.description("The Hostname to bind to. If not specified, will bind to all hosts")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(false)
.build();
public static final PropertyDescriptor HTTP_CONTEXT_MAP =
new PropertyDescriptor.Builder()
.name("HTTP Context Map")
.description(
"The HTTP Context Map Controller Service to use for caching the HTTP Request Information")
.required(true)
.identifiesControllerService(HttpContextMap.class)
.build();
public static final PropertyDescriptor SSL_CONTEXT =
new PropertyDescriptor.Builder()
.name("SSL Context Service")
.description(
"The SSL Context Service to use in order to secure the server. If specified, the server will accept only HTTPS requests; "
+ "otherwise, the server will accept only HTTP requests")
.required(false)
.identifiesControllerService(SSLContextService.class)
.build();
public static final PropertyDescriptor URL_CHARACTER_SET =
new PropertyDescriptor.Builder()
.name("Default URL Character Set")
.description(
"The character set to use for decoding URL parameters if the HTTP Request does not supply one")
.required(true)
.defaultValue("UTF-8")
.addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
.build();
public static final PropertyDescriptor PATH_REGEX =
new PropertyDescriptor.Builder()
.name("Allowed Paths")
.description(
"A Regular Expression that specifies the valid HTTP Paths that are allowed in the incoming URL Requests. If this value is "
+ "specified and the path of the HTTP Requests does not match this Regular Expression, the Processor will respond with a "
+ "404: NotFound")
.required(false)
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
.expressionLanguageSupported(false)
.build();
public static final PropertyDescriptor ALLOW_GET =
new PropertyDescriptor.Builder()
.name("Allow GET")
.description("Allow HTTP GET Method")
.required(true)
.allowableValues("true", "false")
.defaultValue("true")
.build();
public static final PropertyDescriptor ALLOW_POST =
new PropertyDescriptor.Builder()
.name("Allow POST")
.description("Allow HTTP POST Method")
.required(true)
.allowableValues("true", "false")
.defaultValue("true")
.build();
public static final PropertyDescriptor ALLOW_PUT =
new PropertyDescriptor.Builder()
.name("Allow PUT")
.description("Allow HTTP PUT Method")
.required(true)
.allowableValues("true", "false")
.defaultValue("true")
.build();
public static final PropertyDescriptor ALLOW_DELETE =
new PropertyDescriptor.Builder()
.name("Allow DELETE")
.description("Allow HTTP DELETE Method")
.required(true)
.allowableValues("true", "false")
.defaultValue("true")
.build();
public static final PropertyDescriptor ALLOW_HEAD =
new PropertyDescriptor.Builder()
.name("Allow HEAD")
.description("Allow HTTP HEAD Method")
.required(true)
.allowableValues("true", "false")
.defaultValue("false")
.build();
public static final PropertyDescriptor ALLOW_OPTIONS =
new PropertyDescriptor.Builder()
.name("Allow OPTIONS")
.description("Allow HTTP OPTIONS Method")
.required(true)
.allowableValues("true", "false")
.defaultValue("false")
.build();
public static final PropertyDescriptor ADDITIONAL_METHODS =
new PropertyDescriptor.Builder()
.name("Additional HTTP Methods")
.description("A comma-separated list of non-standard HTTP Methods that should be allowed")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(false)
.build();
public static final PropertyDescriptor CLIENT_AUTH =
new PropertyDescriptor.Builder()
.name("Client Authentication")
.description(
"Specifies whether or not the Processor should authenticate clients. This value is ignored if the <SSL Context Service> "
+ "Property is not specified or the SSL Context provided uses only a KeyStore and not a TrustStore.")
.required(true)
.allowableValues(CLIENT_NONE, CLIENT_WANT, CLIENT_NEED)
.defaultValue(CLIENT_NONE.getValue())
.build();
public static final PropertyDescriptor CONTAINER_QUEUE_SIZE =
new PropertyDescriptor.Builder()
.name("container-queue-size")
.displayName("Container Queue Size")
.description("The size of the queue for Http Request Containers")
.required(true)
.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
.defaultValue("50")
.build();
public static final Relationship REL_SUCCESS =
new Relationship.Builder()
.name("success")
.description("All content that is received is routed to the 'success' relationship")
.build();
private static final List<PropertyDescriptor> propertyDescriptors;
static {
List<PropertyDescriptor> descriptors = new ArrayList<>();
descriptors.add(PORT);
descriptors.add(HOSTNAME);
descriptors.add(SSL_CONTEXT);
descriptors.add(HTTP_CONTEXT_MAP);
descriptors.add(PATH_REGEX);
descriptors.add(URL_CHARACTER_SET);
descriptors.add(ALLOW_GET);
descriptors.add(ALLOW_POST);
descriptors.add(ALLOW_PUT);
descriptors.add(ALLOW_DELETE);
descriptors.add(ALLOW_HEAD);
descriptors.add(ALLOW_OPTIONS);
descriptors.add(ADDITIONAL_METHODS);
descriptors.add(CLIENT_AUTH);
descriptors.add(CONTAINER_QUEUE_SIZE);
propertyDescriptors = Collections.unmodifiableList(descriptors);
}
private volatile Server server;
private AtomicBoolean initialized = new AtomicBoolean(false);
private volatile BlockingQueue<HttpRequestContainer> containerQueue;
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return propertyDescriptors;
}
@Override
public Set<Relationship> getRelationships() {
return Collections.singleton(REL_SUCCESS);
}
@OnScheduled
public void clearInit() {
initialized.set(false);
}
private synchronized void initializeServer(final ProcessContext context) throws Exception {
if (initialized.get()) {
return;
}
this.containerQueue =
new LinkedBlockingQueue<>(context.getProperty(CONTAINER_QUEUE_SIZE).asInteger());
final String host = context.getProperty(HOSTNAME).getValue();
final int port = context.getProperty(PORT).asInteger();
final SSLContextService sslService =
context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class);
final String clientAuthValue = context.getProperty(CLIENT_AUTH).getValue();
final boolean need;
final boolean want;
if (CLIENT_NEED.equals(clientAuthValue)) {
need = true;
want = false;
} else if (CLIENT_WANT.equals(clientAuthValue)) {
need = false;
want = true;
} else {
need = false;
want = false;
}
final SslContextFactory sslFactory =
(sslService == null) ? null : createSslFactory(sslService, need, want);
final Server server = new Server(port);
// create the http configuration
final HttpConfiguration httpConfiguration = new HttpConfiguration();
if (sslFactory == null) {
// create the connector
final ServerConnector http =
new ServerConnector(server, new HttpConnectionFactory(httpConfiguration));
// set host and port
if (StringUtils.isNotBlank(host)) {
http.setHost(host);
}
http.setPort(port);
// add this connector
server.setConnectors(new Connector[] {http});
} else {
// add some secure config
final HttpConfiguration httpsConfiguration = new HttpConfiguration(httpConfiguration);
httpsConfiguration.setSecureScheme("https");
httpsConfiguration.setSecurePort(port);
httpsConfiguration.addCustomizer(new SecureRequestCustomizer());
// build the connector
final ServerConnector https =
new ServerConnector(
server,
new SslConnectionFactory(sslFactory, "http/1.1"),
new HttpConnectionFactory(httpsConfiguration));
// set host and port
if (StringUtils.isNotBlank(host)) {
https.setHost(host);
}
https.setPort(port);
// add this connector
server.setConnectors(new Connector[] {https});
}
final Set<String> allowedMethods = new HashSet<>();
if (context.getProperty(ALLOW_GET).asBoolean()) {
allowedMethods.add("GET");
}
if (context.getProperty(ALLOW_POST).asBoolean()) {
allowedMethods.add("POST");
}
if (context.getProperty(ALLOW_PUT).asBoolean()) {
allowedMethods.add("PUT");
}
if (context.getProperty(ALLOW_DELETE).asBoolean()) {
allowedMethods.add("DELETE");
}
if (context.getProperty(ALLOW_HEAD).asBoolean()) {
allowedMethods.add("HEAD");
}
if (context.getProperty(ALLOW_OPTIONS).asBoolean()) {
allowedMethods.add("OPTIONS");
}
final String additionalMethods = context.getProperty(ADDITIONAL_METHODS).getValue();
if (additionalMethods != null) {
for (final String additionalMethod : additionalMethods.split(",")) {
final String trimmed = additionalMethod.trim();
if (!trimmed.isEmpty()) {
allowedMethods.add(trimmed.toUpperCase());
}
}
}
final String pathRegex = context.getProperty(PATH_REGEX).getValue();
final Pattern pathPattern = (pathRegex == null) ? null : Pattern.compile(pathRegex);
server.setHandler(
new AbstractHandler() {
@Override
public void handle(
final String target,
final Request baseRequest,
final HttpServletRequest request,
final HttpServletResponse response)
throws IOException, ServletException {
final String requestUri = request.getRequestURI();
if (!allowedMethods.contains(request.getMethod().toUpperCase())) {
getLogger()
.info(
"Sending back METHOD_NOT_ALLOWED response to {}; method was {}; request URI was {}",
new Object[] {request.getRemoteAddr(), request.getMethod(), requestUri});
response.sendError(Status.METHOD_NOT_ALLOWED.getStatusCode());
return;
}
if (pathPattern != null) {
final URI uri;
try {
uri = new URI(requestUri);
} catch (final URISyntaxException e) {
throw new ServletException(e);
}
if (!pathPattern.matcher(uri.getPath()).matches()) {
response.sendError(Status.NOT_FOUND.getStatusCode());
getLogger()
.info(
"Sending back NOT_FOUND response to {}; request was {} {}",
new Object[] {request.getRemoteAddr(), request.getMethod(), requestUri});
return;
}
}
// If destination queues full, send back a 503: Service Unavailable.
if (context.getAvailableRelationships().isEmpty()) {
response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
return;
}
// Right now, that information, though, is only in the ProcessSession, not the
// ProcessContext,
// so it is not known to us. Should see if it can be added to the ProcessContext.
final AsyncContext async = baseRequest.startAsync();
async.setTimeout(Long.MAX_VALUE); // timeout is handled by HttpContextMap
final boolean added =
containerQueue.offer(new HttpRequestContainer(request, response, async));
if (added) {
getLogger()
.debug(
"Added Http Request to queue for {} {} from {}",
new Object[] {request.getMethod(), requestUri, request.getRemoteAddr()});
} else {
getLogger()
.info(
"Sending back a SERVICE_UNAVAILABLE response to {}; request was {} {}",
new Object[] {
request.getRemoteAddr(), request.getMethod(), request.getRemoteAddr()
});
response.sendError(Status.SERVICE_UNAVAILABLE.getStatusCode());
response.flushBuffer();
async.complete();
}
}
});
this.server = server;
server.start();
getLogger().info("Server started and listening on port " + getPort());
initialized.set(true);
}
protected int getPort() {
for (final Connector connector : server.getConnectors()) {
if (connector instanceof ServerConnector) {
return ((ServerConnector) connector).getLocalPort();
}
}
throw new IllegalStateException("Server is not listening on any ports");
}
protected int getRequestQueueSize() {
return containerQueue.size();
}
private SslContextFactory createSslFactory(
final SSLContextService sslService,
final boolean needClientAuth,
final boolean wantClientAuth) {
final SslContextFactory sslFactory = new SslContextFactory();
sslFactory.setNeedClientAuth(needClientAuth);
sslFactory.setWantClientAuth(wantClientAuth);
if (sslService.isKeyStoreConfigured()) {
sslFactory.setKeyStorePath(sslService.getKeyStoreFile());
sslFactory.setKeyStorePassword(sslService.getKeyStorePassword());
sslFactory.setKeyStoreType(sslService.getKeyStoreType());
}
if (sslService.isTrustStoreConfigured()) {
sslFactory.setTrustStorePath(sslService.getTrustStoreFile());
sslFactory.setTrustStorePassword(sslService.getTrustStorePassword());
sslFactory.setTrustStoreType(sslService.getTrustStoreType());
}
return sslFactory;
}
@OnStopped
public void shutdown() throws Exception {
if (server != null) {
getLogger().debug("Shutting down server");
server.stop();
server.destroy();
server.join();
getLogger().info("Shut down {}", new Object[] {server});
}
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session)
throws ProcessException {
try {
if (!initialized.get()) {
initializeServer(context);
}
} catch (Exception e) {
context.yield();
throw new ProcessException("Failed to initialize the server", e);
}
final HttpRequestContainer container = containerQueue.poll();
if (container == null) {
return;
}
final long start = System.nanoTime();
final HttpServletRequest request = container.getRequest();
FlowFile flowFile = session.create();
try {
flowFile = session.importFrom(request.getInputStream(), flowFile);
} catch (final IOException e) {
getLogger()
.error(
"Failed to receive content from HTTP Request from {} due to {}",
new Object[] {request.getRemoteAddr(), e});
session.remove(flowFile);
return;
}
final String charset =
request.getCharacterEncoding() == null
? context.getProperty(URL_CHARACTER_SET).getValue()
: request.getCharacterEncoding();
final String contextIdentifier = UUID.randomUUID().toString();
final Map<String, String> attributes = new HashMap<>();
try {
putAttribute(attributes, HTTPUtils.HTTP_CONTEXT_ID, contextIdentifier);
putAttribute(attributes, "mime.type", request.getContentType());
putAttribute(attributes, "http.servlet.path", request.getServletPath());
putAttribute(attributes, "http.context.path", request.getContextPath());
putAttribute(attributes, "http.method", request.getMethod());
putAttribute(attributes, "http.local.addr", request.getLocalAddr());
putAttribute(attributes, HTTPUtils.HTTP_LOCAL_NAME, request.getLocalName());
if (request.getQueryString() != null) {
putAttribute(
attributes, "http.query.string", URLDecoder.decode(request.getQueryString(), charset));
}
putAttribute(attributes, HTTPUtils.HTTP_REMOTE_HOST, request.getRemoteHost());
putAttribute(attributes, "http.remote.addr", request.getRemoteAddr());
putAttribute(attributes, "http.remote.user", request.getRemoteUser());
putAttribute(attributes, HTTPUtils.HTTP_REQUEST_URI, request.getRequestURI());
putAttribute(attributes, "http.request.url", request.getRequestURL().toString());
putAttribute(attributes, "http.auth.type", request.getAuthType());
putAttribute(attributes, "http.requested.session.id", request.getRequestedSessionId());
if (request.getDispatcherType() != null) {
putAttribute(attributes, "http.dispatcher.type", request.getDispatcherType().name());
}
putAttribute(attributes, "http.character.encoding", request.getCharacterEncoding());
putAttribute(attributes, "http.locale", request.getLocale());
putAttribute(attributes, "http.server.name", request.getServerName());
putAttribute(attributes, HTTPUtils.HTTP_PORT, request.getServerPort());
final Enumeration<String> paramEnumeration = request.getParameterNames();
while (paramEnumeration.hasMoreElements()) {
final String paramName = paramEnumeration.nextElement();
final String value = request.getParameter(paramName);
attributes.put("http.param." + paramName, value);
}
final Cookie[] cookies = request.getCookies();
if (cookies != null) {
for (final Cookie cookie : cookies) {
final String name = cookie.getName();
final String cookiePrefix = "http.cookie." + name + ".";
attributes.put(cookiePrefix + "value", cookie.getValue());
attributes.put(cookiePrefix + "domain", cookie.getDomain());
attributes.put(cookiePrefix + "path", cookie.getPath());
attributes.put(cookiePrefix + "max.age", String.valueOf(cookie.getMaxAge()));
attributes.put(cookiePrefix + "version", String.valueOf(cookie.getVersion()));
attributes.put(cookiePrefix + "secure", String.valueOf(cookie.getSecure()));
}
}
final String queryString = request.getQueryString();
if (queryString != null) {
final String[] params = URL_QUERY_PARAM_DELIMITER.split(queryString);
for (final String keyValueString : params) {
final int indexOf = keyValueString.indexOf("=");
if (indexOf < 0) {
// no =, then it's just a key with no value
attributes.put("http.query.param." + URLDecoder.decode(keyValueString, charset), "");
} else {
final String key = keyValueString.substring(0, indexOf);
final String value;
if (indexOf == keyValueString.length() - 1) {
value = "";
} else {
value = keyValueString.substring(indexOf + 1);
}
attributes.put(
"http.query.param." + URLDecoder.decode(key, charset),
URLDecoder.decode(value, charset));
}
}
}
} catch (final UnsupportedEncodingException uee) {
throw new ProcessException(
"Invalid character encoding", uee); // won't happen because charset has been validated
}
final Enumeration<String> headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
final String headerName = headerNames.nextElement();
final String headerValue = request.getHeader(headerName);
putAttribute(attributes, "http.headers." + headerName, headerValue);
}
final Principal principal = request.getUserPrincipal();
if (principal != null) {
putAttribute(attributes, "http.principal.name", principal.getName());
}
final X509Certificate certs[] =
(X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
final String subjectDn;
if (certs != null && certs.length > 0) {
final X509Certificate cert = certs[0];
subjectDn = cert.getSubjectDN().getName();
final String issuerDn = cert.getIssuerDN().getName();
putAttribute(attributes, HTTPUtils.HTTP_SSL_CERT, subjectDn);
putAttribute(attributes, "http.issuer.dn", issuerDn);
} else {
subjectDn = null;
}
flowFile = session.putAllAttributes(flowFile, attributes);
final HttpContextMap contextMap =
context.getProperty(HTTP_CONTEXT_MAP).asControllerService(HttpContextMap.class);
final boolean registered =
contextMap.register(
contextIdentifier, request, container.getResponse(), container.getContext());
if (!registered) {
getLogger()
.warn(
"Received request from {} but could not process it because too many requests are already outstanding; responding with SERVICE_UNAVAILABLE",
new Object[] {request.getRemoteAddr()});
try {
container.getResponse().setStatus(Status.SERVICE_UNAVAILABLE.getStatusCode());
container.getResponse().flushBuffer();
container.getContext().complete();
} catch (final Exception e) {
getLogger()
.warn(
"Failed to respond with SERVICE_UNAVAILABLE message to {} due to {}",
new Object[] {request.getRemoteAddr(), e});
}
session.remove(flowFile);
return;
}
final long receiveMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
session
.getProvenanceReporter()
.receive(
flowFile,
HTTPUtils.getURI(attributes),
"Received from "
+ request.getRemoteAddr()
+ (subjectDn == null ? "" : " with DN=" + subjectDn),
receiveMillis);
session.transfer(flowFile, REL_SUCCESS);
getLogger()
.info(
"Transferring {} to 'success'; received from {}",
new Object[] {flowFile, request.getRemoteAddr()});
}
private void putAttribute(final Map<String, String> map, final String key, final Object value) {
if (value == null) {
return;
}
putAttribute(map, key, value.toString());
}
private void putAttribute(final Map<String, String> map, final String key, final String value) {
if (value == null) {
return;
}
map.put(key, value);
}
private static class HttpRequestContainer {
private final HttpServletRequest request;
private final HttpServletResponse response;
private final AsyncContext context;
public HttpRequestContainer(
final HttpServletRequest request,
final HttpServletResponse response,
final AsyncContext async) {
this.request = request;
this.response = response;
this.context = async;
}
public HttpServletRequest getRequest() {
return request;
}
public HttpServletResponse getResponse() {
return response;
}
public AsyncContext getContext() {
return context;
}
}
}
/** A base class for processors that interact with Apache Solr. */
public abstract class SolrProcessor extends AbstractProcessor {
public static final AllowableValue SOLR_TYPE_CLOUD =
new AllowableValue("Cloud", "Cloud", "A SolrCloud instance.");
public static final AllowableValue SOLR_TYPE_STANDARD =
new AllowableValue("Standard", "Standard", "A stand-alone Solr instance.");
public static final PropertyDescriptor SOLR_TYPE =
new PropertyDescriptor.Builder()
.name("Solr Type")
.description("The type of Solr instance, Cloud or Standard.")
.required(true)
.allowableValues(SOLR_TYPE_CLOUD, SOLR_TYPE_STANDARD)
.defaultValue(SOLR_TYPE_STANDARD.getValue())
.build();
public static final PropertyDescriptor SOLR_LOCATION =
new PropertyDescriptor.Builder()
.name("Solr Location")
.description(
"The Solr url for a Solr Type of Standard (ex: http://localhost:8984/solr/gettingstarted), "
+ "or the ZooKeeper hosts for a Solr Type of Cloud (ex: localhost:9983).")
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
public static final PropertyDescriptor COLLECTION =
new PropertyDescriptor.Builder()
.name("Collection")
.description("The Solr collection name, only used with a Solr Type of Cloud")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(true)
.build();
private volatile SolrClient solrClient;
@OnScheduled
public final void onScheduled(final ProcessContext context) throws IOException {
this.solrClient = createSolrClient(context);
}
/**
* Create a SolrClient based on the type of Solr specified.
*
* @param context The context
* @return an HttpSolrClient or CloudSolrClient
*/
protected SolrClient createSolrClient(final ProcessContext context) {
if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
return new HttpSolrClient(context.getProperty(SOLR_LOCATION).getValue());
} else {
CloudSolrClient cloudSolrClient =
new CloudSolrClient(context.getProperty(SOLR_LOCATION).getValue());
cloudSolrClient.setDefaultCollection(
context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue());
return cloudSolrClient;
}
}
/**
* Returns the {@link org.apache.solr.client.solrj.SolrClient} that was created by the {@link
* #createSolrClient(org.apache.nifi.processor.ProcessContext)} method
*
* @return an HttpSolrClient or CloudSolrClient
*/
protected final SolrClient getSolrClient() {
return solrClient;
}
@Override
protected final Collection<ValidationResult> customValidate(ValidationContext context) {
final List<ValidationResult> problems = new ArrayList<>();
if (SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue())) {
final String collection = context.getProperty(COLLECTION).getValue();
if (collection == null || collection.trim().isEmpty()) {
problems.add(
new ValidationResult.Builder()
.subject(COLLECTION.getName())
.input(collection)
.valid(false)
.explanation("A collection must specified for Solr Type of Cloud")
.build());
}
}
Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context);
if (otherProblems != null) {
problems.addAll(otherProblems);
}
return problems;
}
/**
* Allows additional custom validation to be done. This will be called from the parent's
* customValidation method.
*
* @param context The context
* @return Validation results indicating problems
*/
protected Collection<ValidationResult> additionalCustomValidation(ValidationContext context) {
return new ArrayList<>();
}
}