Ejemplo n.º 1
0
 private void updateDataDogTransport(ReportingContext context) throws IOException {
   String dataDogTransport = context.getProperty(DATADOG_TRANSPORT).getValue();
   if (dataDogTransport.equalsIgnoreCase(DATADOG_AGENT.getValue())) {
     ddMetricRegistryBuilder.build("agent");
   } else if (dataDogTransport.equalsIgnoreCase(DATADOG_HTTP.getValue())
       && context.getProperty(API_KEY).isSet()) {
     ddMetricRegistryBuilder.build(context.getProperty(API_KEY).getValue());
   }
 }
Ejemplo n.º 2
0
 @Override
 public ValidationResult setProperty(
     final ControllerService service,
     final PropertyDescriptor property,
     final AllowableValue value) {
   return setProperty(service, property, value.getValue());
 }
Ejemplo n.º 3
0
 /**
  * Create a SolrClient based on the type of Solr specified.
  *
  * @param context The context
  * @return an HttpSolrClient or CloudSolrClient
  */
 protected SolrClient createSolrClient(final ProcessContext context) {
   if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
     return new HttpSolrClient(context.getProperty(SOLR_LOCATION).getValue());
   } else {
     CloudSolrClient cloudSolrClient =
         new CloudSolrClient(context.getProperty(SOLR_LOCATION).getValue());
     cloudSolrClient.setDefaultCollection(
         context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue());
     return cloudSolrClient;
   }
 }
Ejemplo n.º 4
0
  @Override
  protected final Collection<ValidationResult> customValidate(ValidationContext context) {
    final List<ValidationResult> problems = new ArrayList<>();

    if (SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue())) {
      final String collection = context.getProperty(COLLECTION).getValue();
      if (collection == null || collection.trim().isEmpty()) {
        problems.add(
            new ValidationResult.Builder()
                .subject(COLLECTION.getName())
                .input(collection)
                .valid(false)
                .explanation("A collection must specified for Solr Type of Cloud")
                .build());
      }
    }

    Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context);
    if (otherProblems != null) {
      problems.addAll(otherProblems);
    }

    return problems;
  }
Ejemplo n.º 5
0
 @Override
 public ValidationResult setProperty(
     final PropertyDescriptor descriptor, final AllowableValue value) {
   return context.setProperty(descriptor, value.getValue());
 }
Ejemplo n.º 6
0
@EventDriven
@SupportsBatching
@Tags({"map", "cache", "put", "distributed"})
@InputRequirement(Requirement.INPUT_REQUIRED)
@CapabilityDescription(
    "Gets the content of a FlowFile and puts it to a distributed map cache, using a cache key "
        + "computed from FlowFile attributes. If the cache already contains the entry and the cache update strategy is "
        + "'keep original' the entry is not replaced.'")
@WritesAttribute(
    attribute = "cached",
    description =
        "All FlowFiles will have an attribute 'cached'. The value of this "
            + "attribute is true, is the FlowFile is cached, otherwise false.")
@SeeAlso(
    classNames = {
      "org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService",
      "org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer",
      "org.apache.nifi.processors.standard.FetchDistributedMapCache"
    })
public class PutDistributedMapCache extends AbstractProcessor {

  public static final String CACHED_ATTRIBUTE_NAME = "cached";

  // Identifies the distributed map cache client
  public static final PropertyDescriptor DISTRIBUTED_CACHE_SERVICE =
      new PropertyDescriptor.Builder()
          .name("Distributed Cache Service")
          .description("The Controller Service that is used to cache flow files")
          .required(true)
          .identifiesControllerService(DistributedMapCacheClient.class)
          .build();

  // Selects the FlowFile attribute, whose value is used as cache key
  public static final PropertyDescriptor CACHE_ENTRY_IDENTIFIER =
      new PropertyDescriptor.Builder()
          .name("Cache Entry Identifier")
          .description(
              "A FlowFile attribute, or the results of an Attribute Expression Language statement, which will "
                  + "be evaluated against a FlowFile in order to determine the cache key")
          .required(true)
          .addValidator(
              StandardValidators.createAttributeExpressionLanguageValidator(
                  ResultType.STRING, true))
          .expressionLanguageSupported(true)
          .build();

  public static final AllowableValue CACHE_UPDATE_REPLACE =
      new AllowableValue(
          "replace",
          "Replace if present",
          "Adds the specified entry to the cache, replacing any value that is currently set.");

  public static final AllowableValue CACHE_UPDATE_KEEP_ORIGINAL =
      new AllowableValue(
          "keeporiginal",
          "Keep original",
          "Adds the specified entry to the cache, if the key does not exist.");

  public static final PropertyDescriptor CACHE_UPDATE_STRATEGY =
      new PropertyDescriptor.Builder()
          .name("Cache update strategy")
          .description(
              "Determines how the cache is updated if the cache already contains the entry")
          .required(true)
          .allowableValues(CACHE_UPDATE_REPLACE, CACHE_UPDATE_KEEP_ORIGINAL)
          .defaultValue(CACHE_UPDATE_REPLACE.getValue())
          .build();

  public static final PropertyDescriptor CACHE_ENTRY_MAX_BYTES =
      new PropertyDescriptor.Builder()
          .name("Max cache entry size")
          .description("The maximum amount of data to put into cache")
          .required(false)
          .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
          .defaultValue("1 MB")
          .expressionLanguageSupported(false)
          .build();

  public static final Relationship REL_SUCCESS =
      new Relationship.Builder()
          .name("success")
          .description(
              "Any FlowFile that is successfully inserted into cache will be routed to this relationship")
          .build();

  public static final Relationship REL_FAILURE =
      new Relationship.Builder()
          .name("failure")
          .description(
              "Any FlowFile that cannot be inserted into the cache will be routed to this relationship")
          .build();
  private final Set<Relationship> relationships;

  private final Serializer<String> keySerializer = new StringSerializer();
  private final Serializer<byte[]> valueSerializer = new CacheValueSerializer();
  private final Deserializer<byte[]> valueDeserializer = new CacheValueDeserializer();

  public PutDistributedMapCache() {
    final Set<Relationship> rels = new HashSet<>();
    rels.add(REL_SUCCESS);
    rels.add(REL_FAILURE);
    relationships = Collections.unmodifiableSet(rels);
  }

  @Override
  protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    final List<PropertyDescriptor> descriptors = new ArrayList<>();
    descriptors.add(CACHE_ENTRY_IDENTIFIER);
    descriptors.add(DISTRIBUTED_CACHE_SERVICE);
    descriptors.add(CACHE_UPDATE_STRATEGY);
    descriptors.add(CACHE_ENTRY_MAX_BYTES);
    return descriptors;
  }

  @Override
  public Set<Relationship> getRelationships() {
    return relationships;
  }

  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session)
      throws ProcessException {

    FlowFile flowFile = session.get();
    if (flowFile == null) {
      return;
    }

    final ProcessorLog logger = getLogger();

    // cache key is computed from attribute 'CACHE_ENTRY_IDENTIFIER' with expression language
    // support
    final String cacheKey =
        context
            .getProperty(CACHE_ENTRY_IDENTIFIER)
            .evaluateAttributeExpressions(flowFile)
            .getValue();

    // if the computed value is null, or empty, we transfer the flow file to failure relationship
    if (StringUtils.isBlank(cacheKey)) {
      logger.error(
          "FlowFile {} has no attribute for given Cache Entry Identifier", new Object[] {flowFile});
      flowFile = session.penalize(flowFile);
      session.transfer(flowFile, REL_FAILURE);
      return;
    }

    // the cache client used to interact with the distributed cache
    final DistributedMapCacheClient cache =
        context
            .getProperty(DISTRIBUTED_CACHE_SERVICE)
            .asControllerService(DistributedMapCacheClient.class);

    try {

      final long maxCacheEntrySize =
          context.getProperty(CACHE_ENTRY_MAX_BYTES).asDataSize(DataUnit.B).longValue();
      long flowFileSize = flowFile.getSize();

      // too big flow file
      if (flowFileSize > maxCacheEntrySize) {
        logger.warn(
            "Flow file {} size {} exceeds the max cache entry size ({} B).",
            new Object[] {flowFile, flowFileSize, maxCacheEntrySize});
        session.transfer(flowFile, REL_FAILURE);
        return;
      }

      if (flowFileSize == 0) {
        logger.warn("Flow file {} is empty, there is nothing to cache.", new Object[] {flowFile});
        session.transfer(flowFile, REL_FAILURE);
        return;
      }

      // get flow file content
      final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
      session.exportTo(flowFile, byteStream);
      byte[] cacheValue = byteStream.toByteArray();
      final String updateStrategy = context.getProperty(CACHE_UPDATE_STRATEGY).getValue();
      boolean cached = false;

      if (updateStrategy.equals(CACHE_UPDATE_REPLACE.getValue())) {
        cache.put(cacheKey, cacheValue, keySerializer, valueSerializer);
        cached = true;
      } else if (updateStrategy.equals(CACHE_UPDATE_KEEP_ORIGINAL.getValue())) {
        final byte[] oldValue =
            cache.getAndPutIfAbsent(
                cacheKey, cacheValue, keySerializer, valueSerializer, valueDeserializer);
        if (oldValue == null) {
          cached = true;
        }
      }

      // set 'cached' attribute
      flowFile = session.putAttribute(flowFile, CACHED_ATTRIBUTE_NAME, String.valueOf(cached));

      if (cached) {
        session.transfer(flowFile, REL_SUCCESS);
      } else {
        session.transfer(flowFile, REL_FAILURE);
      }

    } catch (final IOException e) {
      flowFile = session.penalize(flowFile);
      session.transfer(flowFile, REL_FAILURE);
      logger.error(
          "Unable to communicate with cache when processing {} due to {}",
          new Object[] {flowFile, e});
    }
  }

  public static class CacheValueSerializer implements Serializer<byte[]> {

    @Override
    public void serialize(final byte[] bytes, final OutputStream out)
        throws SerializationException, IOException {
      out.write(bytes);
    }
  }

  public static class CacheValueDeserializer implements Deserializer<byte[]> {

    @Override
    public byte[] deserialize(final byte[] input) throws DeserializationException, IOException {
      if (input == null || input.length == 0) {
        return null;
      }
      return input;
    }
  }

  /** Simple string serializer, used for serializing the cache key */
  public static class StringSerializer implements Serializer<String> {

    @Override
    public void serialize(final String value, final OutputStream out)
        throws SerializationException, IOException {
      out.write(value.getBytes(StandardCharsets.UTF_8));
    }
  }
}
Ejemplo n.º 7
0
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session)
      throws ProcessException {

    FlowFile flowFile = session.get();
    if (flowFile == null) {
      return;
    }

    final ProcessorLog logger = getLogger();

    // cache key is computed from attribute 'CACHE_ENTRY_IDENTIFIER' with expression language
    // support
    final String cacheKey =
        context
            .getProperty(CACHE_ENTRY_IDENTIFIER)
            .evaluateAttributeExpressions(flowFile)
            .getValue();

    // if the computed value is null, or empty, we transfer the flow file to failure relationship
    if (StringUtils.isBlank(cacheKey)) {
      logger.error(
          "FlowFile {} has no attribute for given Cache Entry Identifier", new Object[] {flowFile});
      flowFile = session.penalize(flowFile);
      session.transfer(flowFile, REL_FAILURE);
      return;
    }

    // the cache client used to interact with the distributed cache
    final DistributedMapCacheClient cache =
        context
            .getProperty(DISTRIBUTED_CACHE_SERVICE)
            .asControllerService(DistributedMapCacheClient.class);

    try {

      final long maxCacheEntrySize =
          context.getProperty(CACHE_ENTRY_MAX_BYTES).asDataSize(DataUnit.B).longValue();
      long flowFileSize = flowFile.getSize();

      // too big flow file
      if (flowFileSize > maxCacheEntrySize) {
        logger.warn(
            "Flow file {} size {} exceeds the max cache entry size ({} B).",
            new Object[] {flowFile, flowFileSize, maxCacheEntrySize});
        session.transfer(flowFile, REL_FAILURE);
        return;
      }

      if (flowFileSize == 0) {
        logger.warn("Flow file {} is empty, there is nothing to cache.", new Object[] {flowFile});
        session.transfer(flowFile, REL_FAILURE);
        return;
      }

      // get flow file content
      final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
      session.exportTo(flowFile, byteStream);
      byte[] cacheValue = byteStream.toByteArray();
      final String updateStrategy = context.getProperty(CACHE_UPDATE_STRATEGY).getValue();
      boolean cached = false;

      if (updateStrategy.equals(CACHE_UPDATE_REPLACE.getValue())) {
        cache.put(cacheKey, cacheValue, keySerializer, valueSerializer);
        cached = true;
      } else if (updateStrategy.equals(CACHE_UPDATE_KEEP_ORIGINAL.getValue())) {
        final byte[] oldValue =
            cache.getAndPutIfAbsent(
                cacheKey, cacheValue, keySerializer, valueSerializer, valueDeserializer);
        if (oldValue == null) {
          cached = true;
        }
      }

      // set 'cached' attribute
      flowFile = session.putAttribute(flowFile, CACHED_ATTRIBUTE_NAME, String.valueOf(cached));

      if (cached) {
        session.transfer(flowFile, REL_SUCCESS);
      } else {
        session.transfer(flowFile, REL_FAILURE);
      }

    } catch (final IOException e) {
      flowFile = session.penalize(flowFile);
      session.transfer(flowFile, REL_FAILURE);
      logger.error(
          "Unable to communicate with cache when processing {} due to {}",
          new Object[] {flowFile, e});
    }
  }
Ejemplo n.º 8
0
@Tags({"reporting", "datadog", "metrics"})
@CapabilityDescription(
    "Publishes metrics from NiFi to datadog. For accurate and informative reporting, components should have unique names.")
public class DataDogReportingTask extends AbstractReportingTask {

  static final AllowableValue DATADOG_AGENT =
      new AllowableValue(
          "Datadog Agent",
          "Datadog Agent",
          "Metrics will be sent via locally installed Datadog agent. "
              + "Datadog agent needs to be installed manually before using this option");

  static final AllowableValue DATADOG_HTTP =
      new AllowableValue(
          "Datadog HTTP",
          "Datadog HTTP",
          "Metrics will be sent via HTTP transport with no need of Agent installed. "
              + "Datadog API key needs to be set");

  static final PropertyDescriptor DATADOG_TRANSPORT =
      new PropertyDescriptor.Builder()
          .name("Datadog transport")
          .description("Transport through which metrics will be sent to Datadog")
          .required(true)
          .allowableValues(DATADOG_AGENT, DATADOG_HTTP)
          .defaultValue(DATADOG_HTTP.getValue())
          .build();

  static final PropertyDescriptor API_KEY =
      new PropertyDescriptor.Builder()
          .name("API key")
          .description(
              "Datadog API key. If specified value is 'agent', local Datadog agent will be used.")
          .expressionLanguageSupported(false)
          .required(false)
          .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
          .build();

  static final PropertyDescriptor METRICS_PREFIX =
      new PropertyDescriptor.Builder()
          .name("Metrics prefix")
          .description("Prefix to be added before every metric")
          .required(true)
          .expressionLanguageSupported(true)
          .defaultValue("nifi")
          .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
          .build();

  static final PropertyDescriptor ENVIRONMENT =
      new PropertyDescriptor.Builder()
          .name("Environment")
          .description(
              "Environment, dataflow is running in. "
                  + "This property will be included as metrics tag.")
          .required(true)
          .expressionLanguageSupported(true)
          .defaultValue("dev")
          .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
          .build();

  private MetricsService metricsService;
  private DDMetricRegistryBuilder ddMetricRegistryBuilder;
  private MetricRegistry metricRegistry;
  private String metricsPrefix;
  private String environment;
  private String statusId;
  private ConcurrentHashMap<String, AtomicDouble> metricsMap;
  private Map<String, String> defaultTags;
  private volatile VirtualMachineMetrics virtualMachineMetrics;
  private Logger logger = LoggerFactory.getLogger(getClass().getName());

  @OnScheduled
  public void setup(final ConfigurationContext context) {
    metricsService = getMetricsService();
    ddMetricRegistryBuilder = getMetricRegistryBuilder();
    metricRegistry = getMetricRegistry();
    metricsMap = getMetricsMap();
    metricsPrefix = METRICS_PREFIX.getDefaultValue();
    environment = ENVIRONMENT.getDefaultValue();
    virtualMachineMetrics = VirtualMachineMetrics.getInstance();
    ddMetricRegistryBuilder
        .setMetricRegistry(metricRegistry)
        .setTags(metricsService.getAllTagsList());
  }

  @Override
  protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    final List<PropertyDescriptor> properties = new ArrayList<>();
    properties.add(METRICS_PREFIX);
    properties.add(ENVIRONMENT);
    properties.add(API_KEY);
    properties.add(DATADOG_TRANSPORT);
    return properties;
  }

  @Override
  public void onTrigger(ReportingContext context) {
    final ProcessGroupStatus status = context.getEventAccess().getControllerStatus();

    metricsPrefix = context.getProperty(METRICS_PREFIX).evaluateAttributeExpressions().getValue();
    environment = context.getProperty(ENVIRONMENT).evaluateAttributeExpressions().getValue();
    statusId = status.getId();
    defaultTags = ImmutableMap.of("env", environment, "dataflow_id", statusId);
    try {
      updateDataDogTransport(context);
    } catch (IOException e) {
      e.printStackTrace();
    }
    updateAllMetricGroups(status);
    ddMetricRegistryBuilder.getDatadogReporter().report();
  }

  protected void updateMetrics(
      Map<String, Double> metrics, Optional<String> processorName, Map<String, String> tags) {
    for (Map.Entry<String, Double> entry : metrics.entrySet()) {
      final String metricName = buildMetricName(processorName, entry.getKey());
      logger.debug(metricName + ": " + entry.getValue());
      // if metric is not registered yet - register it
      if (!metricsMap.containsKey(metricName)) {
        metricsMap.put(metricName, new AtomicDouble(entry.getValue()));
        metricRegistry.register(metricName, new MetricGauge(metricName, tags));
      }
      // set real time value to metrics map
      metricsMap.get(metricName).set(entry.getValue());
    }
  }

  private void updateAllMetricGroups(ProcessGroupStatus processGroupStatus) {
    final List<ProcessorStatus> processorStatuses = new ArrayList<>();
    populateProcessorStatuses(processGroupStatus, processorStatuses);
    for (final ProcessorStatus processorStatus : processorStatuses) {
      updateMetrics(
          metricsService.getProcessorMetrics(processorStatus),
          Optional.of(processorStatus.getName()),
          defaultTags);
    }

    final List<ConnectionStatus> connectionStatuses = new ArrayList<>();
    populateConnectionStatuses(processGroupStatus, connectionStatuses);
    for (ConnectionStatus connectionStatus : connectionStatuses) {
      Map<String, String> connectionStatusTags = new HashMap<>(defaultTags);
      connectionStatusTags.putAll(metricsService.getConnectionStatusTags(connectionStatus));
      updateMetrics(
          metricsService.getConnectionStatusMetrics(connectionStatus),
          Optional.<String>absent(),
          connectionStatusTags);
    }

    final List<PortStatus> inputPortStatuses = new ArrayList<>();
    populateInputPortStatuses(processGroupStatus, inputPortStatuses);
    for (PortStatus portStatus : inputPortStatuses) {
      Map<String, String> portTags = new HashMap<>(defaultTags);
      portTags.putAll(metricsService.getPortStatusTags(portStatus));
      updateMetrics(
          metricsService.getPortStatusMetrics(portStatus), Optional.<String>absent(), portTags);
    }

    final List<PortStatus> outputPortStatuses = new ArrayList<>();
    populateOutputPortStatuses(processGroupStatus, outputPortStatuses);
    for (PortStatus portStatus : outputPortStatuses) {
      Map<String, String> portTags = new HashMap<>(defaultTags);
      portTags.putAll(metricsService.getPortStatusTags(portStatus));
      updateMetrics(
          metricsService.getPortStatusMetrics(portStatus), Optional.<String>absent(), portTags);
    }

    updateMetrics(
        metricsService.getJVMMetrics(virtualMachineMetrics),
        Optional.<String>absent(),
        defaultTags);
    updateMetrics(
        metricsService.getDataFlowMetrics(processGroupStatus),
        Optional.<String>absent(),
        defaultTags);
  }

  private class MetricGauge implements Gauge, DynamicTagsCallback {
    private Map<String, String> tags;
    private String metricName;

    public MetricGauge(String metricName, Map<String, String> tagsMap) {
      this.tags = tagsMap;
      this.metricName = metricName;
    }

    @Override
    public Object getValue() {
      return metricsMap.get(metricName).get();
    }

    @Override
    public List<String> getTags() {
      List<String> tagsList = Lists.newArrayList();
      for (Map.Entry<String, String> entry : tags.entrySet()) {
        tagsList.add(entry.getKey() + ":" + entry.getValue());
      }
      return tagsList;
    }
  }

  private void updateDataDogTransport(ReportingContext context) throws IOException {
    String dataDogTransport = context.getProperty(DATADOG_TRANSPORT).getValue();
    if (dataDogTransport.equalsIgnoreCase(DATADOG_AGENT.getValue())) {
      ddMetricRegistryBuilder.build("agent");
    } else if (dataDogTransport.equalsIgnoreCase(DATADOG_HTTP.getValue())
        && context.getProperty(API_KEY).isSet()) {
      ddMetricRegistryBuilder.build(context.getProperty(API_KEY).getValue());
    }
  }

  private void populateProcessorStatuses(
      final ProcessGroupStatus groupStatus, final List<ProcessorStatus> statuses) {
    statuses.addAll(groupStatus.getProcessorStatus());
    for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) {
      populateProcessorStatuses(childGroupStatus, statuses);
    }
  }

  private void populateConnectionStatuses(
      final ProcessGroupStatus groupStatus, final List<ConnectionStatus> statuses) {
    statuses.addAll(groupStatus.getConnectionStatus());
    for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) {
      populateConnectionStatuses(childGroupStatus, statuses);
    }
  }

  private void populateInputPortStatuses(
      final ProcessGroupStatus groupStatus, final List<PortStatus> statuses) {
    statuses.addAll(groupStatus.getInputPortStatus());
    for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) {
      populateInputPortStatuses(childGroupStatus, statuses);
    }
  }

  private void populateOutputPortStatuses(
      final ProcessGroupStatus groupStatus, final List<PortStatus> statuses) {
    statuses.addAll(groupStatus.getOutputPortStatus());
    for (final ProcessGroupStatus childGroupStatus : groupStatus.getProcessGroupStatus()) {
      populateOutputPortStatuses(childGroupStatus, statuses);
    }
  }

  private String buildMetricName(Optional<String> processorName, String metricName) {
    return metricsPrefix + "." + processorName.or("flow") + "." + metricName;
  }

  protected MetricsService getMetricsService() {
    return new MetricsService();
  }

  protected DDMetricRegistryBuilder getMetricRegistryBuilder() {
    return new DDMetricRegistryBuilder();
  }

  protected MetricRegistry getMetricRegistry() {
    return new MetricRegistry();
  }

  protected ConcurrentHashMap<String, AtomicDouble> getMetricsMap() {
    return new ConcurrentHashMap<>();
  }
}
Ejemplo n.º 9
0
@InputRequirement(Requirement.INPUT_FORBIDDEN)
@Tags({"http", "https", "request", "listen", "ingress", "web service"})
@CapabilityDescription(
    "Starts an HTTP Server and listens for HTTP Requests. For each request, creates a FlowFile and transfers to 'success'. "
        + "This Processor is designed to be used in conjunction with the HandleHttpResponse Processor in order to create a Web Service")
@WritesAttributes({
  @WritesAttribute(
      attribute = HTTPUtils.HTTP_CONTEXT_ID,
      description =
          "An identifier that allows the HandleHttpRequest and HandleHttpResponse "
              + "to coordinate which FlowFile belongs to which HTTP Request/Response."),
  @WritesAttribute(
      attribute = "mime.type",
      description = "The MIME Type of the data, according to the HTTP Header \"Content-Type\""),
  @WritesAttribute(
      attribute = "http.servlet.path",
      description = "The part of the request URL that is considered the Servlet Path"),
  @WritesAttribute(
      attribute = "http.context.path",
      description = "The part of the request URL that is considered to be the Context Path"),
  @WritesAttribute(
      attribute = "http.method",
      description = "The HTTP Method that was used for the request, such as GET or POST"),
  @WritesAttribute(
      attribute = HTTPUtils.HTTP_LOCAL_NAME,
      description = "IP address/hostname of the server"),
  @WritesAttribute(attribute = HTTPUtils.HTTP_PORT, description = "Listening port of the server"),
  @WritesAttribute(
      attribute = "http.query.string",
      description = "The query string portion of hte Request URL"),
  @WritesAttribute(
      attribute = HTTPUtils.HTTP_REMOTE_HOST,
      description = "The hostname of the requestor"),
  @WritesAttribute(
      attribute = "http.remote.addr",
      description = "The hostname:port combination of the requestor"),
  @WritesAttribute(attribute = "http.remote.user", description = "The username of the requestor"),
  @WritesAttribute(attribute = HTTPUtils.HTTP_REQUEST_URI, description = "The full Request URL"),
  @WritesAttribute(
      attribute = "http.auth.type",
      description = "The type of HTTP Authorization used"),
  @WritesAttribute(
      attribute = "http.principal.name",
      description = "The name of the authenticated user making the request"),
  @WritesAttribute(
      attribute = HTTPUtils.HTTP_SSL_CERT,
      description =
          "The Distinguished Name of the requestor. This value will not be populated "
              + "unless the Processor is configured to use an SSLContext Service"),
  @WritesAttribute(
      attribute = "http.issuer.dn",
      description =
          "The Distinguished Name of the entity that issued the Subject's certificate. "
              + "This value will not be populated unless the Processor is configured to use an SSLContext Service"),
  @WritesAttribute(
      attribute = "http.headers.XXX",
      description =
          "Each of the HTTP Headers that is received in the request will be added as an "
              + "attribute, prefixed with \"http.headers.\" For example, if the request contains an HTTP Header named \"x-my-header\", then the value "
              + "will be added to an attribute named \"http.headers.x-my-header\"")
})
@SeeAlso(
    value = {HandleHttpResponse.class},
    classNames = {
      "org.apache.nifi.http.StandardHttpContextMap",
      "org.apache.nifi.ssl.StandardSSLContextService"
    })
public class HandleHttpRequest extends AbstractProcessor {

  private static final Pattern URL_QUERY_PARAM_DELIMITER = Pattern.compile("&");

  // Allowable values for client auth
  public static final AllowableValue CLIENT_NONE =
      new AllowableValue(
          "No Authentication",
          "No Authentication",
          "Processor will not authenticate clients. Anyone can communicate with this Processor anonymously");
  public static final AllowableValue CLIENT_WANT =
      new AllowableValue(
          "Want Authentication",
          "Want Authentication",
          "Processor will try to verify the client but if unable to verify will allow the client to communicate anonymously");
  public static final AllowableValue CLIENT_NEED =
      new AllowableValue(
          "Need Authentication",
          "Need Authentication",
          "Processor will reject communications from any client unless the client provides a certificate that is trusted by the TrustStore"
              + "specified in the SSL Context Service");

  public static final PropertyDescriptor PORT =
      new PropertyDescriptor.Builder()
          .name("Listening Port")
          .description("The Port to listen on for incoming HTTP requests")
          .required(true)
          .addValidator(StandardValidators.createLongValidator(0L, 65535L, true))
          .expressionLanguageSupported(false)
          .defaultValue("80")
          .build();
  public static final PropertyDescriptor HOSTNAME =
      new PropertyDescriptor.Builder()
          .name("Hostname")
          .description("The Hostname to bind to. If not specified, will bind to all hosts")
          .required(false)
          .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
          .expressionLanguageSupported(false)
          .build();
  public static final PropertyDescriptor HTTP_CONTEXT_MAP =
      new PropertyDescriptor.Builder()
          .name("HTTP Context Map")
          .description(
              "The HTTP Context Map Controller Service to use for caching the HTTP Request Information")
          .required(true)
          .identifiesControllerService(HttpContextMap.class)
          .build();
  public static final PropertyDescriptor SSL_CONTEXT =
      new PropertyDescriptor.Builder()
          .name("SSL Context Service")
          .description(
              "The SSL Context Service to use in order to secure the server. If specified, the server will accept only HTTPS requests; "
                  + "otherwise, the server will accept only HTTP requests")
          .required(false)
          .identifiesControllerService(SSLContextService.class)
          .build();
  public static final PropertyDescriptor URL_CHARACTER_SET =
      new PropertyDescriptor.Builder()
          .name("Default URL Character Set")
          .description(
              "The character set to use for decoding URL parameters if the HTTP Request does not supply one")
          .required(true)
          .defaultValue("UTF-8")
          .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
          .build();
  public static final PropertyDescriptor PATH_REGEX =
      new PropertyDescriptor.Builder()
          .name("Allowed Paths")
          .description(
              "A Regular Expression that specifies the valid HTTP Paths that are allowed in the incoming URL Requests. If this value is "
                  + "specified and the path of the HTTP Requests does not match this Regular Expression, the Processor will respond with a "
                  + "404: NotFound")
          .required(false)
          .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
          .expressionLanguageSupported(false)
          .build();
  public static final PropertyDescriptor ALLOW_GET =
      new PropertyDescriptor.Builder()
          .name("Allow GET")
          .description("Allow HTTP GET Method")
          .required(true)
          .allowableValues("true", "false")
          .defaultValue("true")
          .build();
  public static final PropertyDescriptor ALLOW_POST =
      new PropertyDescriptor.Builder()
          .name("Allow POST")
          .description("Allow HTTP POST Method")
          .required(true)
          .allowableValues("true", "false")
          .defaultValue("true")
          .build();
  public static final PropertyDescriptor ALLOW_PUT =
      new PropertyDescriptor.Builder()
          .name("Allow PUT")
          .description("Allow HTTP PUT Method")
          .required(true)
          .allowableValues("true", "false")
          .defaultValue("true")
          .build();
  public static final PropertyDescriptor ALLOW_DELETE =
      new PropertyDescriptor.Builder()
          .name("Allow DELETE")
          .description("Allow HTTP DELETE Method")
          .required(true)
          .allowableValues("true", "false")
          .defaultValue("true")
          .build();
  public static final PropertyDescriptor ALLOW_HEAD =
      new PropertyDescriptor.Builder()
          .name("Allow HEAD")
          .description("Allow HTTP HEAD Method")
          .required(true)
          .allowableValues("true", "false")
          .defaultValue("false")
          .build();
  public static final PropertyDescriptor ALLOW_OPTIONS =
      new PropertyDescriptor.Builder()
          .name("Allow OPTIONS")
          .description("Allow HTTP OPTIONS Method")
          .required(true)
          .allowableValues("true", "false")
          .defaultValue("false")
          .build();
  public static final PropertyDescriptor ADDITIONAL_METHODS =
      new PropertyDescriptor.Builder()
          .name("Additional HTTP Methods")
          .description("A comma-separated list of non-standard HTTP Methods that should be allowed")
          .required(false)
          .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
          .expressionLanguageSupported(false)
          .build();
  public static final PropertyDescriptor CLIENT_AUTH =
      new PropertyDescriptor.Builder()
          .name("Client Authentication")
          .description(
              "Specifies whether or not the Processor should authenticate clients. This value is ignored if the <SSL Context Service> "
                  + "Property is not specified or the SSL Context provided uses only a KeyStore and not a TrustStore.")
          .required(true)
          .allowableValues(CLIENT_NONE, CLIENT_WANT, CLIENT_NEED)
          .defaultValue(CLIENT_NONE.getValue())
          .build();
  public static final PropertyDescriptor CONTAINER_QUEUE_SIZE =
      new PropertyDescriptor.Builder()
          .name("container-queue-size")
          .displayName("Container Queue Size")
          .description("The size of the queue for Http Request Containers")
          .required(true)
          .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
          .defaultValue("50")
          .build();

  public static final Relationship REL_SUCCESS =
      new Relationship.Builder()
          .name("success")
          .description("All content that is received is routed to the 'success' relationship")
          .build();

  private static final List<PropertyDescriptor> propertyDescriptors;

  static {
    List<PropertyDescriptor> descriptors = new ArrayList<>();
    descriptors.add(PORT);
    descriptors.add(HOSTNAME);
    descriptors.add(SSL_CONTEXT);
    descriptors.add(HTTP_CONTEXT_MAP);
    descriptors.add(PATH_REGEX);
    descriptors.add(URL_CHARACTER_SET);
    descriptors.add(ALLOW_GET);
    descriptors.add(ALLOW_POST);
    descriptors.add(ALLOW_PUT);
    descriptors.add(ALLOW_DELETE);
    descriptors.add(ALLOW_HEAD);
    descriptors.add(ALLOW_OPTIONS);
    descriptors.add(ADDITIONAL_METHODS);
    descriptors.add(CLIENT_AUTH);
    descriptors.add(CONTAINER_QUEUE_SIZE);
    propertyDescriptors = Collections.unmodifiableList(descriptors);
  }

  private volatile Server server;
  private AtomicBoolean initialized = new AtomicBoolean(false);
  private volatile BlockingQueue<HttpRequestContainer> containerQueue;

  @Override
  protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    return propertyDescriptors;
  }

  @Override
  public Set<Relationship> getRelationships() {
    return Collections.singleton(REL_SUCCESS);
  }

  @OnScheduled
  public void clearInit() {
    initialized.set(false);
  }

  private synchronized void initializeServer(final ProcessContext context) throws Exception {
    if (initialized.get()) {
      return;
    }
    this.containerQueue =
        new LinkedBlockingQueue<>(context.getProperty(CONTAINER_QUEUE_SIZE).asInteger());
    final String host = context.getProperty(HOSTNAME).getValue();
    final int port = context.getProperty(PORT).asInteger();
    final SSLContextService sslService =
        context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class);

    final String clientAuthValue = context.getProperty(CLIENT_AUTH).getValue();
    final boolean need;
    final boolean want;
    if (CLIENT_NEED.equals(clientAuthValue)) {
      need = true;
      want = false;
    } else if (CLIENT_WANT.equals(clientAuthValue)) {
      need = false;
      want = true;
    } else {
      need = false;
      want = false;
    }

    final SslContextFactory sslFactory =
        (sslService == null) ? null : createSslFactory(sslService, need, want);
    final Server server = new Server(port);

    // create the http configuration
    final HttpConfiguration httpConfiguration = new HttpConfiguration();
    if (sslFactory == null) {
      // create the connector
      final ServerConnector http =
          new ServerConnector(server, new HttpConnectionFactory(httpConfiguration));

      // set host and port
      if (StringUtils.isNotBlank(host)) {
        http.setHost(host);
      }
      http.setPort(port);

      // add this connector
      server.setConnectors(new Connector[] {http});
    } else {
      // add some secure config
      final HttpConfiguration httpsConfiguration = new HttpConfiguration(httpConfiguration);
      httpsConfiguration.setSecureScheme("https");
      httpsConfiguration.setSecurePort(port);
      httpsConfiguration.addCustomizer(new SecureRequestCustomizer());

      // build the connector
      final ServerConnector https =
          new ServerConnector(
              server,
              new SslConnectionFactory(sslFactory, "http/1.1"),
              new HttpConnectionFactory(httpsConfiguration));

      // set host and port
      if (StringUtils.isNotBlank(host)) {
        https.setHost(host);
      }
      https.setPort(port);

      // add this connector
      server.setConnectors(new Connector[] {https});
    }

    final Set<String> allowedMethods = new HashSet<>();
    if (context.getProperty(ALLOW_GET).asBoolean()) {
      allowedMethods.add("GET");
    }
    if (context.getProperty(ALLOW_POST).asBoolean()) {
      allowedMethods.add("POST");
    }
    if (context.getProperty(ALLOW_PUT).asBoolean()) {
      allowedMethods.add("PUT");
    }
    if (context.getProperty(ALLOW_DELETE).asBoolean()) {
      allowedMethods.add("DELETE");
    }
    if (context.getProperty(ALLOW_HEAD).asBoolean()) {
      allowedMethods.add("HEAD");
    }
    if (context.getProperty(ALLOW_OPTIONS).asBoolean()) {
      allowedMethods.add("OPTIONS");
    }

    final String additionalMethods = context.getProperty(ADDITIONAL_METHODS).getValue();
    if (additionalMethods != null) {
      for (final String additionalMethod : additionalMethods.split(",")) {
        final String trimmed = additionalMethod.trim();
        if (!trimmed.isEmpty()) {
          allowedMethods.add(trimmed.toUpperCase());
        }
      }
    }

    final String pathRegex = context.getProperty(PATH_REGEX).getValue();
    final Pattern pathPattern = (pathRegex == null) ? null : Pattern.compile(pathRegex);

    server.setHandler(
        new AbstractHandler() {
          @Override
          public void handle(
              final String target,
              final Request baseRequest,
              final HttpServletRequest request,
              final HttpServletResponse response)
              throws IOException, ServletException {

            final String requestUri = request.getRequestURI();
            if (!allowedMethods.contains(request.getMethod().toUpperCase())) {
              getLogger()
                  .info(
                      "Sending back METHOD_NOT_ALLOWED response to {}; method was {}; request URI was {}",
                      new Object[] {request.getRemoteAddr(), request.getMethod(), requestUri});
              response.sendError(Status.METHOD_NOT_ALLOWED.getStatusCode());
              return;
            }

            if (pathPattern != null) {
              final URI uri;
              try {
                uri = new URI(requestUri);
              } catch (final URISyntaxException e) {
                throw new ServletException(e);
              }

              if (!pathPattern.matcher(uri.getPath()).matches()) {
                response.sendError(Status.NOT_FOUND.getStatusCode());
                getLogger()
                    .info(
                        "Sending back NOT_FOUND response to {}; request was {} {}",
                        new Object[] {request.getRemoteAddr(), request.getMethod(), requestUri});
                return;
              }
            }

            // If destination queues full, send back a 503: Service Unavailable.
            if (context.getAvailableRelationships().isEmpty()) {
              response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
              return;
            }

            // Right now, that information, though, is only in the ProcessSession, not the
            // ProcessContext,
            // so it is not known to us. Should see if it can be added to the ProcessContext.
            final AsyncContext async = baseRequest.startAsync();
            async.setTimeout(Long.MAX_VALUE); // timeout is handled by HttpContextMap
            final boolean added =
                containerQueue.offer(new HttpRequestContainer(request, response, async));

            if (added) {
              getLogger()
                  .debug(
                      "Added Http Request to queue for {} {} from {}",
                      new Object[] {request.getMethod(), requestUri, request.getRemoteAddr()});
            } else {
              getLogger()
                  .info(
                      "Sending back a SERVICE_UNAVAILABLE response to {}; request was {} {}",
                      new Object[] {
                        request.getRemoteAddr(), request.getMethod(), request.getRemoteAddr()
                      });

              response.sendError(Status.SERVICE_UNAVAILABLE.getStatusCode());
              response.flushBuffer();
              async.complete();
            }
          }
        });

    this.server = server;
    server.start();

    getLogger().info("Server started and listening on port " + getPort());

    initialized.set(true);
  }

  protected int getPort() {
    for (final Connector connector : server.getConnectors()) {
      if (connector instanceof ServerConnector) {
        return ((ServerConnector) connector).getLocalPort();
      }
    }

    throw new IllegalStateException("Server is not listening on any ports");
  }

  protected int getRequestQueueSize() {
    return containerQueue.size();
  }

  private SslContextFactory createSslFactory(
      final SSLContextService sslService,
      final boolean needClientAuth,
      final boolean wantClientAuth) {
    final SslContextFactory sslFactory = new SslContextFactory();

    sslFactory.setNeedClientAuth(needClientAuth);
    sslFactory.setWantClientAuth(wantClientAuth);

    if (sslService.isKeyStoreConfigured()) {
      sslFactory.setKeyStorePath(sslService.getKeyStoreFile());
      sslFactory.setKeyStorePassword(sslService.getKeyStorePassword());
      sslFactory.setKeyStoreType(sslService.getKeyStoreType());
    }

    if (sslService.isTrustStoreConfigured()) {
      sslFactory.setTrustStorePath(sslService.getTrustStoreFile());
      sslFactory.setTrustStorePassword(sslService.getTrustStorePassword());
      sslFactory.setTrustStoreType(sslService.getTrustStoreType());
    }

    return sslFactory;
  }

  @OnStopped
  public void shutdown() throws Exception {
    if (server != null) {
      getLogger().debug("Shutting down server");
      server.stop();
      server.destroy();
      server.join();
      getLogger().info("Shut down {}", new Object[] {server});
    }
  }

  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session)
      throws ProcessException {
    try {
      if (!initialized.get()) {
        initializeServer(context);
      }
    } catch (Exception e) {
      context.yield();
      throw new ProcessException("Failed to initialize the server", e);
    }

    final HttpRequestContainer container = containerQueue.poll();
    if (container == null) {
      return;
    }

    final long start = System.nanoTime();
    final HttpServletRequest request = container.getRequest();
    FlowFile flowFile = session.create();
    try {
      flowFile = session.importFrom(request.getInputStream(), flowFile);
    } catch (final IOException e) {
      getLogger()
          .error(
              "Failed to receive content from HTTP Request from {} due to {}",
              new Object[] {request.getRemoteAddr(), e});
      session.remove(flowFile);
      return;
    }

    final String charset =
        request.getCharacterEncoding() == null
            ? context.getProperty(URL_CHARACTER_SET).getValue()
            : request.getCharacterEncoding();

    final String contextIdentifier = UUID.randomUUID().toString();
    final Map<String, String> attributes = new HashMap<>();
    try {
      putAttribute(attributes, HTTPUtils.HTTP_CONTEXT_ID, contextIdentifier);
      putAttribute(attributes, "mime.type", request.getContentType());
      putAttribute(attributes, "http.servlet.path", request.getServletPath());
      putAttribute(attributes, "http.context.path", request.getContextPath());
      putAttribute(attributes, "http.method", request.getMethod());
      putAttribute(attributes, "http.local.addr", request.getLocalAddr());
      putAttribute(attributes, HTTPUtils.HTTP_LOCAL_NAME, request.getLocalName());
      if (request.getQueryString() != null) {
        putAttribute(
            attributes, "http.query.string", URLDecoder.decode(request.getQueryString(), charset));
      }
      putAttribute(attributes, HTTPUtils.HTTP_REMOTE_HOST, request.getRemoteHost());
      putAttribute(attributes, "http.remote.addr", request.getRemoteAddr());
      putAttribute(attributes, "http.remote.user", request.getRemoteUser());
      putAttribute(attributes, HTTPUtils.HTTP_REQUEST_URI, request.getRequestURI());
      putAttribute(attributes, "http.request.url", request.getRequestURL().toString());
      putAttribute(attributes, "http.auth.type", request.getAuthType());

      putAttribute(attributes, "http.requested.session.id", request.getRequestedSessionId());
      if (request.getDispatcherType() != null) {
        putAttribute(attributes, "http.dispatcher.type", request.getDispatcherType().name());
      }
      putAttribute(attributes, "http.character.encoding", request.getCharacterEncoding());
      putAttribute(attributes, "http.locale", request.getLocale());
      putAttribute(attributes, "http.server.name", request.getServerName());
      putAttribute(attributes, HTTPUtils.HTTP_PORT, request.getServerPort());

      final Enumeration<String> paramEnumeration = request.getParameterNames();
      while (paramEnumeration.hasMoreElements()) {
        final String paramName = paramEnumeration.nextElement();
        final String value = request.getParameter(paramName);
        attributes.put("http.param." + paramName, value);
      }

      final Cookie[] cookies = request.getCookies();
      if (cookies != null) {
        for (final Cookie cookie : cookies) {
          final String name = cookie.getName();
          final String cookiePrefix = "http.cookie." + name + ".";
          attributes.put(cookiePrefix + "value", cookie.getValue());
          attributes.put(cookiePrefix + "domain", cookie.getDomain());
          attributes.put(cookiePrefix + "path", cookie.getPath());
          attributes.put(cookiePrefix + "max.age", String.valueOf(cookie.getMaxAge()));
          attributes.put(cookiePrefix + "version", String.valueOf(cookie.getVersion()));
          attributes.put(cookiePrefix + "secure", String.valueOf(cookie.getSecure()));
        }
      }

      final String queryString = request.getQueryString();
      if (queryString != null) {
        final String[] params = URL_QUERY_PARAM_DELIMITER.split(queryString);
        for (final String keyValueString : params) {
          final int indexOf = keyValueString.indexOf("=");
          if (indexOf < 0) {
            // no =, then it's just a key with no value
            attributes.put("http.query.param." + URLDecoder.decode(keyValueString, charset), "");
          } else {
            final String key = keyValueString.substring(0, indexOf);
            final String value;

            if (indexOf == keyValueString.length() - 1) {
              value = "";
            } else {
              value = keyValueString.substring(indexOf + 1);
            }

            attributes.put(
                "http.query.param." + URLDecoder.decode(key, charset),
                URLDecoder.decode(value, charset));
          }
        }
      }
    } catch (final UnsupportedEncodingException uee) {
      throw new ProcessException(
          "Invalid character encoding", uee); // won't happen because charset has been validated
    }

    final Enumeration<String> headerNames = request.getHeaderNames();
    while (headerNames.hasMoreElements()) {
      final String headerName = headerNames.nextElement();
      final String headerValue = request.getHeader(headerName);
      putAttribute(attributes, "http.headers." + headerName, headerValue);
    }

    final Principal principal = request.getUserPrincipal();
    if (principal != null) {
      putAttribute(attributes, "http.principal.name", principal.getName());
    }

    final X509Certificate certs[] =
        (X509Certificate[]) request.getAttribute("javax.servlet.request.X509Certificate");
    final String subjectDn;
    if (certs != null && certs.length > 0) {
      final X509Certificate cert = certs[0];
      subjectDn = cert.getSubjectDN().getName();
      final String issuerDn = cert.getIssuerDN().getName();

      putAttribute(attributes, HTTPUtils.HTTP_SSL_CERT, subjectDn);
      putAttribute(attributes, "http.issuer.dn", issuerDn);
    } else {
      subjectDn = null;
    }

    flowFile = session.putAllAttributes(flowFile, attributes);

    final HttpContextMap contextMap =
        context.getProperty(HTTP_CONTEXT_MAP).asControllerService(HttpContextMap.class);
    final boolean registered =
        contextMap.register(
            contextIdentifier, request, container.getResponse(), container.getContext());

    if (!registered) {
      getLogger()
          .warn(
              "Received request from {} but could not process it because too many requests are already outstanding; responding with SERVICE_UNAVAILABLE",
              new Object[] {request.getRemoteAddr()});

      try {
        container.getResponse().setStatus(Status.SERVICE_UNAVAILABLE.getStatusCode());
        container.getResponse().flushBuffer();
        container.getContext().complete();
      } catch (final Exception e) {
        getLogger()
            .warn(
                "Failed to respond with SERVICE_UNAVAILABLE message to {} due to {}",
                new Object[] {request.getRemoteAddr(), e});
      }

      session.remove(flowFile);
      return;
    }

    final long receiveMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    session
        .getProvenanceReporter()
        .receive(
            flowFile,
            HTTPUtils.getURI(attributes),
            "Received from "
                + request.getRemoteAddr()
                + (subjectDn == null ? "" : " with DN=" + subjectDn),
            receiveMillis);
    session.transfer(flowFile, REL_SUCCESS);
    getLogger()
        .info(
            "Transferring {} to 'success'; received from {}",
            new Object[] {flowFile, request.getRemoteAddr()});
  }

  private void putAttribute(final Map<String, String> map, final String key, final Object value) {
    if (value == null) {
      return;
    }

    putAttribute(map, key, value.toString());
  }

  private void putAttribute(final Map<String, String> map, final String key, final String value) {
    if (value == null) {
      return;
    }

    map.put(key, value);
  }

  private static class HttpRequestContainer {

    private final HttpServletRequest request;
    private final HttpServletResponse response;
    private final AsyncContext context;

    public HttpRequestContainer(
        final HttpServletRequest request,
        final HttpServletResponse response,
        final AsyncContext async) {
      this.request = request;
      this.response = response;
      this.context = async;
    }

    public HttpServletRequest getRequest() {
      return request;
    }

    public HttpServletResponse getResponse() {
      return response;
    }

    public AsyncContext getContext() {
      return context;
    }
  }
}
Ejemplo n.º 10
0
  private synchronized void initializeServer(final ProcessContext context) throws Exception {
    if (initialized.get()) {
      return;
    }
    this.containerQueue =
        new LinkedBlockingQueue<>(context.getProperty(CONTAINER_QUEUE_SIZE).asInteger());
    final String host = context.getProperty(HOSTNAME).getValue();
    final int port = context.getProperty(PORT).asInteger();
    final SSLContextService sslService =
        context.getProperty(SSL_CONTEXT).asControllerService(SSLContextService.class);

    final String clientAuthValue = context.getProperty(CLIENT_AUTH).getValue();
    final boolean need;
    final boolean want;
    if (CLIENT_NEED.equals(clientAuthValue)) {
      need = true;
      want = false;
    } else if (CLIENT_WANT.equals(clientAuthValue)) {
      need = false;
      want = true;
    } else {
      need = false;
      want = false;
    }

    final SslContextFactory sslFactory =
        (sslService == null) ? null : createSslFactory(sslService, need, want);
    final Server server = new Server(port);

    // create the http configuration
    final HttpConfiguration httpConfiguration = new HttpConfiguration();
    if (sslFactory == null) {
      // create the connector
      final ServerConnector http =
          new ServerConnector(server, new HttpConnectionFactory(httpConfiguration));

      // set host and port
      if (StringUtils.isNotBlank(host)) {
        http.setHost(host);
      }
      http.setPort(port);

      // add this connector
      server.setConnectors(new Connector[] {http});
    } else {
      // add some secure config
      final HttpConfiguration httpsConfiguration = new HttpConfiguration(httpConfiguration);
      httpsConfiguration.setSecureScheme("https");
      httpsConfiguration.setSecurePort(port);
      httpsConfiguration.addCustomizer(new SecureRequestCustomizer());

      // build the connector
      final ServerConnector https =
          new ServerConnector(
              server,
              new SslConnectionFactory(sslFactory, "http/1.1"),
              new HttpConnectionFactory(httpsConfiguration));

      // set host and port
      if (StringUtils.isNotBlank(host)) {
        https.setHost(host);
      }
      https.setPort(port);

      // add this connector
      server.setConnectors(new Connector[] {https});
    }

    final Set<String> allowedMethods = new HashSet<>();
    if (context.getProperty(ALLOW_GET).asBoolean()) {
      allowedMethods.add("GET");
    }
    if (context.getProperty(ALLOW_POST).asBoolean()) {
      allowedMethods.add("POST");
    }
    if (context.getProperty(ALLOW_PUT).asBoolean()) {
      allowedMethods.add("PUT");
    }
    if (context.getProperty(ALLOW_DELETE).asBoolean()) {
      allowedMethods.add("DELETE");
    }
    if (context.getProperty(ALLOW_HEAD).asBoolean()) {
      allowedMethods.add("HEAD");
    }
    if (context.getProperty(ALLOW_OPTIONS).asBoolean()) {
      allowedMethods.add("OPTIONS");
    }

    final String additionalMethods = context.getProperty(ADDITIONAL_METHODS).getValue();
    if (additionalMethods != null) {
      for (final String additionalMethod : additionalMethods.split(",")) {
        final String trimmed = additionalMethod.trim();
        if (!trimmed.isEmpty()) {
          allowedMethods.add(trimmed.toUpperCase());
        }
      }
    }

    final String pathRegex = context.getProperty(PATH_REGEX).getValue();
    final Pattern pathPattern = (pathRegex == null) ? null : Pattern.compile(pathRegex);

    server.setHandler(
        new AbstractHandler() {
          @Override
          public void handle(
              final String target,
              final Request baseRequest,
              final HttpServletRequest request,
              final HttpServletResponse response)
              throws IOException, ServletException {

            final String requestUri = request.getRequestURI();
            if (!allowedMethods.contains(request.getMethod().toUpperCase())) {
              getLogger()
                  .info(
                      "Sending back METHOD_NOT_ALLOWED response to {}; method was {}; request URI was {}",
                      new Object[] {request.getRemoteAddr(), request.getMethod(), requestUri});
              response.sendError(Status.METHOD_NOT_ALLOWED.getStatusCode());
              return;
            }

            if (pathPattern != null) {
              final URI uri;
              try {
                uri = new URI(requestUri);
              } catch (final URISyntaxException e) {
                throw new ServletException(e);
              }

              if (!pathPattern.matcher(uri.getPath()).matches()) {
                response.sendError(Status.NOT_FOUND.getStatusCode());
                getLogger()
                    .info(
                        "Sending back NOT_FOUND response to {}; request was {} {}",
                        new Object[] {request.getRemoteAddr(), request.getMethod(), requestUri});
                return;
              }
            }

            // If destination queues full, send back a 503: Service Unavailable.
            if (context.getAvailableRelationships().isEmpty()) {
              response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
              return;
            }

            // Right now, that information, though, is only in the ProcessSession, not the
            // ProcessContext,
            // so it is not known to us. Should see if it can be added to the ProcessContext.
            final AsyncContext async = baseRequest.startAsync();
            async.setTimeout(Long.MAX_VALUE); // timeout is handled by HttpContextMap
            final boolean added =
                containerQueue.offer(new HttpRequestContainer(request, response, async));

            if (added) {
              getLogger()
                  .debug(
                      "Added Http Request to queue for {} {} from {}",
                      new Object[] {request.getMethod(), requestUri, request.getRemoteAddr()});
            } else {
              getLogger()
                  .info(
                      "Sending back a SERVICE_UNAVAILABLE response to {}; request was {} {}",
                      new Object[] {
                        request.getRemoteAddr(), request.getMethod(), request.getRemoteAddr()
                      });

              response.sendError(Status.SERVICE_UNAVAILABLE.getStatusCode());
              response.flushBuffer();
              async.complete();
            }
          }
        });

    this.server = server;
    server.start();

    getLogger().info("Server started and listening on port " + getPort());

    initialized.set(true);
  }
Ejemplo n.º 11
0
/** A base class for processors that interact with Apache Solr. */
public abstract class SolrProcessor extends AbstractProcessor {

  public static final AllowableValue SOLR_TYPE_CLOUD =
      new AllowableValue("Cloud", "Cloud", "A SolrCloud instance.");

  public static final AllowableValue SOLR_TYPE_STANDARD =
      new AllowableValue("Standard", "Standard", "A stand-alone Solr instance.");

  public static final PropertyDescriptor SOLR_TYPE =
      new PropertyDescriptor.Builder()
          .name("Solr Type")
          .description("The type of Solr instance, Cloud or Standard.")
          .required(true)
          .allowableValues(SOLR_TYPE_CLOUD, SOLR_TYPE_STANDARD)
          .defaultValue(SOLR_TYPE_STANDARD.getValue())
          .build();

  public static final PropertyDescriptor SOLR_LOCATION =
      new PropertyDescriptor.Builder()
          .name("Solr Location")
          .description(
              "The Solr url for a Solr Type of Standard (ex: http://localhost:8984/solr/gettingstarted), "
                  + "or the ZooKeeper hosts for a Solr Type of Cloud (ex: localhost:9983).")
          .required(true)
          .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
          .build();

  public static final PropertyDescriptor COLLECTION =
      new PropertyDescriptor.Builder()
          .name("Collection")
          .description("The Solr collection name, only used with a Solr Type of Cloud")
          .required(false)
          .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
          .expressionLanguageSupported(true)
          .build();

  private volatile SolrClient solrClient;

  @OnScheduled
  public final void onScheduled(final ProcessContext context) throws IOException {
    this.solrClient = createSolrClient(context);
  }

  /**
   * Create a SolrClient based on the type of Solr specified.
   *
   * @param context The context
   * @return an HttpSolrClient or CloudSolrClient
   */
  protected SolrClient createSolrClient(final ProcessContext context) {
    if (SOLR_TYPE_STANDARD.equals(context.getProperty(SOLR_TYPE).getValue())) {
      return new HttpSolrClient(context.getProperty(SOLR_LOCATION).getValue());
    } else {
      CloudSolrClient cloudSolrClient =
          new CloudSolrClient(context.getProperty(SOLR_LOCATION).getValue());
      cloudSolrClient.setDefaultCollection(
          context.getProperty(COLLECTION).evaluateAttributeExpressions().getValue());
      return cloudSolrClient;
    }
  }

  /**
   * Returns the {@link org.apache.solr.client.solrj.SolrClient} that was created by the {@link
   * #createSolrClient(org.apache.nifi.processor.ProcessContext)} method
   *
   * @return an HttpSolrClient or CloudSolrClient
   */
  protected final SolrClient getSolrClient() {
    return solrClient;
  }

  @Override
  protected final Collection<ValidationResult> customValidate(ValidationContext context) {
    final List<ValidationResult> problems = new ArrayList<>();

    if (SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue())) {
      final String collection = context.getProperty(COLLECTION).getValue();
      if (collection == null || collection.trim().isEmpty()) {
        problems.add(
            new ValidationResult.Builder()
                .subject(COLLECTION.getName())
                .input(collection)
                .valid(false)
                .explanation("A collection must specified for Solr Type of Cloud")
                .build());
      }
    }

    Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context);
    if (otherProblems != null) {
      problems.addAll(otherProblems);
    }

    return problems;
  }

  /**
   * Allows additional custom validation to be done. This will be called from the parent's
   * customValidation method.
   *
   * @param context The context
   * @return Validation results indicating problems
   */
  protected Collection<ValidationResult> additionalCustomValidation(ValidationContext context) {
    return new ArrayList<>();
  }
}