@Override
  public void onTrigger(final ProcessContext context, final ProcessSession session)
      throws ProcessException {
    final List<FlowFile> flowFiles = session.get(50);
    if (flowFiles.isEmpty()) {
      return;
    }

    final ProcessorLog logger = getLogger();
    try {
      if (fileWatcher.checkAndReset()) {
        this.dictionaryTerms = createDictionary(context);
      }
    } catch (final IOException e) {
      logger.error("Unable to reload dictionary due to {}", e);
    }

    final boolean matchAll =
        context.getProperty(MATCHING_CRITERIA).getValue().equals(MATCH_CRITERIA_ALL);

    for (final FlowFile flowFile : flowFiles) {
      final boolean matched =
          matchAll
              ? allMatch(flowFile, attributePattern, dictionaryTerms)
              : anyMatch(flowFile, attributePattern, dictionaryTerms);
      final Relationship relationship = matched ? REL_MATCHED : REL_UNMATCHED;
      session.getProvenanceReporter().route(flowFile, relationship);
      session.transfer(flowFile, relationship);
      logger.info("Transferred {} to {}", new Object[] {flowFile, relationship});
    }
  }
Beispiel #2
0
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
      return;
    }

    final ProcessorLog logger = getLogger();

    boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
    String encoding = context.getProperty(ENCODING).getValue();
    StreamCallback encoder = null;

    // Select the encoder/decoder to use
    if (encode) {
      if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
        encoder = new EncodeBase64();
      } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
        encoder = new EncodeBase32();
      } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
        encoder = new EncodeHex();
      }
    } else {
      if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
        encoder = new DecodeBase64();
      } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
        encoder = new DecodeBase32();
      } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
        encoder = new DecodeHex();
      }
    }

    if (encoder == null) {
      logger.warn(
          "Unknown operation: {} {}", new Object[] {encode ? "encode" : "decode", encoding});
      return;
    }

    try {
      final StopWatch stopWatch = new StopWatch(true);
      flowFile = session.write(flowFile, encoder);

      logger.info("Successfully {} {}", new Object[] {encode ? "encoded" : "decoded", flowFile});
      session
          .getProvenanceReporter()
          .modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
      session.transfer(flowFile, REL_SUCCESS);
    } catch (Exception e) {
      logger.error(
          "Failed to {} {} due to {}", new Object[] {encode ? "encode" : "decode", flowFile, e});
      session.transfer(flowFile, REL_FAILURE);
    }
  }
  private Map<String, String> sendDirectMessage(
      final ProcessContext context,
      final ProcessSession session,
      final FlowFile flowFile,
      final ProcessorLog logger,
      Message message,
      Recipient recipient) {

    String groupAttribute = context.getProperty(GROUP_ATTRIBUTE_NAME).getValue();
    String groupSufixAttribute =
        context.getProperty(GROUP_SUFIX).isSet()
            ? context.getProperty(GROUP_SUFIX).evaluateAttributeExpressions(flowFile).getValue()
            : "";
    String groupSubjectAttribute = context.getProperty(GROUP_SUBJECT_ATTRIBUTE_NAME).getValue();
    String participantsAttribute = context.getProperty(PARTICIPANTS_ATTRIBUTE_NAME).getValue();
    String groupName =
        message.getHeader().getRelatedConversationId() != null
            ? message.getHeader().getRelatedConversationId()
            : UUID.randomUUID().toString();
    groupName += groupSufixAttribute;
    String groupSubject =
        message.getHeader().getSubject() != null
            ? message.getHeader().getSubject()
            : "Private Message";

    String referenceAttributeName = context.getProperty(REFERENCE_ATTRIBUTE_NAME).getValue();

    Map<String, String> generatedReferences = new HashMap<>();

    // create references
    generatedReferences.put(recipient.getRecipientId(), UUID.randomUUID().toString());

    FlowFile clone = session.clone(flowFile);
    session.getProvenanceReporter().clone(flowFile, clone);

    clone = session.putAttribute(clone, groupAttribute, groupName);
    clone = session.putAttribute(clone, groupSubjectAttribute, groupSubject);
    clone =
        session.putAttribute(
            clone,
            referenceAttributeName,
            generatedReferences.values().stream().collect(Collectors.joining(",")));
    clone =
        session.putAttribute(
            clone,
            participantsAttribute,
            recipient.getDeliveryAddress().getPhysicalAddress().getAddress());
    session.getProvenanceReporter().modifyAttributes(clone);

    logger.debug("Routing message {} to {}.", new Object[] {clone, REL_DIRECT_MESSAGE});
    session.transfer(clone, REL_DIRECT_MESSAGE);
    session.getProvenanceReporter().route(clone, REL_DIRECT_MESSAGE);

    return generatedReferences;
  }
  public static JmsProcessingSummary map2FlowFile(
      final ProcessContext context,
      final ProcessSession session,
      final Message message,
      final boolean addAttributes,
      ProcessorLog logger)
      throws Exception {

    // Currently not very useful, because always one Message == one FlowFile
    final IntegerHolder msgsThisFlowFile = new IntegerHolder(1);

    FlowFile flowFile = session.create();
    try {
      // MapMessage is exception, add only name-value pairs to FlowFile attributes
      if (message instanceof MapMessage) {
        MapMessage mapMessage = (MapMessage) message;
        flowFile = session.putAllAttributes(flowFile, createMapMessageValues(mapMessage));
      } else { // all other message types, write Message body to FlowFile content
        flowFile =
            session.write(
                flowFile,
                new OutputStreamCallback() {
                  @Override
                  public void process(final OutputStream rawOut) throws IOException {
                    try (final OutputStream out = new BufferedOutputStream(rawOut, 65536)) {
                      final byte[] messageBody = JmsFactory.createByteArray(message);
                      out.write(messageBody);
                    } catch (final JMSException e) {
                      throw new ProcessException("Failed to receive JMS Message due to {}", e);
                    }
                  }
                });
      }

      if (addAttributes) {
        flowFile = session.putAllAttributes(flowFile, JmsFactory.createAttributeMap(message));
      }

      session.getProvenanceReporter().receive(flowFile, context.getProperty(URL).getValue());
      session.transfer(flowFile, REL_SUCCESS);
      logger.info(
          "Created {} from {} messages received from JMS Server and transferred to 'success'",
          new Object[] {flowFile, msgsThisFlowFile.get()});

      return new JmsProcessingSummary(flowFile.getSize(), message, flowFile);

    } catch (Exception e) {
      session.remove(flowFile);
      throw e;
    }
  }
  private boolean reloadDictionary(
      final ProcessContext context, final boolean force, final ProcessorLog logger)
      throws IOException {
    boolean obtainedLock;
    if (force) {
      dictionaryUpdateLock.lock();
      obtainedLock = true;
    } else {
      obtainedLock = dictionaryUpdateLock.tryLock();
    }

    if (obtainedLock) {
      try {
        final Search<byte[]> search = new AhoCorasick<>();
        final Set<SearchTerm<byte[]>> terms = new HashSet<>();

        final InputStream inStream =
            Files.newInputStream(
                Paths.get(context.getProperty(DICTIONARY).getValue()), StandardOpenOption.READ);

        final TermLoader termLoader;
        if (context.getProperty(DICTIONARY_ENCODING).getValue().equalsIgnoreCase(TEXT_ENCODING)) {
          termLoader = new TextualTermLoader(inStream);
        } else {
          termLoader = new BinaryTermLoader(inStream);
        }

        try {
          SearchTerm<byte[]> term;
          while ((term = termLoader.nextTerm()) != null) {
            terms.add(term);
          }

          search.initializeDictionary(terms);
          searchRef.set(search);
          logger.info(
              "Loaded search dictionary from {}",
              new Object[] {context.getProperty(DICTIONARY).getValue()});
          return true;
        } finally {
          termLoader.close();
        }
      } finally {
        dictionaryUpdateLock.unlock();
      }
    } else {
      return false;
    }
  }
  private Map<String, String> sendMessagesToPreStablishedGroups(
      final ProcessContext context,
      final ProcessSession session,
      final FlowFile flowFile,
      final ProcessorLog logger,
      List<Recipient> recipients) {

    String groupAttribute = context.getProperty(GROUP_ATTRIBUTE_NAME).getValue();
    String groupSufixAttribute =
        context.getProperty(GROUP_SUFIX).isSet()
            ? context.getProperty(GROUP_SUFIX).evaluateAttributeExpressions(flowFile).getValue()
            : "";
    String referenceAttributeName = context.getProperty(REFERENCE_ATTRIBUTE_NAME).getValue();

    Map<String, String> generatedReferences = new HashMap<>();

    recipients
        .stream()
        .forEach(
            r -> {
              generatedReferences.put(r.getRecipientId(), UUID.randomUUID().toString());

              String address = r.getDeliveryAddress().getPhysicalAddress().getAddress();

              FlowFile clone = session.clone(flowFile);
              session.getProvenanceReporter().clone(flowFile, clone);

              clone =
                  session.putAttribute(
                      clone,
                      groupAttribute,
                      convertAddressToGroupName(address, groupSufixAttribute));
              clone =
                  session.putAttribute(
                      clone, referenceAttributeName, generatedReferences.get(r.getRecipientId()));
              session.getProvenanceReporter().modifyAttributes(clone);

              logger.debug("Routing message {} to {}.", new Object[] {clone, REL_PERMANENT_GROUP});
              session.transfer(clone, REL_PERMANENT_GROUP);
              session.getProvenanceReporter().route(clone, REL_PERMANENT_GROUP);
            });

    return generatedReferences;
  }
  private Map<String, String> sendMessageToDynamicGroup(
      final ProcessContext context,
      final ProcessSession session,
      final FlowFile flowFile,
      final ProcessorLog logger,
      Message message,
      List<Recipient> recipients) {

    String groupAttribute = context.getProperty(GROUP_ATTRIBUTE_NAME).getValue();
    String groupSufixAttribute =
        context.getProperty(GROUP_SUFIX).isSet()
            ? context.getProperty(GROUP_SUFIX).evaluateAttributeExpressions(flowFile).getValue()
            : "";
    String groupSubjectAttribute = context.getProperty(GROUP_SUBJECT_ATTRIBUTE_NAME).getValue();
    String participantsAttribute = context.getProperty(PARTICIPANTS_ATTRIBUTE_NAME).getValue();
    String groupName =
        message.getHeader().getRelatedConversationId() != null
            ? message.getHeader().getRelatedConversationId()
            : UUID.randomUUID().toString();
    groupName += groupSufixAttribute;
    String groupSubject =
        message.getHeader().getSubject() != null
            ? message.getHeader().getSubject()
            : "Group Message";

    String referenceAttributeName = context.getProperty(REFERENCE_ATTRIBUTE_NAME).getValue();

    Map<String, String> generatedReferences = new HashMap<>();

    // create references
    recipients
        .stream()
        .forEach(r -> generatedReferences.put(r.getRecipientId(), UUID.randomUUID().toString()));

    // convert the recipients insto a JSON array.
    JsonArray participants = new JsonArray();
    recipients
        .stream()
        .map(
            r -> {
              JsonObject jo = new JsonObject();
              jo.addProperty(
                  "participant", r.getDeliveryAddress().getPhysicalAddress().getAddress());
              return jo;
            })
        .forEach(participants::add);

    FlowFile clone = session.clone(flowFile);
    session.getProvenanceReporter().clone(flowFile, clone);

    clone = session.putAttribute(clone, groupAttribute, groupName);
    clone = session.putAttribute(clone, groupSubjectAttribute, groupSubject);
    clone =
        session.putAttribute(
            clone,
            referenceAttributeName,
            generatedReferences.values().stream().collect(Collectors.joining(",")));
    clone = session.putAttribute(clone, participantsAttribute, new Gson().toJson(participants));
    session.getProvenanceReporter().modifyAttributes(clone);

    logger.debug("Routing message {} to {}.", new Object[] {clone, REL_DYNAMIC_GROUP});
    session.transfer(clone, REL_DYNAMIC_GROUP);
    session.getProvenanceReporter().route(clone, REL_DYNAMIC_GROUP);

    return generatedReferences;
  }
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
      return;
    }

    final ProcessorLog logger = getLogger();

    final ObjectHolder<Throwable> errorHolder = new ObjectHolder<>(null);
    final ObjectHolder<MessageWrapper> messageWrapperHolder = new ObjectHolder<>(null);

    session.read(
        flowFile,
        (final InputStream rawIn) -> {
          try {
            messageWrapperHolder.set(MessageSerializer.deserializeMessageWrapper(rawIn));
          } catch (MessageSerializationException ex) {
            errorHolder.set(
                new RuntimeException(
                    "Error deserializing FlowFile content into a MessageWrapper instance. Routing to FAILURE",
                    ex));
          }
        });

    if (errorHolder.get() != null) {
      UCSCreateException.routeFlowFileToException(
          context,
          session,
          logger,
          flowFile,
          REL_FAILURE,
          null,
          "Error in message deserialization: " + errorHolder.get().getCause() != null
              ? errorHolder.get().getCause().getMessage()
              : errorHolder.get().getMessage(),
          ExceptionType.InvalidMessage,
          null,
          null);
      return;
    }

    Message message = messageWrapperHolder.get().getMessage();

    // resolve the sender. We couldn't resolved it before because we needed
    // the specific serviceId.
    UCSController ucsService =
        context.getProperty(UCS_CONTROLLER_SERVICE).asControllerService(UCSController.class);
    UserContactInfo uci =
        ucsService.resolveUserContactInfo(
            message.getHeader().getSender().getPhysicalAddress().getAddress());

    if (uci == null) {
      UCSCreateException.routeFlowFileToException(
          context,
          session,
          logger,
          flowFile,
          REL_FAILURE,
          null,
          "Unknown User: "******"Unknown Service "
              + context.getProperty(SERVICE_ID).getValue()
              + " for  User: "******"GROUP:").
    Map<Boolean, List<Recipient>> chatRecipients =
        message
            .getHeader()
            .getRecipientsList()
            .stream()
            .filter(
                r ->
                    r.getDeliveryAddress() != null
                        && r.getDeliveryAddress().getPhysicalAddress() != null)
            .filter(
                r ->
                    context
                        .getProperty(SERVICE_ID)
                        .getValue()
                        .equals(r.getDeliveryAddress().getPhysicalAddress().getServiceId()))
            .collect(
                Collectors.groupingBy(
                    r ->
                        r.getDeliveryAddress()
                            .getPhysicalAddress()
                            .getAddress()
                            .startsWith("GROUP:")));

    Map<String, String> generatedReferences = new HashMap<>();

    if (chatRecipients.containsKey(Boolean.TRUE)) {
      generatedReferences.putAll(
          this.sendMessagesToPreStablishedGroups(
              context, session, flowFile, logger, chatRecipients.get(Boolean.TRUE)));
    }

    if (chatRecipients.containsKey(Boolean.FALSE)) {
      List<Recipient> recipients = chatRecipients.get(Boolean.FALSE);
      if (recipients.size() == 1) {
        generatedReferences.putAll(
            this.sendDirectMessage(context, session, flowFile, logger, message, recipients.get(0)));
      } else {
        generatedReferences.putAll(
            this.sendMessageToDynamicGroup(
                context, session, flowFile, logger, message, recipients));
      }
    }

    logger.debug("Removing original FlowFile");
    session.remove(flowFile);

    // keep track of the generated references
    // TODO: is this check correct/enough?
    if (message.getHeader().isReceiptNotification()) {
      logger.debug(
          "The message has ReceiptNotification flag enabled -> We are persisting its references.");
      generatedReferences
          .entrySet()
          .stream()
          .forEach(
              (gr) -> {
                ucsService.saveMessageReference(message, gr.getKey(), gr.getValue());
              });
    } else {
      logger.debug(
          "The message doesn't have ReceiptNotification flag enabled -> We are not persisting its references.");
    }
  }
Beispiel #9
0
  private DestinationAccepts getDestinationAcceptance(
      final HttpClient client,
      final String uri,
      final ProcessorLog logger,
      final String transactionId)
      throws IOException {
    final HttpHead head = new HttpHead(uri);
    head.addHeader(TRANSACTION_ID_HEADER, transactionId);
    final HttpResponse response = client.execute(head);

    final int statusCode = response.getStatusLine().getStatusCode();
    if (statusCode == Status.METHOD_NOT_ALLOWED.getStatusCode()) {
      // we assume that the destination can support FlowFile v1 always.
      return new DestinationAccepts(false, false, true, false, null);
    } else if (statusCode == Status.OK.getStatusCode()) {
      boolean acceptsFlowFileV3 = false;
      boolean acceptsFlowFileV2 = false;
      boolean acceptsFlowFileV1 = true;
      boolean acceptsGzip = false;
      Integer protocolVersion = null;

      Header[] headers = response.getHeaders(ACCEPT);
      if (headers != null) {
        for (final Header header : headers) {
          for (final String accepted : header.getValue().split(",")) {
            final String trimmed = accepted.trim();
            if (trimmed.equals(APPLICATION_FLOW_FILE_V3)) {
              acceptsFlowFileV3 = true;
            } else if (trimmed.equals(APPLICATION_FLOW_FILE_V2)) {
              acceptsFlowFileV2 = true;
            } else {
              // we assume that the destination accepts FlowFile V1 because legacy versions
              // of NiFi that accepted V1 did not use an Accept header to indicate it... or
              // any other header. So the bets thing we can do is just assume that V1 is
              // accepted, if we're going to send as FlowFile.
              acceptsFlowFileV1 = true;
            }
          }
        }
      }

      final Header destinationVersion = response.getFirstHeader(PROTOCOL_VERSION_HEADER);
      if (destinationVersion != null) {
        try {
          protocolVersion = Integer.valueOf(destinationVersion.getValue());
        } catch (final NumberFormatException e) {
          // nothing to do here really.... it's an invalid value, so treat the same as if not
          // specified
        }
      }

      if (acceptsFlowFileV3) {
        logger.debug(
            "Connection to URI "
                + uri
                + " will be using Content Type "
                + APPLICATION_FLOW_FILE_V3
                + " if sending data as FlowFile");
      } else if (acceptsFlowFileV2) {
        logger.debug(
            "Connection to URI "
                + uri
                + " will be using Content Type "
                + APPLICATION_FLOW_FILE_V2
                + " if sending data as FlowFile");
      } else if (acceptsFlowFileV1) {
        logger.debug(
            "Connection to URI "
                + uri
                + " will be using Content Type "
                + APPLICATION_FLOW_FILE_V1
                + " if sending data as FlowFile");
      }

      headers = response.getHeaders(ACCEPT_ENCODING);
      if (headers != null) {
        for (final Header header : headers) {
          for (final String accepted : header.getValue().split(",")) {
            if (accepted.equalsIgnoreCase("gzip")) {
              acceptsGzip = true;
            }
          }
        }
      }

      if (acceptsGzip) {
        logger.debug(
            "Connection to URI " + uri + " indicates that inline GZIP compression is supported");
      } else {
        logger.debug(
            "Connection to URI "
                + uri
                + " indicates that it does NOT support inline GZIP compression");
      }

      return new DestinationAccepts(
          acceptsFlowFileV3, acceptsFlowFileV2, acceptsFlowFileV1, acceptsGzip, protocolVersion);
    } else {
      logger.warn(
          "Unable to communicate with destination; when attempting to perform an HTTP HEAD, got unexpected response code of "
              + statusCode
              + ": "
              + response.getStatusLine().getReasonPhrase());
      return new DestinationAccepts(false, false, false, false, null);
    }
  }
Beispiel #10
0
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
    final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
    final String userAgent = context.getProperty(USER_AGENT).getValue();

    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(
        context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(
        context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder.setSocketTimeout(
        context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();

    final StreamThrottler throttler = throttlerRef.get();
    final ProcessorLog logger = getLogger();

    final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
    String lastUrl = null;
    long bytesToSend = 0L;

    final List<FlowFile> toSend = new ArrayList<>();
    DestinationAccepts destinationAccepts = null;
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();

    final ObjectHolder<String> dnHolder = new ObjectHolder<>("none");
    while (true) {
      FlowFile flowFile = session.get();
      if (flowFile == null) {
        break;
      }

      final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
      try {
        new java.net.URL(url);
      } catch (final MalformedURLException e) {
        logger.error(
            "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
            new Object[] {flowFile, url});
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        continue;
      }

      // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing
      // FlowFiles
      if (lastUrl != null && !lastUrl.equals(url)) {
        session.transfer(flowFile);
        break;
      }

      lastUrl = url;
      toSend.add(flowFile);

      if (client == null || destinationAccepts == null) {
        final Config config = getConfig(url, context);
        final HttpClientConnectionManager conMan = config.getConnectionManager();

        final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
        clientBuilder.setConnectionManager(conMan);
        clientBuilder.setUserAgent(userAgent);
        clientBuilder.addInterceptorFirst(
            new HttpResponseInterceptor() {
              @Override
              public void process(final HttpResponse response, final HttpContext httpContext)
                  throws HttpException, IOException {
                HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
                ManagedHttpClientConnection conn =
                    coreContext.getConnection(ManagedHttpClientConnection.class);
                if (!conn.isOpen()) {
                  return;
                }

                SSLSession sslSession = conn.getSSLSession();

                if (sslSession != null) {
                  final X509Certificate[] certChain = sslSession.getPeerCertificateChain();
                  if (certChain == null || certChain.length == 0) {
                    throw new SSLPeerUnverifiedException("No certificates found");
                  }

                  final X509Certificate cert = certChain[0];
                  dnHolder.set(cert.getSubjectDN().getName().trim());
                }
              }
            });

        clientBuilder.disableAutomaticRetries();
        clientBuilder.disableContentCompression();

        final String username = context.getProperty(USERNAME).getValue();
        final String password = context.getProperty(PASSWORD).getValue();
        // set the credentials if appropriate
        if (username != null) {
          final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
          if (password == null) {
            credentialsProvider.setCredentials(
                AuthScope.ANY, new UsernamePasswordCredentials(username));
          } else {
            credentialsProvider.setCredentials(
                AuthScope.ANY, new UsernamePasswordCredentials(username, password));
          }
          clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }
        client = clientBuilder.build();

        // determine whether or not destination accepts flowfile/gzip
        destinationAccepts = config.getDestinationAccepts();
        if (destinationAccepts == null) {
          try {
            if (sendAsFlowFile) {
              destinationAccepts =
                  getDestinationAcceptance(client, url, getLogger(), transactionId);
            } else {
              destinationAccepts = new DestinationAccepts(false, false, false, false, null);
            }

            config.setDestinationAccepts(destinationAccepts);
          } catch (IOException e) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            logger.error(
                "Unable to communicate with destination {} to determine whether or not it can accept "
                    + "flowfiles/gzip; routing {} to failure due to {}",
                new Object[] {url, flowFile, e});
            context.yield();
            return;
          }
        }
      }

      // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2
      // (streaming) format,
      // then only use a single FlowFile
      if (!sendAsFlowFile
          || (!destinationAccepts.isFlowFileV3Accepted()
              && !destinationAccepts.isFlowFileV2Accepted())) {
        break;
      }

      bytesToSend += flowFile.getSize();
      if (bytesToSend > maxBatchBytes.longValue()) {
        break;
      }
    }

    if (toSend.isEmpty()) {
      return;
    }

    final String url = lastUrl;
    final HttpPost post = new HttpPost(url);
    final List<FlowFile> flowFileList = toSend;
    final DestinationAccepts accepts = destinationAccepts;
    final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;

    final EntityTemplate entity =
        new EntityTemplate(
            new ContentProducer() {
              @Override
              public void writeTo(final OutputStream rawOut) throws IOException {
                final OutputStream throttled =
                    (throttler == null) ? rawOut : throttler.newThrottledOutputStream(rawOut);
                OutputStream wrappedOut = new BufferedOutputStream(throttled);
                if (compressionLevel > 0 && accepts.isGzipAccepted()) {
                  wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
                }

                try (final OutputStream out = wrappedOut) {
                  for (final FlowFile flowFile : flowFileList) {
                    session.read(
                        flowFile,
                        new InputStreamCallback() {
                          @Override
                          public void process(final InputStream rawIn) throws IOException {
                            try (final InputStream in = new BufferedInputStream(rawIn)) {

                              FlowFilePackager packager = null;
                              if (!sendAsFlowFile) {
                                packager = null;
                              } else if (accepts.isFlowFileV3Accepted()) {
                                packager = new FlowFilePackagerV3();
                              } else if (accepts.isFlowFileV2Accepted()) {
                                packager = new FlowFilePackagerV2();
                              } else if (accepts.isFlowFileV1Accepted()) {
                                packager = new FlowFilePackagerV1();
                              }

                              // if none of the above conditions is met, we should never get here,
                              // because
                              // we will have already verified that at least 1 of the FlowFile
                              // packaging
                              // formats is acceptable if sending as FlowFile.
                              if (packager == null) {
                                StreamUtils.copy(in, out);
                              } else {
                                final Map<String, String> flowFileAttributes;
                                if (isDestinationLegacyNiFi) {
                                  // Old versions of NiFi expect nf.file.name and nf.file.path to
                                  // indicate filename & path;
                                  // in order to maintain backward compatibility, we copy the
                                  // filename & path to those attribute keys.
                                  flowFileAttributes = new HashMap<>(flowFile.getAttributes());
                                  flowFileAttributes.put(
                                      "nf.file.name",
                                      flowFile.getAttribute(CoreAttributes.FILENAME.key()));
                                  flowFileAttributes.put(
                                      "nf.file.path",
                                      flowFile.getAttribute(CoreAttributes.PATH.key()));
                                } else {
                                  flowFileAttributes = flowFile.getAttributes();
                                }

                                packager.packageFlowFile(
                                    in, out, flowFileAttributes, flowFile.getSize());
                              }
                            }
                          }
                        });
                  }

                  out.flush();
                }
              }
            });

    entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
    post.setEntity(entity);
    post.setConfig(requestConfig);

    final String contentType;
    if (sendAsFlowFile) {
      if (accepts.isFlowFileV3Accepted()) {
        contentType = APPLICATION_FLOW_FILE_V3;
      } else if (accepts.isFlowFileV2Accepted()) {
        contentType = APPLICATION_FLOW_FILE_V2;
      } else if (accepts.isFlowFileV1Accepted()) {
        contentType = APPLICATION_FLOW_FILE_V1;
      } else {
        logger.error(
            "Cannot send data to {} because the destination does not accept FlowFiles and this processor is "
                + "configured to deliver FlowFiles; rolling back session",
            new Object[] {url});
        session.rollback();
        context.yield();
        return;
      }
    } else {
      final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key());
      contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue;
    }

    final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
    if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) {
      final Pattern pattern = Pattern.compile(attributeHeaderRegex);

      final Map<String, String> attributes = flowFileList.get(0).getAttributes();
      for (final Map.Entry<String, String> entry : attributes.entrySet()) {
        final String key = entry.getKey();
        if (pattern.matcher(key).matches()) {
          post.setHeader(entry.getKey(), entry.getValue());
        }
      }
    }

    post.setHeader(CONTENT_TYPE, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);
    if (compressionLevel > 0 && accepts.isGzipAccepted()) {
      post.setHeader(GZIPPED_HEADER, "true");
    }

    // Do the actual POST
    final String flowFileDescription =
        toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";

    final String uploadDataRate;
    final long uploadMillis;
    CloseableHttpResponse response = null;
    try {
      final StopWatch stopWatch = new StopWatch(true);
      response = client.execute(post);

      // consume input stream entirely, ignoring its contents. If we
      // don't do this, the Connection will not be returned to the pool
      EntityUtils.consume(response.getEntity());
      stopWatch.stop();
      uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
      uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
      logger.error(
          "Failed to Post {} due to {}; transferring to failure",
          new Object[] {flowFileDescription, e});
      context.yield();
      for (FlowFile flowFile : toSend) {
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
      }
      return;
    } finally {
      if (response != null) {
        try {
          response.close();
        } catch (IOException e) {
          getLogger().warn("Failed to close HTTP Response due to {}", new Object[] {e});
        }
      }
    }

    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
      final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
      if (locationUriHeader != null) {
        if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
          final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
          if (holdUriHeader != null) {
            holdUri = holdUriHeader.getValue();
          }
        }
      }

      if (holdUri == null) {
        for (FlowFile flowFile : toSend) {
          flowFile = session.penalize(flowFile);
          logger.error(
              "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
              new Object[] {flowFile, url, responseCode, responseReason});
          session.transfer(flowFile, REL_FAILURE);
        }
        return;
      }
    }

    if (holdUri == null) {
      if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
        for (FlowFile flowFile : toSend) {
          flowFile = session.penalize(flowFile);
          logger.error(
              "Failed to Post {} to {}: response code was {}:{}; will yield processing, "
                  + "since the destination is temporarily unavailable",
              new Object[] {flowFile, url, responseCode, responseReason});
          session.transfer(flowFile, REL_FAILURE);
        }
        context.yield();
        return;
      }

      if (responseCode >= 300) {
        for (FlowFile flowFile : toSend) {
          flowFile = session.penalize(flowFile);
          logger.error(
              "Failed to Post {} to {}: response code was {}:{}",
              new Object[] {flowFile, url, responseCode, responseReason});
          session.transfer(flowFile, REL_FAILURE);
        }
        return;
      }

      logger.info(
          "Successfully Posted {} to {} in {} at a rate of {}",
          new Object[] {
            flowFileDescription,
            url,
            FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS),
            uploadDataRate
          });

      for (final FlowFile flowFile : toSend) {
        session
            .getProvenanceReporter()
            .send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true);
        session.transfer(flowFile, REL_SUCCESS);
      }
      return;
    }

    //
    // the response indicated a Hold URI; delete the Hold.
    //
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are
    // returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
      // If the Hold URI that we get starts with /contentListener, it may not really be
      // /contentListener,
      // as this really indicates that it should be whatever we posted to -- if posting directly to
      // the
      // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer,
      // we may
      // be posting to some other URL.
      fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
      // URL indicates the full path but not hostname or port; use the same hostname & port that we
      // posted
      // to but use the full path indicated by the response.
      int firstSlash = url.indexOf("/", 8);
      if (firstSlash < 0) {
        firstSlash = url.length();
      }
      final String beforeSlash = url.substring(0, firstSlash);
      fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
      // Absolute URL
      fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }

    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);

    while (true) {
      try {
        final HttpResponse holdResponse = client.execute(delete);
        EntityUtils.consume(holdResponse.getEntity());
        final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
        final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
        if (holdStatusCode >= 300) {
          logger.error(
              "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
              new Object[] {flowFileDescription, holdStatusCode, holdReason});

          for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
          }
          return;
        }

        logger.info(
            "Successfully Posted {} to {} in {} milliseconds at a rate of {}",
            new Object[] {flowFileDescription, url, uploadMillis, uploadDataRate});

        for (FlowFile flowFile : toSend) {
          session.getProvenanceReporter().send(flowFile, url);
          session.transfer(flowFile, REL_SUCCESS);
        }
        return;
      } catch (final IOException e) {
        logger.warn(
            "Failed to delete Hold that destination placed on {} due to {}",
            new Object[] {flowFileDescription, e});
      }

      if (!isScheduled()) {
        context.yield();
        logger.warn(
            "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
            new Object[] {flowFileDescription});
        for (FlowFile flowFile : toSend) {
          flowFile = session.penalize(flowFile);
          session.transfer(flowFile, REL_FAILURE);
        }
        return;
      }
    }
  }
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session)
      throws ProcessException {
    final ProcessorLog logger = getLogger();
    final SynchronousFileWatcher fileWatcher = fileWatcherRef.get();
    try {
      if (fileWatcher.checkAndReset()) {
        reloadDictionary(context, true, logger);
      }
    } catch (final IOException e) {
      throw new ProcessException(e);
    }

    Search<byte[]> search = searchRef.get();
    try {
      if (search == null) {
        if (reloadDictionary(context, false, logger)) {
          search = searchRef.get();
        }
      }
    } catch (final IOException e) {
      throw new ProcessException(e);
    }

    if (search == null) {
      return;
    }

    FlowFile flowFile = session.get();
    if (flowFile == null) {
      return;
    }

    final Search<byte[]> finalSearch = search;
    final ObjectHolder<SearchTerm<byte[]>> termRef = new ObjectHolder<>(null);
    termRef.set(null);

    session.read(
        flowFile,
        new InputStreamCallback() {
          @Override
          public void process(final InputStream rawIn) throws IOException {
            try (final InputStream in = new BufferedInputStream(rawIn)) {
              final SearchState<byte[]> searchResult = finalSearch.search(in, false);
              if (searchResult.foundMatch()) {
                termRef.set(searchResult.getResults().keySet().iterator().next());
              }
            }
          }
        });

    final SearchTerm<byte[]> matchingTerm = termRef.get();
    if (matchingTerm == null) {
      logger.info("Routing {} to 'unmatched'", new Object[] {flowFile});
      session.getProvenanceReporter().route(flowFile, REL_NO_MATCH);
      session.transfer(flowFile, REL_NO_MATCH);
    } else {
      final String matchingTermString = matchingTerm.toString(UTF8);
      logger.info(
          "Routing {} to 'matched' because it matched term {}",
          new Object[] {flowFile, matchingTermString});
      flowFile = session.putAttribute(flowFile, MATCH_ATTRIBUTE_KEY, matchingTermString);
      session.getProvenanceReporter().route(flowFile, REL_MATCH);
      session.transfer(flowFile, REL_MATCH);
    }
  }
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session)
      throws ProcessException {

    FlowFile flowFile = session.get();
    if (flowFile == null) {
      return;
    }

    final ProcessorLog logger = getLogger();

    // cache key is computed from attribute 'CACHE_ENTRY_IDENTIFIER' with expression language
    // support
    final String cacheKey =
        context
            .getProperty(CACHE_ENTRY_IDENTIFIER)
            .evaluateAttributeExpressions(flowFile)
            .getValue();

    // if the computed value is null, or empty, we transfer the flow file to failure relationship
    if (StringUtils.isBlank(cacheKey)) {
      logger.error(
          "FlowFile {} has no attribute for given Cache Entry Identifier", new Object[] {flowFile});
      flowFile = session.penalize(flowFile);
      session.transfer(flowFile, REL_FAILURE);
      return;
    }

    // the cache client used to interact with the distributed cache
    final DistributedMapCacheClient cache =
        context
            .getProperty(DISTRIBUTED_CACHE_SERVICE)
            .asControllerService(DistributedMapCacheClient.class);

    try {

      final long maxCacheEntrySize =
          context.getProperty(CACHE_ENTRY_MAX_BYTES).asDataSize(DataUnit.B).longValue();
      long flowFileSize = flowFile.getSize();

      // too big flow file
      if (flowFileSize > maxCacheEntrySize) {
        logger.warn(
            "Flow file {} size {} exceeds the max cache entry size ({} B).",
            new Object[] {flowFile, flowFileSize, maxCacheEntrySize});
        session.transfer(flowFile, REL_FAILURE);
        return;
      }

      if (flowFileSize == 0) {
        logger.warn("Flow file {} is empty, there is nothing to cache.", new Object[] {flowFile});
        session.transfer(flowFile, REL_FAILURE);
        return;
      }

      // get flow file content
      final ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
      session.exportTo(flowFile, byteStream);
      byte[] cacheValue = byteStream.toByteArray();
      final String updateStrategy = context.getProperty(CACHE_UPDATE_STRATEGY).getValue();
      boolean cached = false;

      if (updateStrategy.equals(CACHE_UPDATE_REPLACE.getValue())) {
        cache.put(cacheKey, cacheValue, keySerializer, valueSerializer);
        cached = true;
      } else if (updateStrategy.equals(CACHE_UPDATE_KEEP_ORIGINAL.getValue())) {
        final byte[] oldValue =
            cache.getAndPutIfAbsent(
                cacheKey, cacheValue, keySerializer, valueSerializer, valueDeserializer);
        if (oldValue == null) {
          cached = true;
        }
      }

      // set 'cached' attribute
      flowFile = session.putAttribute(flowFile, CACHED_ATTRIBUTE_NAME, String.valueOf(cached));

      if (cached) {
        session.transfer(flowFile, REL_SUCCESS);
      } else {
        session.transfer(flowFile, REL_FAILURE);
      }

    } catch (final IOException e) {
      flowFile = session.penalize(flowFile);
      session.transfer(flowFile, REL_FAILURE);
      logger.error(
          "Unable to communicate with cache when processing {} due to {}",
          new Object[] {flowFile, e});
    }
  }
  public void consume(
      final ProcessContext context,
      final ProcessSession session,
      final WrappedMessageConsumer wrappedConsumer)
      throws ProcessException {
    final ProcessorLog logger = getLogger();

    final MessageConsumer consumer = wrappedConsumer.getConsumer();
    final boolean clientAcknowledge =
        context.getProperty(ACKNOWLEDGEMENT_MODE).getValue().equalsIgnoreCase(ACK_MODE_CLIENT);
    final long timeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS);
    final boolean addAttributes = context.getProperty(JMS_PROPS_TO_ATTRIBUTES).asBoolean();
    final int batchSize = context.getProperty(BATCH_SIZE).asInteger();

    final JmsProcessingSummary processingSummary = new JmsProcessingSummary();

    final StopWatch stopWatch = new StopWatch(true);
    for (int i = 0; i < batchSize; i++) {

      final Message message;
      try {
        // If we haven't received a message, wait until one is available. If we have already
        // received at least one
        // message, then we are not willing to wait for more to become available, but we are willing
        // to keep receiving
        // all messages that are immediately available.
        if (processingSummary.getMessagesReceived() == 0) {
          message = consumer.receive(timeout);
        } else {
          message = consumer.receiveNoWait();
        }
      } catch (final JMSException e) {
        logger.error("Failed to receive JMS Message due to {}", e);
        wrappedConsumer.close(logger);
        break;
      }

      if (message == null) { // if no messages, we're done
        break;
      }

      try {
        processingSummary.add(map2FlowFile(context, session, message, addAttributes, logger));
      } catch (Exception e) {
        logger.error("Failed to receive JMS Message due to {}", e);
        wrappedConsumer.close(logger);
        break;
      }
    }

    if (processingSummary.getFlowFilesCreated() == 0) {
      context.yield();
      return;
    }

    session.commit();

    stopWatch.stop();
    if (processingSummary.getFlowFilesCreated() > 0) {
      final float secs = ((float) stopWatch.getDuration(TimeUnit.MILLISECONDS) / 1000F);
      float messagesPerSec = ((float) processingSummary.getMessagesReceived()) / secs;
      final String dataRate = stopWatch.calculateDataRate(processingSummary.getBytesReceived());
      logger.info(
          "Received {} messages in {} milliseconds, at a rate of {} messages/sec or {}",
          new Object[] {
            processingSummary.getMessagesReceived(),
            stopWatch.getDuration(TimeUnit.MILLISECONDS),
            messagesPerSec,
            dataRate
          });
    }

    // if we need to acknowledge the messages, do so now.
    final Message lastMessage = processingSummary.getLastMessageReceived();
    if (clientAcknowledge && lastMessage != null) {
      try {
        lastMessage
            .acknowledge(); // acknowledge all received messages by acknowledging only the last.
      } catch (final JMSException e) {
        logger.error(
            "Failed to acknowledge {} JMS Message(s). This may result in duplicate messages. Reason for failure: {}",
            new Object[] {processingSummary.getMessagesReceived(), e});
      }
    }
  }