Exemple #1
0
  @Override
  public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final ComponentLog logger = getLogger();
    final Criteria criteria = criteriaCache.get();

    List<FlowFile> flowFiles = session.get(100);
    if (flowFiles.isEmpty()) {
      return;
    }

    final Map<PropertyDescriptor, String> properties = context.getProperties();

    // get the default actions
    final Map<String, Action> defaultActions = getDefaultActions(properties);

    // record which rule should be applied to which flow file - when operating
    // in 'use clone' mode, this collection will contain a number of entries
    // that map to single element lists. this is because the original flowfile
    // is cloned for each matching rule. in 'use original' mode, this collection
    // will contain a single entry that maps a list of multiple rules. this is
    // because is the original flowfile is used for all matching rules. in this
    // case the order of the matching rules is perserved in the list
    final Map<FlowFile, List<Rule>> matchedRules = new HashMap<>();

    for (FlowFile flowFile : flowFiles) {
      matchedRules.clear();

      // if there is update criteria specified, evaluate it
      if (criteria != null
          && evaluateCriteria(session, context, criteria, flowFile, matchedRules)) {
        // apply the actions for each rule and transfer the flowfile
        for (final Map.Entry<FlowFile, List<Rule>> entry : matchedRules.entrySet()) {
          FlowFile match = entry.getKey();
          final List<Rule> rules = entry.getValue();

          // execute each matching rule(s)
          match = executeActions(session, context, rules, defaultActions, match);
          logger.info(
              "Updated attributes for {}; transferring to '{}'",
              new Object[] {match, REL_SUCCESS.getName()});

          // transfer the match
          session.getProvenanceReporter().modifyAttributes(match);
          session.transfer(match, REL_SUCCESS);
        }
      } else {
        // transfer the flowfile to no match (that has the default actions applied)
        flowFile = executeActions(session, context, null, defaultActions, flowFile);
        logger.info(
            "Updated attributes for {}; transferring to '{}'",
            new Object[] {flowFile, REL_SUCCESS.getName()});
        session.getProvenanceReporter().modifyAttributes(flowFile);
        session.transfer(flowFile, REL_SUCCESS);
      }
    }
  }
  private void addProcessor(final Element parentElement, final ProcessorNode processor) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement("processor");
    parentElement.appendChild(element);
    addTextElement(element, "id", processor.getIdentifier());
    addTextElement(element, "name", processor.getName());

    addPosition(element, processor.getPosition());
    addStyle(element, processor.getStyle());

    addTextElement(element, "comment", processor.getComments());
    addTextElement(element, "class", processor.getProcessor().getClass().getCanonicalName());
    addTextElement(element, "maxConcurrentTasks", processor.getMaxConcurrentTasks());
    addTextElement(element, "schedulingPeriod", processor.getSchedulingPeriod());
    addTextElement(element, "penalizationPeriod", processor.getPenalizationPeriod());
    addTextElement(element, "yieldPeriod", processor.getYieldPeriod());
    addTextElement(element, "bulletinLevel", processor.getBulletinLevel().toString());
    addTextElement(element, "lossTolerant", String.valueOf(processor.isLossTolerant()));
    addTextElement(element, "scheduledState", processor.getScheduledState().name());
    addTextElement(element, "schedulingStrategy", processor.getSchedulingStrategy().name());
    addTextElement(element, "runDurationNanos", processor.getRunDuration(TimeUnit.NANOSECONDS));

    addConfiguration(element, processor.getProperties(), processor.getAnnotationData(), encryptor);

    for (final Relationship rel : processor.getAutoTerminatedRelationships()) {
      addTextElement(element, "autoTerminatedRelationship", rel.getName());
    }
  }
  @Override
  public void addConnection(final Connection connection) {
    Objects.requireNonNull(connection, "connection cannot be null");

    if (!connection.getSource().equals(this) && !connection.getDestination().equals(this)) {
      throw new IllegalStateException(
          "Cannot a connection to a ProcessorNode for which the ProcessorNode is neither the Source nor the Destination");
    }

    writeLock.lock();
    try {
      List<Connection> updatedIncoming = null;
      if (connection.getDestination().equals(this)) {
        // don't add the connection twice. This may occur if we have a self-loop because we will be
        // told
        // to add the connection once because we are the source and again because we are the
        // destination.
        final List<Connection> incomingConnections = incomingConnectionsRef.get();
        updatedIncoming = new ArrayList<>(incomingConnections);
        if (!updatedIncoming.contains(connection)) {
          updatedIncoming.add(connection);
        }
      }

      if (connection.getSource().equals(this)) {
        // don't add the connection twice. This may occur if we have a self-loop because we will be
        // told
        // to add the connection once because we are the source and again because we are the
        // destination.
        if (!destinations.containsKey(connection)) {
          for (final Relationship relationship : connection.getRelationships()) {
            final Relationship rel = getRelationship(relationship.getName());
            Set<Connection> set = connections.get(rel);
            if (set == null) {
              set = new HashSet<>();
              connections.put(rel, set);
            }

            set.add(connection);

            destinations.put(connection, connection.getDestination());
          }

          final Set<Relationship> autoTerminated = this.undefinedRelationshipsToTerminate.get();
          if (autoTerminated != null) {
            autoTerminated.removeAll(connection.getRelationships());
            this.undefinedRelationshipsToTerminate.set(autoTerminated);
          }
        }
      }

      if (updatedIncoming != null) {
        incomingConnectionsRef.set(Collections.unmodifiableList(updatedIncoming));
      }
    } finally {
      writeLock.unlock();
    }
  }
  @Override
  public void setAutoTerminatedRelationships(final Set<Relationship> terminate) {
    writeLock.lock();
    try {
      if (isRunning()) {
        throw new IllegalStateException(
            "Cannot modify Processor configuration while the Processor is running");
      }

      for (final Relationship rel : terminate) {
        if (!getConnections(rel).isEmpty()) {
          throw new IllegalStateException(
              "Cannot mark relationship '"
                  + rel.getName()
                  + "' as auto-terminated because Connection already exists with this relationship");
        }
      }
      undefinedRelationshipsToTerminate.set(new HashSet<>(terminate));
    } finally {
      writeLock.unlock();
    }
  }
  private void addConnection(final Element parentElement, final Connection connection) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement("connection");
    parentElement.appendChild(element);
    addTextElement(element, "id", connection.getIdentifier());
    addTextElement(element, "name", connection.getName());

    final Element bendPointsElement = doc.createElement("bendPoints");
    element.appendChild(bendPointsElement);
    for (final Position bendPoint : connection.getBendPoints()) {
      addPosition(bendPointsElement, bendPoint, "bendPoint");
    }

    addTextElement(element, "labelIndex", connection.getLabelIndex());
    addTextElement(element, "zIndex", connection.getZIndex());

    final String sourceId = connection.getSource().getIdentifier();
    final ConnectableType sourceType = connection.getSource().getConnectableType();
    final String sourceGroupId;
    if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) {
      sourceGroupId =
          ((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier();
    } else {
      sourceGroupId = connection.getSource().getProcessGroup().getIdentifier();
    }

    final ConnectableType destinationType = connection.getDestination().getConnectableType();
    final String destinationId = connection.getDestination().getIdentifier();
    final String destinationGroupId;
    if (destinationType == ConnectableType.REMOTE_INPUT_PORT) {
      destinationGroupId =
          ((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier();
    } else {
      destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier();
    }

    addTextElement(element, "sourceId", sourceId);
    addTextElement(element, "sourceGroupId", sourceGroupId);
    addTextElement(element, "sourceType", sourceType.toString());

    addTextElement(element, "destinationId", destinationId);
    addTextElement(element, "destinationGroupId", destinationGroupId);
    addTextElement(element, "destinationType", destinationType.toString());

    for (final Relationship relationship : connection.getRelationships()) {
      addTextElement(element, "relationship", relationship.getName());
    }

    addTextElement(
        element,
        "maxWorkQueueSize",
        connection.getFlowFileQueue().getBackPressureObjectThreshold());
    addTextElement(
        element,
        "maxWorkQueueDataSize",
        connection.getFlowFileQueue().getBackPressureDataSizeThreshold());

    addTextElement(
        element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration());
    for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
      final String className = comparator.getClass().getCanonicalName();
      addTextElement(element, "queuePrioritizerClass", className);
    }

    parentElement.appendChild(element);
  }
Exemple #6
0
  /** Extracts the values for the configured properties from the specified Processor. */
  private Map<String, String> extractConfiguredPropertyValues(
      ProcessorNode processor, ProcessorDTO processorDTO) {
    Map<String, String> values = new HashMap<>();

    if (processorDTO.getName() != null) {
      values.put(NAME, processor.getName());
    }
    if (processorDTO.getConfig() != null) {
      ProcessorConfigDTO newConfig = processorDTO.getConfig();
      if (newConfig.getConcurrentlySchedulableTaskCount() != null) {
        values.put(
            CONCURRENTLY_SCHEDULABLE_TASKS, String.valueOf(processor.getMaxConcurrentTasks()));
      }
      if (newConfig.getPenaltyDuration() != null) {
        values.put(PENALTY_DURATION, processor.getPenalizationPeriod());
      }
      if (newConfig.getYieldDuration() != null) {
        values.put(YIELD_DURATION, processor.getYieldPeriod());
      }
      if (newConfig.getBulletinLevel() != null) {
        values.put(BULLETIN_LEVEL, processor.getBulletinLevel().name());
      }
      if (newConfig.getAnnotationData() != null) {
        values.put(ANNOTATION_DATA, processor.getAnnotationData());
      }
      if (newConfig.getSchedulingPeriod() != null) {
        values.put(SCHEDULING_PERIOD, String.valueOf(processor.getSchedulingPeriod()));
      }
      if (newConfig.getAutoTerminatedRelationships() != null) {
        // get each of the auto terminated relationship names
        final Set<Relationship> autoTerminatedRelationships =
            processor.getAutoTerminatedRelationships();
        final List<String> autoTerminatedRelationshipNames =
            new ArrayList<>(autoTerminatedRelationships.size());
        for (final Relationship relationship : autoTerminatedRelationships) {
          autoTerminatedRelationshipNames.add(relationship.getName());
        }

        // sort them and include in the configuration
        Collections.sort(autoTerminatedRelationshipNames, Collator.getInstance(Locale.US));
        values.put(
            AUTO_TERMINATED_RELATIONSHIPS, StringUtils.join(autoTerminatedRelationshipNames, ", "));
      }
      if (newConfig.getProperties() != null) {
        // for each property specified, extract its configured value
        Map<String, String> properties = newConfig.getProperties();
        Map<PropertyDescriptor, String> configuredProperties = processor.getProperties();
        for (String propertyName : properties.keySet()) {
          // build a descriptor for getting the configured value
          PropertyDescriptor propertyDescriptor =
              new PropertyDescriptor.Builder().name(propertyName).build();
          String configuredPropertyValue = configuredProperties.get(propertyDescriptor);

          // if the configured value couldn't be found, use the default value from the actual
          // descriptor
          if (configuredPropertyValue == null) {
            propertyDescriptor =
                locatePropertyDescriptor(configuredProperties.keySet(), propertyDescriptor);
            configuredPropertyValue = propertyDescriptor.getDefaultValue();
          }
          values.put(propertyName, configuredPropertyValue);
        }
      }
      if (newConfig.getComments() != null) {
        values.put(COMMENTS, processor.getComments());
      }
      if (newConfig.getSchedulingStrategy() != null) {
        values.put(SCHEDULING_STRATEGY, processor.getSchedulingStrategy().toString());
      }
    }

    return values;
  }
  @Override
  public void updateConnection(final Connection connection) throws IllegalStateException {
    if (requireNonNull(connection).getSource().equals(this)) {
      writeLock.lock();
      try {
        //
        // update any relationships
        //
        // first check if any relations were removed.
        final List<Relationship> existingRelationships = new ArrayList<>();
        for (final Map.Entry<Relationship, Set<Connection>> entry : connections.entrySet()) {
          if (entry.getValue().contains(connection)) {
            existingRelationships.add(entry.getKey());
          }
        }

        for (final Relationship rel : connection.getRelationships()) {
          if (!existingRelationships.contains(rel)) {
            // relationship was removed. Check if this is legal.
            final Set<Connection> connectionsForRelationship = getConnections(rel);
            if (connectionsForRelationship != null
                && connectionsForRelationship.size() == 1
                && this.isRunning()
                && !isAutoTerminated(rel)
                && getRelationships().contains(rel)) {
              // if we are running and we do not terminate undefined relationships and this is the
              // only
              // connection that defines the given relationship, and that relationship is required,
              // then it is not legal to remove this relationship from this connection.
              throw new IllegalStateException(
                  "Cannot remove relationship "
                      + rel.getName()
                      + " from Connection because doing so would invalidate Processor "
                      + this
                      + ", which is currently running");
            }
          }
        }

        // remove the connection from any list that currently contains
        for (final Set<Connection> list : connections.values()) {
          list.remove(connection);
        }

        // add the connection in for all relationships listed.
        for (final Relationship rel : connection.getRelationships()) {
          Set<Connection> set = connections.get(rel);
          if (set == null) {
            set = new HashSet<>();
            connections.put(rel, set);
          }
          set.add(connection);
        }

        // update to the new destination
        destinations.put(connection, connection.getDestination());

        final Set<Relationship> autoTerminated = this.undefinedRelationshipsToTerminate.get();
        if (autoTerminated != null) {
          autoTerminated.removeAll(connection.getRelationships());
          this.undefinedRelationshipsToTerminate.set(autoTerminated);
        }
      } finally {
        writeLock.unlock();
      }
    }

    if (connection.getDestination().equals(this)) {
      writeLock.lock();
      try {
        // update our incoming connections -- we can just remove & re-add the connection to
        // update the list.
        final List<Connection> incomingConnections = incomingConnectionsRef.get();
        final List<Connection> updatedIncoming = new ArrayList<>(incomingConnections);
        updatedIncoming.remove(connection);
        updatedIncoming.add(connection);
        incomingConnectionsRef.set(Collections.unmodifiableList(updatedIncoming));
      } finally {
        writeLock.unlock();
      }
    }
  }
  @Override
  public Collection<ValidationResult> getValidationErrors() {
    final List<ValidationResult> results = new ArrayList<>();
    readLock.lock();
    try {
      final ValidationContext validationContext =
          validationContextFactory.newValidationContext(getProperties(), getAnnotationData());

      final Collection<ValidationResult> validationResults;
      try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
        validationResults = getProcessor().validate(validationContext);
      }

      for (final ValidationResult result : validationResults) {
        if (!result.isValid()) {
          results.add(result);
        }
      }

      for (final Relationship relationship : getUndefinedRelationships()) {
        if (!isAutoTerminated(relationship)) {
          final ValidationResult error =
              new ValidationResult.Builder()
                  .explanation(
                      "Relationship '"
                          + relationship.getName()
                          + "' is not connected to any component and is not auto-terminated")
                  .subject("Relationship " + relationship.getName())
                  .valid(false)
                  .build();
          results.add(error);
        }
      }

      switch (getInputRequirement()) {
        case INPUT_ALLOWED:
          break;
        case INPUT_FORBIDDEN:
          {
            final int incomingConnCount = getIncomingNonLoopConnections().size();
            if (incomingConnCount != 0) {
              results.add(
                  new ValidationResult.Builder()
                      .explanation(
                          "Processor does not allow upstream connections but currently has "
                              + incomingConnCount)
                      .subject("Upstream Connections")
                      .valid(false)
                      .build());
            }
            break;
          }
        case INPUT_REQUIRED:
          {
            if (getIncomingNonLoopConnections().isEmpty()) {
              results.add(
                  new ValidationResult.Builder()
                      .explanation(
                          "Processor requires an upstream connection but currently has none")
                      .subject("Upstream Connections")
                      .valid(false)
                      .build());
            }
            break;
          }
      }
    } catch (final Throwable t) {
      results.add(
          new ValidationResult.Builder()
              .explanation("Failed to run validation due to " + t.toString())
              .valid(false)
              .build());
    } finally {
      readLock.unlock();
    }
    return results;
  }