private void addProcessor(final Element parentElement, final ProcessorNode processor) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement("processor");
parentElement.appendChild(element);
addTextElement(element, "id", processor.getIdentifier());
addTextElement(element, "name", processor.getName());
addPosition(element, processor.getPosition());
addStyle(element, processor.getStyle());
addTextElement(element, "comment", processor.getComments());
addTextElement(element, "class", processor.getProcessor().getClass().getCanonicalName());
addTextElement(element, "maxConcurrentTasks", processor.getMaxConcurrentTasks());
addTextElement(element, "schedulingPeriod", processor.getSchedulingPeriod());
addTextElement(element, "penalizationPeriod", processor.getPenalizationPeriod());
addTextElement(element, "yieldPeriod", processor.getYieldPeriod());
addTextElement(element, "bulletinLevel", processor.getBulletinLevel().toString());
addTextElement(element, "lossTolerant", String.valueOf(processor.isLossTolerant()));
addTextElement(element, "scheduledState", processor.getScheduledState().name());
addTextElement(element, "schedulingStrategy", processor.getSchedulingStrategy().name());
addTextElement(element, "runDurationNanos", processor.getRunDuration(TimeUnit.NANOSECONDS));
addConfiguration(element, processor.getProperties(), processor.getAnnotationData(), encryptor);
for (final Relationship rel : processor.getAutoTerminatedRelationships()) {
addTextElement(element, "autoTerminatedRelationship", rel.getName());
}
}
@Override
public void addConnection(final Connection connection) {
Objects.requireNonNull(connection, "connection cannot be null");
if (!connection.getSource().equals(this) && !connection.getDestination().equals(this)) {
throw new IllegalStateException(
"Cannot a connection to a ProcessorNode for which the ProcessorNode is neither the Source nor the Destination");
}
writeLock.lock();
try {
List<Connection> updatedIncoming = null;
if (connection.getDestination().equals(this)) {
// don't add the connection twice. This may occur if we have a self-loop because we will be
// told
// to add the connection once because we are the source and again because we are the
// destination.
final List<Connection> incomingConnections = incomingConnectionsRef.get();
updatedIncoming = new ArrayList<>(incomingConnections);
if (!updatedIncoming.contains(connection)) {
updatedIncoming.add(connection);
}
}
if (connection.getSource().equals(this)) {
// don't add the connection twice. This may occur if we have a self-loop because we will be
// told
// to add the connection once because we are the source and again because we are the
// destination.
if (!destinations.containsKey(connection)) {
for (final Relationship relationship : connection.getRelationships()) {
final Relationship rel = getRelationship(relationship.getName());
Set<Connection> set = connections.get(rel);
if (set == null) {
set = new HashSet<>();
connections.put(rel, set);
}
set.add(connection);
destinations.put(connection, connection.getDestination());
}
final Set<Relationship> autoTerminated = this.undefinedRelationshipsToTerminate.get();
if (autoTerminated != null) {
autoTerminated.removeAll(connection.getRelationships());
this.undefinedRelationshipsToTerminate.set(autoTerminated);
}
}
}
if (updatedIncoming != null) {
incomingConnectionsRef.set(Collections.unmodifiableList(updatedIncoming));
}
} finally {
writeLock.unlock();
}
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
final ComponentLog logger = getLogger();
final Criteria criteria = criteriaCache.get();
List<FlowFile> flowFiles = session.get(100);
if (flowFiles.isEmpty()) {
return;
}
final Map<PropertyDescriptor, String> properties = context.getProperties();
// get the default actions
final Map<String, Action> defaultActions = getDefaultActions(properties);
// record which rule should be applied to which flow file - when operating
// in 'use clone' mode, this collection will contain a number of entries
// that map to single element lists. this is because the original flowfile
// is cloned for each matching rule. in 'use original' mode, this collection
// will contain a single entry that maps a list of multiple rules. this is
// because is the original flowfile is used for all matching rules. in this
// case the order of the matching rules is perserved in the list
final Map<FlowFile, List<Rule>> matchedRules = new HashMap<>();
for (FlowFile flowFile : flowFiles) {
matchedRules.clear();
// if there is update criteria specified, evaluate it
if (criteria != null
&& evaluateCriteria(session, context, criteria, flowFile, matchedRules)) {
// apply the actions for each rule and transfer the flowfile
for (final Map.Entry<FlowFile, List<Rule>> entry : matchedRules.entrySet()) {
FlowFile match = entry.getKey();
final List<Rule> rules = entry.getValue();
// execute each matching rule(s)
match = executeActions(session, context, rules, defaultActions, match);
logger.info(
"Updated attributes for {}; transferring to '{}'",
new Object[] {match, REL_SUCCESS.getName()});
// transfer the match
session.getProvenanceReporter().modifyAttributes(match);
session.transfer(match, REL_SUCCESS);
}
} else {
// transfer the flowfile to no match (that has the default actions applied)
flowFile = executeActions(session, context, null, defaultActions, flowFile);
logger.info(
"Updated attributes for {}; transferring to '{}'",
new Object[] {flowFile, REL_SUCCESS.getName()});
session.getProvenanceReporter().modifyAttributes(flowFile);
session.transfer(flowFile, REL_SUCCESS);
}
}
}
/**
* Asserts that all FlowFiles that were transferred were transferred to the given relationship
*
* @param relationship to validate
*/
public void assertAllFlowFilesTransferred(final Relationship relationship) {
for (final Map.Entry<Relationship, List<MockFlowFile>> entry : transferMap.entrySet()) {
final Relationship rel = entry.getKey();
final List<MockFlowFile> flowFiles = entry.getValue();
if (!rel.equals(relationship) && flowFiles != null && !flowFiles.isEmpty()) {
Assert.fail(
"Expected all Transferred FlowFiles to go to "
+ relationship
+ " but "
+ flowFiles.size()
+ " were routed to "
+ rel);
}
}
}
/**
* @param relationshipName name
* @return the relationship for this nodes processor for the given name or creates a new
* relationship for the given name
*/
@Override
public Relationship getRelationship(final String relationshipName) {
final Relationship specRel = new Relationship.Builder().name(relationshipName).build();
Relationship returnRel = specRel;
final Set<Relationship> relationships;
try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
relationships = processor.getRelationships();
}
for (final Relationship rel : relationships) {
if (rel.equals(specRel)) {
returnRel = rel;
break;
}
}
return returnRel;
}
@Override
public void setAutoTerminatedRelationships(final Set<Relationship> terminate) {
writeLock.lock();
try {
if (isRunning()) {
throw new IllegalStateException(
"Cannot modify Processor configuration while the Processor is running");
}
for (final Relationship rel : terminate) {
if (!getConnections(rel).isEmpty()) {
throw new IllegalStateException(
"Cannot mark relationship '"
+ rel.getName()
+ "' as auto-terminated because Connection already exists with this relationship");
}
}
undefinedRelationshipsToTerminate.set(new HashSet<>(terminate));
} finally {
writeLock.unlock();
}
}
private void addConnection(final Element parentElement, final Connection connection) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement("connection");
parentElement.appendChild(element);
addTextElement(element, "id", connection.getIdentifier());
addTextElement(element, "name", connection.getName());
final Element bendPointsElement = doc.createElement("bendPoints");
element.appendChild(bendPointsElement);
for (final Position bendPoint : connection.getBendPoints()) {
addPosition(bendPointsElement, bendPoint, "bendPoint");
}
addTextElement(element, "labelIndex", connection.getLabelIndex());
addTextElement(element, "zIndex", connection.getZIndex());
final String sourceId = connection.getSource().getIdentifier();
final ConnectableType sourceType = connection.getSource().getConnectableType();
final String sourceGroupId;
if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) {
sourceGroupId =
((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier();
} else {
sourceGroupId = connection.getSource().getProcessGroup().getIdentifier();
}
final ConnectableType destinationType = connection.getDestination().getConnectableType();
final String destinationId = connection.getDestination().getIdentifier();
final String destinationGroupId;
if (destinationType == ConnectableType.REMOTE_INPUT_PORT) {
destinationGroupId =
((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier();
} else {
destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier();
}
addTextElement(element, "sourceId", sourceId);
addTextElement(element, "sourceGroupId", sourceGroupId);
addTextElement(element, "sourceType", sourceType.toString());
addTextElement(element, "destinationId", destinationId);
addTextElement(element, "destinationGroupId", destinationGroupId);
addTextElement(element, "destinationType", destinationType.toString());
for (final Relationship relationship : connection.getRelationships()) {
addTextElement(element, "relationship", relationship.getName());
}
addTextElement(
element,
"maxWorkQueueSize",
connection.getFlowFileQueue().getBackPressureObjectThreshold());
addTextElement(
element,
"maxWorkQueueDataSize",
connection.getFlowFileQueue().getBackPressureDataSizeThreshold());
addTextElement(
element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration());
for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
final String className = comparator.getClass().getCanonicalName();
addTextElement(element, "queuePrioritizerClass", className);
}
parentElement.appendChild(element);
}
/** Extracts the values for the configured properties from the specified Processor. */
private Map<String, String> extractConfiguredPropertyValues(
ProcessorNode processor, ProcessorDTO processorDTO) {
Map<String, String> values = new HashMap<>();
if (processorDTO.getName() != null) {
values.put(NAME, processor.getName());
}
if (processorDTO.getConfig() != null) {
ProcessorConfigDTO newConfig = processorDTO.getConfig();
if (newConfig.getConcurrentlySchedulableTaskCount() != null) {
values.put(
CONCURRENTLY_SCHEDULABLE_TASKS, String.valueOf(processor.getMaxConcurrentTasks()));
}
if (newConfig.getPenaltyDuration() != null) {
values.put(PENALTY_DURATION, processor.getPenalizationPeriod());
}
if (newConfig.getYieldDuration() != null) {
values.put(YIELD_DURATION, processor.getYieldPeriod());
}
if (newConfig.getBulletinLevel() != null) {
values.put(BULLETIN_LEVEL, processor.getBulletinLevel().name());
}
if (newConfig.getAnnotationData() != null) {
values.put(ANNOTATION_DATA, processor.getAnnotationData());
}
if (newConfig.getSchedulingPeriod() != null) {
values.put(SCHEDULING_PERIOD, String.valueOf(processor.getSchedulingPeriod()));
}
if (newConfig.getAutoTerminatedRelationships() != null) {
// get each of the auto terminated relationship names
final Set<Relationship> autoTerminatedRelationships =
processor.getAutoTerminatedRelationships();
final List<String> autoTerminatedRelationshipNames =
new ArrayList<>(autoTerminatedRelationships.size());
for (final Relationship relationship : autoTerminatedRelationships) {
autoTerminatedRelationshipNames.add(relationship.getName());
}
// sort them and include in the configuration
Collections.sort(autoTerminatedRelationshipNames, Collator.getInstance(Locale.US));
values.put(
AUTO_TERMINATED_RELATIONSHIPS, StringUtils.join(autoTerminatedRelationshipNames, ", "));
}
if (newConfig.getProperties() != null) {
// for each property specified, extract its configured value
Map<String, String> properties = newConfig.getProperties();
Map<PropertyDescriptor, String> configuredProperties = processor.getProperties();
for (String propertyName : properties.keySet()) {
// build a descriptor for getting the configured value
PropertyDescriptor propertyDescriptor =
new PropertyDescriptor.Builder().name(propertyName).build();
String configuredPropertyValue = configuredProperties.get(propertyDescriptor);
// if the configured value couldn't be found, use the default value from the actual
// descriptor
if (configuredPropertyValue == null) {
propertyDescriptor =
locatePropertyDescriptor(configuredProperties.keySet(), propertyDescriptor);
configuredPropertyValue = propertyDescriptor.getDefaultValue();
}
values.put(propertyName, configuredPropertyValue);
}
}
if (newConfig.getComments() != null) {
values.put(COMMENTS, processor.getComments());
}
if (newConfig.getSchedulingStrategy() != null) {
values.put(SCHEDULING_STRATEGY, processor.getSchedulingStrategy().toString());
}
}
return values;
}
@Override
public void updateConnection(final Connection connection) throws IllegalStateException {
if (requireNonNull(connection).getSource().equals(this)) {
writeLock.lock();
try {
//
// update any relationships
//
// first check if any relations were removed.
final List<Relationship> existingRelationships = new ArrayList<>();
for (final Map.Entry<Relationship, Set<Connection>> entry : connections.entrySet()) {
if (entry.getValue().contains(connection)) {
existingRelationships.add(entry.getKey());
}
}
for (final Relationship rel : connection.getRelationships()) {
if (!existingRelationships.contains(rel)) {
// relationship was removed. Check if this is legal.
final Set<Connection> connectionsForRelationship = getConnections(rel);
if (connectionsForRelationship != null
&& connectionsForRelationship.size() == 1
&& this.isRunning()
&& !isAutoTerminated(rel)
&& getRelationships().contains(rel)) {
// if we are running and we do not terminate undefined relationships and this is the
// only
// connection that defines the given relationship, and that relationship is required,
// then it is not legal to remove this relationship from this connection.
throw new IllegalStateException(
"Cannot remove relationship "
+ rel.getName()
+ " from Connection because doing so would invalidate Processor "
+ this
+ ", which is currently running");
}
}
}
// remove the connection from any list that currently contains
for (final Set<Connection> list : connections.values()) {
list.remove(connection);
}
// add the connection in for all relationships listed.
for (final Relationship rel : connection.getRelationships()) {
Set<Connection> set = connections.get(rel);
if (set == null) {
set = new HashSet<>();
connections.put(rel, set);
}
set.add(connection);
}
// update to the new destination
destinations.put(connection, connection.getDestination());
final Set<Relationship> autoTerminated = this.undefinedRelationshipsToTerminate.get();
if (autoTerminated != null) {
autoTerminated.removeAll(connection.getRelationships());
this.undefinedRelationshipsToTerminate.set(autoTerminated);
}
} finally {
writeLock.unlock();
}
}
if (connection.getDestination().equals(this)) {
writeLock.lock();
try {
// update our incoming connections -- we can just remove & re-add the connection to
// update the list.
final List<Connection> incomingConnections = incomingConnectionsRef.get();
final List<Connection> updatedIncoming = new ArrayList<>(incomingConnections);
updatedIncoming.remove(connection);
updatedIncoming.add(connection);
incomingConnectionsRef.set(Collections.unmodifiableList(updatedIncoming));
} finally {
writeLock.unlock();
}
}
}
@Override
public Collection<ValidationResult> getValidationErrors() {
final List<ValidationResult> results = new ArrayList<>();
readLock.lock();
try {
final ValidationContext validationContext =
validationContextFactory.newValidationContext(getProperties(), getAnnotationData());
final Collection<ValidationResult> validationResults;
try (final NarCloseable narCloseable = NarCloseable.withNarLoader()) {
validationResults = getProcessor().validate(validationContext);
}
for (final ValidationResult result : validationResults) {
if (!result.isValid()) {
results.add(result);
}
}
for (final Relationship relationship : getUndefinedRelationships()) {
if (!isAutoTerminated(relationship)) {
final ValidationResult error =
new ValidationResult.Builder()
.explanation(
"Relationship '"
+ relationship.getName()
+ "' is not connected to any component and is not auto-terminated")
.subject("Relationship " + relationship.getName())
.valid(false)
.build();
results.add(error);
}
}
switch (getInputRequirement()) {
case INPUT_ALLOWED:
break;
case INPUT_FORBIDDEN:
{
final int incomingConnCount = getIncomingNonLoopConnections().size();
if (incomingConnCount != 0) {
results.add(
new ValidationResult.Builder()
.explanation(
"Processor does not allow upstream connections but currently has "
+ incomingConnCount)
.subject("Upstream Connections")
.valid(false)
.build());
}
break;
}
case INPUT_REQUIRED:
{
if (getIncomingNonLoopConnections().isEmpty()) {
results.add(
new ValidationResult.Builder()
.explanation(
"Processor requires an upstream connection but currently has none")
.subject("Upstream Connections")
.valid(false)
.build());
}
break;
}
}
} catch (final Throwable t) {
results.add(
new ValidationResult.Builder()
.explanation("Failed to run validation due to " + t.toString())
.valid(false)
.build());
} finally {
readLock.unlock();
}
return results;
}