private static void addConfiguration(
final Element element,
final Map<PropertyDescriptor, String> properties,
final String annotationData,
final StringEncryptor encryptor) {
final Document doc = element.getOwnerDocument();
for (final Map.Entry<PropertyDescriptor, String> entry : properties.entrySet()) {
final PropertyDescriptor descriptor = entry.getKey();
String value = entry.getValue();
if (value != null && descriptor.isSensitive()) {
value = ENC_PREFIX + encryptor.encrypt(value) + ENC_SUFFIX;
}
if (value == null) {
value = descriptor.getDefaultValue();
}
final Element propElement = doc.createElement("property");
addTextElement(propElement, "name", descriptor.getName());
if (value != null) {
addTextElement(propElement, "value", value);
}
element.appendChild(propElement);
}
if (annotationData != null) {
addTextElement(element, "annotationData", annotationData);
}
}
/**
* Locates the actual property descriptor for the given spec property descriptor.
*
* @param propertyDescriptors descriptors
* @param specDescriptor example descriptor
* @return property
*/
private PropertyDescriptor locatePropertyDescriptor(
Set<PropertyDescriptor> propertyDescriptors, PropertyDescriptor specDescriptor) {
for (PropertyDescriptor propertyDescriptor : propertyDescriptors) {
if (propertyDescriptor.equals(specDescriptor)) {
return propertyDescriptor;
}
}
return specDescriptor;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
final long startNanos = System.nanoTime();
final AmazonSQSClient client = getClient();
final SendMessageBatchRequest request = new SendMessageBatchRequest();
final String queueUrl =
context.getProperty(QUEUE_URL).evaluateAttributeExpressions(flowFile).getValue();
request.setQueueUrl(queueUrl);
final Set<SendMessageBatchRequestEntry> entries = new HashSet<>();
final SendMessageBatchRequestEntry entry = new SendMessageBatchRequestEntry();
entry.setId(flowFile.getAttribute("uuid"));
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
session.exportTo(flowFile, baos);
final String flowFileContent = baos.toString();
entry.setMessageBody(flowFileContent);
final Map<String, MessageAttributeValue> messageAttributes = new HashMap<>();
for (final PropertyDescriptor descriptor : userDefinedProperties) {
final MessageAttributeValue mav = new MessageAttributeValue();
mav.setDataType("String");
mav.setStringValue(
context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue());
messageAttributes.put(descriptor.getName(), mav);
}
entry.setMessageAttributes(messageAttributes);
entry.setDelaySeconds(context.getProperty(DELAY).asTimePeriod(TimeUnit.SECONDS).intValue());
entries.add(entry);
request.setEntries(entries);
try {
client.sendMessageBatch(request);
} catch (final Exception e) {
getLogger()
.error(
"Failed to send messages to Amazon SQS due to {}; routing to failure",
new Object[] {e});
flowFile = session.penalize(flowFile);
session.transfer(flowFile, REL_FAILURE);
return;
}
getLogger()
.info("Successfully published message to Amazon SQS for {}", new Object[] {flowFile});
session.transfer(flowFile, REL_SUCCESS);
final long transmissionMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
session.getProvenanceReporter().send(flowFile, queueUrl, transmissionMillis);
}
@OnScheduled
public void setup(final ProcessContext context) {
userDefinedProperties = new ArrayList<>();
for (final PropertyDescriptor descriptor : context.getProperties().keySet()) {
if (descriptor.isDynamic()) {
userDefinedProperties.add(descriptor);
}
}
}
@OnScheduled
public void setup(final ConfigurationContext context) {
metricsService = getMetricsService();
ddMetricRegistryBuilder = getMetricRegistryBuilder();
metricRegistry = getMetricRegistry();
metricsMap = getMetricsMap();
metricsPrefix = METRICS_PREFIX.getDefaultValue();
environment = ENVIRONMENT.getDefaultValue();
virtualMachineMetrics = VirtualMachineMetrics.getInstance();
ddMetricRegistryBuilder
.setMetricRegistry(metricRegistry)
.setTags(metricsService.getAllTagsList());
}
@Override
public ValidationResult setProperty(
final ControllerService service, final PropertyDescriptor property, final String value) {
final MockStateManager serviceStateManager =
controllerServiceStateManagers.get(service.getIdentifier());
if (serviceStateManager == null) {
throw new IllegalStateException(
"Controller service "
+ service
+ " has not been added to this TestRunner via the #addControllerService method");
}
final ControllerServiceConfiguration configuration = getConfigToUpdate(service);
final Map<PropertyDescriptor, String> curProps = configuration.getProperties();
final Map<PropertyDescriptor, String> updatedProps = new HashMap<>(curProps);
final ValidationContext validationContext =
new MockValidationContext(context, serviceStateManager)
.getControllerServiceValidationContext(service);
final ValidationResult validationResult = property.validate(value, validationContext);
updatedProps.put(property, value);
configuration.setProperties(updatedProps);
return validationResult;
}
@Override
public void onPropertyModified(
final PropertyDescriptor descriptor, final String oldValue, final String newValue) {
if (descriptor.equals(DICTIONARY)) {
fileWatcherRef.set(
new SynchronousFileWatcher(Paths.get(newValue), new LastModifiedMonitor(), 60000L));
}
}
/**
* Extracts the values for the configured properties from the specified ControllerService.
*
* @param controllerService service
* @param controllerServiceDTO dto
* @return properties
*/
private Map<String, String> extractConfiguredPropertyValues(
ControllerServiceNode controllerService, ControllerServiceDTO controllerServiceDTO) {
Map<String, String> values = new HashMap<>();
if (controllerServiceDTO.getName() != null) {
values.put(NAME, controllerService.getName());
}
if (controllerServiceDTO.getAnnotationData() != null) {
values.put(ANNOTATION_DATA, controllerService.getAnnotationData());
}
if (controllerServiceDTO.getProperties() != null) {
// for each property specified, extract its configured value
Map<String, String> properties = controllerServiceDTO.getProperties();
Map<PropertyDescriptor, String> configuredProperties = controllerService.getProperties();
for (String propertyName : properties.keySet()) {
// build a descriptor for getting the configured value
PropertyDescriptor propertyDescriptor =
new PropertyDescriptor.Builder().name(propertyName).build();
String configuredPropertyValue = configuredProperties.get(propertyDescriptor);
// if the configured value couldn't be found, use the default value from the actual
// descriptor
if (configuredPropertyValue == null) {
propertyDescriptor =
locatePropertyDescriptor(configuredProperties.keySet(), propertyDescriptor);
configuredPropertyValue = propertyDescriptor.getDefaultValue();
}
values.put(propertyName, configuredPropertyValue);
}
}
if (controllerServiceDTO.getComments() != null) {
values.put(COMMENTS, controllerService.getComments());
}
return values;
}
@Override
protected final Collection<ValidationResult> customValidate(ValidationContext context) {
final List<ValidationResult> problems = new ArrayList<>();
if (SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue())) {
final String collection = context.getProperty(COLLECTION).getValue();
if (collection == null || collection.trim().isEmpty()) {
problems.add(
new ValidationResult.Builder()
.subject(COLLECTION.getName())
.input(collection)
.valid(false)
.explanation("A collection must specified for Solr Type of Cloud")
.build());
}
}
Collection<ValidationResult> otherProblems = this.additionalCustomValidation(context);
if (otherProblems != null) {
problems.addAll(otherProblems);
}
return problems;
}
/**
* Audits the configuration of a single controller service.
*
* @param proceedingJoinPoint join point
* @param controllerServiceDTO dto
* @param controllerServiceDAO dao
* @return object
* @throws Throwable ex
*/
@Around(
"within(org.apache.nifi.web.dao.ControllerServiceDAO+) && "
+ "execution(org.apache.nifi.controller.service.ControllerServiceNode updateControllerService(org.apache.nifi.web.api.dto.ControllerServiceDTO)) && "
+ "args(controllerServiceDTO) && "
+ "target(controllerServiceDAO)")
public Object updateControllerServiceAdvice(
ProceedingJoinPoint proceedingJoinPoint,
ControllerServiceDTO controllerServiceDTO,
ControllerServiceDAO controllerServiceDAO)
throws Throwable {
// determine the initial values for each property/setting thats changing
ControllerServiceNode controllerService =
controllerServiceDAO.getControllerService(controllerServiceDTO.getId());
final Map<String, String> values =
extractConfiguredPropertyValues(controllerService, controllerServiceDTO);
final boolean isDisabled = isDisabled(controllerService);
// update the controller service state
final ControllerServiceNode updatedControllerService =
(ControllerServiceNode) proceedingJoinPoint.proceed();
// if no exceptions were thrown, add the controller service action...
controllerService =
controllerServiceDAO.getControllerService(updatedControllerService.getIdentifier());
// get the current user
NiFiUser user = NiFiUserUtils.getNiFiUser();
// ensure the user was found
if (user != null) {
// determine the updated values
Map<String, String> updatedValues =
extractConfiguredPropertyValues(controllerService, controllerServiceDTO);
// create the controller service details
FlowChangeExtensionDetails serviceDetails = new FlowChangeExtensionDetails();
serviceDetails.setType(
controllerService.getControllerServiceImplementation().getClass().getSimpleName());
// create a controller service action
Date actionTimestamp = new Date();
Collection<Action> actions = new ArrayList<>();
// go through each updated value
for (String property : updatedValues.keySet()) {
String newValue = updatedValues.get(property);
String oldValue = values.get(property);
Operation operation = null;
// determine the type of operation
if (oldValue == null || newValue == null || !newValue.equals(oldValue)) {
operation = Operation.Configure;
}
// create a configuration action accordingly
if (operation != null) {
// clear the value if this property is sensitive
final PropertyDescriptor propertyDescriptor =
controllerService
.getControllerServiceImplementation()
.getPropertyDescriptor(property);
if (propertyDescriptor != null && propertyDescriptor.isSensitive()) {
if (newValue != null) {
newValue = "********";
}
if (oldValue != null) {
oldValue = "********";
}
} else if (ANNOTATION_DATA.equals(property)) {
if (newValue != null) {
newValue = "<annotation data not shown>";
}
if (oldValue != null) {
oldValue = "<annotation data not shown>";
}
}
final FlowChangeConfigureDetails actionDetails = new FlowChangeConfigureDetails();
actionDetails.setName(property);
actionDetails.setValue(newValue);
actionDetails.setPreviousValue(oldValue);
// create a configuration action
FlowChangeAction configurationAction = new FlowChangeAction();
configurationAction.setUserIdentity(user.getDn());
configurationAction.setUserName(user.getUserName());
configurationAction.setOperation(operation);
configurationAction.setTimestamp(actionTimestamp);
configurationAction.setSourceId(controllerService.getIdentifier());
configurationAction.setSourceName(controllerService.getName());
configurationAction.setSourceType(Component.ControllerService);
configurationAction.setComponentDetails(serviceDetails);
configurationAction.setActionDetails(actionDetails);
actions.add(configurationAction);
}
}
// determine the new executing state
final boolean updateIsDisabled = isDisabled(updatedControllerService);
// determine if the running state has changed and its not disabled
if (isDisabled != updateIsDisabled) {
// create a controller service action
FlowChangeAction serviceAction = new FlowChangeAction();
serviceAction.setUserIdentity(user.getDn());
serviceAction.setUserName(user.getUserName());
serviceAction.setTimestamp(new Date());
serviceAction.setSourceId(controllerService.getIdentifier());
serviceAction.setSourceName(controllerService.getName());
serviceAction.setSourceType(Component.ControllerService);
serviceAction.setComponentDetails(serviceDetails);
// set the operation accordingly
if (updateIsDisabled) {
serviceAction.setOperation(Operation.Disable);
} else {
serviceAction.setOperation(Operation.Enable);
}
actions.add(serviceAction);
}
// ensure there are actions to record
if (!actions.isEmpty()) {
// save the actions
saveActions(actions, logger);
}
}
return updatedControllerService;
}
/** Extracts the values for the configured properties from the specified Processor. */
private Map<String, String> extractConfiguredPropertyValues(
ProcessorNode processor, ProcessorDTO processorDTO) {
Map<String, String> values = new HashMap<>();
if (processorDTO.getName() != null) {
values.put(NAME, processor.getName());
}
if (processorDTO.getConfig() != null) {
ProcessorConfigDTO newConfig = processorDTO.getConfig();
if (newConfig.getConcurrentlySchedulableTaskCount() != null) {
values.put(
CONCURRENTLY_SCHEDULABLE_TASKS, String.valueOf(processor.getMaxConcurrentTasks()));
}
if (newConfig.getPenaltyDuration() != null) {
values.put(PENALTY_DURATION, processor.getPenalizationPeriod());
}
if (newConfig.getYieldDuration() != null) {
values.put(YIELD_DURATION, processor.getYieldPeriod());
}
if (newConfig.getBulletinLevel() != null) {
values.put(BULLETIN_LEVEL, processor.getBulletinLevel().name());
}
if (newConfig.getAnnotationData() != null) {
values.put(ANNOTATION_DATA, processor.getAnnotationData());
}
if (newConfig.getSchedulingPeriod() != null) {
values.put(SCHEDULING_PERIOD, String.valueOf(processor.getSchedulingPeriod()));
}
if (newConfig.getAutoTerminatedRelationships() != null) {
// get each of the auto terminated relationship names
final Set<Relationship> autoTerminatedRelationships =
processor.getAutoTerminatedRelationships();
final List<String> autoTerminatedRelationshipNames =
new ArrayList<>(autoTerminatedRelationships.size());
for (final Relationship relationship : autoTerminatedRelationships) {
autoTerminatedRelationshipNames.add(relationship.getName());
}
// sort them and include in the configuration
Collections.sort(autoTerminatedRelationshipNames, Collator.getInstance(Locale.US));
values.put(
AUTO_TERMINATED_RELATIONSHIPS, StringUtils.join(autoTerminatedRelationshipNames, ", "));
}
if (newConfig.getProperties() != null) {
// for each property specified, extract its configured value
Map<String, String> properties = newConfig.getProperties();
Map<PropertyDescriptor, String> configuredProperties = processor.getProperties();
for (String propertyName : properties.keySet()) {
// build a descriptor for getting the configured value
PropertyDescriptor propertyDescriptor =
new PropertyDescriptor.Builder().name(propertyName).build();
String configuredPropertyValue = configuredProperties.get(propertyDescriptor);
// if the configured value couldn't be found, use the default value from the actual
// descriptor
if (configuredPropertyValue == null) {
propertyDescriptor =
locatePropertyDescriptor(configuredProperties.keySet(), propertyDescriptor);
configuredPropertyValue = propertyDescriptor.getDefaultValue();
}
values.put(propertyName, configuredPropertyValue);
}
}
if (newConfig.getComments() != null) {
values.put(COMMENTS, processor.getComments());
}
if (newConfig.getSchedulingStrategy() != null) {
values.put(SCHEDULING_STRATEGY, processor.getSchedulingStrategy().toString());
}
}
return values;
}
/**
* Audits the configuration of a single processor.
*
* @param proceedingJoinPoint join point
* @param processorDTO dto
* @param processorDAO dao
* @return node
* @throws Throwable ex
*/
@Around(
"within(org.apache.nifi.web.dao.ProcessorDAO+) && "
+ "execution(org.apache.nifi.controller.ProcessorNode updateProcessor(org.apache.nifi.web.api.dto.ProcessorDTO)) && "
+ "args(processorDTO) && "
+ "target(processorDAO)")
public ProcessorNode updateProcessorAdvice(
ProceedingJoinPoint proceedingJoinPoint, ProcessorDTO processorDTO, ProcessorDAO processorDAO)
throws Throwable {
// determine the initial values for each property/setting thats changing
ProcessorNode processor = processorDAO.getProcessor(processorDTO.getId());
final Map<String, String> values = extractConfiguredPropertyValues(processor, processorDTO);
final ScheduledState scheduledState = processor.getScheduledState();
// update the processor state
final ProcessorNode updatedProcessor = (ProcessorNode) proceedingJoinPoint.proceed();
// if no exceptions were thrown, add the processor action...
// get the updated verbose state
processor = processorDAO.getProcessor(updatedProcessor.getIdentifier());
// get the current user
NiFiUser user = NiFiUserUtils.getNiFiUser();
// ensure the user was found
if (user != null) {
// determine the updated values
Map<String, String> updatedValues = extractConfiguredPropertyValues(processor, processorDTO);
// create the processor details
FlowChangeExtensionDetails processorDetails = new FlowChangeExtensionDetails();
processorDetails.setType(processor.getProcessor().getClass().getSimpleName());
// create a processor action
Date actionTimestamp = new Date();
Collection<Action> actions = new ArrayList<>();
// go through each updated value
for (String property : updatedValues.keySet()) {
String newValue = updatedValues.get(property);
String oldValue = values.get(property);
Operation operation = null;
// determine the type of operation
if (oldValue == null || newValue == null || !newValue.equals(oldValue)) {
operation = Operation.Configure;
}
// create a configuration action accordingly
if (operation != null) {
// clear the value if this property is sensitive
final PropertyDescriptor propertyDescriptor =
processor.getProcessor().getPropertyDescriptor(property);
if (propertyDescriptor != null && propertyDescriptor.isSensitive()) {
if (newValue != null) {
newValue = "********";
}
if (oldValue != null) {
oldValue = "********";
}
} else if (ANNOTATION_DATA.equals(property)) {
if (newValue != null) {
newValue = "<annotation data not shown>";
}
if (oldValue != null) {
oldValue = "<annotation data not shown>";
}
}
final FlowChangeConfigureDetails actionDetails = new FlowChangeConfigureDetails();
actionDetails.setName(property);
actionDetails.setValue(newValue);
actionDetails.setPreviousValue(oldValue);
// create a configuration action
FlowChangeAction configurationAction = new FlowChangeAction();
configurationAction.setUserIdentity(user.getIdentity());
configurationAction.setUserName(user.getUserName());
configurationAction.setOperation(operation);
configurationAction.setTimestamp(actionTimestamp);
configurationAction.setSourceId(processor.getIdentifier());
configurationAction.setSourceName(processor.getName());
configurationAction.setSourceType(Component.Processor);
configurationAction.setComponentDetails(processorDetails);
configurationAction.setActionDetails(actionDetails);
actions.add(configurationAction);
}
}
// determine the new executing state
final ScheduledState updatedScheduledState = processor.getScheduledState();
// determine if the running state has changed and its not disabled
if (scheduledState != updatedScheduledState) {
// create a processor action
FlowChangeAction processorAction = new FlowChangeAction();
processorAction.setUserIdentity(user.getIdentity());
processorAction.setUserName(user.getUserName());
processorAction.setTimestamp(new Date());
processorAction.setSourceId(processor.getIdentifier());
processorAction.setSourceName(processor.getName());
processorAction.setSourceType(Component.Processor);
processorAction.setComponentDetails(processorDetails);
// set the operation accordingly
if (ScheduledState.RUNNING.equals(updatedScheduledState)) {
processorAction.setOperation(Operation.Start);
} else if (ScheduledState.DISABLED.equals(updatedScheduledState)) {
processorAction.setOperation(Operation.Disable);
} else {
// state is now stopped... consider the previous state
if (ScheduledState.RUNNING.equals(scheduledState)) {
processorAction.setOperation(Operation.Stop);
} else if (ScheduledState.DISABLED.equals(scheduledState)) {
processorAction.setOperation(Operation.Enable);
}
}
actions.add(processorAction);
}
// ensure there are actions to record
if (!actions.isEmpty()) {
// save the actions
saveActions(actions, logger);
}
}
return updatedProcessor;
}
// Executes the specified action on the specified flowfile.
private FlowFile executeActions(
final ProcessSession session,
final ProcessContext context,
final List<Rule> rules,
final Map<String, Action> defaultActions,
final FlowFile flowfile) {
final ComponentLog logger = getLogger();
final Map<String, Action> actions = new HashMap<>(defaultActions);
final String ruleName =
(rules == null || rules.isEmpty()) ? "default" : rules.get(rules.size() - 1).getName();
// if a rule matched, get its actions and possible overwrite the default ones
if (rules != null && rules.size() > 0) {
// combine all rules actions with the default actions... loop through the rules in order, this
// way
// subsequent matching rules will take precedence over previously matching rules and default
// values
for (final Rule rule : rules) {
for (final Action action : rule.getActions()) {
// store the action and overwrite the previous value (from the defaults or a previously
// matching rule)
actions.put(action.getAttribute(), action);
}
}
// add an action for the matched rule - when matching multiple rules against
// the original flowfile (use original) this will leave the last matching
// rule's name as the value of this attribute. this decision was made since
// this would be the behavior if they user chained multiple UpdateAttributes
// together with 'use clone' specified
final Action matchedRuleAction = new Action();
matchedRuleAction.setAttribute(getClass().getSimpleName() + ".matchedRule");
matchedRuleAction.setValue(ruleName);
actions.put(matchedRuleAction.getAttribute(), matchedRuleAction);
}
// attribute values that will be applied to the flow file
final Map<String, String> attributesToUpdate = new HashMap<>(actions.size());
final Set<String> attributesToDelete = new HashSet<>(actions.size());
// go through each action
for (final Action action : actions.values()) {
if (!action.getAttribute().equals(DELETE_ATTRIBUTES.getName())) {
try {
final String newAttributeValue =
getPropertyValue(action.getValue(), context)
.evaluateAttributeExpressions(flowfile)
.getValue();
// log if appropriate
if (logger.isDebugEnabled()) {
logger.debug(
String.format(
"%s setting attribute '%s' = '%s' for %s per rule '%s'.",
this, action.getAttribute(), newAttributeValue, flowfile, ruleName));
}
attributesToUpdate.put(action.getAttribute(), newAttributeValue);
} catch (final ProcessException pe) {
throw new ProcessException(
String.format(
"Unable to evaluate new value for attribute '%s': %s.",
action.getAttribute(), pe),
pe);
}
} else {
try {
final String actionValue = action.getValue();
final String regex =
(actionValue == null)
? null
: getPropertyValue(actionValue, context)
.evaluateAttributeExpressions(flowfile)
.getValue();
if (regex != null) {
Pattern pattern = Pattern.compile(regex);
final Set<String> attributeKeys = flowfile.getAttributes().keySet();
for (final String key : attributeKeys) {
if (pattern.matcher(key).matches()) {
// log if appropriate
if (logger.isDebugEnabled()) {
logger.debug(
String.format(
"%s deleting attribute '%s' for %s per regex '%s'.",
this, key, flowfile, regex));
}
attributesToDelete.add(key);
}
}
}
} catch (final ProcessException pe) {
throw new ProcessException(
String.format("Unable to delete attribute '%s': %s.", action.getAttribute(), pe), pe);
}
}
}
// If the 'alternate.identifier' attribute is added, then we want to create an ADD_INFO
// provenance event.
final String alternateIdentifierAdd =
attributesToUpdate.get(CoreAttributes.ALTERNATE_IDENTIFIER.key());
if (alternateIdentifierAdd != null) {
try {
final URI uri = new URI(alternateIdentifierAdd);
final String namespace = uri.getScheme();
if (namespace != null) {
final String identifier =
alternateIdentifierAdd.substring(
Math.min(namespace.length() + 1, alternateIdentifierAdd.length() - 1));
session.getProvenanceReporter().associate(flowfile, namespace, identifier);
}
} catch (final URISyntaxException e) {
}
}
// update and delete the flowfile attributes
return session.removeAllAttributes(
session.putAllAttributes(flowfile, attributesToUpdate), attributesToDelete);
}