@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
final ComponentLog logger = getLogger();
final Criteria criteria = criteriaCache.get();
List<FlowFile> flowFiles = session.get(100);
if (flowFiles.isEmpty()) {
return;
}
final Map<PropertyDescriptor, String> properties = context.getProperties();
// get the default actions
final Map<String, Action> defaultActions = getDefaultActions(properties);
// record which rule should be applied to which flow file - when operating
// in 'use clone' mode, this collection will contain a number of entries
// that map to single element lists. this is because the original flowfile
// is cloned for each matching rule. in 'use original' mode, this collection
// will contain a single entry that maps a list of multiple rules. this is
// because is the original flowfile is used for all matching rules. in this
// case the order of the matching rules is perserved in the list
final Map<FlowFile, List<Rule>> matchedRules = new HashMap<>();
for (FlowFile flowFile : flowFiles) {
matchedRules.clear();
// if there is update criteria specified, evaluate it
if (criteria != null
&& evaluateCriteria(session, context, criteria, flowFile, matchedRules)) {
// apply the actions for each rule and transfer the flowfile
for (final Map.Entry<FlowFile, List<Rule>> entry : matchedRules.entrySet()) {
FlowFile match = entry.getKey();
final List<Rule> rules = entry.getValue();
// execute each matching rule(s)
match = executeActions(session, context, rules, defaultActions, match);
logger.info(
"Updated attributes for {}; transferring to '{}'",
new Object[] {match, REL_SUCCESS.getName()});
// transfer the match
session.getProvenanceReporter().modifyAttributes(match);
session.transfer(match, REL_SUCCESS);
}
} else {
// transfer the flowfile to no match (that has the default actions applied)
flowFile = executeActions(session, context, null, defaultActions, flowFile);
logger.info(
"Updated attributes for {}; transferring to '{}'",
new Object[] {flowFile, REL_SUCCESS.getName()});
session.getProvenanceReporter().modifyAttributes(flowFile);
session.transfer(flowFile, REL_SUCCESS);
}
}
}
// Evaluates the specified Criteria on the specified flowfile. Clones the
// specified flow file for each rule that is applied.
private boolean evaluateCriteria(
final ProcessSession session,
final ProcessContext context,
final Criteria criteria,
final FlowFile flowfile,
final Map<FlowFile, List<Rule>> matchedRules) {
final ComponentLog logger = getLogger();
final List<Rule> rules = criteria.getRules();
// consider each rule and hold a copy of the flowfile for each matched rule
for (final Rule rule : rules) {
// evaluate the rule
if (evaluateRule(context, rule, flowfile)) {
final FlowFile flowfileToUse;
// determine if we should use the original flow file or clone
if (FlowFilePolicy.USE_ORIGINAL.equals(criteria.getFlowFilePolicy())
|| matchedRules.isEmpty()) {
flowfileToUse = flowfile;
} else {
// clone the original for this rule
flowfileToUse = session.clone(flowfile);
}
// store the flow file to use when executing this rule
List<Rule> rulesForFlowFile = matchedRules.get(flowfileToUse);
if (rulesForFlowFile == null) {
rulesForFlowFile = new ArrayList<>();
matchedRules.put(flowfileToUse, rulesForFlowFile);
}
rulesForFlowFile.add(rule);
// log if appropriate
if (logger.isDebugEnabled()) {
logger.debug(
this
+ " all conditions met for rule '"
+ rule.getName()
+ "'. Using flow file - "
+ flowfileToUse);
}
}
}
return !matchedRules.isEmpty();
}
@Override
public synchronized int onEvent(
int evtSubscribeNotifyAction, WinDef.PVOID userContext, WinNT.HANDLE eventHandle) {
if (logger.isDebugEnabled()) {
logger.debug("onEvent(" + evtSubscribeNotifyAction + ", " + userContext + ", " + eventHandle);
}
if (evtSubscribeNotifyAction == WEvtApi.EvtSubscribeNotifyAction.ERROR) {
if (eventHandle.getPointer().getInt(0)
== WEvtApi.EvtSubscribeErrors.ERROR_EVT_QUERY_RESULT_STALE) {
logger.error(MISSING_EVENT_MESSAGE);
} else {
logger.error(RECEIVED_THE_FOLLOWING_WIN32_ERROR + eventHandle.getPointer().getInt(0));
}
} else if (evtSubscribeNotifyAction == WEvtApi.EvtSubscribeNotifyAction.DELIVER) {
wEvtApi.EvtRender(
null, eventHandle, WEvtApi.EvtRenderFlags.EVENT_XML, size, buffer, used, propertyCount);
// Not enough room in buffer, resize so it's big enough
if (kernel32.GetLastError() == W32Errors.ERROR_INSUFFICIENT_BUFFER) {
int newMaxSize = used.getInt(0);
// Check for overflow or too big
if (newMaxSize < size || newMaxSize > maxBufferSize) {
logger.error(
"Dropping event "
+ eventHandle
+ " because it couldn't be rendered within "
+ maxBufferSize
+ " bytes.");
// Ignored, see
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa385577(v=vs.85).aspx
return 0;
}
size = newMaxSize;
buffer = new Memory(size);
wEvtApi.EvtRender(
null, eventHandle, WEvtApi.EvtRenderFlags.EVENT_XML, size, buffer, used, propertyCount);
}
int lastError = kernel32.GetLastError();
if (lastError == W32Errors.ERROR_SUCCESS) {
int usedBytes = used.getInt(0);
String string = Charsets.UTF_16LE.decode(buffer.getByteBuffer(0, usedBytes)).toString();
if (string.endsWith("\u0000")) {
string = string.substring(0, string.length() - 1);
}
consumer.accept(string);
} else {
logger.error(
EVT_RENDER_RETURNED_THE_FOLLOWING_ERROR_CODE + errorLookup.getLastError() + ".");
}
}
// Ignored, see https://msdn.microsoft.com/en-us/library/windows/desktop/aa385577(v=vs.85).aspx
return 0;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
final String logLevelValue = context.getProperty(LOG_LEVEL).getValue().toLowerCase();
final DebugLevels logLevel;
try {
logLevel = DebugLevels.valueOf(logLevelValue);
} catch (Exception e) {
throw new ProcessException(e);
}
final ComponentLog LOG = getLogger();
boolean isLogLevelEnabled = false;
switch (logLevel) {
case trace:
isLogLevelEnabled = LOG.isTraceEnabled();
break;
case debug:
isLogLevelEnabled = LOG.isDebugEnabled();
break;
case info:
isLogLevelEnabled = LOG.isInfoEnabled();
break;
case warn:
isLogLevelEnabled = LOG.isWarnEnabled();
break;
case error:
isLogLevelEnabled = LOG.isErrorEnabled();
break;
}
if (!isLogLevelEnabled) {
transferChunk(session);
return;
}
final FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
processFlowFile(LOG, logLevel, flowFile, session, context);
session.transfer(flowFile, REL_SUCCESS);
}
// Executes the specified action on the specified flowfile.
private FlowFile executeActions(
final ProcessSession session,
final ProcessContext context,
final List<Rule> rules,
final Map<String, Action> defaultActions,
final FlowFile flowfile) {
final ComponentLog logger = getLogger();
final Map<String, Action> actions = new HashMap<>(defaultActions);
final String ruleName =
(rules == null || rules.isEmpty()) ? "default" : rules.get(rules.size() - 1).getName();
// if a rule matched, get its actions and possible overwrite the default ones
if (rules != null && rules.size() > 0) {
// combine all rules actions with the default actions... loop through the rules in order, this
// way
// subsequent matching rules will take precedence over previously matching rules and default
// values
for (final Rule rule : rules) {
for (final Action action : rule.getActions()) {
// store the action and overwrite the previous value (from the defaults or a previously
// matching rule)
actions.put(action.getAttribute(), action);
}
}
// add an action for the matched rule - when matching multiple rules against
// the original flowfile (use original) this will leave the last matching
// rule's name as the value of this attribute. this decision was made since
// this would be the behavior if they user chained multiple UpdateAttributes
// together with 'use clone' specified
final Action matchedRuleAction = new Action();
matchedRuleAction.setAttribute(getClass().getSimpleName() + ".matchedRule");
matchedRuleAction.setValue(ruleName);
actions.put(matchedRuleAction.getAttribute(), matchedRuleAction);
}
// attribute values that will be applied to the flow file
final Map<String, String> attributesToUpdate = new HashMap<>(actions.size());
final Set<String> attributesToDelete = new HashSet<>(actions.size());
// go through each action
for (final Action action : actions.values()) {
if (!action.getAttribute().equals(DELETE_ATTRIBUTES.getName())) {
try {
final String newAttributeValue =
getPropertyValue(action.getValue(), context)
.evaluateAttributeExpressions(flowfile)
.getValue();
// log if appropriate
if (logger.isDebugEnabled()) {
logger.debug(
String.format(
"%s setting attribute '%s' = '%s' for %s per rule '%s'.",
this, action.getAttribute(), newAttributeValue, flowfile, ruleName));
}
attributesToUpdate.put(action.getAttribute(), newAttributeValue);
} catch (final ProcessException pe) {
throw new ProcessException(
String.format(
"Unable to evaluate new value for attribute '%s': %s.",
action.getAttribute(), pe),
pe);
}
} else {
try {
final String actionValue = action.getValue();
final String regex =
(actionValue == null)
? null
: getPropertyValue(actionValue, context)
.evaluateAttributeExpressions(flowfile)
.getValue();
if (regex != null) {
Pattern pattern = Pattern.compile(regex);
final Set<String> attributeKeys = flowfile.getAttributes().keySet();
for (final String key : attributeKeys) {
if (pattern.matcher(key).matches()) {
// log if appropriate
if (logger.isDebugEnabled()) {
logger.debug(
String.format(
"%s deleting attribute '%s' for %s per regex '%s'.",
this, key, flowfile, regex));
}
attributesToDelete.add(key);
}
}
}
} catch (final ProcessException pe) {
throw new ProcessException(
String.format("Unable to delete attribute '%s': %s.", action.getAttribute(), pe), pe);
}
}
}
// If the 'alternate.identifier' attribute is added, then we want to create an ADD_INFO
// provenance event.
final String alternateIdentifierAdd =
attributesToUpdate.get(CoreAttributes.ALTERNATE_IDENTIFIER.key());
if (alternateIdentifierAdd != null) {
try {
final URI uri = new URI(alternateIdentifierAdd);
final String namespace = uri.getScheme();
if (namespace != null) {
final String identifier =
alternateIdentifierAdd.substring(
Math.min(namespace.length() + 1, alternateIdentifierAdd.length() - 1));
session.getProvenanceReporter().associate(flowfile, namespace, identifier);
}
} catch (final URISyntaxException e) {
}
}
// update and delete the flowfile attributes
return session.removeAllAttributes(
session.putAllAttributes(flowfile, attributesToUpdate), attributesToDelete);
}
protected String processFlowFile(
final ComponentLog logger,
final DebugLevels logLevel,
final FlowFile flowFile,
final ProcessSession session,
final ProcessContext context) {
final Set<String> attributeKeys =
getAttributesToLog(flowFile.getAttributes().keySet(), context);
final ComponentLog LOG = getLogger();
final String dashedLine;
String logPrefix =
context.getProperty(LOG_PREFIX).evaluateAttributeExpressions(flowFile).getValue();
if (StringUtil.isBlank(logPrefix)) {
dashedLine = StringUtils.repeat('-', 50);
} else {
// abbreviate long lines
logPrefix = StringUtils.abbreviate(logPrefix, 40);
// center the logPrefix and pad with dashes
logPrefix = StringUtils.center(logPrefix, 40, '-');
// five dashes on the left and right side, plus the dashed logPrefix
dashedLine = StringUtils.repeat('-', 5) + logPrefix + StringUtils.repeat('-', 5);
}
// Pretty print metadata
final StringBuilder message = new StringBuilder();
message.append("logging for flow file ").append(flowFile);
message.append("\n");
message.append(dashedLine);
message.append("\nStandard FlowFile Attributes");
message.append(
String.format(
"\nKey: '%1$s'\n\tValue: '%2$s'", "entryDate", new Date(flowFile.getEntryDate())));
message.append(
String.format(
"\nKey: '%1$s'\n\tValue: '%2$s'",
"lineageStartDate", new Date(flowFile.getLineageStartDate())));
message.append(String.format("\nKey: '%1$s'\n\tValue: '%2$s'", "fileSize", flowFile.getSize()));
message.append("\nFlowFile Attribute Map Content");
for (final String key : attributeKeys) {
message.append(
String.format("\nKey: '%1$s'\n\tValue: '%2$s'", key, flowFile.getAttribute(key)));
}
message.append("\n");
message.append(dashedLine);
// The user can request to log the payload
final boolean logPayload = context.getProperty(LOG_PAYLOAD).asBoolean();
if (logPayload) {
message.append("\n");
if (flowFile.getSize() < ONE_MB) {
final FlowFilePayloadCallback callback = new FlowFilePayloadCallback();
session.read(flowFile, callback);
message.append(callback.getContents());
} else {
message.append("\n Not including payload since it is larger than one mb.");
}
}
final String outputMessage = message.toString().trim();
// Uses optional property to specify logging level
switch (logLevel) {
case info:
LOG.info(outputMessage);
break;
case debug:
LOG.debug(outputMessage);
break;
case warn:
LOG.warn(outputMessage);
break;
case trace:
LOG.trace(outputMessage);
break;
case error:
LOG.error(outputMessage);
break;
default:
LOG.debug(outputMessage);
}
return outputMessage;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session)
throws ProcessException {
FlowFile flowFile = null;
if (context.hasIncomingConnection()) {
flowFile = session.get();
// If we have no FlowFile, and all incoming connections are self-loops then we can continue
// on.
// However, if we have no FlowFile and we have connections coming from other Processors, then
// we know that we should run only if we have a FlowFile.
if (flowFile == null && context.hasNonLoopConnection()) {
return;
}
}
OkHttpClient okHttpClient = getClient();
if (flowFile == null) {
flowFile = session.create();
}
final String index =
context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
final String docId =
context.getProperty(DOC_ID).evaluateAttributeExpressions(flowFile).getValue();
final String docType =
context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
final String fields =
context.getProperty(FIELDS).isSet()
? context.getProperty(FIELDS).evaluateAttributeExpressions(flowFile).getValue()
: null;
// Authentication
final String username =
context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();
final ComponentLog logger = getLogger();
Response getResponse = null;
try {
logger.debug("Fetching {}/{}/{} from Elasticsearch", new Object[] {index, docType, docId});
// read the url property from the context
final String urlstr =
StringUtils.trimToEmpty(
context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
final URL url = buildRequestURL(urlstr, docId, index, docType, fields);
final long startNanos = System.nanoTime();
getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "GET", null);
final int statusCode = getResponse.code();
if (isSuccess(statusCode)) {
ResponseBody body = getResponse.body();
final byte[] bodyBytes = body.bytes();
JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
boolean found = responseJson.get("found").asBoolean(false);
String retrievedIndex = responseJson.get("_index").asText();
String retrievedType = responseJson.get("_type").asText();
String retrievedId = responseJson.get("_id").asText();
if (found) {
JsonNode source = responseJson.get("_source");
flowFile = session.putAttribute(flowFile, "filename", retrievedId);
flowFile = session.putAttribute(flowFile, "es.index", retrievedIndex);
flowFile = session.putAttribute(flowFile, "es.type", retrievedType);
if (source != null) {
flowFile =
session.write(
flowFile,
out -> {
out.write(source.toString().getBytes());
});
}
logger.debug("Elasticsearch document " + retrievedId + " fetched, routing to success");
// emit provenance event
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
if (context.hasNonLoopConnection()) {
session.getProvenanceReporter().fetch(flowFile, url.toExternalForm(), millis);
} else {
session.getProvenanceReporter().receive(flowFile, url.toExternalForm(), millis);
}
session.transfer(flowFile, REL_SUCCESS);
} else {
logger.warn(
"Failed to read {}/{}/{} from Elasticsearch: Document not found",
new Object[] {index, docType, docId});
// We couldn't find the document, so send it to "not found"
session.transfer(flowFile, REL_NOT_FOUND);
}
} else {
if (statusCode == 404) {
logger.warn(
"Failed to read {}/{}/{} from Elasticsearch: Document not found",
new Object[] {index, docType, docId});
// We couldn't find the document, so penalize it and send it to "not found"
session.transfer(flowFile, REL_NOT_FOUND);
} else {
// 5xx -> RETRY, but a server error might last a while, so yield
if (statusCode / 100 == 5) {
logger.warn(
"Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...",
new Object[] {statusCode, getResponse.message()});
session.transfer(flowFile, REL_RETRY);
context.yield();
} else if (context.hasIncomingConnection()) { // 1xx, 3xx, 4xx -> NO RETRY
logger.warn(
"Elasticsearch returned code {} with message {}, transferring flow file to failure",
new Object[] {statusCode, getResponse.message()});
session.transfer(flowFile, REL_FAILURE);
} else {
logger.warn(
"Elasticsearch returned code {} with message {}",
new Object[] {statusCode, getResponse.message()});
session.remove(flowFile);
}
}
}
} catch (IOException ioe) {
logger.error(
"Failed to read from Elasticsearch due to {}, this may indicate an error in configuration "
+ "(hosts, username/password, etc.). Routing to retry",
new Object[] {ioe.getLocalizedMessage()},
ioe);
if (context.hasIncomingConnection()) {
session.transfer(flowFile, REL_RETRY);
} else {
session.remove(flowFile);
}
context.yield();
} catch (Exception e) {
logger.error(
"Failed to read {} from Elasticsearch due to {}",
new Object[] {flowFile, e.getLocalizedMessage()},
e);
if (context.hasIncomingConnection()) {
session.transfer(flowFile, REL_FAILURE);
} else {
session.remove(flowFile);
}
context.yield();
} finally {
if (getResponse != null) {
getResponse.close();
}
}
}