@Test
public void testCursorStrategyCopyWithMultipleResults() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.cursorStrategy(COPY)
.sourceField("msg")
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {
new Result("the", "one", 0, 3), new Result("hello", "two", 10, 15),
};
}
})
.build();
final Message msg = createMessage("message");
msg.addField("msg", "the great hello");
extractor.runExtractor(msg);
// With the copy strategy, the source field will not be modified.
assertThat(msg.getField("msg")).isEqualTo("the great hello");
}
@Test
public void testWithMultipleTargetValueResultsAndOneValueIsNull() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {
new Result(1, "one", -1, -1),
new Result(2, "two", -1, -1),
new Result(null, "three", -1, -1)
};
}
})
.build();
final Message msg = createMessage("the hello");
extractor.runExtractor(msg);
// If the extractor returns multiple results and one result value is null, all results will be
// ignored.
// TODO: This is the current behaviour and it is weird. Will be fixed soon.
assertThat(msg.hasField("one")).isFalse();
assertThat(msg.hasField("two")).isFalse();
assertThat(msg.hasField("three")).isFalse();
}
@Test
public void testCursorStrategyCutWithMultipleResults() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.cursorStrategy(CUT)
.sourceField("msg")
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {
new Result("the", "one", 0, 3), new Result("hello", "two", 10, 15),
};
}
})
.build();
final Message msg = createMessage("message");
msg.addField("msg", "the great hello");
extractor.runExtractor(msg);
// With the cut strategy the matched data will be removed from the message.
assertThat(msg.getField("msg")).isEqualTo("great");
}
@Test
public void testCursorStrategyCutIfTargetFieldEqualsSourceField() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.cursorStrategy(CUT)
.sourceField("msg")
.targetField("msg")
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("the", 0, 3)};
}
})
.build();
final Message msg = createMessage("message");
msg.addField("msg", "the hello");
extractor.runExtractor(msg);
// If source and target fields are the same, the field is not touched because it already got set
// to a new value.
assertThat(msg.getField("msg")).isEqualTo("the");
}
@Test
public void testConvertersWithNonStringFieldValue() throws Exception {
final Converter converter =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return "converted";
}
})
.build();
final TestExtractor extractor =
new TestExtractor.Builder()
.converters(Lists.newArrayList(converter))
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result(123, "target", -1, -1)};
}
})
.build();
final Message msg = createMessage("message");
extractor.runExtractor(msg);
// Only string values will be converted.
assertThat(msg.getField("target")).isEqualTo(123);
}
@Test
public void testConvertersThatReturnNullValue() throws Exception {
final Converter converter =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return null;
}
})
.build();
final TestExtractor extractor =
new TestExtractor.Builder()
.converters(Lists.newArrayList(converter))
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("1", -1, -1)};
}
})
.build();
final Message msg = createMessage("message");
extractor.runExtractor(msg);
assertThat(msg.getField("target")).isNull();
}
@Test
public void testWithMultipleTargetValueResults() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {
new Result(1, "one", -1, -1),
new Result("2", "two", -1, -1),
new Result(3, "three", -1, -1)
};
}
})
.build();
final Message msg = createMessage("the hello");
extractor.runExtractor(msg);
assertThat(msg.hasField("target")).isFalse();
assertThat(msg.getField("one")).isEqualTo(1);
assertThat(msg.getField("two")).isEqualTo("2");
assertThat(msg.getField("three")).isEqualTo(3);
}
@Test
public void testConvertersAreExecutedInOrder() throws Exception {
final Converter converter1 =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return ((String) input) + "1";
}
})
.build();
final Converter converter2 =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return ((String) input) + "2";
}
})
.build();
final Converter converter3 =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return ((String) input) + "3";
}
})
.build();
final TestExtractor extractor =
new TestExtractor.Builder()
.converters(Lists.newArrayList(converter1, converter2, converter3))
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("converter", -1, -1)};
}
})
.build();
final Message msg = createMessage("message");
extractor.runExtractor(msg);
assertThat(msg.getField("target")).isEqualTo("converter123");
}
@Test
public void testSuccessfulMatch() {
StreamRule rule = getSampleRule();
rule.setValue("^foo");
Message msg = getSampleMessage();
msg.addField("something", "foobar");
StreamRuleMatcher matcher = getMatcher(rule);
assertTrue(matcher.match(msg, rule));
}
@Test
public void testMissedMatch() {
StreamRule rule = getSampleRule();
rule.setValue("^foo");
Message msg = getSampleMessage();
msg.addField("something", "zomg");
StreamRuleMatcher matcher = getMatcher(rule);
assertFalse(matcher.match(msg, rule));
}
@Test
public void testSuccessfulComplexRegexMatch() {
StreamRule rule = getSampleRule();
rule.setField("some_field");
rule.setValue("foo=^foo|bar\\d.+wat");
Message msg = getSampleMessage();
msg.addField("some_field", "bar1foowat");
StreamRuleMatcher matcher = getMatcher(rule);
assertTrue(matcher.match(msg, rule));
}
@Test
public void testConvertersWithMultipleFields() throws Exception {
final Converter converter =
new TestConverter.Builder()
.multiple(true)
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return ImmutableMap.builder()
.put("one", 1)
.put("two", "2")
.put(
"message",
"message should not be overwritten") // Try to overwrite reserved field.
.build();
}
})
.build();
final TestExtractor extractor =
new TestExtractor.Builder()
.converters(Lists.newArrayList(converter))
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("1", -1, -1)};
}
})
.build();
final Message msg = createMessage("the message");
extractor.runExtractor(msg);
// With a "multiple fields" converter the target field is not touched, only the additional
// fields are added.
assertThat(msg.getField("target")).isEqualTo("1");
assertThat(msg.getField("one")).isEqualTo(1);
assertThat(msg.getField("two")).isEqualTo("2");
// Reserved fields are not overwritten!
assertThat(msg.getField("message")).isEqualTo("the message");
// Attempts to overwrite a reserved field are recorded as converter exception.
assertThat(extractor.getConverterExceptionCount()).isEqualTo(1);
}
@Test
public void testMultipleConvertersWithFirstReturningNullValue() throws Exception {
final Converter converter1 =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return null;
}
})
.build();
final Converter converter2 =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return input + "2";
}
})
.build();
final TestExtractor extractor =
new TestExtractor.Builder()
.converters(Lists.newArrayList(converter1, converter2))
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("converter", -1, -1)};
}
})
.build();
final Message msg = createMessage("message");
extractor.runExtractor(msg);
// If the first converter returns null, the second will not be executed because the value is not
// a string anymore.
assertThat(msg.getField("target")).isNull();
}
@Test
public void runCodegen() throws IOException {
final Rule rule = parser.parseRule(ruleForTest(), true).withId("1");
final String sourceCode = CodeGenerator.sourceCodeForRule(rule);
Files.write(sourceCode, OUTFILE.toFile(), StandardCharsets.UTF_8);
log.info("Code:\n{}", sourceCode);
try {
ClassLoader ruleClassloader = new ClassLoader() {};
//noinspection unchecked
Class<GeneratedRule> rule$1 =
(Class<GeneratedRule>)
JCC.loadFromJava(
ruleClassloader,
"org.graylog.plugins.pipelineprocessor.$dynamic.rules.rule$1",
sourceCode);
//noinspection unchecked
final Set<Constructor> constructors =
ReflectionUtils.getConstructors(rule$1, input -> input.getParameterCount() == 1);
final Constructor onlyElement = Iterables.getOnlyElement(constructors);
final GeneratedRule generatedRule = (GeneratedRule) onlyElement.newInstance(functionRegistry);
final Message message = new Message("hello", "jenkins.torch.sh", Tools.nowUTC());
message.addField("message", "#1234");
message.addField("something_that_doesnt_exist", "foo");
final EvaluationContext context = new EvaluationContext(message);
final boolean when = generatedRule.when(context);
if (when) {
generatedRule.then(context);
}
log.info("created dynamic rule {} matches: {}", generatedRule.name(), when);
assertThat(context.currentMessage().hasField("some_identifier")).isTrue();
} catch (InvocationTargetException
| ClassNotFoundException
| InstantiationException
| IllegalAccessException e) {
log.error("Cannot load dynamically created class!", e);
}
}
@Test
public void testWithStringCondition() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder().conditionType(STRING).conditionValue("hello").build();
// Extractor runs if the message contains the condition value "hello".
final Message msg1 = createMessage("hello world");
extractor.runExtractor(msg1);
assertThat(msg1.hasField("target")).isTrue();
// Extractor does not run if the message does not contain the condition value.
final Message msg2 = createMessage("the message");
extractor.runExtractor(msg2);
assertThat(msg2.hasField("target")).isFalse();
}
@Test
public void testWithRegexpCondition() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder().conditionType(REGEX).conditionValue("^hello").build();
// Extractor runs if the message matches the condition regexp.
final Message msg1 = createMessage("hello world");
extractor.runExtractor(msg1);
assertThat(msg1.hasField("target")).isTrue();
// Extractor does not run if the message does not match the condition regexp.
final Message msg2 = createMessage("the hello");
extractor.runExtractor(msg2);
assertThat(msg2.hasField("target")).isFalse();
}
@Test
public void testWithEmptyResultArray() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[0];
}
})
.build();
final Message msg = createMessage("the hello");
extractor.runExtractor(msg);
assertThat(msg.hasField("target")).isFalse();
}
@Test
public void testWithOneValueOnlyResultsAndValueIsNull() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result(null, -1, -1)};
}
})
.build();
final Message msg = createMessage("the hello");
extractor.runExtractor(msg);
assertThat(msg.hasField("target")).isFalse();
}
@Test
public void testCursorStrategyCutIfSourceFieldIsReservedField() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.cursorStrategy(CUT)
.sourceField("message")
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("the", 0, 3)};
}
})
.build();
final Message msg = createMessage("the hello");
extractor.runExtractor(msg);
// The source value is not modified if it is a reserved field.
assertThat(msg.getField("message")).isEqualTo("the hello");
}
@Override
public boolean match(Message msg, StreamRule rule) {
Double msgVal = getDouble(msg.getField(rule.getField()));
if (msgVal == null) {
return false;
}
Double ruleVal = getDouble(rule.getValue());
if (ruleVal == null) {
return false;
}
return rule.getInverted() ^ (msgVal > ruleVal);
}
@Test
public void testCursorStrategyCutWithAllTextCut() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.cursorStrategy(CUT)
.sourceField("msg")
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("the hello", 0, 9)};
}
})
.build();
final Message msg = createMessage("message");
msg.addField("msg", "the hello");
extractor.runExtractor(msg);
// If all data is cut from the source field, the "fullyCutByExtractor" string gets inserted.
assertThat(msg.getField("msg")).isEqualTo("fullyCutByExtractor");
}
@Test
public void testCursorStrategyCutIfBeginAndEndIndexAreDisabled() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.cursorStrategy(CUT)
.sourceField("msg")
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("the", -1, -1)};
}
})
.build();
final Message msg = createMessage("message");
msg.addField("msg", "the hello");
extractor.runExtractor(msg);
// If the begin and end index is -1, the source field should not be modified.
assertThat(msg.getField("msg")).isEqualTo("the hello");
}
private static boolean checkRouting(String outputTypeClass, Message msg) {
// ElasticSearch gets all messages.
if (outputTypeClass.equals(ES_CLASS_NAME)) {
return true;
}
for (Stream stream : msg.getStreams()) {
if (((StreamImpl) stream).hasConfiguredOutputs(outputTypeClass)) {
return true;
}
}
// No stream had that output configured.
return false;
}
@Override
public boolean filter(Message msg, GraylogServer server) {
for (Blacklist blacklist : Blacklist.fetchAll()) {
for (BlacklistRule rule : blacklist.getRules()) {
if (Pattern.compile(rule.getTerm(), Pattern.DOTALL).matcher(msg.getMessage()).matches()) {
LOG.debug("Message <{}> is blacklisted. First match on {}", this, rule.getTerm());
// Done - This message is blacklisted.
return true;
}
}
}
return false;
}
@Test(expected = StringIndexOutOfBoundsException.class)
public void testCursorStrategyCutIfEndIndexIsDisabled() throws Exception {
final TestExtractor extractor =
new TestExtractor.Builder()
.cursorStrategy(CUT)
.sourceField("msg")
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("the", 0, -1)};
}
})
.build();
final Message msg = createMessage("message");
msg.addField("msg", "the hello");
extractor.runExtractor(msg);
// If the end index is -1, the source field should not be modified.
// TODO: The current implementation only checks if begin index is -1. Needs to be fixed.
assertThat(msg.getField("msg")).isEqualTo("the hello");
}
@Test
public void testRunExtractorCheckSourceValueIsString() throws Exception {
final TestExtractor extractor = new TestExtractor.Builder().sourceField("a_field").build();
// Extractor should not run for source field values that are not strings!
final Message msg1 = createMessage("the message");
msg1.addField("a_field", 1);
extractor.runExtractor(msg1);
assertThat(msg1.hasField("target")).isFalse();
// The extractor should run for a source field value of type string.
final Message msg2 = createMessage("the message");
msg2.addField("a_field", "the source");
extractor.runExtractor(msg2);
assertThat(msg2.hasField("target")).isTrue();
}
@Override
public String apply(final Message input) {
return input.getId();
}
@Override
public void write(Message msg) throws Exception {
if (shutdown || driverFailed) {
return;
}
try {
if (connection == null) {
reconnect();
}
if (connection == null) {
return;
}
synchronized (connection) {
int index = 1;
logInsert.setTimestamp(index++, new Timestamp(msg.getTimestamp().getMillis()));
logInsert.setString(index++, msg.getId());
logInsert.setString(index++, msg.getSource());
String ms = msg.getMessage();
if (ms != null && ms.length() > MAX_MESSAGE) {
ms = ms.substring(0, MAX_MESSAGE);
}
logInsert.setString(index++, ms);
if (fields != null) {
for (String f : fields) {
Object value = msg.getField(f);
String s = value != null ? value.toString() : null;
if (s == null) {
logInsert.setNull(index++, Types.VARCHAR);
} else {
if (s.length() > MAX_VALUE) {
s = s.substring(0, MAX_VALUE);
}
logInsert.setString(index++, s);
}
}
}
logInsert.executeUpdate();
if (logInsertAttribute != null) {
Object id = null;
ResultSet ids = logInsert.getGeneratedKeys();
while (ids != null && ids.next()) {
id = ids.getObject(1);
}
if (id != null) {
for (Entry<String, Object> e : msg.getFieldsEntries()) {
String name = e.getKey();
Object value = e.getValue();
String s = value != null ? value.toString() : null;
logInsertAttribute.setObject(1, id);
logInsertAttribute.setString(2, name);
if (s.length() > MAX_VALUE) {
s = s.substring(0, MAX_VALUE);
}
logInsertAttribute.setString(3, s);
logInsertAttribute.executeUpdate();
}
} else {
throw new SQLException("Failed to generate ID for primary log record!");
}
}
}
} catch (SQLException e) {
log.log(Level.WARNING, "JDBC output error: " + e.getMessage(), e);
if (connection != null) {
try {
connection.rollback();
connection.setAutoCommit(true);
} catch (SQLException ee) {
// Don`t care
}
}
connection = null;
} finally {
if (connection != null) {
connection.commit();
}
}
}
@Test
public void testConvertersWithExceptions() throws Exception {
final Converter converter1 =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
throw new NullPointerException("EEK");
}
})
.build();
final Converter converter2 =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
return input + "2";
}
})
.build();
final Converter converter3 =
new TestConverter.Builder()
.callback(
new Function<Object, Object>() {
@Nullable
@Override
public Object apply(Object input) {
throw new NullPointerException("EEK");
}
})
.build();
final TestExtractor extractor =
new TestExtractor.Builder()
.converters(Lists.newArrayList(converter1, converter2, converter3))
.callback(
new Callable<Result[]>() {
@Override
public Result[] call() throws Exception {
return new Result[] {new Result("converter", -1, -1)};
}
})
.build();
final Message msg = createMessage("message");
extractor.runExtractor(msg);
// The two exceptions should have been recorded.
assertThat(extractor.getConverterExceptionCount()).isEqualTo(2);
// It ignores all converters which throw an exception but executes the ones that don't.
// TODO: Is this really the expected behaviour? The converters are executed in order and
// basically depend on the output of the previous. This might not work for all converters.
assertThat(msg.getField("target")).isEqualTo("converter2");
}