@Override
public void dropOutdated() {
BasicDBObject query = new BasicDBObject();
query.put("last_seen", new BasicDBObject("$lt", Tools.getUTCTimestamp() - pingTimeout));
destroyAll(NodeImpl.class, query);
}
protected String buildDescription(
Stream stream, AlertCondition.CheckResult checkResult, Configuration configuration) {
StringBuilder sb = new StringBuilder();
sb.append(checkResult.getResultDescription());
sb.append("\n\n");
sb.append("*Date:* ").append("\n").append(Tools.iso8601().toString()).append("\n\n");
sb.append("*Stream ID:* ").append("\n").append(stream.getId()).append("\n\n");
sb.append("*Stream title:* ").append("\n").append(stream.getTitle()).append("\n\n");
sb.append("*Stream URL:* ")
.append("\n")
.append(buildStreamURL(configuration.getString(CK_GRAYLOG_URL), stream))
.append("\n\n");
sb.append("*Stream rules:* ").append("\n").append(buildStreamRules(stream)).append("\n\n");
sb.append("*Alert triggered at:* ")
.append("\n")
.append(checkResult.getTriggeredAt())
.append("\n\n");
sb.append("*Triggered condition:* ")
.append("\n")
.append(checkResult.getTriggeredCondition())
.append("\n\n");
return sb.toString();
}
@POST
@Timed
@Consumes(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Update extractor order of an input")
@ApiResponses(value = {@ApiResponse(code = 404, message = "No such input on this node.")})
@Path("order")
public void order(
@ApiParam(name = "inputId", value = "Persist ID (!) of input.", required = true)
@PathParam("inputId")
String inputPersistId,
@ApiParam(name = "JSON body", required = true) OrderExtractorsRequest oer)
throws NotFoundException {
checkPermission(RestPermissions.INPUTS_EDIT, inputPersistId);
final Input mongoInput = inputService.find(inputPersistId);
for (Extractor extractor : inputService.getExtractors(mongoInput)) {
if (oer.order().containsValue(extractor.getId())) {
extractor.setOrder(Tools.getKeyByValue(oer.order(), extractor.getId()));
}
// Docs embedded in MongoDB array cannot be updated atomically... :/
inputService.removeExtractor(mongoInput, extractor.getId());
try {
inputService.addExtractor(mongoInput, extractor);
} catch (ValidationException e) {
LOG.warn("Validation error for extractor update.", e);
}
}
LOG.info("Updated extractor ordering of input <persist:{}>.", inputPersistId);
}
public static AlarmCallbackHistory create(
String id,
AlarmCallbackConfiguration alarmCallbackConfiguration,
Alert alert,
AlertCondition alertCondition,
AlarmCallbackResult result) {
return create(id, alarmCallbackConfiguration, alert, alertCondition, result, Tools.nowUTC());
}
@Override
public boolean isAnyMasterPresent() {
BasicDBObject query = new BasicDBObject();
query.put("type", NodeImpl.Type.SERVER.toString());
query.put("last_seen", new BasicDBObject("$gte", Tools.getUTCTimestamp() - pingTimeout));
query.put("is_master", true);
return query(NodeImpl.class, query).size() > 0;
}
private DateTime parseDateTime(String to) {
final DateTimeFormatter formatter;
if (to.contains("T")) {
formatter = ISODateTimeFormat.dateTime();
} else {
formatter = Tools.timeFormatterWithOptionalMilliseconds();
}
// Use withOffsetParsed() to keep the timezone!
return formatter.withOffsetParsed().parseDateTime(to);
}
/**
* Mark this node as alive and probably update some settings that may have changed since last
* server boot.
*
* @param isMaster
* @param restTransportAddress
*/
@Override
public void markAsAlive(Node node, boolean isMaster, String restTransportAddress) {
node.getFields().put("last_seen", Tools.getUTCTimestamp());
node.getFields().put("is_master", isMaster);
node.getFields().put("transport_address", restTransportAddress);
try {
save(node);
} catch (ValidationException e) {
throw new RuntimeException("Validation failed.", e);
}
}
@Override
public Output create(CreateOutputRequest request, String userId) throws ValidationException {
return create(
OutputImpl.create(
new ObjectId().toHexString(),
request.title(),
request.type(),
userId,
request.configuration(),
Tools.iso8601().toDate(),
request.contentPack()));
}
public String getJSON() {
try {
switch (getGELFType()) {
case ZLIB:
return Tools.decompressZlib(payload);
case GZIP:
return Tools.decompressGzip(payload);
case UNCOMPRESSED:
return new String(payload, StandardCharsets.UTF_8);
case CHUNKED:
case UNSUPPORTED:
throw new IllegalStateException("Unknown GELF type. Not supported.");
}
} catch (final IOException e) {
// Note that the UnsupportedEncodingException thrown by 'new String' can never happen because
// UTF-8
// is a mandatory JRE encoding which is always present. So we only need to mention the
// decompress exceptions here.
throw new IllegalStateException("Failed to decompress the GELF message payload", e);
}
return null;
}
@Override
public String registerRadio(String nodeId, String restTransportUri) {
Map<String, Object> fields = Maps.newHashMap();
fields.put("last_seen", Tools.getUTCTimestamp());
fields.put("node_id", nodeId);
fields.put("type", NodeImpl.Type.RADIO.toString());
fields.put("transport_address", restTransportUri);
try {
return save(new NodeImpl(fields));
} catch (ValidationException e) {
throw new RuntimeException("Validation failed.", e);
}
}
protected void savePidFile(final String pidFile) {
final String pid = Tools.getPID();
final Path pidFilePath = Paths.get(pidFile);
pidFilePath.toFile().deleteOnExit();
try {
if (pid == null || pid.isEmpty() || pid.equals("unknown")) {
throw new Exception("Could not determine PID.");
}
Files.write(pidFilePath, pid.getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
LOG.error("Could not write PID file: " + e.getMessage(), e);
System.exit(1);
}
}
public Map<String, Object> getAsDatabaseObject() {
Map<String, Object> result = Maps.newHashMap();
Map<String, Map<String, Object>> indexInformation = Maps.newHashMap();
for (Map.Entry<String, IndexStats> e : indices.entrySet()) {
indexInformation.put(e.getKey(), getIndexInformation(e.getValue()));
}
result.put("server_id", serverId);
result.put("deflector_target", deflectorTarget);
result.put("max_messages_per_index", maxMessagesPerIndex);
result.put("indices", indexInformation);
result.put("timestamp", Tools.getUTCTimestamp());
return result;
}
@Test
@UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL)
public void testCreate() throws Exception {
final String title = "Dashboard Title";
final String description = "This is the dashboard description";
final String creatorUserId = "foobar";
final DateTime createdAt = Tools.nowUTC();
final Dashboard dashboard =
dashboardService.create(title, description, creatorUserId, createdAt);
assertNotNull(dashboard);
assertEquals(title, dashboard.getTitle());
assertEquals(description, dashboard.getDescription());
assertNotNull(dashboard.getId());
assertEquals(0, dashboardService.count());
}
@Override
public Map<String, Node> allActive(NodeImpl.Type type) {
Map<String, Node> nodes = Maps.newHashMap();
BasicDBObject query = new BasicDBObject();
query.put("last_seen", new BasicDBObject("$gte", Tools.getUTCTimestamp() - pingTimeout));
query.put("type", type.toString());
for (DBObject obj : query(NodeImpl.class, query)) {
Node node = new NodeImpl((ObjectId) obj.get("_id"), obj.toMap());
String nodeId = (String) obj.get("node_id");
nodes.put(nodeId, node);
}
return nodes;
}
@Test
public void runCodegen() throws IOException {
final Rule rule = parser.parseRule(ruleForTest(), true).withId("1");
final String sourceCode = CodeGenerator.sourceCodeForRule(rule);
Files.write(sourceCode, OUTFILE.toFile(), StandardCharsets.UTF_8);
log.info("Code:\n{}", sourceCode);
try {
ClassLoader ruleClassloader = new ClassLoader() {};
//noinspection unchecked
Class<GeneratedRule> rule$1 =
(Class<GeneratedRule>)
JCC.loadFromJava(
ruleClassloader,
"org.graylog.plugins.pipelineprocessor.$dynamic.rules.rule$1",
sourceCode);
//noinspection unchecked
final Set<Constructor> constructors =
ReflectionUtils.getConstructors(rule$1, input -> input.getParameterCount() == 1);
final Constructor onlyElement = Iterables.getOnlyElement(constructors);
final GeneratedRule generatedRule = (GeneratedRule) onlyElement.newInstance(functionRegistry);
final Message message = new Message("hello", "jenkins.torch.sh", Tools.nowUTC());
message.addField("message", "#1234");
message.addField("something_that_doesnt_exist", "foo");
final EvaluationContext context = new EvaluationContext(message);
final boolean when = generatedRule.when(context);
if (when) {
generatedRule.then(context);
}
log.info("created dynamic rule {} matches: {}", generatedRule.name(), when);
assertThat(context.currentMessage().hasField("some_identifier")).isTrue();
} catch (InvocationTargetException
| ClassNotFoundException
| InstantiationException
| IllegalAccessException e) {
log.error("Cannot load dynamically created class!", e);
}
}
private String dumpConfiguration(final Map<String, String> configMap) {
final StringBuilder sb = new StringBuilder();
sb.append("# Configuration of graylog2-")
.append(commandName)
.append(" ")
.append(version)
.append(System.lineSeparator());
sb.append("# Generated on ").append(Tools.iso8601()).append(System.lineSeparator());
for (Map.Entry<String, String> entry : configMap.entrySet()) {
sb.append(entry.getKey())
.append('=')
.append(nullToEmpty(entry.getValue()))
.append(System.lineSeparator());
}
return sb.toString();
}
protected AlertCondition getDummyAlertCondition(Map<String, Object> parameters) {
return new AlertCondition(
stream, CONDITION_ID, null, Tools.iso8601(), STREAM_CREATOR, parameters) {
@Override
public String getDescription() {
return null;
}
@Override
protected CheckResult runCheck(Indexer indexer) {
return null;
}
@Override
public List<ResultMessage> getSearchHits() {
return null;
}
};
}
@Override
protected CheckResult runCheck() {
try {
final String filter = "streams:" + stream.getId();
final CountResult result = searches.count("*", new RelativeRange(time * 60), filter);
final long count = result.getCount();
LOG.debug("Alert check <{}> result: [{}]", id, count);
final boolean triggered;
switch (thresholdType) {
case MORE:
triggered = count > threshold;
break;
case LESS:
triggered = count < threshold;
break;
default:
triggered = false;
}
if (triggered) {
final List<MessageSummary> summaries = Lists.newArrayList();
if (getBacklog() > 0) {
final SearchResult backlogResult =
searches.search(
"*",
filter,
new RelativeRange(time * 60),
getBacklog(),
0,
new Sorting("timestamp", Sorting.Direction.DESC));
for (ResultMessage resultMessage : backlogResult.getResults()) {
final Message msg = new Message(resultMessage.getMessage());
summaries.add(new MessageSummary(resultMessage.getIndex(), msg));
}
}
final String resultDescription =
"Stream had "
+ count
+ " messages in the last "
+ time
+ " minutes with trigger condition "
+ thresholdType.toString().toLowerCase()
+ " than "
+ threshold
+ " messages. "
+ "(Current grace time: "
+ grace
+ " minutes)";
return new CheckResult(true, this, resultDescription, Tools.iso8601(), summaries);
} else {
return new NegativeCheckResult(this);
}
} catch (InvalidRangeParametersException e) {
// cannot happen lol
LOG.error("Invalid timerange.", e);
return null;
} catch (InvalidRangeFormatException e) {
// lol same here
LOG.error("Invalid timerange format.", e);
return null;
}
}
protected FieldValueAlertCondition getFieldValueAlertCondition(Map<String, Object> parameters) {
return new FieldValueAlertCondition(
searches, stream, CONDITION_ID, Tools.nowUTC(), STREAM_CREATOR, parameters);
}
public Map<String, Object> toElasticSearchObject(@Nonnull final Meter invalidTimestampMeter) {
final Map<String, Object> obj =
Maps.newHashMapWithExpectedSize(REQUIRED_FIELDS.size() + fields.size());
for (Map.Entry<String, Object> entry : fields.entrySet()) {
final String key = entry.getKey();
// Elasticsearch does not allow "." characters in keys since version 2.0.
// See:
// https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots
if (key != null && key.contains(".")) {
final String newKey = key.replace('.', KEY_REPLACEMENT_CHAR);
// If the message already contains the transformed key, we skip the field and emit a
// warning.
// This is still not optimal but better than implementing expensive logic with multiple
// replacement
// character options. Conflicts should be rare...
if (!obj.containsKey(newKey)) {
obj.put(newKey, entry.getValue());
} else {
LOG.warn(
"Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"",
key,
entry.getValue(),
getId(),
KEY_REPLACEMENT_CHAR,
newKey,
obj.get(newKey));
LOG.debug("Full message with \".\" in message key: {}", this);
}
} else {
if (key != null && obj.containsKey(key)) {
final String newKey = key.replace(KEY_REPLACEMENT_CHAR, '.');
// Deliberate warning duplicates because the key with the "." might be transformed before
// reaching
// the duplicate original key with a "_". Otherwise we would silently overwrite the
// transformed key.
LOG.warn(
"Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"",
newKey,
fields.get(newKey),
getId(),
KEY_REPLACEMENT_CHAR,
key,
entry.getValue());
LOG.debug("Full message with \".\" in message key: {}", this);
}
obj.put(key, entry.getValue());
}
}
obj.put(FIELD_MESSAGE, getMessage());
obj.put(FIELD_SOURCE, getSource());
final Object timestampValue = getField(FIELD_TIMESTAMP);
DateTime dateTime;
if (timestampValue instanceof Date) {
dateTime = new DateTime(timestampValue);
} else if (timestampValue instanceof DateTime) {
dateTime = (DateTime) timestampValue;
} else if (timestampValue instanceof String) {
// if the timestamp value is a string, we try to parse it in the correct format.
// we fall back to "now", this avoids losing messages which happen to have the wrong timestamp
// format
try {
dateTime = ES_DATE_FORMAT_FORMATTER.parseDateTime((String) timestampValue);
} catch (IllegalArgumentException e) {
LOG.trace(
"Invalid format for field timestamp '{}' in message {}, forcing to current time.",
timestampValue,
getId());
invalidTimestampMeter.mark();
dateTime = Tools.nowUTC();
}
} else {
// don't allow any other types for timestamp, force to "now"
LOG.trace(
"Invalid type for field timestamp '{}' in message {}, forcing to current time.",
timestampValue.getClass().getSimpleName(),
getId());
invalidTimestampMeter.mark();
dateTime = Tools.nowUTC();
}
if (dateTime != null) {
obj.put(FIELD_TIMESTAMP, buildElasticSearchTimeFormat(dateTime.withZone(UTC)));
}
// Manually converting stream ID to string - caused strange problems without it.
if (getStreams().isEmpty()) {
obj.put(FIELD_STREAMS, Collections.emptyList());
} else {
final List<String> streamIds = Lists.newArrayListWithCapacity(streams.size());
for (Stream stream : streams) {
streamIds.add(stream.getId());
}
obj.put(FIELD_STREAMS, streamIds);
}
return obj;
}
public DeadLetter(BulkItemResponse failure, Message message) {
this.timestamp = Tools.iso8601();
this.id = new UUID().toString();
this.failure = failure;
this.message = message;
}
@Override
protected void startCommand() {
LOG.info(
"Graylog " + commandName + " {} starting up. (JRE: {})",
version,
Tools.getSystemInformation());
// Do not use a PID file if the user requested not to
if (!isNoPidFile()) {
savePidFile(getPidFile());
}
final ServerStatus serverStatus = injector.getInstance(ServerStatus.class);
serverStatus.initialize();
startNodeRegistration(injector);
final ActivityWriter activityWriter;
final ServiceManager serviceManager;
try {
activityWriter = injector.getInstance(ActivityWriter.class);
serviceManager = injector.getInstance(ServiceManager.class);
} catch (ProvisionException e) {
LOG.error("Guice error", e);
annotateProvisionException(e);
System.exit(-1);
return;
} catch (Exception e) {
LOG.error("Unexpected exception", e);
System.exit(-1);
return;
}
Runtime.getRuntime().addShutdownHook(new Thread(injector.getInstance(shutdownHook())));
// propagate default size to input plugins
MessageInput.setDefaultRecvBufferSize(configuration.getUdpRecvBufferSizes());
// Start services.
final ServiceManagerListener serviceManagerListener =
injector.getInstance(ServiceManagerListener.class);
serviceManager.addListener(serviceManagerListener);
try {
serviceManager.startAsync().awaitHealthy();
} catch (Exception e) {
try {
serviceManager
.stopAsync()
.awaitStopped(configuration.getShutdownTimeout(), TimeUnit.MILLISECONDS);
} catch (TimeoutException timeoutException) {
LOG.error("Unable to shutdown properly on time. {}", serviceManager.servicesByState());
}
LOG.error("Graylog startup failed. Exiting. Exception was:", e);
System.exit(-1);
}
LOG.info("Services started, startup times in ms: {}", serviceManager.startupTimes());
activityWriter.write(new Activity("Started up.", Main.class));
LOG.info("Graylog " + commandName + " up and running.");
// Block forever.
try {
Thread.currentThread().join();
} catch (InterruptedException e) {
return;
}
}