@Override
  public void dropOutdated() {
    BasicDBObject query = new BasicDBObject();
    query.put("last_seen", new BasicDBObject("$lt", Tools.getUTCTimestamp() - pingTimeout));

    destroyAll(NodeImpl.class, query);
  }
  protected String buildDescription(
      Stream stream, AlertCondition.CheckResult checkResult, Configuration configuration) {
    StringBuilder sb = new StringBuilder();

    sb.append(checkResult.getResultDescription());
    sb.append("\n\n");
    sb.append("*Date:* ").append("\n").append(Tools.iso8601().toString()).append("\n\n");
    sb.append("*Stream ID:* ").append("\n").append(stream.getId()).append("\n\n");
    sb.append("*Stream title:* ").append("\n").append(stream.getTitle()).append("\n\n");
    sb.append("*Stream URL:* ")
        .append("\n")
        .append(buildStreamURL(configuration.getString(CK_GRAYLOG_URL), stream))
        .append("\n\n");
    sb.append("*Stream rules:* ").append("\n").append(buildStreamRules(stream)).append("\n\n");
    sb.append("*Alert triggered at:* ")
        .append("\n")
        .append(checkResult.getTriggeredAt())
        .append("\n\n");
    sb.append("*Triggered condition:* ")
        .append("\n")
        .append(checkResult.getTriggeredCondition())
        .append("\n\n");

    return sb.toString();
  }
  @POST
  @Timed
  @Consumes(MediaType.APPLICATION_JSON)
  @ApiOperation(value = "Update extractor order of an input")
  @ApiResponses(value = {@ApiResponse(code = 404, message = "No such input on this node.")})
  @Path("order")
  public void order(
      @ApiParam(name = "inputId", value = "Persist ID (!) of input.", required = true)
          @PathParam("inputId")
          String inputPersistId,
      @ApiParam(name = "JSON body", required = true) OrderExtractorsRequest oer)
      throws NotFoundException {
    checkPermission(RestPermissions.INPUTS_EDIT, inputPersistId);

    final Input mongoInput = inputService.find(inputPersistId);

    for (Extractor extractor : inputService.getExtractors(mongoInput)) {
      if (oer.order().containsValue(extractor.getId())) {
        extractor.setOrder(Tools.getKeyByValue(oer.order(), extractor.getId()));
      }

      // Docs embedded in MongoDB array cannot be updated atomically... :/
      inputService.removeExtractor(mongoInput, extractor.getId());
      try {
        inputService.addExtractor(mongoInput, extractor);
      } catch (ValidationException e) {
        LOG.warn("Validation error for extractor update.", e);
      }
    }

    LOG.info("Updated extractor ordering of input <persist:{}>.", inputPersistId);
  }
 public static AlarmCallbackHistory create(
     String id,
     AlarmCallbackConfiguration alarmCallbackConfiguration,
     Alert alert,
     AlertCondition alertCondition,
     AlarmCallbackResult result) {
   return create(id, alarmCallbackConfiguration, alert, alertCondition, result, Tools.nowUTC());
 }
  @Override
  public boolean isAnyMasterPresent() {
    BasicDBObject query = new BasicDBObject();
    query.put("type", NodeImpl.Type.SERVER.toString());
    query.put("last_seen", new BasicDBObject("$gte", Tools.getUTCTimestamp() - pingTimeout));
    query.put("is_master", true);

    return query(NodeImpl.class, query).size() > 0;
  }
 private DateTime parseDateTime(String to) {
   final DateTimeFormatter formatter;
   if (to.contains("T")) {
     formatter = ISODateTimeFormat.dateTime();
   } else {
     formatter = Tools.timeFormatterWithOptionalMilliseconds();
   }
   // Use withOffsetParsed() to keep the timezone!
   return formatter.withOffsetParsed().parseDateTime(to);
 }
 /**
  * Mark this node as alive and probably update some settings that may have changed since last
  * server boot.
  *
  * @param isMaster
  * @param restTransportAddress
  */
 @Override
 public void markAsAlive(Node node, boolean isMaster, String restTransportAddress) {
   node.getFields().put("last_seen", Tools.getUTCTimestamp());
   node.getFields().put("is_master", isMaster);
   node.getFields().put("transport_address", restTransportAddress);
   try {
     save(node);
   } catch (ValidationException e) {
     throw new RuntimeException("Validation failed.", e);
   }
 }
 @Override
 public Output create(CreateOutputRequest request, String userId) throws ValidationException {
   return create(
       OutputImpl.create(
           new ObjectId().toHexString(),
           request.title(),
           request.type(),
           userId,
           request.configuration(),
           Tools.iso8601().toDate(),
           request.contentPack()));
 }
 public String getJSON() {
   try {
     switch (getGELFType()) {
       case ZLIB:
         return Tools.decompressZlib(payload);
       case GZIP:
         return Tools.decompressGzip(payload);
       case UNCOMPRESSED:
         return new String(payload, StandardCharsets.UTF_8);
       case CHUNKED:
       case UNSUPPORTED:
         throw new IllegalStateException("Unknown GELF type. Not supported.");
     }
   } catch (final IOException e) {
     // Note that the UnsupportedEncodingException thrown by 'new String' can never happen because
     // UTF-8
     // is a mandatory JRE encoding which is always present. So we only need to mention the
     // decompress exceptions here.
     throw new IllegalStateException("Failed to decompress the GELF message payload", e);
   }
   return null;
 }
  @Override
  public String registerRadio(String nodeId, String restTransportUri) {
    Map<String, Object> fields = Maps.newHashMap();
    fields.put("last_seen", Tools.getUTCTimestamp());
    fields.put("node_id", nodeId);
    fields.put("type", NodeImpl.Type.RADIO.toString());
    fields.put("transport_address", restTransportUri);

    try {
      return save(new NodeImpl(fields));
    } catch (ValidationException e) {
      throw new RuntimeException("Validation failed.", e);
    }
  }
  protected void savePidFile(final String pidFile) {
    final String pid = Tools.getPID();
    final Path pidFilePath = Paths.get(pidFile);
    pidFilePath.toFile().deleteOnExit();

    try {
      if (pid == null || pid.isEmpty() || pid.equals("unknown")) {
        throw new Exception("Could not determine PID.");
      }

      Files.write(pidFilePath, pid.getBytes(StandardCharsets.UTF_8));
    } catch (Exception e) {
      LOG.error("Could not write PID file: " + e.getMessage(), e);
      System.exit(1);
    }
  }
  public Map<String, Object> getAsDatabaseObject() {
    Map<String, Object> result = Maps.newHashMap();

    Map<String, Map<String, Object>> indexInformation = Maps.newHashMap();
    for (Map.Entry<String, IndexStats> e : indices.entrySet()) {
      indexInformation.put(e.getKey(), getIndexInformation(e.getValue()));
    }

    result.put("server_id", serverId);
    result.put("deflector_target", deflectorTarget);
    result.put("max_messages_per_index", maxMessagesPerIndex);
    result.put("indices", indexInformation);
    result.put("timestamp", Tools.getUTCTimestamp());

    return result;
  }
  @Test
  @UsingDataSet(loadStrategy = LoadStrategyEnum.DELETE_ALL)
  public void testCreate() throws Exception {
    final String title = "Dashboard Title";
    final String description = "This is the dashboard description";
    final String creatorUserId = "foobar";
    final DateTime createdAt = Tools.nowUTC();

    final Dashboard dashboard =
        dashboardService.create(title, description, creatorUserId, createdAt);

    assertNotNull(dashboard);
    assertEquals(title, dashboard.getTitle());
    assertEquals(description, dashboard.getDescription());
    assertNotNull(dashboard.getId());
    assertEquals(0, dashboardService.count());
  }
  @Override
  public Map<String, Node> allActive(NodeImpl.Type type) {
    Map<String, Node> nodes = Maps.newHashMap();

    BasicDBObject query = new BasicDBObject();
    query.put("last_seen", new BasicDBObject("$gte", Tools.getUTCTimestamp() - pingTimeout));
    query.put("type", type.toString());

    for (DBObject obj : query(NodeImpl.class, query)) {
      Node node = new NodeImpl((ObjectId) obj.get("_id"), obj.toMap());
      String nodeId = (String) obj.get("node_id");

      nodes.put(nodeId, node);
    }

    return nodes;
  }
  @Test
  public void runCodegen() throws IOException {
    final Rule rule = parser.parseRule(ruleForTest(), true).withId("1");

    final String sourceCode = CodeGenerator.sourceCodeForRule(rule);
    Files.write(sourceCode, OUTFILE.toFile(), StandardCharsets.UTF_8);

    log.info("Code:\n{}", sourceCode);

    try {

      ClassLoader ruleClassloader = new ClassLoader() {};
      //noinspection unchecked
      Class<GeneratedRule> rule$1 =
          (Class<GeneratedRule>)
              JCC.loadFromJava(
                  ruleClassloader,
                  "org.graylog.plugins.pipelineprocessor.$dynamic.rules.rule$1",
                  sourceCode);

      //noinspection unchecked
      final Set<Constructor> constructors =
          ReflectionUtils.getConstructors(rule$1, input -> input.getParameterCount() == 1);
      final Constructor onlyElement = Iterables.getOnlyElement(constructors);
      final GeneratedRule generatedRule = (GeneratedRule) onlyElement.newInstance(functionRegistry);

      final Message message = new Message("hello", "jenkins.torch.sh", Tools.nowUTC());
      message.addField("message", "#1234");
      message.addField("something_that_doesnt_exist", "foo");
      final EvaluationContext context = new EvaluationContext(message);

      final boolean when = generatedRule.when(context);
      if (when) {
        generatedRule.then(context);
      }
      log.info("created dynamic rule {} matches: {}", generatedRule.name(), when);

      assertThat(context.currentMessage().hasField("some_identifier")).isTrue();

    } catch (InvocationTargetException
        | ClassNotFoundException
        | InstantiationException
        | IllegalAccessException e) {
      log.error("Cannot load dynamically created class!", e);
    }
  }
Example #16
0
  private String dumpConfiguration(final Map<String, String> configMap) {
    final StringBuilder sb = new StringBuilder();
    sb.append("# Configuration of graylog2-")
        .append(commandName)
        .append(" ")
        .append(version)
        .append(System.lineSeparator());
    sb.append("# Generated on ").append(Tools.iso8601()).append(System.lineSeparator());

    for (Map.Entry<String, String> entry : configMap.entrySet()) {
      sb.append(entry.getKey())
          .append('=')
          .append(nullToEmpty(entry.getValue()))
          .append(System.lineSeparator());
    }

    return sb.toString();
  }
  protected AlertCondition getDummyAlertCondition(Map<String, Object> parameters) {
    return new AlertCondition(
        stream, CONDITION_ID, null, Tools.iso8601(), STREAM_CREATOR, parameters) {
      @Override
      public String getDescription() {
        return null;
      }

      @Override
      protected CheckResult runCheck(Indexer indexer) {
        return null;
      }

      @Override
      public List<ResultMessage> getSearchHits() {
        return null;
      }
    };
  }
  @Override
  protected CheckResult runCheck() {
    try {
      final String filter = "streams:" + stream.getId();
      final CountResult result = searches.count("*", new RelativeRange(time * 60), filter);
      final long count = result.getCount();

      LOG.debug("Alert check <{}> result: [{}]", id, count);

      final boolean triggered;
      switch (thresholdType) {
        case MORE:
          triggered = count > threshold;
          break;
        case LESS:
          triggered = count < threshold;
          break;
        default:
          triggered = false;
      }

      if (triggered) {
        final List<MessageSummary> summaries = Lists.newArrayList();
        if (getBacklog() > 0) {
          final SearchResult backlogResult =
              searches.search(
                  "*",
                  filter,
                  new RelativeRange(time * 60),
                  getBacklog(),
                  0,
                  new Sorting("timestamp", Sorting.Direction.DESC));
          for (ResultMessage resultMessage : backlogResult.getResults()) {
            final Message msg = new Message(resultMessage.getMessage());
            summaries.add(new MessageSummary(resultMessage.getIndex(), msg));
          }
        }

        final String resultDescription =
            "Stream had "
                + count
                + " messages in the last "
                + time
                + " minutes with trigger condition "
                + thresholdType.toString().toLowerCase()
                + " than "
                + threshold
                + " messages. "
                + "(Current grace time: "
                + grace
                + " minutes)";
        return new CheckResult(true, this, resultDescription, Tools.iso8601(), summaries);
      } else {
        return new NegativeCheckResult(this);
      }
    } catch (InvalidRangeParametersException e) {
      // cannot happen lol
      LOG.error("Invalid timerange.", e);
      return null;
    } catch (InvalidRangeFormatException e) {
      // lol same here
      LOG.error("Invalid timerange format.", e);
      return null;
    }
  }
 protected FieldValueAlertCondition getFieldValueAlertCondition(Map<String, Object> parameters) {
   return new FieldValueAlertCondition(
       searches, stream, CONDITION_ID, Tools.nowUTC(), STREAM_CREATOR, parameters);
 }
Example #20
0
  public Map<String, Object> toElasticSearchObject(@Nonnull final Meter invalidTimestampMeter) {
    final Map<String, Object> obj =
        Maps.newHashMapWithExpectedSize(REQUIRED_FIELDS.size() + fields.size());

    for (Map.Entry<String, Object> entry : fields.entrySet()) {
      final String key = entry.getKey();

      // Elasticsearch does not allow "." characters in keys since version 2.0.
      // See:
      // https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots
      if (key != null && key.contains(".")) {
        final String newKey = key.replace('.', KEY_REPLACEMENT_CHAR);

        // If the message already contains the transformed key, we skip the field and emit a
        // warning.
        // This is still not optimal but better than implementing expensive logic with multiple
        // replacement
        // character options. Conflicts should be rare...
        if (!obj.containsKey(newKey)) {
          obj.put(newKey, entry.getValue());
        } else {
          LOG.warn(
              "Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"",
              key,
              entry.getValue(),
              getId(),
              KEY_REPLACEMENT_CHAR,
              newKey,
              obj.get(newKey));
          LOG.debug("Full message with \".\" in message key: {}", this);
        }
      } else {
        if (key != null && obj.containsKey(key)) {
          final String newKey = key.replace(KEY_REPLACEMENT_CHAR, '.');
          // Deliberate warning duplicates because the key with the "." might be transformed before
          // reaching
          // the duplicate original key with a "_". Otherwise we would silently overwrite the
          // transformed key.
          LOG.warn(
              "Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"",
              newKey,
              fields.get(newKey),
              getId(),
              KEY_REPLACEMENT_CHAR,
              key,
              entry.getValue());
          LOG.debug("Full message with \".\" in message key: {}", this);
        }
        obj.put(key, entry.getValue());
      }
    }

    obj.put(FIELD_MESSAGE, getMessage());
    obj.put(FIELD_SOURCE, getSource());

    final Object timestampValue = getField(FIELD_TIMESTAMP);
    DateTime dateTime;
    if (timestampValue instanceof Date) {
      dateTime = new DateTime(timestampValue);
    } else if (timestampValue instanceof DateTime) {
      dateTime = (DateTime) timestampValue;
    } else if (timestampValue instanceof String) {
      // if the timestamp value is a string, we try to parse it in the correct format.
      // we fall back to "now", this avoids losing messages which happen to have the wrong timestamp
      // format
      try {
        dateTime = ES_DATE_FORMAT_FORMATTER.parseDateTime((String) timestampValue);
      } catch (IllegalArgumentException e) {
        LOG.trace(
            "Invalid format for field timestamp '{}' in message {}, forcing to current time.",
            timestampValue,
            getId());
        invalidTimestampMeter.mark();
        dateTime = Tools.nowUTC();
      }
    } else {
      // don't allow any other types for timestamp, force to "now"
      LOG.trace(
          "Invalid type for field timestamp '{}' in message {}, forcing to current time.",
          timestampValue.getClass().getSimpleName(),
          getId());
      invalidTimestampMeter.mark();
      dateTime = Tools.nowUTC();
    }
    if (dateTime != null) {
      obj.put(FIELD_TIMESTAMP, buildElasticSearchTimeFormat(dateTime.withZone(UTC)));
    }

    // Manually converting stream ID to string - caused strange problems without it.
    if (getStreams().isEmpty()) {
      obj.put(FIELD_STREAMS, Collections.emptyList());
    } else {
      final List<String> streamIds = Lists.newArrayListWithCapacity(streams.size());
      for (Stream stream : streams) {
        streamIds.add(stream.getId());
      }
      obj.put(FIELD_STREAMS, streamIds);
    }

    return obj;
  }
Example #21
0
 public DeadLetter(BulkItemResponse failure, Message message) {
   this.timestamp = Tools.iso8601();
   this.id = new UUID().toString();
   this.failure = failure;
   this.message = message;
 }
  @Override
  protected void startCommand() {
    LOG.info(
        "Graylog " + commandName + " {} starting up. (JRE: {})",
        version,
        Tools.getSystemInformation());

    // Do not use a PID file if the user requested not to
    if (!isNoPidFile()) {
      savePidFile(getPidFile());
    }

    final ServerStatus serverStatus = injector.getInstance(ServerStatus.class);
    serverStatus.initialize();

    startNodeRegistration(injector);

    final ActivityWriter activityWriter;
    final ServiceManager serviceManager;
    try {
      activityWriter = injector.getInstance(ActivityWriter.class);
      serviceManager = injector.getInstance(ServiceManager.class);
    } catch (ProvisionException e) {
      LOG.error("Guice error", e);
      annotateProvisionException(e);
      System.exit(-1);
      return;
    } catch (Exception e) {
      LOG.error("Unexpected exception", e);
      System.exit(-1);
      return;
    }

    Runtime.getRuntime().addShutdownHook(new Thread(injector.getInstance(shutdownHook())));

    // propagate default size to input plugins
    MessageInput.setDefaultRecvBufferSize(configuration.getUdpRecvBufferSizes());

    // Start services.
    final ServiceManagerListener serviceManagerListener =
        injector.getInstance(ServiceManagerListener.class);
    serviceManager.addListener(serviceManagerListener);
    try {
      serviceManager.startAsync().awaitHealthy();
    } catch (Exception e) {
      try {
        serviceManager
            .stopAsync()
            .awaitStopped(configuration.getShutdownTimeout(), TimeUnit.MILLISECONDS);
      } catch (TimeoutException timeoutException) {
        LOG.error("Unable to shutdown properly on time. {}", serviceManager.servicesByState());
      }
      LOG.error("Graylog startup failed. Exiting. Exception was:", e);
      System.exit(-1);
    }
    LOG.info("Services started, startup times in ms: {}", serviceManager.startupTimes());

    activityWriter.write(new Activity("Started up.", Main.class));
    LOG.info("Graylog " + commandName + " up and running.");

    // Block forever.
    try {
      Thread.currentThread().join();
    } catch (InterruptedException e) {
      return;
    }
  }