/** {@inheritDoc} */
  @Override
  protected final void handleMessages(List<T> messages) {
    long start = System.nanoTime();
    // Create a batch of message that we want to write.
    List<RegularStatement> statements = new ArrayList<>();
    for (T t : messages) {
      try {
        handleMessage(statements, t);
      } catch (RuntimeException e) {
        LOG.warn("Failed to write message: " + t, e); // Just in case we cannot process a message
      }
    }

    // Try writing the batch
    try {
      Batch batch = QueryBuilder.batch(statements.toArray(new RegularStatement[statements.size()]));

      long beforeSend = System.nanoTime();

      ResultSetFuture f = connection.getSession().executeAsync(batch);
      f.getUninterruptibly(); // throws QueryValidationExecption etc

      long total = System.nanoTime();
      // Is this an abnormal slow batch?
      boolean isSlow =
          TimeUnit.MILLISECONDS.convert(total - start, TimeUnit.NANOSECONDS) > 200
              || messages.size() >= getBatchSize();
      if (isSlow || lastSlowBatch > 0) {
        LOG.info(
            "Total time: "
                + DurationFormatter.DEFAULT.formatNanos(total - start)
                + ", prepping="
                + DurationFormatter.DEFAULT.formatNanos(beforeSend - start)
                + ", sending="
                + DurationFormatter.DEFAULT.formatNanos(total - beforeSend)
                + ", size="
                + messages.size());
        // makes sure we write 10 info statements after the last slow batch we insert
        lastSlowBatch = isSlow ? 10 : lastSlowBatch - 1;
      }
      persistedCount.mark(messages.size());
      // sink.onSucces(messages);
    } catch (QueryValidationException e) {
      LOG.error("Could not execute query, this is an internal error", e);
    } catch (Exception e) {
      onFailure(messages, e);
      try {
        sleepUntilShutdown(2, TimeUnit.SECONDS);
      } catch (InterruptedException ignore) {
        Thread.interrupted();
      }
    }
  }
Ejemplo n.º 2
0
  /**
   * Query the database for packets from the supplied mmsi numbers and with transmission timestamps
   * at or after t0 and at or before t1.
   *
   * <p>Intended for "small" queries where the query result can be kept in memory.
   *
   * @param sourceFilterPredicate
   * @param t0
   * @param t1
   * @param mmsi
   * @return
   */
  public List<IPositionMessage> findByMmsi(
      Predicate<AisPacketSource> sourceFilterPredicate, Instant t0, Instant t1, int mmsi) {

    AisStoreQueryBuilder query =
        AisStoreQueryBuilder.forMmsi(mmsi).setInterval(t0, t1).setFetchSize(1000);

    AisStoreQueryResult result = cassandraConnection.execute(query);

    List<IPositionMessage> pastPositionMessages =
        StreamSupport.stream(result.spliterator(), false)
            .filter(packet -> sourceFilterPredicate.test(AisPacketSource.create(packet)))
            .map(packet -> packet.tryGetAisMessage())
            .filter(msg -> msg instanceof IPositionMessage)
            .map(msg -> (IPositionMessage) msg)
            .collect(Collectors.toList());

    return pastPositionMessages;
  }