/** {@inheritDoc} */ @Override protected final void handleMessages(List<T> messages) { long start = System.nanoTime(); // Create a batch of message that we want to write. List<RegularStatement> statements = new ArrayList<>(); for (T t : messages) { try { handleMessage(statements, t); } catch (RuntimeException e) { LOG.warn("Failed to write message: " + t, e); // Just in case we cannot process a message } } // Try writing the batch try { Batch batch = QueryBuilder.batch(statements.toArray(new RegularStatement[statements.size()])); long beforeSend = System.nanoTime(); ResultSetFuture f = connection.getSession().executeAsync(batch); f.getUninterruptibly(); // throws QueryValidationExecption etc long total = System.nanoTime(); // Is this an abnormal slow batch? boolean isSlow = TimeUnit.MILLISECONDS.convert(total - start, TimeUnit.NANOSECONDS) > 200 || messages.size() >= getBatchSize(); if (isSlow || lastSlowBatch > 0) { LOG.info( "Total time: " + DurationFormatter.DEFAULT.formatNanos(total - start) + ", prepping=" + DurationFormatter.DEFAULT.formatNanos(beforeSend - start) + ", sending=" + DurationFormatter.DEFAULT.formatNanos(total - beforeSend) + ", size=" + messages.size()); // makes sure we write 10 info statements after the last slow batch we insert lastSlowBatch = isSlow ? 10 : lastSlowBatch - 1; } persistedCount.mark(messages.size()); // sink.onSucces(messages); } catch (QueryValidationException e) { LOG.error("Could not execute query, this is an internal error", e); } catch (Exception e) { onFailure(messages, e); try { sleepUntilShutdown(2, TimeUnit.SECONDS); } catch (InterruptedException ignore) { Thread.interrupted(); } } }
/** * Query the database for packets from the supplied mmsi numbers and with transmission timestamps * at or after t0 and at or before t1. * * <p>Intended for "small" queries where the query result can be kept in memory. * * @param sourceFilterPredicate * @param t0 * @param t1 * @param mmsi * @return */ public List<IPositionMessage> findByMmsi( Predicate<AisPacketSource> sourceFilterPredicate, Instant t0, Instant t1, int mmsi) { AisStoreQueryBuilder query = AisStoreQueryBuilder.forMmsi(mmsi).setInterval(t0, t1).setFetchSize(1000); AisStoreQueryResult result = cassandraConnection.execute(query); List<IPositionMessage> pastPositionMessages = StreamSupport.stream(result.spliterator(), false) .filter(packet -> sourceFilterPredicate.test(AisPacketSource.create(packet))) .map(packet -> packet.tryGetAisMessage()) .filter(msg -> msg instanceof IPositionMessage) .map(msg -> (IPositionMessage) msg) .collect(Collectors.toList()); return pastPositionMessages; }