public void init(FilterConfig filterConfig) throws ServletException { final MetricRegistry metricsRegistry = Metrics.metricRegistry(); this.metersByStatusCode = new ConcurrentHashMap<Integer, Meter>(meterNamesByStatusCode.size()); for (Map.Entry<Integer, String> entry : meterNamesByStatusCode.entrySet()) { metersByStatusCode.put(entry.getKey(), metricsRegistry.meter(name("http", entry.getValue()))); } this.otherMeter = metricsRegistry.meter(name("http", otherMetricName)); this.activeRequests = metricsRegistry.counter(name("http", "activeRequests")); this.requestTimer = metricsRegistry.timer(name("http", "requests")); }
@Override public void doRun() { writtenMessages = metricRegistry.meter(name(InputCacheWorkerThread.class, "writtenMessages")); outOfCapacity = metricRegistry.meter(name(InputCacheWorkerThread.class, "FailedWritesOutOfCapacity")); new Thread( new Runnable() { @Override public void run() { work(inputCache, processBuffer); } }, "master-cache-worker-input") .start(); }
public static void main(String[] args) { // create some various metrics and update them MetricRegistry registry = new MetricRegistry(); final AtomicInteger gaugeInteger = new AtomicInteger(); registry.register( name("gauge"), new Gauge<Integer>() { @Override public Integer getValue() { return gaugeInteger.get(); } }); final Counter counter = registry.counter(name("counter")); final Histogram histogram = registry.histogram(name("histogram")); final Meter meter = registry.meter(name("meter")); final Timer timer = registry.timer(name("timer")); NewRelicReporter reporter = new NewRelicReporter( registry, "new relic reporter", MetricFilter.ALL, new AllEnabledMetricAttributeFilter(), TimeUnit.SECONDS, TimeUnit.MILLISECONDS, "foo/"); reporter.start(60, TimeUnit.SECONDS); ScheduledExecutorService svc = Executors.newScheduledThreadPool(1); final Random random = new Random(); svc.scheduleAtFixedRate( new Runnable() { @Override public void run() { System.out.println("Updating"); gaugeInteger.incrementAndGet(); counter.inc(); histogram.update(random.nextInt(10)); meter.mark(); timer.update(random.nextInt(10), TimeUnit.MILLISECONDS); } }, 0, 1, TimeUnit.SECONDS); }
@Test public void testMeter() { System.out.println("******************************* METER *******************************"); meter = registry.meter("meter"); try { for (int i = 0; i < ITER_COUNT; i++) { meter.mark(); Thread.sleep(SLEEP_MS); } } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } }
@Test public void testMeter() { final Meter meter = registry.meter(name("foo", "bar")); meter.mark(10); meter.mark(20); reportAndRefresh(); SearchResponse searchResponse = client().prepareSearch(indexWithDate).setTypes("meter").execute().actionGet(); org.assertj.core.api.Assertions.assertThat(searchResponse.getHits().totalHits()).isEqualTo(1L); Map<String, Object> hit = searchResponse.getHits().getAt(0).sourceAsMap(); assertTimestamp(hit); assertKey(hit, "name", prefix + ".foo.bar"); assertKey(hit, "count", 30); assertKey(hit, "host", "localhost"); }
public CodaHaleMetricsTracker( final String poolName, final PoolStats poolStats, final MetricRegistry registry) { this.poolName = poolName; this.registry = registry; this.connectionObtainTimer = registry.timer(MetricRegistry.name(poolName, "pool", "Wait")); this.connectionUsage = registry.histogram(MetricRegistry.name(poolName, "pool", "Usage")); this.connectionTimeoutMeter = registry.meter(MetricRegistry.name(poolName, "pool", "ConnectionTimeoutRate")); registry.register( MetricRegistry.name(poolName, "pool", "TotalConnections"), new Gauge<Integer>() { @Override public Integer getValue() { return poolStats.getTotalConnections(); } }); registry.register( MetricRegistry.name(poolName, "pool", "IdleConnections"), new Gauge<Integer>() { @Override public Integer getValue() { return poolStats.getIdleConnections(); } }); registry.register( MetricRegistry.name(poolName, "pool", "ActiveConnections"), new Gauge<Integer>() { @Override public Integer getValue() { return poolStats.getActiveConnections(); } }); registry.register( MetricRegistry.name(poolName, "pool", "PendingConnections"), new Gauge<Integer>() { @Override public Integer getValue() { return poolStats.getPendingThreads(); } }); }
@Override public void updateQueuedSpans(int update) { registry.meter("tracing.reporter.queued.span").mark(); }
@Override public void incrementSpansDropped(int quantity) { registry.meter("tracing.reporter.span.dropped").mark(quantity); }
@Override public void incrementMessagesDropped(Throwable cause) { registry.meter("tracing.reporter.message.dropped").mark(); }
@Override public void incrementMessages() { registry.meter("tracing.reporter.message.accepted").mark(); }
public static void markResourceMeter(String... path) { metricRegistry.meter(name("resource", path)).mark(); }
public static Meter meter(String name) { return metrics.meter(name); }
@Test public void writeThousandsSingleSource() throws InterruptedException, ExecutionException, MigrationException, UnsupportedEncodingException { final Id sourceId = IdGenerator.createId("source"); final String edgeType = "test"; final EdgeGenerator generator = new EdgeGenerator() { @Override public Edge newEdge() { Edge edge = createEdge(sourceId, edgeType, IdGenerator.createId("target")); return edge; } @Override public Observable<MarkedEdge> doSearch(final GraphManager manager) { return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent())); } }; // final int numInjectors = 2; final int numInjectors = 1; /** * create 3 injectors. This way all the caches are independent of one another. This is the same * as multiple nodes */ final List<Injector> injectors = createInjectors(numInjectors); final GraphFig graphFig = getInstance(injectors, GraphFig.class); final long shardSize = graphFig.getShardSize(); // we don't want to starve the cass runtime since it will be on the same box. Only take 50% of // processing // power for writes final int numProcessors = Runtime.getRuntime().availableProcessors() / 2; final int numWorkersPerInjector = numProcessors / numInjectors; /** Do 4x shard size so we should have approximately 4 shards */ final long numberOfEdges = shardSize * 4; final long workerWriteLimit = numberOfEdges / numWorkersPerInjector / numInjectors; final long expectedShardCount = numberOfEdges / shardSize; createExecutor(numWorkersPerInjector); final AtomicLong writeCounter = new AtomicLong(); // min stop time the min delta + 1 cache cycle timeout final long minExecutionTime = graphFig.getShardMinDelta() + graphFig.getShardCacheTimeout(); logger.info( "Writing {} edges per worker on {} workers in {} injectors", workerWriteLimit, numWorkersPerInjector, numInjectors); final List<Future<Boolean>> futures = new ArrayList<>(); for (Injector injector : injectors) { final GraphManagerFactory gmf = injector.getInstance(GraphManagerFactory.class); for (int i = 0; i < numWorkersPerInjector; i++) { Future<Boolean> future = executor.submit( new Worker(gmf, generator, workerWriteLimit, minExecutionTime, writeCounter)); futures.add(future); } } /** Wait for all writes to complete */ for (Future<Boolean> future : futures) { future.get(); } // now get all our shards final NodeShardCache cache = getInstance(injectors, NodeShardCache.class); final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode(sourceId, edgeType); // now submit the readers. final GraphManagerFactory gmf = getInstance(injectors, GraphManagerFactory.class); final long writeCount = writeCounter.get(); final Meter readMeter = registry.meter("readThroughput"); final List<Throwable> failures = new ArrayList<>(); for (int i = 0; i < 2; i++) { /** Start reading continuously while we migrate data to ensure our view is always correct */ final ListenableFuture<Long> future = executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter)); // add the future Futures.addCallback( future, new FutureCallback<Long>() { @Override public void onSuccess(@Nullable final Long result) { logger.info("Successfully ran the read, re-running"); executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter)); } @Override public void onFailure(final Throwable t) { failures.add(t); logger.error("Failed test!", t); } }); } int compactedCount; // now start our readers while (true) { if (!failures.isEmpty()) { StringBuilder builder = new StringBuilder(); builder.append("Read runner failed!\n"); for (Throwable t : failures) { builder.append("Exception is: "); ByteArrayOutputStream output = new ByteArrayOutputStream(); t.printStackTrace(new PrintWriter(output)); builder.append(output.toString("UTF-8")); builder.append("\n\n"); } fail(builder.toString()); } // reset our count. Ultimately we'll have 4 groups once our compaction completes compactedCount = 0; // we have to get it from the cache, because this will trigger the compaction process final Iterator<ShardEntryGroup> groups = cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta); final Set<ShardEntryGroup> shardEntryGroups = new HashSet<>(); while (groups.hasNext()) { final ShardEntryGroup group = groups.next(); shardEntryGroups.add(group); logger.info( "Compaction pending status for group {} is {}", group, group.isCompactionPending()); if (!group.isCompactionPending()) { compactedCount++; } } // we're done if (compactedCount >= expectedShardCount) { logger.info("All compactions complete, sleeping"); // final Object mutex = new Object(); // // synchronized ( mutex ){ // // mutex.wait(); // } break; } Thread.sleep(2000); } // now continue reading everything for 30 seconds Thread.sleep(30000); executor.shutdownNow(); }
/** @author Kasper Nielsen */ public abstract class CassandraBatchedStagedWriter<T> extends AbstractBatchedStage<T> { /** The logger. */ private static final Logger LOG = LoggerFactory.getLogger(CassandraBatchedStagedWriter.class); /** The connection to Cassandra. */ private final CassandraConnection connection; final MetricRegistry metrics = new MetricRegistry(); final Meter persistedCount = metrics.meter( MetricRegistry.name("aistore", "cassandra", "Number of persisted AIS messages")); /** greater than 0 if the last batch was slow. */ private int lastSlowBatch; /** * @param queueSize * @param maxBatchSize */ public CassandraBatchedStagedWriter(CassandraConnection connection, int batchSize) { super(Math.min(100000, batchSize * 100), batchSize); this.connection = requireNonNull(connection); final JmxReporter reporter = JmxReporter.forRegistry(metrics).inDomain("fooo.erer.er").build(); reporter.start(); } /** {@inheritDoc} */ @Override protected final void handleMessages(List<T> messages) { long start = System.nanoTime(); // Create a batch of message that we want to write. List<RegularStatement> statements = new ArrayList<>(); for (T t : messages) { try { handleMessage(statements, t); } catch (RuntimeException e) { LOG.warn("Failed to write message: " + t, e); // Just in case we cannot process a message } } // Try writing the batch try { Batch batch = QueryBuilder.batch(statements.toArray(new RegularStatement[statements.size()])); long beforeSend = System.nanoTime(); ResultSetFuture f = connection.getSession().executeAsync(batch); f.getUninterruptibly(); // throws QueryValidationExecption etc long total = System.nanoTime(); // Is this an abnormal slow batch? boolean isSlow = TimeUnit.MILLISECONDS.convert(total - start, TimeUnit.NANOSECONDS) > 200 || messages.size() >= getBatchSize(); if (isSlow || lastSlowBatch > 0) { LOG.info( "Total time: " + DurationFormatter.DEFAULT.formatNanos(total - start) + ", prepping=" + DurationFormatter.DEFAULT.formatNanos(beforeSend - start) + ", sending=" + DurationFormatter.DEFAULT.formatNanos(total - beforeSend) + ", size=" + messages.size()); // makes sure we write 10 info statements after the last slow batch we insert lastSlowBatch = isSlow ? 10 : lastSlowBatch - 1; } persistedCount.mark(messages.size()); // sink.onSucces(messages); } catch (QueryValidationException e) { LOG.error("Could not execute query, this is an internal error", e); } catch (Exception e) { onFailure(messages, e); try { sleepUntilShutdown(2, TimeUnit.SECONDS); } catch (InterruptedException ignore) { Thread.interrupted(); } } } protected abstract void handleMessage(List<RegularStatement> statements, T message); public abstract void onFailure(List<T> messages, Throwable cause); }
/** * Constructor passing in metric registry. * * @param registry the metric registry */ public Metrics(final MetricRegistry registry) { this.registry = registry; this.httpConnectionAttempt = registry.meter("http-connection-attempt"); this.httpConnectionSucceeded = registry.meter("http-connection-succeeded"); this.httpConnectionFailed = registry.meter("http-connection-failed"); this.httpConnectionClosed = registry.meter("http-connection-closed"); this.sendAttempt = registry.meter("sent-attempt"); this.sendSuccess = registry.meter("send-success"); this.sendError = registry.meter("send-error"); this.sendRateLimit = registry.meter("send-rate-limit"); this.sendException = registry.meter("send-exception"); this.sendLinearBackoff = registry.meter("send-backoff-linear"); this.sendExponentialBackoff = registry.meter("send-backoff-exponential"); this.readKafkaItem = registry.meter("read-kafka-item"); this.passedOnKafkaItem = registry.meter("passed-on-kafka-item"); this.readKafkaItemFromConsumer = registry.meter("read-kafka-item-from-consumer"); this.readKafkaItemsFromConsumer = registry.meter("read-kafka-items-from-consumer"); this.sendHeartbeatAttempt = registry.meter("send-http-heartbeat"); this.sendHeartbeatSuccess = registry.meter("send-http-heartbeat-success"); this.sendHeartbeatFailure = registry.meter("send-http-heartbeat-failure"); this.interruptedService = registry.meter("interrupt-service"); this.interruptedHTTPSending = registry.meter("interrupt-http-send"); this.interruptedShutdown = registry.meter("interrupt-shutdown"); this.shutdown = registry.meter("shutdown"); this.bulkPostTime = registry.timer("bulk-post-time"); }
public class EventHandler extends AbstractHandler { private static final Logger log = LoggerFactory.getLogger(EventHandler.class); private AtomicLong numOps = new AtomicLong(0); private AtomicLong numTxs = new AtomicLong(0); private ConfigurationFactory configurationFactory = new TypesafeConfigFactory(); private Conf configuration; private MessageEncoder messageEncoder; private MessageProducer messageProducer; private MetricRegistry metrics = new MetricRegistry(); private Timer operationProcessingTimer = metrics.timer("operationProcessingTime"); private Timer messageEncodingTimer = metrics.timer("encodingTime"); private Timer messageSendingTimer = metrics.timer("sendingTime"); private Meter operationProcessingErrorMeter = metrics.meter("processingErrors"); private TxFactory txFactory; private ScheduledReporter metricsReporter; private String configurationPath; public EventHandler() { super(TxOpMode.op); log.info("created handler - default mode: " + getMode()); } @VisibleForTesting public EventHandler( Conf configuration, MessageEncoder messageEncoder, MessageProducer messageProducer, TxFactory txFactory) { this.configuration = configuration; this.messageEncoder = messageEncoder; this.messageProducer = messageProducer; this.txFactory = txFactory; } @Override public void init(DsConfiguration conf, DsMetaData metaData) { super.init(conf, metaData); log.info("Initializing handler: Mode =" + getMode()); configuration = configurationFactory.load(configurationPath); messageProducer = KafkaProducerFactory.create(configuration.kafka()); messageEncoder = MessageEncoderFactory.create(configuration); txFactory = new TxFactory(); if (configuration.metrics().isEnabled()) { metricsReporter = MetricsReporterFactory.createReporter(configuration.metrics(), metrics); } } @Override public Status transactionBegin(DsEvent e, DsTransaction transaction) { super.transactionBegin(e, transaction); if (log.isDebugEnabled()) { log.debug( "Received begin tx event, numTx=" + numTxs.get() + " : position=" + transaction.getTranID() + ", totalOps=" + transaction.getTotalOps()); } return Status.OK; } @Override public Status operationAdded(DsEvent e, DsTransaction transaction, DsOperation operation) { Status overallStatus = Status.OK; super.operationAdded(e, transaction, operation); numOps.incrementAndGet(); final Tx tx = new Tx(transaction, getMetaData(), getConfig()); final TableMetaData tMeta = getMetaData().getTableMetaData(operation.getTableName()); final Op op = new Op(operation, tMeta, getConfig()); operation.getTokens(); if (isOperationMode()) { if (log.isDebugEnabled()) { log.debug( " Received operation: table='" + op.getTableName() + "'" + ", pos=" + op.getPosition() + " (total_ops= " + tx.getTotalOps() + ", buffered=" + tx.getSize() + ")" + ", ts=" + op.getTimestamp()); } Status operationStatus = processOperation(tx, op); if (Status.ABEND.equals(operationStatus)) { overallStatus = Status.ABEND; } } return overallStatus; } @Override public Status transactionCommit(DsEvent e, DsTransaction transaction) { Status overallStatus = Status.OK; super.transactionCommit(e, transaction); Tx tx = txFactory.createAdapterTx(transaction, getMetaData(), getConfig()); numTxs.incrementAndGet(); if (log.isDebugEnabled()) { log.debug( "transactionCommit event, tx #" + numTxs.get() + ":" + ", pos=" + tx.getTranID() + " (total_ops= " + tx.getTotalOps() + ", buffered=" + tx.getSize() + ")" + ", ts=" + tx.getTimestamp() + ")"); } if (!isOperationMode()) { for (Op op : tx) { Status operationStatus = processOperation(tx, op); if (Status.ABEND.equals(operationStatus)) { overallStatus = Status.ABEND; } } } return overallStatus; } private Status processOperation(Tx tx, Op op) { Timer.Context timer = operationProcessingTimer.time(); Status status = Status.OK; try { encodeAndSend(tx, op); } catch (RuntimeException re) { operationProcessingErrorMeter.mark(); log.error("Error processing operation: " + op.toString(), re); status = Status.ABEND; } timer.stop(); return status; } @Override public Status metaDataChanged(DsEvent e, DsMetaData meta) { log.debug("Received metadata event: " + e + "; current tables: " + meta.getTableNames().size()); return super.metaDataChanged(e, meta); } @Override public String reportStatus() { String s = "Status report: " + ", mode=" + getMode() + ", transactions=" + numTxs.get() + ", operations=" + numOps.get(); return s; } @Override public void destroy() { log.debug("destroy()... " + reportStatus()); if (configuration.metricsEnabled()) { metricsReporter.stop(); } messageProducer.terminate(); super.destroy(); } private void encodeAndSend(Tx tx, Op op) { if (log.isDebugEnabled()) { log.debug("Processing of transaction " + tx + " and operation " + op); } byte[] encodedMessage = encodeMessage(tx, op); sendMessage(encodedMessage); if (log.isDebugEnabled()) { log.debug("Completed processing of transaction " + tx + " and operation " + op); } } private void sendMessage(byte[] encodedMessage) { Timer.Context sendMessageTimer = messageSendingTimer.time(); messageProducer.produce(encodedMessage); sendMessageTimer.stop(); if (log.isDebugEnabled()) { log.debug("Completed send of message: " + new String(encodedMessage)); } } private byte[] encodeMessage(Tx tx, Op op) { Timer.Context encodingTimer = messageEncodingTimer.time(); byte[] encodedMessage = messageEncoder.encode(tx, op); encodingTimer.stop(); if (log.isTraceEnabled()) { log.trace("Result of message encoding is = " + new String(encodedMessage)); } return encodedMessage; } public void setConfigurationPath(String configurationPath) { this.configurationPath = configurationPath; } }
@SuppressWarnings("OverlyLongMethod") private void run(Config config) { System.err.println( " _\n" + " _ __| |___ __ _\n" + "| '_ \\ / _ \\/ _` |\n" + "| .__/_\\___/\\__, |\n" + "|_| |___/ stress"); final Config stressConfig = config.getConfig("plog.stress"); final int threadCount = stressConfig.getInt("threads"); log.info("Using {} threads", threadCount); final int rate = stressConfig.getInt("rate"); final RateLimiter rateLimiter = RateLimiter.create(rate); final int socketRenewRate = stressConfig.getInt("renew_rate"); final int minSize = stressConfig.getInt("min_size"); final int maxSize = stressConfig.getInt("max_size"); final int sizeIncrements = stressConfig.getInt("size_increments"); final double sizeExponent = stressConfig.getDouble("size_exponent"); final int sizeDelta = maxSize - minSize; final int differentSizes = sizeDelta / sizeIncrements; if (differentSizes == 0) { throw new RuntimeException("No sizes! Decrease plog.stress.size_increments"); } final int stopAfter = stressConfig.getInt("stop_after"); final int packetSize = stressConfig.getInt("udp.size"); final int bufferSize = stressConfig.getInt("udp.SO_SNDBUF"); final Fragmenter fragmenter = new Fragmenter(packetSize); final Random random = new Random(stressConfig.getLong("seed")); final byte[] randomBytes = new byte[maxSize]; random.nextBytes(randomBytes); final ByteBuf randomMessage = Unpooled.wrappedBuffer(randomBytes); log.info("Generating {} different hashes", differentSizes); final int[] precomputedHashes = new int[differentSizes]; for (int i = 0; i < differentSizes; i++) { precomputedHashes[i] = Murmur3.hash32(randomMessage, 0, minSize + sizeIncrements * i, 0); } final ByteBufAllocator allocator = new PooledByteBufAllocator(); final double packetLoss = stressConfig.getDouble("udp.loss"); final Meter socketMeter = registry.meter("Sockets used"); final Meter messageMeter = registry.meter("Messages sent"); final Meter packetMeter = registry.meter("Packets sent"); final Meter sendFailureMeter = registry.meter("Send failures"); final Meter lossMeter = registry.meter("Packets dropped"); final Histogram messageSizeHistogram = registry.histogram("Message size"); final Histogram packetSizeHistogram = registry.histogram("Packet size"); final InetSocketAddress target = new InetSocketAddress(stressConfig.getString("host"), stressConfig.getInt("port")); log.info("Starting with config {}", config); final long consoleRate = stressConfig.getDuration("console.interval", TimeUnit.MILLISECONDS); ConsoleReporter.forRegistry(registry).build().start(consoleRate, TimeUnit.MILLISECONDS); for (int i = 0; i < threadCount; i++) { new Thread("stress_" + i) { private DatagramChannel channel = null; @Override public void run() { try { for (int sent = 0; sent < stopAfter; sent++, messageMeter.mark()) { if (sent % socketRenewRate == 0) { if (channel != null) { channel.close(); } channel = DatagramChannel.open(); channel.socket().setSendBufferSize(bufferSize); socketMeter.mark(); } // global rate limiting rateLimiter.acquire(); final int sizeIndex = (int) (Math.pow(random.nextDouble(), sizeExponent) * differentSizes); final int messageSize = minSize + sizeIncrements * sizeIndex; final int hash = precomputedHashes[sizeIndex]; messageSizeHistogram.update(messageSize); final ByteBuf[] fragments = fragmenter.fragment(allocator, randomMessage, null, sent, messageSize, hash); for (ByteBuf fragment : fragments) { if (random.nextDouble() < packetLoss) { lossMeter.mark(); } else { final int packetSize = fragment.readableBytes(); final ByteBuffer buffer = fragment.nioBuffer(); try { channel.send(buffer, target); packetSizeHistogram.update(packetSize); packetMeter.mark(); } catch (SocketException e) { sendFailureMeter.mark(); } } fragment.release(); } } } catch (Throwable t) { t.printStackTrace(); System.exit(1); } } }.start(); } }
@Test(timeout = 120000) @Category(StressTest.class) public void writeThousandsDelete() throws InterruptedException, ExecutionException, MigrationException, UnsupportedEncodingException { final Id sourceId = IdGenerator.createId("source"); final String edgeType = "test"; final EdgeGenerator generator = new EdgeGenerator() { @Override public Edge newEdge() { Edge edge = createEdge(sourceId, edgeType, IdGenerator.createId("target")); return edge; } @Override public Observable<MarkedEdge> doSearch(final GraphManager manager) { return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent(), false)); } }; // final int numInjectors = 2; final int numInjectors = 1; /** * create 3 injectors. This way all the caches are independent of one another. This is the same * as multiple nodes */ final List<Injector> injectors = createInjectors(numInjectors); final GraphFig graphFig = getInstance(injectors, GraphFig.class); final long shardSize = graphFig.getShardSize(); // we don't want to starve the cass runtime since it will be on the same box. Only take 50% of // processing // power for writes final int numProcessors = Runtime.getRuntime().availableProcessors() / 2; final int numWorkersPerInjector = numProcessors / numInjectors; /** Do 4x shard size so we should have approximately 4 shards */ final long numberOfEdges = shardSize * 4; final long workerWriteLimit = numberOfEdges / numWorkersPerInjector / numInjectors; createExecutor(numWorkersPerInjector); final AtomicLong writeCounter = new AtomicLong(); // min stop time the min delta + 1 cache cycle timeout final long minExecutionTime = graphFig.getShardMinDelta() + graphFig.getShardCacheTimeout(); logger.info( "Writing {} edges per worker on {} workers in {} injectors", workerWriteLimit, numWorkersPerInjector, numInjectors); final List<Future<Boolean>> futures = new ArrayList<>(); for (Injector injector : injectors) { final GraphManagerFactory gmf = injector.getInstance(GraphManagerFactory.class); for (int i = 0; i < numWorkersPerInjector; i++) { Future<Boolean> future = executor.submit( new Worker(gmf, generator, workerWriteLimit, minExecutionTime, writeCounter)); futures.add(future); } } /** Wait for all writes to complete */ for (Future<Boolean> future : futures) { future.get(); } // now get all our shards final NodeShardCache cache = getInstance(injectors, NodeShardCache.class); final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode(sourceId, edgeType); // now submit the readers. final GraphManagerFactory gmf = getInstance(injectors, GraphManagerFactory.class); final long writeCount = writeCounter.get(); final Meter readMeter = registry.meter("readThroughput"); // check our shard state final Iterator<ShardEntryGroup> existingShardGroups = cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta); int shardCount = 0; while (existingShardGroups.hasNext()) { final ShardEntryGroup group = existingShardGroups.next(); shardCount++; logger.info( "Compaction pending status for group {} is {}", group, group.isCompactionPending()); } logger.info("found {} shard groups", shardCount); // now mark and delete all the edges final GraphManager manager = gmf.createEdgeManager(scope); // sleep occasionally to stop pushing cassandra over long count = Long.MAX_VALUE; while (count != 0) { // take 10000 then sleep count = generator .doSearch(manager) .onBackpressureBlock() .take(1000) .flatMap(edge -> manager.markEdge(edge)) .flatMap(edge -> manager.deleteEdge(edge)) .countLong() .toBlocking() .last(); Thread.sleep(500); } // now loop until with a reader until our shards are gone /** Start reading continuously while we migrate data to ensure our view is always correct */ final ListenableFuture<Long> future = executor.submit(new ReadWorker(gmf, generator, 0, readMeter)); final List<Throwable> failures = new ArrayList<>(); // add the future Futures.addCallback( future, new FutureCallback<Long>() { @Override public void onSuccess(@Nullable final Long result) { logger.info("Successfully ran the read, re-running"); executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter)); } @Override public void onFailure(final Throwable t) { failures.add(t); logger.error("Failed test!", t); } }); // now start our readers while (true) { if (!failures.isEmpty()) { StringBuilder builder = new StringBuilder(); builder.append("Read runner failed!\n"); for (Throwable t : failures) { builder.append("Exception is: "); ByteArrayOutputStream output = new ByteArrayOutputStream(); t.printStackTrace(new PrintWriter(output)); builder.append(output.toString("UTF-8")); builder.append("\n\n"); } fail(builder.toString()); } // reset our count. Ultimately we'll have 4 groups once our compaction completes shardCount = 0; // we have to get it from the cache, because this will trigger the compaction process final Iterator<ShardEntryGroup> groups = cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta); ShardEntryGroup group = null; while (groups.hasNext()) { group = groups.next(); logger.info("Shard size for group is {}", group.getReadShards()); shardCount += group.getReadShards().size(); } // we're done, 1 shard remains, we have a group, and it's our default shard if (shardCount == 1 && group != null && group.getMinShard().getShardIndex() == Shard.MIN_SHARD.getShardIndex()) { logger.info("All compactions complete,"); break; } Thread.sleep(2000); } // now that we have finished expanding s executor.shutdownNow(); }
@Override protected void doStart() throws Exception { super.doStart(); final String prefix = name(getHandler().getClass(), name); this.requests = timer(name(prefix, "requests")); this.dispatches = timer(name(prefix, "dispatches")); this.activeRequests = metricRegistry.counter(name(prefix, "active-requests")); this.activeDispatches = metricRegistry.counter(name(prefix, "active-dispatches")); this.activeSuspended = metricRegistry.counter(name(prefix, "active-suspended")); this.asyncDispatches = metricRegistry.meter(name(prefix, "async-dispatches")); this.asyncTimeouts = metricRegistry.meter(name(prefix, "async-timeouts")); this.responses = new Meter[] { metricRegistry.meter(name(prefix, "1xx-responses")), // 1xx metricRegistry.meter(name(prefix, "2xx-responses")), // 2xx metricRegistry.meter(name(prefix, "3xx-responses")), // 3xx metricRegistry.meter(name(prefix, "4xx-responses")), // 4xx metricRegistry.meter(name(prefix, "5xx-responses")) // 5xx }; this.getRequests = timer(name(prefix, "get-requests")); this.postRequests = timer(name(prefix, "post-requests")); this.headRequests = timer(name(prefix, "head-requests")); this.putRequests = timer(name(prefix, "put-requests")); this.deleteRequests = timer(name(prefix, "delete-requests")); this.optionsRequests = timer(name(prefix, "options-requests")); this.traceRequests = timer(name(prefix, "trace-requests")); this.connectRequests = timer(name(prefix, "connect-requests")); this.moveRequests = timer(name(prefix, "move-requests")); this.otherRequests = timer(name(prefix, "other-requests")); this.listener = new AsyncListener() { @Override public void onTimeout(AsyncEvent event) throws IOException { asyncTimeouts.mark(); } @Override public void onStartAsync(AsyncEvent event) throws IOException { event.getAsyncContext().addListener(this); } @Override public void onError(AsyncEvent event) throws IOException {} @Override public void onComplete(AsyncEvent event) throws IOException { final AsyncContextState state = (AsyncContextState) event.getAsyncContext(); final Request request = (Request) state.getRequest(); updateResponses(request); if (!(state.getHttpChannelState().getState() == State.DISPATCHED)) { activeSuspended.dec(); } } }; }
public class GraphManagerShardConsistencyIT { private static final Logger logger = LoggerFactory.getLogger(GraphManagerShardConsistencyIT.class); private static final MetricRegistry registry = new MetricRegistry(); private static final Meter writeMeter = registry.meter("writeThroughput"); private Slf4jReporter reporter; protected ApplicationScope scope; protected Object originalShardSize; protected Object originalShardTimeout; protected Object originalShardDelta; protected ListeningExecutorService executor; @Before public void setupOrg() { originalShardSize = ConfigurationManager.getConfigInstance().getProperty(GraphFig.SHARD_SIZE); originalShardTimeout = ConfigurationManager.getConfigInstance().getProperty(GraphFig.SHARD_CACHE_TIMEOUT); originalShardDelta = ConfigurationManager.getConfigInstance().getProperty(GraphFig.SHARD_MIN_DELTA); ConfigurationManager.getConfigInstance().setProperty(GraphFig.SHARD_SIZE, 500); final long cacheTimeout = 2000; // set our cache timeout to the above value ConfigurationManager.getConfigInstance() .setProperty(GraphFig.SHARD_CACHE_TIMEOUT, cacheTimeout); final long minDelta = (long) (cacheTimeout * 2.5); ConfigurationManager.getConfigInstance().setProperty(GraphFig.SHARD_MIN_DELTA, minDelta); // get the system property of the UUID to use. If one is not set, use the defualt String uuidString = System.getProperty("org.id", "80a42760-b699-11e3-a5e2-0800200c9a66"); scope = new ApplicationScopeImpl(IdGenerator.createId(UUID.fromString(uuidString), "test")); reporter = Slf4jReporter.forRegistry(registry) .outputTo(logger) .convertRatesTo(TimeUnit.SECONDS) .convertDurationsTo(TimeUnit.MILLISECONDS) .build(); reporter.start(10, TimeUnit.SECONDS); } @After public void tearDown() { reporter.stop(); reporter.report(); executor.shutdownNow(); } private void createExecutor(final int size) { executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(size)); } @Test public void writeThousandsSingleSource() throws InterruptedException, ExecutionException, MigrationException, UnsupportedEncodingException { final Id sourceId = IdGenerator.createId("source"); final String edgeType = "test"; final EdgeGenerator generator = new EdgeGenerator() { @Override public Edge newEdge() { Edge edge = createEdge(sourceId, edgeType, IdGenerator.createId("target")); return edge; } @Override public Observable<MarkedEdge> doSearch(final GraphManager manager) { return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent())); } }; // final int numInjectors = 2; final int numInjectors = 1; /** * create 3 injectors. This way all the caches are independent of one another. This is the same * as multiple nodes */ final List<Injector> injectors = createInjectors(numInjectors); final GraphFig graphFig = getInstance(injectors, GraphFig.class); final long shardSize = graphFig.getShardSize(); // we don't want to starve the cass runtime since it will be on the same box. Only take 50% of // processing // power for writes final int numProcessors = Runtime.getRuntime().availableProcessors() / 2; final int numWorkersPerInjector = numProcessors / numInjectors; /** Do 4x shard size so we should have approximately 4 shards */ final long numberOfEdges = shardSize * 4; final long workerWriteLimit = numberOfEdges / numWorkersPerInjector / numInjectors; final long expectedShardCount = numberOfEdges / shardSize; createExecutor(numWorkersPerInjector); final AtomicLong writeCounter = new AtomicLong(); // min stop time the min delta + 1 cache cycle timeout final long minExecutionTime = graphFig.getShardMinDelta() + graphFig.getShardCacheTimeout(); logger.info( "Writing {} edges per worker on {} workers in {} injectors", workerWriteLimit, numWorkersPerInjector, numInjectors); final List<Future<Boolean>> futures = new ArrayList<>(); for (Injector injector : injectors) { final GraphManagerFactory gmf = injector.getInstance(GraphManagerFactory.class); for (int i = 0; i < numWorkersPerInjector; i++) { Future<Boolean> future = executor.submit( new Worker(gmf, generator, workerWriteLimit, minExecutionTime, writeCounter)); futures.add(future); } } /** Wait for all writes to complete */ for (Future<Boolean> future : futures) { future.get(); } // now get all our shards final NodeShardCache cache = getInstance(injectors, NodeShardCache.class); final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode(sourceId, edgeType); // now submit the readers. final GraphManagerFactory gmf = getInstance(injectors, GraphManagerFactory.class); final long writeCount = writeCounter.get(); final Meter readMeter = registry.meter("readThroughput"); final List<Throwable> failures = new ArrayList<>(); for (int i = 0; i < 2; i++) { /** Start reading continuously while we migrate data to ensure our view is always correct */ final ListenableFuture<Long> future = executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter)); // add the future Futures.addCallback( future, new FutureCallback<Long>() { @Override public void onSuccess(@Nullable final Long result) { logger.info("Successfully ran the read, re-running"); executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter)); } @Override public void onFailure(final Throwable t) { failures.add(t); logger.error("Failed test!", t); } }); } int compactedCount; // now start our readers while (true) { if (!failures.isEmpty()) { StringBuilder builder = new StringBuilder(); builder.append("Read runner failed!\n"); for (Throwable t : failures) { builder.append("Exception is: "); ByteArrayOutputStream output = new ByteArrayOutputStream(); t.printStackTrace(new PrintWriter(output)); builder.append(output.toString("UTF-8")); builder.append("\n\n"); } fail(builder.toString()); } // reset our count. Ultimately we'll have 4 groups once our compaction completes compactedCount = 0; // we have to get it from the cache, because this will trigger the compaction process final Iterator<ShardEntryGroup> groups = cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta); final Set<ShardEntryGroup> shardEntryGroups = new HashSet<>(); while (groups.hasNext()) { final ShardEntryGroup group = groups.next(); shardEntryGroups.add(group); logger.info( "Compaction pending status for group {} is {}", group, group.isCompactionPending()); if (!group.isCompactionPending()) { compactedCount++; } } // we're done if (compactedCount >= expectedShardCount) { logger.info("All compactions complete, sleeping"); // final Object mutex = new Object(); // // synchronized ( mutex ){ // // mutex.wait(); // } break; } Thread.sleep(2000); } // now continue reading everything for 30 seconds Thread.sleep(30000); executor.shutdownNow(); } private <T> T getInstance(final List<Injector> injectors, Class<T> clazz) { return injectors.get(0).getInstance(clazz); } /** Create new Guice injector environments and return them */ private List<Injector> createInjectors(int count) throws MigrationException { final List<Injector> injectors = new ArrayList<>(count); for (int i = 0; i < count; i++) { final Injector injector = Guice.createInjector(new TestGraphModule()); injectors.add(injector); } final MigrationManager migrationManager = getInstance(injectors, MigrationManager.class); migrationManager.migrate(); return injectors; } @Test(timeout = 120000) @Category(StressTest.class) public void writeThousandsDelete() throws InterruptedException, ExecutionException, MigrationException, UnsupportedEncodingException { final Id sourceId = IdGenerator.createId("source"); final String edgeType = "test"; final EdgeGenerator generator = new EdgeGenerator() { @Override public Edge newEdge() { Edge edge = createEdge(sourceId, edgeType, IdGenerator.createId("target")); return edge; } @Override public Observable<MarkedEdge> doSearch(final GraphManager manager) { return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent(), false)); } }; // final int numInjectors = 2; final int numInjectors = 1; /** * create 3 injectors. This way all the caches are independent of one another. This is the same * as multiple nodes */ final List<Injector> injectors = createInjectors(numInjectors); final GraphFig graphFig = getInstance(injectors, GraphFig.class); final long shardSize = graphFig.getShardSize(); // we don't want to starve the cass runtime since it will be on the same box. Only take 50% of // processing // power for writes final int numProcessors = Runtime.getRuntime().availableProcessors() / 2; final int numWorkersPerInjector = numProcessors / numInjectors; /** Do 4x shard size so we should have approximately 4 shards */ final long numberOfEdges = shardSize * 4; final long workerWriteLimit = numberOfEdges / numWorkersPerInjector / numInjectors; createExecutor(numWorkersPerInjector); final AtomicLong writeCounter = new AtomicLong(); // min stop time the min delta + 1 cache cycle timeout final long minExecutionTime = graphFig.getShardMinDelta() + graphFig.getShardCacheTimeout(); logger.info( "Writing {} edges per worker on {} workers in {} injectors", workerWriteLimit, numWorkersPerInjector, numInjectors); final List<Future<Boolean>> futures = new ArrayList<>(); for (Injector injector : injectors) { final GraphManagerFactory gmf = injector.getInstance(GraphManagerFactory.class); for (int i = 0; i < numWorkersPerInjector; i++) { Future<Boolean> future = executor.submit( new Worker(gmf, generator, workerWriteLimit, minExecutionTime, writeCounter)); futures.add(future); } } /** Wait for all writes to complete */ for (Future<Boolean> future : futures) { future.get(); } // now get all our shards final NodeShardCache cache = getInstance(injectors, NodeShardCache.class); final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode(sourceId, edgeType); // now submit the readers. final GraphManagerFactory gmf = getInstance(injectors, GraphManagerFactory.class); final long writeCount = writeCounter.get(); final Meter readMeter = registry.meter("readThroughput"); // check our shard state final Iterator<ShardEntryGroup> existingShardGroups = cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta); int shardCount = 0; while (existingShardGroups.hasNext()) { final ShardEntryGroup group = existingShardGroups.next(); shardCount++; logger.info( "Compaction pending status for group {} is {}", group, group.isCompactionPending()); } logger.info("found {} shard groups", shardCount); // now mark and delete all the edges final GraphManager manager = gmf.createEdgeManager(scope); // sleep occasionally to stop pushing cassandra over long count = Long.MAX_VALUE; while (count != 0) { // take 10000 then sleep count = generator .doSearch(manager) .onBackpressureBlock() .take(1000) .flatMap(edge -> manager.markEdge(edge)) .flatMap(edge -> manager.deleteEdge(edge)) .countLong() .toBlocking() .last(); Thread.sleep(500); } // now loop until with a reader until our shards are gone /** Start reading continuously while we migrate data to ensure our view is always correct */ final ListenableFuture<Long> future = executor.submit(new ReadWorker(gmf, generator, 0, readMeter)); final List<Throwable> failures = new ArrayList<>(); // add the future Futures.addCallback( future, new FutureCallback<Long>() { @Override public void onSuccess(@Nullable final Long result) { logger.info("Successfully ran the read, re-running"); executor.submit(new ReadWorker(gmf, generator, writeCount, readMeter)); } @Override public void onFailure(final Throwable t) { failures.add(t); logger.error("Failed test!", t); } }); // now start our readers while (true) { if (!failures.isEmpty()) { StringBuilder builder = new StringBuilder(); builder.append("Read runner failed!\n"); for (Throwable t : failures) { builder.append("Exception is: "); ByteArrayOutputStream output = new ByteArrayOutputStream(); t.printStackTrace(new PrintWriter(output)); builder.append(output.toString("UTF-8")); builder.append("\n\n"); } fail(builder.toString()); } // reset our count. Ultimately we'll have 4 groups once our compaction completes shardCount = 0; // we have to get it from the cache, because this will trigger the compaction process final Iterator<ShardEntryGroup> groups = cache.getReadShardGroup(scope, Long.MAX_VALUE, directedEdgeMeta); ShardEntryGroup group = null; while (groups.hasNext()) { group = groups.next(); logger.info("Shard size for group is {}", group.getReadShards()); shardCount += group.getReadShards().size(); } // we're done, 1 shard remains, we have a group, and it's our default shard if (shardCount == 1 && group != null && group.getMinShard().getShardIndex() == Shard.MIN_SHARD.getShardIndex()) { logger.info("All compactions complete,"); break; } Thread.sleep(2000); } // now that we have finished expanding s executor.shutdownNow(); } private class Worker implements Callable<Boolean> { private final GraphManagerFactory factory; private final EdgeGenerator generator; private final long writeLimit; private final long minExecutionTime; private final AtomicLong writeCounter; private Worker( final GraphManagerFactory factory, final EdgeGenerator generator, final long writeLimit, final long minExecutionTime, final AtomicLong writeCounter) { this.factory = factory; this.generator = generator; this.writeLimit = writeLimit; this.minExecutionTime = minExecutionTime; this.writeCounter = writeCounter; } @Override public Boolean call() throws Exception { GraphManager manager = factory.createEdgeManager(scope); final long startTime = System.currentTimeMillis(); for (long i = 0; i < writeLimit || System.currentTimeMillis() - startTime < minExecutionTime; i++) { Edge edge = generator.newEdge(); Edge returned = manager.writeEdge(edge).toBlocking().last(); assertNotNull("Returned has a version", returned.getTimestamp()); writeMeter.mark(); writeCounter.incrementAndGet(); if (i % 1000 == 0) { logger.info(" Wrote: " + i); } } return true; } } private class ReadWorker implements Callable<Long> { private final GraphManagerFactory factory; private final EdgeGenerator generator; private final long writeCount; private final Meter readMeter; private ReadWorker( final GraphManagerFactory factory, final EdgeGenerator generator, final long writeCount, final Meter readMeter) { this.factory = factory; this.generator = generator; this.writeCount = writeCount; this.readMeter = readMeter; } @Override public Long call() throws Exception { GraphManager gm = factory.createEdgeManager(scope); while (true) { // do a read to eventually trigger our group compaction. Take 2 pages of columns final long returnedEdgeCount = generator .doSearch(gm) .doOnNext(edge -> readMeter.mark()) .countLong() .toBlocking() .last(); logger.info("Completed reading {} edges", returnedEdgeCount); if (writeCount != returnedEdgeCount) { logger.warn( "Unexpected edge count returned!!! Expected {} but was {}", writeCount, returnedEdgeCount); } assertEquals("Expected to read same edge count", writeCount, returnedEdgeCount); } } } private interface EdgeGenerator { /** Create a new edge to persiste */ public Edge newEdge(); /** Perform the search returning an observable edge */ public Observable<MarkedEdge> doSearch(final GraphManager manager); } }