@Override
  public Connector build(
      Server server, MetricRegistry metrics, String name, ThreadPool threadPool) {
    logSupportedParameters();

    final HttpConfiguration httpConfig = buildHttpConfiguration();

    final HttpConnectionFactory httpConnectionFactory = buildHttpConnectionFactory(httpConfig);

    final SslContextFactory sslContextFactory = buildSslContextFactory();
    server.addBean(sslContextFactory);

    final SslConnectionFactory sslConnectionFactory =
        new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.toString());

    final Scheduler scheduler = new ScheduledExecutorScheduler();

    final ByteBufferPool bufferPool = buildBufferPool();

    final String timerName =
        name(
            HttpConnectionFactory.class, getBindHost(), Integer.toString(getPort()), "connections");

    return buildConnector(
        server,
        scheduler,
        bufferPool,
        name,
        threadPool,
        new InstrumentedConnectionFactory(sslConnectionFactory, metrics.timer(timerName)),
        httpConnectionFactory);
  }
 /**
  * Constructor passing in metric registry.
  *
  * @param registry the metric registry
  */
 public Metrics(final MetricRegistry registry) {
   this.registry = registry;
   this.httpConnectionAttempt = registry.meter("http-connection-attempt");
   this.httpConnectionSucceeded = registry.meter("http-connection-succeeded");
   this.httpConnectionFailed = registry.meter("http-connection-failed");
   this.httpConnectionClosed = registry.meter("http-connection-closed");
   this.sendAttempt = registry.meter("sent-attempt");
   this.sendSuccess = registry.meter("send-success");
   this.sendError = registry.meter("send-error");
   this.sendRateLimit = registry.meter("send-rate-limit");
   this.sendException = registry.meter("send-exception");
   this.sendLinearBackoff = registry.meter("send-backoff-linear");
   this.sendExponentialBackoff = registry.meter("send-backoff-exponential");
   this.readKafkaItem = registry.meter("read-kafka-item");
   this.passedOnKafkaItem = registry.meter("passed-on-kafka-item");
   this.readKafkaItemFromConsumer = registry.meter("read-kafka-item-from-consumer");
   this.readKafkaItemsFromConsumer = registry.meter("read-kafka-items-from-consumer");
   this.sendHeartbeatAttempt = registry.meter("send-http-heartbeat");
   this.sendHeartbeatSuccess = registry.meter("send-http-heartbeat-success");
   this.sendHeartbeatFailure = registry.meter("send-http-heartbeat-failure");
   this.interruptedService = registry.meter("interrupt-service");
   this.interruptedHTTPSending = registry.meter("interrupt-http-send");
   this.interruptedShutdown = registry.meter("interrupt-shutdown");
   this.shutdown = registry.meter("shutdown");
   this.bulkPostTime = registry.timer("bulk-post-time");
 }
 @Before
 public void setUp() throws Exception {
   producer = new TimerProducer(endpoint);
   inOrder = Mockito.inOrder(endpoint, exchange, registry, timer, context, in);
   when(endpoint.getRegistry()).thenReturn(registry);
   when(registry.timer(METRICS_NAME)).thenReturn(timer);
   when(timer.time()).thenReturn(context);
   when(exchange.getIn()).thenReturn(in);
 }
  public static void main(String[] args) {

    // create some various metrics and update them

    MetricRegistry registry = new MetricRegistry();

    final AtomicInteger gaugeInteger = new AtomicInteger();

    registry.register(
        name("gauge"),
        new Gauge<Integer>() {

          @Override
          public Integer getValue() {
            return gaugeInteger.get();
          }
        });

    final Counter counter = registry.counter(name("counter"));

    final Histogram histogram = registry.histogram(name("histogram"));

    final Meter meter = registry.meter(name("meter"));

    final Timer timer = registry.timer(name("timer"));

    NewRelicReporter reporter =
        new NewRelicReporter(
            registry,
            "new relic reporter",
            MetricFilter.ALL,
            new AllEnabledMetricAttributeFilter(),
            TimeUnit.SECONDS,
            TimeUnit.MILLISECONDS,
            "foo/");
    reporter.start(60, TimeUnit.SECONDS);

    ScheduledExecutorService svc = Executors.newScheduledThreadPool(1);

    final Random random = new Random();

    svc.scheduleAtFixedRate(
        new Runnable() {
          @Override
          public void run() {
            System.out.println("Updating");
            gaugeInteger.incrementAndGet();
            counter.inc();
            histogram.update(random.nextInt(10));
            meter.mark();
            timer.update(random.nextInt(10), TimeUnit.MILLISECONDS);
          }
        },
        0,
        1,
        TimeUnit.SECONDS);
  }
/**
 * <code>ConnectionPoolCallTest</code> - Checks how connection pool decreases latency
 *
 * @author Vlad Mihalcea
 */
@Ignore
public class ConnectionPoolCallTest extends DataSourceProviderIntegrationTest {

  private final Logger LOGGER = LoggerFactory.getLogger(getClass());

  private MetricRegistry metricRegistry = new MetricRegistry();

  private Timer timer = metricRegistry.timer("callTimer");

  private Slf4jReporter logReporter =
      Slf4jReporter.forRegistry(metricRegistry).outputTo(LOGGER).build();

  private int callCount = 1000;

  public ConnectionPoolCallTest(DataSourceProvider dataSourceProvider) {
    super(dataSourceProvider);
  }

  @Override
  protected Class<?>[] entities() {
    return new Class[] {};
  }

  @Test
  public void testNoPooling() throws SQLException {
    LOGGER.info("Test without pooling for {}", dataSourceProvider().database());
    test(dataSourceProvider().dataSource());
  }

  @Test
  public void testPooling() throws SQLException {
    LOGGER.info("Test with pooling for {}", dataSourceProvider().database());
    test(poolingDataSource());
  }

  private void test(DataSource dataSource) throws SQLException {
    for (int i = 0; i < callCount; i++) {
      long startNanos = System.nanoTime();
      try (Connection connection = dataSource.getConnection()) {}
      timer.update(System.nanoTime() - startNanos, TimeUnit.NANOSECONDS);
    }
    logReporter.report();
  }

  protected HikariDataSource poolingDataSource() {
    Properties properties = new Properties();
    properties.setProperty(
        "dataSourceClassName", dataSourceProvider().dataSourceClassName().getName());
    properties.put("dataSourceProperties", dataSourceProvider().dataSourceProperties());
    properties.setProperty("minimumPoolSize", String.valueOf(1));
    properties.setProperty("maximumPoolSize", String.valueOf(3));
    properties.setProperty("connectionTimeout", String.valueOf(100));
    return new HikariDataSource(new HikariConfig(properties));
  }
}
  public void init(FilterConfig filterConfig) throws ServletException {
    final MetricRegistry metricsRegistry = Metrics.metricRegistry();

    this.metersByStatusCode = new ConcurrentHashMap<Integer, Meter>(meterNamesByStatusCode.size());
    for (Map.Entry<Integer, String> entry : meterNamesByStatusCode.entrySet()) {
      metersByStatusCode.put(entry.getKey(), metricsRegistry.meter(name("http", entry.getValue())));
    }
    this.otherMeter = metricsRegistry.meter(name("http", otherMetricName));
    this.activeRequests = metricsRegistry.counter(name("http", "activeRequests"));
    this.requestTimer = metricsRegistry.timer(name("http", "requests"));
  }
Beispiel #7
0
  private PerfContext start(MetricRegistry metrics, String name) {
    Timer timer = metrics.timer(name);
    final Timer.Context ctx = timer.time();

    return new PerfContext() {
      @Override
      public void end() {
        ctx.stop();
      }
    };
  }
  protected AbstractScheduledProcessor(
      ScheduledExecutorService executor,
      Duration frequency,
      Duration initialDelay,
      String name,
      MetricRegistry metricRegistry) {
    this.executor = executor;
    this.frequency = frequency;
    this.initialDelay = initialDelay;

    timer = metricRegistry.timer(String.format("scheduled-processor-%s", name));
  }
  @Override
  public void process(Request request) {
    String name = getRouteName(request);

    if (name == null || name.isEmpty()) return;

    if (!ROUTE_TIMERS.containsKey(name)) {
      ROUTE_TIMERS.putIfAbsent(name, metrics.timer(getTimerName(name)));
    }

    if (!EXCEPTION_COUNTERS_BY_ROUTE.containsKey(name)) {
      EXCEPTION_COUNTERS_BY_ROUTE.putIfAbsent(name, metrics.counter(getExceptionCounterName(name)));
    }
  }
Beispiel #10
0
  @Test
  public void testTimer() {
    System.out.println("******************************* TIMER *******************************");
    timer = registry.timer("timer");
    try {
      for (int i = 0; i < ITER_COUNT; i++) {
        final Timer.Context context = timer.time();
        Thread.sleep(SLEEP_MS);
        context.stop();
      }

    } catch (InterruptedException ex) {
      Thread.currentThread().interrupt();
    }
  }
 protected BaseAttributeResolver(AttributeResolverDescriptor descriptor) {
   checkNotNull(descriptor);
   this.descriptor =
       new AttributeResolverDescriptorDelegate(descriptor) {
         @Override
         public int getPreferredCacheTTL() {
           return (preferredCacheTTL == null)
               ? super.getPreferredCacheTTL()
               : preferredCacheTTL.get();
         }
       };
   final MetricRegistry registry = getOrCreate();
   this.timer = registry.timer(name("pip.AttributeResolver", descriptor.getId(), "timer"));
   this.histogram =
       registry.histogram(name("pip.AttributeResolver", descriptor.getId(), "histogram"));
 }
  public CodaHaleMetricsTracker(
      final String poolName, final PoolStats poolStats, final MetricRegistry registry) {
    this.poolName = poolName;
    this.registry = registry;
    this.connectionObtainTimer = registry.timer(MetricRegistry.name(poolName, "pool", "Wait"));
    this.connectionUsage = registry.histogram(MetricRegistry.name(poolName, "pool", "Usage"));
    this.connectionTimeoutMeter =
        registry.meter(MetricRegistry.name(poolName, "pool", "ConnectionTimeoutRate"));

    registry.register(
        MetricRegistry.name(poolName, "pool", "TotalConnections"),
        new Gauge<Integer>() {
          @Override
          public Integer getValue() {
            return poolStats.getTotalConnections();
          }
        });

    registry.register(
        MetricRegistry.name(poolName, "pool", "IdleConnections"),
        new Gauge<Integer>() {
          @Override
          public Integer getValue() {
            return poolStats.getIdleConnections();
          }
        });

    registry.register(
        MetricRegistry.name(poolName, "pool", "ActiveConnections"),
        new Gauge<Integer>() {
          @Override
          public Integer getValue() {
            return poolStats.getActiveConnections();
          }
        });

    registry.register(
        MetricRegistry.name(poolName, "pool", "PendingConnections"),
        new Gauge<Integer>() {
          @Override
          public Integer getValue() {
            return poolStats.getPendingThreads();
          }
        });
  }
  @Test
  @SuppressWarnings("squid:S2925")
  public void testTimer() throws Exception {
    final Timer timer = registry.timer(name("foo", "bar"));
    final Timer.Context timerContext = timer.time();
    Thread.sleep(200);
    timerContext.stop();
    reportAndRefresh();

    SearchResponse searchResponse =
        client().prepareSearch(indexWithDate).setTypes("timer").execute().actionGet();
    org.assertj.core.api.Assertions.assertThat(searchResponse.getHits().totalHits()).isEqualTo(1L);

    Map<String, Object> hit = searchResponse.getHits().getAt(0).sourceAsMap();
    assertTimestamp(hit);
    assertKey(hit, "name", prefix + ".foo.bar");
    assertKey(hit, "count", 1);
    assertKey(hit, "host", "localhost");
  }
Beispiel #14
0
 public static Timer timer(String name) {
   return metrics.timer(name);
 }
  private ExtensionResponse tryExecuteGremlinScript(
      final RexsterResourceContext rexsterResourceContext,
      final Graph graph,
      final Vertex vertex,
      final Edge edge,
      final String script) {

    final MetricRegistry metricRegistry = rexsterResourceContext.getMetricRegistry();
    final Timer scriptTimer = metricRegistry.timer(MetricRegistry.name("http", "script-engine"));
    final Counter successfulExecutions =
        metricRegistry.counter(MetricRegistry.name("http", "script-engine", "success"));
    final Counter failedExecutions =
        metricRegistry.counter(MetricRegistry.name("http", "script-engine", "fail"));

    ExtensionResponse extensionResponse;

    final JSONObject requestObject = rexsterResourceContext.getRequestObject();

    // can't initialize this statically because the configure() method won't get called before it.
    // need to think a bit on how to best initialized the controller.
    final EngineController engineController = EngineController.getInstance();

    final boolean showTypes = RequestObjectHelper.getShowTypes(requestObject);
    final long offsetStart = RequestObjectHelper.getStartOffset(requestObject);
    final long offsetEnd = RequestObjectHelper.getEndOffset(requestObject);

    final GraphSONMode mode = showTypes ? GraphSONMode.EXTENDED : GraphSONMode.NORMAL;
    final Set<String> returnKeys = RequestObjectHelper.getReturnKeys(requestObject, WILDCARD);

    final String languageToExecuteWith = getLanguageToExecuteWith(requestObject);
    final EngineHolder engineHolder;
    final ScriptEngine scriptEngine;
    try {
      if (!engineController.isEngineAvailable(languageToExecuteWith)) {
        return ExtensionResponse.error("language requested is not available on the server");
      }

      engineHolder = engineController.getEngineByLanguageName(languageToExecuteWith);
      scriptEngine = engineHolder.getEngine();
    } catch (ScriptException se) {
      return ExtensionResponse.error("could not get request script engine");
    }

    final Bindings bindings = createBindings(graph, vertex, edge, scriptEngine);

    // add all keys not defined by this request as bindings to the script engine
    placeParametersOnBinding(requestObject, bindings, showTypes);

    // get the list of "stored procedures" to run
    final RexsterApplicationGraph rag = rexsterResourceContext.getRexsterApplicationGraph();

    final ExtensionMethod extensionMethod = rexsterResourceContext.getExtensionMethod();
    Map configurationMap = null;

    Iterator<String> scriptsToRun = null;
    try {
      final ExtensionConfiguration extensionConfiguration =
          rag != null ? rag.findExtensionConfiguration(EXTENSION_NAMESPACE, EXTENSION_NAME) : null;
      if (extensionConfiguration != null) {
        configurationMap = extensionConfiguration.tryGetMapFromConfiguration();
        scriptsToRun = getScriptsToRun(requestObject, configurationMap);
      }
    } catch (IOException ioe) {
      return ExtensionResponse.error(
          ioe, generateErrorJson(extensionMethod.getExtensionApiAsJson()));
    }

    if ((script == null || script.isEmpty()) && scriptsToRun == null) {
      return ExtensionResponse.badRequest(
          "no scripts provided", generateErrorJson(extensionMethod.getExtensionApiAsJson()));
    }

    final Timer.Context context = scriptTimer.time();
    try {
      // result is either the ad-hoc script on the query string or the last "stored procedure"
      Object result = null;
      if (scriptsToRun != null) {
        while (scriptsToRun.hasNext()) {
          result = engineHolder.getEngine().eval(scriptsToRun.next(), bindings);
        }
      }

      if (isClientScriptAllowed(configurationMap) && script != null && !script.isEmpty()) {
        result = engineHolder.getEngine().eval(script, bindings);
      }

      final JSONArray results =
          new JSONResultConverter(mode, offsetStart, offsetEnd, returnKeys).convert(result);

      final HashMap<String, Object> resultMap = new HashMap<String, Object>();
      resultMap.put(Tokens.SUCCESS, true);
      resultMap.put(Tokens.RESULTS, results);

      final JSONObject resultObject = new JSONObject(resultMap);
      extensionResponse = ExtensionResponse.ok(resultObject);

      successfulExecutions.inc();

    } catch (Exception e) {
      logger.error(String.format("Gremlin Extension: %s", e.getMessage()), e);
      extensionResponse =
          ExtensionResponse.error(e, generateErrorJson(extensionMethod.getExtensionApiAsJson()));

      failedExecutions.inc();
    } finally {
      context.stop();
    }

    return extensionResponse;
  }
Beispiel #16
0
 /**
  * @param name the metric name
  * @return the timer
  */
 public static Timer workerTimer(String name) {
   return METRIC_REGISTRY.timer(getWorkerMetricName(name));
 }
/**
 * ConnectionProxyFactoryPerformanceTest - ConnectionProxyFactory Performance Test
 *
 * @author Vlad Mihalcea
 */
public class ConnectionProxyFactoryPerformanceTest extends ConnectionDecoratorTest {

  private static final Logger LOGGER =
      LoggerFactory.getLogger(ConnectionProxyFactoryPerformanceTest.class);

  private static MetricRegistry metricRegistry = new MetricRegistry();

  private static Slf4jReporter logReporter =
      Slf4jReporter.forRegistry(metricRegistry).outputTo(LOGGER).build();

  private static Timer noProxyHistogram = metricRegistry.timer("no-proxy");
  private static Timer proxyHistogram = metricRegistry.timer("proxy");

  static {
    logReporter.start(30, TimeUnit.SECONDS);
  }

  private final Connection targetConnection = Mockito.mock(Connection.class);
  private final ConnectionCallback connectionCallback = Mockito.mock(ConnectionCallback.class);

  private final InvocationHandler invocationHandler =
      new InvocationHandler() {
        @Override
        public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
          return method.invoke(targetConnection, args);
        }
      };
  private final Connection proxy =
      (Connection)
          Proxy.newProxyInstance(
              this.getClass().getClassLoader(), new Class[] {Connection.class}, invocationHandler);

  private ConnectionDecorator connectionDecorator =
      new ConnectionDecorator(targetConnection, connectionCallback);

  @Test
  public void testPerformance() {
    if (TestUtils.isPerformanceTesting()) {
      long durationMillis = 30 * 1000;
      timeAllMethodsForDuration(
          new Callable() {
            @Override
            public Object call() throws Exception {
              callConnectionMethods(noProxyHistogram);
              return null;
            }
          },
          durationMillis);
      timeAllMethodsForDuration(
          new Callable() {
            @Override
            public Object call() throws Exception {
              callConnectionMethods(proxyHistogram);
              return null;
            }
          },
          durationMillis);
    }
  }

  private void callConnectionMethods(Timer timer) throws SQLException {
    long startNanos = System.nanoTime();
    try {
      connectionDecorator.getMetaData();
      String javaVersion = System.getProperty("java.version");
      if (javaVersion.contains("1.7") || javaVersion.contains("1.8")) {
        connectionDecorator.setSchema("schema");
        connectionDecorator.abort(null);
      }
    } finally {
      long endNanos = System.nanoTime();
      timer.update((endNanos - startNanos), TimeUnit.NANOSECONDS);
    }
  }

  private void timeAllMethodsForDuration(Callable callable, long durationMillis) {
    long startMillis = System.currentTimeMillis();
    int i = 0;
    while (true) {
      long endMillis = System.currentTimeMillis();
      if (i % 1000 == 0) {
        LOGGER.info("Iteration: {}, total millis {}!", i, endMillis - startMillis);
        if ((endMillis - startMillis) > durationMillis) {
          break;
        }
      }
      try {
        callable.call();
      } catch (Exception e) {
        throw new IllegalArgumentException(e);
      }
      i++;
    }
  }
}
public class EventHandler extends AbstractHandler {
  private static final Logger log = LoggerFactory.getLogger(EventHandler.class);

  private AtomicLong numOps = new AtomicLong(0);
  private AtomicLong numTxs = new AtomicLong(0);

  private ConfigurationFactory configurationFactory = new TypesafeConfigFactory();
  private Conf configuration;
  private MessageEncoder messageEncoder;
  private MessageProducer messageProducer;

  private MetricRegistry metrics = new MetricRegistry();
  private Timer operationProcessingTimer = metrics.timer("operationProcessingTime");
  private Timer messageEncodingTimer = metrics.timer("encodingTime");
  private Timer messageSendingTimer = metrics.timer("sendingTime");
  private Meter operationProcessingErrorMeter = metrics.meter("processingErrors");
  private TxFactory txFactory;
  private ScheduledReporter metricsReporter;

  private String configurationPath;

  public EventHandler() {
    super(TxOpMode.op);
    log.info("created handler - default mode: " + getMode());
  }

  @VisibleForTesting
  public EventHandler(
      Conf configuration,
      MessageEncoder messageEncoder,
      MessageProducer messageProducer,
      TxFactory txFactory) {
    this.configuration = configuration;
    this.messageEncoder = messageEncoder;
    this.messageProducer = messageProducer;
    this.txFactory = txFactory;
  }

  @Override
  public void init(DsConfiguration conf, DsMetaData metaData) {
    super.init(conf, metaData);
    log.info("Initializing handler: Mode =" + getMode());
    configuration = configurationFactory.load(configurationPath);
    messageProducer = KafkaProducerFactory.create(configuration.kafka());
    messageEncoder = MessageEncoderFactory.create(configuration);
    txFactory = new TxFactory();

    if (configuration.metrics().isEnabled()) {
      metricsReporter = MetricsReporterFactory.createReporter(configuration.metrics(), metrics);
    }
  }

  @Override
  public Status transactionBegin(DsEvent e, DsTransaction transaction) {
    super.transactionBegin(e, transaction);

    if (log.isDebugEnabled()) {
      log.debug(
          "Received begin tx event, numTx="
              + numTxs.get()
              + " : position="
              + transaction.getTranID()
              + ", totalOps="
              + transaction.getTotalOps());
    }

    return Status.OK;
  }

  @Override
  public Status operationAdded(DsEvent e, DsTransaction transaction, DsOperation operation) {
    Status overallStatus = Status.OK;
    super.operationAdded(e, transaction, operation);
    numOps.incrementAndGet();

    final Tx tx = new Tx(transaction, getMetaData(), getConfig());
    final TableMetaData tMeta = getMetaData().getTableMetaData(operation.getTableName());
    final Op op = new Op(operation, tMeta, getConfig());

    operation.getTokens();
    if (isOperationMode()) {

      if (log.isDebugEnabled()) {
        log.debug(
            " Received operation: table='"
                + op.getTableName()
                + "'"
                + ", pos="
                + op.getPosition()
                + " (total_ops= "
                + tx.getTotalOps()
                + ", buffered="
                + tx.getSize()
                + ")"
                + ", ts="
                + op.getTimestamp());
      }

      Status operationStatus = processOperation(tx, op);

      if (Status.ABEND.equals(operationStatus)) {
        overallStatus = Status.ABEND;
      }
    }
    return overallStatus;
  }

  @Override
  public Status transactionCommit(DsEvent e, DsTransaction transaction) {
    Status overallStatus = Status.OK;
    super.transactionCommit(e, transaction);

    Tx tx = txFactory.createAdapterTx(transaction, getMetaData(), getConfig());
    numTxs.incrementAndGet();

    if (log.isDebugEnabled()) {
      log.debug(
          "transactionCommit event, tx #"
              + numTxs.get()
              + ":"
              + ", pos="
              + tx.getTranID()
              + " (total_ops= "
              + tx.getTotalOps()
              + ", buffered="
              + tx.getSize()
              + ")"
              + ", ts="
              + tx.getTimestamp()
              + ")");
    }

    if (!isOperationMode()) {
      for (Op op : tx) {
        Status operationStatus = processOperation(tx, op);

        if (Status.ABEND.equals(operationStatus)) {
          overallStatus = Status.ABEND;
        }
      }
    }

    return overallStatus;
  }

  private Status processOperation(Tx tx, Op op) {
    Timer.Context timer = operationProcessingTimer.time();
    Status status = Status.OK;

    try {
      encodeAndSend(tx, op);
    } catch (RuntimeException re) {
      operationProcessingErrorMeter.mark();
      log.error("Error processing operation: " + op.toString(), re);
      status = Status.ABEND;
    }

    timer.stop();
    return status;
  }

  @Override
  public Status metaDataChanged(DsEvent e, DsMetaData meta) {
    log.debug("Received metadata event: " + e + "; current tables: " + meta.getTableNames().size());
    return super.metaDataChanged(e, meta);
  }

  @Override
  public String reportStatus() {
    String s =
        "Status report: "
            + ", mode="
            + getMode()
            + ", transactions="
            + numTxs.get()
            + ", operations="
            + numOps.get();
    return s;
  }

  @Override
  public void destroy() {
    log.debug("destroy()... " + reportStatus());
    if (configuration.metricsEnabled()) {
      metricsReporter.stop();
    }

    messageProducer.terminate();
    super.destroy();
  }

  private void encodeAndSend(Tx tx, Op op) {
    if (log.isDebugEnabled()) {
      log.debug("Processing of transaction " + tx + " and operation " + op);
    }

    byte[] encodedMessage = encodeMessage(tx, op);
    sendMessage(encodedMessage);

    if (log.isDebugEnabled()) {
      log.debug("Completed processing of transaction " + tx + " and operation " + op);
    }
  }

  private void sendMessage(byte[] encodedMessage) {
    Timer.Context sendMessageTimer = messageSendingTimer.time();
    messageProducer.produce(encodedMessage);
    sendMessageTimer.stop();

    if (log.isDebugEnabled()) {
      log.debug("Completed send of message: " + new String(encodedMessage));
    }
  }

  private byte[] encodeMessage(Tx tx, Op op) {
    Timer.Context encodingTimer = messageEncodingTimer.time();
    byte[] encodedMessage = messageEncoder.encode(tx, op);
    encodingTimer.stop();

    if (log.isTraceEnabled()) {
      log.trace("Result of message encoding is = " + new String(encodedMessage));
    }

    return encodedMessage;
  }

  public void setConfigurationPath(String configurationPath) {
    this.configurationPath = configurationPath;
  }
}
Beispiel #19
0
 public static Optional<Timer.Context> startTimer(String className, String methodName) {
   if (metricRegistry != null)
     return Optional.of(
         metricRegistry.timer(String.format("%s.%s", className, methodName)).time());
   return Optional.absent();
 }
Beispiel #20
0
 /**
  * @param name the metric name
  * @return the timer
  */
 public static Timer clientTimer(String name) {
   return METRIC_REGISTRY.timer(getClientMetricName(name));
 }
public class LongBTreeTest {

  private static final boolean DEBUG = false;
  private static int perThreadTrees = 10000;
  private static int minTreeSize = 4;
  private static int maxTreeSize = 10000;
  private static int threads = DEBUG ? 1 : Runtime.getRuntime().availableProcessors() * 8;
  private static final MetricRegistry metrics = new MetricRegistry();
  private static final Timer BTREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "BTREE"));
  private static final Timer TREE_TIMER = metrics.timer(MetricRegistry.name(BTree.class, "TREE"));
  private static final ExecutorService MODIFY =
      Executors.newFixedThreadPool(threads, new NamedThreadFactory("MODIFY"));
  private static final ExecutorService COMPARE =
      DEBUG ? MODIFY : Executors.newFixedThreadPool(threads, new NamedThreadFactory("COMPARE"));
  private static final RandomAbort<Integer> SPORADIC_ABORT =
      new RandomAbort<>(new Random(), 0.0001f);

  static {
    System.setProperty("cassandra.btree.fanfactor", "4");
  }

  /** ************************ TEST ACCESS ******************************************* */
  @Test
  public void testSearchIterator() throws InterruptedException {
    final int perTreeSelections = 100;
    testRandomSelection(
        perThreadTrees,
        perTreeSelections,
        (test) -> {
          IndexedSearchIterator<Integer, Integer> iter1 = test.testAsSet.iterator();
          IndexedSearchIterator<Integer, Integer> iter2 = test.testAsList.iterator();
          return (key) -> {
            Integer found1 = iter1.hasNext() ? iter1.next(key) : null;
            Integer found2 = iter2.hasNext() ? iter2.next(key) : null;
            Assert.assertSame(found1, found2);
            if (found1 != null) Assert.assertEquals(iter1.indexOfCurrent(), iter2.indexOfCurrent());

            int index = Collections.binarySearch(test.canonicalList, key, test.comparator);
            if (index < 0) {
              Assert.assertNull(found1);
            } else {
              Assert.assertEquals(key, found1);
              Assert.assertEquals(index, iter1.indexOfCurrent());
            }

            // check that by advancing the same key again we get null, but only do it on one of the
            // two iterators
            // to ensure they both advance differently
            if (ThreadLocalRandom.current().nextBoolean()) Assert.assertNull(iter1.next(key));
            else Assert.assertNull(iter2.next(key));
          };
        });
  }

  @Test
  public void testInequalityLookups() throws InterruptedException {
    final int perTreeSelections = 2;
    testRandomSelectionOfSet(
        perThreadTrees,
        perTreeSelections,
        (test, canonical) -> {
          if (!canonical.isEmpty() || !test.isEmpty()) {
            Assert.assertEquals(canonical.isEmpty(), test.isEmpty());
            Assert.assertEquals(canonical.first(), test.first());
            Assert.assertEquals(canonical.last(), test.last());
          }
          return (key) -> {
            Assert.assertEquals(test.ceiling(key), canonical.ceiling(key));
            Assert.assertEquals(test.higher(key), canonical.higher(key));
            Assert.assertEquals(test.floor(key), canonical.floor(key));
            Assert.assertEquals(test.lower(key), canonical.lower(key));
          };
        });
  }

  @Test
  public void testListIndexes() throws InterruptedException {
    testRandomSelectionOfList(
        perThreadTrees,
        4,
        (test, canonical, cmp) ->
            (key) -> {
              int javaIndex = Collections.binarySearch(canonical, key, cmp);
              int btreeIndex = test.indexOf(key);
              Assert.assertEquals(javaIndex, btreeIndex);
              if (javaIndex >= 0)
                Assert.assertEquals(canonical.get(javaIndex), test.get(btreeIndex));
            });
  }

  @Test
  public void testToArray() throws InterruptedException {
    testRandomSelection(
        perThreadTrees,
        4,
        (selection) -> {
          Integer[] array = new Integer[selection.canonicalList.size() + 1];
          selection.testAsList.toArray(array, 1);
          Assert.assertEquals(null, array[0]);
          for (int j = 0; j < selection.canonicalList.size(); j++)
            Assert.assertEquals(selection.canonicalList.get(j), array[j + 1]);
        });
  }

  private static final class CountingFunction implements Function<Integer, Integer> {
    final Function<Integer, Integer> wrapped;
    int count = 0;

    protected CountingFunction(Function<Integer, Integer> wrapped) {
      this.wrapped = wrapped;
    }

    public Integer apply(Integer integer) {
      count++;
      return wrapped.apply(integer);
    }
  }

  @Test
  public void testTransformAndFilter() throws InterruptedException {
    testRandomSelection(
        perThreadTrees,
        4,
        false,
        false,
        false,
        (selection) -> {
          Map<Integer, Integer> update = new LinkedHashMap<>();
          for (Integer i : selection.testKeys) update.put(i, new Integer(i));

          CountingFunction function;
          Object[] original = selection.testAsSet.tree();
          Object[] transformed;

          // test replacing none, leaving all present
          function = new CountingFunction((x) -> x);
          transformed = BTree.transformAndFilter(original, function);
          Assert.assertEquals(BTree.size(original), function.count);
          Assert.assertSame(original, transformed);

          // test replacing some, leaving all present
          function = new CountingFunction((x) -> update.containsKey(x) ? update.get(x) : x);
          transformed = BTree.transformAndFilter(original, function);
          Assert.assertEquals(BTree.size(original), function.count);
          assertSame(transform(selection.canonicalList, function.wrapped), iterable(transformed));

          // test replacing some, removing some
          function = new CountingFunction(update::get);
          transformed = BTree.transformAndFilter(original, function);
          Assert.assertEquals(BTree.size(original), function.count);
          assertSame(
              filter(transform(selection.canonicalList, function.wrapped), notNull()),
              iterable(transformed));

          // test replacing none, removing some
          function = new CountingFunction((x) -> update.containsKey(x) ? null : x);
          transformed = BTree.transformAndFilter(selection.testAsList.tree(), function);
          Assert.assertEquals(BTree.size(original), function.count);
          assertSame(
              filter(transform(selection.canonicalList, function.wrapped), notNull()),
              iterable(transformed));
        });
  }

  private static void assertSame(Iterable<Integer> i1, Iterable<Integer> i2) {
    assertSame(i1.iterator(), i2.iterator());
  }

  private static void assertSame(Iterator<Integer> i1, Iterator<Integer> i2) {
    while (i1.hasNext() && i2.hasNext()) Assert.assertSame(i1.next(), i2.next());
    Assert.assertEquals(i1.hasNext(), i2.hasNext());
  }

  private void testRandomSelectionOfList(
      int perThreadTrees, int perTreeSelections, BTreeListTestFactory testRun)
      throws InterruptedException {
    testRandomSelection(
        perThreadTrees,
        perTreeSelections,
        (BTreeTestFactory)
            (selection) ->
                testRun.get(selection.testAsList, selection.canonicalList, selection.comparator));
  }

  private void testRandomSelectionOfSet(
      int perThreadTrees, int perTreeSelections, BTreeSetTestFactory testRun)
      throws InterruptedException {
    testRandomSelection(
        perThreadTrees,
        perTreeSelections,
        (BTreeTestFactory) (selection) -> testRun.get(selection.testAsSet, selection.canonicalSet));
  }

  static interface BTreeSetTestFactory {
    TestEachKey get(BTreeSet<Integer> test, NavigableSet<Integer> canonical);
  }

  static interface BTreeListTestFactory {
    TestEachKey get(
        BTreeSet<Integer> test, List<Integer> canonical, Comparator<Integer> comparator);
  }

  static interface BTreeTestFactory {
    TestEachKey get(RandomSelection test);
  }

  static interface TestEachKey {
    void testOne(Integer value);
  }

  private void testRandomSelection(
      int perThreadTrees, int perTreeSelections, BTreeTestFactory testRun)
      throws InterruptedException {
    testRandomSelection(
        perThreadTrees,
        perTreeSelections,
        (selection) -> {
          TestEachKey testEachKey = testRun.get(selection);
          for (Integer key : selection.testKeys) testEachKey.testOne(key);
        });
  }

  private void testRandomSelection(
      int perThreadTrees, int perTreeSelections, Consumer<RandomSelection> testRun)
      throws InterruptedException {
    testRandomSelection(perThreadTrees, perTreeSelections, true, true, true, testRun);
  }

  private void testRandomSelection(
      int perThreadTrees,
      int perTreeSelections,
      boolean narrow,
      boolean mixInNotPresentItems,
      boolean permitReversal,
      Consumer<RandomSelection> testRun)
      throws InterruptedException {
    int threads = Runtime.getRuntime().availableProcessors();
    final CountDownLatch latch = new CountDownLatch(threads);
    final AtomicLong errors = new AtomicLong();
    final AtomicLong count = new AtomicLong();
    final long totalCount = threads * perThreadTrees * perTreeSelections;
    for (int t = 0; t < threads; t++) {
      Runnable runnable =
          new Runnable() {
            public void run() {
              try {
                for (int i = 0; i < perThreadTrees; i++) {
                  RandomTree tree = randomTree(minTreeSize, maxTreeSize);
                  for (int j = 0; j < perTreeSelections; j++) {
                    testRun.accept(tree.select(narrow, mixInNotPresentItems, permitReversal));
                    count.incrementAndGet();
                  }
                }
              } catch (Throwable t) {
                errors.incrementAndGet();
                t.printStackTrace();
              }
              latch.countDown();
            }
          };
      MODIFY.execute(runnable);
    }
    while (latch.getCount() > 0) {
      for (int i = 0; i < 10L; i++) {
        latch.await(1L, TimeUnit.SECONDS);
        Assert.assertEquals(0, errors.get());
      }
      log(
          "%.1f%% complete %s",
          100 * count.get() / (double) totalCount,
          errors.get() > 0 ? ("Errors: " + errors.get()) : "");
    }
  }

  private static class RandomSelection {
    final List<Integer> testKeys;
    final NavigableSet<Integer> canonicalSet;
    final List<Integer> canonicalList;
    final BTreeSet<Integer> testAsSet;
    final BTreeSet<Integer> testAsList;
    final Comparator<Integer> comparator;

    private RandomSelection(
        List<Integer> testKeys,
        NavigableSet<Integer> canonicalSet,
        BTreeSet<Integer> testAsSet,
        List<Integer> canonicalList,
        BTreeSet<Integer> testAsList,
        Comparator<Integer> comparator) {
      this.testKeys = testKeys;
      this.canonicalList = canonicalList;
      this.canonicalSet = canonicalSet;
      this.testAsSet = testAsSet;
      this.testAsList = testAsList;
      this.comparator = comparator;
    }
  }

  private static class RandomTree {
    final NavigableSet<Integer> canonical;
    final BTreeSet<Integer> test;

    private RandomTree(NavigableSet<Integer> canonical, BTreeSet<Integer> test) {
      this.canonical = canonical;
      this.test = test;
    }

    RandomSelection select(boolean narrow, boolean mixInNotPresentItems, boolean permitReversal) {
      ThreadLocalRandom random = ThreadLocalRandom.current();
      NavigableSet<Integer> canonicalSet = this.canonical;
      BTreeSet<Integer> testAsSet = this.test;
      List<Integer> canonicalList = new ArrayList<>(canonicalSet);
      BTreeSet<Integer> testAsList = this.test;

      Assert.assertEquals(canonicalSet.size(), testAsSet.size());
      Assert.assertEquals(canonicalList.size(), testAsList.size());

      // sometimes select keys first, so we cover full range
      List<Integer> allKeys = randomKeys(canonical, mixInNotPresentItems);
      List<Integer> keys = allKeys;

      int narrowCount = random.nextInt(3);
      while (narrow && canonicalList.size() > 10 && keys.size() > 10 && narrowCount-- > 0) {
        boolean useLb = random.nextBoolean();
        boolean useUb = random.nextBoolean();
        if (!(useLb | useUb)) continue;

        // select a range smaller than the total span when we have more narrowing iterations left
        int indexRange = keys.size() / (narrowCount + 1);

        boolean lbInclusive = true;
        Integer lbKey = canonicalList.get(0);
        int lbKeyIndex = 0, lbIndex = 0;
        boolean ubInclusive = true;
        Integer ubKey = canonicalList.get(canonicalList.size() - 1);
        int ubKeyIndex = keys.size(), ubIndex = canonicalList.size();

        if (useLb) {
          lbKeyIndex = random.nextInt(0, indexRange - 1);
          Integer candidate = keys.get(lbKeyIndex);
          if (useLb = (candidate > lbKey && candidate <= ubKey)) {
            lbInclusive = random.nextBoolean();
            lbKey = keys.get(lbKeyIndex);
            lbIndex = Collections.binarySearch(canonicalList, lbKey);
            if (lbIndex >= 0 && !lbInclusive) lbIndex++;
            else if (lbIndex < 0) lbIndex = -1 - lbIndex;
          }
        }
        if (useUb) {
          ubKeyIndex =
              random.nextInt(Math.max(lbKeyIndex, keys.size() - indexRange), keys.size() - 1);
          Integer candidate = keys.get(ubKeyIndex);
          if (useUb = (candidate < ubKey && candidate >= lbKey)) {
            ubInclusive = random.nextBoolean();
            ubKey = keys.get(ubKeyIndex);
            ubIndex = Collections.binarySearch(canonicalList, ubKey);
            if (ubIndex >= 0 && ubInclusive) {
              ubIndex++;
            } else if (ubIndex < 0) ubIndex = -1 - ubIndex;
          }
        }
        if (ubIndex < lbIndex) {
          ubIndex = lbIndex;
          ubKey = lbKey;
          ubInclusive = false;
        }

        canonicalSet =
            !useLb
                ? canonicalSet.headSet(ubKey, ubInclusive)
                : !useUb
                    ? canonicalSet.tailSet(lbKey, lbInclusive)
                    : canonicalSet.subSet(lbKey, lbInclusive, ubKey, ubInclusive);
        testAsSet =
            !useLb
                ? testAsSet.headSet(ubKey, ubInclusive)
                : !useUb
                    ? testAsSet.tailSet(lbKey, lbInclusive)
                    : testAsSet.subSet(lbKey, lbInclusive, ubKey, ubInclusive);

        keys = keys.subList(lbKeyIndex, ubKeyIndex);
        canonicalList = canonicalList.subList(lbIndex, ubIndex);
        testAsList = testAsList.subList(lbIndex, ubIndex);

        Assert.assertEquals(canonicalSet.size(), testAsSet.size());
        Assert.assertEquals(canonicalList.size(), testAsList.size());
      }

      // possibly restore full set of keys, to test case where we are provided existing keys that
      // are out of bounds
      if (keys != allKeys && random.nextBoolean()) keys = allKeys;

      Comparator<Integer> comparator = naturalOrder();
      if (permitReversal && random.nextBoolean()) {
        if (allKeys != keys) keys = new ArrayList<>(keys);
        if (canonicalSet != canonical) canonicalList = new ArrayList<>(canonicalList);
        Collections.reverse(keys);
        Collections.reverse(canonicalList);
        testAsList = testAsList.descendingSet();

        canonicalSet = canonicalSet.descendingSet();
        testAsSet = testAsSet.descendingSet();
        comparator = reverseOrder();
      }

      Assert.assertEquals(canonicalSet.size(), testAsSet.size());
      Assert.assertEquals(canonicalList.size(), testAsList.size());
      if (!canonicalSet.isEmpty()) {
        Assert.assertEquals(canonicalSet.first(), canonicalList.get(0));
        Assert.assertEquals(canonicalSet.last(), canonicalList.get(canonicalList.size() - 1));
        Assert.assertEquals(canonicalSet.first(), testAsSet.first());
        Assert.assertEquals(canonicalSet.last(), testAsSet.last());
        Assert.assertEquals(canonicalSet.first(), testAsList.get(0));
        Assert.assertEquals(canonicalSet.last(), testAsList.get(testAsList.size() - 1));
      }

      return new RandomSelection(
          keys, canonicalSet, testAsSet, canonicalList, testAsList, comparator);
    }
  }

  private static RandomTree randomTree(int minSize, int maxSize) {
    // perform most of our tree constructions via update, as this is more efficient; since every run
    // uses this
    // we test builder disproportionately more often than if it had its own test anyway
    return ThreadLocalRandom.current().nextFloat() < 0.95
        ? randomTreeByUpdate(minSize, maxSize)
        : randomTreeByBuilder(minSize, maxSize);
  }

  private static RandomTree randomTreeByUpdate(int minSize, int maxSize) {
    assert minSize > 3;
    TreeSet<Integer> canonical = new TreeSet<>();

    ThreadLocalRandom random = ThreadLocalRandom.current();
    int targetSize = random.nextInt(minSize, maxSize);
    int maxModificationSize = random.nextInt(2, targetSize);
    Object[] accmumulate = BTree.empty();
    int curSize = 0;
    while (curSize < targetSize) {
      int nextSize = maxModificationSize == 1 ? 1 : random.nextInt(1, maxModificationSize);
      TreeSet<Integer> build = new TreeSet<>();
      for (int i = 0; i < nextSize; i++) {
        Integer next = random.nextInt();
        build.add(next);
        canonical.add(next);
      }
      accmumulate =
          BTree.update(accmumulate, naturalOrder(), build, UpdateFunction.<Integer>noOp());
      curSize += nextSize;
      maxModificationSize = Math.min(maxModificationSize, targetSize - curSize);
    }
    return new RandomTree(canonical, BTreeSet.<Integer>wrap(accmumulate, naturalOrder()));
  }

  private static RandomTree randomTreeByBuilder(int minSize, int maxSize) {
    assert minSize > 3;
    ThreadLocalRandom random = ThreadLocalRandom.current();
    BTree.Builder<Integer> builder = BTree.builder(naturalOrder());

    int targetSize = random.nextInt(minSize, maxSize);
    int maxModificationSize = (int) Math.sqrt(targetSize);

    TreeSet<Integer> canonical = new TreeSet<>();

    int curSize = 0;
    TreeSet<Integer> ordered = new TreeSet<>();
    List<Integer> shuffled = new ArrayList<>();
    while (curSize < targetSize) {
      int nextSize = maxModificationSize <= 1 ? 1 : random.nextInt(1, maxModificationSize);

      // leave a random selection of previous values
      (random.nextBoolean() ? ordered.headSet(random.nextInt()) : ordered.tailSet(random.nextInt()))
          .clear();
      shuffled =
          new ArrayList<>(
              shuffled.subList(0, shuffled.size() < 2 ? 0 : random.nextInt(shuffled.size() / 2)));

      for (int i = 0; i < nextSize; i++) {
        Integer next = random.nextInt();
        ordered.add(next);
        shuffled.add(next);
        canonical.add(next);
      }

      switch (random.nextInt(5)) {
        case 0:
          builder.addAll(ordered);
          break;
        case 1:
          builder.addAll(BTreeSet.of(ordered));
          break;
        case 2:
          for (Integer i : ordered) builder.add(i);
        case 3:
          builder.addAll(shuffled);
          break;
        case 4:
          for (Integer i : shuffled) builder.add(i);
      }

      curSize += nextSize;
      maxModificationSize = Math.min(maxModificationSize, targetSize - curSize);
    }

    BTreeSet<Integer> btree = BTreeSet.<Integer>wrap(builder.build(), naturalOrder());
    Assert.assertEquals(canonical.size(), btree.size());
    return new RandomTree(canonical, btree);
  }

  // select a random subset of the keys, with an optional random population of keys inbetween those
  // that are present
  // return a value with the search position
  private static List<Integer> randomKeys(
      Iterable<Integer> canonical, boolean mixInNotPresentItems) {
    ThreadLocalRandom rnd = ThreadLocalRandom.current();
    boolean useFake = mixInNotPresentItems && rnd.nextBoolean();
    final float fakeRatio = rnd.nextFloat();
    List<Integer> results = new ArrayList<>();
    Long fakeLb = null, fakeUb = null;
    for (Integer v : canonical) {
      if (!useFake
          || fakeLb == null
          || (fakeUb == null ? v - 1 : fakeUb) <= fakeLb + 1
          || rnd.nextFloat() < fakeRatio) {
        // if we cannot safely construct a fake value, or our randomizer says not to, we emit the
        // next real value
        results.add(v);
        fakeLb = v.longValue();
        fakeUb = null;
      } else {
        // otherwise we emit a fake value in the range immediately proceeding the last real value,
        // and not
        // exceeding the real value that would have proceeded (ignoring any other suppressed real
        // values since)
        if (fakeUb == null) fakeUb = v.longValue() - 1;
        long mid = (fakeLb + fakeUb) / 2;
        assert mid < fakeUb;
        results.add((int) mid);
        fakeLb = mid;
      }
    }
    final float useChance = rnd.nextFloat();
    return Lists.newArrayList(filter(results, (x) -> rnd.nextFloat() < useChance));
  }

  /** ************************ TEST MUTATION ******************************************* */
  @Test
  public void testOversizedMiddleInsert() {
    TreeSet<Integer> canon = new TreeSet<>();
    for (int i = 0; i < 10000000; i++) canon.add(i);
    Object[] btree =
        BTree.build(Arrays.asList(Integer.MIN_VALUE, Integer.MAX_VALUE), UpdateFunction.noOp());
    btree = BTree.update(btree, naturalOrder(), canon, UpdateFunction.<Integer>noOp());
    canon.add(Integer.MIN_VALUE);
    canon.add(Integer.MAX_VALUE);
    assertTrue(BTree.isWellFormed(btree, naturalOrder()));
    testEqual("Oversize", BTree.iterator(btree), canon.iterator());
  }

  @Test
  public void testIndividualInsertsSmallOverlappingRange()
      throws ExecutionException, InterruptedException {
    testInsertions(50, 1, 1, true);
  }

  @Test
  public void testBatchesSmallOverlappingRange() throws ExecutionException, InterruptedException {
    testInsertions(50, 1, 5, true);
  }

  @Test
  public void testIndividualInsertsMediumSparseRange()
      throws ExecutionException, InterruptedException {
    testInsertions(perThreadTrees / 10, 500, 10, 1, true);
  }

  @Test
  public void testBatchesMediumSparseRange() throws ExecutionException, InterruptedException {
    testInsertions(500, 10, 10, true);
  }

  @Test
  public void testLargeBatchesLargeRange() throws ExecutionException, InterruptedException {
    testInsertions(perThreadTrees / 10, Math.max(maxTreeSize, 5000), 3, 100, true);
  }

  @Test
  public void testRandomRangeAndBatches() throws ExecutionException, InterruptedException {
    ThreadLocalRandom random = ThreadLocalRandom.current();
    int treeSize = random.nextInt(maxTreeSize / 10, maxTreeSize * 10);
    for (int i = 0; i < perThreadTrees / 10; i++)
      testInsertions(threads * 10, treeSize, random.nextInt(1, 100) / 10f, treeSize / 100, true);
  }

  @Test
  public void testSlicingSmallRandomTrees() throws ExecutionException, InterruptedException {
    testInsertions(50, 10, 10, false);
  }

  private static void testInsertions(
      int perTestCount, float testKeyRatio, int modificationBatchSize, boolean quickEquality)
      throws ExecutionException, InterruptedException {
    int tests = perThreadTrees * threads;
    testInsertions(tests, perTestCount, testKeyRatio, modificationBatchSize, quickEquality);
  }

  private static void testInsertions(
      int tests,
      int perTestCount,
      float testKeyRatio,
      int modificationBatchSize,
      boolean quickEquality)
      throws ExecutionException, InterruptedException {
    int batchesPerTest = perTestCount / modificationBatchSize;
    int testKeyRange = (int) (perTestCount * testKeyRatio);
    long totalCount = (long) perTestCount * tests;
    log(
        "Performing %d tests of %d operations, with %.2f max size/key-range ratio in batches of ~%d ops",
        tests, perTestCount, 1 / testKeyRatio, modificationBatchSize);

    // if we're not doing quick-equality, we can spam with garbage for all the checks we perform, so
    // we'll split the work into smaller chunks
    int chunkSize = quickEquality ? tests : (int) (100000 / Math.pow(perTestCount, 2));
    for (int chunk = 0; chunk < tests; chunk += chunkSize) {
      final List<ListenableFutureTask<List<ListenableFuture<?>>>> outer = new ArrayList<>();
      for (int i = 0; i < chunkSize; i++) {
        int maxRunLength =
            modificationBatchSize == 1
                ? 1
                : ThreadLocalRandom.current().nextInt(1, modificationBatchSize);
        outer.add(
            doOneTestInsertions(
                testKeyRange, maxRunLength, modificationBatchSize, batchesPerTest, quickEquality));
      }

      final List<ListenableFuture<?>> inner = new ArrayList<>();
      long complete = 0;
      int reportInterval = Math.max(1000, (int) (totalCount / 10000));
      long lastReportAt = 0;
      for (ListenableFutureTask<List<ListenableFuture<?>>> f : outer) {
        inner.addAll(f.get());
        complete += perTestCount;
        if (complete - lastReportAt >= reportInterval) {
          long done = (chunk * perTestCount) + complete;
          float ratio = done / (float) totalCount;
          log("Completed %.1f%% (%d of %d operations)", ratio * 100, done, totalCount);
          lastReportAt = complete;
        }
      }
      Futures.allAsList(inner).get();
    }
    Snapshot snap = BTREE_TIMER.getSnapshot();
    log(
        "btree: %.2fns, %.2fns, %.2fns",
        snap.getMedian(), snap.get95thPercentile(), snap.get999thPercentile());
    snap = TREE_TIMER.getSnapshot();
    log(
        "java: %.2fns, %.2fns, %.2fns",
        snap.getMedian(), snap.get95thPercentile(), snap.get999thPercentile());
    log("Done");
  }

  private static ListenableFutureTask<List<ListenableFuture<?>>> doOneTestInsertions(
      final int upperBound,
      final int maxRunLength,
      final int averageModsPerIteration,
      final int iterations,
      final boolean quickEquality) {
    ListenableFutureTask<List<ListenableFuture<?>>> f =
        ListenableFutureTask.create(
            new Callable<List<ListenableFuture<?>>>() {
              @Override
              public List<ListenableFuture<?>> call() {
                final List<ListenableFuture<?>> r = new ArrayList<>();
                NavigableMap<Integer, Integer> canon = new TreeMap<>();
                Object[] btree = BTree.empty();
                final TreeMap<Integer, Integer> buffer = new TreeMap<>();
                ThreadLocalRandom rnd = ThreadLocalRandom.current();
                for (int i = 0; i < iterations; i++) {
                  buffer.clear();
                  int mods = rnd.nextInt(1, averageModsPerIteration * 2);
                  while (mods > 0) {
                    int v = rnd.nextInt(upperBound);
                    int rc = Math.max(0, Math.min(mods, maxRunLength) - 1);
                    int c = 1 + (rc <= 0 ? 0 : rnd.nextInt(rc));
                    for (int j = 0; j < c; j++) {
                      buffer.put(v, v);
                      v++;
                    }
                    mods -= c;
                  }
                  Timer.Context ctxt;
                  ctxt = TREE_TIMER.time();
                  canon.putAll(buffer);
                  ctxt.stop();
                  ctxt = BTREE_TIMER.time();
                  Object[] next = null;
                  while (next == null)
                    next = BTree.update(btree, naturalOrder(), buffer.keySet(), SPORADIC_ABORT);
                  btree = next;
                  ctxt.stop();

                  if (!BTree.isWellFormed(btree, naturalOrder())) {
                    log("ERROR: Not well formed");
                    throw new AssertionError("Not well formed!");
                  }
                  if (quickEquality)
                    testEqual("", BTree.iterator(btree), canon.keySet().iterator());
                  else r.addAll(testAllSlices("RND", btree, new TreeSet<>(canon.keySet())));
                }
                return r;
              }
            });
    if (DEBUG) f.run();
    else MODIFY.execute(f);
    return f;
  }

  @Test
  public void testSlicingAllSmallTrees() throws ExecutionException, InterruptedException {
    Object[] cur = BTree.empty();
    TreeSet<Integer> canon = new TreeSet<>();
    // we set FAN_FACTOR to 4, so 128 items is four levels deep, three fully populated
    for (int i = 0; i < 128; i++) {
      String id = String.format("[0..%d)", canon.size());
      log("Testing " + id);
      Futures.allAsList(testAllSlices(id, cur, canon)).get();
      Object[] next = null;
      while (next == null)
        next = BTree.update(cur, naturalOrder(), Arrays.asList(i), SPORADIC_ABORT);
      cur = next;
      canon.add(i);
    }
  }

  private static List<ListenableFuture<?>> testAllSlices(
      String id, Object[] btree, NavigableSet<Integer> canon) {
    List<ListenableFuture<?>> waitFor = new ArrayList<>();
    testAllSlices(id + " ASC", new BTreeSet<>(btree, naturalOrder()), canon, true, waitFor);
    testAllSlices(
        id + " DSC",
        new BTreeSet<Integer>(btree, naturalOrder()).descendingSet(),
        canon.descendingSet(),
        false,
        waitFor);
    return waitFor;
  }

  private static void testAllSlices(
      String id,
      NavigableSet<Integer> btree,
      NavigableSet<Integer> canon,
      boolean ascending,
      List<ListenableFuture<?>> results) {
    testOneSlice(id, btree, canon, results);
    for (Integer lb : range(canon.size(), Integer.MIN_VALUE, ascending)) {
      // test head/tail sets
      testOneSlice(
          String.format("%s->[..%d)", id, lb),
          btree.headSet(lb, true),
          canon.headSet(lb, true),
          results);
      testOneSlice(
          String.format("%s->(..%d)", id, lb),
          btree.headSet(lb, false),
          canon.headSet(lb, false),
          results);
      testOneSlice(
          String.format("%s->(%d..]", id, lb),
          btree.tailSet(lb, true),
          canon.tailSet(lb, true),
          results);
      testOneSlice(
          String.format("%s->(%d..]", id, lb),
          btree.tailSet(lb, false),
          canon.tailSet(lb, false),
          results);
      for (Integer ub : range(canon.size(), lb, ascending)) {
        // test subsets
        testOneSlice(
            String.format("%s->[%d..%d]", id, lb, ub),
            btree.subSet(lb, true, ub, true),
            canon.subSet(lb, true, ub, true),
            results);
        testOneSlice(
            String.format("%s->(%d..%d]", id, lb, ub),
            btree.subSet(lb, false, ub, true),
            canon.subSet(lb, false, ub, true),
            results);
        testOneSlice(
            String.format("%s->[%d..%d)", id, lb, ub),
            btree.subSet(lb, true, ub, false),
            canon.subSet(lb, true, ub, false),
            results);
        testOneSlice(
            String.format("%s->(%d..%d)", id, lb, ub),
            btree.subSet(lb, false, ub, false),
            canon.subSet(lb, false, ub, false),
            results);
      }
    }
  }

  private static void testOneSlice(
      final String id,
      final NavigableSet<Integer> test,
      final NavigableSet<Integer> canon,
      List<ListenableFuture<?>> results) {
    ListenableFutureTask<?> f =
        ListenableFutureTask.create(
            new Runnable() {

              @Override
              public void run() {
                test(id + " Count", test.size(), canon.size());
                testEqual(id, test.iterator(), canon.iterator());
                testEqual(id + "->DSCI", test.descendingIterator(), canon.descendingIterator());
                testEqual(
                    id + "->DSCS",
                    test.descendingSet().iterator(),
                    canon.descendingSet().iterator());
                testEqual(
                    id + "->DSCS->DSCI",
                    test.descendingSet().descendingIterator(),
                    canon.descendingSet().descendingIterator());
              }
            },
            null);
    results.add(f);
    if (DEBUG) f.run();
    else COMPARE.execute(f);
  }

  private static void test(String id, int test, int expect) {
    if (test != expect) {
      log("%s: Expected %d, Got %d", id, expect, test);
    }
  }

  private static <V> void testEqual(String id, Iterator<V> btree, Iterator<V> canon) {
    boolean equal = true;
    while (btree.hasNext() && canon.hasNext()) {
      Object i = btree.next();
      Object j = canon.next();
      if (!Objects.equals(i, j)) {
        log("%s: Expected %d, Got %d", id, j, i);
        equal = false;
      }
    }
    while (btree.hasNext()) {
      log("%s: Expected <Nil>, Got %d", id, btree.next());
      equal = false;
    }
    while (canon.hasNext()) {
      log("%s: Expected %d, Got Nil", id, canon.next());
      equal = false;
    }
    if (!equal) throw new AssertionError("Not equal");
  }

  // should only be called on sets that range from 0->N or N->0
  private static final Iterable<Integer> range(
      final int size, final int from, final boolean ascending) {
    return new Iterable<Integer>() {
      int cur;
      int delta;
      int end;

      {
        if (ascending) {
          end = size + 1;
          cur = from == Integer.MIN_VALUE ? -1 : from;
          delta = 1;
        } else {
          end = -2;
          cur = from == Integer.MIN_VALUE ? size : from;
          delta = -1;
        }
      }

      @Override
      public Iterator<Integer> iterator() {
        return new Iterator<Integer>() {
          @Override
          public boolean hasNext() {
            return cur != end;
          }

          @Override
          public Integer next() {
            Integer r = cur;
            cur += delta;
            return r;
          }

          @Override
          public void remove() {
            throw new UnsupportedOperationException();
          }
        };
      }
    };
  }

  private static final class RandomAbort<V> implements UpdateFunction<V, V> {
    final Random rnd;
    final float chance;

    private RandomAbort(Random rnd, float chance) {
      this.rnd = rnd;
      this.chance = chance;
    }

    public V apply(V replacing, V update) {
      return update;
    }

    public boolean abortEarly() {
      return rnd.nextFloat() < chance;
    }

    public void allocated(long heapSize) {}

    public V apply(V v) {
      return v;
    }
  }

  public static void main(String[] args)
      throws ExecutionException, InterruptedException, InvocationTargetException,
          IllegalAccessException {
    for (String arg : args) {
      if (arg.startsWith("fan=")) System.setProperty("cassandra.btree.fanfactor", arg.substring(4));
      else if (arg.startsWith("min=")) minTreeSize = Integer.parseInt(arg.substring(4));
      else if (arg.startsWith("max=")) maxTreeSize = Integer.parseInt(arg.substring(4));
      else if (arg.startsWith("count=")) perThreadTrees = Integer.parseInt(arg.substring(6));
      else exit();
    }

    List<Method> methods = new ArrayList<>();
    for (Method m : LongBTreeTest.class.getDeclaredMethods()) {
      if (m.getParameters().length > 0) continue;
      for (Annotation annotation : m.getAnnotations())
        if (annotation.annotationType() == Test.class) methods.add(m);
    }

    LongBTreeTest test = new LongBTreeTest();
    Collections.sort(methods, (a, b) -> a.getName().compareTo(b.getName()));
    log(Lists.transform(methods, (m) -> m.getName()).toString());
    for (Method m : methods) {
      log(m.getName());
      m.invoke(test);
    }
    log("success");
  }

  private static void exit() {
    log("usage: fan=<int> min=<int> max=<int> count=<int>");
    log("fan:   btree fanout");
    log("min:   minimum btree size (must be >= 4)");
    log("max:   maximum btree size (must be >= 4)");
    log("count: number of trees to assign each core, for each test");
  }

  private static void log(String formatstr, Object... args) {
    args = Arrays.copyOf(args, args.length + 1);
    System.arraycopy(args, 0, args, 1, args.length - 1);
    args[0] = System.currentTimeMillis();
    System.out.printf("%tT: " + formatstr + "\n", args);
  }
}
 /**
  * This is called by RestExpress during it's bind() operation--right before it starts listening.
  */
 @Override
 public void bind(RestExpress server) {
   this.activeRequestsCounter = metrics.counter("active-requests");
   this.allExceptionsCounter = metrics.counter("all-exceptions");
   this.allTimesTimer = metrics.timer("all-times");
 }
Beispiel #23
0
 public void startRequest() {
   rootRequestPerfCtx = start("req");
   Timer metricsTimer = requestsMetrics.timer(pathInfo);
   requestMetricsStartContext = metricsTimer.time();
 }