public static MemcachedCache create(final MemcachedCacheConfig config) { final ConcurrentMap<String, AtomicLong> counters = new ConcurrentHashMap<>(); final ConcurrentMap<String, AtomicLong> meters = new ConcurrentHashMap<>(); final AbstractMonitor monitor = new AbstractMonitor() { final AtomicReference<Map<String, Long>> priorValues = new AtomicReference<Map<String, Long>>(new HashMap<String, Long>()); @Override public boolean doMonitor(ServiceEmitter emitter) { final Map<String, Long> priorValues = this.priorValues.get(); final Map<String, Long> currentValues = getCurrentValues(); final ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder(); for (Map.Entry<String, Long> entry : currentValues.entrySet()) { emitter.emit( builder .setDimension("memcached metric", entry.getKey()) .build("query/cache/memcached/total", entry.getValue())); final Long prior = priorValues.get(entry.getKey()); if (prior != null) { emitter.emit( builder .setDimension("memcached metric", entry.getKey()) .build("query/cache/memcached/delta", entry.getValue() - prior)); } } if (!this.priorValues.compareAndSet(priorValues, currentValues)) { log.error("Prior value changed while I was reporting! updating anyways"); this.priorValues.set(currentValues); } return true; } private Map<String, Long> getCurrentValues() { final ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder(); for (Map.Entry<String, AtomicLong> entry : counters.entrySet()) { builder.put(entry.getKey(), entry.getValue().get()); } for (Map.Entry<String, AtomicLong> entry : meters.entrySet()) { builder.put(entry.getKey(), entry.getValue().get()); } return builder.build(); } }; try { LZ4Transcoder transcoder = new LZ4Transcoder(config.getMaxObjectSize()); // always use compression transcoder.setCompressionThreshold(0); OperationQueueFactory opQueueFactory; long maxQueueBytes = config.getMaxOperationQueueSize(); if (maxQueueBytes > 0) { opQueueFactory = new MemcachedOperationQueueFactory(maxQueueBytes); } else { opQueueFactory = new LinkedOperationQueueFactory(); } final Predicate<String> interesting = new Predicate<String>() { // See net.spy.memcached.MemcachedConnection.registerMetrics() private final Set<String> interestingMetrics = ImmutableSet.of( "[MEM] Reconnecting Nodes (ReconnectQueue)", // "[MEM] Shutting Down Nodes (NodesToShutdown)", // Busted "[MEM] Request Rate: All", "[MEM] Average Bytes written to OS per write", "[MEM] Average Bytes read from OS per read", "[MEM] Average Time on wire for operations (µs)", "[MEM] Response Rate: All (Failure + Success + Retry)", "[MEM] Response Rate: Retry", "[MEM] Response Rate: Failure", "[MEM] Response Rate: Success"); @Override public boolean apply(@Nullable String input) { return input != null && interestingMetrics.contains(input); } }; final MetricCollector metricCollector = new MetricCollector() { @Override public void addCounter(String name) { if (!interesting.apply(name)) { return; } counters.putIfAbsent(name, new AtomicLong(0L)); if (log.isDebugEnabled()) { log.debug("Add Counter [%s]", name); } } @Override public void removeCounter(String name) { if (log.isDebugEnabled()) { log.debug("Ignoring request to remove [%s]", name); } } @Override public void incrementCounter(String name) { if (!interesting.apply(name)) { return; } AtomicLong counter = counters.get(name); if (counter == null) { counters.putIfAbsent(name, new AtomicLong(0)); counter = counters.get(name); } counter.incrementAndGet(); if (log.isDebugEnabled()) { log.debug("Increment [%s]", name); } } @Override public void incrementCounter(String name, int amount) { if (!interesting.apply(name)) { return; } AtomicLong counter = counters.get(name); if (counter == null) { counters.putIfAbsent(name, new AtomicLong(0)); counter = counters.get(name); } counter.addAndGet(amount); if (log.isDebugEnabled()) { log.debug("Increment [%s] %d", name, amount); } } @Override public void decrementCounter(String name) { if (!interesting.apply(name)) { return; } AtomicLong counter = counters.get(name); if (counter == null) { counters.putIfAbsent(name, new AtomicLong(0)); counter = counters.get(name); } counter.decrementAndGet(); if (log.isDebugEnabled()) { log.debug("Decrement [%s]", name); } } @Override public void decrementCounter(String name, int amount) { if (!interesting.apply(name)) { return; } AtomicLong counter = counters.get(name); if (counter == null) { counters.putIfAbsent(name, new AtomicLong(0L)); counter = counters.get(name); } counter.addAndGet(-amount); if (log.isDebugEnabled()) { log.debug("Decrement [%s] %d", name, amount); } } @Override public void addMeter(String name) { if (!interesting.apply(name)) { return; } meters.putIfAbsent(name, new AtomicLong(0L)); if (log.isDebugEnabled()) { log.debug("Adding meter [%s]", name); } } @Override public void removeMeter(String name) { if (!interesting.apply(name)) { return; } if (log.isDebugEnabled()) { log.debug("Ignoring request to remove meter [%s]", name); } } @Override public void markMeter(String name) { if (!interesting.apply(name)) { return; } AtomicLong meter = meters.get(name); if (meter == null) { meters.putIfAbsent(name, new AtomicLong(0L)); meter = meters.get(name); } meter.incrementAndGet(); if (log.isDebugEnabled()) { log.debug("Increment counter [%s]", name); } } @Override public void addHistogram(String name) { log.debug("Ignoring add histogram [%s]", name); } @Override public void removeHistogram(String name) { log.debug("Ignoring remove histogram [%s]", name); } @Override public void updateHistogram(String name, int amount) { log.debug("Ignoring update histogram [%s]: %d", name, amount); } }; final ConnectionFactory connectionFactory = new MemcachedCustomConnectionFactoryBuilder() // 1000 repetitions gives us good distribution with murmur3_128 // (approx < 5% difference in counts across nodes, with 5 cache nodes) .setKetamaNodeRepetitions(1000) .setHashAlg(MURMUR3_128) .setProtocol(ConnectionFactoryBuilder.Protocol.BINARY) .setLocatorType(ConnectionFactoryBuilder.Locator.CONSISTENT) .setDaemon(true) .setFailureMode(FailureMode.Cancel) .setTranscoder(transcoder) .setShouldOptimize(true) .setOpQueueMaxBlockTime(config.getTimeout()) .setOpTimeout(config.getTimeout()) .setReadBufferSize(config.getReadBufferSize()) .setOpQueueFactory(opQueueFactory) .setMetricCollector(metricCollector) .setEnableMetrics(MetricType.DEBUG) // Not as scary as it sounds .build(); final List<InetSocketAddress> hosts = AddrUtil.getAddresses(config.getHosts()); final Supplier<ResourceHolder<MemcachedClientIF>> clientSupplier; if (config.getNumConnections() > 1) { clientSupplier = new LoadBalancingPool<MemcachedClientIF>( config.getNumConnections(), new Supplier<MemcachedClientIF>() { @Override public MemcachedClientIF get() { try { return new MemcachedClient(connectionFactory, hosts); } catch (IOException e) { log.error(e, "Unable to create memcached client"); throw Throwables.propagate(e); } } }); } else { clientSupplier = Suppliers.<ResourceHolder<MemcachedClientIF>>ofInstance( StupidResourceHolder.<MemcachedClientIF>create( new MemcachedClient(connectionFactory, hosts))); } return new MemcachedCache(clientSupplier, config, monitor); } catch (IOException e) { throw Throwables.propagate(e); } }