/**
* Get a CouchbaseClient based on the REST response from a Couchbase server where the username is
* different than the bucket name.
*
* <p>Note that when specifying a ConnectionFactory you must specify a BinaryConnectionFactory.
* Also the ConnectionFactory's protocol and locator values are always overwritten. The protocol
* will always be binary and the locator will be chosen based on the bucket type you are
* connecting to.
*
* <p>To connect to the "default" special bucket for a given cluster, use an empty string as the
* password.
*
* <p>If a password has not been assigned to the bucket, it is typically an empty string.
*
* <p>The subscribe variable is determines whether or not we will subscribe to the configuration
* changes feed. This constructor should be used when calling super from subclasses of
* CouchbaseClient since the subclass might want to start the changes feed later.
*
* @param cf the ConnectionFactory to use to create connections
* @throws IOException if connections could not be made
* @throws ConfigurationException if the configuration provided by the server has issues or is not
* compatible
*/
public CouchbaseClient(CouchbaseConnectionFactory cf) throws IOException {
super(cf, AddrUtil.getAddresses(cf.getVBucketConfig().getServers()));
List<InetSocketAddress> addrs =
AddrUtil.getAddressesFromURL(cf.getVBucketConfig().getCouchServers());
getLogger().info(MODE_ERROR);
vconn = cf.createViewConnection(addrs);
cf.getConfigurationProvider().subscribe(cf.getBucketName(), this);
}
/**
* Reconfigures the connected ViewNodes.
*
* <p>When a reconfiguration event happens, new ViewNodes may need to be added or old ones need to
* be removed from the current configuration. This method takes care that those operations are
* performed in the correct order and are executed in a thread-safe manner.
*
* @param bucket the bucket which has been rebalanced.
*/
public void reconfigure(Bucket bucket) {
reconfiguring = true;
try {
// get a new collection of addresses from the received config
HashSet<SocketAddress> newServerAddresses = new HashSet<SocketAddress>();
List<InetSocketAddress> newServers =
AddrUtil.getAddressesFromURL(bucket.getConfig().getCouchServers());
for (InetSocketAddress server : newServers) {
// add parsed address to our collections
newServerAddresses.add(server);
}
// split current nodes to "odd nodes" and "stay nodes"
ArrayList<ViewNode> shutdownNodes = new ArrayList<ViewNode>();
ArrayList<ViewNode> stayNodes = new ArrayList<ViewNode>();
ArrayList<InetSocketAddress> stayServers = new ArrayList<InetSocketAddress>();
wlock.lock();
try {
for (ViewNode current : couchNodes) {
if (newServerAddresses.contains(current.getSocketAddress())) {
stayNodes.add(current);
stayServers.add((InetSocketAddress) current.getSocketAddress());
} else {
shutdownNodes.add(current);
}
}
// prepare a collection of addresses for new nodes
newServers.removeAll(stayServers);
// create a collection of new nodes
List<ViewNode> newNodes = createConnections(newServers);
// merge stay nodes with new nodes
List<ViewNode> mergedNodes = new ArrayList<ViewNode>();
mergedNodes.addAll(stayNodes);
mergedNodes.addAll(newNodes);
couchNodes = mergedNodes;
} finally {
wlock.unlock();
}
// shutdown for the oddNodes
for (ViewNode qa : shutdownNodes) {
try {
qa.shutdown();
} catch (IOException e) {
getLogger().error("Error shutting down connection to " + qa.getSocketAddress());
}
}
} catch (IOException e) {
getLogger().error("Connection reconfiguration failed", e);
} finally {
reconfiguring = false;
}
}
public static net.spy.memcached.MemcachedClient getSpyClient() {
// MemcachedClient c = new MemcachedClient(AddrUtil.getAddresses("server1:11211
// server2:11211"));
try {
return new net.spy.memcached.MemcachedClient(
net.spy.memcached.AddrUtil.getAddresses(MEMCACHED_SERVER_ADDR));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Creates a new instance that stores tickets in the given memcached hosts.
*
* @param hostnames Array of memcached hosts where each element is of the form host:port.
* @param ticketGrantingTicketTimeOut TGT timeout in seconds.
* @param serviceTicketTimeOut ST timeout in seconds.
*/
public MemCacheTicketRegistry(
final String[] hostnames,
final int ticketGrantingTicketTimeOut,
final int serviceTicketTimeOut) {
try {
this.client = new MemcachedClient(AddrUtil.getAddresses(Arrays.asList(hostnames)));
} catch (final IOException e) {
throw new IllegalArgumentException("Invalid memcached host specification.", e);
}
this.tgtTimeout = ticketGrantingTicketTimeOut;
this.stTimeout = serviceTicketTimeOut;
}
@Override
public void start() {
try {
ExtendedConnectionFactory connectionFactory = new ExtendedKetamaConnectionFactory();
if (config.getTranscoder() != null) {
connectionFactory.setTranscoder(config.getTranscoder());
} else {
connectionFactory.setTranscoder(new KvdbTranscoder());
}
String servers = config.getServers();
if (servers == null) {
throw new RuntimeException("Kvdb server address must be specified.");
}
String[] serverSplits = servers.split(" ");
String writeServer = serverSplits[0].trim();
String readServers = serverSplits.length == 1 ? writeServer : serverSplits[1].trim();
readClient = new MemcachedClient(connectionFactory, AddrUtil.getAddresses(readServers));
writeClient = new MemcachedClient(connectionFactory, AddrUtil.getAddresses(writeServer));
} catch (IOException e) {
throw new RuntimeException("Construct kvdb client failed, cause: ", e);
}
}
public StatefulStreamMapper() {
if (System.getProperty("memcache.address") != null) {
try {
memcache =
new MemcachedClient(AddrUtil.getAddresses(System.getProperty("memcache.address")));
} catch (Exception e) {
e.printStackTrace();
if (memcache != null) {
log.info("Shutting down memcache connection...");
memcache.shutdown();
memcache = null;
}
}
} else memcache = null;
}
public static void main(String[] args) throws Exception {
final MemcachedClient db = new MemcachedClient(AddrUtil.getAddresses(args[0]));
final Transcoder<Object> transcoder =
new Transcoder<Object>() {
public CachedData encode(Object data) {
return new CachedData(0, (byte[]) data, getMaxSize());
}
public Object decode(CachedData cachedData) {
return cachedData.getData();
}
public boolean asyncDecode(CachedData arg0) {
return false;
}
public int getMaxSize() {
return CachedData.MAX_SIZE;
}
};
final String key = "key";
final byte[] value = new byte[128];
value[0] = 1;
value[1] = 2;
value[2] = 3;
db.set(key, 0, value).get();
Runnable task =
new Runnable() {
public void run() {
try {
assertArrayEquals(value, (byte[]) db.asyncGet(key, transcoder).get());
} catch (Exception e) {
e.printStackTrace();
}
}
};
for (int c : new int[] {1, 10, 100, 200, 300}) {
System.out.println(c + ":" + (new Benchmark(c, 10000).run(task)) + "ms");
}
db.shutdown();
}
@Override
public void afterPropertiesSet() throws Exception {
if (!inited && enable) {
try {
ConnectionFactoryBuilder cfBuidler = new ConnectionFactoryBuilder();
cfBuidler.setProtocol(Protocol.BINARY);
cfBuidler.setOpTimeout(3000);
cfBuidler.setTimeoutExceptionThreshold(3000);
if (auth) {
cfBuidler.setAuthDescriptor(
new AuthDescriptor(
new String[] {"PLAIN"}, new PlainCallbackHandler(username, password)));
}
client = new MemcachedClient(cfBuidler.build(), AddrUtil.getAddresses(host + ":" + port));
inited = true;
logger.info("Memcached Client inited...");
} catch (IOException e) {
logger.error("Memcached client initial exception:", e);
e.printStackTrace();
}
}
}
public static net.spy.memcached.MemcachedClient createSpyMemcachedClient(MemcacheConfig config) {
try {
ConnectionFactoryBuilder factoryBuilder = new ConnectionFactoryBuilder();
net.spy.memcached.transcoders.SerializingTranscoder transcoder =
new net.spy.memcached.transcoders.SerializingTranscoder();
transcoder.setCompressionThreshold(config.getCompressionThreshold());
factoryBuilder.setProtocol(ConnectionFactoryBuilder.Protocol.BINARY);
factoryBuilder.setTranscoder(transcoder);
factoryBuilder.setOpTimeout(config.getOpTimeout());
factoryBuilder.setTimeoutExceptionThreshold(5);
factoryBuilder.setLocatorType(ConnectionFactoryBuilder.Locator.CONSISTENT);
factoryBuilder.setFailureMode(FailureMode.Redistribute);
factoryBuilder.setUseNagleAlgorithm(false);
factoryBuilder.setHashAlg(DefaultHashAlgorithm.KETAMA_HASH);
return new net.spy.memcached.MemcachedClient(
factoryBuilder.build(), AddrUtil.getAddresses(config.getServers()));
} catch (Exception ex) {
log.error("Failed to create SpyMemcachedClient", ex);
}
return null;
}
public static MemcachedCache create(final MemcachedCacheConfig config) {
final ConcurrentMap<String, AtomicLong> counters = new ConcurrentHashMap<>();
final ConcurrentMap<String, AtomicLong> meters = new ConcurrentHashMap<>();
final AbstractMonitor monitor =
new AbstractMonitor() {
final AtomicReference<Map<String, Long>> priorValues =
new AtomicReference<Map<String, Long>>(new HashMap<String, Long>());
@Override
public boolean doMonitor(ServiceEmitter emitter) {
final Map<String, Long> priorValues = this.priorValues.get();
final Map<String, Long> currentValues = getCurrentValues();
final ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder();
for (Map.Entry<String, Long> entry : currentValues.entrySet()) {
emitter.emit(
builder
.setDimension("memcached metric", entry.getKey())
.build("query/cache/memcached/total", entry.getValue()));
final Long prior = priorValues.get(entry.getKey());
if (prior != null) {
emitter.emit(
builder
.setDimension("memcached metric", entry.getKey())
.build("query/cache/memcached/delta", entry.getValue() - prior));
}
}
if (!this.priorValues.compareAndSet(priorValues, currentValues)) {
log.error("Prior value changed while I was reporting! updating anyways");
this.priorValues.set(currentValues);
}
return true;
}
private Map<String, Long> getCurrentValues() {
final ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder();
for (Map.Entry<String, AtomicLong> entry : counters.entrySet()) {
builder.put(entry.getKey(), entry.getValue().get());
}
for (Map.Entry<String, AtomicLong> entry : meters.entrySet()) {
builder.put(entry.getKey(), entry.getValue().get());
}
return builder.build();
}
};
try {
LZ4Transcoder transcoder = new LZ4Transcoder(config.getMaxObjectSize());
// always use compression
transcoder.setCompressionThreshold(0);
OperationQueueFactory opQueueFactory;
long maxQueueBytes = config.getMaxOperationQueueSize();
if (maxQueueBytes > 0) {
opQueueFactory = new MemcachedOperationQueueFactory(maxQueueBytes);
} else {
opQueueFactory = new LinkedOperationQueueFactory();
}
final Predicate<String> interesting =
new Predicate<String>() {
// See net.spy.memcached.MemcachedConnection.registerMetrics()
private final Set<String> interestingMetrics =
ImmutableSet.of(
"[MEM] Reconnecting Nodes (ReconnectQueue)",
// "[MEM] Shutting Down Nodes (NodesToShutdown)", // Busted
"[MEM] Request Rate: All",
"[MEM] Average Bytes written to OS per write",
"[MEM] Average Bytes read from OS per read",
"[MEM] Average Time on wire for operations (µs)",
"[MEM] Response Rate: All (Failure + Success + Retry)",
"[MEM] Response Rate: Retry",
"[MEM] Response Rate: Failure",
"[MEM] Response Rate: Success");
@Override
public boolean apply(@Nullable String input) {
return input != null && interestingMetrics.contains(input);
}
};
final MetricCollector metricCollector =
new MetricCollector() {
@Override
public void addCounter(String name) {
if (!interesting.apply(name)) {
return;
}
counters.putIfAbsent(name, new AtomicLong(0L));
if (log.isDebugEnabled()) {
log.debug("Add Counter [%s]", name);
}
}
@Override
public void removeCounter(String name) {
if (log.isDebugEnabled()) {
log.debug("Ignoring request to remove [%s]", name);
}
}
@Override
public void incrementCounter(String name) {
if (!interesting.apply(name)) {
return;
}
AtomicLong counter = counters.get(name);
if (counter == null) {
counters.putIfAbsent(name, new AtomicLong(0));
counter = counters.get(name);
}
counter.incrementAndGet();
if (log.isDebugEnabled()) {
log.debug("Increment [%s]", name);
}
}
@Override
public void incrementCounter(String name, int amount) {
if (!interesting.apply(name)) {
return;
}
AtomicLong counter = counters.get(name);
if (counter == null) {
counters.putIfAbsent(name, new AtomicLong(0));
counter = counters.get(name);
}
counter.addAndGet(amount);
if (log.isDebugEnabled()) {
log.debug("Increment [%s] %d", name, amount);
}
}
@Override
public void decrementCounter(String name) {
if (!interesting.apply(name)) {
return;
}
AtomicLong counter = counters.get(name);
if (counter == null) {
counters.putIfAbsent(name, new AtomicLong(0));
counter = counters.get(name);
}
counter.decrementAndGet();
if (log.isDebugEnabled()) {
log.debug("Decrement [%s]", name);
}
}
@Override
public void decrementCounter(String name, int amount) {
if (!interesting.apply(name)) {
return;
}
AtomicLong counter = counters.get(name);
if (counter == null) {
counters.putIfAbsent(name, new AtomicLong(0L));
counter = counters.get(name);
}
counter.addAndGet(-amount);
if (log.isDebugEnabled()) {
log.debug("Decrement [%s] %d", name, amount);
}
}
@Override
public void addMeter(String name) {
if (!interesting.apply(name)) {
return;
}
meters.putIfAbsent(name, new AtomicLong(0L));
if (log.isDebugEnabled()) {
log.debug("Adding meter [%s]", name);
}
}
@Override
public void removeMeter(String name) {
if (!interesting.apply(name)) {
return;
}
if (log.isDebugEnabled()) {
log.debug("Ignoring request to remove meter [%s]", name);
}
}
@Override
public void markMeter(String name) {
if (!interesting.apply(name)) {
return;
}
AtomicLong meter = meters.get(name);
if (meter == null) {
meters.putIfAbsent(name, new AtomicLong(0L));
meter = meters.get(name);
}
meter.incrementAndGet();
if (log.isDebugEnabled()) {
log.debug("Increment counter [%s]", name);
}
}
@Override
public void addHistogram(String name) {
log.debug("Ignoring add histogram [%s]", name);
}
@Override
public void removeHistogram(String name) {
log.debug("Ignoring remove histogram [%s]", name);
}
@Override
public void updateHistogram(String name, int amount) {
log.debug("Ignoring update histogram [%s]: %d", name, amount);
}
};
final ConnectionFactory connectionFactory =
new MemcachedCustomConnectionFactoryBuilder()
// 1000 repetitions gives us good distribution with murmur3_128
// (approx < 5% difference in counts across nodes, with 5 cache nodes)
.setKetamaNodeRepetitions(1000)
.setHashAlg(MURMUR3_128)
.setProtocol(ConnectionFactoryBuilder.Protocol.BINARY)
.setLocatorType(ConnectionFactoryBuilder.Locator.CONSISTENT)
.setDaemon(true)
.setFailureMode(FailureMode.Cancel)
.setTranscoder(transcoder)
.setShouldOptimize(true)
.setOpQueueMaxBlockTime(config.getTimeout())
.setOpTimeout(config.getTimeout())
.setReadBufferSize(config.getReadBufferSize())
.setOpQueueFactory(opQueueFactory)
.setMetricCollector(metricCollector)
.setEnableMetrics(MetricType.DEBUG) // Not as scary as it sounds
.build();
final List<InetSocketAddress> hosts = AddrUtil.getAddresses(config.getHosts());
final Supplier<ResourceHolder<MemcachedClientIF>> clientSupplier;
if (config.getNumConnections() > 1) {
clientSupplier =
new LoadBalancingPool<MemcachedClientIF>(
config.getNumConnections(),
new Supplier<MemcachedClientIF>() {
@Override
public MemcachedClientIF get() {
try {
return new MemcachedClient(connectionFactory, hosts);
} catch (IOException e) {
log.error(e, "Unable to create memcached client");
throw Throwables.propagate(e);
}
}
});
} else {
clientSupplier =
Suppliers.<ResourceHolder<MemcachedClientIF>>ofInstance(
StupidResourceHolder.<MemcachedClientIF>create(
new MemcachedClient(connectionFactory, hosts)));
}
return new MemcachedCache(clientSupplier, config, monitor);
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
/**
* 私有构造方法,初始化memcached
*
* @throws IOException
*/
private MemCacheServiceImpl(String prop_file) {
// ------------装载memchache信息-----------------------
String server1 = PropertiesReader.getValue(prop_file, "server1");
String server2 = PropertiesReader.getValue(prop_file, "server2");
try {
opTimeout = Integer.parseInt(PropertiesReader.getValue(prop_file, "opTimeout"));
opTimeoutBulk = Integer.parseInt(PropertiesReader.getValue(prop_file, "opTimeoutBulk"));
retry = Integer.parseInt(PropertiesReader.getValue(prop_file, "retry"));
readBufSize = Integer.parseInt(PropertiesReader.getValue(prop_file, "readBufSize"));
opQueueLen = Integer.parseInt(PropertiesReader.getValue(prop_file, "opQueueLen"));
expHour = Integer.parseInt(PropertiesReader.getValue(prop_file, "expHour"));
} catch (Exception e) {
log.error("loading properties fail, use default config!");
}
// 从配置文件中读取相应的配置信息
try {
mc1 =
new MemcachedClient(
new DefaultConnectionFactory() {
@Override
public HashAlgorithm getHashAlg() {
return HashAlgorithm.CRC32_HASH;
}
@Override
public long getOperationTimeout() {
return opTimeout * 1000;
}
@Override
public int getReadBufSize() {
return readBufSize;
}
@Override
public OperationFactory getOperationFactory() {
return super.getOperationFactory();
}
@Override
public int getOpQueueLen() {
return opQueueLen;
}
@Override
public boolean isDaemon() {
return true;
}
},
AddrUtil.getAddresses(server1));
mc2 =
new MemcachedClient(
new DefaultConnectionFactory() {
@Override
public HashAlgorithm getHashAlg() {
return HashAlgorithm.CRC32_HASH;
}
@Override
public long getOperationTimeout() {
return opTimeout * 1000;
}
@Override
public int getReadBufSize() {
return readBufSize;
}
@Override
public OperationFactory getOperationFactory() {
return super.getOperationFactory();
}
@Override
public int getOpQueueLen() {
return opQueueLen;
}
@Override
public boolean isDaemon() {
return true;
}
},
AddrUtil.getAddresses(server2));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 使用gb2312编码
SerializingTranscoder x1 = (SerializingTranscoder) mc1.getTranscoder();
x1.setCharset(ENCODING);
SerializingTranscoder x2 = (SerializingTranscoder) mc2.getTranscoder();
x2.setCharset(ENCODING);
}