コード例 #1
0
  /**
   * Get a CouchbaseClient based on the REST response from a Couchbase server where the username is
   * different than the bucket name.
   *
   * <p>Note that when specifying a ConnectionFactory you must specify a BinaryConnectionFactory.
   * Also the ConnectionFactory's protocol and locator values are always overwritten. The protocol
   * will always be binary and the locator will be chosen based on the bucket type you are
   * connecting to.
   *
   * <p>To connect to the "default" special bucket for a given cluster, use an empty string as the
   * password.
   *
   * <p>If a password has not been assigned to the bucket, it is typically an empty string.
   *
   * <p>The subscribe variable is determines whether or not we will subscribe to the configuration
   * changes feed. This constructor should be used when calling super from subclasses of
   * CouchbaseClient since the subclass might want to start the changes feed later.
   *
   * @param cf the ConnectionFactory to use to create connections
   * @throws IOException if connections could not be made
   * @throws ConfigurationException if the configuration provided by the server has issues or is not
   *     compatible
   */
  public CouchbaseClient(CouchbaseConnectionFactory cf) throws IOException {
    super(cf, AddrUtil.getAddresses(cf.getVBucketConfig().getServers()));
    List<InetSocketAddress> addrs =
        AddrUtil.getAddressesFromURL(cf.getVBucketConfig().getCouchServers());

    getLogger().info(MODE_ERROR);
    vconn = cf.createViewConnection(addrs);
    cf.getConfigurationProvider().subscribe(cf.getBucketName(), this);
  }
コード例 #2
0
  /**
   * Reconfigures the connected ViewNodes.
   *
   * <p>When a reconfiguration event happens, new ViewNodes may need to be added or old ones need to
   * be removed from the current configuration. This method takes care that those operations are
   * performed in the correct order and are executed in a thread-safe manner.
   *
   * @param bucket the bucket which has been rebalanced.
   */
  public void reconfigure(Bucket bucket) {
    reconfiguring = true;

    try {
      // get a new collection of addresses from the received config
      HashSet<SocketAddress> newServerAddresses = new HashSet<SocketAddress>();
      List<InetSocketAddress> newServers =
          AddrUtil.getAddressesFromURL(bucket.getConfig().getCouchServers());
      for (InetSocketAddress server : newServers) {
        // add parsed address to our collections
        newServerAddresses.add(server);
      }

      // split current nodes to "odd nodes" and "stay nodes"
      ArrayList<ViewNode> shutdownNodes = new ArrayList<ViewNode>();
      ArrayList<ViewNode> stayNodes = new ArrayList<ViewNode>();
      ArrayList<InetSocketAddress> stayServers = new ArrayList<InetSocketAddress>();

      wlock.lock();
      try {
        for (ViewNode current : couchNodes) {
          if (newServerAddresses.contains(current.getSocketAddress())) {
            stayNodes.add(current);
            stayServers.add((InetSocketAddress) current.getSocketAddress());
          } else {
            shutdownNodes.add(current);
          }
        }

        // prepare a collection of addresses for new nodes
        newServers.removeAll(stayServers);

        // create a collection of new nodes
        List<ViewNode> newNodes = createConnections(newServers);

        // merge stay nodes with new nodes
        List<ViewNode> mergedNodes = new ArrayList<ViewNode>();
        mergedNodes.addAll(stayNodes);
        mergedNodes.addAll(newNodes);

        couchNodes = mergedNodes;
      } finally {
        wlock.unlock();
      }

      // shutdown for the oddNodes
      for (ViewNode qa : shutdownNodes) {
        try {
          qa.shutdown();
        } catch (IOException e) {
          getLogger().error("Error shutting down connection to " + qa.getSocketAddress());
        }
      }

    } catch (IOException e) {
      getLogger().error("Connection reconfiguration failed", e);
    } finally {
      reconfiguring = false;
    }
  }
コード例 #3
0
 public static net.spy.memcached.MemcachedClient getSpyClient() {
   // MemcachedClient c = new MemcachedClient(AddrUtil.getAddresses("server1:11211
   // server2:11211"));
   try {
     return new net.spy.memcached.MemcachedClient(
         net.spy.memcached.AddrUtil.getAddresses(MEMCACHED_SERVER_ADDR));
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
コード例 #4
0
 /**
  * Creates a new instance that stores tickets in the given memcached hosts.
  *
  * @param hostnames Array of memcached hosts where each element is of the form host:port.
  * @param ticketGrantingTicketTimeOut TGT timeout in seconds.
  * @param serviceTicketTimeOut ST timeout in seconds.
  */
 public MemCacheTicketRegistry(
     final String[] hostnames,
     final int ticketGrantingTicketTimeOut,
     final int serviceTicketTimeOut) {
   try {
     this.client = new MemcachedClient(AddrUtil.getAddresses(Arrays.asList(hostnames)));
   } catch (final IOException e) {
     throw new IllegalArgumentException("Invalid memcached host specification.", e);
   }
   this.tgtTimeout = ticketGrantingTicketTimeOut;
   this.stTimeout = serviceTicketTimeOut;
 }
コード例 #5
0
 @Override
 public void start() {
   try {
     ExtendedConnectionFactory connectionFactory = new ExtendedKetamaConnectionFactory();
     if (config.getTranscoder() != null) {
       connectionFactory.setTranscoder(config.getTranscoder());
     } else {
       connectionFactory.setTranscoder(new KvdbTranscoder());
     }
     String servers = config.getServers();
     if (servers == null) {
       throw new RuntimeException("Kvdb server address must be specified.");
     }
     String[] serverSplits = servers.split(" ");
     String writeServer = serverSplits[0].trim();
     String readServers = serverSplits.length == 1 ? writeServer : serverSplits[1].trim();
     readClient = new MemcachedClient(connectionFactory, AddrUtil.getAddresses(readServers));
     writeClient = new MemcachedClient(connectionFactory, AddrUtil.getAddresses(writeServer));
   } catch (IOException e) {
     throw new RuntimeException("Construct kvdb client failed, cause: ", e);
   }
 }
コード例 #6
0
 public StatefulStreamMapper() {
   if (System.getProperty("memcache.address") != null) {
     try {
       memcache =
           new MemcachedClient(AddrUtil.getAddresses(System.getProperty("memcache.address")));
     } catch (Exception e) {
       e.printStackTrace();
       if (memcache != null) {
         log.info("Shutting down memcache connection...");
         memcache.shutdown();
         memcache = null;
       }
     }
   } else memcache = null;
 }
コード例 #7
0
  public static void main(String[] args) throws Exception {
    final MemcachedClient db = new MemcachedClient(AddrUtil.getAddresses(args[0]));
    final Transcoder<Object> transcoder =
        new Transcoder<Object>() {
          public CachedData encode(Object data) {
            return new CachedData(0, (byte[]) data, getMaxSize());
          }

          public Object decode(CachedData cachedData) {
            return cachedData.getData();
          }

          public boolean asyncDecode(CachedData arg0) {
            return false;
          }

          public int getMaxSize() {
            return CachedData.MAX_SIZE;
          }
        };
    final String key = "key";
    final byte[] value = new byte[128];
    value[0] = 1;
    value[1] = 2;
    value[2] = 3;
    db.set(key, 0, value).get();
    Runnable task =
        new Runnable() {
          public void run() {
            try {
              assertArrayEquals(value, (byte[]) db.asyncGet(key, transcoder).get());
            } catch (Exception e) {
              e.printStackTrace();
            }
          }
        };
    for (int c : new int[] {1, 10, 100, 200, 300}) {
      System.out.println(c + ":" + (new Benchmark(c, 10000).run(task)) + "ms");
    }
    db.shutdown();
  }
コード例 #8
0
 @Override
 public void afterPropertiesSet() throws Exception {
   if (!inited && enable) {
     try {
       ConnectionFactoryBuilder cfBuidler = new ConnectionFactoryBuilder();
       cfBuidler.setProtocol(Protocol.BINARY);
       cfBuidler.setOpTimeout(3000);
       cfBuidler.setTimeoutExceptionThreshold(3000);
       if (auth) {
         cfBuidler.setAuthDescriptor(
             new AuthDescriptor(
                 new String[] {"PLAIN"}, new PlainCallbackHandler(username, password)));
       }
       client = new MemcachedClient(cfBuidler.build(), AddrUtil.getAddresses(host + ":" + port));
       inited = true;
       logger.info("Memcached Client inited...");
     } catch (IOException e) {
       logger.error("Memcached client initial exception:", e);
       e.printStackTrace();
     }
   }
 }
コード例 #9
0
  public static net.spy.memcached.MemcachedClient createSpyMemcachedClient(MemcacheConfig config) {
    try {
      ConnectionFactoryBuilder factoryBuilder = new ConnectionFactoryBuilder();

      net.spy.memcached.transcoders.SerializingTranscoder transcoder =
          new net.spy.memcached.transcoders.SerializingTranscoder();
      transcoder.setCompressionThreshold(config.getCompressionThreshold());

      factoryBuilder.setProtocol(ConnectionFactoryBuilder.Protocol.BINARY);
      factoryBuilder.setTranscoder(transcoder);
      factoryBuilder.setOpTimeout(config.getOpTimeout());
      factoryBuilder.setTimeoutExceptionThreshold(5);
      factoryBuilder.setLocatorType(ConnectionFactoryBuilder.Locator.CONSISTENT);
      factoryBuilder.setFailureMode(FailureMode.Redistribute);
      factoryBuilder.setUseNagleAlgorithm(false);
      factoryBuilder.setHashAlg(DefaultHashAlgorithm.KETAMA_HASH);

      return new net.spy.memcached.MemcachedClient(
          factoryBuilder.build(), AddrUtil.getAddresses(config.getServers()));
    } catch (Exception ex) {
      log.error("Failed to create SpyMemcachedClient", ex);
    }
    return null;
  }
コード例 #10
0
  public static MemcachedCache create(final MemcachedCacheConfig config) {
    final ConcurrentMap<String, AtomicLong> counters = new ConcurrentHashMap<>();
    final ConcurrentMap<String, AtomicLong> meters = new ConcurrentHashMap<>();
    final AbstractMonitor monitor =
        new AbstractMonitor() {
          final AtomicReference<Map<String, Long>> priorValues =
              new AtomicReference<Map<String, Long>>(new HashMap<String, Long>());

          @Override
          public boolean doMonitor(ServiceEmitter emitter) {
            final Map<String, Long> priorValues = this.priorValues.get();
            final Map<String, Long> currentValues = getCurrentValues();
            final ServiceMetricEvent.Builder builder = ServiceMetricEvent.builder();
            for (Map.Entry<String, Long> entry : currentValues.entrySet()) {
              emitter.emit(
                  builder
                      .setDimension("memcached metric", entry.getKey())
                      .build("query/cache/memcached/total", entry.getValue()));
              final Long prior = priorValues.get(entry.getKey());
              if (prior != null) {
                emitter.emit(
                    builder
                        .setDimension("memcached metric", entry.getKey())
                        .build("query/cache/memcached/delta", entry.getValue() - prior));
              }
            }

            if (!this.priorValues.compareAndSet(priorValues, currentValues)) {
              log.error("Prior value changed while I was reporting! updating anyways");
              this.priorValues.set(currentValues);
            }
            return true;
          }

          private Map<String, Long> getCurrentValues() {
            final ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder();
            for (Map.Entry<String, AtomicLong> entry : counters.entrySet()) {
              builder.put(entry.getKey(), entry.getValue().get());
            }
            for (Map.Entry<String, AtomicLong> entry : meters.entrySet()) {
              builder.put(entry.getKey(), entry.getValue().get());
            }
            return builder.build();
          }
        };
    try {
      LZ4Transcoder transcoder = new LZ4Transcoder(config.getMaxObjectSize());

      // always use compression
      transcoder.setCompressionThreshold(0);

      OperationQueueFactory opQueueFactory;
      long maxQueueBytes = config.getMaxOperationQueueSize();
      if (maxQueueBytes > 0) {
        opQueueFactory = new MemcachedOperationQueueFactory(maxQueueBytes);
      } else {
        opQueueFactory = new LinkedOperationQueueFactory();
      }

      final Predicate<String> interesting =
          new Predicate<String>() {
            // See net.spy.memcached.MemcachedConnection.registerMetrics()
            private final Set<String> interestingMetrics =
                ImmutableSet.of(
                    "[MEM] Reconnecting Nodes (ReconnectQueue)",
                    // "[MEM] Shutting Down Nodes (NodesToShutdown)", // Busted
                    "[MEM] Request Rate: All",
                    "[MEM] Average Bytes written to OS per write",
                    "[MEM] Average Bytes read from OS per read",
                    "[MEM] Average Time on wire for operations (µs)",
                    "[MEM] Response Rate: All (Failure + Success + Retry)",
                    "[MEM] Response Rate: Retry",
                    "[MEM] Response Rate: Failure",
                    "[MEM] Response Rate: Success");

            @Override
            public boolean apply(@Nullable String input) {
              return input != null && interestingMetrics.contains(input);
            }
          };

      final MetricCollector metricCollector =
          new MetricCollector() {
            @Override
            public void addCounter(String name) {
              if (!interesting.apply(name)) {
                return;
              }
              counters.putIfAbsent(name, new AtomicLong(0L));

              if (log.isDebugEnabled()) {
                log.debug("Add Counter [%s]", name);
              }
            }

            @Override
            public void removeCounter(String name) {
              if (log.isDebugEnabled()) {
                log.debug("Ignoring request to remove [%s]", name);
              }
            }

            @Override
            public void incrementCounter(String name) {
              if (!interesting.apply(name)) {
                return;
              }
              AtomicLong counter = counters.get(name);
              if (counter == null) {
                counters.putIfAbsent(name, new AtomicLong(0));
                counter = counters.get(name);
              }
              counter.incrementAndGet();

              if (log.isDebugEnabled()) {
                log.debug("Increment [%s]", name);
              }
            }

            @Override
            public void incrementCounter(String name, int amount) {
              if (!interesting.apply(name)) {
                return;
              }
              AtomicLong counter = counters.get(name);
              if (counter == null) {
                counters.putIfAbsent(name, new AtomicLong(0));
                counter = counters.get(name);
              }
              counter.addAndGet(amount);

              if (log.isDebugEnabled()) {
                log.debug("Increment [%s] %d", name, amount);
              }
            }

            @Override
            public void decrementCounter(String name) {
              if (!interesting.apply(name)) {
                return;
              }
              AtomicLong counter = counters.get(name);
              if (counter == null) {
                counters.putIfAbsent(name, new AtomicLong(0));
                counter = counters.get(name);
              }
              counter.decrementAndGet();

              if (log.isDebugEnabled()) {
                log.debug("Decrement [%s]", name);
              }
            }

            @Override
            public void decrementCounter(String name, int amount) {
              if (!interesting.apply(name)) {
                return;
              }
              AtomicLong counter = counters.get(name);
              if (counter == null) {
                counters.putIfAbsent(name, new AtomicLong(0L));
                counter = counters.get(name);
              }
              counter.addAndGet(-amount);

              if (log.isDebugEnabled()) {
                log.debug("Decrement [%s] %d", name, amount);
              }
            }

            @Override
            public void addMeter(String name) {
              if (!interesting.apply(name)) {
                return;
              }
              meters.putIfAbsent(name, new AtomicLong(0L));
              if (log.isDebugEnabled()) {
                log.debug("Adding meter [%s]", name);
              }
            }

            @Override
            public void removeMeter(String name) {
              if (!interesting.apply(name)) {
                return;
              }
              if (log.isDebugEnabled()) {
                log.debug("Ignoring request to remove meter [%s]", name);
              }
            }

            @Override
            public void markMeter(String name) {
              if (!interesting.apply(name)) {
                return;
              }
              AtomicLong meter = meters.get(name);
              if (meter == null) {
                meters.putIfAbsent(name, new AtomicLong(0L));
                meter = meters.get(name);
              }
              meter.incrementAndGet();

              if (log.isDebugEnabled()) {
                log.debug("Increment counter [%s]", name);
              }
            }

            @Override
            public void addHistogram(String name) {
              log.debug("Ignoring add histogram [%s]", name);
            }

            @Override
            public void removeHistogram(String name) {
              log.debug("Ignoring remove histogram [%s]", name);
            }

            @Override
            public void updateHistogram(String name, int amount) {
              log.debug("Ignoring update histogram [%s]: %d", name, amount);
            }
          };

      final ConnectionFactory connectionFactory =
          new MemcachedCustomConnectionFactoryBuilder()
              // 1000 repetitions gives us good distribution with murmur3_128
              // (approx < 5% difference in counts across nodes, with 5 cache nodes)
              .setKetamaNodeRepetitions(1000)
              .setHashAlg(MURMUR3_128)
              .setProtocol(ConnectionFactoryBuilder.Protocol.BINARY)
              .setLocatorType(ConnectionFactoryBuilder.Locator.CONSISTENT)
              .setDaemon(true)
              .setFailureMode(FailureMode.Cancel)
              .setTranscoder(transcoder)
              .setShouldOptimize(true)
              .setOpQueueMaxBlockTime(config.getTimeout())
              .setOpTimeout(config.getTimeout())
              .setReadBufferSize(config.getReadBufferSize())
              .setOpQueueFactory(opQueueFactory)
              .setMetricCollector(metricCollector)
              .setEnableMetrics(MetricType.DEBUG) // Not as scary as it sounds
              .build();

      final List<InetSocketAddress> hosts = AddrUtil.getAddresses(config.getHosts());

      final Supplier<ResourceHolder<MemcachedClientIF>> clientSupplier;

      if (config.getNumConnections() > 1) {
        clientSupplier =
            new LoadBalancingPool<MemcachedClientIF>(
                config.getNumConnections(),
                new Supplier<MemcachedClientIF>() {
                  @Override
                  public MemcachedClientIF get() {
                    try {
                      return new MemcachedClient(connectionFactory, hosts);
                    } catch (IOException e) {
                      log.error(e, "Unable to create memcached client");
                      throw Throwables.propagate(e);
                    }
                  }
                });
      } else {
        clientSupplier =
            Suppliers.<ResourceHolder<MemcachedClientIF>>ofInstance(
                StupidResourceHolder.<MemcachedClientIF>create(
                    new MemcachedClient(connectionFactory, hosts)));
      }

      return new MemcachedCache(clientSupplier, config, monitor);
    } catch (IOException e) {
      throw Throwables.propagate(e);
    }
  }
コード例 #11
0
  /**
   * 私有构造方法,初始化memcached
   *
   * @throws IOException
   */
  private MemCacheServiceImpl(String prop_file) {

    // ------------装载memchache信息-----------------------
    String server1 = PropertiesReader.getValue(prop_file, "server1");
    String server2 = PropertiesReader.getValue(prop_file, "server2");
    try {
      opTimeout = Integer.parseInt(PropertiesReader.getValue(prop_file, "opTimeout"));
      opTimeoutBulk = Integer.parseInt(PropertiesReader.getValue(prop_file, "opTimeoutBulk"));
      retry = Integer.parseInt(PropertiesReader.getValue(prop_file, "retry"));
      readBufSize = Integer.parseInt(PropertiesReader.getValue(prop_file, "readBufSize"));
      opQueueLen = Integer.parseInt(PropertiesReader.getValue(prop_file, "opQueueLen"));
      expHour = Integer.parseInt(PropertiesReader.getValue(prop_file, "expHour"));
    } catch (Exception e) {
      log.error("loading properties fail, use default config!");
    }
    // 从配置文件中读取相应的配置信息
    try {
      mc1 =
          new MemcachedClient(
              new DefaultConnectionFactory() {

                @Override
                public HashAlgorithm getHashAlg() {
                  return HashAlgorithm.CRC32_HASH;
                }

                @Override
                public long getOperationTimeout() {
                  return opTimeout * 1000;
                }

                @Override
                public int getReadBufSize() {
                  return readBufSize;
                }

                @Override
                public OperationFactory getOperationFactory() {
                  return super.getOperationFactory();
                }

                @Override
                public int getOpQueueLen() {
                  return opQueueLen;
                }

                @Override
                public boolean isDaemon() {
                  return true;
                }
              },
              AddrUtil.getAddresses(server1));

      mc2 =
          new MemcachedClient(
              new DefaultConnectionFactory() {

                @Override
                public HashAlgorithm getHashAlg() {
                  return HashAlgorithm.CRC32_HASH;
                }

                @Override
                public long getOperationTimeout() {
                  return opTimeout * 1000;
                }

                @Override
                public int getReadBufSize() {
                  return readBufSize;
                }

                @Override
                public OperationFactory getOperationFactory() {
                  return super.getOperationFactory();
                }

                @Override
                public int getOpQueueLen() {
                  return opQueueLen;
                }

                @Override
                public boolean isDaemon() {
                  return true;
                }
              },
              AddrUtil.getAddresses(server2));

    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    // 使用gb2312编码
    SerializingTranscoder x1 = (SerializingTranscoder) mc1.getTranscoder();
    x1.setCharset(ENCODING);
    SerializingTranscoder x2 = (SerializingTranscoder) mc2.getTranscoder();
    x2.setCharset(ENCODING);
  }