Exemplo n.º 1
0
  @Test
  public void testNamespace() throws Exception {
    Timing timing = new Timing();
    ChildReaper reaper = null;
    CuratorFramework client =
        CuratorFrameworkFactory.builder()
            .connectString(server.getConnectString())
            .sessionTimeoutMs(timing.session())
            .connectionTimeoutMs(timing.connection())
            .retryPolicy(new RetryOneTime(1))
            .namespace("foo")
            .build();
    try {
      client.start();

      for (int i = 0; i < 10; ++i) {
        client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
      }

      reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
      reaper.start();

      timing.forWaiting().sleepABit();

      Stat stat = client.checkExists().forPath("/test");
      Assert.assertEquals(stat.getNumChildren(), 0);

      stat = client.usingNamespace(null).checkExists().forPath("/foo/test");
      Assert.assertNotNull(stat);
      Assert.assertEquals(stat.getNumChildren(), 0);
    } finally {
      Closeables.closeQuietly(reaper);
      Closeables.closeQuietly(client);
    }
  }
  @Test
  public void testCheckVersion() throws Exception {
    CuratorFramework client =
        CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
    client.start();
    try {
      client.create().forPath("/foo");
      Stat stat = client.setData().forPath("/foo", "new".getBytes()); // up the version

      try {
        client
            .inTransaction()
            .check()
            .withVersion(stat.getVersion() + 1)
            .forPath("/foo") // force a bad version
            .and()
            .create()
            .forPath("/bar")
            .and()
            .commit();

        Assert.fail();
      } catch (KeeperException.BadVersionException correct) {
        // correct
      }

      Assert.assertNull(client.checkExists().forPath("/bar"));
    } finally {
      client.close();
    }
  }
Exemplo n.º 3
0
  @Test
  public void testSomeNodes() throws Exception {

    Timing timing = new Timing();
    ChildReaper reaper = null;
    CuratorFramework client =
        CuratorFrameworkFactory.newClient(
            server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
    try {
      client.start();

      Random r = new Random();
      int nonEmptyNodes = 0;
      for (int i = 0; i < 10; ++i) {
        client.create().creatingParentsIfNeeded().forPath("/test/" + Integer.toString(i));
        if (r.nextBoolean()) {
          client.create().forPath("/test/" + Integer.toString(i) + "/foo");
          ++nonEmptyNodes;
        }
      }

      reaper = new ChildReaper(client, "/test", Reaper.Mode.REAP_UNTIL_DELETE, 1);
      reaper.start();

      timing.forWaiting().sleepABit();

      Stat stat = client.checkExists().forPath("/test");
      Assert.assertEquals(stat.getNumChildren(), nonEmptyNodes);
    } finally {
      Closeables.closeQuietly(reaper);
      Closeables.closeQuietly(client);
    }
  }
  @Test
  public void testBasic() throws Exception {
    CuratorFramework client =
        CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
    client.start();
    try {
      Collection<CuratorTransactionResult> results =
          client
              .inTransaction()
              .create()
              .forPath("/foo")
              .and()
              .create()
              .forPath("/foo/bar", "snafu".getBytes())
              .and()
              .commit();

      Assert.assertTrue(client.checkExists().forPath("/foo/bar") != null);
      Assert.assertEquals(client.getData().forPath("/foo/bar"), "snafu".getBytes());

      CuratorTransactionResult fooResult =
          Iterables.find(
              results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/foo"));
      CuratorTransactionResult fooBarResult =
          Iterables.find(
              results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/foo/bar"));
      Assert.assertNotNull(fooResult);
      Assert.assertNotNull(fooBarResult);
      Assert.assertNotSame(fooResult, fooBarResult);
      Assert.assertEquals(fooResult.getResultPath(), "/foo");
      Assert.assertEquals(fooBarResult.getResultPath(), "/foo/bar");
    } finally {
      client.close();
    }
  }
Exemplo n.º 5
0
 @Before
 public void setup() throws Exception {
   zkServer = new TestingServer(2222);
   zkClient =
       CuratorFrameworkFactory.newClient(
           zkServer.getConnectString(), new ExponentialBackoffRetry(300, 5));
   dragonZK = new DragonZooKeeper(zkClient, "/dragon");
 }
 public static IZkConnection createZkConnection(String connectString) {
   Timing timing = new Timing();
   CuratorFramework client =
       CuratorFrameworkFactory.newClient(
           connectString, timing.session(), timing.connection(), new RetryOneTime(1));
   client.start();
   try {
     return new CuratorZKClientBridge(client);
   } catch (Exception e) {
     throw new RuntimeException(e);
   }
 }
 @Before
 public void setUp() throws Exception {
   server = new TestingServer();
   String connectionString = server.getConnectString();
   Map conf = new HashMap();
   conf.put(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT, 1000);
   conf.put(Config.STORM_ZOOKEEPER_RETRY_TIMES, 4);
   conf.put(Config.STORM_ZOOKEEPER_RETRY_INTERVAL, 5);
   ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
   zookeeper = CuratorFrameworkFactory.newClient(connectionString, retryPolicy);
   dynamicBrokersReader = new DynamicBrokersReader(conf, connectionString, masterPath, topic);
   zookeeper.start();
 }
  @Test
  public void testOverSubscribed() throws Exception {
    final Timing timing = new Timing();
    final CuratorFramework client =
        CuratorFrameworkFactory.newClient(
            server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
    ExecutorService service = Executors.newCachedThreadPool();
    ExecutorCompletionService<Void> completionService =
        new ExecutorCompletionService<Void>(service);
    try {
      client.start();

      final Semaphore semaphore = new Semaphore(0);
      final CountDownLatch latch = new CountDownLatch(1);
      for (int i = 0; i < (QTY + 1); ++i) {
        completionService.submit(
            new Callable<Void>() {
              @Override
              public Void call() throws Exception {
                DistributedDoubleBarrier barrier =
                    new DistributedDoubleBarrier(client, "/barrier", QTY) {
                      @Override
                      protected List<String> getChildrenForEntering() throws Exception {
                        semaphore.release();
                        Assert.assertTrue(timing.awaitLatch(latch));
                        return super.getChildrenForEntering();
                      }
                    };
                Assert.assertTrue(barrier.enter(timing.seconds(), TimeUnit.SECONDS));
                Assert.assertTrue(barrier.leave(timing.seconds(), TimeUnit.SECONDS));
                return null;
              }
            });
      }

      Assert.assertTrue(
          semaphore.tryAcquire(
              QTY + 1,
              timing.seconds(),
              TimeUnit.SECONDS)); // wait until all QTY+1 barriers are trying to enter
      latch.countDown();

      for (int i = 0; i < (QTY + 1); ++i) {
        completionService.take().get(); // to check for assertions
      }
    } finally {
      service.shutdown();
      Closeables.close(client, true);
    }
  }
 public KafkaTestBroker() {
   try {
     server = new TestingServer();
     String zookeeperConnectionString = server.getConnectString();
     ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
     zookeeper = CuratorFrameworkFactory.newClient(zookeeperConnectionString, retryPolicy);
     zookeeper.start();
     port = InstanceSpec.getRandomPort();
     kafka.server.KafkaConfig config = buildKafkaConfig(zookeeperConnectionString);
     kafka = new KafkaServerStartable(config);
     kafka.startup();
   } catch (Exception ex) {
     throw new RuntimeException("Could not start test broker", ex);
   }
 }
  /**
   * Create and configure the ZK sync object with the KinesisSpoutConfig.
   *
   * @param config the configuration for the spout.
   */
  ZookeeperShardState(final KinesisSpoutConfig config) {
    this.config = config;
    this.rand = new Random();

    try {
      zk =
          CuratorFrameworkFactory.newClient(
              config.getZookeeperConnectionString(),
              new ExponentialBackoffRetry(BASE_SLEEP_TIME_MS, MAX_NUM_RETRIES));
    } catch (IOException e) {
      LOG.error("Could not connect to ZooKeeper", e);
      throw new KinesisSpoutException(e);
    }
    zk.start();
  }
  @Test
  public void testWithNamespace() throws Exception {
    CuratorFramework client =
        CuratorFrameworkFactory.builder()
            .connectString(server.getConnectString())
            .retryPolicy(new RetryOneTime(1))
            .namespace("galt")
            .build();
    client.start();
    try {
      Collection<CuratorTransactionResult> results =
          client
              .inTransaction()
              .create()
              .forPath("/foo", "one".getBytes())
              .and()
              .create()
              .withMode(CreateMode.PERSISTENT_SEQUENTIAL)
              .forPath("/test-", "one".getBytes())
              .and()
              .setData()
              .forPath("/foo", "two".getBytes())
              .and()
              .create()
              .forPath("/foo/bar")
              .and()
              .delete()
              .forPath("/foo/bar")
              .and()
              .commit();

      Assert.assertTrue(client.checkExists().forPath("/foo") != null);
      Assert.assertTrue(client.nonNamespaceView().checkExists().forPath("/galt/foo") != null);
      Assert.assertEquals(client.getData().forPath("/foo"), "two".getBytes());
      Assert.assertTrue(client.checkExists().forPath("/foo/bar") == null);

      CuratorTransactionResult ephemeralResult =
          Iterables.find(
              results, CuratorTransactionResult.ofTypeAndPath(OperationType.CREATE, "/test-"));
      Assert.assertNotNull(ephemeralResult);
      Assert.assertNotEquals(ephemeralResult.getResultPath(), "/test-");
      Assert.assertTrue(ephemeralResult.getResultPath().startsWith("/test-"));
    } finally {
      client.close();
    }
  }
Exemplo n.º 12
0
  private void initZooKeeper() {
    getLogger().info("Init ZooKeeper");
    this.zooKeeperClient =
        CuratorFrameworkFactory.builder()
            .connectString("elab.kr:2181")
            .retryPolicy(new ExponentialBackoffRetry(1000, 3))
            .aclProvider(new ServiceACLProvider())
            .sessionTimeoutMs(10 * 1000)
            .build();
    this.zooKeeperClient.start();
    try {
      this.zooKeeperClient.getZookeeperClient().blockUntilConnectedOrTimedOut();
    } catch (InterruptedException e) {
      getLogger().fatal("Not Connected ZooKeeper", e);
    }

    this.registerService();
  }
Exemplo n.º 13
0
  @Test
  public void testInCluster() throws Exception {
    final int PARTICIPANT_QTY = 3;

    List<ClientAndLatch> clients = Lists.newArrayList();
    Timing timing = new Timing();
    TestingCluster cluster = new TestingCluster(PARTICIPANT_QTY);
    try {
      cluster.start();

      List<InstanceSpec> instances = Lists.newArrayList(cluster.getInstances());
      for (int i = 0; i < PARTICIPANT_QTY; ++i) {
        CuratorFramework client =
            CuratorFrameworkFactory.newClient(
                instances.get(i).getConnectString(),
                timing.session(),
                timing.connection(),
                new RetryOneTime(1));
        LeaderLatch latch = new LeaderLatch(client, "/latch");

        clients.add(new ClientAndLatch(client, latch, i));
        client.start();
        latch.start();
      }

      ClientAndLatch leader = waitForALeader(clients, timing);
      Assert.assertNotNull(leader);

      cluster.killServer(instances.get(leader.index));

      Thread.sleep(timing.multiple(2).session());

      leader = waitForALeader(clients, timing);
      Assert.assertNotNull(leader);

      Assert.assertEquals(getLeaders(clients).size(), 1);
    } finally {
      for (ClientAndLatch client : clients) {
        Closeables.close(client.latch, true);
        Closeables.close(client.client, true);
      }
      Closeables.close(cluster, true);
    }
  }
  /** Amoeba启动的时候,初始化zookeeper客户端,另外,给zookeeper预热 */
  public void init() throws Exception {

    Properties props = Utils.readGlobalSeqConfigProps();
    String zkHosts = props.getProperty("zkHosts", "127.0.0.1:2181");
    Integer connTimeOut = Integer.valueOf(props.getProperty("zkConnTimeOut", "60"));

    log.info(String.format("zookeeper config: connect string= %s", zkHosts));
    log.info(String.format("zookeeper config: connect timeout= %d seconds", connTimeOut));

    client =
        CuratorFrameworkFactory.builder()
            .connectString(zkHosts)
            .retryPolicy(new ExponentialBackoffRetry(BASE_SLEEP_TIME, RETRY_TIMES))
            .connectionTimeoutMs(connTimeOut * 1000)
            .build(); // 时间要由秒转成毫秒

    client.start();

    // 只是为client热身而调用
    client.checkExists().forPath(ROOT_PATH);
  }
Exemplo n.º 15
0
 public static CuratorFramework newCurator(
     Map conf, List<String> servers, Object port, String root) {
   List<String> serverPorts = new ArrayList<String>();
   for (String zkServer : (List<String>) servers) {
     serverPorts.add(zkServer + ":" + Utils.getInt(port));
   }
   String zkStr = StringUtils.join(serverPorts, ",") + root;
   try {
     CuratorFramework ret =
         CuratorFrameworkFactory.newClient(
             zkStr,
             Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
             15000,
             new RetryNTimes(
                 Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
                 Utils.getInt(conf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
     return ret;
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
Exemplo n.º 16
0
  @SuppressWarnings("unchecked")
  private CuratorFramework newCurator(Map stateConf) {
    Integer port = (Integer) stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_PORT);
    String zkServerPorts = "";
    for (String server : (List<String>) stateConf.get(Config.TRANSACTIONAL_ZOOKEEPER_SERVERS)) {
      zkServerPorts += server + ":" + port + ",";
    }

    // LOG.info("@@@@@@@@@@@@@@@" + zkServerPorts);

    try {
      return CuratorFrameworkFactory.newClient(
          zkServerPorts,
          Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_SESSION_TIMEOUT)),
          20000,
          new RetryNTimes(
              Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_TIMES)),
              Utils.getInt(stateConf.get(Config.STORM_ZOOKEEPER_RETRY_INTERVAL))));
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
  @Test
  public void testBasic() throws Exception {
    final Timing timing = new Timing();
    final List<Closeable> closeables = Lists.newArrayList();
    final CuratorFramework client =
        CuratorFrameworkFactory.newClient(
            server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1));
    try {
      closeables.add(client);
      client.start();

      final CountDownLatch postEnterLatch = new CountDownLatch(QTY);
      final CountDownLatch postLeaveLatch = new CountDownLatch(QTY);
      final AtomicInteger count = new AtomicInteger(0);
      final AtomicInteger max = new AtomicInteger(0);
      List<Future<Void>> futures = Lists.newArrayList();
      ExecutorService service = Executors.newCachedThreadPool();
      for (int i = 0; i < QTY; ++i) {
        Future<Void> future =
            service.submit(
                new Callable<Void>() {
                  @Override
                  public Void call() throws Exception {
                    DistributedDoubleBarrier barrier =
                        new DistributedDoubleBarrier(client, "/barrier", QTY);

                    Assert.assertTrue(barrier.enter(timing.seconds(), TimeUnit.SECONDS));

                    synchronized (TestDistributedDoubleBarrier.this) {
                      int thisCount = count.incrementAndGet();
                      if (thisCount > max.get()) {
                        max.set(thisCount);
                      }
                    }

                    postEnterLatch.countDown();
                    Assert.assertTrue(timing.awaitLatch(postEnterLatch));

                    Assert.assertEquals(count.get(), QTY);

                    Assert.assertTrue(barrier.leave(10, TimeUnit.SECONDS));
                    count.decrementAndGet();

                    postLeaveLatch.countDown();
                    Assert.assertTrue(timing.awaitLatch(postLeaveLatch));

                    return null;
                  }
                });
        futures.add(future);
      }

      for (Future<Void> f : futures) {
        f.get();
      }
      Assert.assertEquals(count.get(), 0);
      Assert.assertEquals(max.get(), QTY);
    } finally {
      for (Closeable c : closeables) {
        Closeables.close(c, true);
      }
    }
  }
  private void buildZKClient(final String zkFullPath) {
    final String[] zkFullPathItems = zkFullPath.split(ZK_FULL_PATH_DELIMITER);
    zkHost_ = zkFullPathItems[1];
    zkNamespace_ =
        zkFullPathItems[2].substring(
            0, zkFullPathItems[2].lastIndexOf(ZK_FULL_PATH_NAMESPACE_SEPERATOR));
    zkNodePath_ =
        zkFullPathItems[2].substring(
            zkFullPathItems[2].lastIndexOf(ZK_FULL_PATH_NAMESPACE_SEPERATOR));
    if (zkClient_ == null) {
      try {
        zkClient_ =
            CuratorFrameworkFactory.builder()
                .connectString(zkHost_)
                .namespace(zkNamespace_)
                .retryPolicy(new ExponentialBackoffRetry(1000, 60))
                .connectionTimeoutMs(2 * 3600 * 1000)
                .build();
        zkClient_
            .getConnectionStateListenable()
            .addListener(
                new ConnectionStateListener() {

                  @Override
                  public void stateChanged(CuratorFramework client, ConnectionState newState) {
                    // TODO Auto-generated method stub
                    if (newState == ConnectionState.LOST) {
                      try {
                        synchronized (this) {
                          while (!client.getZookeeperClient().blockUntilConnectedOrTimedOut()) {
                            if (logger_ != null) {
                              if (logger_.isInfoEnabled()) logger_.info("waitting reconnected...");
                            }
                            wait(1000);
                          }
                        }
                        if (logger_ != null) {
                          if (logger_.isInfoEnabled()) logger_.info("to zookeeper reconnected.");
                        }
                        doNodesDiscovery();
                      } catch (Exception e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                        if (logger_ != null) {
                          if (logger_.isInfoEnabled())
                            logger_.info(
                                String.format(
                                    "To re connect to zookeeper to generate an error, cause:%s",
                                    e));
                        }
                      }
                    }
                  }
                });
        zkClient_.start();
      } catch (Throwable t) {
        zkClient_.close();
        throw new IllegalStateException("building zookeeper client failed.");
      }
    }
  }