@Bean(destroyMethod = "close")
  @ConditionalOnMissingBean
  @SneakyThrows
  public CuratorFramework curatorFramework(
      RetryPolicy retryPolicy, ZookeeperDiscoveryProperties properties) {
    CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
    if (ensembleProvider != null) {
      builder.ensembleProvider(ensembleProvider);
    }
    CuratorFramework curator =
        builder
            .retryPolicy(retryPolicy)
            .connectString(zookeeperProperties.getConnectString())
            .build();
    curator.start();

    log.trace(
        "blocking until connected to zookeeper for "
            + properties.getBlockUntilConnectedWait()
            + properties.getBlockUntilConnectedUnit());
    curator.blockUntilConnected(
        properties.getBlockUntilConnectedWait(), properties.getBlockUntilConnectedUnit());
    log.trace("connected to zookeeper");
    return curator;
  }
  public static void main(String[] args) throws Exception {
    CuratorFramework client = null;
    DistributedDelayQueue<String> queue = null;

    try {
      client = CuratorClientFactory.newClient();
      client
          .getCuratorListenable()
          .addListener(
              new CuratorListener() {
                @Override
                public void eventReceived(CuratorFramework client, CuratorEvent event)
                    throws Exception {
                  System.out.println("CuratorEvent: " + event.getType().name());
                }
              });
      client.start();

      QueueConsumer<String> consumer = createQueueConsumer();
      QueueBuilder<String> builder =
          QueueBuilder.builder(client, consumer, createQueueSerializer(), PATH);
      queue = builder.buildDelayQueue();
      queue.start();

      for (int i = 0; i < 10; i++) {
        queue.put("test-" + i, System.currentTimeMillis() + 10000);
      }
      System.out.println(getCurrentTimeStr() + ": 所有的数据已经生产完毕");

      Thread.sleep(20000);
    } finally {
      CloseableUtils.closeQuietly(client);
      CloseableUtils.closeQuietly(queue);
    }
  }
  public void addServer(
      String zookeeperInstance,
      String joiningServerIp,
      String joiningServerZookeeperPort,
      Integer myId,
      FutureCallback callback) {
    AsyncCallback.DataCallback dataCallback = getDataCallback(callback);

    try (CuratorFramework zkClient = connectToZookeeper(zookeeperInstance)) {
      List<String> joiningServers = new ArrayList<>();
      String serverStr =
          "server." + myId + "=" + joiningServerIp + ":2888:3888;" + joiningServerZookeeperPort;
      logger.info("Adding server: " + serverStr);
      joiningServers.add(serverStr);
      LinkedList<Integer> results = new LinkedList<Integer>();
      zkClient
          .getZookeeperClient()
          .getZooKeeper()
          .reconfig(joiningServers, null, null, -1, dataCallback, results);

    } catch (Exception e) {
      logger.error("Rethrowing error ", e);
      throw new RuntimeException(e);
    }
  }
Esempio n. 4
0
 protected String getFirstService(String containerPath) throws Exception {
   CuratorFramework curatorFramework = curator.get();
   if (curatorFramework != null) {
     byte[] data = curatorFramework.getData().forPath(containerPath);
     if (data != null && data.length > 0) {
       String text = new String(data).trim();
       if (!text.isEmpty()) {
         ObjectMapper mapper = new ObjectMapper();
         Map<String, Object> map = mapper.readValue(data, HashMap.class);
         Object serviceValue = map.get("services");
         if (serviceValue instanceof List) {
           List services = (List) serviceValue;
           if (services != null) {
             if (!services.isEmpty()) {
               List<String> serviceTexts = new ArrayList<String>();
               for (Object service : services) {
                 String serviceText = getSubstitutedData(curatorFramework, service.toString());
                 if (io.fabric8.common.util.Strings.isNotBlank(serviceText)) {
                   return serviceText;
                 }
               }
             }
           }
         }
       }
     }
   }
   return null;
 }
 /**
  * @Title: registerListeners @TitleExplain: 向zk注册监听 @Description: 注册监听
  *
  * @param client
  * @author wudan-mac
  */
 private void registerListeners(CuratorFramework client) {
   /*
    * 注册监听者
    */
   client
       .getConnectionStateListenable()
       .addListener(
           new ConnectionStateListener() {
             @Override
             public void stateChanged(
                 CuratorFramework client, ConnectionState newState) { // 当状态发生变化时通知监听者
               logger.debug("CuratorFramework state changed: {}", newState);
               if (newState == ConnectionState.CONNECTED
                   || newState == ConnectionState.RECONNECTED) {
                 for (IZKListener listener : listeners) {
                   listener.executor(client);
                   logger.debug("Listener {} executed!", listener.getClass().getName());
                 }
               }
             }
           });
   /*
    *异常处理
    */
   client
       .getUnhandledErrorListenable()
       .addListener(
           new UnhandledErrorListener() {
             @Override
             public void unhandledError(String message, Throwable e) {
               logger.debug("CuratorFramework unhandledError: {}", message);
             }
           });
 }
Esempio n. 6
0
  /**
   * 连接
   *
   * @return
   */
  public boolean connect() {
    try {
      client
          .getConnectionStateListenable()
          .addListener(
              new ConnectionStateListener() {

                public void stateChanged(CuratorFramework framework, ConnectionState state) {
                  logger.info("Zookeeper状态:" + state.name());
                  if (state.equals(ConnectionState.RECONNECTED)) {
                    logger.info("Zookeeper状态:检查节点");
                    for (String key : backup.keySet()) {
                      String value = backup.get(key);
                      createNode(key, value);
                    }
                  }
                }
              });
      client.start();
      client.getZookeeperClient().blockUntilConnectedOrTimedOut();
      createRoomProcessorNode();
      return true;
    } catch (Exception e) {
      logger.error("ZooKeeperWrapper.connect出现异常", e);
      return false;
    }
  }
  @Test
  public void testBasic() throws Exception {
    CuratorFramework client =
        CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
    try {
      client.start();

      final DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
      barrier.setBarrier();

      ExecutorService service = Executors.newSingleThreadExecutor();
      service.submit(
          new Callable<Object>() {
            @Override
            public Object call() throws Exception {
              Thread.sleep(1000);
              barrier.removeBarrier();
              return null;
            }
          });

      Assert.assertTrue(barrier.waitOnBarrier(10, TimeUnit.SECONDS));
    } finally {
      client.close();
    }
  }
  @Test
  public void testNoBarrier() throws Exception {
    CuratorFramework client =
        CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
    try {
      client.start();

      final DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
      Assert.assertTrue(barrier.waitOnBarrier(10, TimeUnit.SECONDS));

      // just for grins, test the infinite wait
      ExecutorService service = Executors.newSingleThreadExecutor();
      Future<Object> future =
          service.submit(
              new Callable<Object>() {
                @Override
                public Object call() throws Exception {
                  barrier.waitOnBarrier();
                  return "";
                }
              });
      Assert.assertTrue(future.get(10, TimeUnit.SECONDS) != null);
    } finally {
      client.close();
    }
  }
  @Override
  public <S extends StreamDefinition> S save(S entity) {
    try {
      Map<String, String> map = new HashMap<>();
      map.put(DEFINITION_KEY, entity.getDefinition());

      map.put(
          MODULE_DEFINITIONS_KEY, objectWriter.writeValueAsString(entity.getModuleDefinitions()));

      CuratorFramework client = zkConnection.getClient();
      String path = Paths.build(Paths.STREAMS, entity.getName());
      byte[] binary = ZooKeeperUtils.mapToBytes(map);

      BackgroundPathAndBytesable<?> op =
          client.checkExists().forPath(path) == null ? client.create() : client.setData();

      op.forPath(path, binary);

      logger.trace("Saved stream {} with properties {}", path, map);

      StreamDefinitionRepositoryUtils.saveDependencies(moduleDependencyRepository, entity);
    } catch (Exception e) {
      // NodeExistsException indicates that we tried to create the
      // path just after another thread/jvm successfully created it
      ZooKeeperUtils.wrapAndThrowIgnoring(e, NodeExistsException.class);
    }
    return entity;
  }
  public static long getOffsetFromZooKeeper(
      CuratorFramework curatorClient, String groupId, String topic, int partition)
      throws Exception {
    ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
    String path = topicDirs.consumerOffsetDir() + "/" + partition;
    curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());

    byte[] data = curatorClient.getData().forPath(path);

    if (data == null) {
      return OFFSET_NOT_SET;
    } else {
      String asString = new String(data);
      if (asString.length() == 0) {
        return OFFSET_NOT_SET;
      } else {
        try {
          return Long.parseLong(asString);
        } catch (NumberFormatException e) {
          throw new Exception(
              String.format(
                  "The offset in ZooKeeper for group '%s', topic '%s', partition %d is a malformed string: %s",
                  groupId, topic, partition, asString));
        }
      }
    }
  }
 public RunningJobManager(
     String zkQuorum,
     int zkSessionTimeoutMs,
     int zkRetryTimes,
     int zkRetryInterval,
     String zkRoot,
     String lockPath) {
   this.zkRoot = zkRoot;
   curator = newCurator(zkQuorum, zkSessionTimeoutMs, zkRetryTimes, zkRetryInterval);
   try {
     curator.start();
   } catch (Exception e) {
     LOG.error("curator start error {}", e);
   }
   LOG.info("InterProcessMutex lock path is " + lockPath);
   lock = new InterProcessMutex(curator, lockPath);
   try {
     if (curator.checkExists().forPath(this.zkRoot) == null) {
       curator
           .create()
           .creatingParentsIfNeeded()
           .withMode(CreateMode.PERSISTENT)
           .forPath(this.zkRoot);
     }
   } catch (Exception e) {
     LOG.warn("{}", e);
   }
 }
Esempio n. 12
0
  public Map<String, Map<String, ClientLeaseInfo>> loadAndWatchTopicExistingLeases(String topic)
      throws Exception {
    Map<String, Map<String, ClientLeaseInfo>> topicExistingLeases = new HashMap<>();

    CuratorFramework client = m_zkClient.get();
    String topicPath = ZKPaths.makePath(ZKPathUtils.getBrokerLeaseRootZkPath(), topic);

    client.getData().usingWatcher(m_brokerLeaseChangedWatcher).forPath(topicPath);
    m_brokerLeaseChangedWatcher.addWatchedPath(topicPath);
    addWatchedTopic(topic);

    List<String> partitions = client.getChildren().forPath(topicPath);

    if (partitions != null && !partitions.isEmpty()) {
      for (String partition : partitions) {
        String partitionPath = ZKPaths.makePath(topicPath, partition);
        byte[] data = client.getData().forPath(partitionPath);
        Map<String, ClientLeaseInfo> existingLeases = deserializeExistingLeases(data);
        if (existingLeases != null) {
          topicExistingLeases.put(partitionPath, existingLeases);
        }
      }
    }

    return topicExistingLeases;
  }
  public ZKNamedMutexFactory(
      String connectString, int connectionTimeout, int sessionTimeout, String namespace)
      throws GlobalMutexAcquireException {

    this.connectString = connectString;
    this.namespace = namespace;

    try {

      curatorClient =
          CuratorFrameworkFactory.builder()
              .connectString(connectString)
              .retryPolicy(retryPolicy)
              .connectionTimeoutMs(connectionTimeout)
              .sessionTimeoutMs(sessionTimeout)
              .namespace(namespace)
              .build();

      curatorClient.start();

      if (curatorClient.checkExists().forPath(BASEDIR) == null) {
        curatorClient.create().forPath(BASEDIR);
      }

    } catch (Exception e) {
      throw new GlobalMutexAcquireException("Unable to : ", e);
    }
  }
Esempio n. 14
0
  @Test(timeout = 60000)
  public void testInvalidOffset() throws Exception {
    final String topic = "invalidOffsetTopic";
    final int parallelism = 1;

    // create topic
    createTestTopic(topic, parallelism, 1);

    final StreamExecutionEnvironment env =
        StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);

    // write 20 messages into topic:
    writeSequence(env, topic, 20, parallelism);

    // set invalid offset:
    CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
    ZookeeperOffsetHandler.setOffsetInZooKeeper(
        curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234);
    curatorClient.close();

    // read from topic
    final int valuesCount = 20;
    final int startFrom = 0;
    readSequence(env, standardProps, parallelism, topic, valuesCount, startFrom);

    deleteTestTopic(topic);
  }
Esempio n. 15
0
  public static void main(String[] args) throws Exception {

    // RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
    // RetryPolicy retryPolicy = new RetryNTimes(5, 1000);
    RetryPolicy retryPolicy = new RetryUntilElapsed(5000, 1000);
    //		CuratorFramework client = CuratorFrameworkFactory
    //				.newClient("192.168.1.105:2181",5000,5000, retryPolicy);

    CuratorFramework client =
        CuratorFrameworkFactory.builder()
            .connectString("192.168.1.105:2181")
            .sessionTimeoutMs(5000)
            .connectionTimeoutMs(5000)
            .retryPolicy(retryPolicy)
            .build();

    client.start();

    String path =
        client
            .create()
            .creatingParentsIfNeeded()
            .withMode(CreateMode.EPHEMERAL)
            .forPath("/jike/1", "123".getBytes());

    System.out.println(path);

    Thread.sleep(Integer.MAX_VALUE);
  }
Esempio n. 16
0
    @Override
    public void eventReceived(CuratorFramework client, CuratorEvent event) throws Exception {
      if (event.getType() == CuratorEventType.CHILDREN) {
        if (event.getChildren().isEmpty()) {
          client.getChildren().inBackground(event.getContext()).forPath(event.getPath());
        } else {
          String path = event.getPath() + "/" + event.getChildren().get(0);
          LOG.info("Operations Node registered in ZK. Waiting for transports configration");
          client.getData().inBackground(event.getContext()).forPath(path);
        }
      } else if (event.getType() == CuratorEventType.GET_DATA) {
        if (event.getData() == null) {
          client.getData().inBackground(event.getContext()).forPath(event.getPath());
        } else {
          OperationsNodeInfo nodeInfo =
              OPERATIONS_NODE_INFO_CONVERTER.fromByteArray(event.getData());
          boolean isTransportInitialized = !nodeInfo.getTransports().isEmpty();

          if (isTransportInitialized) {
            LOG.info("Operations Node updated tarnsports configuration in ZK");
            ((CountDownLatch) event.getContext()).countDown();
          } else {
            client.getData().inBackground(event.getContext()).forPath(event.getPath());
          }
        }
      }
    }
  @Test(expected = RuntimeException.class)
  public void assertExecuteInTransactionFailure() throws Exception {
    CuratorFramework client = mock(CuratorFramework.class);
    CuratorTransaction curatorTransaction = mock(CuratorTransaction.class);
    TransactionCheckBuilder transactionCheckBuilder = mock(TransactionCheckBuilder.class);
    CuratorTransactionBridge curatorTransactionBridge = mock(CuratorTransactionBridge.class);
    CuratorTransactionFinal curatorTransactionFinal = mock(CuratorTransactionFinal.class);
    when(coordinatorRegistryCenter.getRawClient()).thenReturn(client);
    when(client.inTransaction()).thenReturn(curatorTransaction);
    when(curatorTransaction.check()).thenReturn(transactionCheckBuilder);
    when(transactionCheckBuilder.forPath("/")).thenReturn(curatorTransactionBridge);
    when(curatorTransactionBridge.and()).thenReturn(curatorTransactionFinal);
    TransactionCreateBuilder transactionCreateBuilder = mock(TransactionCreateBuilder.class);
    when(curatorTransactionFinal.create()).thenReturn(transactionCreateBuilder);
    when(transactionCreateBuilder.forPath("/test_transaction"))
        .thenReturn(curatorTransactionBridge);
    when(curatorTransactionBridge.and()).thenThrow(new RuntimeException());
    jobNodeStorage.executeInTransaction(
        new TransactionExecutionCallback() {

          @Override
          public void execute(final CuratorTransactionFinal curatorTransactionFinal)
              throws Exception {
            curatorTransactionFinal.create().forPath("/test_transaction").and();
          }
        });
    verify(coordinatorRegistryCenter).getRawClient();
    verify(client).inTransaction();
    verify(curatorTransaction).check();
    verify(transactionCheckBuilder).forPath("/");
    verify(curatorTransactionBridge, times(2)).and();
    verify(curatorTransactionFinal).create();
    verify(transactionCreateBuilder).forPath("/test_transaction");
    verify(curatorTransactionFinal, times(0)).commit();
  }
Esempio n. 18
0
  @Test
  public void runOffsetManipulationInZooKeeperTest() {
    try {
      final String topicName = "ZookeeperOffsetHandlerTest-Topic";
      final String groupId = "ZookeeperOffsetHandlerTest-Group";

      final Long offset = (long) (Math.random() * Long.MAX_VALUE);

      CuratorFramework curatorFramework =
          ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
      kafkaServer.createTestTopic(topicName, 3, 2);

      ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorFramework, groupId, topicName, 0, offset);

      Long fetchedOffset =
          ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, groupId, topicName, 0);

      curatorFramework.close();

      assertEquals(offset, fetchedOffset);
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
Esempio n. 19
0
  /**
   * connect ZK, register Watch/unhandle Watch
   *
   * @return
   */
  public CuratorFramework mkClient(
      Map conf, List<String> servers, Object port, String root, final WatcherCallBack watcher) {

    CuratorFramework fk = Utils.newCurator(conf, servers, port, root);

    fk.getCuratorListenable()
        .addListener(
            new CuratorListener() {
              @Override
              public void eventReceived(CuratorFramework _fk, CuratorEvent e) throws Exception {
                if (e.getType().equals(CuratorEventType.WATCHED)) {
                  WatchedEvent event = e.getWatchedEvent();

                  watcher.execute(event.getState(), event.getType(), event.getPath());
                }
              }
            });

    fk.getUnhandledErrorListenable()
        .addListener(
            new UnhandledErrorListener() {
              @Override
              public void unhandledError(String msg, Throwable error) {
                String errmsg = "Unrecoverable Zookeeper error, halting process: " + msg;
                LOG.error(errmsg, error);
                JStormUtils.halt_process(1, "Unrecoverable Zookeeper error");
              }
            });
    fk.start();
    return fk;
  }
  @Test
  public void test() throws Exception {
    Timing timing = new Timing();
    LeaderSelector leaderSelector = null;
    CuratorFramework client =
        CuratorFrameworkFactory.builder()
            .retryPolicy(new ExponentialBackoffRetry(100, 3))
            .connectString(server.getConnectString())
            .sessionTimeoutMs(timing.session())
            .connectionTimeoutMs(timing.connection())
            .build();
    try {
      client.start();

      MyLeaderSelectorListener listener = new MyLeaderSelectorListener();
      ExecutorService executorPool = Executors.newFixedThreadPool(20);
      leaderSelector = new LeaderSelector(client, "/test", threadFactory, executorPool, listener);

      leaderSelector.autoRequeue();
      leaderSelector.start();

      timing.sleepABit();

      Assert.assertEquals(listener.getLeaderCount(), 1);
    } finally {
      CloseableUtils.closeQuietly(leaderSelector);
      CloseableUtils.closeQuietly(client);
    }
  }
  public Map<String, Set<String>> getServices(String bizCode) {
    try {
      List<String> children = client.getChildren().forPath("/" + bizCode);
      System.out.println("-----------services----------");
      for (String bizCh : children) {
        String servicepath = bizCh;
        addChildWatcher("/" + bizCode + "/" + bizCh);
        // System.out.println("-------bizCh-----------"+bizCh);
        if (servicepath.endsWith(".do")) {
          servicepath = servicepath.replace(".", "/");
          servicepath = servicepath.replace("/do", ".do");
          if (!servicepath.startsWith("/")) servicepath = "/" + servicepath;
        } else {
          servicepath = servicepath.replace(".", "/");
        }
        if (!services.containsKey(servicepath)) services.put(servicepath, new HashSet<String>());
        // System.out.println("------------------"+servicepath);

        List<String> providers = client.getChildren().forPath(bizCode + "/" + bizCh);
        for (String p : providers) {
          // System.out.println("------------------------"+p);
          services.get(servicepath).add(p);
          if (!servicesByP.containsKey(p)) {
            servicesByP.put(p, new HashSet<String>());
          }
          servicesByP.get(p).add(servicepath);
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
    }

    printService();
    return services;
  }
Esempio n. 22
0
  @Before
  public void setUp() throws Exception {
    sfb = new ZKServerFactoryBean();
    delete(sfb.getDataDir());
    delete(sfb.getDataLogDir());
    sfb.afterPropertiesSet();

    CuratorFrameworkFactory.Builder builder =
        CuratorFrameworkFactory.builder()
            .connectString("localhost:" + sfb.getClientPortAddress().getPort())
            .retryPolicy(new RetryOneTime(1000))
            .connectionTimeoutMs(360000);

    curator = builder.build();
    curator.start();
    curator.getZookeeperClient().blockUntilConnectedOrTimedOut();

    // setup a local and remote git repo
    basedir = System.getProperty("basedir", ".");
    File root = new File(basedir + "/target/git").getCanonicalFile();
    delete(root);

    new File(root, "remote").mkdirs();
    remote = Git.init().setDirectory(new File(root, "remote")).call();
    remote.commit().setMessage("First Commit").setCommitter("fabric", "user@fabric").call();
    String remoteUrl = "file://" + new File(root, "remote").getCanonicalPath();

    new File(root, "local").mkdirs();
    git = Git.init().setDirectory(new File(root, "local")).call();
    git.commit().setMessage("First Commit").setCommitter("fabric", "user@fabric").call();
    StoredConfig config = git.getRepository().getConfig();
    config.setString("remote", "origin", "url", remoteUrl);
    config.setString("remote", "origin", "fetch", "+refs/heads/*:refs/remotes/origin/*");
    config.save();

    DefaultRuntimeProperties sysprops = new DefaultRuntimeProperties();
    sysprops.setProperty(SystemProperties.KARAF_DATA, "target/data");
    FabricGitServiceImpl gitService = new FabricGitServiceImpl();
    gitService.bindRuntimeProperties(sysprops);
    gitService.activate();
    gitService.setGitForTesting(git);

    DataStoreTemplateRegistry registrationHandler = new DataStoreTemplateRegistry();
    registrationHandler.activateComponent();

    dataStore = new CachingGitDataStore();
    dataStore.bindCurator(curator);
    dataStore.bindGitService(gitService);
    dataStore.bindRegistrationHandler(registrationHandler);
    dataStore.bindRuntimeProperties(sysprops);
    dataStore.bindConfigurer(
        new Configurer() {
          @Override
          public <T> void configure(Map<String, ?> configuration, T target) throws Exception {}
        });
    Map<String, String> datastoreProperties = new HashMap<String, String>();
    datastoreProperties.put(GitDataStore.GIT_REMOTE_URL, remoteUrl);
    dataStore.activate(datastoreProperties);
  }
Esempio n. 23
0
 public void pub(String nameSpace, String path, String value) throws Exception {
   Assert.notNull(path, "path不能为空!");
   Assert.hasText(nameSpace, "nameSpace不能为空");
   if (curatorFramework.usingNamespace(nameSpace).checkExists().forPath(path) != null) {
     curatorFramework.usingNamespace(nameSpace).delete().forPath(path);
   }
   invokeCreate(nameSpace, path, value, CreateMode.EPHEMERAL);
 }
 /** Only for the 0.8 server we need access to the zk client. */
 public CuratorFramework createCuratorClient() {
   RetryPolicy retryPolicy = new ExponentialBackoffRetry(100, 10);
   CuratorFramework curatorClient =
       CuratorFrameworkFactory.newClient(
           standardProps.getProperty("zookeeper.connect"), retryPolicy);
   curatorClient.start();
   return curatorClient;
 }
  @Test
  public void testMultiClient() throws Exception {
    CuratorFramework client1 = null;
    CuratorFramework client2 = null;
    try {
      {
        CuratorFramework client =
            CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
        try {
          client.start();
          DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
          barrier.setBarrier();
        } finally {
          CloseableUtils.closeQuietly(client);
        }
      }

      client1 = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
      client2 = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));

      List<Future<Object>> futures = Lists.newArrayList();
      ExecutorService service = Executors.newCachedThreadPool();
      for (final CuratorFramework c : new CuratorFramework[] {client1, client2}) {
        Future<Object> future =
            service.submit(
                new Callable<Object>() {
                  @Override
                  public Object call() throws Exception {
                    c.start();
                    DistributedBarrier barrier = new DistributedBarrier(c, "/barrier");
                    barrier.waitOnBarrier(10, TimeUnit.MILLISECONDS);
                    return null;
                  }
                });
        futures.add(future);
      }

      Thread.sleep(1000);
      {
        CuratorFramework client =
            CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
        try {
          client.start();
          DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
          barrier.removeBarrier();
        } finally {
          CloseableUtils.closeQuietly(client);
        }
      }

      for (Future<Object> f : futures) {
        f.get();
      }
    } finally {
      CloseableUtils.closeQuietly(client1);
      CloseableUtils.closeQuietly(client2);
    }
  }
Esempio n. 26
0
  /**
   * Tests that offsets are properly committed to ZooKeeper and initial offsets are read from
   * ZooKeeper.
   *
   * <p>This test is only applicable if the Flink Kafka Consumer uses the ZooKeeperOffsetHandler.
   */
  @Test(timeout = 60000)
  public void testOffsetInZookeeper() throws Exception {
    final int parallelism = 3;

    // write a sequence from 0 to 99 to each of the 3 partitions.
    final String topicName = writeSequence("testOffsetInZK", 100, parallelism, 1);

    StreamExecutionEnvironment env1 =
        StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env1.getConfig().disableSysoutLogging();
    env1.enableCheckpointing(50);
    env1.getConfig().setRestartStrategy(RestartStrategies.noRestart());
    env1.setParallelism(parallelism);

    StreamExecutionEnvironment env2 =
        StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env2.getConfig().disableSysoutLogging();
    env2.enableCheckpointing(50);
    env2.getConfig().setRestartStrategy(RestartStrategies.noRestart());
    env2.setParallelism(parallelism);

    readSequence(env1, standardProps, parallelism, topicName, 100, 0);

    CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();

    Long o1 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorClient, standardProps.getProperty("group.id"), topicName, 0);
    Long o2 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorClient, standardProps.getProperty("group.id"), topicName, 1);
    Long o3 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorClient, standardProps.getProperty("group.id"), topicName, 2);

    LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);

    assertTrue(o1 == null || (o1 >= 0 && o1 <= 100));
    assertTrue(o2 == null || (o2 >= 0 && o2 <= 100));
    assertTrue(o3 == null || (o3 >= 0 && o3 <= 100));

    LOG.info("Manipulating offsets");

    // set the offset to 50 for the three partitions
    ZookeeperOffsetHandler.setOffsetInZooKeeper(
        curatorClient, standardProps.getProperty("group.id"), topicName, 0, 49);
    ZookeeperOffsetHandler.setOffsetInZooKeeper(
        curatorClient, standardProps.getProperty("group.id"), topicName, 1, 49);
    ZookeeperOffsetHandler.setOffsetInZooKeeper(
        curatorClient, standardProps.getProperty("group.id"), topicName, 2, 49);

    curatorClient.close();

    // create new env
    readSequence(env2, standardProps, parallelism, topicName, 50, 50);

    deleteTestTopic(topicName);
  }
 public static void setOffsetInZooKeeper(
     CuratorFramework curatorClient, String groupId, String topic, int partition, long offset)
     throws Exception {
   ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
   String path = topicDirs.consumerOffsetDir() + "/" + partition;
   curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());
   byte[] data = Long.toString(offset).getBytes();
   curatorClient.setData().forPath(path, data);
 }
Esempio n. 28
0
 public boolean existsNode(CuratorFramework zk, String path, boolean watch) throws Exception {
   Stat stat = null;
   if (watch) {
     stat = zk.checkExists().watched().forPath(PathUtils.normalize_path(path));
   } else {
     stat = zk.checkExists().forPath(PathUtils.normalize_path(path));
   }
   return stat != null;
 }
Esempio n. 29
0
  public static void main(String[] args) {

    /*
     * 在curator中,CuratorFramework作为zookeeper的client接口
     */
    String connectString = ZKCommons.ZK_SERVER_IP + ":" + ZKCommons.ZK_SERVER_PORT;
    RetryPolicy retryPolicy = new ExponentialBackoffRetry(5000, 3);

    //		CuratorFrameworkFactory.newClient(connectString, 5000, 5000, retryPolicy);

    final CuratorFramework client =
        CuratorFrameworkFactory.builder()
            .connectString(connectString)
            .sessionTimeoutMs(5000)
            .connectionTimeoutMs(5000)
            .retryPolicy(retryPolicy)
            .build();

    client.start();
    try {
      AfterConnectionEstablished.execute(
          client,
          new Runnable() {

            @Override
            public void run() {
              try {
                Stat stat = new Stat();
                byte[] data = client.getData().storingStatIn(stat).forPath("/");
                System.out.println(new String(data));
                System.out.println(stat);
              } catch (Exception e) {
                e.printStackTrace();
              }
              System.out.println("do something after get connect");
            }
          });
    } catch (Exception e1) {
      e1.printStackTrace();
    }

    try {
      States stat = client.getZookeeperClient().getZooKeeper().getState();
      //			CuratorFrameworkState stat = client.getState();
      System.out.println(stat);
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }

    try {
      Thread.sleep(Integer.MAX_VALUE);
    } catch (InterruptedException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
  }
Esempio n. 30
0
 private void doFixACLs(String path, boolean recursive) throws Exception {
   List<ACL> aclList = getAclForPath(path);
   curator.setACL().withACL(aclList).forPath(path);
   if (recursive) {
     for (String child : curator.getChildren().forPath(path)) {
       doFixACLs(path.equals("/") ? "/" + child : path + "/" + child, recursive);
     }
   }
 }