@Override
  public void createTestTopic(
      String topic, int numberOfPartitions, int replicationFactor, Properties topicConfig) {
    // create topic with one client
    LOG.info("Creating topic {}", topic);

    ZkClient creator = createZkClient();

    AdminUtils.createTopic(creator, topic, numberOfPartitions, replicationFactor, topicConfig);
    creator.close();

    // validate that the topic has been created
    final long deadline = System.currentTimeMillis() + 30000;
    do {
      try {
        Thread.sleep(100);
      } catch (InterruptedException e) {
        // restore interrupted state
      }
      List<KafkaTopicPartitionLeader> partitions =
          FlinkKafkaConsumer08.getPartitionsForTopic(
              Collections.singletonList(topic), standardProps);
      if (partitions != null && partitions.size() > 0) {
        return;
      }
    } while (System.currentTimeMillis() < deadline);
    fail("Test topic could not be created");
  }
  @SuppressWarnings("unchecked")
  public T poll() throws Exception {

    try {

      List<String> list = zkClient.getChildren(root);
      if (list.size() == 0) {
        return null;
      }
      Collections.sort(
          list,
          new Comparator<String>() {
            public int compare(String lhs, String rhs) {
              return getNodeNumber(lhs, Node_NAME).compareTo(getNodeNumber(rhs, Node_NAME));
            }
          });

      for (String nodeName : list) {

        String nodeFullPath = root.concat("/").concat(nodeName);
        try {
          T node = (T) zkClient.readData(nodeFullPath);
          zkClient.delete(nodeFullPath);
          return node;
        } catch (ZkNoNodeException e) {
          // ignore
        }
      }

      return null;

    } catch (Exception e) {
      throw ExceptionUtil.convertToRuntimeException(e);
    }
  }
  @Override
  public void deleteTestTopic(String topic) {
    LOG.info("Deleting topic {}", topic);

    ZkClient zk = createZkClient();
    AdminUtils.deleteTopic(zk, topic);
    zk.close();
  }
Example #4
0
 @Export
 public void clean() {
   List<String> groups = zkClient.getChildren(ZkPaths.rootPath());
   for (String group : groups) {
     if (zkClient.countChildren(ZkPaths.groupPath(group)) == 0) {
       zkClient.delete(ZkPaths.groupPath(group));
     }
   }
 }
Example #5
0
 @SuppressWarnings("unchecked")
 public List<T> getElements() {
   List<String> paths = _zkClient.getChildren(_elementsPath);
   List<T> elements = new ArrayList<T>();
   for (String path : paths) {
     elements.add((T) _zkClient.readData(path));
   }
   return elements;
 }
Example #6
0
 public ZooKeeperQueue(ZkClient zkClient, String rootPath, boolean isBlocking) {
   _zkClient = zkClient;
   _rootPath = rootPath;
   _isBlocking = isBlocking;
   _elementsPath = rootPath + "/queue";
   if (!_zkClient.exists(rootPath)) {
     _zkClient.createPersistent(rootPath, true);
     _zkClient.createPersistent(_elementsPath, true);
   }
 }
  /**
   * Creates a Kafka topic if needed, or try to increase its partition count to the desired number.
   */
  private Collection<Partition> ensureTopicCreated(
      final String topicName, final int numPartitions, int replicationFactor) {

    final int sessionTimeoutMs = 10000;
    final int connectionTimeoutMs = 10000;
    final ZkClient zkClient =
        new ZkClient(zkAddress, sessionTimeoutMs, connectionTimeoutMs, utf8Serializer);
    try {
      // The following is basically copy/paste from AdminUtils.createTopic() with
      // createOrUpdateTopicPartitionAssignmentPathInZK(..., update=true)
      final Properties topicConfig = new Properties();
      Seq<Object> brokerList = ZkUtils.getSortedBrokerList(zkClient);
      final scala.collection.Map<Object, Seq<Object>> replicaAssignment =
          AdminUtils.assignReplicasToBrokers(brokerList, numPartitions, replicationFactor, -1, -1);
      retryOperations.execute(
          new RetryCallback<Object, RuntimeException>() {

            @Override
            public Object doWithRetry(RetryContext context) throws RuntimeException {
              AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(
                  zkClient, topicName, replicaAssignment, topicConfig, true);
              return null;
            }
          });
      try {
        Collection<Partition> partitions =
            retryOperations.execute(
                new RetryCallback<Collection<Partition>, Exception>() {

                  @Override
                  public Collection<Partition> doWithRetry(RetryContext context) throws Exception {
                    connectionFactory.refreshMetadata(Collections.singleton(topicName));
                    Collection<Partition> partitions = connectionFactory.getPartitions(topicName);
                    if (partitions.size() < numPartitions) {
                      throw new IllegalStateException(
                          "The number of expected partitions was: "
                              + numPartitions
                              + ", but "
                              + partitions.size()
                              + " have been found instead");
                    }
                    connectionFactory.getLeaders(partitions);
                    return partitions;
                  }
                });
        return partitions;
      } catch (Exception e) {
        logger.error("Cannot initialize Binder", e);
        throw new BinderException("Cannot initialize binder:", e);
      }

    } finally {
      zkClient.close();
    }
  }
Example #8
0
 public LinkedHashMap<String, List<String>> listAll() {
   List<String> groups = zkClient.getChildren(ZkPaths.rootPath());
   LinkedHashMap<String, List<String>> result = Maps.newLinkedHashMap();
   for (String group : groups) {
     List<String> cells = zkClient.getChildren(ZkPaths.groupPath(group));
     if (cells != null && !cells.isEmpty()) { // 过滤空 group
       result.put(group, cells);
     }
   }
   return result;
 }
Example #9
0
  public static void main(String[] args) {

    ZkClient zc = new ZkClient("localhost:2181", 1000);
    // 创建监控节点
    zc.create("/monitor/client/clientA", "clientA", CreateMode.EPHEMERAL);
    try {
      System.in.read();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Example #10
0
 @Export
 public List<String> listGroups() {
   List<String> groups = zkClient.getChildren(ZkPaths.rootPath());
   List<String> notEmptyGroups = Lists.newArrayList();
   for (String group : groups) {
     if (zkClient.countChildren(ZkPaths.groupPath(group)) > 0) {
       notEmptyGroups.add(group);
     }
   }
   return notEmptyGroups;
 }
 public static void writeData(String path, Object object, boolean createPathIfNotExist) {
   try {
     zkClient.writeData(path, JsonUtils.marshalToByte(object, SerializerFeature.WriteClassName));
   } catch (ZkException e) {
     if (createPathIfNotExist) {
       createIfNotExist(path);
       zkClient.writeData(path, JsonUtils.marshalToByte(object, SerializerFeature.WriteClassName));
     } else {
       throw e;
     }
   }
 }
Example #12
0
 /** 使用正确的密码删除节点 */
 static void deleteParent() throws Exception {
   try {
     zkClient = new ZkClient(SERVER_LIST, 50000);
     // zkClient.addAuthInfo(authentication_type, correctAuthentication.getBytes());
     if (zkClient.exists(PATH)) {
       zkClient.delete(PATH);
       System.out.println(PATH + "删除成功");
     }
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
  public boolean offer(T element) throws Exception {

    String nodeFullPath = root.concat("/").concat(Node_NAME);
    try {
      zkClient.createPersistentSequential(nodeFullPath, element);
    } catch (ZkNoNodeException e) {
      zkClient.createPersistent(root);
      offer(element);
    } catch (Exception e) {
      throw ExceptionUtil.convertToRuntimeException(e);
    }
    return true;
  }
Example #14
0
  /** 不使用密码 删除节点 */
  static void deleteNodeByNoAuthentication() throws Exception {

    String prefix = "[不使用任何授权信息]";

    try {
      System.out.println(prefix + "删除节点:" + PATH_DEL);
      zkClient = new ZkClient(SERVER_LIST, 50000);
      if (zkClient.exists(PATH_DEL)) {
        zkClient.delete(PATH_DEL);
        System.out.println(prefix + "删除成功");
      }
    } catch (Exception e) {
      System.err.println(prefix + "删除失败,原因是:" + e.getMessage());
    }
  }
 public static void main(String[] args) throws Exception {
   // 连接zookeeper服务器
   ZkClient zkClient = new ZkClient("127.0.0.1:2181");
   // 读取本地spring配置
   String filePath = "c:/applicationContext.xml";
   // zookeeper保存spring配置的znode
   String zkPath = ZOOKEEPER_NODE_PATH + "/app.xml";
   ZkObject object = new ZkObject();
   object.setData(readFile(filePath).trim());
   System.out.println(object.getData());
   // 上传配置
   zkClient.writeData(zkPath, object);
   zkClient.close();
   System.out.println("配置上传完成");
 }
Example #16
0
  /** 更新数据:不采用密码 */
  static void updateDataByNoAuthentication() {

    String prefix = "[不使用任何授权信息]";

    System.out.println(prefix + "更新数据: " + PATH);
    try {
      zkClient = new ZkClient(SERVER_LIST, 50000);
      if (zkClient.exists(PATH)) {
        zkClient.writeData(PATH, prefix);
        System.out.println(prefix + "更新成功");
      }
    } catch (Exception e) {
      System.err.println(prefix + "更新失败,原因是:" + e.getMessage());
    }
  }
Example #17
0
  /** 更新数据:采用正确的密码 */
  static void updateDataByCorrectAuthentication() {

    String prefix = "[使用正确的授权信息]";

    System.out.println(prefix + "更新数据:" + PATH);
    try {
      zkClient = new ZkClient(SERVER_LIST, 50000);
      // zkClient.addAuthInfo(authentication_type, correctAuthentication.getBytes());
      if (zkClient.exists(PATH)) {
        zkClient.writeData(PATH, prefix);
        System.out.println(prefix + "更新成功");
      }
    } catch (Exception e) {
      System.err.println(prefix + "更新失败,原因是:" + e.getMessage());
    }
  }
Example #18
0
  /** 使用正确的密码删除节点 */
  static void deleteNodeByCorrectAuthentication() throws Exception {

    String prefix = "[使用正确的授权信息]";

    try {
      System.out.println(prefix + "删除节点:" + PATH_DEL);
      zkClient = new ZkClient(SERVER_LIST, 50000);
      // zkClient.addAuthInfo(authentication_type, correctAuthentication.getBytes());
      if (zkClient.exists(PATH_DEL)) {
        zkClient.delete(PATH_DEL);
        System.out.println(prefix + "删除成功");
      }
    } catch (Exception e) {
      System.out.println(prefix + "删除失败,原因是:" + e.getMessage());
    }
  }
 public List<String> getChildren(String path) {
   try {
     return client.getChildren(path);
   } catch (ZkNoNodeException e) {
     return null;
   }
 }
 public static void createIfNotExist(String path) {
   try {
     zkClient.createPersistent(path, true);
   } catch (Exception e) {
     //
   }
 }
 public static <T> T readData(String path, Class<T> dataClass) {
   byte[] data = zkClient.readData(path, true);
   if (data != null && data.length != 0) {
     return JsonUtils.unmarshalFromByte(data, dataClass);
   }
   return null;
 }
 /*
  * (non-Javadoc)
  *
  * @see com.taobao.metamorphosis.client.SessionFactory#close()
  */
 @Override
 public void shutdown() throws MetaClientException {
   if (shutdown) {
     return;
   }
   shutdown = true;
   // if (this.diamondManager != null) {
   // this.diamondManager.close();
   // }
   recoverManager.shutdown();
   // this.localMessageStorageManager.shutdown();
   for (final Shutdownable child : children) {
     child.shutdown();
   }
   try {
     remotingClient.stop();
   } catch (final NotifyRemotingException e) {
     throw new NetworkException("Stop remoting client failed", e);
   }
   if (zkClient != null) {
     zkClient.close();
   }
   if (!isHutdownHookCalled) {
     Runtime.getRuntime().removeShutdownHook(shutdownHook);
   }
 }
Example #23
0
 @Override
 public void delete(String path) {
   try {
     client.delete(path);
   } catch (ZkNoNodeException e) {
   }
 }
Example #24
0
 @Override
 public void createEphemeral(String path, Object data) {
   try {
     client.createEphemeral(path, data);
   } catch (ZkNodeExistsException e) {
   }
 }
Example #25
0
 public String enqueue(T element) {
   try {
     String sequential = _zkClient.createPersistentSequential(getElementRoughPath(), element);
     String elementId = sequential.substring(sequential.lastIndexOf('/') + 1);
     return elementId;
   } catch (Exception e) {
     throw ExceptionUtil.convertToRuntimeException(e);
   }
 }
Example #26
0
 /** 获取数据:不采用密码 */
 static void getDataByNoAuthentication() {
   String prefix = "[不使用任何授权信息]";
   try {
     System.out.println(prefix + "获取数据:" + PATH);
     zkClient = new ZkClient(SERVER_LIST, 50000);
     System.out.println(prefix + "成功获取数据:" + zkClient.readData(PATH));
   } catch (Exception e) {
     System.err.println(prefix + "获取数据失败,原因:" + e.getMessage());
   }
 }
Example #27
0
 private List<String> getConnectedHosts() {
   List<String> ips = zkClient.getChildren(WATCH_PATH);
   for (int i = 0; i < ips.size(); i++) {
     if (!isConnected(ips.get(i))) {
       ips.remove(i);
       i--;
     }
   }
   return ips;
 }
  public ZookeeprTickCondition(String workPath, ZkClient zkClient) {
    this.zkClient = zkClient;
    this.workPath = ZookeeperConstant.ROOT + "/" + workPath;

    zkClient.createPersistent(this.workPath, true);
    zkClient.subscribeChildChanges(
        this.workPath,
        new IZkChildListener() {

          @Override
          public void handleChildChange(String parentPath, List<String> currentChilds)
              throws Exception {
            if (currentChilds != null && !currentChilds.isEmpty()) {
              return;
            }
            tryOccupy();
          }
        });
  }
  @Override
  public int getLeaderToShutDown(String topic) throws Exception {
    ZkClient zkClient = createZkClient();
    PartitionMetadata firstPart = null;
    do {
      if (firstPart != null) {
        LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
        // not the first try. Sleep a bit
        Thread.sleep(150);
      }

      Seq<PartitionMetadata> partitionMetadata =
          AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
      firstPart = partitionMetadata.head();
    } while (firstPart.errorCode() != 0);
    zkClient.close();

    return firstPart.leader().get().id();
  }
Example #30
0
  public static void main(String[] args) throws Exception {

    List<ACL> acls = new ArrayList<ACL>(1);
    for (ACL ids_acl : Ids.CREATOR_ALL_ACL) {
      acls.add(ids_acl);
    }

    try {
      zkClient = new ZkClient(SERVER_LIST, 50000);
      // zkClient.addAuthInfo(authentication_type, correctAuthentication.getBytes());
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }

    try {
      // zkClient.createPersistent(PATH, acls, "init content");
      System.out.println(
          "使用授权key:" + correctAuthentication + "创建节点:" + PATH + ", 初始内容是: init content");
    } catch (Exception e) {
      e.printStackTrace();
    }
    try {
      // zkClient.createPersistent(PATH_DEL, acls, "待删节点");
      System.out.println(
          "使用授权key:" + correctAuthentication + "创建节点:" + PATH_DEL + ", 初始内容是: init content");
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }

    // 获取数据
    // getDataByNoAuthentication();
    // getDataByBadAuthentication();
    // getDataByCorrectAuthentication();

    // 更新数据
    // updateDataByNoAuthentication();
    // updateDataByBadAuthentication();
    // updateDataByCorrectAuthentication();

    // 获取数据
    // getDataByNoAuthentication();
    // getDataByBadAuthentication();
    // getDataByCorrectAuthentication();

    // 删除数据
    deleteNodeByBadAuthentication();
    deleteNodeByNoAuthentication();
    deleteNodeByCorrectAuthentication();

    deleteParent();

    zkClient.close();
  }