@Bean(destroyMethod = "close")
@ConditionalOnMissingBean
@SneakyThrows
public CuratorFramework curatorFramework(
RetryPolicy retryPolicy, ZookeeperDiscoveryProperties properties) {
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
if (ensembleProvider != null) {
builder.ensembleProvider(ensembleProvider);
}
CuratorFramework curator =
builder
.retryPolicy(retryPolicy)
.connectString(zookeeperProperties.getConnectString())
.build();
curator.start();
log.trace(
"blocking until connected to zookeeper for "
+ properties.getBlockUntilConnectedWait()
+ properties.getBlockUntilConnectedUnit());
curator.blockUntilConnected(
properties.getBlockUntilConnectedWait(), properties.getBlockUntilConnectedUnit());
log.trace("connected to zookeeper");
return curator;
}
public static void main(String[] args) throws Exception {
CuratorFramework client = null;
DistributedDelayQueue<String> queue = null;
try {
client = CuratorClientFactory.newClient();
client
.getCuratorListenable()
.addListener(
new CuratorListener() {
@Override
public void eventReceived(CuratorFramework client, CuratorEvent event)
throws Exception {
System.out.println("CuratorEvent: " + event.getType().name());
}
});
client.start();
QueueConsumer<String> consumer = createQueueConsumer();
QueueBuilder<String> builder =
QueueBuilder.builder(client, consumer, createQueueSerializer(), PATH);
queue = builder.buildDelayQueue();
queue.start();
for (int i = 0; i < 10; i++) {
queue.put("test-" + i, System.currentTimeMillis() + 10000);
}
System.out.println(getCurrentTimeStr() + ": 所有的数据已经生产完毕");
Thread.sleep(20000);
} finally {
CloseableUtils.closeQuietly(client);
CloseableUtils.closeQuietly(queue);
}
}
public void addServer(
String zookeeperInstance,
String joiningServerIp,
String joiningServerZookeeperPort,
Integer myId,
FutureCallback callback) {
AsyncCallback.DataCallback dataCallback = getDataCallback(callback);
try (CuratorFramework zkClient = connectToZookeeper(zookeeperInstance)) {
List<String> joiningServers = new ArrayList<>();
String serverStr =
"server." + myId + "=" + joiningServerIp + ":2888:3888;" + joiningServerZookeeperPort;
logger.info("Adding server: " + serverStr);
joiningServers.add(serverStr);
LinkedList<Integer> results = new LinkedList<Integer>();
zkClient
.getZookeeperClient()
.getZooKeeper()
.reconfig(joiningServers, null, null, -1, dataCallback, results);
} catch (Exception e) {
logger.error("Rethrowing error ", e);
throw new RuntimeException(e);
}
}
protected String getFirstService(String containerPath) throws Exception {
CuratorFramework curatorFramework = curator.get();
if (curatorFramework != null) {
byte[] data = curatorFramework.getData().forPath(containerPath);
if (data != null && data.length > 0) {
String text = new String(data).trim();
if (!text.isEmpty()) {
ObjectMapper mapper = new ObjectMapper();
Map<String, Object> map = mapper.readValue(data, HashMap.class);
Object serviceValue = map.get("services");
if (serviceValue instanceof List) {
List services = (List) serviceValue;
if (services != null) {
if (!services.isEmpty()) {
List<String> serviceTexts = new ArrayList<String>();
for (Object service : services) {
String serviceText = getSubstitutedData(curatorFramework, service.toString());
if (io.fabric8.common.util.Strings.isNotBlank(serviceText)) {
return serviceText;
}
}
}
}
}
}
}
}
return null;
}
/**
* @Title: registerListeners @TitleExplain: 向zk注册监听 @Description: 注册监听
*
* @param client
* @author wudan-mac
*/
private void registerListeners(CuratorFramework client) {
/*
* 注册监听者
*/
client
.getConnectionStateListenable()
.addListener(
new ConnectionStateListener() {
@Override
public void stateChanged(
CuratorFramework client, ConnectionState newState) { // 当状态发生变化时通知监听者
logger.debug("CuratorFramework state changed: {}", newState);
if (newState == ConnectionState.CONNECTED
|| newState == ConnectionState.RECONNECTED) {
for (IZKListener listener : listeners) {
listener.executor(client);
logger.debug("Listener {} executed!", listener.getClass().getName());
}
}
}
});
/*
*异常处理
*/
client
.getUnhandledErrorListenable()
.addListener(
new UnhandledErrorListener() {
@Override
public void unhandledError(String message, Throwable e) {
logger.debug("CuratorFramework unhandledError: {}", message);
}
});
}
/**
* 连接
*
* @return
*/
public boolean connect() {
try {
client
.getConnectionStateListenable()
.addListener(
new ConnectionStateListener() {
public void stateChanged(CuratorFramework framework, ConnectionState state) {
logger.info("Zookeeper状态：" + state.name());
if (state.equals(ConnectionState.RECONNECTED)) {
logger.info("Zookeeper状态：检查节点");
for (String key : backup.keySet()) {
String value = backup.get(key);
createNode(key, value);
}
}
}
});
client.start();
client.getZookeeperClient().blockUntilConnectedOrTimedOut();
createRoomProcessorNode();
return true;
} catch (Exception e) {
logger.error("ZooKeeperWrapper.connect出现异常", e);
return false;
}
}
@Test
public void testBasic() throws Exception {
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
try {
client.start();
final DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
barrier.setBarrier();
ExecutorService service = Executors.newSingleThreadExecutor();
service.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
Thread.sleep(1000);
barrier.removeBarrier();
return null;
}
});
Assert.assertTrue(barrier.waitOnBarrier(10, TimeUnit.SECONDS));
} finally {
client.close();
}
}
@Test
public void testNoBarrier() throws Exception {
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
try {
client.start();
final DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
Assert.assertTrue(barrier.waitOnBarrier(10, TimeUnit.SECONDS));
// just for grins, test the infinite wait
ExecutorService service = Executors.newSingleThreadExecutor();
Future<Object> future =
service.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
barrier.waitOnBarrier();
return "";
}
});
Assert.assertTrue(future.get(10, TimeUnit.SECONDS) != null);
} finally {
client.close();
}
}
@Override
public <S extends StreamDefinition> S save(S entity) {
try {
Map<String, String> map = new HashMap<>();
map.put(DEFINITION_KEY, entity.getDefinition());
map.put(
MODULE_DEFINITIONS_KEY, objectWriter.writeValueAsString(entity.getModuleDefinitions()));
CuratorFramework client = zkConnection.getClient();
String path = Paths.build(Paths.STREAMS, entity.getName());
byte[] binary = ZooKeeperUtils.mapToBytes(map);
BackgroundPathAndBytesable<?> op =
client.checkExists().forPath(path) == null ? client.create() : client.setData();
op.forPath(path, binary);
logger.trace("Saved stream {} with properties {}", path, map);
StreamDefinitionRepositoryUtils.saveDependencies(moduleDependencyRepository, entity);
} catch (Exception e) {
// NodeExistsException indicates that we tried to create the
// path just after another thread/jvm successfully created it
ZooKeeperUtils.wrapAndThrowIgnoring(e, NodeExistsException.class);
}
return entity;
}
public static long getOffsetFromZooKeeper(
CuratorFramework curatorClient, String groupId, String topic, int partition)
throws Exception {
ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
String path = topicDirs.consumerOffsetDir() + "/" + partition;
curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());
byte[] data = curatorClient.getData().forPath(path);
if (data == null) {
return OFFSET_NOT_SET;
} else {
String asString = new String(data);
if (asString.length() == 0) {
return OFFSET_NOT_SET;
} else {
try {
return Long.parseLong(asString);
} catch (NumberFormatException e) {
throw new Exception(
String.format(
"The offset in ZooKeeper for group '%s', topic '%s', partition %d is a malformed string: %s",
groupId, topic, partition, asString));
}
}
}
}
public RunningJobManager(
String zkQuorum,
int zkSessionTimeoutMs,
int zkRetryTimes,
int zkRetryInterval,
String zkRoot,
String lockPath) {
this.zkRoot = zkRoot;
curator = newCurator(zkQuorum, zkSessionTimeoutMs, zkRetryTimes, zkRetryInterval);
try {
curator.start();
} catch (Exception e) {
LOG.error("curator start error {}", e);
}
LOG.info("InterProcessMutex lock path is " + lockPath);
lock = new InterProcessMutex(curator, lockPath);
try {
if (curator.checkExists().forPath(this.zkRoot) == null) {
curator
.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.PERSISTENT)
.forPath(this.zkRoot);
}
} catch (Exception e) {
LOG.warn("{}", e);
}
}
public Map<String, Map<String, ClientLeaseInfo>> loadAndWatchTopicExistingLeases(String topic)
throws Exception {
Map<String, Map<String, ClientLeaseInfo>> topicExistingLeases = new HashMap<>();
CuratorFramework client = m_zkClient.get();
String topicPath = ZKPaths.makePath(ZKPathUtils.getBrokerLeaseRootZkPath(), topic);
client.getData().usingWatcher(m_brokerLeaseChangedWatcher).forPath(topicPath);
m_brokerLeaseChangedWatcher.addWatchedPath(topicPath);
addWatchedTopic(topic);
List<String> partitions = client.getChildren().forPath(topicPath);
if (partitions != null && !partitions.isEmpty()) {
for (String partition : partitions) {
String partitionPath = ZKPaths.makePath(topicPath, partition);
byte[] data = client.getData().forPath(partitionPath);
Map<String, ClientLeaseInfo> existingLeases = deserializeExistingLeases(data);
if (existingLeases != null) {
topicExistingLeases.put(partitionPath, existingLeases);
}
}
}
return topicExistingLeases;
}
public ZKNamedMutexFactory(
String connectString, int connectionTimeout, int sessionTimeout, String namespace)
throws GlobalMutexAcquireException {
this.connectString = connectString;
this.namespace = namespace;
try {
curatorClient =
CuratorFrameworkFactory.builder()
.connectString(connectString)
.retryPolicy(retryPolicy)
.connectionTimeoutMs(connectionTimeout)
.sessionTimeoutMs(sessionTimeout)
.namespace(namespace)
.build();
curatorClient.start();
if (curatorClient.checkExists().forPath(BASEDIR) == null) {
curatorClient.create().forPath(BASEDIR);
}
} catch (Exception e) {
throw new GlobalMutexAcquireException("Unable to : ", e);
}
}
@Test(timeout = 60000)
public void testInvalidOffset() throws Exception {
final String topic = "invalidOffsetTopic";
final int parallelism = 1;
// create topic
createTestTopic(topic, parallelism, 1);
final StreamExecutionEnvironment env =
StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
// write 20 messages into topic:
writeSequence(env, topic, 20, parallelism);
// set invalid offset:
CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
ZookeeperOffsetHandler.setOffsetInZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234);
curatorClient.close();
// read from topic
final int valuesCount = 20;
final int startFrom = 0;
readSequence(env, standardProps, parallelism, topic, valuesCount, startFrom);
deleteTestTopic(topic);
}
public static void main(String[] args) throws Exception {
// RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
// RetryPolicy retryPolicy = new RetryNTimes(5, 1000);
RetryPolicy retryPolicy = new RetryUntilElapsed(5000, 1000);
// CuratorFramework client = CuratorFrameworkFactory
// .newClient("192.168.1.105:2181",5000,5000, retryPolicy);
CuratorFramework client =
CuratorFrameworkFactory.builder()
.connectString("192.168.1.105:2181")
.sessionTimeoutMs(5000)
.connectionTimeoutMs(5000)
.retryPolicy(retryPolicy)
.build();
client.start();
String path =
client
.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.EPHEMERAL)
.forPath("/jike/1", "123".getBytes());
System.out.println(path);
Thread.sleep(Integer.MAX_VALUE);
}
@Override
public void eventReceived(CuratorFramework client, CuratorEvent event) throws Exception {
if (event.getType() == CuratorEventType.CHILDREN) {
if (event.getChildren().isEmpty()) {
client.getChildren().inBackground(event.getContext()).forPath(event.getPath());
} else {
String path = event.getPath() + "/" + event.getChildren().get(0);
LOG.info("Operations Node registered in ZK. Waiting for transports configration");
client.getData().inBackground(event.getContext()).forPath(path);
}
} else if (event.getType() == CuratorEventType.GET_DATA) {
if (event.getData() == null) {
client.getData().inBackground(event.getContext()).forPath(event.getPath());
} else {
OperationsNodeInfo nodeInfo =
OPERATIONS_NODE_INFO_CONVERTER.fromByteArray(event.getData());
boolean isTransportInitialized = !nodeInfo.getTransports().isEmpty();
if (isTransportInitialized) {
LOG.info("Operations Node updated tarnsports configuration in ZK");
((CountDownLatch) event.getContext()).countDown();
} else {
client.getData().inBackground(event.getContext()).forPath(event.getPath());
}
}
}
}
@Test(expected = RuntimeException.class)
public void assertExecuteInTransactionFailure() throws Exception {
CuratorFramework client = mock(CuratorFramework.class);
CuratorTransaction curatorTransaction = mock(CuratorTransaction.class);
TransactionCheckBuilder transactionCheckBuilder = mock(TransactionCheckBuilder.class);
CuratorTransactionBridge curatorTransactionBridge = mock(CuratorTransactionBridge.class);
CuratorTransactionFinal curatorTransactionFinal = mock(CuratorTransactionFinal.class);
when(coordinatorRegistryCenter.getRawClient()).thenReturn(client);
when(client.inTransaction()).thenReturn(curatorTransaction);
when(curatorTransaction.check()).thenReturn(transactionCheckBuilder);
when(transactionCheckBuilder.forPath("/")).thenReturn(curatorTransactionBridge);
when(curatorTransactionBridge.and()).thenReturn(curatorTransactionFinal);
TransactionCreateBuilder transactionCreateBuilder = mock(TransactionCreateBuilder.class);
when(curatorTransactionFinal.create()).thenReturn(transactionCreateBuilder);
when(transactionCreateBuilder.forPath("/test_transaction"))
.thenReturn(curatorTransactionBridge);
when(curatorTransactionBridge.and()).thenThrow(new RuntimeException());
jobNodeStorage.executeInTransaction(
new TransactionExecutionCallback() {
@Override
public void execute(final CuratorTransactionFinal curatorTransactionFinal)
throws Exception {
curatorTransactionFinal.create().forPath("/test_transaction").and();
}
});
verify(coordinatorRegistryCenter).getRawClient();
verify(client).inTransaction();
verify(curatorTransaction).check();
verify(transactionCheckBuilder).forPath("/");
verify(curatorTransactionBridge, times(2)).and();
verify(curatorTransactionFinal).create();
verify(transactionCreateBuilder).forPath("/test_transaction");
verify(curatorTransactionFinal, times(0)).commit();
}
@Test
public void runOffsetManipulationInZooKeeperTest() {
try {
final String topicName = "ZookeeperOffsetHandlerTest-Topic";
final String groupId = "ZookeeperOffsetHandlerTest-Group";
final Long offset = (long) (Math.random() * Long.MAX_VALUE);
CuratorFramework curatorFramework =
((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
kafkaServer.createTestTopic(topicName, 3, 2);
ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorFramework, groupId, topicName, 0, offset);
Long fetchedOffset =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, groupId, topicName, 0);
curatorFramework.close();
assertEquals(offset, fetchedOffset);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
/**
* connect ZK, register Watch/unhandle Watch
*
* @return
*/
public CuratorFramework mkClient(
Map conf, List<String> servers, Object port, String root, final WatcherCallBack watcher) {
CuratorFramework fk = Utils.newCurator(conf, servers, port, root);
fk.getCuratorListenable()
.addListener(
new CuratorListener() {
@Override
public void eventReceived(CuratorFramework _fk, CuratorEvent e) throws Exception {
if (e.getType().equals(CuratorEventType.WATCHED)) {
WatchedEvent event = e.getWatchedEvent();
watcher.execute(event.getState(), event.getType(), event.getPath());
}
}
});
fk.getUnhandledErrorListenable()
.addListener(
new UnhandledErrorListener() {
@Override
public void unhandledError(String msg, Throwable error) {
String errmsg = "Unrecoverable Zookeeper error, halting process: " + msg;
LOG.error(errmsg, error);
JStormUtils.halt_process(1, "Unrecoverable Zookeeper error");
}
});
fk.start();
return fk;
}
@Test
public void test() throws Exception {
Timing timing = new Timing();
LeaderSelector leaderSelector = null;
CuratorFramework client =
CuratorFrameworkFactory.builder()
.retryPolicy(new ExponentialBackoffRetry(100, 3))
.connectString(server.getConnectString())
.sessionTimeoutMs(timing.session())
.connectionTimeoutMs(timing.connection())
.build();
try {
client.start();
MyLeaderSelectorListener listener = new MyLeaderSelectorListener();
ExecutorService executorPool = Executors.newFixedThreadPool(20);
leaderSelector = new LeaderSelector(client, "/test", threadFactory, executorPool, listener);
leaderSelector.autoRequeue();
leaderSelector.start();
timing.sleepABit();
Assert.assertEquals(listener.getLeaderCount(), 1);
} finally {
CloseableUtils.closeQuietly(leaderSelector);
CloseableUtils.closeQuietly(client);
}
}
public Map<String, Set<String>> getServices(String bizCode) {
try {
List<String> children = client.getChildren().forPath("/" + bizCode);
System.out.println("-----------services----------");
for (String bizCh : children) {
String servicepath = bizCh;
addChildWatcher("/" + bizCode + "/" + bizCh);
// System.out.println("-------bizCh-----------"+bizCh);
if (servicepath.endsWith(".do")) {
servicepath = servicepath.replace(".", "/");
servicepath = servicepath.replace("/do", ".do");
if (!servicepath.startsWith("/")) servicepath = "/" + servicepath;
} else {
servicepath = servicepath.replace(".", "/");
}
if (!services.containsKey(servicepath)) services.put(servicepath, new HashSet<String>());
// System.out.println("------------------"+servicepath);
List<String> providers = client.getChildren().forPath(bizCode + "/" + bizCh);
for (String p : providers) {
// System.out.println("------------------------"+p);
services.get(servicepath).add(p);
if (!servicesByP.containsKey(p)) {
servicesByP.put(p, new HashSet<String>());
}
servicesByP.get(p).add(servicepath);
}
}
} catch (Exception e) {
e.printStackTrace();
}
printService();
return services;
}
@Before
public void setUp() throws Exception {
sfb = new ZKServerFactoryBean();
delete(sfb.getDataDir());
delete(sfb.getDataLogDir());
sfb.afterPropertiesSet();
CuratorFrameworkFactory.Builder builder =
CuratorFrameworkFactory.builder()
.connectString("localhost:" + sfb.getClientPortAddress().getPort())
.retryPolicy(new RetryOneTime(1000))
.connectionTimeoutMs(360000);
curator = builder.build();
curator.start();
curator.getZookeeperClient().blockUntilConnectedOrTimedOut();
// setup a local and remote git repo
basedir = System.getProperty("basedir", ".");
File root = new File(basedir + "/target/git").getCanonicalFile();
delete(root);
new File(root, "remote").mkdirs();
remote = Git.init().setDirectory(new File(root, "remote")).call();
remote.commit().setMessage("First Commit").setCommitter("fabric", "user@fabric").call();
String remoteUrl = "file://" + new File(root, "remote").getCanonicalPath();
new File(root, "local").mkdirs();
git = Git.init().setDirectory(new File(root, "local")).call();
git.commit().setMessage("First Commit").setCommitter("fabric", "user@fabric").call();
StoredConfig config = git.getRepository().getConfig();
config.setString("remote", "origin", "url", remoteUrl);
config.setString("remote", "origin", "fetch", "+refs/heads/*:refs/remotes/origin/*");
config.save();
DefaultRuntimeProperties sysprops = new DefaultRuntimeProperties();
sysprops.setProperty(SystemProperties.KARAF_DATA, "target/data");
FabricGitServiceImpl gitService = new FabricGitServiceImpl();
gitService.bindRuntimeProperties(sysprops);
gitService.activate();
gitService.setGitForTesting(git);
DataStoreTemplateRegistry registrationHandler = new DataStoreTemplateRegistry();
registrationHandler.activateComponent();
dataStore = new CachingGitDataStore();
dataStore.bindCurator(curator);
dataStore.bindGitService(gitService);
dataStore.bindRegistrationHandler(registrationHandler);
dataStore.bindRuntimeProperties(sysprops);
dataStore.bindConfigurer(
new Configurer() {
@Override
public <T> void configure(Map<String, ?> configuration, T target) throws Exception {}
});
Map<String, String> datastoreProperties = new HashMap<String, String>();
datastoreProperties.put(GitDataStore.GIT_REMOTE_URL, remoteUrl);
dataStore.activate(datastoreProperties);
}
public void pub(String nameSpace, String path, String value) throws Exception {
Assert.notNull(path, "path不能为空！");
Assert.hasText(nameSpace, "nameSpace不能为空");
if (curatorFramework.usingNamespace(nameSpace).checkExists().forPath(path) != null) {
curatorFramework.usingNamespace(nameSpace).delete().forPath(path);
}
invokeCreate(nameSpace, path, value, CreateMode.EPHEMERAL);
}
/** Only for the 0.8 server we need access to the zk client. */
public CuratorFramework createCuratorClient() {
RetryPolicy retryPolicy = new ExponentialBackoffRetry(100, 10);
CuratorFramework curatorClient =
CuratorFrameworkFactory.newClient(
standardProps.getProperty("zookeeper.connect"), retryPolicy);
curatorClient.start();
return curatorClient;
}
@Test
public void testMultiClient() throws Exception {
CuratorFramework client1 = null;
CuratorFramework client2 = null;
try {
{
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
try {
client.start();
DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
barrier.setBarrier();
} finally {
CloseableUtils.closeQuietly(client);
}
}
client1 = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
client2 = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
List<Future<Object>> futures = Lists.newArrayList();
ExecutorService service = Executors.newCachedThreadPool();
for (final CuratorFramework c : new CuratorFramework[] {client1, client2}) {
Future<Object> future =
service.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
c.start();
DistributedBarrier barrier = new DistributedBarrier(c, "/barrier");
barrier.waitOnBarrier(10, TimeUnit.MILLISECONDS);
return null;
}
});
futures.add(future);
}
Thread.sleep(1000);
{
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
try {
client.start();
DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
barrier.removeBarrier();
} finally {
CloseableUtils.closeQuietly(client);
}
}
for (Future<Object> f : futures) {
f.get();
}
} finally {
CloseableUtils.closeQuietly(client1);
CloseableUtils.closeQuietly(client2);
}
}
/**
* Tests that offsets are properly committed to ZooKeeper and initial offsets are read from
* ZooKeeper.
*
* <p>This test is only applicable if the Flink Kafka Consumer uses the ZooKeeperOffsetHandler.
*/
@Test(timeout = 60000)
public void testOffsetInZookeeper() throws Exception {
final int parallelism = 3;
// write a sequence from 0 to 99 to each of the 3 partitions.
final String topicName = writeSequence("testOffsetInZK", 100, parallelism, 1);
StreamExecutionEnvironment env1 =
StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
env1.getConfig().disableSysoutLogging();
env1.enableCheckpointing(50);
env1.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env1.setParallelism(parallelism);
StreamExecutionEnvironment env2 =
StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
env2.getConfig().disableSysoutLogging();
env2.enableCheckpointing(50);
env2.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env2.setParallelism(parallelism);
readSequence(env1, standardProps, parallelism, topicName, 100, 0);
CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
Long o1 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 0);
Long o2 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 1);
Long o3 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 2);
LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
assertTrue(o1 == null || (o1 >= 0 && o1 <= 100));
assertTrue(o2 == null || (o2 >= 0 && o2 <= 100));
assertTrue(o3 == null || (o3 >= 0 && o3 <= 100));
LOG.info("Manipulating offsets");
// set the offset to 50 for the three partitions
ZookeeperOffsetHandler.setOffsetInZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 0, 49);
ZookeeperOffsetHandler.setOffsetInZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 1, 49);
ZookeeperOffsetHandler.setOffsetInZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 2, 49);
curatorClient.close();
// create new env
readSequence(env2, standardProps, parallelism, topicName, 50, 50);
deleteTestTopic(topicName);
}
public static void setOffsetInZooKeeper(
CuratorFramework curatorClient, String groupId, String topic, int partition, long offset)
throws Exception {
ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, topic);
String path = topicDirs.consumerOffsetDir() + "/" + partition;
curatorClient.newNamespaceAwareEnsurePath(path).ensure(curatorClient.getZookeeperClient());
byte[] data = Long.toString(offset).getBytes();
curatorClient.setData().forPath(path, data);
}
public boolean existsNode(CuratorFramework zk, String path, boolean watch) throws Exception {
Stat stat = null;
if (watch) {
stat = zk.checkExists().watched().forPath(PathUtils.normalize_path(path));
} else {
stat = zk.checkExists().forPath(PathUtils.normalize_path(path));
}
return stat != null;
}
public static void main(String[] args) {
/*
* 在curator中，CuratorFramework作为zookeeper的client接口
*/
String connectString = ZKCommons.ZK_SERVER_IP + ":" + ZKCommons.ZK_SERVER_PORT;
RetryPolicy retryPolicy = new ExponentialBackoffRetry(5000, 3);
// CuratorFrameworkFactory.newClient(connectString, 5000, 5000, retryPolicy);
final CuratorFramework client =
CuratorFrameworkFactory.builder()
.connectString(connectString)
.sessionTimeoutMs(5000)
.connectionTimeoutMs(5000)
.retryPolicy(retryPolicy)
.build();
client.start();
try {
AfterConnectionEstablished.execute(
client,
new Runnable() {
@Override
public void run() {
try {
Stat stat = new Stat();
byte[] data = client.getData().storingStatIn(stat).forPath("/");
System.out.println(new String(data));
System.out.println(stat);
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("do something after get connect");
}
});
} catch (Exception e1) {
e1.printStackTrace();
}
try {
States stat = client.getZookeeperClient().getZooKeeper().getState();
// CuratorFrameworkState stat = client.getState();
System.out.println(stat);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
Thread.sleep(Integer.MAX_VALUE);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private void doFixACLs(String path, boolean recursive) throws Exception {
List<ACL> aclList = getAclForPath(path);
curator.setACL().withACL(aclList).forPath(path);
if (recursive) {
for (String child : curator.getChildren().forPath(path)) {
doFixACLs(path.equals("/") ? "/" + child : path + "/" + child, recursive);
}
}
}