@Test
public void runOffsetManipulationInZooKeeperTest() {
try {
final String topicName = "ZookeeperOffsetHandlerTest-Topic";
final String groupId = "ZookeeperOffsetHandlerTest-Group";
final Long offset = (long) (Math.random() * Long.MAX_VALUE);
CuratorFramework curatorFramework =
((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
kafkaServer.createTestTopic(topicName, 3, 2);
ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorFramework, groupId, topicName, 0, offset);
Long fetchedOffset =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, groupId, topicName, 0);
curatorFramework.close();
assertEquals(offset, fetchedOffset);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
@Test(timeout = 60000)
public void testInvalidOffset() throws Exception {
final String topic = "invalidOffsetTopic";
final int parallelism = 1;
// create topic
createTestTopic(topic, parallelism, 1);
final StreamExecutionEnvironment env =
StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
// write 20 messages into topic:
writeSequence(env, topic, 20, parallelism);
// set invalid offset:
CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
ZookeeperOffsetHandler.setOffsetInZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234);
curatorClient.close();
// read from topic
final int valuesCount = 20;
final int startFrom = 0;
readSequence(env, standardProps, parallelism, topic, valuesCount, startFrom);
deleteTestTopic(topic);
}
@Test
public void testNoBarrier() throws Exception {
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
try {
client.start();
final DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
Assert.assertTrue(barrier.waitOnBarrier(10, TimeUnit.SECONDS));
// just for grins, test the infinite wait
ExecutorService service = Executors.newSingleThreadExecutor();
Future<Object> future =
service.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
barrier.waitOnBarrier();
return "";
}
});
Assert.assertTrue(future.get(10, TimeUnit.SECONDS) != null);
} finally {
client.close();
}
}
@Test
public void testBasic() throws Exception {
CuratorFramework client =
CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1));
try {
client.start();
final DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
barrier.setBarrier();
ExecutorService service = Executors.newSingleThreadExecutor();
service.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
Thread.sleep(1000);
barrier.removeBarrier();
return null;
}
});
Assert.assertTrue(barrier.waitOnBarrier(10, TimeUnit.SECONDS));
} finally {
client.close();
}
}
@Override
public void close() {
leaderSelector.close();
client.close();
webServer.close();
}
/**
* Tests that offsets are properly committed to ZooKeeper and initial offsets are read from
* ZooKeeper.
*
* <p>This test is only applicable if the Flink Kafka Consumer uses the ZooKeeperOffsetHandler.
*/
@Test(timeout = 60000)
public void testOffsetInZookeeper() throws Exception {
final int parallelism = 3;
// write a sequence from 0 to 99 to each of the 3 partitions.
final String topicName = writeSequence("testOffsetInZK", 100, parallelism, 1);
StreamExecutionEnvironment env1 =
StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
env1.getConfig().disableSysoutLogging();
env1.enableCheckpointing(50);
env1.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env1.setParallelism(parallelism);
StreamExecutionEnvironment env2 =
StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
env2.getConfig().disableSysoutLogging();
env2.enableCheckpointing(50);
env2.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env2.setParallelism(parallelism);
readSequence(env1, standardProps, parallelism, topicName, 100, 0);
CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
Long o1 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 0);
Long o2 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 1);
Long o3 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 2);
LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
assertTrue(o1 == null || (o1 >= 0 && o1 <= 100));
assertTrue(o2 == null || (o2 >= 0 && o2 <= 100));
assertTrue(o3 == null || (o3 >= 0 && o3 <= 100));
LOG.info("Manipulating offsets");
// set the offset to 50 for the three partitions
ZookeeperOffsetHandler.setOffsetInZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 0, 49);
ZookeeperOffsetHandler.setOffsetInZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 1, 49);
ZookeeperOffsetHandler.setOffsetInZooKeeper(
curatorClient, standardProps.getProperty("group.id"), topicName, 2, 49);
curatorClient.close();
// create new env
readSequence(env2, standardProps, parallelism, topicName, 50, 50);
deleteTestTopic(topicName);
}
@Test(timeout = 60000)
public void testOffsetAutocommitTest() throws Exception {
final int parallelism = 3;
// write a sequence from 0 to 99 to each of the 3 partitions.
final String topicName = writeSequence("testOffsetAutocommit", 100, parallelism, 1);
StreamExecutionEnvironment env =
StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
// NOTE: We are not enabling the checkpointing!
env.getConfig().disableSysoutLogging();
env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env.setParallelism(parallelism);
// the readSequence operation sleeps for 20 ms between each record.
// setting a delay of 25*20 = 500 for the commit interval makes
// sure that we commit roughly 3-4 times while reading, however
// at least once.
Properties readProps = new Properties();
readProps.putAll(standardProps);
readProps.setProperty("auto.commit.interval.ms", "500");
// read so that the offset can be committed to ZK
readSequence(env, readProps, parallelism, topicName, 100, 0);
// get the offset
CuratorFramework curatorFramework =
((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
Long o1 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 0);
Long o2 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 1);
Long o3 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 2);
curatorFramework.close();
LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
// ensure that the offset has been committed
boolean atLeastOneOffsetSet =
(o1 != null && o1 > 0 && o1 <= 100)
|| (o2 != null && o2 > 0 && o2 <= 100)
|| (o3 != null && o3 > 0 && o3 <= 100);
assertTrue(
"Expecting at least one offset to be set o1=" + o1 + " o2=" + o2 + " o3=" + o3,
atLeastOneOffsetSet);
deleteTestTopic(topicName);
}
@Override
public void close() {
if (state.compareAndSet(State.OPEN, State.CLOSED)) {
for (Map.Entry<String, Entry> mapEntry : things.entrySet()) {
Entry entry = mapEntry.getValue();
if (entry.closer != null) {
log.debug(
String.format(
"Closing left over thing. Type: %s - Id: %s",
entry.thing.getClass(), mapEntry.getKey()));
entry.closer.close();
}
}
things.clear();
client.close();
events.clear();
}
}
public DistributedClusterState(Map<Object, Object> _conf) throws Exception {
conf = _conf;
// just mkdir STORM_ZOOKEEPER_ROOT dir
CuratorFramework _zk = mkZk();
String path = String.valueOf(conf.get(Config.STORM_ZOOKEEPER_ROOT));
zkobj.mkdirs(_zk, path);
_zk.close();
active = new AtomicBoolean(true);
watcher =
new WatcherCallBack() {
@Override
public void execute(KeeperState state, EventType type, String path) {
if (active.get()) {
if (!(state.equals(KeeperState.SyncConnected))) {
LOG.warn(
"Received event "
+ state
+ ":"
+ type
+ ":"
+ path
+ " with disconnected Zookeeper.");
} else {
LOG.info("Received event " + state + ":" + type + ":" + path);
}
if (!type.equals(EventType.None)) {
for (Entry<UUID, ClusterStateCallback> e : callbacks.entrySet()) {
ClusterStateCallback fn = e.getValue();
fn.execute(type, path);
}
}
}
}
};
zk = null;
zk = mkZk(watcher);
}
void doEvaluate(Statement base) throws Throwable {
try {
cluster = new TestingCluster(3);
cluster.start();
client = newClient(cluster.getConnectString(), new RetryOneTime(200 /* ms */));
client.start();
checkState(
client.blockUntilConnected(5, TimeUnit.SECONDS),
"failed to connect to zookeeper in 5 seconds");
base.evaluate();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Interrupted while connecting to ZooKeeper", e);
} finally {
client.close();
cluster.close();
}
}
@Test
public void testServerCrash() throws Exception {
final int TIMEOUT = 1000;
final CuratorFramework client =
CuratorFrameworkFactory.builder()
.connectString(server.getConnectString())
.connectionTimeoutMs(TIMEOUT)
.retryPolicy(new RetryOneTime(1))
.build();
try {
client.start();
final DistributedBarrier barrier = new DistributedBarrier(client, "/barrier");
barrier.setBarrier();
final ExecutorService service = Executors.newSingleThreadExecutor();
Future<Object> future =
service.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
Thread.sleep(TIMEOUT / 2);
server.stop();
return null;
}
});
barrier.waitOnBarrier(TIMEOUT * 2, TimeUnit.SECONDS);
future.get();
Assert.fail();
} catch (KeeperException.ConnectionLossException expected) {
// expected
} finally {
client.close();
}
}
@Override
public void close() throws IOException {
curatorClient.close();
}
@Override
public void close() {
this.active.set(false);
zk.close();
}
@Test
public void testRun() throws Exception {
TestingCluster localCluster = new TestingCluster(1);
localCluster.start();
CuratorFramework localCf =
CuratorFrameworkFactory.builder()
.connectString(localCluster.getConnectString())
.retryPolicy(new ExponentialBackoffRetry(1, 10))
.compressionProvider(new PotentiallyGzippedCompressionProvider(false))
.build();
localCf.start();
TestingCluster remoteCluster = new TestingCluster(1);
remoteCluster.start();
CuratorFramework remoteCf =
CuratorFrameworkFactory.builder()
.connectString(remoteCluster.getConnectString())
.retryPolicy(new ExponentialBackoffRetry(1, 10))
.compressionProvider(new PotentiallyGzippedCompressionProvider(false))
.build();
remoteCf.start();
ObjectMapper jsonMapper = new DefaultObjectMapper();
DruidClusterBridgeConfig config =
new DruidClusterBridgeConfig() {
@Override
public String getTier() {
return DruidServer.DEFAULT_TIER;
}
@Override
public Duration getStartDelay() {
return new Duration(0);
}
@Override
public Duration getPeriod() {
return new Duration(Long.MAX_VALUE);
}
@Override
public String getBrokerServiceName() {
return "testz0rz";
}
@Override
public int getPriority() {
return 0;
}
};
ScheduledExecutorFactory factory = ScheduledExecutors.createFactory(new Lifecycle());
DruidNode me = new DruidNode("me", "localhost", 8080);
AtomicReference<LeaderLatch> leaderLatch =
new AtomicReference<>(new LeaderLatch(localCf, "test"));
ZkPathsConfig zkPathsConfig =
new ZkPathsConfig() {
@Override
public String getZkBasePath() {
return "/druid";
}
};
DruidServerMetadata metadata =
new DruidServerMetadata("test", "localhost", 1000, "bridge", DruidServer.DEFAULT_TIER, 0);
DbSegmentPublisher dbSegmentPublisher = EasyMock.createMock(DbSegmentPublisher.class);
EasyMock.replay(dbSegmentPublisher);
DatabaseSegmentManager databaseSegmentManager =
EasyMock.createMock(DatabaseSegmentManager.class);
EasyMock.replay(databaseSegmentManager);
ServerView serverView = EasyMock.createMock(ServerView.class);
EasyMock.replay(serverView);
BridgeZkCoordinator bridgeZkCoordinator =
new BridgeZkCoordinator(
jsonMapper,
zkPathsConfig,
new SegmentLoaderConfig(),
metadata,
remoteCf,
dbSegmentPublisher,
databaseSegmentManager,
serverView);
Announcer announcer = new Announcer(remoteCf, Executors.newSingleThreadExecutor());
announcer.start();
announcer.announce(
zkPathsConfig.getAnnouncementsPath() + "/" + me.getHost(),
jsonMapper.writeValueAsBytes(me));
BatchDataSegmentAnnouncer batchDataSegmentAnnouncer =
EasyMock.createMock(BatchDataSegmentAnnouncer.class);
BatchServerInventoryView batchServerInventoryView =
EasyMock.createMock(BatchServerInventoryView.class);
EasyMock.expect(batchServerInventoryView.getInventory())
.andReturn(
Arrays.asList(
new DruidServer("1", "localhost", 117, "historical", DruidServer.DEFAULT_TIER, 0),
new DruidServer("2", "localhost", 1, "historical", DruidServer.DEFAULT_TIER, 0)));
batchServerInventoryView.registerSegmentCallback(
EasyMock.<Executor>anyObject(), EasyMock.<ServerView.SegmentCallback>anyObject());
batchServerInventoryView.registerServerCallback(
EasyMock.<Executor>anyObject(), EasyMock.<ServerView.ServerCallback>anyObject());
EasyMock.expectLastCall();
batchServerInventoryView.start();
EasyMock.expectLastCall();
batchServerInventoryView.stop();
EasyMock.expectLastCall();
EasyMock.replay(batchServerInventoryView);
DruidClusterBridge bridge =
new DruidClusterBridge(
jsonMapper,
config,
factory,
me,
localCf,
leaderLatch,
bridgeZkCoordinator,
announcer,
batchDataSegmentAnnouncer,
batchServerInventoryView);
bridge.start();
int retry = 0;
while (!bridge.isLeader()) {
if (retry > 5) {
throw new ISE("Unable to become leader");
}
Thread.sleep(100);
retry++;
}
String path = "/druid/announcements/localhost:8080";
retry = 0;
while (remoteCf.checkExists().forPath(path) == null) {
if (retry > 5) {
throw new ISE("Unable to announce");
}
Thread.sleep(100);
retry++;
}
boolean verified = verifyUpdate(jsonMapper, path, remoteCf);
retry = 0;
while (!verified) {
if (retry > 5) {
throw new ISE("No updates to bridge node occurred");
}
Thread.sleep(100);
retry++;
verified = verifyUpdate(jsonMapper, path, remoteCf);
}
announcer.stop();
bridge.stop();
remoteCf.close();
remoteCluster.close();
localCf.close();
localCluster.close();
EasyMock.verify(batchServerInventoryView);
EasyMock.verify(dbSegmentPublisher);
EasyMock.verify(databaseSegmentManager);
EasyMock.verify(serverView);
}
/**
* This test ensures that when the consumers retrieve some start offset from kafka (earliest,
* latest), that this offset is committed to Zookeeper, even if some partitions are not read
*
* <p>Test: - Create 3 topics - write 50 messages into each. - Start three consumers with
* auto.offset.reset='latest' and wait until they committed into ZK. - Check if the offsets in ZK
* are set to 50 for the three partitions
*
* <p>See FLINK-3440 as well
*/
@Test(timeout = 60000)
public void testKafkaOffsetRetrievalToZookeeper() throws Exception {
final int parallelism = 3;
// write a sequence from 0 to 49 to each of the 3 partitions.
final String topicName = writeSequence("testKafkaOffsetToZk", 50, parallelism, 1);
final StreamExecutionEnvironment env2 =
StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
env2.getConfig().disableSysoutLogging();
env2.getConfig().setRestartStrategy(RestartStrategies.noRestart());
env2.setParallelism(parallelism);
env2.enableCheckpointing(200);
Properties readProps = new Properties();
readProps.putAll(standardProps);
readProps.setProperty("auto.offset.reset", "latest");
DataStream<String> stream =
env2.addSource(kafkaServer.getConsumer(topicName, new SimpleStringSchema(), readProps));
stream.addSink(new DiscardingSink<String>());
final AtomicReference<Throwable> errorRef = new AtomicReference<>();
final Thread runner =
new Thread("runner") {
@Override
public void run() {
try {
env2.execute();
} catch (Throwable t) {
if (!(t.getCause() instanceof JobCancellationException)) {
errorRef.set(t);
}
}
}
};
runner.start();
final CuratorFramework curatorFramework =
((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
final Long l49 = 49L;
final long deadline = 30000 + System.currentTimeMillis();
do {
Long o1 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 0);
Long o2 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 1);
Long o3 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 2);
if (l49.equals(o1) && l49.equals(o2) && l49.equals(o3)) {
break;
}
Thread.sleep(100);
} while (System.currentTimeMillis() < deadline);
// cancel the job
JobManagerCommunicationUtils.cancelCurrentJob(flink.getLeaderGateway(timeout));
final Throwable t = errorRef.get();
if (t != null) {
throw new RuntimeException("Job failed with an exception", t);
}
// check if offsets are correctly in ZK
Long o1 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 0);
Long o2 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 1);
Long o3 =
ZookeeperOffsetHandler.getOffsetFromZooKeeper(
curatorFramework, standardProps.getProperty("group.id"), topicName, 2);
Assert.assertEquals(Long.valueOf(49L), o1);
Assert.assertEquals(Long.valueOf(49L), o2);
Assert.assertEquals(Long.valueOf(49L), o3);
curatorFramework.close();
}
@After
public void tearDown() throws Exception {
workerTaskMonitor.stop();
cf.close();
testingCluster.stop();
}
@Override
public void close() {
curatorFramework.close();
}
@Override
public void destroy() throws Exception {
zkClient.close();
}
public void close() {
client.close();
}
public void close() {
_curator.close();
_curator = null;
}