@Test public void testReigsterSlaveConsumer_thenMasterDown_thenMasterStart() throws Exception { this.testReigsterSlaveConsumer_thenMasterDown(); this.mocksControl.reset(); this.mockConnect("meta://localhost:0"); this.mockCommitOffsets(GROUP, new ArrayList<TopicPartitionRegInfo>()); this.mockLoadNullInitOffset("topic1", GROUP, new Partition("0-0")); this.mockLoadNullInitOffset("topic1", GROUP, new Partition("0-1")); this.mockLoadNullInitOffset("topic1", GROUP, new Partition("0-2")); this.mockLoadNullInitOffset("topic2", GROUP, new Partition("0-0")); this.mockFetchManagerRestart(); this.mockAddFetchRequest( new FetchRequest( new Broker(0, "meta://localhost:0"), 0, new TopicPartitionRegInfo("topic1", new Partition("0-0"), 0), 1024 * 1024)); this.mockAddFetchRequest( new FetchRequest( new Broker(0, "meta://localhost:0"), 0, new TopicPartitionRegInfo("topic1", new Partition("0-1"), 0), 1024 * 1024)); this.mockAddFetchRequest( new FetchRequest( new Broker(0, "meta://localhost:0"), 0, new TopicPartitionRegInfo("topic1", new Partition("0-2"), 0), 1024 * 1024)); this.mockAddFetchRequest( new FetchRequest( new Broker(0, "meta://localhost:0"), 0, new TopicPartitionRegInfo("topic2", new Partition("0-0"), 0), 1024 * 1024)); this.mocksControl.replay(); ZkUtils.createEphemeralPath( this.client, this.metaZookeeper.brokerIdsPath + "/0/master", "meta://localhost:0"); this.client.createEphemeral(this.metaZookeeper.brokerTopicsPath + "/topic1/0-m", "3"); this.client.createEphemeral(this.metaZookeeper.brokerTopicsPath + "/topic2/0-m", "1"); Thread.sleep(5000); this.mocksControl.verify(); // 恢复到testReigsterSlaveConsumer时的状态 final SlaveZKLoadRebalanceListener listener = this.checkTopic(); final Set<Broker> brokerSet = ConsumerZooKeeperAccessor.getOldBrokerSet(listener); assertEquals(1, brokerSet.size()); assertTrue(brokerSet.contains(new Broker(0, "meta://localhost:0"))); assertFalse(brokerSet.contains(new Broker(1, "meta://localhost:1"))); }
@Test public void testReigsterSlaveConsumer_thenOtherMasterDown() throws Exception { this.testReigsterSlaveConsumer(); this.mocksControl.reset(); // mock nothing this.mocksControl.replay(); // other master down,no problem ZkUtils.deletePath(this.client, this.metaZookeeper.brokerIdsPath + "/1/master"); ZkUtils.deletePath(this.client, this.metaZookeeper.brokerTopicsPath + "/topic2/1-m"); Thread.sleep(5000); this.mocksControl.verify(); final SlaveZKLoadRebalanceListener listener = this.checkTopic(); final Set<Broker> brokerSet = ConsumerZooKeeperAccessor.getOldBrokerSet(listener); assertEquals(1, brokerSet.size()); assertTrue(brokerSet.contains(new Broker(0, "meta://localhost:0"))); assertFalse(brokerSet.contains(new Broker(1, "meta://localhost:1"))); }
@Test public void testReigsterSlaveConsumer_thenMasterDown() throws Exception { this.testReigsterSlaveConsumer(); this.mocksControl.reset(); this.mockCommitOffsets( GROUP, ConsumerZooKeeperAccessor.getTopicPartitionRegInfos( this.slaveConsumerZooKeeper, this.fetchManager)); this.mockFetchManagerRestartAnyTimes(); this.mockConnectCloseAnyTimes("meta://localhost:0"); this.mocksControl.replay(); // master down ZkUtils.deletePath(this.client, this.metaZookeeper.brokerIdsPath + "/0/master"); // 这里topic的两次删除(挂掉或人工停掉),可能会引起两次负载均衡 ZkUtils.deletePath(this.client, this.metaZookeeper.brokerTopicsPath + "/topic1/0-m"); ZkUtils.deletePath(this.client, this.metaZookeeper.brokerTopicsPath + "/topic2/0-m"); Thread.sleep(5000); this.mocksControl.verify(); // master 挂掉或人工停掉,TopicPartitionRegInfo清空了 final SlaveZKLoadRebalanceListener listener = (SlaveZKLoadRebalanceListener) ConsumerZooKeeperAccessor.getBrokerConnectionListenerForTest( this.slaveConsumerZooKeeper, this.fetchManager); assertNotNull(listener); final ConcurrentHashMap<String /* topic */, ConcurrentHashMap<Partition, TopicPartitionRegInfo>> topicRegistry = ConsumerZooKeeperAccessor.getTopicRegistry(listener); assertNotNull(topicRegistry); assertTrue(topicRegistry.isEmpty()); // assertEquals(2, topicRegistry.size()); // // assertTrue(topicRegistry.containsKey("topic1")); // assertTrue(topicRegistry.containsKey("topic2")); // assertEquals(0, topicRegistry.get("topic1").size()); // assertEquals(0, topicRegistry.get("topic2").size()); final Set<Broker> brokerSet = ConsumerZooKeeperAccessor.getOldBrokerSet(listener); assertEquals(0, brokerSet.size()); assertFalse(brokerSet.contains(new Broker(0, "meta://localhost:0"))); assertFalse(brokerSet.contains(new Broker(1, "meta://localhost:1"))); }
@Test public void testReigsterSlaveConsumer() throws Exception { final ConsumerConfig consumerConfig = new ConsumerConfig(); consumerConfig.setGroup(GROUP); final ConcurrentHashMap<String /* topic */, SubscriberInfo> topicSubcriberRegistry = new ConcurrentHashMap<String, SubscriberInfo>(); topicSubcriberRegistry.put("topic1", new SubscriberInfo(null, 1024 * 1024)); topicSubcriberRegistry.put("topic2", new SubscriberInfo(null, 1024 * 1024)); // 假设集群里有两台master,topic1在master里有3个分区; // topic2在master里有1个分区,在另一个不相关的master里有1个分区 ZkUtils.createEphemeralPath( this.client, this.metaZookeeper.brokerIdsPath + "/0/master", "meta://localhost:0"); ZkUtils.createEphemeralPath( this.client, this.metaZookeeper.brokerIdsPath + "/1/master", "meta://localhost:1"); this.client.createEphemeral(this.metaZookeeper.brokerTopicsPath + "/topic1/0-m", "3"); this.client.createEphemeral(this.metaZookeeper.brokerTopicsPath + "/topic2/0-m", "1"); this.client.createEphemeral(this.metaZookeeper.brokerTopicsPath + "/topic2/1-m", "1"); this.mockConnect("meta://localhost:0"); // this.mockConnect("meta://localhost:1");不连接到另外一个master this.mockCommitOffsets(GROUP, new ArrayList<TopicPartitionRegInfo>()); this.mockLoadNullInitOffset("topic1", GROUP, new Partition("0-0")); this.mockLoadNullInitOffset("topic1", GROUP, new Partition("0-1")); this.mockLoadNullInitOffset("topic1", GROUP, new Partition("0-2")); this.mockLoadNullInitOffset("topic2", GROUP, new Partition("0-0")); // this.mockLoadNullInitOffset("topic2", GROUP, new // Partition("1-0"));不load另外一个master的分区 this.mockFetchManagerRestart(); this.mockAddFetchRequest( new FetchRequest( new Broker(0, "meta://localhost:0"), 0, new TopicPartitionRegInfo("topic1", new Partition("0-0"), 0), 1024 * 1024)); this.mockAddFetchRequest( new FetchRequest( new Broker(0, "meta://localhost:0"), 0, new TopicPartitionRegInfo("topic1", new Partition("0-1"), 0), 1024 * 1024)); this.mockAddFetchRequest( new FetchRequest( new Broker(0, "meta://localhost:0"), 0, new TopicPartitionRegInfo("topic1", new Partition("0-2"), 0), 1024 * 1024)); this.mockAddFetchRequest( new FetchRequest( new Broker(0, "meta://localhost:0"), 0, new TopicPartitionRegInfo("topic2", new Partition("0-0"), 0), 1024 * 1024)); // this.mockAddFetchRequest(new FetchRequest(new Broker(1, // "meta://localhost:1"), 0, new TopicPartitionRegInfo( // "topic2", new Partition("1-0"), 0), 1024 * 1024));不向另外一个master抓取消息 this.mocksControl.replay(); this.slaveConsumerZooKeeper.registerConsumer( consumerConfig, this.fetchManager, topicSubcriberRegistry, this.offsetStorage, this.loadBalanceStrategy); this.mocksControl.verify(); // 验证订阅者分配,只订阅自己master下的所有分区下 final SlaveZKLoadRebalanceListener listener = this.checkTopic(); final Set<Broker> brokerSet = ConsumerZooKeeperAccessor.getOldBrokerSet(listener); assertEquals(1, brokerSet.size()); assertTrue(brokerSet.contains(new Broker(0, "meta://localhost:0"))); assertFalse(brokerSet.contains(new Broker(1, "meta://localhost:1"))); }