/** Tests that no checkpoints happen when the fetcher is not running. */ @Test public void checkRestoredNullCheckpointWhenFetcherNotReady() throws Exception { FlinkKafkaConsumerBase<String> consumer = getConsumer(null, new LinkedMap(), true); OperatorStateStore operatorStateStore = mock(OperatorStateStore.class); TestingListState<Serializable> listState = new TestingListState<>(); when(operatorStateStore.getSerializableListState(Matchers.any(String.class))) .thenReturn(listState); consumer.initializeState(operatorStateStore); consumer.prepareSnapshot(17L, 17L); assertFalse(listState.get().iterator().hasNext()); }
/** Tests that no checkpoints happen when the fetcher is not running. */ @Test public void checkRestoredCheckpointWhenFetcherNotReady() throws Exception { OperatorStateStore operatorStateStore = mock(OperatorStateStore.class); TestingListState<Serializable> expectedState = new TestingListState<>(); expectedState.add(Tuple2.of(new KafkaTopicPartition("abc", 13), 16768L)); expectedState.add(Tuple2.of(new KafkaTopicPartition("def", 7), 987654321L)); TestingListState<Serializable> listState = new TestingListState<>(); FlinkKafkaConsumerBase<String> consumer = getConsumer(null, new LinkedMap(), true); when(operatorStateStore.getSerializableListState(Matchers.any(String.class))) .thenReturn(expectedState); consumer.initializeState(operatorStateStore); when(operatorStateStore.getSerializableListState(Matchers.any(String.class))) .thenReturn(listState); consumer.prepareSnapshot(17L, 17L); Set<Serializable> expected = new HashSet<>(); for (Serializable serializable : expectedState.get()) { expected.add(serializable); } int counter = 0; for (Serializable serializable : listState.get()) { assertTrue(expected.contains(serializable)); counter++; } assertEquals(expected.size(), counter); }
/** Tests that on snapshots, states and offsets to commit to Kafka are correct */ @Test @SuppressWarnings("unchecked") public void testSnapshotState() throws Exception { // -------------------------------------------------------------------- // prepare fake states // -------------------------------------------------------------------- final HashMap<KafkaTopicPartition, Long> state1 = new HashMap<>(); state1.put(new KafkaTopicPartition("abc", 13), 16768L); state1.put(new KafkaTopicPartition("def", 7), 987654321L); final HashMap<KafkaTopicPartition, Long> state2 = new HashMap<>(); state2.put(new KafkaTopicPartition("abc", 13), 16770L); state2.put(new KafkaTopicPartition("def", 7), 987654329L); final HashMap<KafkaTopicPartition, Long> state3 = new HashMap<>(); state3.put(new KafkaTopicPartition("abc", 13), 16780L); state3.put(new KafkaTopicPartition("def", 7), 987654377L); // -------------------------------------------------------------------- final AbstractFetcher<String, ?> fetcher = mock(AbstractFetcher.class); when(fetcher.snapshotCurrentState()).thenReturn(state1, state2, state3); final LinkedMap pendingOffsetsToCommit = new LinkedMap(); FlinkKafkaConsumerBase<String> consumer = getConsumer(fetcher, pendingOffsetsToCommit, true); assertEquals(0, pendingOffsetsToCommit.size()); OperatorStateStore backend = mock(OperatorStateStore.class); TestingListState<Serializable> init = new TestingListState<>(); TestingListState<Serializable> listState1 = new TestingListState<>(); TestingListState<Serializable> listState2 = new TestingListState<>(); TestingListState<Serializable> listState3 = new TestingListState<>(); when(backend.getSerializableListState(Matchers.any(String.class))) .thenReturn(init, listState1, listState2, listState3); consumer.initializeState(backend); // checkpoint 1 consumer.prepareSnapshot(138L, 138L); HashMap<KafkaTopicPartition, Long> snapshot1 = new HashMap<>(); for (Serializable serializable : listState1.get()) { Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable; snapshot1.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1); } assertEquals(state1, snapshot1); assertEquals(1, pendingOffsetsToCommit.size()); assertEquals(state1, pendingOffsetsToCommit.get(138L)); // checkpoint 2 consumer.prepareSnapshot(140L, 140L); HashMap<KafkaTopicPartition, Long> snapshot2 = new HashMap<>(); for (Serializable serializable : listState2.get()) { Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable; snapshot2.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1); } assertEquals(state2, snapshot2); assertEquals(2, pendingOffsetsToCommit.size()); assertEquals(state2, pendingOffsetsToCommit.get(140L)); // ack checkpoint 1 consumer.notifyCheckpointComplete(138L); assertEquals(1, pendingOffsetsToCommit.size()); assertTrue(pendingOffsetsToCommit.containsKey(140L)); // checkpoint 3 consumer.prepareSnapshot(141L, 141L); HashMap<KafkaTopicPartition, Long> snapshot3 = new HashMap<>(); for (Serializable serializable : listState3.get()) { Tuple2<KafkaTopicPartition, Long> kafkaTopicPartitionLongTuple2 = (Tuple2<KafkaTopicPartition, Long>) serializable; snapshot3.put(kafkaTopicPartitionLongTuple2.f0, kafkaTopicPartitionLongTuple2.f1); } assertEquals(state3, snapshot3); assertEquals(2, pendingOffsetsToCommit.size()); assertEquals(state3, pendingOffsetsToCommit.get(141L)); // ack checkpoint 3, subsumes number 2 consumer.notifyCheckpointComplete(141L); assertEquals(0, pendingOffsetsToCommit.size()); consumer.notifyCheckpointComplete(666); // invalid checkpoint assertEquals(0, pendingOffsetsToCommit.size()); OperatorStateStore operatorStateStore = mock(OperatorStateStore.class); TestingListState<Tuple2<KafkaTopicPartition, Long>> listState = new TestingListState<>(); when(operatorStateStore.getOperatorState(Matchers.any(ListStateDescriptor.class))) .thenReturn(listState); // create 500 snapshots for (int i = 100; i < 600; i++) { consumer.prepareSnapshot(i, i); listState.clear(); } assertEquals(FlinkKafkaConsumerBase.MAX_NUM_PENDING_CHECKPOINTS, pendingOffsetsToCommit.size()); // commit only the second last consumer.notifyCheckpointComplete(598); assertEquals(1, pendingOffsetsToCommit.size()); // access invalid checkpoint consumer.notifyCheckpointComplete(590); // and the last consumer.notifyCheckpointComplete(599); assertEquals(0, pendingOffsetsToCommit.size()); }