コード例 #1
0
ファイル: DataGenerators.java プロジェクト: lizhiyong/flink
  public static void generateLongStringTupleSequence(
      StreamExecutionEnvironment env,
      String brokerConnection,
      String topic,
      int numPartitions,
      final int from,
      final int to)
      throws Exception {

    TypeInformation<Tuple2<Integer, Integer>> resultType =
        TypeInfoParser.parse("Tuple2<Integer, Integer>");

    env.setParallelism(numPartitions);
    env.getConfig().disableSysoutLogging();
    env.setNumberOfExecutionRetries(0);

    DataStream<Tuple2<Integer, Integer>> stream =
        env.addSource(
            new RichParallelSourceFunction<Tuple2<Integer, Integer>>() {

              private volatile boolean running = true;

              @Override
              public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
                int cnt = from;
                int partition = getRuntimeContext().getIndexOfThisSubtask();

                while (running && cnt <= to) {
                  ctx.collect(new Tuple2<Integer, Integer>(partition, cnt));
                  cnt++;
                }
              }

              @Override
              public void cancel() {
                running = false;
              }
            });

    stream.addSink(
        new FlinkKafkaProducer<>(
            topic,
            new TypeInformationSerializationSchema<>(resultType, env.getConfig()),
            FlinkKafkaProducer.getPropertiesFromBrokerList(brokerConnection),
            new Tuple2Partitioner(numPartitions)));

    env.execute("Data generator (Int, Int) stream to topic " + topic);
  }
コード例 #2
0
  @Test
  public void testTumblingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 100;
    final int NUM_KEYS = 100;
    FailingSource.reset();

    try {
      StreamExecutionEnvironment env =
          StreamExecutionEnvironment.createRemoteEnvironment(
              "localhost", cluster.getLeaderRPCPort());

      env.setParallelism(PARALLELISM);
      env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
      env.enableCheckpointing(100);
      env.setNumberOfExecutionRetries(3);
      env.getConfig().disableSysoutLogging();

      env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3))
          .rebalance()
          .keyBy(0)
          .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS))
          .apply(
              new RichWindowFunction<
                  Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

                private boolean open = false;

                @Override
                public void open(Configuration parameters) {
                  assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                  open = true;
                }

                @Override
                public void apply(
                    Tuple tuple,
                    TimeWindow window,
                    Iterable<Tuple2<Long, IntType>> values,
                    Collector<Tuple4<Long, Long, Long, IntType>> out) {

                  // validate that the function has been opened properly
                  assertTrue(open);

                  int sum = 0;
                  long key = -1;

                  for (Tuple2<Long, IntType> value : values) {
                    sum += value.f1.value;
                    key = value.f0;
                  }
                  out.collect(
                      new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum)));
                }
              })
          .addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE))
          .setParallelism(1);

      tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
コード例 #3
0
  @Test
  public void testTumblingTimeWindowWithKVState() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 100;
    final int NUM_KEYS = 100;
    FailingSource.reset();

    try {
      StreamExecutionEnvironment env =
          StreamExecutionEnvironment.createRemoteEnvironment(
              "localhost", cluster.getLeaderRPCPort());

      env.setParallelism(PARALLELISM);
      env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
      env.enableCheckpointing(100);
      env.setNumberOfExecutionRetries(3);
      env.getConfig().disableSysoutLogging();

      env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3))
          .rebalance()
          .keyBy(0)
          .timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS))
          .apply(
              new RichWindowFunction<
                  Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

                private boolean open = false;

                private OperatorState<Integer> count;

                @Override
                public void open(Configuration parameters) {
                  assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                  open = true;
                  count = getRuntimeContext().getKeyValueState("count", Integer.class, 0);
                }

                @Override
                public void apply(
                    Tuple tuple,
                    TimeWindow window,
                    Iterable<Tuple2<Long, IntType>> values,
                    Collector<Tuple4<Long, Long, Long, IntType>> out)
                    throws Exception {

                  // the window count state starts with the key, so that we get
                  // different count results for each key
                  if (count.value() == 0) {
                    count.update(tuple.<Long>getField(0).intValue());
                  }

                  // validate that the function has been opened properly
                  assertTrue(open);

                  count.update(count.value() + 1);
                  out.collect(
                      new Tuple4<>(
                          tuple.<Long>getField(0),
                          window.getStart(),
                          window.getEnd(),
                          new IntType(count.value())));
                }
              })
          .addSink(new CountValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE))
          .setParallelism(1);

      tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
      e.printStackTrace();
      fail(e.getMessage());
    }
  }
コード例 #4
0
ファイル: Kafka08ITCase.java プロジェクト: Ravenr/flink
  @Test(timeout = 60000)
  public void testOffsetAutocommitTest() throws Exception {
    final String topicName = "testOffsetAutocommit";
    final int parallelism = 3;

    createTestTopic(topicName, parallelism, 1);

    StreamExecutionEnvironment env1 =
        StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env1.getConfig().disableSysoutLogging();
    env1.setNumberOfExecutionRetries(0);
    env1.setParallelism(parallelism);

    StreamExecutionEnvironment env2 =
        StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    // NOTE: We are not enabling the checkpointing!
    env2.getConfig().disableSysoutLogging();
    env2.setNumberOfExecutionRetries(0);
    env2.setParallelism(parallelism);

    // write a sequence from 0 to 99 to each of the 3 partitions.
    writeSequence(env1, topicName, 100, parallelism);

    // the readSequence operation sleeps for 20 ms between each record.
    // setting a delay of 25*20 = 500 for the commit interval makes
    // sure that we commit roughly 3-4 times while reading, however
    // at least once.
    Properties readProps = new Properties();
    readProps.putAll(standardProps);
    readProps.setProperty("auto.commit.interval.ms", "500");

    // read so that the offset can be committed to ZK
    readSequence(env2, readProps, parallelism, topicName, 100, 0);

    // get the offset
    CuratorFramework curatorFramework =
        ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();

    long o1 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorFramework, standardProps.getProperty("group.id"), topicName, 0);
    long o2 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorFramework, standardProps.getProperty("group.id"), topicName, 1);
    long o3 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorFramework, standardProps.getProperty("group.id"), topicName, 2);

    LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);

    // ensure that the offset has been committed
    boolean atLeastOneOffsetSet =
        (o1 > 0 && o1 <= 100) || (o2 > 0 && o2 <= 100) || (o3 > 0 && o3 <= 100);
    assertTrue(
        "Expecting at least one offset to be set o1=" + o1 + " o2=" + o2 + " o3=" + o3,
        atLeastOneOffsetSet);
    // assertTrue("Offset of o1=" + o1 + " was not in range", o1 > 0 && o1 <= 100);
    // assertTrue("Offset of o2=" + o2 + " was not in range", o2 > 0 && o2 <= 100);
    // assertTrue("Offset of o3=" + o3 + " was not in range", o3 > 0 && o3 <= 100);

    deleteTestTopic(topicName);
  }
コード例 #5
0
ファイル: Kafka08ITCase.java プロジェクト: Ravenr/flink
  /**
   * Tests that offsets are properly committed to ZooKeeper and initial offsets are read from
   * ZooKeeper.
   *
   * <p>This test is only applicable if the Flink Kafka Consumer uses the ZooKeeperOffsetHandler.
   */
  @Test(timeout = 60000)
  public void testOffsetInZookeeper() throws Exception {
    final String topicName = "testOffsetInZK";
    final int parallelism = 3;

    createTestTopic(topicName, parallelism, 1);

    StreamExecutionEnvironment env1 =
        StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env1.getConfig().disableSysoutLogging();
    env1.enableCheckpointing(50);
    env1.setNumberOfExecutionRetries(0);
    env1.setParallelism(parallelism);

    StreamExecutionEnvironment env2 =
        StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env2.getConfig().disableSysoutLogging();
    env2.enableCheckpointing(50);
    env2.setNumberOfExecutionRetries(0);
    env2.setParallelism(parallelism);

    StreamExecutionEnvironment env3 =
        StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env3.getConfig().disableSysoutLogging();
    env3.enableCheckpointing(50);
    env3.setNumberOfExecutionRetries(0);
    env3.setParallelism(parallelism);

    // write a sequence from 0 to 99 to each of the 3 partitions.
    writeSequence(env1, topicName, 100, parallelism);

    readSequence(env2, standardProps, parallelism, topicName, 100, 0);

    CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();

    long o1 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorClient, standardProps.getProperty("group.id"), topicName, 0);
    long o2 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorClient, standardProps.getProperty("group.id"), topicName, 1);
    long o3 =
        ZookeeperOffsetHandler.getOffsetFromZooKeeper(
            curatorClient, standardProps.getProperty("group.id"), topicName, 2);

    LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);

    assertTrue(o1 == FlinkKafkaConsumer08.OFFSET_NOT_SET || (o1 >= 0 && o1 <= 100));
    assertTrue(o2 == FlinkKafkaConsumer08.OFFSET_NOT_SET || (o2 >= 0 && o2 <= 100));
    assertTrue(o3 == FlinkKafkaConsumer08.OFFSET_NOT_SET || (o3 >= 0 && o3 <= 100));

    LOG.info("Manipulating offsets");

    // set the offset to 50 for the three partitions
    ZookeeperOffsetHandler.setOffsetInZooKeeper(
        curatorClient, standardProps.getProperty("group.id"), topicName, 0, 49);
    ZookeeperOffsetHandler.setOffsetInZooKeeper(
        curatorClient, standardProps.getProperty("group.id"), topicName, 1, 49);
    ZookeeperOffsetHandler.setOffsetInZooKeeper(
        curatorClient, standardProps.getProperty("group.id"), topicName, 2, 49);

    curatorClient.close();

    // create new env
    readSequence(env3, standardProps, parallelism, topicName, 50, 50);

    deleteTestTopic(topicName);
  }
コード例 #6
0
ファイル: DataGenerators.java プロジェクト: lizhiyong/flink
  public static void generateRandomizedIntegerSequence(
      StreamExecutionEnvironment env,
      String brokerConnection,
      String topic,
      final int numPartitions,
      final int numElements,
      final boolean randomizeOrder)
      throws Exception {
    env.setParallelism(numPartitions);
    env.getConfig().disableSysoutLogging();
    env.setNumberOfExecutionRetries(0);

    DataStream<Integer> stream =
        env.addSource(
            new RichParallelSourceFunction<Integer>() {

              private volatile boolean running = true;

              @Override
              public void run(SourceContext<Integer> ctx) {
                // create a sequence
                int[] elements = new int[numElements];
                for (int i = 0, val = getRuntimeContext().getIndexOfThisSubtask();
                    i < numElements;
                    i++, val += getRuntimeContext().getNumberOfParallelSubtasks()) {

                  elements[i] = val;
                }

                // scramble the sequence
                if (randomizeOrder) {
                  Random rnd = new Random();
                  for (int i = 0; i < elements.length; i++) {
                    int otherPos = rnd.nextInt(elements.length);

                    int tmp = elements[i];
                    elements[i] = elements[otherPos];
                    elements[otherPos] = tmp;
                  }
                }

                // emit the sequence
                int pos = 0;
                while (running && pos < elements.length) {
                  ctx.collect(elements[pos++]);
                }
              }

              @Override
              public void cancel() {
                running = false;
              }
            });

    stream
        .rebalance()
        .addSink(
            new FlinkKafkaProducer<>(
                topic,
                new TypeInformationSerializationSchema<>(
                    BasicTypeInfo.INT_TYPE_INFO, env.getConfig()),
                FlinkKafkaProducer.getPropertiesFromBrokerList(brokerConnection),
                new KafkaPartitioner() {
                  @Override
                  public int partition(Object key, int numPartitions) {
                    return ((Integer) key) % numPartitions;
                  }
                }));

    env.execute("Scrambles int sequence generator");
  }