コード例 #1
0
  @Test
  public void testMultiDimensionalArray() {
    // tuple
    TypeInformation<?> ti = TypeInfoParser.parse("Tuple2<Integer, Double>[][]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<ObjectArrayTypeInfo<Java Tuple2<Integer, Double>>>", ti.toString());

    // pojos
    ti =
        TypeInfoParser.parse(
            "org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyPojo<basic=String>[][][]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<"
            + "PojoType<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyPojo, fields = [basic: String]>"
            + ">>>",
        ti.toString());

    // basic types
    ti = TypeInfoParser.parse("Float[][][]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<ObjectArrayTypeInfo<BasicArrayTypeInfo<Float>>>", ti.toString());
    ti = TypeInfoParser.parse("String[][][]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<ObjectArrayTypeInfo<BasicArrayTypeInfo<String>>>", ti.toString());
    ti = TypeInfoParser.parse("Date[][][]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<Date>>>", ti.toString());

    // primitive types
    ti = TypeInfoParser.parse("int[][][]");
    Assert.assertEquals("ObjectArrayTypeInfo<ObjectArrayTypeInfo<int[]>>", ti.toString());
    ti = TypeInfoParser.parse("boolean[][][]");
    Assert.assertEquals("ObjectArrayTypeInfo<ObjectArrayTypeInfo<boolean[]>>", ti.toString());

    // value types
    ti = TypeInfoParser.parse("IntValue[][][]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<ValueType<IntValue>>>>",
        ti.toString());

    // writable types
    ti =
        TypeInfoParser.parse(
            "Writable<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyWritable>[][][]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<"
            + "WritableType<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyWritable>"
            + ">>>",
        ti.toString());

    // enum types
    ti =
        TypeInfoParser.parse(
            "Enum<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>[][][]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<ObjectArrayTypeInfo<ObjectArrayTypeInfo<"
            + "EnumTypeInfo<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>"
            + ">>>",
        ti.toString());
  }
コード例 #2
0
  @Test
  public void testException() {
    try {
      TypeInfoParser.parse("THIS_CLASS_DOES_NOT_EXIST");
      Assert.fail("exception expected");
    } catch (IllegalArgumentException e) {
      // right
    }

    try {
      TypeInfoParser.parse("Tuple2<Integer>");
      Assert.fail("exception expected");
    } catch (IllegalArgumentException e) {
      // right
    }

    try {
      TypeInfoParser.parse("Tuple3<Integer,,>");
      Assert.fail("exception expected");
    } catch (IllegalArgumentException e) {
      // right
    }

    try {
      TypeInfoParser.parse("Tuple1<Integer,Double>");
      Assert.fail("exception expected");
    } catch (IllegalArgumentException e) {
      // right
    }
  }
コード例 #3
0
  @Test
  public void testTuples() {
    TypeInformation<?> ti = TypeInfoParser.parse("Tuple2<Integer, Long>");
    Assert.assertEquals(2, ti.getArity());
    Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, ((TupleTypeInfo<?>) ti).getTypeAt(0));
    Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, ((TupleTypeInfo<?>) ti).getTypeAt(1));

    ti = TypeInfoParser.parse("Tuple3<Tuple1<String>, Tuple1<Integer>, Tuple2<Long, Long>>");
    Assert.assertEquals(
        "Java Tuple3<Java Tuple1<String>, Java Tuple1<Integer>, Java Tuple2<Long, Long>>",
        ti.toString());
  }
コード例 #4
0
  @Test
  public void testEnumType() {
    TypeInformation<?> ti =
        TypeInfoParser.parse("Enum<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>");
    Assert.assertEquals(
        "EnumTypeInfo<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>",
        ti.toString());

    TypeInformation<?> ti2 =
        TypeInfoParser.parse(
            "java.lang.Enum<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>");
    Assert.assertEquals(
        "EnumTypeInfo<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyEnum>",
        ti2.toString());
  }
コード例 #5
0
  @Test
  @SuppressWarnings("unchecked")
  public void testTumblingEventTimeWindowsApply() throws Exception {
    closeCalled.set(0);

    final int WINDOW_SIZE = 3;

    TypeInformation<Tuple2<String, Integer>> inputType =
        TypeInfoParser.parse("Tuple2<String, Integer>");

    ListStateDescriptor<Tuple2<String, Integer>> stateDesc =
        new ListStateDescriptor<>(
            "window-contents", inputType.createSerializer(new ExecutionConfig()));

    WindowOperator<
            String,
            Tuple2<String, Integer>,
            Iterable<Tuple2<String, Integer>>,
            Tuple2<String, Integer>,
            TimeWindow>
        operator =
            new WindowOperator<>(
                TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)),
                new TimeWindow.Serializer(),
                new TupleKeySelector(),
                BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
                stateDesc,
                new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>()),
                EventTimeTrigger.create());

    operator.setInputType(
        TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"),
        new ExecutionConfig());

    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
        testHarness = new OneInputStreamOperatorTestHarness<>(operator);

    testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

    testHarness.open();

    testTumblingEventTimeWindows(testHarness);

    testHarness.close();

    Assert.assertEquals("Close was not called.", 1, closeCalled.get());
  }
コード例 #6
0
 @Test
 public void testWritableType() {
   TypeInformation<?> ti =
       TypeInfoParser.parse(
           "Writable<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyWritable>");
   Assert.assertTrue(ti instanceof WritableTypeInfo<?>);
   Assert.assertEquals(MyWritable.class, ((WritableTypeInfo<?>) ti).getTypeClass());
 }
コード例 #7
0
  @Test
  public void testObjectArrays() {
    TypeInformation<?> ti = TypeInfoParser.parse("java.lang.Class[]");

    Assert.assertTrue(ti instanceof ObjectArrayTypeInfo<?, ?>);
    Assert.assertEquals(Class.class, ((ObjectArrayTypeInfo<?, ?>) ti).getComponentType());

    TypeInformation<?> ti2 = TypeInfoParser.parse("Tuple2<Integer,Double>[]");

    Assert.assertTrue(ti2 instanceof ObjectArrayTypeInfo<?, ?>);
    Assert.assertTrue(
        ((ObjectArrayTypeInfo<?, ?>) ti2).getComponentInfo() instanceof TupleTypeInfo);

    TypeInformation<?> ti3 = TypeInfoParser.parse("Tuple2<Integer[],Double>[]");
    Assert.assertEquals(
        "ObjectArrayTypeInfo<Java Tuple2<BasicArrayTypeInfo<Integer>, Double>>", ti3.toString());
  }
コード例 #8
0
 @Test
 public void testLargeMixedTuple() {
   TypeInformation<?> ti =
       TypeInfoParser.parse(
           "org.apache.flink.api.java.tuple.Tuple4<Double,java.lang.Class[],StringValue,Tuple1<int>>[]");
   Assert.assertEquals(
       "ObjectArrayTypeInfo<Java Tuple4<Double, ObjectArrayTypeInfo<GenericType<java.lang.Class>>, ValueType<StringValue>, Java Tuple1<Integer>>>",
       ti.toString());
 }
コード例 #9
0
ファイル: DataGenerators.java プロジェクト: lizhiyong/flink
  public static void generateLongStringTupleSequence(
      StreamExecutionEnvironment env,
      String brokerConnection,
      String topic,
      int numPartitions,
      final int from,
      final int to)
      throws Exception {

    TypeInformation<Tuple2<Integer, Integer>> resultType =
        TypeInfoParser.parse("Tuple2<Integer, Integer>");

    env.setParallelism(numPartitions);
    env.getConfig().disableSysoutLogging();
    env.setNumberOfExecutionRetries(0);

    DataStream<Tuple2<Integer, Integer>> stream =
        env.addSource(
            new RichParallelSourceFunction<Tuple2<Integer, Integer>>() {

              private volatile boolean running = true;

              @Override
              public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
                int cnt = from;
                int partition = getRuntimeContext().getIndexOfThisSubtask();

                while (running && cnt <= to) {
                  ctx.collect(new Tuple2<Integer, Integer>(partition, cnt));
                  cnt++;
                }
              }

              @Override
              public void cancel() {
                running = false;
              }
            });

    stream.addSink(
        new FlinkKafkaProducer<>(
            topic,
            new TypeInformationSerializationSchema<>(resultType, env.getConfig()),
            FlinkKafkaProducer.getPropertiesFromBrokerList(brokerConnection),
            new Tuple2Partitioner(numPartitions)));

    env.execute("Data generator (Int, Int) stream to topic " + topic);
  }
コード例 #10
0
 @Test
 public void testPojoType2() {
   TypeInformation<?> ti =
       TypeInfoParser.parse(
           "Tuple2<String,Tuple2<Integer,org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyPojo<basic=String>>>");
   Assert.assertTrue(ti instanceof TupleTypeInfo);
   TupleTypeInfo<?> tti = (TupleTypeInfo<?>) ti;
   Assert.assertTrue(tti.getTypeAt(0) instanceof BasicTypeInfo);
   Assert.assertTrue(tti.getTypeAt(1) instanceof TupleTypeInfo);
   TupleTypeInfo<?> tti2 = (TupleTypeInfo<?>) tti.getTypeAt(1);
   Assert.assertTrue(tti2.getTypeAt(0) instanceof BasicTypeInfo);
   Assert.assertTrue(tti2.getTypeAt(1) instanceof PojoTypeInfo);
   PojoTypeInfo<?> pti = (PojoTypeInfo<?>) tti2.getTypeAt(1);
   Assert.assertEquals("basic", pti.getPojoFieldAt(0).field.getName());
   Assert.assertTrue(pti.getPojoFieldAt(0).type instanceof BasicTypeInfo);
 }
コード例 #11
0
  @Test
  @SuppressWarnings("unchecked")
  public void testSlidingEventTimeWindowsReduce() throws Exception {
    closeCalled.set(0);

    final int WINDOW_SIZE = 3;
    final int WINDOW_SLIDE = 1;

    TypeInformation<Tuple2<String, Integer>> inputType =
        TypeInfoParser.parse("Tuple2<String, Integer>");

    ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc =
        new ReducingStateDescriptor<>(
            "window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));

    WindowOperator<
            String,
            Tuple2<String, Integer>,
            Tuple2<String, Integer>,
            Tuple2<String, Integer>,
            TimeWindow>
        operator =
            new WindowOperator<>(
                SlidingEventTimeWindows.of(
                    Time.of(WINDOW_SIZE, TimeUnit.SECONDS),
                    Time.of(WINDOW_SLIDE, TimeUnit.SECONDS)),
                new TimeWindow.Serializer(),
                new TupleKeySelector(),
                BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
                stateDesc,
                new InternalSingleValueWindowFunction<>(
                    new PassThroughWindowFunction<String, TimeWindow, Tuple2<String, Integer>>()),
                EventTimeTrigger.create());

    operator.setInputType(inputType, new ExecutionConfig());

    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
        testHarness = new OneInputStreamOperatorTestHarness<>(operator);

    testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

    testHarness.open();

    testSlidingEventTimeWindows(testHarness);

    testHarness.close();
  }
コード例 #12
0
 @Test
 public void testPojoType() {
   TypeInformation<?> ti =
       TypeInfoParser.parse(
           "org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyPojo<"
               + "basic=Integer,"
               + "tuple=Tuple2<String, Integer>,"
               + "hadoopCitizen=Writable<org.apache.flink.api.java.typeutils.TypeInfoParserTest$MyWritable>,"
               + "array=String[]"
               + ">");
   Assert.assertTrue(ti instanceof PojoTypeInfo);
   PojoTypeInfo<?> pti = (PojoTypeInfo<?>) ti;
   Assert.assertEquals("array", pti.getPojoFieldAt(0).field.getName());
   Assert.assertTrue(pti.getPojoFieldAt(0).type instanceof BasicArrayTypeInfo);
   Assert.assertEquals("basic", pti.getPojoFieldAt(1).field.getName());
   Assert.assertTrue(pti.getPojoFieldAt(1).type instanceof BasicTypeInfo);
   Assert.assertEquals("hadoopCitizen", pti.getPojoFieldAt(2).field.getName());
   Assert.assertTrue(pti.getPojoFieldAt(2).type instanceof WritableTypeInfo);
   Assert.assertEquals("tuple", pti.getPojoFieldAt(3).field.getName());
   Assert.assertTrue(pti.getPojoFieldAt(3).type instanceof TupleTypeInfo);
 }
コード例 #13
0
  @Test
  public void testBasicTypes() {
    Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, TypeInfoParser.parse("Integer"));
    Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, TypeInfoParser.parse("Double"));
    Assert.assertEquals(BasicTypeInfo.BYTE_TYPE_INFO, TypeInfoParser.parse("Byte"));
    Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, TypeInfoParser.parse("Float"));
    Assert.assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, TypeInfoParser.parse("Short"));
    Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, TypeInfoParser.parse("Long"));
    Assert.assertEquals(BasicTypeInfo.CHAR_TYPE_INFO, TypeInfoParser.parse("Character"));
    Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, TypeInfoParser.parse("String"));
    Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, TypeInfoParser.parse("Boolean"));
    Assert.assertEquals(BasicTypeInfo.VOID_TYPE_INFO, TypeInfoParser.parse("Void"));
    Assert.assertEquals(BasicTypeInfo.DATE_TYPE_INFO, TypeInfoParser.parse("Date"));

    Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, TypeInfoParser.parse("java.lang.Integer"));
    Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, TypeInfoParser.parse("java.lang.Double"));
    Assert.assertEquals(BasicTypeInfo.BYTE_TYPE_INFO, TypeInfoParser.parse("java.lang.Byte"));
    Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, TypeInfoParser.parse("java.lang.Float"));
    Assert.assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, TypeInfoParser.parse("java.lang.Short"));
    Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, TypeInfoParser.parse("java.lang.Long"));
    Assert.assertEquals(BasicTypeInfo.CHAR_TYPE_INFO, TypeInfoParser.parse("java.lang.Character"));
    Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, TypeInfoParser.parse("java.lang.String"));
    Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, TypeInfoParser.parse("java.lang.Boolean"));
    Assert.assertEquals(BasicTypeInfo.VOID_TYPE_INFO, TypeInfoParser.parse("java.lang.Void"));
    Assert.assertEquals(BasicTypeInfo.DATE_TYPE_INFO, TypeInfoParser.parse("java.util.Date"));

    Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, TypeInfoParser.parse("int"));
    Assert.assertEquals(BasicTypeInfo.DOUBLE_TYPE_INFO, TypeInfoParser.parse("double"));
    Assert.assertEquals(BasicTypeInfo.BYTE_TYPE_INFO, TypeInfoParser.parse("byte"));
    Assert.assertEquals(BasicTypeInfo.FLOAT_TYPE_INFO, TypeInfoParser.parse("float"));
    Assert.assertEquals(BasicTypeInfo.SHORT_TYPE_INFO, TypeInfoParser.parse("short"));
    Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, TypeInfoParser.parse("long"));
    Assert.assertEquals(BasicTypeInfo.CHAR_TYPE_INFO, TypeInfoParser.parse("char"));
    Assert.assertEquals(BasicTypeInfo.BOOLEAN_TYPE_INFO, TypeInfoParser.parse("boolean"));
    Assert.assertEquals(BasicTypeInfo.VOID_TYPE_INFO, TypeInfoParser.parse("void"));
  }
コード例 #14
0
 @Test
 public void testGenericType() {
   TypeInformation<?> ti = TypeInfoParser.parse("java.lang.Class");
   Assert.assertTrue(ti instanceof GenericTypeInfo);
   Assert.assertEquals(Class.class, ((GenericTypeInfo<?>) ti).getTypeClass());
 }
コード例 #15
0
  @Test
  public void testBasicArrays() {
    Assert.assertEquals(BasicArrayTypeInfo.INT_ARRAY_TYPE_INFO, TypeInfoParser.parse("Integer[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO, TypeInfoParser.parse("Double[]"));
    Assert.assertEquals(BasicArrayTypeInfo.BYTE_ARRAY_TYPE_INFO, TypeInfoParser.parse("Byte[]"));
    Assert.assertEquals(BasicArrayTypeInfo.FLOAT_ARRAY_TYPE_INFO, TypeInfoParser.parse("Float[]"));
    Assert.assertEquals(BasicArrayTypeInfo.SHORT_ARRAY_TYPE_INFO, TypeInfoParser.parse("Short[]"));
    Assert.assertEquals(BasicArrayTypeInfo.LONG_ARRAY_TYPE_INFO, TypeInfoParser.parse("Long[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.CHAR_ARRAY_TYPE_INFO, TypeInfoParser.parse("Character[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, TypeInfoParser.parse("String[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.BOOLEAN_ARRAY_TYPE_INFO, TypeInfoParser.parse("Boolean[]"));

    Assert.assertEquals(
        BasicArrayTypeInfo.INT_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Integer[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Double[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.BYTE_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Byte[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.FLOAT_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Float[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.SHORT_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Short[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.LONG_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Long[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.CHAR_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Character[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.STRING_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.String[]"));
    Assert.assertEquals(
        BasicArrayTypeInfo.BOOLEAN_ARRAY_TYPE_INFO, TypeInfoParser.parse("java.lang.Boolean[]"));

    Assert.assertEquals(
        PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("int[]"));
    Assert.assertEquals(
        PrimitiveArrayTypeInfo.DOUBLE_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("double[]"));
    Assert.assertEquals(
        PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("byte[]"));
    Assert.assertEquals(
        PrimitiveArrayTypeInfo.FLOAT_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("float[]"));
    Assert.assertEquals(
        PrimitiveArrayTypeInfo.SHORT_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("short[]"));
    Assert.assertEquals(
        PrimitiveArrayTypeInfo.LONG_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("long[]"));
    Assert.assertEquals(
        PrimitiveArrayTypeInfo.CHAR_PRIMITIVE_ARRAY_TYPE_INFO, TypeInfoParser.parse("char[]"));
    Assert.assertEquals(
        PrimitiveArrayTypeInfo.BOOLEAN_PRIMITIVE_ARRAY_TYPE_INFO,
        TypeInfoParser.parse("boolean[]"));
  }
コード例 #16
0
 private static void helperValueType(String str, Class<?> clazz) {
   TypeInformation<?> ti = TypeInfoParser.parse(str);
   Assert.assertTrue(ti instanceof ValueTypeInfo);
   ValueTypeInfo<?> vti = (ValueTypeInfo<?>) ti;
   Assert.assertEquals(clazz, vti.getTypeClass());
 }
コード例 #17
0
  @Test
  @SuppressWarnings("unchecked")
  public void testCountTrigger() throws Exception {
    closeCalled.set(0);

    final int WINDOW_SIZE = 4;

    TypeInformation<Tuple2<String, Integer>> inputType =
        TypeInfoParser.parse("Tuple2<String, Integer>");

    ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc =
        new ReducingStateDescriptor<>(
            "window-contents", new SumReducer(), inputType.createSerializer(new ExecutionConfig()));

    WindowOperator<
            String,
            Tuple2<String, Integer>,
            Tuple2<String, Integer>,
            Tuple2<String, Integer>,
            GlobalWindow>
        operator =
            new WindowOperator<>(
                GlobalWindows.create(),
                new GlobalWindow.Serializer(),
                new TupleKeySelector(),
                BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()),
                stateDesc,
                new InternalSingleValueWindowFunction<>(
                    new PassThroughWindowFunction<String, GlobalWindow, Tuple2<String, Integer>>()),
                PurgingTrigger.of(CountTrigger.of(WINDOW_SIZE)));

    operator.setInputType(
        TypeInfoParser.<Tuple2<String, Integer>>parse("Tuple2<String, Integer>"),
        new ExecutionConfig());

    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>>
        testHarness = new OneInputStreamOperatorTestHarness<>(operator);

    testHarness.configureForKeyedStream(new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);

    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

    testHarness.open();

    // The global window actually ignores these timestamps...

    // add elements out-of-order
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 3999));

    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 20));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 999));

    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1998));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));

    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE));

    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10999));

    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 1000));

    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), Long.MAX_VALUE));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 4), Long.MAX_VALUE));

    TestHarnessUtil.assertOutputEqualsSorted(
        "Output was not correct.",
        expectedOutput,
        testHarness.getOutput(),
        new ResultSortComparator());

    testHarness.close();
  }
コード例 #18
0
public class SlidingTimePreReducerTest {

  TypeSerializer<Integer> serializer = TypeExtractor.getForObject(1).createSerializer(null);
  TypeInformation<Tuple2<Integer, Integer>> tupleType =
      TypeInfoParser.parse("Tuple2<Integer,Integer>");

  ReduceFunction<Integer> reducer = new SumReducer();
  ReduceFunction<Tuple2<Integer, Integer>> tupleReducer = new TupleSumReducer();

  @Test
  @SuppressWarnings("unchecked")
  public void testPreReduce1() throws Exception {
    // This ensures that the buffer is properly cleared after a burst of elements by
    // replaying the same sequence of elements with a later timestamp and expecting the same
    // result.

    TestOutput<StreamWindow<Tuple2<Integer, Integer>>> collector =
        new TestOutput<StreamWindow<Tuple2<Integer, Integer>>>();

    SlidingTimePreReducer<Tuple2<Integer, Integer>> preReducer =
        new SlidingTimePreReducer<Tuple2<Integer, Integer>>(
            tupleReducer,
            tupleType.createSerializer(new ExecutionConfig()),
            3,
            2,
            new TimestampWrapper<Tuple2<Integer, Integer>>(
                new Timestamp<Tuple2<Integer, Integer>>() {

                  private static final long serialVersionUID = 1L;

                  @Override
                  public long getTimestamp(Tuple2<Integer, Integer> value) {
                    return value.f0;
                  }
                },
                1));

    int timeOffset = 0;

    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 1, 1));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 2, 2));
    preReducer.emitWindow(collector);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 3, 3));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 4, 4));
    preReducer.evict(1);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 5, 5));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 6, 6));
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 7, 7));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 8, 8));
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 9, 9));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 10, 10));
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 11, 11));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 12, 12));
    preReducer.emitWindow(collector);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 13, 13));

    // ensure that everything is cleared out
    preReducer.evict(100);

    timeOffset = 25; // a little while later...

    // Repeat the same sequence, this should produce the same result
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 1, 1));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 2, 2));
    preReducer.emitWindow(collector);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 3, 3));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 4, 4));
    preReducer.evict(1);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 5, 5));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 6, 6));
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 7, 7));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 8, 8));
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 9, 9));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 10, 10));
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 11, 11));
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 12, 12));
    preReducer.emitWindow(collector);
    preReducer.store(new Tuple2<Integer, Integer>(timeOffset + 13, 13));

    List<StreamWindow<Tuple2<Integer, Integer>>> expected =
        new ArrayList<StreamWindow<Tuple2<Integer, Integer>>>();
    timeOffset = 0; // rewind ...
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 1, 3)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 2, 9)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 4, 15)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 6, 21)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 8, 27)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 10, 33)));

    timeOffset = 25; // and back to the future ...
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 1, 3)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 2, 9)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 4, 15)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 6, 21)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 8, 27)));
    expected.add(StreamWindow.fromElements(new Tuple2<Integer, Integer>(timeOffset + 10, 33)));

    assertEquals(expected, collector.getCollected());
  }

  @Test
  public void testPreReduce2() throws Exception {
    TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();

    SlidingTimePreReducer<Integer> preReducer =
        new SlidingTimePreReducer<Integer>(
            reducer,
            serializer,
            5,
            2,
            new TimestampWrapper<Integer>(
                new Timestamp<Integer>() {

                  private static final long serialVersionUID = 1L;

                  @Override
                  public long getTimestamp(Integer value) {
                    return value;
                  }
                },
                1));

    preReducer.store(1);
    preReducer.store(2);
    preReducer.emitWindow(collector);
    preReducer.store(3);
    preReducer.store(4);
    preReducer.emitWindow(collector);
    preReducer.store(5);
    preReducer.store(6);
    preReducer.evict(1);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(7);
    preReducer.store(8);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(9);
    preReducer.store(10);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(11);
    preReducer.store(12);
    preReducer.emitWindow(collector);
    preReducer.store(13);

    List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
    expected.add(StreamWindow.fromElements(3));
    expected.add(StreamWindow.fromElements(10));
    expected.add(StreamWindow.fromElements(20));
    expected.add(StreamWindow.fromElements(30));
    expected.add(StreamWindow.fromElements(40));
    expected.add(StreamWindow.fromElements(50));

    assertEquals(expected, collector.getCollected());
  }

  @Test
  public void testPreReduce3() throws Exception {
    TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();

    SlidingTimePreReducer<Integer> preReducer =
        new SlidingTimePreReducer<Integer>(
            reducer,
            serializer,
            6,
            3,
            new TimestampWrapper<Integer>(
                new Timestamp<Integer>() {

                  private static final long serialVersionUID = 1L;

                  @Override
                  public long getTimestamp(Integer value) {
                    return value;
                  }
                },
                1));

    preReducer.store(1);
    preReducer.store(2);
    preReducer.store(3);
    preReducer.emitWindow(collector);
    preReducer.store(4);
    preReducer.store(5);
    preReducer.store(6);
    preReducer.emitWindow(collector);
    preReducer.evict(3);
    preReducer.store(7);
    preReducer.store(8);
    preReducer.store(9);
    preReducer.emitWindow(collector);
    preReducer.evict(3);
    preReducer.store(10);
    preReducer.store(11);
    preReducer.store(12);
    preReducer.emitWindow(collector);
    preReducer.evict(3);
    preReducer.store(13);

    List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
    expected.add(StreamWindow.fromElements(6));
    expected.add(StreamWindow.fromElements(21));
    expected.add(StreamWindow.fromElements(39));
    expected.add(StreamWindow.fromElements(57));

    assertEquals(expected, collector.getCollected());
  }

  @Test
  public void testPreReduce4() throws Exception {
    TestOutput<StreamWindow<Integer>> collector = new TestOutput<StreamWindow<Integer>>();

    SlidingTimePreReducer<Integer> preReducer =
        new SlidingTimePreReducer<Integer>(
            reducer,
            serializer,
            3,
            2,
            new TimestampWrapper<Integer>(
                new Timestamp<Integer>() {

                  private static final long serialVersionUID = 1L;

                  @Override
                  public long getTimestamp(Integer value) {
                    return value;
                  }
                },
                1));

    preReducer.store(1);
    preReducer.store(2);
    preReducer.emitWindow(collector);
    preReducer.store(3);
    preReducer.store(4);
    preReducer.evict(1);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(5);
    preReducer.store(6);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(7);
    preReducer.store(8);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.emitWindow(collector);
    preReducer.emitWindow(collector);
    preReducer.evict(2);
    preReducer.store(14);
    preReducer.emitWindow(collector);
    preReducer.emitWindow(collector);
    preReducer.evict(1);
    preReducer.emitWindow(collector);
    preReducer.emitWindow(collector);
    preReducer.store(21);
    preReducer.emitWindow(collector);
    preReducer.evict(1);
    preReducer.emitWindow(collector);

    preReducer.store(9);

    List<StreamWindow<Integer>> expected = new ArrayList<StreamWindow<Integer>>();
    expected.add(StreamWindow.fromElements(3));
    expected.add(StreamWindow.fromElements(9));
    expected.add(StreamWindow.fromElements(15));
    expected.add(StreamWindow.fromElements(21));
    expected.add(StreamWindow.fromElements(8));
    expected.add(StreamWindow.fromElements(8));
    expected.add(StreamWindow.fromElements(14));
    expected.add(StreamWindow.fromElements(14));
    expected.add(StreamWindow.fromElements(21));

    assertEquals(expected, collector.getCollected());
  }

  private static class SumReducer implements ReduceFunction<Integer> {

    private static final long serialVersionUID = 1L;

    @Override
    public Integer reduce(Integer value1, Integer value2) throws Exception {
      return value1 + value2;
    }
  }

  private static class TupleSumReducer implements ReduceFunction<Tuple2<Integer, Integer>> {

    private static final long serialVersionUID = 1L;

    @Override
    public Tuple2<Integer, Integer> reduce(
        Tuple2<Integer, Integer> value1, Tuple2<Integer, Integer> value2) throws Exception {
      return new Tuple2<Integer, Integer>(value1.f0, value1.f1 + value2.f1);
    }
  }
}