@Override
    public Tuple2<Integer, Integer> reduce(
        Tuple2<Integer, Integer> value1, Tuple2<Integer, Integer> value2) throws Exception {
      state.update(state.value() + 1);
      globalCounts.put(value1.f0, state.value());

      return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
    }
    @Override
    public void apply(
        Integer key, TimeWindow window, Iterable<Integer> values, Collector<Integer> out)
        throws Exception {
      for (Integer i : values) {
        // we need to update this state before emitting elements. Else, the test's main
        // thread will have received all output elements before the state is updated and
        // the checks may fail
        state.update(state.value() + 1);
        globalCounts.put(key, state.value());

        out.collect(i);
      }
    }
  @Override
  public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx)
      throws Exception {

    OperatorState<Boolean> first = ctx.getKeyValueState("first", true);

    if (first.value()) {
      long start = timestamp - (timestamp % interval);
      long nextFireTimestamp = start + interval;

      ctx.registerEventTimeTimer(nextFireTimestamp);

      first.update(false);
      return TriggerResult.CONTINUE;
    }
    return TriggerResult.CONTINUE;
  }
    @Override
    public void run(SourceContext<String> ctx) throws Exception {
      final Object lockingObject = ctx.getCheckpointLock();

      while (isRunning && index.value() < numElements) {
        char first = (char) ((index.value() % 40) + 40);

        stringBuilder.setLength(0);
        stringBuilder.append(first);

        String result = randomString(stringBuilder, rnd);

        synchronized (lockingObject) {
          index.update(index.value() + step);
          ctx.collect(result);
        }
      }
    }
Beispiel #5
0
  @Override
  public void run(SourceContext<OUT> ctx) throws Exception {
    if (iteratorToRead == null) {
      throw new IllegalStateException("Kafka iterator not initialized properly.");
    }

    final Object checkpointLock = ctx.getCheckpointLock();

    while (running && iteratorToRead.hasNext()) {
      MessageAndMetadata<byte[], byte[]> message = iteratorToRead.next();
      if (lastOffsets.getState()[message.partition()] >= message.offset()) {
        LOG.info(
            "Skipping message with offset {} from partition {}",
            message.offset(),
            message.partition());
        continue;
      }
      OUT next = deserializationSchema.deserialize(message.message());

      if (deserializationSchema.isEndOfStream(next)) {
        LOG.info("DeserializationSchema signaled end of stream for this source");
        break;
      }

      // make the state update and the element emission atomic
      synchronized (checkpointLock) {
        lastOffsets.getState()[message.partition()] = message.offset();
        ctx.collect(next);
      }

      if (LOG.isTraceEnabled()) {
        LOG.trace(
            "Processed record with offset {} from partition {}",
            message.offset(),
            message.partition());
      }
    }
  }
 @Override
 public void close() throws IOException {
   counts[getRuntimeContext().getIndexOfThisSubtask()] = count.value();
 }
 @Override
 public PrefixCount map(String value) throws IOException {
   count.update(count.value() + 1);
   return new PrefixCount(value.substring(0, 1), value, 1L);
 }
 @Override
 public PrefixCount map(PrefixCount value) throws Exception {
   count.update(count.value() + 1);
   return value;
 }
Beispiel #9
0
  @Override
  public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    ConsumerConnector consumer = Consumer.createJavaConsumerConnector(this.consumerConfig);
    // we request only one stream per consumer instance. Kafka will make sure that each consumer
    // group
    // will see each message only once.
    Map<String, Integer> topicCountMap = Collections.singletonMap(topicName, 1);
    Map<String, List<KafkaStream<byte[], byte[]>>> streams =
        consumer.createMessageStreams(topicCountMap);
    if (streams.size() != 1) {
      throw new RuntimeException("Expected only one message stream but got " + streams.size());
    }
    List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
    if (kafkaStreams == null) {
      throw new RuntimeException(
          "Requested stream not available. Available streams: " + streams.toString());
    }
    if (kafkaStreams.size() != 1) {
      throw new RuntimeException(
          "Requested 1 stream from Kafka, bot got " + kafkaStreams.size() + " streams");
    }
    LOG.info(
        "Opening Consumer instance for topic '{}' on group '{}'",
        topicName,
        consumerConfig.groupId());
    this.iteratorToRead = kafkaStreams.get(0).iterator();
    this.consumer = consumer;

    zkClient =
        new ZkClient(
            consumerConfig.zkConnect(),
            consumerConfig.zkSessionTimeoutMs(),
            consumerConfig.zkConnectionTimeoutMs(),
            new KafkaZKStringSerializer());

    // most likely the number of offsets we're going to store here will be lower than the number of
    // partitions.
    int numPartitions = getNumberOfPartitions();
    LOG.debug("The topic {} has {} partitions", topicName, numPartitions);
    this.lastOffsets =
        getRuntimeContext().getOperatorState("offset", new long[numPartitions], false);
    this.commitedOffsets = new long[numPartitions];
    // check if there are offsets to restore
    if (!Arrays.equals(lastOffsets.getState(), new long[numPartitions])) {
      if (lastOffsets.getState().length != numPartitions) {
        throw new IllegalStateException(
            "There are "
                + lastOffsets.getState().length
                + " offsets to restore for topic "
                + topicName
                + " but "
                + "there are only "
                + numPartitions
                + " in the topic");
      }

      LOG.info("Setting restored offsets {} in ZooKeeper", Arrays.toString(lastOffsets.getState()));
      setOffsetsInZooKeeper(lastOffsets.getState());
    } else {
      // initialize empty offsets
      Arrays.fill(this.lastOffsets.getState(), -1);
    }
    Arrays.fill(this.commitedOffsets, 0); // just to make it clear

    running = true;
  }