@Override
    public void writeRecord(Record rec) throws IOException {
      IntValue key = rec.getField(0, IntValue.class);
      IntValue value = rec.getField(1, IntValue.class);

      this.bld.setLength(0);
      this.bld.append(key.getValue());
      this.bld.append('_');
      this.bld.append(value.getValue());
      this.bld.append('\n');

      byte[] bytes = this.bld.toString().getBytes();

      this.stream.write(bytes);
    }
示例#2
0
    public static void prepareInputFile(
        MutableObjectIterator<Record> inIt, String inputFilePath, boolean insertInvalidData)
        throws IOException {
      FileWriter fw = new FileWriter(inputFilePath);
      BufferedWriter bw = new BufferedWriter(fw);

      if (insertInvalidData) {
        bw.write("####_I_AM_INVALID_########\n");
      }

      Record rec = new Record();
      while ((rec = inIt.next(rec)) != null) {
        IntValue key = rec.getField(0, IntValue.class);
        IntValue value = rec.getField(1, IntValue.class);

        bw.write(key.getValue() + "_" + value.getValue() + "\n");
      }
      if (insertInvalidData) {
        bw.write("####_I_AM_INVALID_########\n");
      }

      bw.flush();
      bw.close();
    }
    @Override
    public void invoke() throws Exception {
      RecordReader<IntValue> reader =
          new RecordReader<>(getEnvironment().getInputGate(0), IntValue.class);

      try {
        final int numberOfSubtaskIndexesToReceive =
            getTaskConfiguration().getInteger(CONFIG_KEY, 0);
        final BitSet receivedSubtaskIndexes = new BitSet(numberOfSubtaskIndexesToReceive);

        IntValue record;

        int numberOfReceivedSubtaskIndexes = 0;

        while ((record = reader.next()) != null) {
          // Check that we don't receive more than expected
          numberOfReceivedSubtaskIndexes++;

          if (numberOfReceivedSubtaskIndexes > numberOfSubtaskIndexesToReceive) {
            throw new IllegalStateException("Received more records than expected.");
          }

          int subtaskIndex = record.getValue();

          // Check that we only receive each subtask index once
          if (receivedSubtaskIndexes.get(subtaskIndex)) {
            throw new IllegalStateException("Received expected subtask index twice.");
          } else {
            receivedSubtaskIndexes.set(subtaskIndex, true);
          }
        }

        // Check that we have received all expected subtask indexes
        if (receivedSubtaskIndexes.cardinality() != numberOfSubtaskIndexesToReceive) {
          throw new IllegalStateException(
              "Finished receive, but did not receive " + "all expected subtask indexes.");
        }
      } finally {
        reader.clearBuffers();
      }
    }