@After
  public void shutdownAll() throws Exception {
    // 1st, shutdown sorters
    for (UnilateralSortMerger<?> sorter : this.sorters) {
      if (sorter != null) {
        sorter.close();
      }
    }
    this.sorters.clear();

    // 2nd, shutdown I/O
    this.ioManager.shutdown();
    Assert.assertTrue(
        "I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown());

    // last, verify all memory is returned and shutdown mem manager
    MemoryManager memMan = getMemoryManager();
    if (memMan != null) {
      Assert.assertTrue(
          "Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
      memMan.shutdown();
    }
  }
示例#2
0
  @Test
  public void testSortBothMerge() {
    try {

      Generator generator1 =
          new Generator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
      Generator generator2 =
          new Generator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);

      final TestData.GeneratorIterator input1 =
          new TestData.GeneratorIterator(generator1, INPUT_1_SIZE);
      final TestData.GeneratorIterator input2 =
          new TestData.GeneratorIterator(generator2, INPUT_2_SIZE);

      final JoinFunction matcher = new NoOpMatcher();
      final Collector<Record> collector = new DiscardingOutputCollector<Record>();

      long start = System.nanoTime();

      final UnilateralSortMerger<Record> sorter1 =
          new UnilateralSortMerger<Record>(
              this.memoryManager,
              this.ioManager,
              input1,
              this.parentTask,
              this.serializer1,
              this.comparator1.duplicate(),
              MEMORY_FOR_SORTER,
              128,
              0.8f);

      final UnilateralSortMerger<Record> sorter2 =
          new UnilateralSortMerger<Record>(
              this.memoryManager,
              this.ioManager,
              input2,
              this.parentTask,
              this.serializer2,
              this.comparator2.duplicate(),
              MEMORY_FOR_SORTER,
              128,
              0.8f);

      final MutableObjectIterator<Record> sortedInput1 = sorter1.getIterator();
      final MutableObjectIterator<Record> sortedInput2 = sorter2.getIterator();

      // compare with iterator values
      ReusingMergeInnerJoinIterator<Record, Record, Record> iterator =
          new ReusingMergeInnerJoinIterator<Record, Record, Record>(
              sortedInput1,
              sortedInput2,
              this.serializer1.getSerializer(),
              this.comparator1,
              this.serializer2.getSerializer(),
              this.comparator2,
              this.pairComparator11,
              this.memoryManager,
              this.ioManager,
              MEMORY_PAGES_FOR_MERGE,
              this.parentTask);

      iterator.open();

      while (iterator.callWithNextKey(matcher, collector)) ;

      iterator.close();
      sorter1.close();
      sorter2.close();

      long elapsed = System.nanoTime() - start;
      double msecs = elapsed / (1000 * 1000);

      System.out.println("Sort-Merge Took " + msecs + " msecs.");
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail("An exception occurred during the test: " + e.getMessage());
    }
  }
示例#3
0
  @Override
  public void invoke() throws Exception {
    if (LOG.isDebugEnabled()) {
      LOG.debug(getLogString("Starting data sink operator"));
    }

    ExecutionConfig executionConfig;
    try {
      ExecutionConfig c =
          (ExecutionConfig)
              InstantiationUtil.readObjectFromConfig(
                  getJobConfiguration(), ExecutionConfig.CONFIG_KEY, getUserCodeClassLoader());
      if (c != null) {
        executionConfig = c;
      } else {
        LOG.warn("The execution config returned by the configuration was null");
        executionConfig = new ExecutionConfig();
      }
    } catch (IOException e) {
      throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e);
    } catch (ClassNotFoundException e) {
      throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e);
    }
    boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();

    try {

      // initialize local strategies
      MutableObjectIterator<IT> input1;
      switch (this.config.getInputLocalStrategy(0)) {
        case NONE:
          // nothing to do
          localStrategy = null;
          input1 = reader;
          break;
        case SORT:
          // initialize sort local strategy
          try {
            // get type comparator
            TypeComparatorFactory<IT> compFact =
                this.config.getInputComparator(0, getUserCodeClassLoader());
            if (compFact == null) {
              throw new Exception("Missing comparator factory for local strategy on input " + 0);
            }

            // initialize sorter
            UnilateralSortMerger<IT> sorter =
                new UnilateralSortMerger<IT>(
                    getEnvironment().getMemoryManager(),
                    getEnvironment().getIOManager(),
                    this.reader,
                    this,
                    this.inputTypeSerializerFactory,
                    compFact.createComparator(),
                    this.config.getRelativeMemoryInput(0),
                    this.config.getFilehandlesInput(0),
                    this.config.getSpillingThresholdInput(0));

            this.localStrategy = sorter;
            input1 = sorter.getIterator();
          } catch (Exception e) {
            throw new RuntimeException(
                "Initializing the input processing failed"
                    + (e.getMessage() == null ? "." : ": " + e.getMessage()),
                e);
          }
          break;
        default:
          throw new RuntimeException("Invalid local strategy for DataSinkTask");
      }

      // read the reader and write it to the output

      final TypeSerializer<IT> serializer = this.inputTypeSerializerFactory.getSerializer();
      final MutableObjectIterator<IT> input = input1;
      final OutputFormat<IT> format = this.format;

      // check if task has been canceled
      if (this.taskCanceled) {
        return;
      }

      if (LOG.isDebugEnabled()) {
        LOG.debug(getLogString("Starting to produce output"));
      }

      // open
      format.open(
          this.getEnvironment().getIndexInSubtaskGroup(),
          this.getEnvironment().getNumberOfSubtasks());

      if (objectReuseEnabled) {
        IT record = serializer.createInstance();

        // work!
        while (!this.taskCanceled && ((record = input.next(record)) != null)) {
          format.writeRecord(record);
        }
      } else {
        IT record;

        // work!
        while (!this.taskCanceled && ((record = input.next()) != null)) {
          format.writeRecord(record);
        }
      }

      // close. We close here such that a regular close throwing an exception marks a task as
      // failed.
      if (!this.taskCanceled) {
        this.format.close();
        this.format = null;
      }
    } catch (Exception ex) {

      // make a best effort to clean up
      try {
        if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
          cleanupCalled = true;
          ((CleanupWhenUnsuccessful) format).tryCleanupOnError();
        }
      } catch (Throwable t) {
        LOG.error("Cleanup on error failed.", t);
      }

      ex = ExceptionInChainedStubException.exceptionUnwrap(ex);

      if (ex instanceof CancelTaskException) {
        // forward canceling exception
        throw ex;
      }
      // drop, if the task was canceled
      else if (!this.taskCanceled) {
        if (LOG.isErrorEnabled()) {
          LOG.error(getLogString("Error in user code: " + ex.getMessage()), ex);
        }
        throw ex;
      }
    } finally {
      if (this.format != null) {
        // close format, if it has not been closed, yet.
        // This should only be the case if we had a previous error, or were canceled.
        try {
          this.format.close();
        } catch (Throwable t) {
          if (LOG.isWarnEnabled()) {
            LOG.warn(getLogString("Error closing the output format"), t);
          }
        }
      }
      // close local strategy if necessary
      if (localStrategy != null) {
        try {
          this.localStrategy.close();
        } catch (Throwable t) {
          LOG.error("Error closing local strategy", t);
        }
      }

      RegularPactTask.clearReaders(new MutableReader[] {inputReader});
    }

    if (!this.taskCanceled) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(getLogString("Finished data sink operator"));
      }
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug(getLogString("Data sink operator cancelled"));
      }
    }
  }