Ejemplo n.º 1
0
    @Override
    public void notifyWriteSpillBegin(Spillable spillable, int spillSize, String spillReason) {
      int numFiles = spillable.spillCount();

      if (numFiles % 10 == 0) {
        LOG.info(
            "spilling group: {}, on grouping: {}, num times: {}, with reason: {}",
            new Object[] {
              joinField.printVerbose(), spillable.getGrouping().print(), numFiles + 1, spillReason
            });

        Runtime runtime = Runtime.getRuntime();
        long freeMem = runtime.freeMemory() / 1024 / 1024;
        long maxMem = runtime.maxMemory() / 1024 / 1024;
        long totalMem = runtime.totalMemory() / 1024 / 1024;

        LOG.info(
            "mem on spill (mb), free: " + freeMem + ", total: " + totalMem + ", max: " + maxMem);
      }

      LOG.info("spilling {} tuples in list to file number {}", spillSize, numFiles + 1);

      flowProcess.increment(Spill.Num_Spills_Written, 1);
      flowProcess.increment(Spill.Num_Tuples_Spilled, spillSize);
    }
Ejemplo n.º 2
0
  protected void clearGroups() {
    for (Collection<Tuple> collection : collections) {
      collection.clear();

      if (collection instanceof Spillable) ((Spillable) collection).setGrouping(grouping);
    }
  }
Ejemplo n.º 3
0
  protected Collection<Tuple> createTupleCollection(Fields joinField) {
    Collection<Tuple> collection = tupleCollectionFactory.create(flowProcess);

    if (collection instanceof Spillable)
      ((Spillable) collection).setSpillListener(createListener(joinField));

    return collection;
  }