Exemple #1
0
  @Override
  public void onPush(Row m) {
    List<Object> keys = EMPTY_KEY;
    if (clauseCount > 0) {
      keys = new ArrayList<Object>(clauseCount);

      for (String clause : clauses) {
        Object keyValue = m.get(clause);
        if (discardNullGroup && keyValue == null) return;

        keys.add(keyValue);
      }
    }

    try {
      inputCount++;

      AggregationFunction[] fs = buffer.get(keys);
      if (fs == null) {
        fs = new AggregationFunction[funcs.length];
        for (int i = 0; i < fs.length; i++) fs[i] = funcs[i].clone();

        buffer.put(keys, fs);
      }

      for (AggregationFunction f : fs) f.apply(m);

      // flush
      if (buffer.size() > 50000) flush();

    } catch (IOException e) {
      throw new IllegalStateException("stats failed, query " + query, e);
    }
  }
Exemple #2
0
  private void flush() throws IOException {
    if (logger.isDebugEnabled())
      logger.debug("araqne logdb: flushing stats buffer, [{}] keys", buffer.keySet().size());

    for (List<Object> keys : buffer.keySet()) {
      AggregationFunction[] fs = buffer.get(keys);
      Object[] l = new Object[fs.length];
      int i = 0;
      for (AggregationFunction f : fs) l[i++] = f.serialize();

      sorter.add(new Item(keys.toArray(), l));
    }

    buffer.clear();
  }
Exemple #3
0
  @Override
  public void onStart() {
    inputCount = 0;
    sorter = new ParallelMergeSorter(new ItemComparer());

    int queryId = 0;
    if (getQuery() != null) queryId = getQuery().getId();

    SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd_HHmmss");
    sorter.setTag("_" + queryId + "_" + df.format(new Date()) + "_");

    this.buffer = new ConcurrentHashMap<List<Object>, AggregationFunction[]>();

    for (AggregationFunction f : funcs) f.clean();
  }
Exemple #4
0
  @Override
  public void onClose(QueryStopReason reason) {
    // command is not started
    if (sorter == null) return;

    if (reason != QueryStopReason.End && reason != QueryStopReason.PartialFetch) {
      try {
        sorter.cancel();
        sorter = null;
      } catch (Throwable t) {
        logger.warn(
            "araqne logdb: cannot close stats sorter, query ["
                + getQuery().getId()
                + ":"
                + getQuery().getQueryString()
                + "]",
            t);
      }

      return;
    }

    logger.debug("araqne logdb: stats sort input count [{}]", inputCount);
    CloseableIterator it = null;
    try {
      // last flush
      flush();

      // reclaim buffer (GC support)
      buffer = new ConcurrentHashMap<List<Object>, AggregationFunction[]>();

      // sort
      it = sorter.sort();

      Object[] lastKeys = null;
      AggregationFunction[] fs = null;
      Item item = null;
      int count = 0;
      while (it.hasNext()) {
        item = (Item) it.next();
        count++;

        // first record or need to change merge set?
        if (lastKeys == null || !Arrays.equals(lastKeys, (Object[]) item.getKey())) {
          if (compareLogger.isDebugEnabled() && lastKeys != null)
            compareLogger.debug(
                "araqne logdb: stats key compare [{}] != [{}]",
                lastKeys[0],
                ((Object[]) item.getKey())[0]);

          // finalize last record (only if changing set)
          if (fs != null) {
            pass(fs, lastKeys);
          }

          // load new record
          fs = new AggregationFunction[funcs.length];
          int i = 0;
          Object[] rawFuncs = (Object[]) item.getValue();
          for (Object rawFunc : rawFuncs) {
            Object[] l = (Object[]) rawFunc;
            AggregationFunction f = funcs[i].clone();
            f.deserialize(l);
            fs[i++] = f;
          }
        } else {
          // merge
          int i = 0;
          for (AggregationFunction f : fs) {
            Object[] l = (Object[]) ((Object[]) item.getValue())[i];
            AggregationFunction f2 = funcs[i].clone();
            f2.deserialize(l);
            f.merge(f2);
            i++;
          }
        }

        lastKeys = (Object[]) item.getKey();
      }

      // write last merge set
      if (item != null) pass(fs, lastKeys);

      // write result for empty data set (only for no group clause)
      if (inputCount == 0 && clauses.size() == 0) {
        // write initial function values
        pass(funcs, null);
      }

      logger.debug("araqne logdb: sorted stats input [{}]", count);
    } catch (Throwable t) {
      getQuery().cancel(t);
      throw new IllegalStateException("sort failed, query " + query, t);
    } finally {
      if (it != null) {
        try {
          // close and delete final sorted run file
          it.close();
        } catch (IOException e) {
        }
      }

      // support sorter cache GC when query processing is ended
      sorter = null;
    }
  }
Exemple #5
0
  @Override
  public void onPush(RowBatch rowBatch) {
    List<Object> keys = EMPTY_KEY;

    if (useClause) keys = new ArrayList<Object>(clauseCount);

    if (rowBatch.selectedInUse) {
      for (int index = 0; index < rowBatch.size; index++) {
        keys.clear();
        Row row = rowBatch.rows[rowBatch.selected[index]];
        if (useClause) {
          boolean isNullGroup = false;
          for (String clause : clauses) {
            Object keyValue = row.get(clause);
            if (discardNullGroup && keyValue == null) {
              isNullGroup = true;
              break;
            }

            keys.add(keyValue);
          }

          if (isNullGroup) continue;
        }

        inputCount++;

        AggregationFunction[] fs = buffer.get(keys);
        if (fs == null) {
          fs = new AggregationFunction[funcs.length];
          for (int i = 0; i < fs.length; i++) fs[i] = funcs[i].clone();

          buffer.put(new ArrayList<Object>(keys), fs);
        }

        for (AggregationFunction f : fs) f.apply(row);
      }
    } else {
      for (int i = 0; i < rowBatch.size; i++) {
        Row m = rowBatch.rows[i];
        if (useClause) {
          keys.clear();
          boolean isNullGroup = false;
          for (String clause : clauses) {
            Object keyValue = m.get(clause);
            if (discardNullGroup && keyValue == null) {
              isNullGroup = true;
              break;
            }

            keys.add(keyValue);
          }

          if (isNullGroup) continue;
        }

        inputCount++;

        AggregationFunction[] fs = buffer.get(keys);
        if (fs == null) {
          fs = new AggregationFunction[funcs.length];
          for (int j = 0; j < fs.length; j++) fs[j] = funcs[j].clone();

          buffer.put(new ArrayList<Object>(keys), fs);
        }

        for (AggregationFunction f : fs) f.apply(m);
      }
    }

    try {
      // flush
      if (buffer.size() > 50000) flush();
    } catch (IOException e) {
      throw new IllegalStateException("stats failed, query " + query, e);
    }
  }