private void flush() throws IOException {
if (logger.isDebugEnabled())
logger.debug("araqne logdb: flushing stats buffer, [{}] keys", buffer.keySet().size());
for (List<Object> keys : buffer.keySet()) {
AggregationFunction[] fs = buffer.get(keys);
Object[] l = new Object[fs.length];
int i = 0;
for (AggregationFunction f : fs) l[i++] = f.serialize();
sorter.add(new Item(keys.toArray(), l));
}
buffer.clear();
}
@Override
public void onStart() {
inputCount = 0;
sorter = new ParallelMergeSorter(new ItemComparer());
int queryId = 0;
if (getQuery() != null) queryId = getQuery().getId();
SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd_HHmmss");
sorter.setTag("_" + queryId + "_" + df.format(new Date()) + "_");
this.buffer = new ConcurrentHashMap<List<Object>, AggregationFunction[]>();
for (AggregationFunction f : funcs) f.clean();
}
@Override
public void onClose(QueryStopReason reason) {
// command is not started
if (sorter == null) return;
if (reason != QueryStopReason.End && reason != QueryStopReason.PartialFetch) {
try {
sorter.cancel();
sorter = null;
} catch (Throwable t) {
logger.warn(
"araqne logdb: cannot close stats sorter, query ["
+ getQuery().getId()
+ ":"
+ getQuery().getQueryString()
+ "]",
t);
}
return;
}
logger.debug("araqne logdb: stats sort input count [{}]", inputCount);
CloseableIterator it = null;
try {
// last flush
flush();
// reclaim buffer (GC support)
buffer = new ConcurrentHashMap<List<Object>, AggregationFunction[]>();
// sort
it = sorter.sort();
Object[] lastKeys = null;
AggregationFunction[] fs = null;
Item item = null;
int count = 0;
while (it.hasNext()) {
item = (Item) it.next();
count++;
// first record or need to change merge set?
if (lastKeys == null || !Arrays.equals(lastKeys, (Object[]) item.getKey())) {
if (compareLogger.isDebugEnabled() && lastKeys != null)
compareLogger.debug(
"araqne logdb: stats key compare [{}] != [{}]",
lastKeys[0],
((Object[]) item.getKey())[0]);
// finalize last record (only if changing set)
if (fs != null) {
pass(fs, lastKeys);
}
// load new record
fs = new AggregationFunction[funcs.length];
int i = 0;
Object[] rawFuncs = (Object[]) item.getValue();
for (Object rawFunc : rawFuncs) {
Object[] l = (Object[]) rawFunc;
AggregationFunction f = funcs[i].clone();
f.deserialize(l);
fs[i++] = f;
}
} else {
// merge
int i = 0;
for (AggregationFunction f : fs) {
Object[] l = (Object[]) ((Object[]) item.getValue())[i];
AggregationFunction f2 = funcs[i].clone();
f2.deserialize(l);
f.merge(f2);
i++;
}
}
lastKeys = (Object[]) item.getKey();
}
// write last merge set
if (item != null) pass(fs, lastKeys);
// write result for empty data set (only for no group clause)
if (inputCount == 0 && clauses.size() == 0) {
// write initial function values
pass(funcs, null);
}
logger.debug("araqne logdb: sorted stats input [{}]", count);
} catch (Throwable t) {
getQuery().cancel(t);
throw new IllegalStateException("sort failed, query " + query, t);
} finally {
if (it != null) {
try {
// close and delete final sorted run file
it.close();
} catch (IOException e) {
}
}
// support sorter cache GC when query processing is ended
sorter = null;
}
}