@Override
public void onPush(Row m) {
String line = (String) m.get("line");
if (line == null) return;
String sig = makeSignature(line);
m.put("signature", sig);
pushPipe(m);
}
@Override
public void onPush(Row m) {
List<Object> keys = EMPTY_KEY;
if (clauseCount > 0) {
keys = new ArrayList<Object>(clauseCount);
for (String clause : clauses) {
Object keyValue = m.get(clause);
if (discardNullGroup && keyValue == null) return;
keys.add(keyValue);
}
}
try {
inputCount++;
AggregationFunction[] fs = buffer.get(keys);
if (fs == null) {
fs = new AggregationFunction[funcs.length];
for (int i = 0; i < fs.length; i++) fs[i] = funcs[i].clone();
buffer.put(keys, fs);
}
for (AggregationFunction f : fs) f.apply(m);
// flush
if (buffer.size() > 50000) flush();
} catch (IOException e) {
throw new IllegalStateException("stats failed, query " + query, e);
}
}
@Override
public void onPush(RowBatch rowBatch) {
if (rowBatch.selectedInUse) {
for (int i = 0; i < rowBatch.size; i++) {
int p = rowBatch.selected[i];
Row row = rowBatch.rows[p];
String line = (String) row.get("line");
if (line == null) continue;
String sig = makeSignature(line);
row.put("signature", sig);
}
} else {
for (int i = 0; i < rowBatch.size; i++) {
Row row = rowBatch.rows[i];
String line = (String) row.get("line");
if (line == null) continue;
String sig = makeSignature(line);
row.put("signature", sig);
}
}
pushPipe(rowBatch);
}
private void writeLog(Row m) throws IOException {
LineWriter writer = this.writer;
if (usePartition) {
List<String> key = new ArrayList<String>(holders.size());
Date date = m.getDate();
for (PartitionPlaceholder holder : holders) key.add(holder.getKey(date));
PartitionOutput output = outputs.get(key);
if (output == null) {
output = new PartitionOutput(writerFactory, pathToken, tmpPath, date, encoding, overwrite);
outputs.put(key, output);
if (logger.isDebugEnabled())
logger.debug(
"araqne logdb: new partition found key [{}] tmpPath [{}] filePath [{}] date [{}]",
new Object[] {key, tmpPath, pathToken, date});
}
writer = output.getWriter();
}
writer.write(m);
}
@Override
public void onPush(RowBatch rowBatch) {
List<Object> keys = EMPTY_KEY;
if (useClause) keys = new ArrayList<Object>(clauseCount);
if (rowBatch.selectedInUse) {
for (int index = 0; index < rowBatch.size; index++) {
keys.clear();
Row row = rowBatch.rows[rowBatch.selected[index]];
if (useClause) {
boolean isNullGroup = false;
for (String clause : clauses) {
Object keyValue = row.get(clause);
if (discardNullGroup && keyValue == null) {
isNullGroup = true;
break;
}
keys.add(keyValue);
}
if (isNullGroup) continue;
}
inputCount++;
AggregationFunction[] fs = buffer.get(keys);
if (fs == null) {
fs = new AggregationFunction[funcs.length];
for (int i = 0; i < fs.length; i++) fs[i] = funcs[i].clone();
buffer.put(new ArrayList<Object>(keys), fs);
}
for (AggregationFunction f : fs) f.apply(row);
}
} else {
for (int i = 0; i < rowBatch.size; i++) {
Row m = rowBatch.rows[i];
if (useClause) {
keys.clear();
boolean isNullGroup = false;
for (String clause : clauses) {
Object keyValue = m.get(clause);
if (discardNullGroup && keyValue == null) {
isNullGroup = true;
break;
}
keys.add(keyValue);
}
if (isNullGroup) continue;
}
inputCount++;
AggregationFunction[] fs = buffer.get(keys);
if (fs == null) {
fs = new AggregationFunction[funcs.length];
for (int j = 0; j < fs.length; j++) fs[j] = funcs[j].clone();
buffer.put(new ArrayList<Object>(keys), fs);
}
for (AggregationFunction f : fs) f.apply(m);
}
}
try {
// flush
if (buffer.size() > 50000) flush();
} catch (IOException e) {
throw new IllegalStateException("stats failed, query " + query, e);
}
}