private void checkpoint(boolean last) throws PEException {
final long total = globalCounters.getOccurances(GlobalCounters.LINES);
if (last || ((total % checkpointInterval) == 0)) {
outputStream.println("Writing checkpoint file, total stmts: " + total);
final String corpusString = PEXmlUtils.marshalJAXB(corpus);
final File prevFile = new File(checkpointFile.getAbsolutePath() + ".prev");
if (prevFile.exists()) {
prevFile.delete();
}
if (checkpointFile.exists()) {
checkpointFile.renameTo(prevFile);
}
PEFileUtils.writeToFile(checkpointFile, corpusString, true);
}
if (last) {
final String corpusString = PEXmlUtils.marshalJAXB(sortByFrequency());
PEFileUtils.writeToFile(corpusFile, corpusString, true);
printSummary(outputStream);
}
flushErrorLog();
}
/** This is a simple format: sql: <the sql> error | warn: <the message> */
private void logError(String sql, SourcePosition sp, String message, boolean error) {
if (error) {
globalCounters.increment(GlobalCounters.ERRORS);
} else {
globalCounters.increment(GlobalCounters.WARNINGS);
}
try {
final String type = error ? "error" : "warn";
final String line = "(line=" + sp.getPosition() + ")";
errorLog.write("---- " + type + " " + line + " ---------------");
errorLog.newLine();
errorLog.write("msg: " + message);
errorLog.newLine();
errorLog.write("sql: " + sql);
errorLog.newLine();
} catch (final Throwable ct) {
logger.error("Unable to write error log record", ct);
}
}
private void trackNonDML(Statement s, String sql) {
String rep = null;
if (s instanceof SchemaQueryStatement) {
final SchemaQueryStatement sqs = (SchemaQueryStatement) s;
rep = "show " + sqs.getTag();
} else {
rep = sql;
}
nonDMLBreakout.increment(lookupNonDMLCounter(s));
trackOther(rep);
}
private static void printSummary(final PrintStream outputStream) {
nonDMLBreakout.printTo("\nNon-DML Breakout : %s\n", " %-20s : %s\n", outputStream);
dmlBreakout.printTo("\nDML Breakout : %s\n", " %-20s : %s\n", outputStream);
outputStream.println();
outputStream.println("Literal counts / shrunk DML statements");
outputStream.printf(" %-20s : %s\n", "count", literalCounts.getTotalOccurances());
outputStream.printf(" %-20s : %s\n", "min", literalCounts.getMinimum());
outputStream.printf(" %-20s : %4.4f\n", "avg", literalCounts.getAverage());
outputStream.printf(
" %-20s : %4.4f\n", "std dev(approx)", literalCounts.getStandardDeviation());
outputStream.printf(" %-20s : %s\n", "90.0%", literalCounts.getPercentile(0.900d));
outputStream.printf(" %-20s : %s\n", "99.0%", literalCounts.getPercentile(0.990d));
outputStream.printf(" %-20s : %s\n", "99.9%", literalCounts.getPercentile(0.999d));
outputStream.printf(
" %-20s : %8.8f%%\n", "percentile of 100", literalCounts.findPercentile(100) * 100.0d);
outputStream.printf(" %-20s : %s\n", "max", literalCounts.getMaximum());
globalCounters.printTo("\nSummary : \n", " %-20s : %s\n", outputStream);
}
@Override
public boolean omit(LineInfo info, String sql) {
globalCounters.increment(GlobalCounters.LINES);
final StatementCounter sc = (StatementCounter) sink;
if (CandidateParser.isInsert(sql)) {
return false;
}
final CandidateParser cp = new CandidateParser(sql);
if (cp.shrink()) {
final String shrunk = cp.getShrunk();
final NonInsertEntry se = sc.byShrunk.get(shrunk);
if (se != null) {
globalCounters.increment(GlobalCounters.SHRINK_CACHE_HITS);
sc.saveIntermediateCheckpoint();
se.bump(dmlBreakout, literalCounts);
return true;
}
globalCounters.increment(GlobalCounters.SHRINK_CACHE_MISSES);
}
return false;
}
public void bump(EnumStatsCollector<DMLCounters> dmlBreakout, IntegerHistogram literalCounts) {
populationObject.setFreq(populationObject.getFreq() + 1);
dmlBreakout.increment(dmlCategory);
literalCounts.sample(populationObject.getLiteralCount(), 1L);
}
@Override
public void onStatement(String sql, SourcePosition sp, Statement s) throws Throwable {
saveIntermediateCheckpoint();
globalCounters.increment(GlobalCounters.PROCESSED_STATEMENTS);
if (s instanceof EmptyStatement) {
globalCounters.increment(GlobalCounters.EMPTY_STATEMENTS);
return;
}
if (!s.isDML()) {
trackNonDML(s, sql);
return;
}
final Emitter emitter = Singletons.require(HostService.class).getDBNative().getEmitter();
emitter.setOptions(emitOptions);
final StringBuilder buf = new StringBuilder();
final DMLStatement dmls = (DMLStatement) s;
final DMLCounters incCounter = lookupDMLCounter(dmls);
if (dmls instanceof InsertIntoValuesStatement) {
final InsertIntoValuesStatement iivs = (InsertIntoValuesStatement) dmls;
emitter.emitInsertPrefix(tee.getPersistenceContext(), iivs, buf);
final String prefix = buf.toString();
InsertEntry ie = inserts.get(prefix);
if (ie == null) {
final Database<?> db = dmls.getDatabase(tee.getPersistenceContext());
ie =
new InsertEntry(
corpus,
prefix,
iivs.getColumnSpecification().size(),
iivs.getClass().getSimpleName(),
(db == null ? null : db.getName().get()));
inserts.put(prefix, ie);
}
ie.bump(iivs.getValues().size());
} else {
emitter.emitStatement(tee.getPersistenceContext(), dmls, buf);
final String p = buf.toString();
NonInsertEntry se = byParam.get(p);
if (se == null) {
String shrunk = null;
int litCount = -1;
if (!(dmls instanceof InsertIntoValuesStatement)) {
final CandidateParser cp = new CandidateParser(sql);
if (cp.shrink()) {
shrunk = cp.getShrunk();
// also verify we get the same number of literals
final ValueManager valueManager = tee.getPersistenceContext().getValueManager();
litCount = cp.getLiterals().size();
if (litCount != valueManager.getNumberOfLiterals()) {
final ValueManager.CacheStatus cacheStatus = valueManager.getCacheStatus();
String reason;
switch (cacheStatus) {
case NOCACHE_DYNAMIC_FUNCTION:
reason = "contains a non-cacheable dynamic function";
break;
case NOCACHE_TOO_MANY_LITERALS:
reason = "literal count exceeded configured max_cached_plan_literals";
break;
case CACHEABLE:
default:
reason = "unknown";
}
logError(
sql,
sp,
"Mismatched literal size; parse="
+ valueManager.getNumberOfLiterals()
+ "/shrink="
+ litCount
+ " , reason="
+ reason,
false);
}
} else {
logError(sql, sp, "Unable to shrink", false);
}
}
final Database<?> db = dmls.getDatabase(tee.getPersistenceContext());
se =
new NonInsertEntry(
corpus,
sql,
dmls.getClass().getSimpleName(),
(db == null ? null : db.getName().get()),
litCount,
incCounter);
byParam.put(p, se);
if (shrunk != null) {
globalCounters.increment(GlobalCounters.SHRINK_CACHE_ADDS);
byShrunk.put(shrunk, se);
}
}
if (se.populationObject.getLiteralCount() >= 0) {
literalCounts.sample(se.populationObject.getLiteralCount());
}
se.bump(dmlBreakout, literalCounts);
}
}