Ejemplo n.º 1
0
  private void checkpoint(boolean last) throws PEException {
    final long total = globalCounters.getOccurances(GlobalCounters.LINES);

    if (last || ((total % checkpointInterval) == 0)) {
      outputStream.println("Writing checkpoint file, total stmts: " + total);

      final String corpusString = PEXmlUtils.marshalJAXB(corpus);

      final File prevFile = new File(checkpointFile.getAbsolutePath() + ".prev");
      if (prevFile.exists()) {
        prevFile.delete();
      }

      if (checkpointFile.exists()) {
        checkpointFile.renameTo(prevFile);
      }

      PEFileUtils.writeToFile(checkpointFile, corpusString, true);
    }

    if (last) {
      final String corpusString = PEXmlUtils.marshalJAXB(sortByFrequency());
      PEFileUtils.writeToFile(corpusFile, corpusString, true);
      printSummary(outputStream);
    }

    flushErrorLog();
  }
Ejemplo n.º 2
0
  /** This is a simple format: sql: <the sql> error | warn: <the message> */
  private void logError(String sql, SourcePosition sp, String message, boolean error) {
    if (error) {
      globalCounters.increment(GlobalCounters.ERRORS);
    } else {
      globalCounters.increment(GlobalCounters.WARNINGS);
    }

    try {
      final String type = error ? "error" : "warn";
      final String line = "(line=" + sp.getPosition() + ")";
      errorLog.write("---- " + type + " " + line + " ---------------");
      errorLog.newLine();
      errorLog.write("msg: " + message);
      errorLog.newLine();
      errorLog.write("sql: " + sql);
      errorLog.newLine();
    } catch (final Throwable ct) {
      logger.error("Unable to write error log record", ct);
    }
  }
Ejemplo n.º 3
0
  private void trackNonDML(Statement s, String sql) {
    String rep = null;
    if (s instanceof SchemaQueryStatement) {
      final SchemaQueryStatement sqs = (SchemaQueryStatement) s;
      rep = "show " + sqs.getTag();
    } else {
      rep = sql;
    }

    nonDMLBreakout.increment(lookupNonDMLCounter(s));
    trackOther(rep);
  }
Ejemplo n.º 4
0
  private static void printSummary(final PrintStream outputStream) {
    nonDMLBreakout.printTo("\nNon-DML Breakout         : %s\n", "    %-20s : %s\n", outputStream);

    dmlBreakout.printTo("\nDML Breakout             : %s\n", "    %-20s : %s\n", outputStream);

    outputStream.println();
    outputStream.println("Literal counts / shrunk DML statements");
    outputStream.printf("    %-20s : %s\n", "count", literalCounts.getTotalOccurances());
    outputStream.printf("    %-20s : %s\n", "min", literalCounts.getMinimum());
    outputStream.printf("    %-20s : %4.4f\n", "avg", literalCounts.getAverage());
    outputStream.printf(
        "    %-20s : %4.4f\n", "std dev(approx)", literalCounts.getStandardDeviation());
    outputStream.printf("    %-20s : %s\n", "90.0%", literalCounts.getPercentile(0.900d));
    outputStream.printf("    %-20s : %s\n", "99.0%", literalCounts.getPercentile(0.990d));
    outputStream.printf("    %-20s : %s\n", "99.9%", literalCounts.getPercentile(0.999d));
    outputStream.printf(
        "    %-20s : %8.8f%%\n", "percentile of 100", literalCounts.findPercentile(100) * 100.0d);
    outputStream.printf("    %-20s : %s\n", "max", literalCounts.getMaximum());

    globalCounters.printTo("\nSummary                  : \n", "    %-20s : %s\n", outputStream);
  }
Ejemplo n.º 5
0
    @Override
    public boolean omit(LineInfo info, String sql) {
      globalCounters.increment(GlobalCounters.LINES);
      final StatementCounter sc = (StatementCounter) sink;
      if (CandidateParser.isInsert(sql)) {
        return false;
      }
      final CandidateParser cp = new CandidateParser(sql);
      if (cp.shrink()) {
        final String shrunk = cp.getShrunk();
        final NonInsertEntry se = sc.byShrunk.get(shrunk);
        if (se != null) {
          globalCounters.increment(GlobalCounters.SHRINK_CACHE_HITS);
          sc.saveIntermediateCheckpoint();
          se.bump(dmlBreakout, literalCounts);
          return true;
        }

        globalCounters.increment(GlobalCounters.SHRINK_CACHE_MISSES);
      }
      return false;
    }
Ejemplo n.º 6
0
 public void bump(EnumStatsCollector<DMLCounters> dmlBreakout, IntegerHistogram literalCounts) {
   populationObject.setFreq(populationObject.getFreq() + 1);
   dmlBreakout.increment(dmlCategory);
   literalCounts.sample(populationObject.getLiteralCount(), 1L);
 }
Ejemplo n.º 7
0
  @Override
  public void onStatement(String sql, SourcePosition sp, Statement s) throws Throwable {
    saveIntermediateCheckpoint();
    globalCounters.increment(GlobalCounters.PROCESSED_STATEMENTS);
    if (s instanceof EmptyStatement) {
      globalCounters.increment(GlobalCounters.EMPTY_STATEMENTS);
      return;
    }
    if (!s.isDML()) {
      trackNonDML(s, sql);
      return;
    }
    final Emitter emitter = Singletons.require(HostService.class).getDBNative().getEmitter();
    emitter.setOptions(emitOptions);
    final StringBuilder buf = new StringBuilder();
    final DMLStatement dmls = (DMLStatement) s;

    final DMLCounters incCounter = lookupDMLCounter(dmls);

    if (dmls instanceof InsertIntoValuesStatement) {
      final InsertIntoValuesStatement iivs = (InsertIntoValuesStatement) dmls;
      emitter.emitInsertPrefix(tee.getPersistenceContext(), iivs, buf);
      final String prefix = buf.toString();
      InsertEntry ie = inserts.get(prefix);
      if (ie == null) {
        final Database<?> db = dmls.getDatabase(tee.getPersistenceContext());
        ie =
            new InsertEntry(
                corpus,
                prefix,
                iivs.getColumnSpecification().size(),
                iivs.getClass().getSimpleName(),
                (db == null ? null : db.getName().get()));
        inserts.put(prefix, ie);
      }
      ie.bump(iivs.getValues().size());
    } else {
      emitter.emitStatement(tee.getPersistenceContext(), dmls, buf);
      final String p = buf.toString();
      NonInsertEntry se = byParam.get(p);
      if (se == null) {
        String shrunk = null;
        int litCount = -1;
        if (!(dmls instanceof InsertIntoValuesStatement)) {
          final CandidateParser cp = new CandidateParser(sql);
          if (cp.shrink()) {
            shrunk = cp.getShrunk();
            // also verify we get the same number of literals
            final ValueManager valueManager = tee.getPersistenceContext().getValueManager();
            litCount = cp.getLiterals().size();
            if (litCount != valueManager.getNumberOfLiterals()) {
              final ValueManager.CacheStatus cacheStatus = valueManager.getCacheStatus();
              String reason;
              switch (cacheStatus) {
                case NOCACHE_DYNAMIC_FUNCTION:
                  reason = "contains a non-cacheable dynamic function";
                  break;
                case NOCACHE_TOO_MANY_LITERALS:
                  reason = "literal count exceeded configured max_cached_plan_literals";
                  break;
                case CACHEABLE:
                default:
                  reason = "unknown";
              }
              logError(
                  sql,
                  sp,
                  "Mismatched literal size; parse="
                      + valueManager.getNumberOfLiterals()
                      + "/shrink="
                      + litCount
                      + " , reason="
                      + reason,
                  false);
            }
          } else {
            logError(sql, sp, "Unable to shrink", false);
          }
        }
        final Database<?> db = dmls.getDatabase(tee.getPersistenceContext());
        se =
            new NonInsertEntry(
                corpus,
                sql,
                dmls.getClass().getSimpleName(),
                (db == null ? null : db.getName().get()),
                litCount,
                incCounter);
        byParam.put(p, se);
        if (shrunk != null) {
          globalCounters.increment(GlobalCounters.SHRINK_CACHE_ADDS);
          byShrunk.put(shrunk, se);
        }
      }
      if (se.populationObject.getLiteralCount() >= 0) {
        literalCounts.sample(se.populationObject.getLiteralCount());
      }
      se.bump(dmlBreakout, literalCounts);
    }
  }