Beispiel #1
0
 /**
  * Main setUp method for test cases. Given the ProjectType we will populate the static catalog
  * field members The full_catalog flag is a hack to work around OutofMemory issues with TPC-E
  *
  * @param type
  * @param fkeys
  * @param full_catalog
  * @throws Exception
  */
 protected void setUp(ProjectType type, boolean fkeys, boolean full_catalog) throws Exception {
   super.setUp();
   is_first = (is_first == null ? true : false);
   this.last_type = type;
   catalog = project_catalogs.get(type);
   catalog_db = project_databases.get(type);
   p_estimator = project_p_estimators.get(type);
   if (catalog == null) {
     AbstractProjectBuilder projectBuilder = AbstractProjectBuilder.getProjectBuilder(type);
     if (ENABLE_JAR_REUSE) {
       File jar_path = projectBuilder.getJarPath(true);
       if (jar_path.exists()) {
         LOG.debug("LOAD CACHE JAR: " + jar_path.getAbsolutePath());
         catalog = CatalogUtil.loadCatalogFromJar(jar_path.getAbsolutePath());
       } else {
         LOG.debug("MISSING JAR: " + jar_path.getAbsolutePath());
       }
     }
     if (catalog == null) {
       switch (type) {
         case TPCC:
           catalog = TPCCProjectBuilder.getTPCCSchemaCatalog(true);
           // Update the ProcParameter mapping used in the catalogs
           //
           // ParametersUtil.populateCatalog(CatalogUtil.getDatabase(catalog),
           // ParametersUtil.getParameterMapping(type));
           break;
         case TPCE:
           catalog = projectBuilder.createCatalog(fkeys, full_catalog);
           break;
         case TM1:
         case SEATS:
         case AUCTIONMARK:
         case MARKOV:
         case LOCALITY:
         case MAPREDUCE:
           catalog = projectBuilder.getFullCatalog(fkeys);
           break;
         default:
           assert (false) : "Invalid project type - " + type;
       } // SWITCH
     }
     if (type == ProjectType.TPCC)
       ParametersUtil.populateCatalog(
           CatalogUtil.getDatabase(catalog), ParametersUtil.getParameterMapping(type));
     this.init(type, catalog);
   }
 }
Beispiel #2
0
  protected void setUp(AbstractProjectBuilder projectBuilder, boolean force) throws Exception {
    super.setUp();
    is_first = (is_first == null ? true : false);
    this.last_type = ProjectType.TEST;
    if (force == false) {
      catalog = project_catalogs.get(this.last_type);
      catalog_db = project_databases.get(this.last_type);
      p_estimator = project_p_estimators.get(this.last_type);
    }
    if (catalog == null || force) {
      String catalogJar = new File(projectBuilder.getJarName(true)).getAbsolutePath();
      try {
        boolean status = projectBuilder.compile(catalogJar);
        assert (status);
      } catch (Exception ex) {
        throw new RuntimeException(
            "Failed to create " + projectBuilder.getProjectName() + " catalog [" + catalogJar + "]",
            ex);
      }

      catalog = new Catalog();
      try {
        // read in the catalog
        String serializedCatalog =
            JarReader.readFileFromJarfile(catalogJar, CatalogUtil.CATALOG_FILENAME);
        // create the catalog (that will be passed to the ClientInterface
        catalog.execute(serializedCatalog);
      } catch (Exception ex) {
        throw new RuntimeException(
            "Failed to load " + projectBuilder.getProjectName() + " catalog [" + catalogJar + "]",
            ex);
      }

      this.init(this.last_type, catalog);
    }
  }
Beispiel #3
0
  /** @throws Exception */
  private void loadWorkload() throws Exception {
    final boolean debug = LOG.isDebugEnabled();
    // Workload Trace
    if (this.params.containsKey(PARAM_WORKLOAD)) {
      assert (this.catalog_db != null) : "Missing catalog!";
      String path = new File(this.params.get(PARAM_WORKLOAD)).getAbsolutePath();

      boolean weightedTxns = this.getBooleanParam(PARAM_WORKLOAD_XACT_WEIGHTS, false);
      if (debug) LOG.debug("Use Transaction Weights in Limits: " + weightedTxns);

      // This will prune out duplicate trace records...
      if (params.containsKey(PARAM_WORKLOAD_REMOVE_DUPES)) {
        DuplicateTraceFilter filter = new DuplicateTraceFilter();
        this.workload_filter =
            (this.workload_filter != null ? filter.attach(this.workload_filter) : filter);
        if (debug) LOG.debug("Attached " + filter.debugImpl());
      }

      // TRANSACTION OFFSET
      if (params.containsKey(PARAM_WORKLOAD_XACT_OFFSET)) {
        this.workload_xact_offset = Long.parseLong(params.get(PARAM_WORKLOAD_XACT_OFFSET));
        ProcedureLimitFilter filter =
            new ProcedureLimitFilter(-1l, this.workload_xact_offset, weightedTxns);
        // Important! The offset should go in the front!
        this.workload_filter =
            (this.workload_filter != null ? filter.attach(this.workload_filter) : filter);
        if (debug) LOG.debug("Attached " + filter.debugImpl());
      }

      // BASE PARTITIONS
      if (params.containsKey(PARAM_WORKLOAD_RANDOM_PARTITIONS)
          || params.containsKey(PARAM_WORKLOAD_BASE_PARTITIONS)) {
        BasePartitionTxnFilter filter =
            new BasePartitionTxnFilter(new PartitionEstimator(catalog_db));

        // FIXED LIST
        if (params.containsKey(PARAM_WORKLOAD_BASE_PARTITIONS)) {
          for (String p_str : this.getParam(PARAM_WORKLOAD_BASE_PARTITIONS).split(",")) {
            workload_base_partitions.add(Integer.valueOf(p_str));
          } // FOR
          // RANDOM
        } else {
          double factor = this.getDoubleParam(PARAM_WORKLOAD_RANDOM_PARTITIONS);
          List<Integer> all_partitions =
              new ArrayList<Integer>(CatalogUtil.getAllPartitionIds(catalog_db));
          Collections.shuffle(all_partitions, new Random());
          workload_base_partitions.addAll(
              all_partitions.subList(0, (int) (all_partitions.size() * factor)));
        }
        filter.addPartitions(workload_base_partitions);
        this.workload_filter =
            (this.workload_filter != null ? this.workload_filter.attach(filter) : filter);
        if (debug) LOG.debug("Attached " + filter.debugImpl());
      }

      // Txn Limit
      this.workload_xact_limit = this.getLongParam(PARAM_WORKLOAD_XACT_LIMIT);
      Histogram<String> proc_histogram = null;

      // Include/exclude procedures from the traces
      if (params.containsKey(PARAM_WORKLOAD_PROC_INCLUDE)
          || params.containsKey(PARAM_WORKLOAD_PROC_EXCLUDE)) {
        Filter filter = new ProcedureNameFilter(weightedTxns);

        // INCLUDE
        String temp = params.get(PARAM_WORKLOAD_PROC_INCLUDE);
        if (temp != null && !temp.equals(ProcedureNameFilter.INCLUDE_ALL)) {

          // We can take the counts for PROC_INCLUDE and scale them
          // with the multiplier
          double multiplier = 1.0d;
          if (this.hasDoubleParam(PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER)) {
            multiplier = this.getDoubleParam(PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER);
            if (debug) LOG.debug("Workload Procedure Multiplier: " + multiplier);
          }

          // Default Txn Frequencies
          String procinclude = params.get(PARAM_WORKLOAD_PROC_INCLUDE);
          if (procinclude.equalsIgnoreCase("default")) {
            procinclude =
                AbstractProjectBuilder.getProjectBuilder(catalog_type)
                    .getTransactionFrequencyString();
          }

          Map<String, Integer> limits = new HashMap<String, Integer>();
          int total_unlimited = 0;
          int total = 0;
          for (String proc_name : procinclude.split(",")) {
            int limit = -1;
            // Check if there is a limit for this procedure
            if (proc_name.contains(":")) {
              String pieces[] = proc_name.split(":");
              proc_name = pieces[0];
              limit = (int) Math.round(Integer.parseInt(pieces[1]) * multiplier);
            }

            if (limit < 0) {
              if (proc_histogram == null) {
                if (debug) LOG.debug("Generating procedure histogram from workload file");
                proc_histogram = WorkloadUtil.getProcedureHistogram(new File(path));
              }
              limit = (int) proc_histogram.get(proc_name, 0);
              total_unlimited += limit;
            } else {
              total += limit;
            }
            limits.put(proc_name, limit);
          } // FOR
          // If we have a workload limit and some txns that we want
          // to get unlimited
          // records from, then we want to modify the other txns so
          // that we fill in the "gap"
          if (this.workload_xact_limit != null && total_unlimited > 0) {
            int remaining = this.workload_xact_limit.intValue() - total - total_unlimited;
            if (remaining > 0) {
              for (Entry<String, Integer> e : limits.entrySet()) {
                double ratio = e.getValue() / (double) total;
                e.setValue((int) Math.ceil(e.getValue() + (ratio * remaining)));
              } // FOR
            }
          }

          Histogram<String> proc_multiplier_histogram = null;
          if (debug) {
            if (proc_histogram != null) LOG.debug("Full Workload Histogram:\n" + proc_histogram);
            proc_multiplier_histogram = new Histogram<String>();
          }
          total = 0;
          for (Entry<String, Integer> e : limits.entrySet()) {
            if (debug) proc_multiplier_histogram.put(e.getKey(), e.getValue());
            ((ProcedureNameFilter) filter).include(e.getKey(), e.getValue());
            total += e.getValue();
          } // FOR
          if (debug)
            LOG.debug("Multiplier Histogram [total=" + total + "]:\n" + proc_multiplier_histogram);
        }

        // EXCLUDE
        temp = params.get(PARAM_WORKLOAD_PROC_EXCLUDE);
        if (temp != null) {
          for (String proc_name : params.get(PARAM_WORKLOAD_PROC_EXCLUDE).split(",")) {
            ((ProcedureNameFilter) filter).exclude(proc_name);
          } // FOR
        }

        // Sampling!!
        if (this.getBooleanParam(PARAM_WORKLOAD_PROC_SAMPLE, false)) {
          if (debug) LOG.debug("Attaching sampling filter");
          if (proc_histogram == null)
            proc_histogram = WorkloadUtil.getProcedureHistogram(new File(path));
          Map<String, Integer> proc_includes = ((ProcedureNameFilter) filter).getProcIncludes();
          SamplingFilter sampling_filter = new SamplingFilter(proc_includes, proc_histogram);
          filter = sampling_filter;
          if (debug) LOG.debug("Workload Procedure Histogram:\n" + proc_histogram);
        }

        // Attach our new filter to the chain (or make it the head if
        // it's the first one)
        this.workload_filter =
            (this.workload_filter != null ? this.workload_filter.attach(filter) : filter);
        if (debug) LOG.debug("Attached " + filter.debugImpl());
      }

      // TRANSACTION LIMIT
      if (this.workload_xact_limit != null) {
        ProcedureLimitFilter filter =
            new ProcedureLimitFilter(this.workload_xact_limit, weightedTxns);
        this.workload_filter =
            (this.workload_filter != null ? this.workload_filter.attach(filter) : filter);
        if (debug) LOG.debug("Attached " + filter.debugImpl());
      }

      // QUERY LIMIT
      if (params.containsKey(PARAM_WORKLOAD_QUERY_LIMIT)) {
        this.workload_query_limit = Long.parseLong(params.get(PARAM_WORKLOAD_QUERY_LIMIT));
        QueryLimitFilter filter = new QueryLimitFilter(this.workload_query_limit);
        this.workload_filter =
            (this.workload_filter != null ? this.workload_filter.attach(filter) : filter);
      }

      if (this.workload_filter != null && debug)
        LOG.debug("Workload Filters: " + this.workload_filter.toString());
      this.workload = new Workload(this.catalog);
      this.workload.load(path, this.catalog_db, this.workload_filter);
      this.workload_path = new File(path).getAbsolutePath();
      if (this.workload_filter != null) this.workload_filter.reset();
    }

    // Workload Statistics
    if (this.catalog_db != null) {
      this.stats = new WorkloadStatistics(this.catalog_db);
      if (this.params.containsKey(PARAM_STATS)) {
        String path = this.params.get(PARAM_STATS);
        if (debug) LOG.debug("Loading in workload statistics from '" + path + "'");
        this.stats_path = new File(path).getAbsolutePath();
        try {
          this.stats.load(path, this.catalog_db);
        } catch (Throwable ex) {
          throw new RuntimeException("Failed to load stats file '" + this.stats_path + "'", ex);
        }
      }

      // Scaling
      if (this.params.containsKey(PARAM_STATS_SCALE_FACTOR)) {
        double scale_factor = this.getDoubleParam(PARAM_STATS_SCALE_FACTOR);
        LOG.info("Scaling TableStatistics: " + scale_factor);
        AbstractTableStatisticsGenerator generator =
            AbstractTableStatisticsGenerator.factory(
                this.catalog_db, this.catalog_type, scale_factor);
        generator.apply(this.stats);
      }
    }
  }