/** @throws Exception */ private void loadWorkload() throws Exception { final boolean debug = LOG.isDebugEnabled(); // Workload Trace if (this.params.containsKey(PARAM_WORKLOAD)) { assert (this.catalog_db != null) : "Missing catalog!"; String path = new File(this.params.get(PARAM_WORKLOAD)).getAbsolutePath(); boolean weightedTxns = this.getBooleanParam(PARAM_WORKLOAD_XACT_WEIGHTS, false); if (debug) LOG.debug("Use Transaction Weights in Limits: " + weightedTxns); // This will prune out duplicate trace records... if (params.containsKey(PARAM_WORKLOAD_REMOVE_DUPES)) { DuplicateTraceFilter filter = new DuplicateTraceFilter(); this.workload_filter = (this.workload_filter != null ? filter.attach(this.workload_filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // TRANSACTION OFFSET if (params.containsKey(PARAM_WORKLOAD_XACT_OFFSET)) { this.workload_xact_offset = Long.parseLong(params.get(PARAM_WORKLOAD_XACT_OFFSET)); ProcedureLimitFilter filter = new ProcedureLimitFilter(-1l, this.workload_xact_offset, weightedTxns); // Important! The offset should go in the front! this.workload_filter = (this.workload_filter != null ? filter.attach(this.workload_filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // BASE PARTITIONS if (params.containsKey(PARAM_WORKLOAD_RANDOM_PARTITIONS) || params.containsKey(PARAM_WORKLOAD_BASE_PARTITIONS)) { BasePartitionTxnFilter filter = new BasePartitionTxnFilter(new PartitionEstimator(catalog_db)); // FIXED LIST if (params.containsKey(PARAM_WORKLOAD_BASE_PARTITIONS)) { for (String p_str : this.getParam(PARAM_WORKLOAD_BASE_PARTITIONS).split(",")) { workload_base_partitions.add(Integer.valueOf(p_str)); } // FOR // RANDOM } else { double factor = this.getDoubleParam(PARAM_WORKLOAD_RANDOM_PARTITIONS); List<Integer> all_partitions = new ArrayList<Integer>(CatalogUtil.getAllPartitionIds(catalog_db)); Collections.shuffle(all_partitions, new Random()); workload_base_partitions.addAll( all_partitions.subList(0, (int) (all_partitions.size() * factor))); } filter.addPartitions(workload_base_partitions); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // Txn Limit this.workload_xact_limit = this.getLongParam(PARAM_WORKLOAD_XACT_LIMIT); Histogram<String> proc_histogram = null; // Include/exclude procedures from the traces if (params.containsKey(PARAM_WORKLOAD_PROC_INCLUDE) || params.containsKey(PARAM_WORKLOAD_PROC_EXCLUDE)) { Filter filter = new ProcedureNameFilter(weightedTxns); // INCLUDE String temp = params.get(PARAM_WORKLOAD_PROC_INCLUDE); if (temp != null && !temp.equals(ProcedureNameFilter.INCLUDE_ALL)) { // We can take the counts for PROC_INCLUDE and scale them // with the multiplier double multiplier = 1.0d; if (this.hasDoubleParam(PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER)) { multiplier = this.getDoubleParam(PARAM_WORKLOAD_PROC_INCLUDE_MULTIPLIER); if (debug) LOG.debug("Workload Procedure Multiplier: " + multiplier); } // Default Txn Frequencies String procinclude = params.get(PARAM_WORKLOAD_PROC_INCLUDE); if (procinclude.equalsIgnoreCase("default")) { procinclude = AbstractProjectBuilder.getProjectBuilder(catalog_type) .getTransactionFrequencyString(); } Map<String, Integer> limits = new HashMap<String, Integer>(); int total_unlimited = 0; int total = 0; for (String proc_name : procinclude.split(",")) { int limit = -1; // Check if there is a limit for this procedure if (proc_name.contains(":")) { String pieces[] = proc_name.split(":"); proc_name = pieces[0]; limit = (int) Math.round(Integer.parseInt(pieces[1]) * multiplier); } if (limit < 0) { if (proc_histogram == null) { if (debug) LOG.debug("Generating procedure histogram from workload file"); proc_histogram = WorkloadUtil.getProcedureHistogram(new File(path)); } limit = (int) proc_histogram.get(proc_name, 0); total_unlimited += limit; } else { total += limit; } limits.put(proc_name, limit); } // FOR // If we have a workload limit and some txns that we want // to get unlimited // records from, then we want to modify the other txns so // that we fill in the "gap" if (this.workload_xact_limit != null && total_unlimited > 0) { int remaining = this.workload_xact_limit.intValue() - total - total_unlimited; if (remaining > 0) { for (Entry<String, Integer> e : limits.entrySet()) { double ratio = e.getValue() / (double) total; e.setValue((int) Math.ceil(e.getValue() + (ratio * remaining))); } // FOR } } Histogram<String> proc_multiplier_histogram = null; if (debug) { if (proc_histogram != null) LOG.debug("Full Workload Histogram:\n" + proc_histogram); proc_multiplier_histogram = new Histogram<String>(); } total = 0; for (Entry<String, Integer> e : limits.entrySet()) { if (debug) proc_multiplier_histogram.put(e.getKey(), e.getValue()); ((ProcedureNameFilter) filter).include(e.getKey(), e.getValue()); total += e.getValue(); } // FOR if (debug) LOG.debug("Multiplier Histogram [total=" + total + "]:\n" + proc_multiplier_histogram); } // EXCLUDE temp = params.get(PARAM_WORKLOAD_PROC_EXCLUDE); if (temp != null) { for (String proc_name : params.get(PARAM_WORKLOAD_PROC_EXCLUDE).split(",")) { ((ProcedureNameFilter) filter).exclude(proc_name); } // FOR } // Sampling!! if (this.getBooleanParam(PARAM_WORKLOAD_PROC_SAMPLE, false)) { if (debug) LOG.debug("Attaching sampling filter"); if (proc_histogram == null) proc_histogram = WorkloadUtil.getProcedureHistogram(new File(path)); Map<String, Integer> proc_includes = ((ProcedureNameFilter) filter).getProcIncludes(); SamplingFilter sampling_filter = new SamplingFilter(proc_includes, proc_histogram); filter = sampling_filter; if (debug) LOG.debug("Workload Procedure Histogram:\n" + proc_histogram); } // Attach our new filter to the chain (or make it the head if // it's the first one) this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // TRANSACTION LIMIT if (this.workload_xact_limit != null) { ProcedureLimitFilter filter = new ProcedureLimitFilter(this.workload_xact_limit, weightedTxns); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); if (debug) LOG.debug("Attached " + filter.debugImpl()); } // QUERY LIMIT if (params.containsKey(PARAM_WORKLOAD_QUERY_LIMIT)) { this.workload_query_limit = Long.parseLong(params.get(PARAM_WORKLOAD_QUERY_LIMIT)); QueryLimitFilter filter = new QueryLimitFilter(this.workload_query_limit); this.workload_filter = (this.workload_filter != null ? this.workload_filter.attach(filter) : filter); } if (this.workload_filter != null && debug) LOG.debug("Workload Filters: " + this.workload_filter.toString()); this.workload = new Workload(this.catalog); this.workload.load(path, this.catalog_db, this.workload_filter); this.workload_path = new File(path).getAbsolutePath(); if (this.workload_filter != null) this.workload_filter.reset(); } // Workload Statistics if (this.catalog_db != null) { this.stats = new WorkloadStatistics(this.catalog_db); if (this.params.containsKey(PARAM_STATS)) { String path = this.params.get(PARAM_STATS); if (debug) LOG.debug("Loading in workload statistics from '" + path + "'"); this.stats_path = new File(path).getAbsolutePath(); try { this.stats.load(path, this.catalog_db); } catch (Throwable ex) { throw new RuntimeException("Failed to load stats file '" + this.stats_path + "'", ex); } } // Scaling if (this.params.containsKey(PARAM_STATS_SCALE_FACTOR)) { double scale_factor = this.getDoubleParam(PARAM_STATS_SCALE_FACTOR); LOG.info("Scaling TableStatistics: " + scale_factor); AbstractTableStatisticsGenerator generator = AbstractTableStatisticsGenerator.factory( this.catalog_db, this.catalog_type, scale_factor); generator.apply(this.stats); } } }
@Override protected double estimateWorkloadCostImpl( final CatalogContext catalogContext, final Workload workload, final Filter filter, final Double upper_bound) throws Exception { if (debug.val) LOG.debug( "Calculating workload execution cost across " + num_intervals + " intervals for " + num_partitions + " partitions"); // (1) Grab the costs at the different time intervals // Also create the ratios that we will use to weight the interval costs final AtomicLong total_txns = new AtomicLong(0); // final HashSet<Long> trace_ids[] = new HashSet[num_intervals]; for (int i = 0; i < num_intervals; i++) { total_interval_txns[i] = 0; total_interval_queries[i] = 0; singlepartition_ctrs[i] = 0; singlepartition_with_partitions_ctrs[i] = 0; multipartition_ctrs[i] = 0; partitions_touched[i] = 0; incomplete_txn_ctrs[i] = 0; exec_mismatch_ctrs[i] = 0; incomplete_txn_histogram[i].clear(); missing_txn_histogram[i].clear(); exec_histogram[i].clear(); } // FOR // (2) Now go through the workload and estimate the partitions that each txn // will touch for the given catalog setups if (trace.val) { LOG.trace("Total # of Txns in Workload: " + workload.getTransactionCount()); if (filter != null) LOG.trace( "Workload Filter Chain: " + StringUtil.join(" ", "\n", filter.getFilters())); } // QUEUING THREAD tmp_consumers.clear(); Producer<TransactionTrace, Pair<TransactionTrace, Integer>> producer = new Producer<TransactionTrace, Pair<TransactionTrace, Integer>>( CollectionUtil.iterable(workload.iterator(filter))) { @Override public Pair<Consumer<Pair<TransactionTrace, Integer>>, Pair<TransactionTrace, Integer>> transform(TransactionTrace txn_trace) { int i = workload.getTimeInterval(txn_trace, num_intervals); assert (i >= 0) : "Invalid time interval '" + i + "'\n" + txn_trace.debug(catalogContext.database); assert (i < num_intervals) : "Invalid interval: " + i + "\n" + txn_trace.debug(catalogContext.database); total_txns.incrementAndGet(); Pair<TransactionTrace, Integer> p = Pair.of(txn_trace, i); return (Pair.of(tmp_consumers.get(i), p)); } }; // PROCESSING THREADS final int num_threads = ThreadUtil.getMaxGlobalThreads(); int interval_ctr = 0; for (int thread = 0; thread < num_threads; thread++) { // First create a new IntervalProcessor/Consumer IntervalProcessor ip = new IntervalProcessor(catalogContext, workload, filter); // Then assign it to some number of intervals for (int i = 0, cnt = (int) Math.ceil(num_intervals / (double) num_threads); i < cnt; i++) { if (interval_ctr > num_intervals) break; tmp_consumers.put(interval_ctr++, ip); if (trace.val) LOG.trace( String.format("Interval #%02d => IntervalProcessor #%02d", interval_ctr - 1, thread)); } // FOR // And make sure that we queue it up too producer.addConsumer(ip); } // FOR (threads) ThreadUtil.runGlobalPool(producer.getRunnablesList()); // BLOCKING if (debug.val) { int processed = 0; for (Consumer<?> c : producer.getConsumers()) { processed += c.getProcessedCounter(); } // FOR assert (total_txns.get() == processed) : String.format("Expected[%d] != Processed[%d]", total_txns.get(), processed); } // We have to convert all of the costs into the range of [0.0, 1.0] // For each interval, divide the number of partitions touched by the total number // of partitions that the interval could have touched (worst case scenario) final double execution_costs[] = new double[num_intervals]; StringBuilder sb = (this.isDebugEnabled() || debug.get() ? new StringBuilder() : null); Map<String, Object> debug_m = null; if (sb != null) { debug_m = new LinkedHashMap<String, Object>(); } if (debug.val) LOG.debug("Calculating execution cost for " + this.num_intervals + " intervals..."); long total_multipartition_txns = 0; for (int i = 0; i < this.num_intervals; i++) { interval_weights[i] = total_interval_txns[i] / (double) total_txns.get(); long total_txns_in_interval = (long) total_interval_txns[i]; long total_queries_in_interval = (long) total_interval_queries[i]; long num_txns = this.cost_models[i].txn_ctr.get(); long potential_txn_touches = (total_txns_in_interval * num_partitions); // TXNS double penalty = 0.0d; total_multipartition_txns += multipartition_ctrs[i]; // Divide the total number of partitions touched by... // This is the total number of partitions that we could have touched // in this interval // And this is the total number of partitions that we did actually touch if (multipartition_ctrs[i] > 0) { assert (partitions_touched[i] > 0) : "No touched partitions for interval " + i; double cost = (partitions_touched[i] / (double) potential_txn_touches); if (this.use_multitpartition_penalty) { penalty = this.multipartition_penalty * (1.0d + (multipartition_ctrs[i] / (double) total_txns_in_interval)); assert (penalty >= 1.0) : "The multipartition penalty is less than one: " + penalty; cost *= penalty; } execution_costs[i] = Math.min(cost, (double) potential_txn_touches); } // For each txn that wasn't even evaluated, add all of the // partitions to the incomplete histogram if (num_txns < total_txns_in_interval) { if (trace.val) LOG.trace( "Adding " + (total_txns_in_interval - num_txns) + " entries to the incomplete histogram for interval #" + i); for (long ii = num_txns; ii < total_txns_in_interval; ii++) { missing_txn_histogram[i].put(all_partitions); } // WHILE } if (sb != null) { tmp_penalties.add(penalty); tmp_total.add(total_txns_in_interval); tmp_touched.add(partitions_touched[i]); tmp_potential.add(potential_txn_touches); Map<String, Object> inner = new LinkedHashMap<String, Object>(); inner.put("Partitions Touched", partitions_touched[i]); inner.put("Potential Touched", potential_txn_touches); inner.put("Multi-Partition Txns", multipartition_ctrs[i]); inner.put("Total Txns", total_txns_in_interval); inner.put("Total Queries", total_queries_in_interval); inner.put("Missing Txns", (total_txns_in_interval - num_txns)); inner.put("Cost", String.format("%.05f", execution_costs[i])); inner.put("Exec Txns", exec_histogram[i].getSampleCount()); debug_m.put("Interval #" + i, inner); } } // FOR if (sb != null) { Map<String, Object> m0 = new LinkedHashMap<String, Object>(); m0.put("SinglePartition Txns", (total_txns.get() - total_multipartition_txns)); m0.put("MultiPartition Txns", total_multipartition_txns); m0.put( "Total Txns", String.format( "%d [%.06f]", total_txns.get(), (1.0d - (total_multipartition_txns / (double) total_txns.get())))); Map<String, Object> m1 = new LinkedHashMap<String, Object>(); m1.put("Touched Partitions", tmp_touched); m1.put("Potential Partitions", tmp_potential); m1.put("Total Partitions", tmp_total); m1.put("Penalties", tmp_penalties); sb.append(StringUtil.formatMaps(debug_m, m0, m1)); if (debug.val) LOG.debug("**** Execution Cost ****\n" + sb); this.appendDebugMessage(sb); } // LOG.debug("Execution By Intervals:\n" + sb.toString()); // (3) We then need to go through and grab the histograms of partitions were accessed if (sb != null) { if (debug.val) LOG.debug("Calculating skew factor for " + this.num_intervals + " intervals..."); debug_histograms.clear(); sb = new StringBuilder(); } for (int i = 0; i < this.num_intervals; i++) { ObjectHistogram<Integer> histogram_txn = this.cost_models[i].getTxnPartitionAccessHistogram(); ObjectHistogram<Integer> histogram_query = this.cost_models[i].getQueryPartitionAccessHistogram(); this.histogram_query_partitions.put(histogram_query); long num_queries = this.cost_models[i].query_ctr.get(); this.query_ctr.addAndGet(num_queries); // DEBUG SingleSitedCostModel inner_costModel = (SingleSitedCostModel) this.cost_models[i]; boolean is_valid = (partitions_touched[i] + singlepartition_with_partitions_ctrs[i]) == (this.cost_models[i].getTxnPartitionAccessHistogram().getSampleCount() + exec_mismatch_ctrs[i]); if (!is_valid) { LOG.error("Transaction Entries: " + inner_costModel.getTransactionCacheEntries().size()); ObjectHistogram<Integer> check = new ObjectHistogram<Integer>(); for (TransactionCacheEntry tce : inner_costModel.getTransactionCacheEntries()) { check.put(tce.getTouchedPartitions()); // LOG.error(tce.debug() + "\n"); } LOG.error( "Check Touched Partitions: sample=" + check.getSampleCount() + ", values=" + check.getValueCount()); LOG.error( "Cache Touched Partitions: sample=" + this.cost_models[i].getTxnPartitionAccessHistogram().getSampleCount() + ", values=" + this.cost_models[i].getTxnPartitionAccessHistogram().getValueCount()); int qtotal = inner_costModel.getAllQueryCacheEntries().size(); int ctr = 0; int multip = 0; for (QueryCacheEntry qce : inner_costModel.getAllQueryCacheEntries()) { ctr += (qce.getAllPartitions().isEmpty() ? 0 : 1); multip += (qce.getAllPartitions().size() > 1 ? 1 : 0); } // FOR LOG.error("# of QueryCacheEntries with Touched Partitions: " + ctr + " / " + qtotal); LOG.error("# of MultiP QueryCacheEntries: " + multip); } assert (is_valid) : String.format( "Partitions Touched by Txns Mismatch in Interval #%d\n" + "(partitions_touched[%d] + singlepartition_with_partitions_ctrs[%d]) != " + "(histogram_txn[%d] + exec_mismatch_ctrs[%d])", i, partitions_touched[i], singlepartition_with_partitions_ctrs[i], this.cost_models[i].getTxnPartitionAccessHistogram().getSampleCount(), exec_mismatch_ctrs[i]); this.histogram_java_partitions.put(this.cost_models[i].getJavaExecutionHistogram()); this.histogram_txn_partitions.put(histogram_txn); long num_txns = this.cost_models[i].txn_ctr.get(); assert (num_txns >= 0) : "The transaction counter at interval #" + i + " is " + num_txns; this.txn_ctr.addAndGet(num_txns); // Calculate the skew factor at this time interval // XXX: Should the number of txns be the total number of unique txns // that were executed or the total number of times a txn touched the partitions? // XXX: What do we do when the number of elements that we are examining is zero? // I guess the cost just needs to be zero? // XXX: What histogram do we want to use? target_histogram.clear(); target_histogram.put(histogram_txn); // For each txn that we haven't gotten an estimate for at this interval, // we're going mark it as being broadcast to all partitions. That way the access // histogram will look uniform. Then as more information is added, we will // This is an attempt to make sure that the skew cost never decreases but only increases long total_txns_in_interval = (long) total_interval_txns[i]; if (sb != null) { debug_histograms.put("Incomplete Txns", incomplete_txn_histogram[i]); debug_histograms.put("Missing Txns", missing_txn_histogram[i]); debug_histograms.put( "Target Partitions (BEFORE)", new ObjectHistogram<Integer>(target_histogram)); debug_histograms.put("Target Partitions (AFTER)", target_histogram); } // Merge the values from incomplete histogram into the target // histogram target_histogram.put(incomplete_txn_histogram[i]); target_histogram.put(missing_txn_histogram[i]); exec_histogram[i].put(missing_txn_histogram[i]); long num_elements = target_histogram.getSampleCount(); // The number of partition touches should never be greater than our // potential touches assert (num_elements <= (total_txns_in_interval * num_partitions)) : "New Partitions Touched Sample Count [" + num_elements + "] < " + "Maximum Potential Touched Count [" + (total_txns_in_interval * num_partitions) + "]"; if (sb != null) { Map<String, Object> m = new LinkedHashMap<String, Object>(); for (String key : debug_histograms.keySet()) { ObjectHistogram<?> h = debug_histograms.get(key); m.put( key, String.format("[Sample=%d, Value=%d]\n%s", h.getSampleCount(), h.getValueCount(), h)); } // FOR sb.append( String.format( "INTERVAL #%d [total_txns_in_interval=%d, num_txns=%d, incomplete_txns=%d]\n%s", i, total_txns_in_interval, num_txns, incomplete_txn_ctrs[i], StringUtil.formatMaps(m))); } // Txn Skew if (num_elements == 0) { txn_skews[i] = 0.0d; } else { txn_skews[i] = SkewFactorUtil.calculateSkew(num_partitions, num_elements, target_histogram); } // Exec Skew if (exec_histogram[i].getSampleCount() == 0) { exec_skews[i] = 0.0d; } else { exec_skews[i] = SkewFactorUtil.calculateSkew( num_partitions, exec_histogram[i].getSampleCount(), exec_histogram[i]); } total_skews[i] = (0.5 * exec_skews[i]) + (0.5 * txn_skews[i]); if (sb != null) { sb.append("Txn Skew = " + MathUtil.roundToDecimals(txn_skews[i], 6) + "\n"); sb.append("Exec Skew = " + MathUtil.roundToDecimals(exec_skews[i], 6) + "\n"); sb.append("Total Skew = " + MathUtil.roundToDecimals(total_skews[i], 6) + "\n"); sb.append(StringUtil.DOUBLE_LINE); } } // FOR if (sb != null && sb.length() > 0) { if (debug.val) LOG.debug("**** Skew Factor ****\n" + sb); this.appendDebugMessage(sb); } if (trace.val) { for (int i = 0; i < num_intervals; i++) { LOG.trace( "Time Interval #" + i + "\n" + "Total # of Txns: " + this.cost_models[i].txn_ctr.get() + "\n" + "Multi-Partition Txns: " + multipartition_ctrs[i] + "\n" + "Execution Cost: " + execution_costs[i] + "\n" + "ProcHistogram:\n" + this.cost_models[i].getProcedureHistogram().toString() + "\n" + // "TransactionsPerPartitionHistogram:\n" + // this.cost_models[i].getTxnPartitionAccessHistogram() // + "\n" + StringUtil.SINGLE_LINE); } } // (3) We can now calculate the final total estimate cost of this workload as the following // Just take the simple ratio of mp txns / all txns this.last_execution_cost = MathUtil.weightedMean( execution_costs, total_interval_txns); // MathUtil.roundToDecimals(MathUtil.geometricMean(execution_costs, // MathUtil.GEOMETRIC_MEAN_ZERO), // 10); // The final skew cost needs to be weighted by the percentage of txns running in that interval // This will cause the partitions with few txns this.last_skew_cost = MathUtil.weightedMean( total_skews, total_interval_txns); // roundToDecimals(MathUtil.geometricMean(entropies, // MathUtil.GEOMETRIC_MEAN_ZERO), // 10); double new_final_cost = (this.use_execution ? (this.execution_weight * this.last_execution_cost) : 0) + (this.use_skew ? (this.skew_weight * this.last_skew_cost) : 0); if (sb != null) { Map<String, Object> m = new LinkedHashMap<String, Object>(); m.put("Total Txns", total_txns.get()); m.put("Interval Txns", Arrays.toString(total_interval_txns)); m.put("Execution Costs", Arrays.toString(execution_costs)); m.put("Skew Factors", Arrays.toString(total_skews)); m.put("Txn Skew", Arrays.toString(txn_skews)); m.put("Exec Skew", Arrays.toString(exec_skews)); m.put("Interval Weights", Arrays.toString(interval_weights)); m.put( "Final Cost", String.format( "%f = %f + %f", new_final_cost, this.last_execution_cost, this.last_skew_cost)); if (debug.val) LOG.debug(StringUtil.formatMaps(m)); this.appendDebugMessage(StringUtil.formatMaps(m)); } this.last_final_cost = new_final_cost; return (MathUtil.roundToDecimals(this.last_final_cost, 5)); }