@SuppressWarnings("unchecked") protected Map<String, Object>[] getDebugMaps() { List<Map<String, Object>> maps = new ArrayList<Map<String, Object>>(); Map<String, Object> m; m = new LinkedHashMap<String, Object>(); m.put("Transaction Id", this.txn_id); m.put("Procedure", this.catalog_proc); m.put("Base Partition", this.base_partition); m.put("Hash Code", this.hashCode()); m.put("Pending Error", this.pending_error); maps.add(m); // Predictions m = new LinkedHashMap<String, Object>(); m.put("Predict Single-Partitioned", this.predict_singlePartition); m.put("Predict Touched Partitions", this.getPredictTouchedPartitions()); m.put("Predict Read Only", this.isPredictReadOnly()); m.put("Predict Abortable", this.isPredictAbortable()); maps.add(m); // Global State m = new LinkedHashMap<String, Object>(); m.put("Marked Released", PartitionSet.toString(this.released)); m.put("Marked Prepared", PartitionSet.toString(this.prepared)); m.put("Marked Finished", PartitionSet.toString(this.finished)); m.put("Marked Deletable", this.checkDeletableFlag()); maps.add(m); // Prefetch State if (this.prefetch != null) { m = new LinkedHashMap<String, Object>(); m.put("Prefetch Partitions", this.prefetch.partitions); m.put("Prefetch Fragments", StringUtil.join("\n", this.prefetch.fragments)); m.put("Prefetch Parameters", StringUtil.join("\n", this.prefetch.params)); m.put("Prefetch Raw Parameters", StringUtil.join("\n", this.prefetch.paramsRaw)); maps.add(m); } // Partition Execution State m = new LinkedHashMap<String, Object>(); m.put("Current Round State", Arrays.toString(this.round_state)); m.put("Exec Read-Only", PartitionSet.toString(this.exec_readOnly)); m.put("First UndoToken", Arrays.toString(this.exec_firstUndoToken)); m.put("Last UndoToken", Arrays.toString(this.exec_lastUndoToken)); m.put("No Undo Buffer", PartitionSet.toString(this.exec_noUndoBuffer)); m.put("# of Rounds", Arrays.toString(this.round_ctr)); m.put("Executed Work", PartitionSet.toString(this.exec_eeWork)); maps.add(m); return ((Map<String, Object>[]) maps.toArray(new Map[0])); }
/** * Call VoltDB.crashVoltDB on behalf of the EE * * @param reason Reason the EE crashed */ public static void crashVoltDB(String reason, String traces[], String filename, int lineno) { if (reason != null) { LOG.fatal("ExecutionEngine requested that we crash: " + reason); LOG.fatal("Error was in " + filename + ":" + lineno + "\n" + StringUtil.join("\n", traces)); } VoltDB.crashVoltDB(); }
/** testLoadAirportFlights */ public void testLoadAirportFlights() throws Exception { Map<String, Histogram<String>> histograms = SEATSHistogramUtil.loadAirportFlights(AIRLINE_DATA_DIR); assertNotNull(histograms); assertFalse(histograms.isEmpty()); assert (histograms.size() >= 200); // Just some airports that we expect to be in there String airports[] = {"BWI", "LAX", "JFK", "MDW", "ATL", "SFO", "ORD"}; for (String a : airports) { assertTrue(a, histograms.containsKey(a)); } // FOR System.err.println(StringUtil.formatMaps(histograms)); // We expect ATL to be the max // assertEquals("ATL", histogram.getMaxCountValue()); // Make sure the values are formatted correctly // ListOrderedMap<String, Histogram<String>> m = new ListOrderedMap<String, // Histogram<String>>(); Pattern p = Pattern.compile("[\\d\\w]{3,3}"); for (String s_airport : histograms.keySet()) { assert (p.matcher(s_airport).matches()) : "Invalid source airport: " + s_airport; // m.put(s_airport, histograms.get(s_airport)); for (Object value : histograms.get(s_airport).values()) { assert (p.matcher(value.toString()).matches()) : "Invalid destination airport: " + value; } // FOR } // FOR // System.err.println(StringUtil.formatMaps(m)); }
public synchronized void tick(int counter) { this.tick_counter = counter; if (config.warehouse_debug) { Map<String, Histogram<Integer>> m = new ListOrderedMap<String, Histogram<Integer>>(); m.put( String.format( "LAST ROUND\n - SampleCount=%d", this.lastWarehouseHistory.getSampleCount()), this.lastWarehouseHistory); m.put( String.format("TOTAL\n - SampleCount=%d", this.totalWarehouseHistory.getSampleCount()), this.totalWarehouseHistory); long total = this.totalWarehouseHistory.getSampleCount(); LOG.info( String.format( "ROUND #%02d - Warehouse Temporal Skew - %d / %d [%.2f]\n%s", this.tick_counter, this.temporal_counter, total, (this.temporal_counter / (double) total), StringUtil.formatMaps(m))); LOG.info(StringUtil.SINGLE_LINE); this.lastWarehouseHistory.clearValues(); } }
/** * Called by Java to store dependencies for the EE. Creates a private list of dependencies to be * manipulated by the tracker. Does not copy the table data - references WorkUnit's tables. * * @param dependencies */ public void stashWorkUnitDependencies(final Map<Integer, List<VoltTable>> dependencies) { if (d) LOG.debug( String.format( "Stashing %d InputDependencies:\n%s", dependencies.size(), StringUtil.formatMaps(dependencies))); m_dependencyTracker.trackNewWorkUnit(dependencies); }
private Map<String, Long> getCodeXref(String col_name) { Map<String, Long> m = this.code_id_xref.get(col_name); assert (m != null) : "Invalid code xref mapping column '" + col_name + "'"; assert (m.isEmpty() == false) : "Empty code xref mapping for column '" + col_name + "'\n" + StringUtil.formatMaps(this.code_id_xref); return (m); }
public String debug() { Map<String, Object> m = new LinkedHashMap<String, Object>(); m.put("PartitionId", this.partitionId); m.put("# of Elements", this.size()); m.put("Wait Time", this.waitTime); m.put("Next Time Remaining", Math.max(0, EstTime.currentTimeMillis() - this.blockTime)); m.put("Next Txn", this.nextTxn); m.put("Last Popped Txn", this.lastTxnPopped); m.put("Last Seen Txn", this.lastSeenTxn); return (StringUtil.formatMaps(m)); }
protected QueueState checkQueueState() { QueueState newState = QueueState.UNBLOCKED; AbstractTransaction ts = super.peek(); if (ts == null) { if (t) LOG.trace(String.format("Partition %d :: Queue is empty.", this.partitionId)); newState = QueueState.BLOCKED_EMPTY; } // Check whether can unblock now else if (ts == this.nextTxn && this.state != QueueState.UNBLOCKED) { if (EstTime.currentTimeMillis() < this.blockTime) { newState = QueueState.BLOCKED_SAFETY; } else if (d) { LOG.debug( String.format( "Partition %d :: Wait time for %s has passed. Unblocking...", this.partitionId, this.nextTxn)); } } // This is a new txn and we should wait... else if (this.nextTxn != ts) { long txnTimestamp = TransactionIdManager.getTimestampFromTransactionId(ts.getTransactionId().longValue()); long timestamp = EstTime.currentTimeMillis(); long waitTime = Math.max(0, this.waitTime - (timestamp - txnTimestamp)); newState = (waitTime > 0 ? QueueState.BLOCKED_SAFETY : QueueState.UNBLOCKED); this.blockTime = timestamp + waitTime; this.nextTxn = ts; if (d) { String debug = ""; if (t) { Map<String, Object> m = new LinkedHashMap<String, Object>(); m.put("Txn Init Timestamp", txnTimestamp); m.put("Current Timestamp", timestamp); m.put("Block Time Remaining", (this.blockTime - timestamp)); debug = "\n" + StringUtil.formatMaps(m); } LOG.debug( String.format( "Partition %d :: Blocking %s for %d ms [maxWait=%d]%s", this.partitionId, ts, (this.blockTime - timestamp), this.waitTime, debug)); } } if (newState != this.state) { this.state = newState; if (d) LOG.debug( String.format( "Partition %d :: State:%s / NextTxn:%s", this.partitionId, this.state, this.nextTxn)); } return this.state; }
@Override public String toString() { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Scale Factor", this.scale_factor); m.put("# of Reservations", this.num_reservations); m.put("Flight Start Date", this.flight_start_date); m.put("Flight Upcoming Date", this.flight_upcoming_date); m.put("Flight Past Days", this.flight_past_days); m.put("Flight Future Days", this.flight_future_days); m.put("Num Flights", this.num_flights); m.put("Num Customers", this.num_customers); m.put("Num Reservations", this.num_reservations); return (StringUtil.formatMaps(m)); }
@Override public String toString() { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Warehouses", parameters.warehouses); m.put( "W_ID Range", String.format("[%d, %d]", parameters.starting_warehouse, parameters.last_warehouse)); m.put("Districts per Warehouse", parameters.districtsPerWarehouse); m.put("Custers per District", parameters.customersPerDistrict); m.put("Initial Orders per District", parameters.newOrdersPerDistrict); m.put("Items", parameters.items); m.put("Affine Warehouse", lastAssignedWarehouseId); m.put("Skew Factor", this.skewFactor); if (this.zipf != null && this.zipf.isHistoryEnabled()) { m.put("Skewed Warehouses", this.zipf.getHistory()); } return ("TPCC Simulator Options\n" + StringUtil.formatMaps(m, this.config.debugMap())); }
/** * Store dependency tables for later retrieval by the EE. * * @param workunit */ void trackNewWorkUnit(final Map<Integer, List<VoltTable>> dependencies) { for (final Entry<Integer, List<VoltTable>> e : dependencies.entrySet()) { // could do this optionally - debug only. if (d) verifyDependencySanity(e.getKey(), e.getValue()); // create a new list of references to the workunit's table // to avoid any changes to the WorkUnit's list. But do not // copy the table data. ArrayDeque<VoltTable> deque = m_depsById.get(e.getKey()); if (deque == null) { deque = new ArrayDeque<VoltTable>(); // intentionally overwrite the previous dependency id. // would a lookup and a clear() be faster? m_depsById.put(e.getKey(), deque); } else { deque.clear(); } deque.addAll(e.getValue()); } if (d) LOG.debug("Current InputDepencies:\n" + StringUtil.formatMaps(m_depsById)); }
public String debug() { long timestamp = System.currentTimeMillis(); AbstractTransaction peek = super.peek(); @SuppressWarnings("unchecked") Map<String, Object> m[] = new Map[3]; int i = -1; m[++i] = new LinkedHashMap<String, Object>(); m[i].put("PartitionId", this.partitionId); m[i].put("Current State", this.state); m[i].put("# of Elements", this.size()); m[i].put("# of Popped", this.txnsPopped); m[i].put("Last Popped Txn", this.lastTxnPopped); m[i].put("Last Seen Txn", this.lastSeenTxnId); m[i].put("Last Safe Txn", this.lastSafeTxnId); m[++i] = new LinkedHashMap<String, Object>(); m[i].put("Throttled", super.isThrottled()); m[i].put("Threshold", super.getThrottleThreshold()); m[i].put("Release", super.getThrottleRelease()); m[i].put("Increase Delta", super.getThrottleThresholdIncreaseDelta()); m[i].put("Max Size", super.getThrottleThresholdMaxSize()); m[++i] = new LinkedHashMap<String, Object>(); m[i].put("Peek Txn", (peek == null ? "null" : peek)); m[i].put("Wait Time", this.waitTime + " ms"); m[i].put("Current Time", timestamp); m[i].put( "Blocked Time", (this.blockTimestamp > 0 ? this.blockTimestamp + (this.blockTimestamp < timestamp ? " **PASSED**" : "") : "--")); m[i].put( "Blocked Remaining", (this.blockTimestamp > 0 ? Math.max(0, this.blockTimestamp - timestamp) + " ms" : "--")); return (StringUtil.formatMaps(m)); }
/** * This is the most important method of the queue. This will figure out the next state and how * long we must wait until we can release the next transaction. <B>Note:</B> I believe that this * is the only thing that needs to be synchronized * * @param afterRemoval If this flag is set to true, then it means that who ever is calling this * method just removed something from the queue. That means that we need to go and check * whether the lastSafeTxnId should change. * @return */ private QueueState checkQueueState(boolean afterRemoval) { if (trace.val && super.isEmpty() == false) LOG.trace( String.format( "Partition %d :: checkQueueState(afterPoll=%s) [current=%s]", this.partitionId, afterRemoval, this.state)); QueueState newState = (afterRemoval ? QueueState.BLOCKED_SAFETY : QueueState.UNBLOCKED); long currentTimestamp = -1l; AbstractTransaction ts = super.peek(); // BLOCKING Long txnId = null; if (ts == null) { // if (trace.val) // LOG.trace(String.format("Partition %d :: Queue is empty.", // this.partitionId)); newState = QueueState.BLOCKED_EMPTY; } // Check whether can unblock now else { assert (ts.isInitialized()) : String.format( "Unexpected uninitialized transaction %s [partition=%d]", ts, this.partitionId); txnId = ts.getTransactionId(); // HACK: Ignore null txnIds if (txnId == null) { LOG.warn( String.format( "Partition %d :: Uninitialized transaction handle %s", this.partitionId, ts)); return (this.state); } assert (txnId != null) : "Null transaction id from " + txnId; // If this txnId is greater than the last safe one that we've seen, then we know // that the lastSafeTxnId has been polled. That means that we need to // wait for an appropriate amount of time before we're allow to be executed. if (txnId.compareTo(this.lastSafeTxnId) > 0 && afterRemoval == false) { newState = QueueState.BLOCKED_ORDERING; if (debug.val) LOG.debug( String.format( "Partition %d :: txnId[%d] > lastSafeTxnId[%d]", this.partitionId, txnId, this.lastSafeTxnId)); } // If our current block time is negative, then we know that we're the first txnId // that's been in the system. We'll also want to wait a bit before we're // allowed to be executed. else if (this.blockTimestamp == NULL_BLOCK_TIMESTAMP) { newState = QueueState.BLOCKED_SAFETY; if (debug.val) LOG.debug( String.format( "Partition %d :: txnId[%d] ==> %s (blockTime=%d)", this.partitionId, txnId, newState, this.blockTimestamp)); } // Check whether it's safe to unblock this mofo else if ((currentTimestamp = System.currentTimeMillis()) < this.blockTimestamp) { newState = QueueState.BLOCKED_SAFETY; if (debug.val) LOG.debug( String.format( "Partition %d :: txnId[%d] ==> %s (blockTime[%d] - current[%d] = %d)", this.partitionId, txnId, newState, this.blockTimestamp, currentTimestamp, Math.max(0, this.blockTimestamp - currentTimestamp))); } // We didn't find any reason to block this txn, so it's sail yo for it... else if (debug.val) { LOG.debug( String.format( "Partition %d :: Safe to Execute %d [currentTime=%d]", this.partitionId, txnId, System.currentTimeMillis())); } } if (newState != this.state) { // note if we get non-empty but blocked if ((newState == QueueState.BLOCKED_ORDERING) || (newState == QueueState.BLOCKED_SAFETY)) { if (trace.val) LOG.trace( String.format("Partition %d :: NewState=%s --> %s", this.partitionId, newState, ts)); long txnTimestamp = TransactionIdManager.getTimestampFromTransactionId(txnId.longValue()); if (currentTimestamp == -1) currentTimestamp = System.currentTimeMillis(); // Calculate how long we need to wait before this txn is safe to run // If we're blocking on "safety", then we can use an offset based // on when the txnId was created. If we're blocking for "ordering", // then we'll want to wait for the full wait time. int waitTime = this.waitTime; if (newState == QueueState.BLOCKED_SAFETY) { waitTime = (int) Math.max(0, this.waitTime - (currentTimestamp - txnTimestamp)); } this.blockTimestamp = currentTimestamp + waitTime; if (trace.val) LOG.trace( String.format( "Partition %d :: SET blockTimestamp = %d --> %s", this.partitionId, this.blockTimestamp, ts)); if (this.blockTimestamp <= currentTimestamp) { newState = QueueState.UNBLOCKED; } if (this.profiler != null && this.lastSafeTxnId.equals(txnId) == false) this.profiler.waitTimes.put(newState == QueueState.UNBLOCKED ? 0 : waitTime); if (debug.val) LOG.debug( String.format( "Partition %d :: SET lastSafeTxnId = %d --> %s", this.partitionId, this.lastSafeTxnId, ts)); if (trace.val) { LOG.trace( String.format( "Partition %d :: SET lastSafeTxnId = %d --> %s", this.partitionId, this.lastSafeTxnId, ts)); String debug = ""; if (trace.val) { Map<String, Object> m = new LinkedHashMap<String, Object>(); m.put("Txn Init Timestamp", txnTimestamp); m.put("Current Timestamp", currentTimestamp); m.put("Block Time Remaining", (this.blockTimestamp - currentTimestamp)); debug = "\n" + StringUtil.formatMaps(m); } LOG.trace( String.format( "Partition %d :: Blocking %s for %d ms " + "[maxWait=%d, origState=%s, newState=%s]\n%s%s", this.partitionId, ts, (this.blockTimestamp - currentTimestamp), this.waitTime, this.state, newState, this.debug(), debug)); } } else if (newState == QueueState.UNBLOCKED) { if (currentTimestamp == -1) currentTimestamp = System.currentTimeMillis(); if (this.blockTimestamp > currentTimestamp) { newState = QueueState.BLOCKED_SAFETY; } } } // IF // This txn should always becomes our next safeTxnId. // This is essentially the next txn // that should be executed, but somebody *could* come along and add in // a new txn with a lower id. But that's ok because we've synchronized setting // the id up above. This is actually probably the only part of this entire method // that needs to be protected... if (txnId != null) this.lastSafeTxnId = txnId; // Set the new state if (newState != this.state) { if (trace.val) LOG.trace( String.format( "Partition %d :: ORIG[%s]->NEW[%s] / LastSafeTxn:%d", this.partitionId, this.state, newState, this.lastSafeTxnId)); if (this.profiler != null) { this.profiler.queueStates.get(this.state).stopIfStarted(); this.profiler.queueStates.get(newState).start(); } this.state = newState; // Always poke anybody that is blocking on this queue. // The txn may not be ready to run just yet, but at least they'll be // able to recompute a new sleep time. this.isReady.signal(); } else if (this.profiler != null) { this.profiler.queueStates.get(this.state).restart(); } // Sanity Check if ((this.state == QueueState.BLOCKED_ORDERING) || (this.state == QueueState.BLOCKED_SAFETY)) { assert (this.state != QueueState.BLOCKED_EMPTY); } // Make sure that we're always in a valid state to avoid livelock problems assert (this.state != QueueState.BLOCKED_SAFETY || (this.state == QueueState.BLOCKED_SAFETY && this.blockTimestamp != NULL_BLOCK_TIMESTAMP)) : String.format("Invalid state %s with NULL blocked timestamp", this.state); assert (this.state != QueueState.BLOCKED_ORDERING || (this.state == QueueState.BLOCKED_ORDERING && this.blockTimestamp != NULL_BLOCK_TIMESTAMP)) : String.format("Invalid state %s with NULL blocked timestamp", this.state); return this.state; }
@Override protected double estimateWorkloadCostImpl( final CatalogContext catalogContext, final Workload workload, final Filter filter, final Double upper_bound) throws Exception { if (debug.val) LOG.debug( "Calculating workload execution cost across " + num_intervals + " intervals for " + num_partitions + " partitions"); // (1) Grab the costs at the different time intervals // Also create the ratios that we will use to weight the interval costs final AtomicLong total_txns = new AtomicLong(0); // final HashSet<Long> trace_ids[] = new HashSet[num_intervals]; for (int i = 0; i < num_intervals; i++) { total_interval_txns[i] = 0; total_interval_queries[i] = 0; singlepartition_ctrs[i] = 0; singlepartition_with_partitions_ctrs[i] = 0; multipartition_ctrs[i] = 0; partitions_touched[i] = 0; incomplete_txn_ctrs[i] = 0; exec_mismatch_ctrs[i] = 0; incomplete_txn_histogram[i].clear(); missing_txn_histogram[i].clear(); exec_histogram[i].clear(); } // FOR // (2) Now go through the workload and estimate the partitions that each txn // will touch for the given catalog setups if (trace.val) { LOG.trace("Total # of Txns in Workload: " + workload.getTransactionCount()); if (filter != null) LOG.trace( "Workload Filter Chain: " + StringUtil.join(" ", "\n", filter.getFilters())); } // QUEUING THREAD tmp_consumers.clear(); Producer<TransactionTrace, Pair<TransactionTrace, Integer>> producer = new Producer<TransactionTrace, Pair<TransactionTrace, Integer>>( CollectionUtil.iterable(workload.iterator(filter))) { @Override public Pair<Consumer<Pair<TransactionTrace, Integer>>, Pair<TransactionTrace, Integer>> transform(TransactionTrace txn_trace) { int i = workload.getTimeInterval(txn_trace, num_intervals); assert (i >= 0) : "Invalid time interval '" + i + "'\n" + txn_trace.debug(catalogContext.database); assert (i < num_intervals) : "Invalid interval: " + i + "\n" + txn_trace.debug(catalogContext.database); total_txns.incrementAndGet(); Pair<TransactionTrace, Integer> p = Pair.of(txn_trace, i); return (Pair.of(tmp_consumers.get(i), p)); } }; // PROCESSING THREADS final int num_threads = ThreadUtil.getMaxGlobalThreads(); int interval_ctr = 0; for (int thread = 0; thread < num_threads; thread++) { // First create a new IntervalProcessor/Consumer IntervalProcessor ip = new IntervalProcessor(catalogContext, workload, filter); // Then assign it to some number of intervals for (int i = 0, cnt = (int) Math.ceil(num_intervals / (double) num_threads); i < cnt; i++) { if (interval_ctr > num_intervals) break; tmp_consumers.put(interval_ctr++, ip); if (trace.val) LOG.trace( String.format("Interval #%02d => IntervalProcessor #%02d", interval_ctr - 1, thread)); } // FOR // And make sure that we queue it up too producer.addConsumer(ip); } // FOR (threads) ThreadUtil.runGlobalPool(producer.getRunnablesList()); // BLOCKING if (debug.val) { int processed = 0; for (Consumer<?> c : producer.getConsumers()) { processed += c.getProcessedCounter(); } // FOR assert (total_txns.get() == processed) : String.format("Expected[%d] != Processed[%d]", total_txns.get(), processed); } // We have to convert all of the costs into the range of [0.0, 1.0] // For each interval, divide the number of partitions touched by the total number // of partitions that the interval could have touched (worst case scenario) final double execution_costs[] = new double[num_intervals]; StringBuilder sb = (this.isDebugEnabled() || debug.get() ? new StringBuilder() : null); Map<String, Object> debug_m = null; if (sb != null) { debug_m = new LinkedHashMap<String, Object>(); } if (debug.val) LOG.debug("Calculating execution cost for " + this.num_intervals + " intervals..."); long total_multipartition_txns = 0; for (int i = 0; i < this.num_intervals; i++) { interval_weights[i] = total_interval_txns[i] / (double) total_txns.get(); long total_txns_in_interval = (long) total_interval_txns[i]; long total_queries_in_interval = (long) total_interval_queries[i]; long num_txns = this.cost_models[i].txn_ctr.get(); long potential_txn_touches = (total_txns_in_interval * num_partitions); // TXNS double penalty = 0.0d; total_multipartition_txns += multipartition_ctrs[i]; // Divide the total number of partitions touched by... // This is the total number of partitions that we could have touched // in this interval // And this is the total number of partitions that we did actually touch if (multipartition_ctrs[i] > 0) { assert (partitions_touched[i] > 0) : "No touched partitions for interval " + i; double cost = (partitions_touched[i] / (double) potential_txn_touches); if (this.use_multitpartition_penalty) { penalty = this.multipartition_penalty * (1.0d + (multipartition_ctrs[i] / (double) total_txns_in_interval)); assert (penalty >= 1.0) : "The multipartition penalty is less than one: " + penalty; cost *= penalty; } execution_costs[i] = Math.min(cost, (double) potential_txn_touches); } // For each txn that wasn't even evaluated, add all of the // partitions to the incomplete histogram if (num_txns < total_txns_in_interval) { if (trace.val) LOG.trace( "Adding " + (total_txns_in_interval - num_txns) + " entries to the incomplete histogram for interval #" + i); for (long ii = num_txns; ii < total_txns_in_interval; ii++) { missing_txn_histogram[i].put(all_partitions); } // WHILE } if (sb != null) { tmp_penalties.add(penalty); tmp_total.add(total_txns_in_interval); tmp_touched.add(partitions_touched[i]); tmp_potential.add(potential_txn_touches); Map<String, Object> inner = new LinkedHashMap<String, Object>(); inner.put("Partitions Touched", partitions_touched[i]); inner.put("Potential Touched", potential_txn_touches); inner.put("Multi-Partition Txns", multipartition_ctrs[i]); inner.put("Total Txns", total_txns_in_interval); inner.put("Total Queries", total_queries_in_interval); inner.put("Missing Txns", (total_txns_in_interval - num_txns)); inner.put("Cost", String.format("%.05f", execution_costs[i])); inner.put("Exec Txns", exec_histogram[i].getSampleCount()); debug_m.put("Interval #" + i, inner); } } // FOR if (sb != null) { Map<String, Object> m0 = new LinkedHashMap<String, Object>(); m0.put("SinglePartition Txns", (total_txns.get() - total_multipartition_txns)); m0.put("MultiPartition Txns", total_multipartition_txns); m0.put( "Total Txns", String.format( "%d [%.06f]", total_txns.get(), (1.0d - (total_multipartition_txns / (double) total_txns.get())))); Map<String, Object> m1 = new LinkedHashMap<String, Object>(); m1.put("Touched Partitions", tmp_touched); m1.put("Potential Partitions", tmp_potential); m1.put("Total Partitions", tmp_total); m1.put("Penalties", tmp_penalties); sb.append(StringUtil.formatMaps(debug_m, m0, m1)); if (debug.val) LOG.debug("**** Execution Cost ****\n" + sb); this.appendDebugMessage(sb); } // LOG.debug("Execution By Intervals:\n" + sb.toString()); // (3) We then need to go through and grab the histograms of partitions were accessed if (sb != null) { if (debug.val) LOG.debug("Calculating skew factor for " + this.num_intervals + " intervals..."); debug_histograms.clear(); sb = new StringBuilder(); } for (int i = 0; i < this.num_intervals; i++) { ObjectHistogram<Integer> histogram_txn = this.cost_models[i].getTxnPartitionAccessHistogram(); ObjectHistogram<Integer> histogram_query = this.cost_models[i].getQueryPartitionAccessHistogram(); this.histogram_query_partitions.put(histogram_query); long num_queries = this.cost_models[i].query_ctr.get(); this.query_ctr.addAndGet(num_queries); // DEBUG SingleSitedCostModel inner_costModel = (SingleSitedCostModel) this.cost_models[i]; boolean is_valid = (partitions_touched[i] + singlepartition_with_partitions_ctrs[i]) == (this.cost_models[i].getTxnPartitionAccessHistogram().getSampleCount() + exec_mismatch_ctrs[i]); if (!is_valid) { LOG.error("Transaction Entries: " + inner_costModel.getTransactionCacheEntries().size()); ObjectHistogram<Integer> check = new ObjectHistogram<Integer>(); for (TransactionCacheEntry tce : inner_costModel.getTransactionCacheEntries()) { check.put(tce.getTouchedPartitions()); // LOG.error(tce.debug() + "\n"); } LOG.error( "Check Touched Partitions: sample=" + check.getSampleCount() + ", values=" + check.getValueCount()); LOG.error( "Cache Touched Partitions: sample=" + this.cost_models[i].getTxnPartitionAccessHistogram().getSampleCount() + ", values=" + this.cost_models[i].getTxnPartitionAccessHistogram().getValueCount()); int qtotal = inner_costModel.getAllQueryCacheEntries().size(); int ctr = 0; int multip = 0; for (QueryCacheEntry qce : inner_costModel.getAllQueryCacheEntries()) { ctr += (qce.getAllPartitions().isEmpty() ? 0 : 1); multip += (qce.getAllPartitions().size() > 1 ? 1 : 0); } // FOR LOG.error("# of QueryCacheEntries with Touched Partitions: " + ctr + " / " + qtotal); LOG.error("# of MultiP QueryCacheEntries: " + multip); } assert (is_valid) : String.format( "Partitions Touched by Txns Mismatch in Interval #%d\n" + "(partitions_touched[%d] + singlepartition_with_partitions_ctrs[%d]) != " + "(histogram_txn[%d] + exec_mismatch_ctrs[%d])", i, partitions_touched[i], singlepartition_with_partitions_ctrs[i], this.cost_models[i].getTxnPartitionAccessHistogram().getSampleCount(), exec_mismatch_ctrs[i]); this.histogram_java_partitions.put(this.cost_models[i].getJavaExecutionHistogram()); this.histogram_txn_partitions.put(histogram_txn); long num_txns = this.cost_models[i].txn_ctr.get(); assert (num_txns >= 0) : "The transaction counter at interval #" + i + " is " + num_txns; this.txn_ctr.addAndGet(num_txns); // Calculate the skew factor at this time interval // XXX: Should the number of txns be the total number of unique txns // that were executed or the total number of times a txn touched the partitions? // XXX: What do we do when the number of elements that we are examining is zero? // I guess the cost just needs to be zero? // XXX: What histogram do we want to use? target_histogram.clear(); target_histogram.put(histogram_txn); // For each txn that we haven't gotten an estimate for at this interval, // we're going mark it as being broadcast to all partitions. That way the access // histogram will look uniform. Then as more information is added, we will // This is an attempt to make sure that the skew cost never decreases but only increases long total_txns_in_interval = (long) total_interval_txns[i]; if (sb != null) { debug_histograms.put("Incomplete Txns", incomplete_txn_histogram[i]); debug_histograms.put("Missing Txns", missing_txn_histogram[i]); debug_histograms.put( "Target Partitions (BEFORE)", new ObjectHistogram<Integer>(target_histogram)); debug_histograms.put("Target Partitions (AFTER)", target_histogram); } // Merge the values from incomplete histogram into the target // histogram target_histogram.put(incomplete_txn_histogram[i]); target_histogram.put(missing_txn_histogram[i]); exec_histogram[i].put(missing_txn_histogram[i]); long num_elements = target_histogram.getSampleCount(); // The number of partition touches should never be greater than our // potential touches assert (num_elements <= (total_txns_in_interval * num_partitions)) : "New Partitions Touched Sample Count [" + num_elements + "] < " + "Maximum Potential Touched Count [" + (total_txns_in_interval * num_partitions) + "]"; if (sb != null) { Map<String, Object> m = new LinkedHashMap<String, Object>(); for (String key : debug_histograms.keySet()) { ObjectHistogram<?> h = debug_histograms.get(key); m.put( key, String.format("[Sample=%d, Value=%d]\n%s", h.getSampleCount(), h.getValueCount(), h)); } // FOR sb.append( String.format( "INTERVAL #%d [total_txns_in_interval=%d, num_txns=%d, incomplete_txns=%d]\n%s", i, total_txns_in_interval, num_txns, incomplete_txn_ctrs[i], StringUtil.formatMaps(m))); } // Txn Skew if (num_elements == 0) { txn_skews[i] = 0.0d; } else { txn_skews[i] = SkewFactorUtil.calculateSkew(num_partitions, num_elements, target_histogram); } // Exec Skew if (exec_histogram[i].getSampleCount() == 0) { exec_skews[i] = 0.0d; } else { exec_skews[i] = SkewFactorUtil.calculateSkew( num_partitions, exec_histogram[i].getSampleCount(), exec_histogram[i]); } total_skews[i] = (0.5 * exec_skews[i]) + (0.5 * txn_skews[i]); if (sb != null) { sb.append("Txn Skew = " + MathUtil.roundToDecimals(txn_skews[i], 6) + "\n"); sb.append("Exec Skew = " + MathUtil.roundToDecimals(exec_skews[i], 6) + "\n"); sb.append("Total Skew = " + MathUtil.roundToDecimals(total_skews[i], 6) + "\n"); sb.append(StringUtil.DOUBLE_LINE); } } // FOR if (sb != null && sb.length() > 0) { if (debug.val) LOG.debug("**** Skew Factor ****\n" + sb); this.appendDebugMessage(sb); } if (trace.val) { for (int i = 0; i < num_intervals; i++) { LOG.trace( "Time Interval #" + i + "\n" + "Total # of Txns: " + this.cost_models[i].txn_ctr.get() + "\n" + "Multi-Partition Txns: " + multipartition_ctrs[i] + "\n" + "Execution Cost: " + execution_costs[i] + "\n" + "ProcHistogram:\n" + this.cost_models[i].getProcedureHistogram().toString() + "\n" + // "TransactionsPerPartitionHistogram:\n" + // this.cost_models[i].getTxnPartitionAccessHistogram() // + "\n" + StringUtil.SINGLE_LINE); } } // (3) We can now calculate the final total estimate cost of this workload as the following // Just take the simple ratio of mp txns / all txns this.last_execution_cost = MathUtil.weightedMean( execution_costs, total_interval_txns); // MathUtil.roundToDecimals(MathUtil.geometricMean(execution_costs, // MathUtil.GEOMETRIC_MEAN_ZERO), // 10); // The final skew cost needs to be weighted by the percentage of txns running in that interval // This will cause the partitions with few txns this.last_skew_cost = MathUtil.weightedMean( total_skews, total_interval_txns); // roundToDecimals(MathUtil.geometricMean(entropies, // MathUtil.GEOMETRIC_MEAN_ZERO), // 10); double new_final_cost = (this.use_execution ? (this.execution_weight * this.last_execution_cost) : 0) + (this.use_skew ? (this.skew_weight * this.last_skew_cost) : 0); if (sb != null) { Map<String, Object> m = new LinkedHashMap<String, Object>(); m.put("Total Txns", total_txns.get()); m.put("Interval Txns", Arrays.toString(total_interval_txns)); m.put("Execution Costs", Arrays.toString(execution_costs)); m.put("Skew Factors", Arrays.toString(total_skews)); m.put("Txn Skew", Arrays.toString(txn_skews)); m.put("Exec Skew", Arrays.toString(exec_skews)); m.put("Interval Weights", Arrays.toString(interval_weights)); m.put( "Final Cost", String.format( "%f = %f + %f", new_final_cost, this.last_execution_cost, this.last_skew_cost)); if (debug.val) LOG.debug(StringUtil.formatMaps(m)); this.appendDebugMessage(StringUtil.formatMaps(m)); } this.last_final_cost = new_final_cost; return (MathUtil.roundToDecimals(this.last_final_cost, 5)); }
@Override public void process(Pair<TransactionTrace, Integer> p) { assert (p != null); final TransactionTrace txn_trace = p.getFirst(); final int i = p.getSecond(); // Interval final int txn_weight = (use_txn_weights ? txn_trace.getWeight() : 1); final String proc_key = CatalogKey.createKey(CatalogUtil.DEFAULT_DATABASE_NAME, txn_trace.getCatalogItemName()); // Terrible Hack: Assume that we are using the SingleSitedCostModel // and that // it will return fixed values based on whether the txn is // single-partitioned or not SingleSitedCostModel singlesited_cost_model = (SingleSitedCostModel) cost_models[i]; total_interval_txns[i] += txn_weight; total_interval_queries[i] += (txn_trace.getQueryCount() * txn_weight); histogram_procs.put(proc_key, txn_weight); try { singlesited_cost_model.estimateTransactionCost(catalogContext, workload, filter, txn_trace); TransactionCacheEntry txn_entry = singlesited_cost_model.getTransactionCacheEntry(txn_trace); assert (txn_entry != null) : "No txn entry for " + txn_trace; Collection<Integer> partitions = txn_entry.getTouchedPartitions(); // If the txn runs on only one partition, then the cost is // nothing if (txn_entry.isSinglePartitioned()) { singlepartition_ctrs[i] += txn_weight; if (!partitions.isEmpty()) { assert (txn_entry.getAllTouchedPartitionsHistogram().getValueCount() == 1) : txn_entry + " says it was single-partitioned but the partition count says otherwise:\n" + txn_entry.debug(); singlepartition_with_partitions_ctrs[i] += txn_weight; } histogram_sp_procs.put(proc_key, txn_weight); // If the txn runs on multiple partitions, then the cost // is... // XXX 2010-06-28: The number of partitions that the txn // touches divided by the total number of partitions // XXX 2010-07-02: The histogram for the total number of // partitions touched by all of the queries // in the transaction. This ensures that txns with just one // multi-partition query // isn't weighted the same as a txn with many // multi-partition queries } else { assert (!partitions.isEmpty()) : "No touched partitions for " + txn_trace; if (partitions.size() == 1 && txn_entry.getExecutionPartition() != HStoreConstants.NULL_PARTITION_ID) { assert (CollectionUtil.first(partitions) != txn_entry.getExecutionPartition()) : txn_entry.debug(); exec_mismatch_ctrs[i] += txn_weight; partitions_touched[i] += txn_weight; } else { assert (partitions.size() > 1) : String.format( "%s is not marked as single-partition but it only touches one partition\n%s", txn_trace, txn_entry.debug()); } partitions_touched[i] += (partitions.size() * txn_weight); // Txns multipartition_ctrs[i] += txn_weight; histogram_mp_procs.put(proc_key, txn_weight); } Integer base_partition = txn_entry.getExecutionPartition(); if (base_partition != null) { exec_histogram[i].put(base_partition, txn_weight); } else { exec_histogram[i].put(all_partitions, txn_weight); } if (debug.val) { // && // txn_trace.getCatalogItemName().equalsIgnoreCase("DeleteCallForwarding")) // { Procedure catalog_proc = txn_trace.getCatalogItem(catalogContext.database); Map<String, Object> inner = new LinkedHashMap<String, Object>(); for (Statement catalog_stmt : catalog_proc.getStatements()) { inner.put(catalog_stmt.fullName(), CatalogUtil.getReferencedTables(catalog_stmt)); } Map<String, Object> m = new LinkedHashMap<String, Object>(); m.put(txn_trace.toString(), null); m.put("Interval", i); m.put("Single-Partition", txn_entry.isSinglePartitioned()); m.put("Base Partition", base_partition); m.put("Touched Partitions", partitions); m.put(catalog_proc.fullName(), inner); LOG.debug(StringUtil.formatMaps(m)); } // We need to keep a count of the number txns that didn't have // all of its queries estimated // completely so that we can update the access histograms down // below for entropy calculations // Note that this is at the txn level, not the query level. if (!txn_entry.isComplete()) { incomplete_txn_ctrs[i] += txn_weight; tmp_missingPartitions.clear(); tmp_missingPartitions.addAll(all_partitions); tmp_missingPartitions.removeAll(txn_entry.getTouchedPartitions()); // Update the histogram for this interval to keep track of // how many times we need to // increase the partition access histogram incomplete_txn_histogram[i].put(tmp_missingPartitions, txn_weight); if (trace.val) { Map<String, Object> m = new LinkedHashMap<String, Object>(); m.put(String.format("Marking %s as incomplete in interval #%d", txn_trace, i), null); m.put("Examined Queries", txn_entry.getExaminedQueryCount()); m.put("Total Queries", txn_entry.getTotalQueryCount()); m.put("Touched Partitions", txn_entry.getTouchedPartitions()); m.put("Missing Partitions", tmp_missingPartitions); LOG.trace(StringUtil.formatMaps(m)); } } } catch (Exception ex) { CatalogUtil.saveCatalog(catalogContext.catalog, CatalogUtil.CATALOG_FILENAME); throw new RuntimeException( "Failed to estimate cost for " + txn_trace.getCatalogItemName() + " at interval " + i, ex); } }
private void buildDatabaseElement(Document doc, final Element database) { // /project/database/users final Element users = doc.createElement("users"); database.appendChild(users); // users/user if (m_users.isEmpty()) { final Element user = doc.createElement("user"); user.setAttribute("name", "default"); user.setAttribute("groups", "default"); user.setAttribute("password", ""); user.setAttribute("sysproc", "true"); user.setAttribute("adhoc", "true"); users.appendChild(user); } else { for (final UserInfo info : m_users) { final Element user = doc.createElement("user"); user.setAttribute("name", info.name); user.setAttribute("password", info.password); user.setAttribute("sysproc", info.sysproc ? "true" : "false"); user.setAttribute("adhoc", info.adhoc ? "true" : "false"); // build up user/@groups. This attribute must be redesigned if (info.groups.length > 0) { final StringBuilder groups = new StringBuilder(); for (final String group : info.groups) { if (groups.length() > 0) groups.append(","); groups.append(group); } user.setAttribute("groups", groups.toString()); } users.appendChild(user); } } // /project/database/groups final Element groups = doc.createElement("groups"); database.appendChild(groups); // groups/group if (m_groups.isEmpty()) { final Element group = doc.createElement("group"); group.setAttribute("name", "default"); group.setAttribute("sysproc", "true"); group.setAttribute("adhoc", "true"); groups.appendChild(group); } else { for (final GroupInfo info : m_groups) { final Element group = doc.createElement("group"); group.setAttribute("name", info.name); group.setAttribute("sysproc", info.sysproc ? "true" : "false"); group.setAttribute("adhoc", info.adhoc ? "true" : "false"); groups.appendChild(group); } } // /project/database/schemas final Element schemas = doc.createElement("schemas"); database.appendChild(schemas); // schemas/schema for (final String schemaPath : m_schemas) { final Element schema = doc.createElement("schema"); schema.setAttribute("path", schemaPath); schemas.appendChild(schema); } // /project/database/procedures final Element procedures = doc.createElement("procedures"); database.appendChild(procedures); // procedures/procedure for (final ProcedureInfo procedure : m_procedures) { if (procedure.cls == null) continue; assert (procedure.sql == null); final Element proc = doc.createElement("procedure"); proc.setAttribute("class", procedure.cls.getName()); // build up @users. This attribute should be redesigned if (procedure.users.length > 0) { final StringBuilder userattr = new StringBuilder(); for (final String user : procedure.users) { if (userattr.length() > 0) userattr.append(","); userattr.append(user); } proc.setAttribute("users", userattr.toString()); } // build up @groups. This attribute should be redesigned if (procedure.groups.length > 0) { final StringBuilder groupattr = new StringBuilder(); for (final String group : procedure.groups) { if (groupattr.length() > 0) groupattr.append(","); groupattr.append(group); } proc.setAttribute("groups", groupattr.toString()); } // HACK: Prefetchable Statements if (m_prefetchQueries.containsKey(procedure.cls.getSimpleName())) { Collection<String> stmtNames = m_prefetchQueries.get(procedure.cls.getSimpleName()); proc.setAttribute("prefetchable", StringUtil.join(",", stmtNames)); } // HACK: Deferrable Statements if (m_deferQueries.containsKey(procedure.cls.getSimpleName())) { Collection<String> stmtNames = m_deferQueries.get(procedure.cls.getSimpleName()); proc.setAttribute("deferrable", StringUtil.join(",", stmtNames)); } procedures.appendChild(proc); } // procedures/procedures (that are stmtprocedures) for (final ProcedureInfo procedure : m_procedures) { if (procedure.sql == null) continue; assert (procedure.cls == null); final Element proc = doc.createElement("procedure"); proc.setAttribute("class", procedure.name); if (procedure.partitionInfo != null) ; proc.setAttribute("partitioninfo", procedure.partitionInfo); // build up @users. This attribute should be redesigned if (procedure.users.length > 0) { final StringBuilder userattr = new StringBuilder(); for (final String user : procedure.users) { if (userattr.length() > 0) userattr.append(","); userattr.append(user); } proc.setAttribute("users", userattr.toString()); } // build up @groups. This attribute should be redesigned if (procedure.groups.length > 0) { final StringBuilder groupattr = new StringBuilder(); for (final String group : procedure.groups) { if (groupattr.length() > 0) groupattr.append(","); groupattr.append(group); } proc.setAttribute("groups", groupattr.toString()); } final Element sql = doc.createElement("sql"); proc.appendChild(sql); final Text sqltext = doc.createTextNode(procedure.sql); sql.appendChild(sqltext); procedures.appendChild(proc); } if (m_partitionInfos.size() > 0) { // /project/database/partitions final Element partitions = doc.createElement("partitions"); database.appendChild(partitions); // partitions/table for (final Entry<String, String> partitionInfo : m_partitionInfos.entrySet()) { final Element table = doc.createElement("partition"); table.setAttribute("table", partitionInfo.getKey()); table.setAttribute("column", partitionInfo.getValue()); partitions.appendChild(table); } } // Evictable Tables if (m_evictableTables.isEmpty() == false) { final Element evictables = doc.createElement("evictables"); database.appendChild(evictables); // Table entries for (String tableName : m_evictableTables) { final Element table = doc.createElement("evictable"); table.setAttribute("table", tableName); evictables.appendChild(table); } } // Vertical Partitions if (m_replicatedSecondaryIndexes.size() > 0) { // /project/database/partitions final Element verticalpartitions = doc.createElement("verticalpartitions"); database.appendChild(verticalpartitions); // partitions/table for (String tableName : m_replicatedSecondaryIndexes.keySet()) { Pair<Boolean, Collection<String>> p = m_replicatedSecondaryIndexes.get(tableName); Boolean createIndex = p.getFirst(); Collection<String> columnNames = p.getSecond(); final Element vp = doc.createElement("verticalpartition"); vp.setAttribute("table", tableName); vp.setAttribute("indexed", createIndex.toString()); for (final String columnName : columnNames) { final Element column = doc.createElement("column"); column.setTextContent(columnName); vp.appendChild(column); } // FOR (cols) verticalpartitions.appendChild(vp); } // FOR (tables) } // /project/database/classdependencies final Element classdeps = doc.createElement("classdependencies"); database.appendChild(classdeps); // classdependency for (final Class<?> supplemental : m_supplementals) { final Element supp = doc.createElement("classdependency"); supp.setAttribute("class", supplemental.getName()); classdeps.appendChild(supp); } // project/database/exports if (m_elloader != null) { final Element exports = doc.createElement("exports"); database.appendChild(exports); final Element conn = doc.createElement("connector"); conn.setAttribute("class", m_elloader); conn.setAttribute("enabled", m_elenabled ? "true" : "false"); // turn list into stupid comma separated attribute list String usersattr = ""; if (m_elAuthUsers != null) { for (String s : m_elAuthUsers) { if (usersattr.isEmpty()) { usersattr += s; } else { usersattr += "," + s; } } conn.setAttribute("users", usersattr); } // turn list into stupid comma separated attribute list String groupsattr = ""; if (m_elAuthGroups != null) { for (String s : m_elAuthGroups) { if (groupsattr.isEmpty()) { groupsattr += s; } else { groupsattr += "," + s; } } conn.setAttribute("groups", groupsattr); } exports.appendChild(conn); if (m_eltTables.size() > 0) { final Element tables = doc.createElement("tables"); conn.appendChild(tables); for (ELTTableInfo info : m_eltTables) { final Element table = doc.createElement("table"); table.setAttribute("name", info.m_tablename); table.setAttribute("exportonly", info.m_export_only ? "true" : "false"); tables.appendChild(table); } } } if (m_snapshotPath != null) { final Element snapshot = doc.createElement("snapshot"); snapshot.setAttribute("frequency", m_snapshotFrequency); snapshot.setAttribute("path", m_snapshotPath); snapshot.setAttribute("prefix", m_snapshotPrefix); snapshot.setAttribute("retain", Integer.toString(m_snapshotRetain)); database.appendChild(snapshot); } }
public static MaterializedViewInfo addVerticalPartition( final Table catalog_tbl, final Collection<Column> catalog_cols, final boolean createIndex) throws Exception { assert (catalog_cols.isEmpty() == false); Database catalog_db = ((Database) catalog_tbl.getParent()); String viewName = getNextVerticalPartitionName(catalog_tbl, catalog_cols); if (debug.get()) LOG.debug( String.format( "Adding Vertical Partition %s for %s: %s", viewName, catalog_tbl, catalog_cols)); // Create a new virtual table Table virtual_tbl = catalog_db.getTables().get(viewName); if (virtual_tbl == null) { virtual_tbl = catalog_db.getTables().add(viewName); } virtual_tbl.setIsreplicated(true); virtual_tbl.setMaterializer(catalog_tbl); virtual_tbl.setSystable(true); virtual_tbl.getColumns().clear(); // Create MaterializedView and link it to the virtual table MaterializedViewInfo catalog_view = catalog_tbl.getViews().add(viewName); catalog_view.setVerticalpartition(true); catalog_view.setDest(virtual_tbl); List<Column> indexColumns = new ArrayList<Column>(); Column partition_col = catalog_tbl.getPartitioncolumn(); if (partition_col instanceof VerticalPartitionColumn) { partition_col = ((VerticalPartitionColumn) partition_col).getHorizontalColumn(); } if (debug.get()) LOG.debug(catalog_tbl.getName() + " Partition Column: " + partition_col); int i = 0; assert (catalog_cols != null); assert (catalog_cols.isEmpty() == false) : "No vertical partitioning columns for " + catalog_view.fullName(); for (Column catalog_col : catalog_cols) { // MaterializedView ColumnRef ColumnRef catalog_ref = catalog_view.getGroupbycols().add(catalog_col.getName()); catalog_ref.setColumn(catalog_col); catalog_ref.setIndex(i++); // VirtualTable Column Column virtual_col = virtual_tbl.getColumns().add(catalog_col.getName()); virtual_col.setDefaulttype(catalog_col.getDefaulttype()); virtual_col.setDefaultvalue(catalog_col.getDefaultvalue()); virtual_col.setIndex(catalog_col.getIndex()); virtual_col.setNullable(catalog_col.getNullable()); virtual_col.setSize(catalog_col.getSize()); virtual_col.setType(catalog_col.getType()); if (debug.get()) LOG.debug(String.format("Added VerticalPartition column %s", virtual_col.fullName())); // If they want an index, then we'll make one based on every column except for the column // that the table is partitioned on if (createIndex) { boolean include = true; if (partition_col instanceof MultiColumn) { include = (((MultiColumn) partition_col).contains(catalog_col) == false); } else if (catalog_col.equals(partition_col)) { include = false; } if (include) indexColumns.add(virtual_col); } } // FOR if (createIndex) { if (indexColumns.isEmpty()) { Map<String, Object> m = new ListOrderedMap<String, Object>(); m.put("Partition Column", partition_col); m.put("VP Table Columns", virtual_tbl.getColumns()); m.put("Passed-in Columns", CatalogUtil.debug(catalog_cols)); LOG.error("Failed to find index columns\n" + StringUtil.formatMaps(m)); throw new Exception(String.format("No columns selected for index on %s", viewName)); } String idxName = "SYS_IDX_" + viewName; Index virtual_idx = virtual_tbl.getIndexes().get(idxName); if (virtual_idx == null) { virtual_idx = virtual_tbl.getIndexes().add(idxName); } virtual_idx.getColumns().clear(); IndexType idxType = (indexColumns.size() == 1 ? IndexType.HASH_TABLE : IndexType.BALANCED_TREE); virtual_idx.setType(idxType.getValue()); i = 0; for (Column catalog_col : indexColumns) { ColumnRef cref = virtual_idx.getColumns().add(catalog_col.getTypeName()); cref.setColumn(catalog_col); cref.setIndex(i++); } // FOR if (debug.get()) LOG.debug( String.format( "Created %s index '%s' for vertical partition '%s'", idxType, idxName, viewName)); } return (catalog_view); }
/** * @param projectFileURL URL of the project file. * @param clusterConfig Object containing desired physical cluster parameters * @param jarOutputPath The location to put the finished JAR to. * @param output Where to print status/errors to, usually stdout. * @param procInfoOverrides Optional overridden values for procedure annotations. */ public boolean compile( final String projectFileURL, final ClusterConfig clusterConfig, final String jarOutputPath, final PrintStream output, final Map<String, ProcInfoData> procInfoOverrides) { m_hsql = null; m_projectFileURL = projectFileURL; m_jarOutputPath = jarOutputPath; m_outputStream = output; // use this map as default annotation values m_procInfoOverrides = procInfoOverrides; LOG.l7dlog( Level.DEBUG, LogKeys.compiler_VoltCompiler_LeaderAndHostCountAndSitesPerHost.name(), new Object[] { clusterConfig.getLeaderAddress(), clusterConfig.getHostCount(), clusterConfig.getSitesPerHost() }, null); // do all the work to get the catalog final Catalog catalog = compileCatalog(projectFileURL, clusterConfig); if (catalog == null) { LOG.error( "VoltCompiler had " + m_errors.size() + " errors\n" + StringUtil.join("\n", m_errors)); return (false); } // WRITE CATALOG TO JAR HERE final String catalogCommands = catalog.serialize(); byte[] catalogBytes = null; try { catalogBytes = catalogCommands.getBytes("UTF-8"); } catch (final UnsupportedEncodingException e1) { addErr("Can't encode the compiled catalog file correctly"); return false; } // Create Dtxn.Coordinator configuration for cluster // byte[] dtxnConfBytes = null; // try { // dtxnConfBytes = HStoreDtxnConf.toHStoreDtxnConf(catalog).getBytes("UTF-8"); // } catch (final Exception e1) { // addErr("Can't encode the Dtxn.Coordinator configuration file correctly"); // return false; // } try { // m_jarBuilder.addEntry("dtxn.conf", dtxnConfBytes); m_jarBuilder.addEntry(CatalogUtil.CATALOG_FILENAME, catalogBytes); m_jarBuilder.addEntry("project.xml", new File(projectFileURL)); for (final Entry<String, String> e : m_ddlFilePaths.entrySet()) m_jarBuilder.addEntry(e.getKey(), new File(e.getValue())); m_jarBuilder.writeJarToDisk(jarOutputPath); } catch (final VoltCompilerException e) { return false; } assert (!hasErrors()); if (hasErrors()) { return false; } return true; }
public TPCCSimulation( TPCCSimulation.ProcCaller client, RandomGenerator generator, Clock clock, ScaleParameters parameters, TPCCConfig config, double skewFactor, Catalog catalog) { assert parameters != null; this.client = client; this.generator = generator; this.clock = clock; this.parameters = parameters; this.affineWarehouse = lastAssignedWarehouseId; this.skewFactor = skewFactor; this.config = config; if (config.neworder_skew_warehouse) { if (debug.val) LOG.debug("Enabling W_ID Zipfian Skew: " + skewFactor); this.zipf = new RandomDistribution.Zipf( new Random(), parameters.starting_warehouse, parameters.last_warehouse + 1, Math.max(1.001d, this.skewFactor)); this.custom_skew = new RandomDistribution.HotWarmCold( new Random(), parameters.starting_warehouse + 1, parameters.last_warehouse, TPCCConstants.HOT_DATA_WORKLOAD_SKEW, TPCCConstants.HOT_DATA_SIZE, TPCCConstants.WARM_DATA_WORKLOAD_SKEW, TPCCConstants.WARM_DATA_SIZE); } if (config.warehouse_debug) { LOG.info("Enabling WAREHOUSE debug mode"); } lastAssignedWarehouseId += 1; if (lastAssignedWarehouseId > parameters.last_warehouse) lastAssignedWarehouseId = 1; if (debug.val) { LOG.debug(this.toString()); } if (config.neworder_multip_remote) { synchronized (TPCCSimulation.class) { if (remoteWarehouseIds == null) { remoteWarehouseIds = new HashMap<Integer, List<Integer>>(); HashMap<Integer, Integer> partitionToSite = new HashMap<Integer, Integer>(); Database catalog_db = CatalogUtil.getDatabase(catalog); DefaultHasher hasher = new DefaultHasher(catalog_db); for (Site s : CatalogUtil.getCluster(catalog_db).getSites()) { for (Partition p : s.getPartitions()) partitionToSite.put(p.getId(), s.getId()); } // FOR for (int w_id0 = parameters.starting_warehouse; w_id0 <= parameters.last_warehouse; w_id0++) { final int partition0 = hasher.hash(w_id0); final int site0 = partitionToSite.get(partition0); final List<Integer> rList = new ArrayList<Integer>(); for (int w_id1 = parameters.starting_warehouse; w_id1 <= parameters.last_warehouse; w_id1++) { // Figure out what partition this W_ID maps to int partition1 = hasher.hash(w_id1); // Check whether this partition is on our same local site int site1 = partitionToSite.get(partition1); if (site0 != site1) rList.add(w_id1); } // FOR remoteWarehouseIds.put(w_id0, rList); } // FOR LOG.debug("NewOrder Remote W_ID Mapping\n" + StringUtil.formatMaps(remoteWarehouseIds)); } } // SYNCH } }
/** * MAIN! * * @param vargs * @throws Exception */ public static void main(String[] vargs) throws Exception { ArgumentsParser args = ArgumentsParser.load(vargs); args.require( ArgumentsParser.PARAM_CATALOG, ArgumentsParser.PARAM_WORKLOAD, ArgumentsParser.PARAM_PARTITION_PLAN, ArgumentsParser.PARAM_DESIGNER_INTERVALS // ArgumentsParser.PARAM_DESIGNER_HINTS ); assert (args.workload.getTransactionCount() > 0) : "No transactions were loaded from " + args.workload; if (args.hasParam(ArgumentsParser.PARAM_CATALOG_HOSTS)) { ClusterConfiguration cc = new ClusterConfiguration(args.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS)); args.updateCatalog(FixCatalog.cloneCatalog(args.catalog, cc), null); } // If given a PartitionPlan, then update the catalog File pplan_path = new File(args.getParam(ArgumentsParser.PARAM_PARTITION_PLAN)); if (pplan_path.exists()) { PartitionPlan pplan = new PartitionPlan(); pplan.load(pplan_path, args.catalog_db); if (args.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_REMOVE_PROCS, false)) { for (Procedure catalog_proc : pplan.proc_entries.keySet()) { pplan.setNullProcParameter(catalog_proc); } // FOR } if (args.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_RANDOM_PROCS, false)) { for (Procedure catalog_proc : pplan.proc_entries.keySet()) { pplan.setRandomProcParameter(catalog_proc); } // FOR } pplan.apply(args.catalog_db); System.out.println("Applied PartitionPlan '" + pplan_path + "' to catalog\n" + pplan); System.out.print(StringUtil.DOUBLE_LINE); if (args.hasParam(ArgumentsParser.PARAM_PARTITION_PLAN_OUTPUT)) { String output = args.getParam(ArgumentsParser.PARAM_PARTITION_PLAN_OUTPUT); if (output.equals("-")) output = pplan_path.getAbsolutePath(); pplan.save(new File(output)); System.out.println("Saved PartitionPlan to '" + output + "'"); } } else { System.err.println("PartitionPlan file '" + pplan_path + "' does not exist. Ignoring..."); } System.out.flush(); int num_intervals = args.num_intervals; // getIntParam(ArgumentsParser.PARAM_DESIGNER_INTERVALS); TimeIntervalCostModel<SingleSitedCostModel> costmodel = new TimeIntervalCostModel<SingleSitedCostModel>( args.catalogContext, SingleSitedCostModel.class, num_intervals); if (args.hasParam(ArgumentsParser.PARAM_DESIGNER_HINTS)) costmodel.applyDesignerHints(args.designer_hints); double cost = costmodel.estimateWorkloadCost(args.catalogContext, args.workload); Map<String, Object> m = new LinkedHashMap<String, Object>(); m.put("PARTITIONS", CatalogUtil.getNumberOfPartitions(args.catalog_db)); m.put("INTERVALS", args.num_intervals); m.put("EXEC COST", costmodel.last_execution_cost); m.put("SKEW COST", costmodel.last_skew_cost); m.put("TOTAL COST", cost); m.put("PARTITIONS TOUCHED", costmodel.getTxnPartitionAccessHistogram().getSampleCount()); System.out.println(StringUtil.formatMaps(m)); // long total = 0; m.clear(); // for (int i = 0; i < num_intervals; i++) { // SingleSitedCostModel cm = costmodel.getCostModel(i); // Histogram<Integer> h = cm.getTxnPartitionAccessHistogram(); // m.put(String.format("Interval %02d", i), // cm.getTxnPartitionAccessHistogram()); // total += h.getSampleCount(); // h.setKeepZeroEntries(true); // for (Integer partition : // CatalogUtil.getAllPartitionIds(args.catalog_db)) { // if (h.contains(partition) == false) h.put(partition, 0); // } // System.out.println(StringUtil.box("Interval #" + i, "+", 100) + "\n" // + h); // System.out.println(); // } // FOR // System.out.println(StringUtil.formatMaps(m)); // System.err.println("TOTAL: " + total); }