/** Save the profile information into the database */
protected final void saveProfile(BenchmarkComponent baseClient) {
// CONFIG_PROFILE
Table catalog_tbl =
catalogContext.database.getTables().get(SEATSConstants.TABLENAME_CONFIG_PROFILE);
VoltTable vt = CatalogUtil.getVoltTable(catalog_tbl);
assert (vt != null);
vt.addRow(
this.scale_factor, // CFP_SCALE_FACTOR
this.airport_max_customer_id.toJSONString(), // CFP_AIPORT_MAX_CUSTOMER
this.flight_start_date, // CFP_FLIGHT_START
this.flight_upcoming_date, // CFP_FLIGHT_UPCOMING
this.flight_past_days, // CFP_FLIGHT_PAST_DAYS
this.flight_future_days, // CFP_FLIGHT_FUTURE_DAYS
this.num_flights, // CFP_NUM_FLIGHTS
this.num_customers, // CFP_NUM_CUSTOMERS
this.num_reservations, // CFP_NUM_RESERVATIONS
JSONUtil.toJSONString(this.code_id_xref) // CFP_CODE_ID_XREF
);
if (debug.val)
LOG.debug(String.format("Saving profile information into %s\n%s", catalog_tbl, this));
baseClient.loadVoltTable(catalog_tbl.getName(), vt);
// CONFIG_HISTOGRAMS
catalog_tbl =
catalogContext.database.getTables().get(SEATSConstants.TABLENAME_CONFIG_HISTOGRAMS);
vt = CatalogUtil.getVoltTable(catalog_tbl);
assert (vt != null);
for (Entry<String, Histogram<String>> e : this.airport_histograms.entrySet()) {
vt.addRow(
e.getKey(), // CFH_NAME
e.getValue().toJSONString(), // CFH_DATA
1 // CFH_IS_AIRPORT
);
} // FOR
if (debug.val) LOG.debug("Saving airport histogram information into " + catalog_tbl);
baseClient.loadVoltTable(catalog_tbl.getName(), vt);
for (Entry<String, Histogram<String>> e : this.histograms.entrySet()) {
vt.addRow(
e.getKey(), // CFH_NAME
e.getValue().toJSONString(), // CFH_DATA
0 // CFH_IS_AIRPORT
);
} // FOR
if (debug.val) LOG.debug("Saving benchmark histogram information into " + catalog_tbl);
baseClient.loadVoltTable(catalog_tbl.getName(), vt);
return;
}
/** testEvictTuples */
@Test
public void testEvictTuples() throws Exception {
// First load some stuff into the database
for (String tableName : TARGET_TABLES) {
this.loadData(this.getTable(tableName));
}
// Then make sure that we can evict from each of them
for (String tableName : TARGET_TABLES) {
Table catalog_tbl = this.getTable(tableName);
VoltTable evictResult = this.evictData(catalog_tbl);
evictResult.advanceRow();
// Our stats should now come back with at least one block evicted
VoltTable results[] = this.ee.getStats(SysProcSelector.TABLE, this.locators, false, 0L);
assertEquals(1, results.length);
// System.err.println("-------------------------------");
// System.err.println(VoltTableUtil.format(results));
while (results[0].advanceRow()) {
if (results[0].getString("TABLE_NAME").equalsIgnoreCase(catalog_tbl.getName()) == false)
continue;
for (String col : statsFields) {
assertEquals(col, evictResult.getLong(col), results[0].getLong(col));
if (col == "ANTICACHE_BLOCKS_EVICTED") {
assertEquals(col, 1, results[0].getLong(col));
} else {
assertNotSame(col, 0, results[0].getLong(col));
}
} // FOR
} // WHILE
} // FOR
}
private VoltTable evictData(Table catalog_tbl) throws Exception {
VoltTable results[] = this.ee.getStats(SysProcSelector.TABLE, this.locators, false, 0L);
assertEquals(1, results.length);
// System.err.println(VoltTableUtil.format(results));
while (results[0].advanceRow()) {
if (results[0].getString("TABLE_NAME").equalsIgnoreCase(catalog_tbl.getName()) == false)
continue;
for (String col : statsFields) {
int idx = results[0].getColumnIndex(col);
assertEquals(0, results[0].getLong(idx));
} // FOR
} // WHILE
// Now force the EE to evict our boys out
// We'll tell it to remove 1MB, which is guaranteed to include all of our tuples
VoltTable evictResult = this.ee.antiCacheEvictBlock(catalog_tbl, 1024 * 1024, 1);
// System.err.println("-------------------------------");
// System.err.println(VoltTableUtil.format(evictResult));
assertNotNull(evictResult);
assertEquals(1, evictResult.getRowCount());
assertNotSame(results[0].getColumnCount(), evictResult.getColumnCount());
evictResult.resetRowPosition();
boolean adv = evictResult.advanceRow();
assertTrue(adv);
return (evictResult);
}
protected Column getColumn(Database catalog_db, Table catalog_tbl, String col_name) {
assertNotNull(catalog_db);
assertNotNull(catalog_tbl);
Column catalog_col = catalog_tbl.getColumns().getIgnoreCase(col_name);
assert (catalog_col != null)
: "Failed to retrieve Column '" + col_name + "' from Table '" + catalog_tbl.getName() + "'";
return (catalog_col);
}
/**
* Return the name of the vertical partition for the given table name
*
* @param tableName
* @return
*/
private static String getNextVerticalPartitionName(
Table catalog_tbl, Collection<Column> catalog_cols) {
Database catalog_db = ((Database) catalog_tbl.getParent());
Collection<String> colNames = new HashSet<String>();
for (Column catalog_col : catalog_cols) {
colNames.add(catalog_col.getName());
}
// Figure out how many vertical partition tables already exist for this table
int next = 0;
String prefix = "SYS_VP_" + catalog_tbl.getName() + "_";
Pattern p = Pattern.compile(Pattern.quote(prefix) + "[\\d]+");
for (Table otherTable : CatalogUtil.getSysTables(catalog_db)) {
if (debug.get())
LOG.debug(String.format("Checking whether '%s' matches prefix '%s'", otherTable, prefix));
Matcher m = p.matcher(otherTable.getName());
if (m.matches() == false) continue;
// Check to make sure it's not the same vertical partition
Collection<Column> otherColumns = otherTable.getColumns();
if (debug.get())
LOG.debug(
String.format(
"%s.%s <-> %s.%s",
catalog_tbl.getName(), catalog_cols, otherTable.getName(), otherColumns));
if (otherColumns.size() != colNames.size()) continue;
boolean fail = false;
for (Column otherCol : otherColumns) {
if (colNames.contains(otherCol.getName()) == false) {
fail = true;
break;
}
}
if (fail) continue;
next++;
} // FOR
String viewName = String.format("%s%02d", prefix, next);
assert (catalog_tbl.getViews().contains(viewName) == false);
if (debug.get())
LOG.debug(String.format("Next VerticalPartition name '%s' for %s", viewName, catalog_tbl));
return (viewName);
}
/**
* For a given table catalog object, return an Iterable that will iterate through tuples in the
* generated flat files.
*
* @param catalog_tbl
* @return
* @throws Exception
*/
public Iterable<Object[]> getTable(final Table catalog_tbl) throws Exception {
File file =
new File(
EGenLoader.this.output_path
+ File.separator
+ catalog_tbl.getName().toUpperCase()
+ ".txt");
return (new TableDataIterable(catalog_tbl, file));
}
public void addTablePartitionInfo(Table catalog_tbl, Column catalog_col) {
assert (catalog_col != null) : "Unexpected null partition column for " + catalog_tbl;
// TODO: Support special columns
if (catalog_col instanceof VerticalPartitionColumn) {
catalog_col = ((VerticalPartitionColumn) catalog_col).getHorizontalColumn();
}
if (catalog_col instanceof MultiColumn) {
catalog_col = ((MultiColumn) catalog_col).get(0);
}
this.addTablePartitionInfo(catalog_tbl.getName(), catalog_col.getName());
}
protected Column getColumn(Table catalog_tbl, int col_idx) {
int num_columns = catalog_tbl.getColumns().size();
if (col_idx < 0) col_idx = num_columns + col_idx; // Python!
assert (col_idx >= 0 && col_idx < num_columns)
: "Invalid column index for " + catalog_tbl + ": " + col_idx;
Column catalog_col = catalog_tbl.getColumns().get(col_idx);
assert (catalog_col != null)
: "Failed to retrieve Column at '"
+ col_idx
+ "' from Table '"
+ catalog_tbl.getName()
+ "'";
return (catalog_col);
}
private void loadData(Table catalog_tbl) throws Exception {
// Load in a bunch of dummy data for this table
VoltTable vt = CatalogUtil.getVoltTable(catalog_tbl);
assertNotNull(vt);
for (int i = 0; i < NUM_TUPLES; i++) {
Object row[] = VoltTableUtil.getRandomRow(catalog_tbl);
if (catalog_tbl.getName().equalsIgnoreCase(TPCCConstants.TABLENAME_ORDER_LINE)) {
row[0] = i; // OL_O_ID
row[1] = (byte) i; // OL_D_ID
row[2] = (short) i; // OL_W_ID
}
vt.addRow(row);
} // FOR
this.executor.loadTable(1000l, catalog_tbl, vt, false);
int statsLocators[] = {catalog_tbl.getRelativeIndex()};
VoltTable stats[] = this.ee.getStats(SysProcSelector.TABLE, statsLocators, false, 0L);
assertEquals(1, stats.length);
// System.err.println(VoltTableUtil.format(stats));
}
public static MaterializedViewInfo addVerticalPartition(
final Table catalog_tbl, final Collection<Column> catalog_cols, final boolean createIndex)
throws Exception {
assert (catalog_cols.isEmpty() == false);
Database catalog_db = ((Database) catalog_tbl.getParent());
String viewName = getNextVerticalPartitionName(catalog_tbl, catalog_cols);
if (debug.get())
LOG.debug(
String.format(
"Adding Vertical Partition %s for %s: %s", viewName, catalog_tbl, catalog_cols));
// Create a new virtual table
Table virtual_tbl = catalog_db.getTables().get(viewName);
if (virtual_tbl == null) {
virtual_tbl = catalog_db.getTables().add(viewName);
}
virtual_tbl.setIsreplicated(true);
virtual_tbl.setMaterializer(catalog_tbl);
virtual_tbl.setSystable(true);
virtual_tbl.getColumns().clear();
// Create MaterializedView and link it to the virtual table
MaterializedViewInfo catalog_view = catalog_tbl.getViews().add(viewName);
catalog_view.setVerticalpartition(true);
catalog_view.setDest(virtual_tbl);
List<Column> indexColumns = new ArrayList<Column>();
Column partition_col = catalog_tbl.getPartitioncolumn();
if (partition_col instanceof VerticalPartitionColumn) {
partition_col = ((VerticalPartitionColumn) partition_col).getHorizontalColumn();
}
if (debug.get()) LOG.debug(catalog_tbl.getName() + " Partition Column: " + partition_col);
int i = 0;
assert (catalog_cols != null);
assert (catalog_cols.isEmpty() == false)
: "No vertical partitioning columns for " + catalog_view.fullName();
for (Column catalog_col : catalog_cols) {
// MaterializedView ColumnRef
ColumnRef catalog_ref = catalog_view.getGroupbycols().add(catalog_col.getName());
catalog_ref.setColumn(catalog_col);
catalog_ref.setIndex(i++);
// VirtualTable Column
Column virtual_col = virtual_tbl.getColumns().add(catalog_col.getName());
virtual_col.setDefaulttype(catalog_col.getDefaulttype());
virtual_col.setDefaultvalue(catalog_col.getDefaultvalue());
virtual_col.setIndex(catalog_col.getIndex());
virtual_col.setNullable(catalog_col.getNullable());
virtual_col.setSize(catalog_col.getSize());
virtual_col.setType(catalog_col.getType());
if (debug.get())
LOG.debug(String.format("Added VerticalPartition column %s", virtual_col.fullName()));
// If they want an index, then we'll make one based on every column except for the column
// that the table is partitioned on
if (createIndex) {
boolean include = true;
if (partition_col instanceof MultiColumn) {
include = (((MultiColumn) partition_col).contains(catalog_col) == false);
} else if (catalog_col.equals(partition_col)) {
include = false;
}
if (include) indexColumns.add(virtual_col);
}
} // FOR
if (createIndex) {
if (indexColumns.isEmpty()) {
Map<String, Object> m = new ListOrderedMap<String, Object>();
m.put("Partition Column", partition_col);
m.put("VP Table Columns", virtual_tbl.getColumns());
m.put("Passed-in Columns", CatalogUtil.debug(catalog_cols));
LOG.error("Failed to find index columns\n" + StringUtil.formatMaps(m));
throw new Exception(String.format("No columns selected for index on %s", viewName));
}
String idxName = "SYS_IDX_" + viewName;
Index virtual_idx = virtual_tbl.getIndexes().get(idxName);
if (virtual_idx == null) {
virtual_idx = virtual_tbl.getIndexes().add(idxName);
}
virtual_idx.getColumns().clear();
IndexType idxType =
(indexColumns.size() == 1 ? IndexType.HASH_TABLE : IndexType.BALANCED_TREE);
virtual_idx.setType(idxType.getValue());
i = 0;
for (Column catalog_col : indexColumns) {
ColumnRef cref = virtual_idx.getColumns().add(catalog_col.getTypeName());
cref.setColumn(catalog_col);
cref.setIndex(i++);
} // FOR
if (debug.get())
LOG.debug(
String.format(
"Created %s index '%s' for vertical partition '%s'", idxType, idxName, viewName));
}
return (catalog_view);
}