@Override
public boolean compile(VoltProjectBuilder builder) {
if (m_compiled) {
return true;
}
m_compiled =
builder.compile(m_jarFileName, m_partitionPerSite, m_siteCount, m_replication, "localhost");
// (1) Load catalog from Jar
Catalog tmpCatalog = CatalogUtil.loadCatalogFromJar(m_jarFileName);
// (2) Update catalog to include target cluster configuration
ClusterConfiguration cc = new ClusterConfiguration();
// Update cc with a bunch of hosts/sites/partitions
for (int site = 0, currentPartition = 0; site < m_siteCount; ++site) {
for (int partition = 0; partition < m_partitionPerSite; ++partition, ++currentPartition) {
cc.addPartition("localhost", site, currentPartition);
}
}
this.catalog = FixCatalog.addHostInfo(tmpCatalog, cc);
// (3) Write updated catalog back out to jar file
try {
CatalogUtil.updateCatalogInJar(m_jarFileName, catalog);
} catch (Exception e) {
throw new RuntimeException(e);
}
tmpCatalog = CatalogUtil.loadCatalogFromJar(m_jarFileName);
// System.err.println(CatalogInfo.getInfo(this.catalog, new File(m_jarFileName)));
// Construct the base command that we will want to use to start
// all of the "remote" HStoreSites
List<String> siteCommand = new ArrayList<String>();
CollectionUtil.addAll(siteCommand, "ant", "hstore-site", "-Djar=" + m_jarFileName);
// Be sure to include our HStoreConf parameters
for (Entry<String, String> e : this.confParams.entrySet()) {
siteCommand.add(String.format("-D%s=%s", e.getKey(), e.getValue()));
}
// Lastly, we will include the site.id as the last parameter
// so that we can easily change it
siteCommand.add("-Dsite.id=-1");
m_procBuilder = new ProcessBuilder(siteCommand.toArray(new String[0]));
m_procBuilder.redirectErrorStream(true);
// set the working directory to obj/release/prod
// m_procBuilder.directory(new File(m_buildDir + File.separator + "prod"));
return m_compiled;
}
/**
* Add fake partitions to the loaded catalog Assuming that there is one partition per site
*
* @param num_partitions
*/
protected void addPartitions(int num_partitions) throws Exception {
// HACK! If we already have this many partitions in the catalog, then we won't recreate it
// This fixes problems where we need to reference the same catalog objects in multiple test
// cases
if (CatalogUtil.getNumberOfPartitions(catalog_db) != num_partitions) {
ClusterConfiguration cc = new ClusterConfiguration();
for (Integer i = 0; i < num_partitions; i++) {
cc.addPartition("localhost", 0, i);
// System.err.println("[" + i + "] " + Arrays.toString(triplets.lastElement()));
} // FOR
catalog = FixCatalog.addHostInfo(catalog, cc);
this.init(this.last_type, catalog);
}
Cluster cluster = CatalogUtil.getCluster(catalog_db);
assertEquals(num_partitions, cluster.getNum_partitions());
assertEquals(num_partitions, CatalogUtil.getNumberOfPartitions(cluster));
}
protected void initializeCluster(int num_hosts, int num_sites, int num_partitions)
throws Exception {
// HACK! If we already have this many partitions in the catalog, then we won't recreate it
// This fixes problems where we need to reference the same catalog objects in multiple test
// cases
if (CatalogUtil.getNumberOfHosts(catalog_db) != num_hosts
|| CatalogUtil.getNumberOfSites(catalog_db) != (num_hosts * num_sites)
|| CatalogUtil.getNumberOfPartitions(catalog_db)
!= (num_hosts * num_sites * num_partitions)) {
catalog = FixCatalog.addHostInfo(catalog, "localhost", num_hosts, num_sites, num_partitions);
this.init(this.last_type, catalog);
}
Cluster cluster = CatalogUtil.getCluster(catalog_db);
assertEquals(num_hosts, CatalogUtil.getNumberOfHosts(catalog_db));
assertEquals((num_hosts * num_sites), CatalogUtil.getNumberOfSites(catalog_db));
assertEquals(
(num_hosts * num_sites * num_partitions), CatalogUtil.getNumberOfPartitions(cluster));
assertEquals((num_hosts * num_sites * num_partitions), cluster.getNum_partitions());
}
/**
* @param args
* @throws Exception
*/
@SuppressWarnings("unchecked")
public void process(String[] args, String... required) throws Exception {
final boolean debug = LOG.isDebugEnabled();
if (debug) LOG.debug("Processing " + args.length + " parameters...");
final Pattern p = Pattern.compile("=");
for (int i = 0, cnt = args.length; i < cnt; i++) {
final String arg = args[i];
final String[] parts = p.split(arg, 2);
if (parts[0].startsWith("-")) parts[0] = parts[0].substring(1);
if (parts.length == 1) {
if (parts[0].startsWith("${") == false) this.opt_params.add(parts[0]);
continue;
} else if (parts[0].equalsIgnoreCase("tag")) {
continue;
} else if (parts[1].startsWith("${") || parts[0].startsWith("#")) {
continue;
}
if (debug) LOG.debug(String.format("%-35s = %s", parts[0], parts[1]));
// DesignerHints Override
if (parts[0].startsWith(PARAM_DESIGNER_HINTS_PREFIX)) {
String param = parts[0].replace(PARAM_DESIGNER_HINTS_PREFIX, "").toLowerCase();
try {
Field f = DesignerHints.class.getField(param);
this.hints_params.put(f.getName(), parts[1]);
if (debug) LOG.debug(String.format("DesignerHints.%s = %s", param, parts[1]));
} catch (NoSuchFieldException ex) {
throw new Exception("Unknown DesignerHints parameter: " + param, ex);
}
}
// HStoreConf Parameter
else if (HStoreConf.isConfParameter(parts[0])) {
this.conf_params.put(parts[0].toLowerCase(), parts[1]);
}
// ArgumentsParser Parameter
else if (PARAMS.contains(parts[0].toLowerCase())) {
this.params.put(parts[0].toLowerCase(), parts[1]);
}
// Invalid!
else {
String suggestions = "";
i = 0;
String end = CollectionUtil.last(parts[0].split("\\."));
for (String param : PARAMS) {
String param_end = CollectionUtil.last(param.split("\\."));
if (param.startsWith(parts[0])
|| (end != null && param.endsWith(end))
|| (end != null && param_end != null && param_end.startsWith(end))) {
if (suggestions.isEmpty()) suggestions = ". Possible Matches:";
suggestions += String.format("\n [%02d] %s", ++i, param);
}
} // FOR
throw new Exception("Unknown parameter '" + parts[0] + "'" + suggestions);
}
} // FOR
// -------------------------------------------------------
// CATALOGS
// -------------------------------------------------------
// Text File
if (this.params.containsKey(PARAM_CATALOG)) {
String path = this.params.get(PARAM_CATALOG);
if (debug) LOG.debug("Loading catalog from file '" + path + "'");
Catalog catalog = CatalogUtil.loadCatalog(path);
if (catalog == null)
throw new Exception("Failed to load catalog object from file '" + path + "'");
this.updateCatalog(catalog, new File(path));
}
// Jar File
else if (this.params.containsKey(PARAM_CATALOG_JAR)) {
String path = this.params.get(PARAM_CATALOG_JAR);
this.params.put(PARAM_CATALOG, path);
File jar_file = new File(path);
Catalog catalog = CatalogUtil.loadCatalogFromJar(path);
if (catalog == null)
throw new Exception("Failed to load catalog object from jar file '" + path + "'");
if (debug) LOG.debug("Loaded catalog from jar file '" + path + "'");
this.updateCatalog(catalog, jar_file);
if (!this.params.containsKey(PARAM_CATALOG_TYPE)) {
String jar_name = jar_file.getName();
int jar_idx = jar_name.lastIndexOf(".jar");
if (jar_idx != -1) {
ProjectType type = ProjectType.get(jar_name.substring(0, jar_idx));
if (type != null) {
if (debug) LOG.debug("Set catalog type '" + type + "' from catalog jar file name");
this.catalog_type = type;
this.params.put(PARAM_CATALOG_TYPE, this.catalog_type.toString());
}
}
}
}
// Schema File
else if (this.params.containsKey(PARAM_CATALOG_SCHEMA)) {
String path = this.params.get(PARAM_CATALOG_SCHEMA);
Catalog catalog = CompilerUtil.compileCatalog(path);
if (catalog == null) throw new Exception("Failed to load schema from '" + path + "'");
if (debug) LOG.debug("Loaded catalog from schema file '" + path + "'");
this.updateCatalog(catalog, new File(path));
}
// Catalog Type
if (this.params.containsKey(PARAM_CATALOG_TYPE)) {
String catalog_type = this.params.get(PARAM_CATALOG_TYPE);
ProjectType type = ProjectType.get(catalog_type);
if (type == null) {
throw new Exception("Unknown catalog type '" + catalog_type + "'");
}
this.catalog_type = type;
}
// Update Cluster Configuration
if (this.params.containsKey(ArgumentsParser.PARAM_CATALOG_HOSTS)) {
ClusterConfiguration cc =
new ClusterConfiguration(this.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS));
this.updateCatalog(FixCatalog.addHostInfo(this.catalog, cc), null);
}
// Check the requirements after loading the catalog, because some of the
// above parameters will set the catalog one
if (required != null && required.length > 0) this.require(required);
// -------------------------------------------------------
// PHYSICAL DESIGN COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_PARTITION_PLAN)) {
assert (this.catalog_db != null);
File path = new File(this.params.get(PARAM_PARTITION_PLAN));
boolean ignoreMissing =
this.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_IGNORE_MISSING, false);
if (path.exists() || (path.exists() == false && ignoreMissing == false)) {
if (debug) LOG.debug("Loading in partition plan from '" + path + "'");
this.pplan = new PartitionPlan();
this.pplan.load(path.getAbsolutePath(), this.catalog_db);
// Apply!
if (this.params.containsKey(PARAM_PARTITION_PLAN_APPLY)
&& this.getBooleanParam(PARAM_PARTITION_PLAN_APPLY)) {
boolean secondaryIndexes =
this.getBooleanParam(PARAM_PARTITION_PLAN_NO_SECONDARY, false) == false;
LOG.info(
String.format(
"Applying PartitionPlan '%s' to catalog [enableSecondary=%s]",
path.getName(), secondaryIndexes));
this.pplan.apply(this.catalog_db, secondaryIndexes);
}
}
}
// -------------------------------------------------------
// DESIGNER COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_DESIGNER_THREADS)) {
this.max_concurrent = Integer.valueOf(this.params.get(PARAM_DESIGNER_THREADS));
}
if (this.params.containsKey(PARAM_DESIGNER_INTERVALS)) {
this.num_intervals = Integer.valueOf(this.params.get(PARAM_DESIGNER_INTERVALS));
}
if (this.params.containsKey(PARAM_DESIGNER_HINTS)) {
String path = this.params.get(PARAM_DESIGNER_HINTS);
if (debug)
LOG.debug(
"Loading in designer hints from '"
+ path
+ "'.\nForced Values:\n"
+ StringUtil.formatMaps(this.hints_params));
this.designer_hints.load(path, catalog_db, this.hints_params);
}
if (this.params.containsKey(PARAM_DESIGNER_CHECKPOINT)) {
this.designer_checkpoint = new File(this.params.get(PARAM_DESIGNER_CHECKPOINT));
}
String designer_attributes[] = {
PARAM_DESIGNER_PARTITIONER,
PARAM_DESIGNER_MAPPER,
PARAM_DESIGNER_INDEXER,
PARAM_DESIGNER_COSTMODEL
};
ClassLoader loader = ClassLoader.getSystemClassLoader();
for (String key : designer_attributes) {
if (this.params.containsKey(key)) {
String target_name = this.params.get(key);
Class<?> target_class = loader.loadClass(target_name);
assert (target_class != null);
if (debug) LOG.debug("Set " + key + " class to " + target_class.getName());
if (key.equals(PARAM_DESIGNER_PARTITIONER)) {
this.partitioner_class = (Class<? extends AbstractPartitioner>) target_class;
} else if (key.equals(PARAM_DESIGNER_MAPPER)) {
this.mapper_class = (Class<? extends AbstractMapper>) target_class;
} else if (key.equals(PARAM_DESIGNER_INDEXER)) {
this.indexer_class = (Class<? extends AbstractIndexSelector>) target_class;
} else if (key.equals(PARAM_DESIGNER_COSTMODEL)) {
this.costmodel_class = (Class<? extends AbstractCostModel>) target_class;
// Special Case: TimeIntervalCostModel
if (target_name.endsWith(TimeIntervalCostModel.class.getSimpleName())) {
this.costmodel =
new TimeIntervalCostModel<SingleSitedCostModel>(
this.catalog_db, SingleSitedCostModel.class, this.num_intervals);
} else {
this.costmodel =
ClassUtil.newInstance(
this.costmodel_class,
new Object[] {this.catalog_db},
new Class[] {Database.class});
}
} else {
assert (false) : "Invalid key '" + key + "'";
}
}
} // FOR
// -------------------------------------------------------
// TRANSACTION ESTIMATION COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_MAPPINGS)) {
assert (this.catalog_db != null);
File path = new File(this.params.get(PARAM_MAPPINGS));
if (path.exists()) {
this.param_mappings.load(path.getAbsolutePath(), this.catalog_db);
} else {
LOG.warn("The ParameterMappings file '" + path + "' does not exist");
}
}
if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS_VALUE)) {
assert (this.catalog_db != null);
float defaultValue = this.getDoubleParam(PARAM_MARKOV_THRESHOLDS_VALUE).floatValue();
this.thresholds = new EstimationThresholds(defaultValue);
this.params.put(PARAM_MARKOV_THRESHOLDS, this.thresholds.toString());
LOG.debug("CREATED THRESHOLDS: " + this.thresholds);
} else if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS)) {
assert (this.catalog_db != null);
this.thresholds = new EstimationThresholds();
File path = new File(this.params.get(PARAM_MARKOV_THRESHOLDS));
if (path.exists()) {
this.thresholds.load(path.getAbsolutePath(), this.catalog_db);
} else {
LOG.warn("The estimation thresholds file '" + path + "' does not exist");
}
LOG.debug("LOADED THRESHOLDS: " + this.thresholds);
}
// -------------------------------------------------------
// HASHER
// -------------------------------------------------------
if (this.catalog != null) {
if (this.params.containsKey(PARAM_HASHER_CLASS)) {
String hasherClassName = this.params.get(PARAM_HASHER_CLASS);
this.hasher_class = (Class<? extends AbstractHasher>) loader.loadClass(hasherClassName);
}
Constructor<? extends AbstractHasher> constructor =
this.hasher_class.getConstructor(new Class[] {Database.class, Integer.class});
int num_partitions = CatalogUtil.getNumberOfPartitions(this.catalog_db);
this.hasher = constructor.newInstance(new Object[] {this.catalog_db, num_partitions});
if (!(this.hasher instanceof DefaultHasher))
LOG.debug("Loaded hasher " + this.hasher.getClass());
if (this.params.containsKey(PARAM_HASHER_PROFILE)) {
this.hasher.load(this.params.get(PARAM_HASHER_PROFILE), null);
}
}
// -------------------------------------------------------
// SAMPLE WORKLOAD TRACE
// -------------------------------------------------------
this.loadWorkload();
}
/**
* MAIN!
*
* @param vargs
* @throws Exception
*/
public static void main(String[] vargs) throws Exception {
ArgumentsParser args = ArgumentsParser.load(vargs);
args.require(
ArgumentsParser.PARAM_CATALOG,
ArgumentsParser.PARAM_WORKLOAD,
ArgumentsParser.PARAM_PARTITION_PLAN,
ArgumentsParser.PARAM_DESIGNER_INTERVALS
// ArgumentsParser.PARAM_DESIGNER_HINTS
);
assert (args.workload.getTransactionCount() > 0)
: "No transactions were loaded from " + args.workload;
if (args.hasParam(ArgumentsParser.PARAM_CATALOG_HOSTS)) {
ClusterConfiguration cc =
new ClusterConfiguration(args.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS));
args.updateCatalog(FixCatalog.cloneCatalog(args.catalog, cc), null);
}
// If given a PartitionPlan, then update the catalog
File pplan_path = new File(args.getParam(ArgumentsParser.PARAM_PARTITION_PLAN));
if (pplan_path.exists()) {
PartitionPlan pplan = new PartitionPlan();
pplan.load(pplan_path, args.catalog_db);
if (args.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_REMOVE_PROCS, false)) {
for (Procedure catalog_proc : pplan.proc_entries.keySet()) {
pplan.setNullProcParameter(catalog_proc);
} // FOR
}
if (args.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_RANDOM_PROCS, false)) {
for (Procedure catalog_proc : pplan.proc_entries.keySet()) {
pplan.setRandomProcParameter(catalog_proc);
} // FOR
}
pplan.apply(args.catalog_db);
System.out.println("Applied PartitionPlan '" + pplan_path + "' to catalog\n" + pplan);
System.out.print(StringUtil.DOUBLE_LINE);
if (args.hasParam(ArgumentsParser.PARAM_PARTITION_PLAN_OUTPUT)) {
String output = args.getParam(ArgumentsParser.PARAM_PARTITION_PLAN_OUTPUT);
if (output.equals("-")) output = pplan_path.getAbsolutePath();
pplan.save(new File(output));
System.out.println("Saved PartitionPlan to '" + output + "'");
}
} else {
System.err.println("PartitionPlan file '" + pplan_path + "' does not exist. Ignoring...");
}
System.out.flush();
int num_intervals =
args.num_intervals; // getIntParam(ArgumentsParser.PARAM_DESIGNER_INTERVALS);
TimeIntervalCostModel<SingleSitedCostModel> costmodel =
new TimeIntervalCostModel<SingleSitedCostModel>(
args.catalogContext, SingleSitedCostModel.class, num_intervals);
if (args.hasParam(ArgumentsParser.PARAM_DESIGNER_HINTS))
costmodel.applyDesignerHints(args.designer_hints);
double cost = costmodel.estimateWorkloadCost(args.catalogContext, args.workload);
Map<String, Object> m = new LinkedHashMap<String, Object>();
m.put("PARTITIONS", CatalogUtil.getNumberOfPartitions(args.catalog_db));
m.put("INTERVALS", args.num_intervals);
m.put("EXEC COST", costmodel.last_execution_cost);
m.put("SKEW COST", costmodel.last_skew_cost);
m.put("TOTAL COST", cost);
m.put("PARTITIONS TOUCHED", costmodel.getTxnPartitionAccessHistogram().getSampleCount());
System.out.println(StringUtil.formatMaps(m));
// long total = 0;
m.clear();
// for (int i = 0; i < num_intervals; i++) {
// SingleSitedCostModel cm = costmodel.getCostModel(i);
// Histogram<Integer> h = cm.getTxnPartitionAccessHistogram();
// m.put(String.format("Interval %02d", i),
// cm.getTxnPartitionAccessHistogram());
// total += h.getSampleCount();
// h.setKeepZeroEntries(true);
// for (Integer partition :
// CatalogUtil.getAllPartitionIds(args.catalog_db)) {
// if (h.contains(partition) == false) h.put(partition, 0);
// }
// System.out.println(StringUtil.box("Interval #" + i, "+", 100) + "\n"
// + h);
// System.out.println();
// } // FOR
// System.out.println(StringUtil.formatMaps(m));
// System.err.println("TOTAL: " + total);
}