/**
* @param args
* @throws Exception
*/
@SuppressWarnings("unchecked")
public void process(String[] args, String... required) throws Exception {
final boolean debug = LOG.isDebugEnabled();
if (debug) LOG.debug("Processing " + args.length + " parameters...");
final Pattern p = Pattern.compile("=");
for (int i = 0, cnt = args.length; i < cnt; i++) {
final String arg = args[i];
final String[] parts = p.split(arg, 2);
if (parts[0].startsWith("-")) parts[0] = parts[0].substring(1);
if (parts.length == 1) {
if (parts[0].startsWith("${") == false) this.opt_params.add(parts[0]);
continue;
} else if (parts[0].equalsIgnoreCase("tag")) {
continue;
} else if (parts[1].startsWith("${") || parts[0].startsWith("#")) {
continue;
}
if (debug) LOG.debug(String.format("%-35s = %s", parts[0], parts[1]));
// DesignerHints Override
if (parts[0].startsWith(PARAM_DESIGNER_HINTS_PREFIX)) {
String param = parts[0].replace(PARAM_DESIGNER_HINTS_PREFIX, "").toLowerCase();
try {
Field f = DesignerHints.class.getField(param);
this.hints_params.put(f.getName(), parts[1]);
if (debug) LOG.debug(String.format("DesignerHints.%s = %s", param, parts[1]));
} catch (NoSuchFieldException ex) {
throw new Exception("Unknown DesignerHints parameter: " + param, ex);
}
}
// HStoreConf Parameter
else if (HStoreConf.isConfParameter(parts[0])) {
this.conf_params.put(parts[0].toLowerCase(), parts[1]);
}
// ArgumentsParser Parameter
else if (PARAMS.contains(parts[0].toLowerCase())) {
this.params.put(parts[0].toLowerCase(), parts[1]);
}
// Invalid!
else {
String suggestions = "";
i = 0;
String end = CollectionUtil.last(parts[0].split("\\."));
for (String param : PARAMS) {
String param_end = CollectionUtil.last(param.split("\\."));
if (param.startsWith(parts[0])
|| (end != null && param.endsWith(end))
|| (end != null && param_end != null && param_end.startsWith(end))) {
if (suggestions.isEmpty()) suggestions = ". Possible Matches:";
suggestions += String.format("\n [%02d] %s", ++i, param);
}
} // FOR
throw new Exception("Unknown parameter '" + parts[0] + "'" + suggestions);
}
} // FOR
// -------------------------------------------------------
// CATALOGS
// -------------------------------------------------------
// Text File
if (this.params.containsKey(PARAM_CATALOG)) {
String path = this.params.get(PARAM_CATALOG);
if (debug) LOG.debug("Loading catalog from file '" + path + "'");
Catalog catalog = CatalogUtil.loadCatalog(path);
if (catalog == null)
throw new Exception("Failed to load catalog object from file '" + path + "'");
this.updateCatalog(catalog, new File(path));
}
// Jar File
else if (this.params.containsKey(PARAM_CATALOG_JAR)) {
String path = this.params.get(PARAM_CATALOG_JAR);
this.params.put(PARAM_CATALOG, path);
File jar_file = new File(path);
Catalog catalog = CatalogUtil.loadCatalogFromJar(path);
if (catalog == null)
throw new Exception("Failed to load catalog object from jar file '" + path + "'");
if (debug) LOG.debug("Loaded catalog from jar file '" + path + "'");
this.updateCatalog(catalog, jar_file);
if (!this.params.containsKey(PARAM_CATALOG_TYPE)) {
String jar_name = jar_file.getName();
int jar_idx = jar_name.lastIndexOf(".jar");
if (jar_idx != -1) {
ProjectType type = ProjectType.get(jar_name.substring(0, jar_idx));
if (type != null) {
if (debug) LOG.debug("Set catalog type '" + type + "' from catalog jar file name");
this.catalog_type = type;
this.params.put(PARAM_CATALOG_TYPE, this.catalog_type.toString());
}
}
}
}
// Schema File
else if (this.params.containsKey(PARAM_CATALOG_SCHEMA)) {
String path = this.params.get(PARAM_CATALOG_SCHEMA);
Catalog catalog = CompilerUtil.compileCatalog(path);
if (catalog == null) throw new Exception("Failed to load schema from '" + path + "'");
if (debug) LOG.debug("Loaded catalog from schema file '" + path + "'");
this.updateCatalog(catalog, new File(path));
}
// Catalog Type
if (this.params.containsKey(PARAM_CATALOG_TYPE)) {
String catalog_type = this.params.get(PARAM_CATALOG_TYPE);
ProjectType type = ProjectType.get(catalog_type);
if (type == null) {
throw new Exception("Unknown catalog type '" + catalog_type + "'");
}
this.catalog_type = type;
}
// Update Cluster Configuration
if (this.params.containsKey(ArgumentsParser.PARAM_CATALOG_HOSTS)) {
ClusterConfiguration cc =
new ClusterConfiguration(this.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS));
this.updateCatalog(FixCatalog.addHostInfo(this.catalog, cc), null);
}
// Check the requirements after loading the catalog, because some of the
// above parameters will set the catalog one
if (required != null && required.length > 0) this.require(required);
// -------------------------------------------------------
// PHYSICAL DESIGN COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_PARTITION_PLAN)) {
assert (this.catalog_db != null);
File path = new File(this.params.get(PARAM_PARTITION_PLAN));
boolean ignoreMissing =
this.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_IGNORE_MISSING, false);
if (path.exists() || (path.exists() == false && ignoreMissing == false)) {
if (debug) LOG.debug("Loading in partition plan from '" + path + "'");
this.pplan = new PartitionPlan();
this.pplan.load(path.getAbsolutePath(), this.catalog_db);
// Apply!
if (this.params.containsKey(PARAM_PARTITION_PLAN_APPLY)
&& this.getBooleanParam(PARAM_PARTITION_PLAN_APPLY)) {
boolean secondaryIndexes =
this.getBooleanParam(PARAM_PARTITION_PLAN_NO_SECONDARY, false) == false;
LOG.info(
String.format(
"Applying PartitionPlan '%s' to catalog [enableSecondary=%s]",
path.getName(), secondaryIndexes));
this.pplan.apply(this.catalog_db, secondaryIndexes);
}
}
}
// -------------------------------------------------------
// DESIGNER COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_DESIGNER_THREADS)) {
this.max_concurrent = Integer.valueOf(this.params.get(PARAM_DESIGNER_THREADS));
}
if (this.params.containsKey(PARAM_DESIGNER_INTERVALS)) {
this.num_intervals = Integer.valueOf(this.params.get(PARAM_DESIGNER_INTERVALS));
}
if (this.params.containsKey(PARAM_DESIGNER_HINTS)) {
String path = this.params.get(PARAM_DESIGNER_HINTS);
if (debug)
LOG.debug(
"Loading in designer hints from '"
+ path
+ "'.\nForced Values:\n"
+ StringUtil.formatMaps(this.hints_params));
this.designer_hints.load(path, catalog_db, this.hints_params);
}
if (this.params.containsKey(PARAM_DESIGNER_CHECKPOINT)) {
this.designer_checkpoint = new File(this.params.get(PARAM_DESIGNER_CHECKPOINT));
}
String designer_attributes[] = {
PARAM_DESIGNER_PARTITIONER,
PARAM_DESIGNER_MAPPER,
PARAM_DESIGNER_INDEXER,
PARAM_DESIGNER_COSTMODEL
};
ClassLoader loader = ClassLoader.getSystemClassLoader();
for (String key : designer_attributes) {
if (this.params.containsKey(key)) {
String target_name = this.params.get(key);
Class<?> target_class = loader.loadClass(target_name);
assert (target_class != null);
if (debug) LOG.debug("Set " + key + " class to " + target_class.getName());
if (key.equals(PARAM_DESIGNER_PARTITIONER)) {
this.partitioner_class = (Class<? extends AbstractPartitioner>) target_class;
} else if (key.equals(PARAM_DESIGNER_MAPPER)) {
this.mapper_class = (Class<? extends AbstractMapper>) target_class;
} else if (key.equals(PARAM_DESIGNER_INDEXER)) {
this.indexer_class = (Class<? extends AbstractIndexSelector>) target_class;
} else if (key.equals(PARAM_DESIGNER_COSTMODEL)) {
this.costmodel_class = (Class<? extends AbstractCostModel>) target_class;
// Special Case: TimeIntervalCostModel
if (target_name.endsWith(TimeIntervalCostModel.class.getSimpleName())) {
this.costmodel =
new TimeIntervalCostModel<SingleSitedCostModel>(
this.catalog_db, SingleSitedCostModel.class, this.num_intervals);
} else {
this.costmodel =
ClassUtil.newInstance(
this.costmodel_class,
new Object[] {this.catalog_db},
new Class[] {Database.class});
}
} else {
assert (false) : "Invalid key '" + key + "'";
}
}
} // FOR
// -------------------------------------------------------
// TRANSACTION ESTIMATION COMPONENTS
// -------------------------------------------------------
if (this.params.containsKey(PARAM_MAPPINGS)) {
assert (this.catalog_db != null);
File path = new File(this.params.get(PARAM_MAPPINGS));
if (path.exists()) {
this.param_mappings.load(path.getAbsolutePath(), this.catalog_db);
} else {
LOG.warn("The ParameterMappings file '" + path + "' does not exist");
}
}
if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS_VALUE)) {
assert (this.catalog_db != null);
float defaultValue = this.getDoubleParam(PARAM_MARKOV_THRESHOLDS_VALUE).floatValue();
this.thresholds = new EstimationThresholds(defaultValue);
this.params.put(PARAM_MARKOV_THRESHOLDS, this.thresholds.toString());
LOG.debug("CREATED THRESHOLDS: " + this.thresholds);
} else if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS)) {
assert (this.catalog_db != null);
this.thresholds = new EstimationThresholds();
File path = new File(this.params.get(PARAM_MARKOV_THRESHOLDS));
if (path.exists()) {
this.thresholds.load(path.getAbsolutePath(), this.catalog_db);
} else {
LOG.warn("The estimation thresholds file '" + path + "' does not exist");
}
LOG.debug("LOADED THRESHOLDS: " + this.thresholds);
}
// -------------------------------------------------------
// HASHER
// -------------------------------------------------------
if (this.catalog != null) {
if (this.params.containsKey(PARAM_HASHER_CLASS)) {
String hasherClassName = this.params.get(PARAM_HASHER_CLASS);
this.hasher_class = (Class<? extends AbstractHasher>) loader.loadClass(hasherClassName);
}
Constructor<? extends AbstractHasher> constructor =
this.hasher_class.getConstructor(new Class[] {Database.class, Integer.class});
int num_partitions = CatalogUtil.getNumberOfPartitions(this.catalog_db);
this.hasher = constructor.newInstance(new Object[] {this.catalog_db, num_partitions});
if (!(this.hasher instanceof DefaultHasher))
LOG.debug("Loaded hasher " + this.hasher.getClass());
if (this.params.containsKey(PARAM_HASHER_PROFILE)) {
this.hasher.load(this.params.get(PARAM_HASHER_PROFILE), null);
}
}
// -------------------------------------------------------
// SAMPLE WORKLOAD TRACE
// -------------------------------------------------------
this.loadWorkload();
}
/**
* Return key/value maps into a nicely formatted table The maps are displayed in order from first
* to last, and there will be a spacer created between each map. The format for each record is:
*
* <p><KEY><DELIMITER><SPACING><VALUE>
*
* <p>If the delimiter is an equal sign, then the format is:
*
* <p><KEY><SPACING><DELIMITER><VALUE>
*
* @param delimiter
* @param upper Upper-case all keys
* @param box Box results
* @param border_top TODO
* @param border_bottom TODO
* @param recursive TODO
* @param first_element_title TODO
* @param maps
* @return
*/
@SuppressWarnings("unchecked")
public static String formatMaps(
String delimiter,
boolean upper,
boolean box,
boolean border_top,
boolean border_bottom,
boolean recursive,
boolean first_element_title,
Map<?, ?>... maps) {
boolean need_divider = (maps.length > 1 || border_bottom || border_top);
// Figure out the largest key size so we can get spacing right
int max_key_size = 0;
int max_title_size = 0;
final Map<Object, String[]> map_keys[] = (Map<Object, String[]>[]) new Map[maps.length];
final boolean map_titles[] = new boolean[maps.length];
for (int i = 0; i < maps.length; i++) {
Map<?, ?> m = maps[i];
if (m == null) continue;
Map<Object, String[]> keys = new HashMap<Object, String[]>();
boolean first = true;
for (Object k : m.keySet()) {
String k_str[] = LINE_SPLIT.split(k != null ? k.toString() : "");
keys.put(k, k_str);
// If the first element has a null value, then we can let it be the title for this map
// It's length doesn't affect the other keys, but will affect the total size of the map
if (first && first_element_title && m.get(k) == null) {
for (String line : k_str) {
max_title_size = Math.max(max_title_size, line.length());
} // FOR
map_titles[i] = true;
} else {
for (String line : k_str) {
max_key_size = Math.max(max_key_size, line.length());
} // FOR
if (first) map_titles[i] = false;
}
first = false;
} // FOR
map_keys[i] = keys;
} // FOR
boolean equalsDelimiter = delimiter.equals("=");
final String f =
"%-"
+ (max_key_size + delimiter.length() + 1)
+ "s"
+ (equalsDelimiter ? "= " : "")
+ "%s\n";
// Now make StringBuilder blocks for each map
// We do it in this way so that we can get the max length of the values
int max_value_size = 0;
StringBuilder blocks[] = new StringBuilder[maps.length];
for (int map_i = 0; map_i < maps.length; map_i++) {
blocks[map_i] = new StringBuilder();
Map<?, ?> m = maps[map_i];
if (m == null) continue;
Map<Object, String[]> keys = map_keys[map_i];
boolean first = true;
for (Entry<?, ?> e : m.entrySet()) {
String key[] = keys.get(e.getKey());
if (first && map_titles[map_i]) {
blocks[map_i].append(StringUtil.join("\n", key));
if (CollectionUtil.last(key).endsWith("\n") == false) blocks[map_i].append("\n");
} else {
Object v_obj = e.getValue();
String v = null;
if (recursive && v_obj instanceof Map<?, ?>) {
v =
formatMaps(
delimiter,
upper,
box,
border_top,
border_bottom,
recursive,
first_element_title,
(Map<?, ?>) v_obj)
.trim();
} else if (key.length == 1 && key[0].trim().isEmpty() && v_obj == null) {
blocks[map_i].append("\n");
continue;
} else if (v_obj == null) {
v = "null";
} else {
v = v_obj.toString();
}
// If the key or value is multiple lines, format them nicely!
String value[] = LINE_SPLIT.split(v);
int total_lines = Math.max(key.length, value.length);
for (int line_i = 0; line_i < total_lines; line_i++) {
String k_line = (line_i < key.length ? key[line_i] : "");
if (upper) k_line = k_line.toUpperCase();
String v_line = (line_i < value.length ? value[line_i] : "");
if (line_i == (key.length - 1)
&& (first == false || (first && v_line.isEmpty() == false))) {
if (equalsDelimiter == false && k_line.trim().isEmpty() == false) k_line += ":";
}
blocks[map_i].append(String.format(f, k_line, v_line));
if (need_divider) max_value_size = Math.max(max_value_size, v_line.length());
} // FOR
if (v.endsWith("\n")) blocks[map_i].append("\n");
}
first = false;
}
} // FOR
// Put it all together!
// System.err.println("max_title_size=" + max_title_size + ", max_key_size=" +
// max_key_size + ", max_value_size=" + max_value_size + ", delimiter=" + delimiter.length());
int total_width =
Math.max(max_title_size, (max_key_size + max_value_size + delimiter.length())) + 1;
String dividing_line = (need_divider ? repeat("-", total_width) : "");
StringBuilder sb = null;
if (maps.length == 1) {
sb = blocks[0];
} else {
sb = new StringBuilder();
for (int i = 0; i < maps.length; i++) {
if (blocks[i].length() == 0) continue;
if (i != 0 && maps[i].size() > 0) sb.append(dividing_line).append("\n");
sb.append(blocks[i]);
} // FOR
}
return (box
? StringUtil.box(sb.toString())
: (border_top ? dividing_line + "\n" : "")
+ sb.toString()
+ (border_bottom ? dividing_line : ""));
}
