/** * @param args * @throws Exception */ @SuppressWarnings("unchecked") public void process(String[] args, String... required) throws Exception { final boolean debug = LOG.isDebugEnabled(); if (debug) LOG.debug("Processing " + args.length + " parameters..."); final Pattern p = Pattern.compile("="); for (int i = 0, cnt = args.length; i < cnt; i++) { final String arg = args[i]; final String[] parts = p.split(arg, 2); if (parts[0].startsWith("-")) parts[0] = parts[0].substring(1); if (parts.length == 1) { if (parts[0].startsWith("${") == false) this.opt_params.add(parts[0]); continue; } else if (parts[0].equalsIgnoreCase("tag")) { continue; } else if (parts[1].startsWith("${") || parts[0].startsWith("#")) { continue; } if (debug) LOG.debug(String.format("%-35s = %s", parts[0], parts[1])); // DesignerHints Override if (parts[0].startsWith(PARAM_DESIGNER_HINTS_PREFIX)) { String param = parts[0].replace(PARAM_DESIGNER_HINTS_PREFIX, "").toLowerCase(); try { Field f = DesignerHints.class.getField(param); this.hints_params.put(f.getName(), parts[1]); if (debug) LOG.debug(String.format("DesignerHints.%s = %s", param, parts[1])); } catch (NoSuchFieldException ex) { throw new Exception("Unknown DesignerHints parameter: " + param, ex); } } // HStoreConf Parameter else if (HStoreConf.isConfParameter(parts[0])) { this.conf_params.put(parts[0].toLowerCase(), parts[1]); } // ArgumentsParser Parameter else if (PARAMS.contains(parts[0].toLowerCase())) { this.params.put(parts[0].toLowerCase(), parts[1]); } // Invalid! else { String suggestions = ""; i = 0; String end = CollectionUtil.last(parts[0].split("\\.")); for (String param : PARAMS) { String param_end = CollectionUtil.last(param.split("\\.")); if (param.startsWith(parts[0]) || (end != null && param.endsWith(end)) || (end != null && param_end != null && param_end.startsWith(end))) { if (suggestions.isEmpty()) suggestions = ". Possible Matches:"; suggestions += String.format("\n [%02d] %s", ++i, param); } } // FOR throw new Exception("Unknown parameter '" + parts[0] + "'" + suggestions); } } // FOR // ------------------------------------------------------- // CATALOGS // ------------------------------------------------------- // Text File if (this.params.containsKey(PARAM_CATALOG)) { String path = this.params.get(PARAM_CATALOG); if (debug) LOG.debug("Loading catalog from file '" + path + "'"); Catalog catalog = CatalogUtil.loadCatalog(path); if (catalog == null) throw new Exception("Failed to load catalog object from file '" + path + "'"); this.updateCatalog(catalog, new File(path)); } // Jar File else if (this.params.containsKey(PARAM_CATALOG_JAR)) { String path = this.params.get(PARAM_CATALOG_JAR); this.params.put(PARAM_CATALOG, path); File jar_file = new File(path); Catalog catalog = CatalogUtil.loadCatalogFromJar(path); if (catalog == null) throw new Exception("Failed to load catalog object from jar file '" + path + "'"); if (debug) LOG.debug("Loaded catalog from jar file '" + path + "'"); this.updateCatalog(catalog, jar_file); if (!this.params.containsKey(PARAM_CATALOG_TYPE)) { String jar_name = jar_file.getName(); int jar_idx = jar_name.lastIndexOf(".jar"); if (jar_idx != -1) { ProjectType type = ProjectType.get(jar_name.substring(0, jar_idx)); if (type != null) { if (debug) LOG.debug("Set catalog type '" + type + "' from catalog jar file name"); this.catalog_type = type; this.params.put(PARAM_CATALOG_TYPE, this.catalog_type.toString()); } } } } // Schema File else if (this.params.containsKey(PARAM_CATALOG_SCHEMA)) { String path = this.params.get(PARAM_CATALOG_SCHEMA); Catalog catalog = CompilerUtil.compileCatalog(path); if (catalog == null) throw new Exception("Failed to load schema from '" + path + "'"); if (debug) LOG.debug("Loaded catalog from schema file '" + path + "'"); this.updateCatalog(catalog, new File(path)); } // Catalog Type if (this.params.containsKey(PARAM_CATALOG_TYPE)) { String catalog_type = this.params.get(PARAM_CATALOG_TYPE); ProjectType type = ProjectType.get(catalog_type); if (type == null) { throw new Exception("Unknown catalog type '" + catalog_type + "'"); } this.catalog_type = type; } // Update Cluster Configuration if (this.params.containsKey(ArgumentsParser.PARAM_CATALOG_HOSTS)) { ClusterConfiguration cc = new ClusterConfiguration(this.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS)); this.updateCatalog(FixCatalog.addHostInfo(this.catalog, cc), null); } // Check the requirements after loading the catalog, because some of the // above parameters will set the catalog one if (required != null && required.length > 0) this.require(required); // ------------------------------------------------------- // PHYSICAL DESIGN COMPONENTS // ------------------------------------------------------- if (this.params.containsKey(PARAM_PARTITION_PLAN)) { assert (this.catalog_db != null); File path = new File(this.params.get(PARAM_PARTITION_PLAN)); boolean ignoreMissing = this.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_IGNORE_MISSING, false); if (path.exists() || (path.exists() == false && ignoreMissing == false)) { if (debug) LOG.debug("Loading in partition plan from '" + path + "'"); this.pplan = new PartitionPlan(); this.pplan.load(path.getAbsolutePath(), this.catalog_db); // Apply! if (this.params.containsKey(PARAM_PARTITION_PLAN_APPLY) && this.getBooleanParam(PARAM_PARTITION_PLAN_APPLY)) { boolean secondaryIndexes = this.getBooleanParam(PARAM_PARTITION_PLAN_NO_SECONDARY, false) == false; LOG.info( String.format( "Applying PartitionPlan '%s' to catalog [enableSecondary=%s]", path.getName(), secondaryIndexes)); this.pplan.apply(this.catalog_db, secondaryIndexes); } } } // ------------------------------------------------------- // DESIGNER COMPONENTS // ------------------------------------------------------- if (this.params.containsKey(PARAM_DESIGNER_THREADS)) { this.max_concurrent = Integer.valueOf(this.params.get(PARAM_DESIGNER_THREADS)); } if (this.params.containsKey(PARAM_DESIGNER_INTERVALS)) { this.num_intervals = Integer.valueOf(this.params.get(PARAM_DESIGNER_INTERVALS)); } if (this.params.containsKey(PARAM_DESIGNER_HINTS)) { String path = this.params.get(PARAM_DESIGNER_HINTS); if (debug) LOG.debug( "Loading in designer hints from '" + path + "'.\nForced Values:\n" + StringUtil.formatMaps(this.hints_params)); this.designer_hints.load(path, catalog_db, this.hints_params); } if (this.params.containsKey(PARAM_DESIGNER_CHECKPOINT)) { this.designer_checkpoint = new File(this.params.get(PARAM_DESIGNER_CHECKPOINT)); } String designer_attributes[] = { PARAM_DESIGNER_PARTITIONER, PARAM_DESIGNER_MAPPER, PARAM_DESIGNER_INDEXER, PARAM_DESIGNER_COSTMODEL }; ClassLoader loader = ClassLoader.getSystemClassLoader(); for (String key : designer_attributes) { if (this.params.containsKey(key)) { String target_name = this.params.get(key); Class<?> target_class = loader.loadClass(target_name); assert (target_class != null); if (debug) LOG.debug("Set " + key + " class to " + target_class.getName()); if (key.equals(PARAM_DESIGNER_PARTITIONER)) { this.partitioner_class = (Class<? extends AbstractPartitioner>) target_class; } else if (key.equals(PARAM_DESIGNER_MAPPER)) { this.mapper_class = (Class<? extends AbstractMapper>) target_class; } else if (key.equals(PARAM_DESIGNER_INDEXER)) { this.indexer_class = (Class<? extends AbstractIndexSelector>) target_class; } else if (key.equals(PARAM_DESIGNER_COSTMODEL)) { this.costmodel_class = (Class<? extends AbstractCostModel>) target_class; // Special Case: TimeIntervalCostModel if (target_name.endsWith(TimeIntervalCostModel.class.getSimpleName())) { this.costmodel = new TimeIntervalCostModel<SingleSitedCostModel>( this.catalog_db, SingleSitedCostModel.class, this.num_intervals); } else { this.costmodel = ClassUtil.newInstance( this.costmodel_class, new Object[] {this.catalog_db}, new Class[] {Database.class}); } } else { assert (false) : "Invalid key '" + key + "'"; } } } // FOR // ------------------------------------------------------- // TRANSACTION ESTIMATION COMPONENTS // ------------------------------------------------------- if (this.params.containsKey(PARAM_MAPPINGS)) { assert (this.catalog_db != null); File path = new File(this.params.get(PARAM_MAPPINGS)); if (path.exists()) { this.param_mappings.load(path.getAbsolutePath(), this.catalog_db); } else { LOG.warn("The ParameterMappings file '" + path + "' does not exist"); } } if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS_VALUE)) { assert (this.catalog_db != null); float defaultValue = this.getDoubleParam(PARAM_MARKOV_THRESHOLDS_VALUE).floatValue(); this.thresholds = new EstimationThresholds(defaultValue); this.params.put(PARAM_MARKOV_THRESHOLDS, this.thresholds.toString()); LOG.debug("CREATED THRESHOLDS: " + this.thresholds); } else if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS)) { assert (this.catalog_db != null); this.thresholds = new EstimationThresholds(); File path = new File(this.params.get(PARAM_MARKOV_THRESHOLDS)); if (path.exists()) { this.thresholds.load(path.getAbsolutePath(), this.catalog_db); } else { LOG.warn("The estimation thresholds file '" + path + "' does not exist"); } LOG.debug("LOADED THRESHOLDS: " + this.thresholds); } // ------------------------------------------------------- // HASHER // ------------------------------------------------------- if (this.catalog != null) { if (this.params.containsKey(PARAM_HASHER_CLASS)) { String hasherClassName = this.params.get(PARAM_HASHER_CLASS); this.hasher_class = (Class<? extends AbstractHasher>) loader.loadClass(hasherClassName); } Constructor<? extends AbstractHasher> constructor = this.hasher_class.getConstructor(new Class[] {Database.class, Integer.class}); int num_partitions = CatalogUtil.getNumberOfPartitions(this.catalog_db); this.hasher = constructor.newInstance(new Object[] {this.catalog_db, num_partitions}); if (!(this.hasher instanceof DefaultHasher)) LOG.debug("Loaded hasher " + this.hasher.getClass()); if (this.params.containsKey(PARAM_HASHER_PROFILE)) { this.hasher.load(this.params.get(PARAM_HASHER_PROFILE), null); } } // ------------------------------------------------------- // SAMPLE WORKLOAD TRACE // ------------------------------------------------------- this.loadWorkload(); }
/** * Return key/value maps into a nicely formatted table The maps are displayed in order from first * to last, and there will be a spacer created between each map. The format for each record is: * * <p><KEY><DELIMITER><SPACING><VALUE> * * <p>If the delimiter is an equal sign, then the format is: * * <p><KEY><SPACING><DELIMITER><VALUE> * * @param delimiter * @param upper Upper-case all keys * @param box Box results * @param border_top TODO * @param border_bottom TODO * @param recursive TODO * @param first_element_title TODO * @param maps * @return */ @SuppressWarnings("unchecked") public static String formatMaps( String delimiter, boolean upper, boolean box, boolean border_top, boolean border_bottom, boolean recursive, boolean first_element_title, Map<?, ?>... maps) { boolean need_divider = (maps.length > 1 || border_bottom || border_top); // Figure out the largest key size so we can get spacing right int max_key_size = 0; int max_title_size = 0; final Map<Object, String[]> map_keys[] = (Map<Object, String[]>[]) new Map[maps.length]; final boolean map_titles[] = new boolean[maps.length]; for (int i = 0; i < maps.length; i++) { Map<?, ?> m = maps[i]; if (m == null) continue; Map<Object, String[]> keys = new HashMap<Object, String[]>(); boolean first = true; for (Object k : m.keySet()) { String k_str[] = LINE_SPLIT.split(k != null ? k.toString() : ""); keys.put(k, k_str); // If the first element has a null value, then we can let it be the title for this map // It's length doesn't affect the other keys, but will affect the total size of the map if (first && first_element_title && m.get(k) == null) { for (String line : k_str) { max_title_size = Math.max(max_title_size, line.length()); } // FOR map_titles[i] = true; } else { for (String line : k_str) { max_key_size = Math.max(max_key_size, line.length()); } // FOR if (first) map_titles[i] = false; } first = false; } // FOR map_keys[i] = keys; } // FOR boolean equalsDelimiter = delimiter.equals("="); final String f = "%-" + (max_key_size + delimiter.length() + 1) + "s" + (equalsDelimiter ? "= " : "") + "%s\n"; // Now make StringBuilder blocks for each map // We do it in this way so that we can get the max length of the values int max_value_size = 0; StringBuilder blocks[] = new StringBuilder[maps.length]; for (int map_i = 0; map_i < maps.length; map_i++) { blocks[map_i] = new StringBuilder(); Map<?, ?> m = maps[map_i]; if (m == null) continue; Map<Object, String[]> keys = map_keys[map_i]; boolean first = true; for (Entry<?, ?> e : m.entrySet()) { String key[] = keys.get(e.getKey()); if (first && map_titles[map_i]) { blocks[map_i].append(StringUtil.join("\n", key)); if (CollectionUtil.last(key).endsWith("\n") == false) blocks[map_i].append("\n"); } else { Object v_obj = e.getValue(); String v = null; if (recursive && v_obj instanceof Map<?, ?>) { v = formatMaps( delimiter, upper, box, border_top, border_bottom, recursive, first_element_title, (Map<?, ?>) v_obj) .trim(); } else if (key.length == 1 && key[0].trim().isEmpty() && v_obj == null) { blocks[map_i].append("\n"); continue; } else if (v_obj == null) { v = "null"; } else { v = v_obj.toString(); } // If the key or value is multiple lines, format them nicely! String value[] = LINE_SPLIT.split(v); int total_lines = Math.max(key.length, value.length); for (int line_i = 0; line_i < total_lines; line_i++) { String k_line = (line_i < key.length ? key[line_i] : ""); if (upper) k_line = k_line.toUpperCase(); String v_line = (line_i < value.length ? value[line_i] : ""); if (line_i == (key.length - 1) && (first == false || (first && v_line.isEmpty() == false))) { if (equalsDelimiter == false && k_line.trim().isEmpty() == false) k_line += ":"; } blocks[map_i].append(String.format(f, k_line, v_line)); if (need_divider) max_value_size = Math.max(max_value_size, v_line.length()); } // FOR if (v.endsWith("\n")) blocks[map_i].append("\n"); } first = false; } } // FOR // Put it all together! // System.err.println("max_title_size=" + max_title_size + ", max_key_size=" + // max_key_size + ", max_value_size=" + max_value_size + ", delimiter=" + delimiter.length()); int total_width = Math.max(max_title_size, (max_key_size + max_value_size + delimiter.length())) + 1; String dividing_line = (need_divider ? repeat("-", total_width) : ""); StringBuilder sb = null; if (maps.length == 1) { sb = blocks[0]; } else { sb = new StringBuilder(); for (int i = 0; i < maps.length; i++) { if (blocks[i].length() == 0) continue; if (i != 0 && maps[i].size() > 0) sb.append(dividing_line).append("\n"); sb.append(blocks[i]); } // FOR } return (box ? StringUtil.box(sb.toString()) : (border_top ? dividing_line + "\n" : "") + sb.toString() + (border_bottom ? dividing_line : "")); }