Exemple #1
0
  @Override
  public boolean compile(VoltProjectBuilder builder) {
    if (m_compiled) {
      return true;
    }
    m_compiled =
        builder.compile(m_jarFileName, m_partitionPerSite, m_siteCount, m_replication, "localhost");

    // (1) Load catalog from Jar
    Catalog tmpCatalog = CatalogUtil.loadCatalogFromJar(m_jarFileName);

    // (2) Update catalog to include target cluster configuration
    ClusterConfiguration cc = new ClusterConfiguration();
    // Update cc with a bunch of hosts/sites/partitions
    for (int site = 0, currentPartition = 0; site < m_siteCount; ++site) {
      for (int partition = 0; partition < m_partitionPerSite; ++partition, ++currentPartition) {
        cc.addPartition("localhost", site, currentPartition);
      }
    }
    this.catalog = FixCatalog.addHostInfo(tmpCatalog, cc);

    // (3) Write updated catalog back out to jar file
    try {
      CatalogUtil.updateCatalogInJar(m_jarFileName, catalog);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }

    tmpCatalog = CatalogUtil.loadCatalogFromJar(m_jarFileName);
    // System.err.println(CatalogInfo.getInfo(this.catalog, new File(m_jarFileName)));

    // Construct the base command that we will want to use to start
    // all of the "remote" HStoreSites
    List<String> siteCommand = new ArrayList<String>();
    CollectionUtil.addAll(siteCommand, "ant", "hstore-site", "-Djar=" + m_jarFileName);
    // Be sure to include our HStoreConf parameters
    for (Entry<String, String> e : this.confParams.entrySet()) {
      siteCommand.add(String.format("-D%s=%s", e.getKey(), e.getValue()));
    }
    // Lastly, we will include the site.id as the last parameter
    // so that we can easily change it
    siteCommand.add("-Dsite.id=-1");

    m_procBuilder = new ProcessBuilder(siteCommand.toArray(new String[0]));
    m_procBuilder.redirectErrorStream(true);
    // set the working directory to obj/release/prod
    // m_procBuilder.directory(new File(m_buildDir + File.separator + "prod"));

    return m_compiled;
  }
Exemple #2
0
 /**
  * Main setUp method for test cases. Given the ProjectType we will populate the static catalog
  * field members The full_catalog flag is a hack to work around OutofMemory issues with TPC-E
  *
  * @param type
  * @param fkeys
  * @param full_catalog
  * @throws Exception
  */
 protected void setUp(ProjectType type, boolean fkeys, boolean full_catalog) throws Exception {
   super.setUp();
   is_first = (is_first == null ? true : false);
   this.last_type = type;
   catalog = project_catalogs.get(type);
   catalog_db = project_databases.get(type);
   p_estimator = project_p_estimators.get(type);
   if (catalog == null) {
     AbstractProjectBuilder projectBuilder = AbstractProjectBuilder.getProjectBuilder(type);
     if (ENABLE_JAR_REUSE) {
       File jar_path = projectBuilder.getJarPath(true);
       if (jar_path.exists()) {
         LOG.debug("LOAD CACHE JAR: " + jar_path.getAbsolutePath());
         catalog = CatalogUtil.loadCatalogFromJar(jar_path.getAbsolutePath());
       } else {
         LOG.debug("MISSING JAR: " + jar_path.getAbsolutePath());
       }
     }
     if (catalog == null) {
       switch (type) {
         case TPCC:
           catalog = TPCCProjectBuilder.getTPCCSchemaCatalog(true);
           // Update the ProcParameter mapping used in the catalogs
           //
           // ParametersUtil.populateCatalog(CatalogUtil.getDatabase(catalog),
           // ParametersUtil.getParameterMapping(type));
           break;
         case TPCE:
           catalog = projectBuilder.createCatalog(fkeys, full_catalog);
           break;
         case TM1:
         case SEATS:
         case AUCTIONMARK:
         case MARKOV:
         case LOCALITY:
         case MAPREDUCE:
           catalog = projectBuilder.getFullCatalog(fkeys);
           break;
         default:
           assert (false) : "Invalid project type - " + type;
       } // SWITCH
     }
     if (type == ProjectType.TPCC)
       ParametersUtil.populateCatalog(
           CatalogUtil.getDatabase(catalog), ParametersUtil.getParameterMapping(type));
     this.init(type, catalog);
   }
 }
  public boolean compile(
      final VoltCompiler compiler,
      final String jarPath,
      final int sitesPerHost,
      final int hostCount,
      final int replication,
      final String leaderAddress) {
    assert (jarPath != null);
    assert (sitesPerHost >= 1);
    assert (hostCount >= 1);
    assert (leaderAddress != null);

    // this stuff could all be converted to org.voltdb.compiler.projectfile.*
    // jaxb objects and (WE ARE!) marshaled to XML. Just needs some elbow grease.

    DocumentBuilderFactory docFactory;
    DocumentBuilder docBuilder;
    Document doc;
    try {
      docFactory = DocumentBuilderFactory.newInstance();
      docBuilder = docFactory.newDocumentBuilder();
      doc = docBuilder.newDocument();
    } catch (final ParserConfigurationException e) {
      e.printStackTrace();
      return false;
    }

    // <project>
    final Element project = doc.createElement("project");
    doc.appendChild(project);

    // <security>
    final Element security = doc.createElement("security");
    security.setAttribute("enabled", Boolean.valueOf(m_securityEnabled).toString());
    project.appendChild(security);

    // <database>
    final Element database = doc.createElement("database");
    database.setAttribute("name", "database");
    database.setAttribute("project", this.project_name);
    project.appendChild(database);
    buildDatabaseElement(doc, database);

    // boilerplate to write this DOM object to file.
    StreamResult result;
    try {
      final Transformer transformer = TransformerFactory.newInstance().newTransformer();
      transformer.setOutputProperty(OutputKeys.INDENT, "yes");
      result = new StreamResult(new StringWriter());
      final DOMSource domSource = new DOMSource(doc);
      transformer.transform(domSource, result);
    } catch (final TransformerConfigurationException e) {
      e.printStackTrace();
      return false;
    } catch (final TransformerFactoryConfigurationError e) {
      e.printStackTrace();
      return false;
    } catch (final TransformerException e) {
      e.printStackTrace();
      return false;
    }

    //        String xml = result.getWriter().toString();
    //        System.out.println(xml);

    final File projectFile = writeStringToTempFile(result.getWriter().toString());
    final String projectPath = projectFile.getPath();
    LOG.debug("PROJECT XML: " + projectPath);

    ClusterConfig cc =
        (this.cluster_config.isEmpty()
            ? new ClusterConfig(hostCount, sitesPerHost, replication, leaderAddress)
            : this.cluster_config);
    final boolean success =
        compiler.compile(projectPath, cc, jarPath, m_compilerDebugPrintStream, m_procInfoOverrides);

    // HACK: If we have a ParameterMappingsSet that we need to apply
    // either from a file or a fixed mappings, then we have
    // to load the catalog into this JVM, apply the mappings, and then
    // update the jar file with the new catalog
    if (m_paramMappingsFile != null || m_paramMappings.isEmpty() == false) {
      File jarFile = new File(jarPath);
      Catalog catalog = CatalogUtil.loadCatalogFromJar(jarFile);
      assert (catalog != null);
      Database catalog_db = CatalogUtil.getDatabase(catalog);

      this.applyParameterMappings(catalog_db);

      // Construct a List of prefetchable Statements
      this.applyPrefetchableFlags(catalog_db);

      // Write it out!
      try {
        CatalogUtil.updateCatalogInJar(jarFile, catalog, m_paramMappingsFile);
      } catch (Exception ex) {
        String msg = "Failed to updated Catalog in jar file '" + jarPath + "'";
        throw new RuntimeException(msg, ex);
      }
    }

    return success;
  }
  /**
   * @param args
   * @throws Exception
   */
  @SuppressWarnings("unchecked")
  public void process(String[] args, String... required) throws Exception {
    final boolean debug = LOG.isDebugEnabled();

    if (debug) LOG.debug("Processing " + args.length + " parameters...");
    final Pattern p = Pattern.compile("=");
    for (int i = 0, cnt = args.length; i < cnt; i++) {
      final String arg = args[i];
      final String[] parts = p.split(arg, 2);
      if (parts[0].startsWith("-")) parts[0] = parts[0].substring(1);

      if (parts.length == 1) {
        if (parts[0].startsWith("${") == false) this.opt_params.add(parts[0]);
        continue;
      } else if (parts[0].equalsIgnoreCase("tag")) {
        continue;
      } else if (parts[1].startsWith("${") || parts[0].startsWith("#")) {
        continue;
      }
      if (debug) LOG.debug(String.format("%-35s = %s", parts[0], parts[1]));

      // DesignerHints Override
      if (parts[0].startsWith(PARAM_DESIGNER_HINTS_PREFIX)) {
        String param = parts[0].replace(PARAM_DESIGNER_HINTS_PREFIX, "").toLowerCase();
        try {
          Field f = DesignerHints.class.getField(param);
          this.hints_params.put(f.getName(), parts[1]);
          if (debug) LOG.debug(String.format("DesignerHints.%s = %s", param, parts[1]));
        } catch (NoSuchFieldException ex) {
          throw new Exception("Unknown DesignerHints parameter: " + param, ex);
        }

      }
      // HStoreConf Parameter
      else if (HStoreConf.isConfParameter(parts[0])) {
        this.conf_params.put(parts[0].toLowerCase(), parts[1]);
      }
      // ArgumentsParser Parameter
      else if (PARAMS.contains(parts[0].toLowerCase())) {
        this.params.put(parts[0].toLowerCase(), parts[1]);
      }
      // Invalid!
      else {
        String suggestions = "";
        i = 0;
        String end = CollectionUtil.last(parts[0].split("\\."));
        for (String param : PARAMS) {
          String param_end = CollectionUtil.last(param.split("\\."));
          if (param.startsWith(parts[0])
              || (end != null && param.endsWith(end))
              || (end != null && param_end != null && param_end.startsWith(end))) {
            if (suggestions.isEmpty()) suggestions = ". Possible Matches:";
            suggestions += String.format("\n [%02d] %s", ++i, param);
          }
        } // FOR
        throw new Exception("Unknown parameter '" + parts[0] + "'" + suggestions);
      }
    } // FOR

    // -------------------------------------------------------
    // CATALOGS
    // -------------------------------------------------------

    // Text File
    if (this.params.containsKey(PARAM_CATALOG)) {
      String path = this.params.get(PARAM_CATALOG);
      if (debug) LOG.debug("Loading catalog from file '" + path + "'");
      Catalog catalog = CatalogUtil.loadCatalog(path);
      if (catalog == null)
        throw new Exception("Failed to load catalog object from file '" + path + "'");
      this.updateCatalog(catalog, new File(path));
    }
    // Jar File
    else if (this.params.containsKey(PARAM_CATALOG_JAR)) {
      String path = this.params.get(PARAM_CATALOG_JAR);
      this.params.put(PARAM_CATALOG, path);
      File jar_file = new File(path);
      Catalog catalog = CatalogUtil.loadCatalogFromJar(path);
      if (catalog == null)
        throw new Exception("Failed to load catalog object from jar file '" + path + "'");
      if (debug) LOG.debug("Loaded catalog from jar file '" + path + "'");
      this.updateCatalog(catalog, jar_file);

      if (!this.params.containsKey(PARAM_CATALOG_TYPE)) {
        String jar_name = jar_file.getName();
        int jar_idx = jar_name.lastIndexOf(".jar");
        if (jar_idx != -1) {
          ProjectType type = ProjectType.get(jar_name.substring(0, jar_idx));
          if (type != null) {
            if (debug) LOG.debug("Set catalog type '" + type + "' from catalog jar file name");
            this.catalog_type = type;
            this.params.put(PARAM_CATALOG_TYPE, this.catalog_type.toString());
          }
        }
      }
    }
    // Schema File
    else if (this.params.containsKey(PARAM_CATALOG_SCHEMA)) {
      String path = this.params.get(PARAM_CATALOG_SCHEMA);
      Catalog catalog = CompilerUtil.compileCatalog(path);
      if (catalog == null) throw new Exception("Failed to load schema from '" + path + "'");
      if (debug) LOG.debug("Loaded catalog from schema file '" + path + "'");
      this.updateCatalog(catalog, new File(path));
    }

    // Catalog Type
    if (this.params.containsKey(PARAM_CATALOG_TYPE)) {
      String catalog_type = this.params.get(PARAM_CATALOG_TYPE);
      ProjectType type = ProjectType.get(catalog_type);
      if (type == null) {
        throw new Exception("Unknown catalog type '" + catalog_type + "'");
      }
      this.catalog_type = type;
    }

    // Update Cluster Configuration
    if (this.params.containsKey(ArgumentsParser.PARAM_CATALOG_HOSTS)) {
      ClusterConfiguration cc =
          new ClusterConfiguration(this.getParam(ArgumentsParser.PARAM_CATALOG_HOSTS));
      this.updateCatalog(FixCatalog.addHostInfo(this.catalog, cc), null);
    }

    // Check the requirements after loading the catalog, because some of the
    // above parameters will set the catalog one
    if (required != null && required.length > 0) this.require(required);

    // -------------------------------------------------------
    // PHYSICAL DESIGN COMPONENTS
    // -------------------------------------------------------
    if (this.params.containsKey(PARAM_PARTITION_PLAN)) {
      assert (this.catalog_db != null);
      File path = new File(this.params.get(PARAM_PARTITION_PLAN));
      boolean ignoreMissing =
          this.getBooleanParam(ArgumentsParser.PARAM_PARTITION_PLAN_IGNORE_MISSING, false);
      if (path.exists() || (path.exists() == false && ignoreMissing == false)) {
        if (debug) LOG.debug("Loading in partition plan from '" + path + "'");
        this.pplan = new PartitionPlan();
        this.pplan.load(path.getAbsolutePath(), this.catalog_db);

        // Apply!
        if (this.params.containsKey(PARAM_PARTITION_PLAN_APPLY)
            && this.getBooleanParam(PARAM_PARTITION_PLAN_APPLY)) {
          boolean secondaryIndexes =
              this.getBooleanParam(PARAM_PARTITION_PLAN_NO_SECONDARY, false) == false;
          LOG.info(
              String.format(
                  "Applying PartitionPlan '%s' to catalog [enableSecondary=%s]",
                  path.getName(), secondaryIndexes));
          this.pplan.apply(this.catalog_db, secondaryIndexes);
        }
      }
    }

    // -------------------------------------------------------
    // DESIGNER COMPONENTS
    // -------------------------------------------------------

    if (this.params.containsKey(PARAM_DESIGNER_THREADS)) {
      this.max_concurrent = Integer.valueOf(this.params.get(PARAM_DESIGNER_THREADS));
    }
    if (this.params.containsKey(PARAM_DESIGNER_INTERVALS)) {
      this.num_intervals = Integer.valueOf(this.params.get(PARAM_DESIGNER_INTERVALS));
    }
    if (this.params.containsKey(PARAM_DESIGNER_HINTS)) {
      String path = this.params.get(PARAM_DESIGNER_HINTS);
      if (debug)
        LOG.debug(
            "Loading in designer hints from '"
                + path
                + "'.\nForced Values:\n"
                + StringUtil.formatMaps(this.hints_params));
      this.designer_hints.load(path, catalog_db, this.hints_params);
    }
    if (this.params.containsKey(PARAM_DESIGNER_CHECKPOINT)) {
      this.designer_checkpoint = new File(this.params.get(PARAM_DESIGNER_CHECKPOINT));
    }

    String designer_attributes[] = {
      PARAM_DESIGNER_PARTITIONER,
      PARAM_DESIGNER_MAPPER,
      PARAM_DESIGNER_INDEXER,
      PARAM_DESIGNER_COSTMODEL
    };
    ClassLoader loader = ClassLoader.getSystemClassLoader();
    for (String key : designer_attributes) {
      if (this.params.containsKey(key)) {
        String target_name = this.params.get(key);
        Class<?> target_class = loader.loadClass(target_name);
        assert (target_class != null);
        if (debug) LOG.debug("Set " + key + " class to " + target_class.getName());

        if (key.equals(PARAM_DESIGNER_PARTITIONER)) {
          this.partitioner_class = (Class<? extends AbstractPartitioner>) target_class;
        } else if (key.equals(PARAM_DESIGNER_MAPPER)) {
          this.mapper_class = (Class<? extends AbstractMapper>) target_class;
        } else if (key.equals(PARAM_DESIGNER_INDEXER)) {
          this.indexer_class = (Class<? extends AbstractIndexSelector>) target_class;
        } else if (key.equals(PARAM_DESIGNER_COSTMODEL)) {
          this.costmodel_class = (Class<? extends AbstractCostModel>) target_class;

          // Special Case: TimeIntervalCostModel
          if (target_name.endsWith(TimeIntervalCostModel.class.getSimpleName())) {
            this.costmodel =
                new TimeIntervalCostModel<SingleSitedCostModel>(
                    this.catalog_db, SingleSitedCostModel.class, this.num_intervals);
          } else {
            this.costmodel =
                ClassUtil.newInstance(
                    this.costmodel_class,
                    new Object[] {this.catalog_db},
                    new Class[] {Database.class});
          }
        } else {
          assert (false) : "Invalid key '" + key + "'";
        }
      }
    } // FOR

    // -------------------------------------------------------
    // TRANSACTION ESTIMATION COMPONENTS
    // -------------------------------------------------------
    if (this.params.containsKey(PARAM_MAPPINGS)) {
      assert (this.catalog_db != null);
      File path = new File(this.params.get(PARAM_MAPPINGS));
      if (path.exists()) {
        this.param_mappings.load(path.getAbsolutePath(), this.catalog_db);
      } else {
        LOG.warn("The ParameterMappings file '" + path + "' does not exist");
      }
    }
    if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS_VALUE)) {
      assert (this.catalog_db != null);
      float defaultValue = this.getDoubleParam(PARAM_MARKOV_THRESHOLDS_VALUE).floatValue();
      this.thresholds = new EstimationThresholds(defaultValue);
      this.params.put(PARAM_MARKOV_THRESHOLDS, this.thresholds.toString());
      LOG.debug("CREATED THRESHOLDS: " + this.thresholds);

    } else if (this.params.containsKey(PARAM_MARKOV_THRESHOLDS)) {
      assert (this.catalog_db != null);
      this.thresholds = new EstimationThresholds();
      File path = new File(this.params.get(PARAM_MARKOV_THRESHOLDS));
      if (path.exists()) {
        this.thresholds.load(path.getAbsolutePath(), this.catalog_db);
      } else {
        LOG.warn("The estimation thresholds file '" + path + "' does not exist");
      }
      LOG.debug("LOADED THRESHOLDS: " + this.thresholds);
    }

    // -------------------------------------------------------
    // HASHER
    // -------------------------------------------------------
    if (this.catalog != null) {
      if (this.params.containsKey(PARAM_HASHER_CLASS)) {
        String hasherClassName = this.params.get(PARAM_HASHER_CLASS);
        this.hasher_class = (Class<? extends AbstractHasher>) loader.loadClass(hasherClassName);
      }
      Constructor<? extends AbstractHasher> constructor =
          this.hasher_class.getConstructor(new Class[] {Database.class, Integer.class});
      int num_partitions = CatalogUtil.getNumberOfPartitions(this.catalog_db);
      this.hasher = constructor.newInstance(new Object[] {this.catalog_db, num_partitions});
      if (!(this.hasher instanceof DefaultHasher))
        LOG.debug("Loaded hasher " + this.hasher.getClass());

      if (this.params.containsKey(PARAM_HASHER_PROFILE)) {
        this.hasher.load(this.params.get(PARAM_HASHER_PROFILE), null);
      }
    }

    // -------------------------------------------------------
    // SAMPLE WORKLOAD TRACE
    // -------------------------------------------------------
    this.loadWorkload();
  }