Ejemplo n.º 1
0
  private static void buildCatalog() throws IOException {
    // build a real catalog
    File cat = File.createTempFile("temp-log-reinitiator", "catalog");
    cat.deleteOnExit();

    VoltProjectBuilder builder = new VoltProjectBuilder();
    String schema = "create table A (i integer not null, primary key (i));";
    builder.addLiteralSchema(schema);
    builder.addPartitionInfo("A", "i");
    builder.addStmtProcedure("hello", "select * from A where i = ?", "A.i: 0");

    if (!builder.compile(cat.getAbsolutePath())) {
      throw new IOException();
    }

    byte[] bytes = CatalogUtil.toBytes(cat);
    String serializedCat = CatalogUtil.loadCatalogFromJar(bytes, null);
    assertNotNull(serializedCat);
    Catalog catalog = new Catalog();
    catalog.execute(serializedCat);

    String deploymentPath = builder.getPathToDeployment();
    CatalogUtil.compileDeploymentAndGetCRC(catalog, deploymentPath, true);

    m_context = new CatalogContext(0, 0, catalog, bytes, 0, 0, 0);
    TheHashinator.initialize(LegacyHashinator.class, LegacyHashinator.getConfigureBytes(3));
  }
Ejemplo n.º 2
0
    @Override
    public void run() {
      CatalogAndIds catalogStuff = null;
      do {
        try {
          catalogStuff = CatalogUtil.getCatalogFromZK(m_rvdb.getHostMessenger().getZK());
        } catch (org.apache.zookeeper_voltpatches.KeeperException.NoNodeException e) {
        } catch (Exception e) {
          VoltDB.crashLocalVoltDB(
              "System was interrupted while waiting for a catalog.", false, null);
        }
      } while (catalogStuff == null || catalogStuff.catalogBytes.length == 0);

      String serializedCatalog = null;
      byte[] catalogJarBytes = catalogStuff.catalogBytes;
      try {
        Pair<InMemoryJarfile, String> loadResults =
            CatalogUtil.loadAndUpgradeCatalogFromJar(catalogStuff.catalogBytes);
        serializedCatalog = CatalogUtil.getSerializedCatalogStringFromJar(loadResults.getFirst());
        catalogJarBytes = loadResults.getFirst().getFullJarBytes();
      } catch (IOException e) {
        VoltDB.crashLocalVoltDB("Unable to load catalog", false, e);
      }

      if ((serializedCatalog == null) || (serializedCatalog.length() == 0))
        VoltDB.crashLocalVoltDB("Catalog loading failure", false, null);

      /* N.B. node recovery requires discovering the current catalog version. */
      Catalog catalog = new Catalog();
      catalog.execute(serializedCatalog);
      serializedCatalog = null;

      // note if this fails it will print an error first
      // This is where we compile real catalog and create runtime
      // catalog context. To validate deployment we compile and create
      // a starter context which uses a placeholder catalog.
      String result = CatalogUtil.compileDeployment(catalog, m_deployment, false);
      if (result != null) {
        hostLog.fatal(result);
        VoltDB.crashLocalVoltDB(result);
      }

      try {
        m_rvdb.m_catalogContext =
            new CatalogContext(
                catalogStuff.txnId,
                catalogStuff.uniqueId,
                catalog,
                catalogJarBytes,
                // Our starter catalog has set the deployment stuff, just yoink it out for now
                m_rvdb.m_catalogContext.getDeploymentBytes(),
                catalogStuff.version,
                -1);
      } catch (Exception e) {
        VoltDB.crashLocalVoltDB("Error agreeing on starting catalog version", true, e);
      }
    }
Ejemplo n.º 3
0
  public MockVoltDB(int clientPort, int adminPort, int httpPort, int drPort) {
    try {
      JSONObject obj = new JSONObject();
      JSONArray jsonArray = new JSONArray();
      jsonArray.put("127.0.0.1");
      obj.put("interfaces", jsonArray);
      obj.put("clientPort", clientPort);
      obj.put("adminPort", adminPort);
      obj.put("httpPort", httpPort);
      obj.put("drPort", drPort);
      m_localMetadata = obj.toString(4);

      m_catalog = new Catalog();
      m_catalog.execute("add / clusters " + m_clusterName);
      m_catalog.execute(
          "add "
              + m_catalog.getClusters().get(m_clusterName).getPath()
              + " databases "
              + m_databaseName);
      Cluster cluster = m_catalog.getClusters().get(m_clusterName);
      // Set a sane default for TestMessaging (at least)
      cluster.setHeartbeattimeout(10000);
      assert (cluster != null);

      try {
        m_hostMessenger.start();
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
      VoltZK.createPersistentZKNodes(m_hostMessenger.getZK());
      m_hostMessenger
          .getZK()
          .create(
              VoltZK.cluster_metadata + "/" + m_hostMessenger.getHostId(),
              getLocalMetadata().getBytes("UTF-8"),
              Ids.OPEN_ACL_UNSAFE,
              CreateMode.EPHEMERAL);

      m_hostMessenger.generateMailboxId(
          m_hostMessenger.getHSIdForLocalSite(HostMessenger.STATS_SITE_ID));
      m_statsAgent = new StatsAgent();
      m_statsAgent.registerMailbox(
          m_hostMessenger, m_hostMessenger.getHSIdForLocalSite(HostMessenger.STATS_SITE_ID));
      for (MailboxType type : MailboxType.values()) {
        m_mailboxMap.put(type, new LinkedList<MailboxNodeContent>());
      }
      m_mailboxMap
          .get(MailboxType.StatsAgent)
          .add(
              new MailboxNodeContent(
                  m_hostMessenger.getHSIdForLocalSite(HostMessenger.STATS_SITE_ID), null));
      m_siteTracker = new SiteTracker(m_hostId, m_mailboxMap);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
Ejemplo n.º 4
0
  /**
   * Loads the schema at ddlurl and setups a voltcompiler / hsql instance.
   *
   * @param ddlurl URL to the schema/ddl file.
   * @param basename Unique string, JSON plans [basename]-stmt-#_json.txt on disk
   * @throws Exception
   */
  public PlannerTestAideDeCamp(URL ddlurl, String basename) throws Exception {
    catalog = new Catalog();
    catalog.execute("add / clusters cluster");
    catalog.execute("add /clusters[cluster] databases database");
    db = catalog.getClusters().get("cluster").getDatabases().get("database");
    proc = db.getProcedures().add(basename);

    String schemaPath = URLDecoder.decode(ddlurl.getPath(), "UTF-8");

    VoltCompiler compiler = new VoltCompiler();
    hsql = HSQLInterface.loadHsqldb();
    // hsql.runDDLFile(schemaPath);
    VoltDDLElementTracker partitionMap = new VoltDDLElementTracker(compiler);
    DDLCompiler ddl_compiler = new DDLCompiler(compiler, hsql, partitionMap, db);
    ddl_compiler.loadSchema(schemaPath);
    ddl_compiler.compileToCatalog(catalog, db);
  }
Ejemplo n.º 5
0
 public synchronized void killSite(long siteId) {
   m_catalog = m_catalog.deepCopy();
   for (List<MailboxNodeContent> lmnc : m_mailboxMap.values()) {
     Iterator<MailboxNodeContent> iter = lmnc.iterator();
     while (iter.hasNext()) {
       if (iter.next().HSId == siteId) {
         iter.remove();
       }
     }
   }
   m_siteTracker = new SiteTracker(m_hostId, m_mailboxMap);
 }
Ejemplo n.º 6
0
  void compileXMLRootNode(ProjectType project) throws VoltCompilerException {
    m_catalog = new Catalog();
    temporaryCatalogInit();

    SecurityType security = project.getSecurity();
    if (security != null) {
      m_catalog.getClusters().get("cluster").setSecurityenabled(security.isEnabled());
    }

    DatabaseType database = project.getDatabase();
    if (database != null) {
      compileDatabaseNode(database);
    }
  }
Ejemplo n.º 7
0
  /** Generate the HTML catalog report from a newly compiled VoltDB catalog */
  public static String report(Catalog catalog, ArrayList<Feedback> warnings) throws IOException {
    // asynchronously get platform properties
    new Thread() {
      @Override
      public void run() {
        PlatformProperties.getPlatformProperties();
      }
    }.start();

    URL url = Resources.getResource(ReportMaker.class, "template.html");
    String contents = Resources.toString(url, Charsets.UTF_8);

    Cluster cluster = catalog.getClusters().get("cluster");
    assert (cluster != null);
    Database db = cluster.getDatabases().get("database");
    assert (db != null);

    String statsData = getStatsHTML(db, warnings);
    contents = contents.replace("##STATS##", statsData);

    String schemaData = generateSchemaTable(db.getTables(), db.getConnectors());
    contents = contents.replace("##SCHEMA##", schemaData);

    String procData = generateProceduresTable(db.getProcedures());
    contents = contents.replace("##PROCS##", procData);

    DatabaseSizes sizes = CatalogSizing.getCatalogSizes(db);

    String sizeData = generateSizeTable(sizes);
    contents = contents.replace("##SIZES##", sizeData);

    String sizeSummary = generateSizeSummary(sizes);
    contents = contents.replace("##SIZESUMMARY##", sizeSummary);

    String platformData = PlatformProperties.getPlatformProperties().toHTML();
    contents = contents.replace("##PLATFORM##", platformData);

    contents = contents.replace("##VERSION##", VoltDB.instance().getVersionString());

    DateFormat df = new SimpleDateFormat("d MMM yyyy HH:mm:ss z");
    contents = contents.replace("##TIMESTAMP##", df.format(m_timestamp));

    String msg =
        Encoder.hexEncode(VoltDB.instance().getVersionString() + "," + System.currentTimeMillis());
    contents = contents.replace("get.py?a=KEY&", String.format("get.py?a=%s&", msg));

    return contents;
  }
Ejemplo n.º 8
0
  protected void setUp(AbstractProjectBuilder projectBuilder, boolean force) throws Exception {
    super.setUp();
    is_first = (is_first == null ? true : false);
    this.last_type = ProjectType.TEST;
    if (force == false) {
      catalog = project_catalogs.get(this.last_type);
      catalog_db = project_databases.get(this.last_type);
      p_estimator = project_p_estimators.get(this.last_type);
    }
    if (catalog == null || force) {
      String catalogJar = new File(projectBuilder.getJarName(true)).getAbsolutePath();
      try {
        boolean status = projectBuilder.compile(catalogJar);
        assert (status);
      } catch (Exception ex) {
        throw new RuntimeException(
            "Failed to create " + projectBuilder.getProjectName() + " catalog [" + catalogJar + "]",
            ex);
      }

      catalog = new Catalog();
      try {
        // read in the catalog
        String serializedCatalog =
            JarReader.readFileFromJarfile(catalogJar, CatalogUtil.CATALOG_FILENAME);
        // create the catalog (that will be passed to the ClientInterface
        catalog.execute(serializedCatalog);
      } catch (Exception ex) {
        throw new RuntimeException(
            "Failed to load " + projectBuilder.getProjectName() + " catalog [" + catalogJar + "]",
            ex);
      }

      this.init(this.last_type, catalog);
    }
  }
Ejemplo n.º 9
0
 public Cluster getCluster() {
   return m_catalog.getClusters().get(m_clusterName);
 }
Ejemplo n.º 10
0
  @SuppressWarnings("deprecation")
  @Override
  public void setUp() throws IOException, InterruptedException {
    VoltDB.instance().readBuildInfo("Test");

    // compile a catalog
    String testDir = BuildDirectoryUtils.getBuildDirectoryPath();
    String catalogJar = testDir + File.separator + JAR;

    TPCCProjectBuilder pb = new TPCCProjectBuilder();
    pb.addDefaultSchema();
    pb.addDefaultPartitioning();
    pb.addProcedures(MultiSiteSelect.class, InsertNewOrder.class);

    pb.compile(catalogJar, 2, 0);

    // load a catalog
    byte[] bytes = CatalogUtil.toBytes(new File(catalogJar));
    String serializedCatalog = CatalogUtil.loadCatalogFromJar(bytes, null);

    // create the catalog (that will be passed to the ClientInterface
    catalog = new Catalog();
    catalog.execute(serializedCatalog);

    // update the catalog with the data from the deployment file
    String pathToDeployment = pb.getPathToDeployment();
    assertTrue(CatalogUtil.compileDeploymentAndGetCRC(catalog, pathToDeployment, true) >= 0);

    cluster = catalog.getClusters().get("cluster");
    CatalogMap<Procedure> procedures = cluster.getDatabases().get("database").getProcedures();
    Procedure insertProc = procedures.get("InsertNewOrder");
    assert (insertProc != null);
    selectProc = procedures.get("MultiSiteSelect");
    assert (selectProc != null);

    // Each EE needs its own thread for correct initialization.
    final AtomicReference<ExecutionEngine> site1Reference = new AtomicReference<ExecutionEngine>();
    final byte configBytes[] = LegacyHashinator.getConfigureBytes(2);
    Thread site1Thread =
        new Thread() {
          @Override
          public void run() {
            site1Reference.set(
                new ExecutionEngineJNI(
                    cluster.getRelativeIndex(),
                    1,
                    0,
                    0,
                    "",
                    100,
                    HashinatorType.LEGACY,
                    configBytes));
          }
        };
    site1Thread.start();
    site1Thread.join();

    final AtomicReference<ExecutionEngine> site2Reference = new AtomicReference<ExecutionEngine>();
    Thread site2Thread =
        new Thread() {
          @Override
          public void run() {
            site2Reference.set(
                new ExecutionEngineJNI(
                    cluster.getRelativeIndex(),
                    2,
                    1,
                    0,
                    "",
                    100,
                    HashinatorType.LEGACY,
                    configBytes));
          }
        };
    site2Thread.start();
    site2Thread.join();

    // create two EEs
    site1 = new ExecutionSite(0); // site 0
    ee1 = site1Reference.get();
    ee1.loadCatalog(0, catalog.serialize());
    site2 = new ExecutionSite(1); // site 1
    ee2 = site2Reference.get();
    ee2.loadCatalog(0, catalog.serialize());

    // cache some plan fragments
    selectStmt = selectProc.getStatements().get("selectAll");
    assert (selectStmt != null);
    int i = 0;
    // this kinda assumes the right order
    for (PlanFragment f : selectStmt.getFragments()) {
      if (i == 0) selectTopFrag = f;
      else selectBottomFrag = f;
      i++;
    }
    assert (selectTopFrag != null);
    assert (selectBottomFrag != null);

    if (selectTopFrag.getHasdependencies() == false) {
      PlanFragment temp = selectTopFrag;
      selectTopFrag = selectBottomFrag;
      selectBottomFrag = temp;
    }

    // get the insert frag
    Statement insertStmt = insertProc.getStatements().get("insert");
    assert (insertStmt != null);

    for (PlanFragment f : insertStmt.getFragments()) insertFrag = f;

    // populate plan cache
    ActivePlanRepository.clear();
    ActivePlanRepository.addFragmentForTest(
        CatalogUtil.getUniqueIdForFragment(selectBottomFrag),
        Encoder.base64Decode(selectBottomFrag.getPlannodetree()));
    ActivePlanRepository.addFragmentForTest(
        CatalogUtil.getUniqueIdForFragment(selectTopFrag),
        Encoder.base64Decode(selectTopFrag.getPlannodetree()));
    ActivePlanRepository.addFragmentForTest(
        CatalogUtil.getUniqueIdForFragment(insertFrag),
        Encoder.base64Decode(insertFrag.getPlannodetree()));

    // insert some data
    ParameterSet params = ParameterSet.fromArrayNoCopy(1L, 1L, 1L);

    VoltTable[] results =
        ee2.executePlanFragments(
            1,
            new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
            null,
            new ParameterSet[] {params},
            1,
            0,
            42,
            Long.MAX_VALUE);
    assert (results.length == 1);
    assert (results[0].asScalarLong() == 1L);

    params = ParameterSet.fromArrayNoCopy(2L, 2L, 2L);

    results =
        ee1.executePlanFragments(
            1,
            new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
            null,
            new ParameterSet[] {params},
            2,
            1,
            42,
            Long.MAX_VALUE);
    assert (results.length == 1);
    assert (results[0].asScalarLong() == 1L);
  }
Ejemplo n.º 11
0
 /**
  * Initialize TheHashinator
  *
  * @param catalog A pointer to the catalog data structure.
  */
 public static void initialize(Catalog catalog) {
   Cluster cluster = catalog.getClusters().get("cluster");
   partitionCount = cluster.getNum_partitions();
 }
Ejemplo n.º 12
0
 public CatalogContext update(String pathToNewJar, String diffCommands) {
   Catalog newCatalog = catalog.deepCopy();
   newCatalog.execute(diffCommands);
   CatalogContext retval = new CatalogContext(newCatalog, pathToNewJar);
   return retval;
 }
Ejemplo n.º 13
0
  @SuppressWarnings("unchecked")
  public Catalog compileCatalog(final String projectFileURL, final ClusterConfig clusterConfig) {
    if (!clusterConfig.validate()) {
      addErr(clusterConfig.getErrorMsg());
      return null;
    }

    // Compiler instance is reusable. Clear the cache.
    cachedAddedClasses.clear();
    m_currentFilename = new File(projectFileURL).getName();
    m_jarBuilder = new JarBuilder(this);

    if (m_outputStream != null) {
      m_outputStream.println("\n** BEGIN PROJECT COMPILE: " + m_currentFilename + " **");
    }

    ProjectType project = null;

    try {
      JAXBContext jc = JAXBContext.newInstance("org.voltdb.compiler.projectfile");
      // This schema shot the sheriff.
      SchemaFactory sf = SchemaFactory.newInstance(javax.xml.XMLConstants.W3C_XML_SCHEMA_NS_URI);
      Schema schema = sf.newSchema(this.getClass().getResource("ProjectFileSchema.xsd"));
      Unmarshaller unmarshaller = jc.createUnmarshaller();
      // But did not shoot unmarshaller!
      unmarshaller.setSchema(schema);
      JAXBElement<ProjectType> result =
          (JAXBElement<ProjectType>) unmarshaller.unmarshal(new File(projectFileURL));
      project = result.getValue();
    } catch (JAXBException e) {
      // Convert some linked exceptions to more friendly errors.
      if (e.getLinkedException() instanceof java.io.FileNotFoundException) {
        addErr(e.getLinkedException().getMessage());
        return null;
      }
      if (e.getLinkedException() instanceof org.xml.sax.SAXParseException) {
        addErr("Error schema validating project.xml file. " + e.getLinkedException().getMessage());
        return null;
      }
      throw new RuntimeException(e);
    } catch (SAXException e) {
      addErr("Error schema validating project.xml file. " + e.getMessage());
      return null;
    }

    try {
      compileXMLRootNode(project);
    } catch (final VoltCompilerException e) {
      //            compilerLog.l7dlog( Level.ERROR,
      // LogKeys.compiler_VoltCompiler_FailedToCompileXML.name(), null);
      LOG.error(e.getMessage(), e);
      // e.printStackTrace();
      return null;
    }
    assert (m_catalog != null);

    try {
      ClusterCompiler.compile(m_catalog, clusterConfig);
    } catch (RuntimeException e) {
      addErr(e.getMessage());
      return null;
    }

    // Optimization: Vertical Partitioning
    if (m_enableVerticalPartitionOptimizations) {
      if (m_verticalPartitionPlanner == null) {
        m_verticalPartitionPlanner =
            new VerticalPartitionPlanner(CatalogUtil.getDatabase(m_catalog), true);
      }
      try {
        m_verticalPartitionPlanner.optimizeDatabase();
      } catch (Exception ex) {
        LOG.warn("Unexpected error", ex);
        addErr("Failed to apply vertical partition optimizations");
      }
    }

    // add epoch info to catalog
    final int epoch = (int) (TransactionIdManager.getEpoch() / 1000);
    m_catalog.getClusters().get("cluster").setLocalepoch(epoch);

    // done handling files
    m_currentFilename = null;
    return m_catalog;
  }
Ejemplo n.º 14
0
  /**
   * Compile and cache the statement and plan and return the final plan graph.
   *
   * @param sql
   * @param paramCount
   */
  public List<AbstractPlanNode> compile(
      String sql,
      int paramCount,
      String joinOrder,
      Object partitionParameter,
      boolean inferSP,
      boolean lockInSP) {
    Statement catalogStmt = proc.getStatements().add("stmt-" + String.valueOf(compileCounter++));
    catalogStmt.setSqltext(sql);
    catalogStmt.setSinglepartition(partitionParameter != null);
    catalogStmt.setBatched(false);
    catalogStmt.setParamnum(paramCount);

    // determine the type of the query
    QueryType qtype = QueryType.SELECT;
    catalogStmt.setReadonly(true);
    if (sql.toLowerCase().startsWith("insert")) {
      qtype = QueryType.INSERT;
      catalogStmt.setReadonly(false);
    }
    if (sql.toLowerCase().startsWith("update")) {
      qtype = QueryType.UPDATE;
      catalogStmt.setReadonly(false);
    }
    if (sql.toLowerCase().startsWith("delete")) {
      qtype = QueryType.DELETE;
      catalogStmt.setReadonly(false);
    }
    catalogStmt.setQuerytype(qtype.getValue());
    // name will look like "basename-stmt-#"
    String name = catalogStmt.getParent().getTypeName() + "-" + catalogStmt.getTypeName();

    DatabaseEstimates estimates = new DatabaseEstimates();
    TrivialCostModel costModel = new TrivialCostModel();
    PartitioningForStatement partitioning =
        new PartitioningForStatement(partitionParameter, inferSP, lockInSP);
    QueryPlanner planner =
        new QueryPlanner(
            catalogStmt.getSqltext(),
            catalogStmt.getTypeName(),
            catalogStmt.getParent().getTypeName(),
            catalog.getClusters().get("cluster"),
            db,
            partitioning,
            hsql,
            estimates,
            false,
            StatementCompiler.DEFAULT_MAX_JOIN_TABLES,
            costModel,
            null,
            joinOrder);

    CompiledPlan plan = null;
    planner.parse();
    plan = planner.plan();
    assert (plan != null);

    // Input Parameters
    // We will need to update the system catalogs with this new information
    // If this is an adhoc query then there won't be any parameters
    for (int i = 0; i < plan.parameters.length; ++i) {
      StmtParameter catalogParam = catalogStmt.getParameters().add(String.valueOf(i));
      catalogParam.setJavatype(plan.parameters[i].getValue());
      catalogParam.setIndex(i);
    }

    // Output Columns
    int index = 0;
    for (SchemaColumn col : plan.columns.getColumns()) {
      Column catColumn = catalogStmt.getOutput_columns().add(String.valueOf(index));
      catColumn.setNullable(false);
      catColumn.setIndex(index);
      catColumn.setName(col.getColumnName());
      catColumn.setType(col.getType().getValue());
      catColumn.setSize(col.getSize());
      index++;
    }

    List<PlanNodeList> nodeLists = new ArrayList<PlanNodeList>();
    nodeLists.add(new PlanNodeList(plan.rootPlanGraph));
    if (plan.subPlanGraph != null) {
      nodeLists.add(new PlanNodeList(plan.subPlanGraph));
    }

    // Store the list of parameters types and indexes in the plan node list.
    List<Pair<Integer, VoltType>> parameters = nodeLists.get(0).getParameters();
    for (int i = 0; i < plan.parameters.length; ++i) {
      Pair<Integer, VoltType> parameter = new Pair<Integer, VoltType>(i, plan.parameters[i]);
      parameters.add(parameter);
    }

    // Now update our catalog information
    // HACK: We're using the node_tree's hashCode() as it's name. It would be really
    //     nice if the Catalog code give us an guid without needing a name first...

    String json = null;
    try {
      JSONObject jobj = new JSONObject(nodeLists.get(0).toJSONString());
      json = jobj.toString(4);
    } catch (JSONException e2) {
      // TODO Auto-generated catch block
      e2.printStackTrace();
      System.exit(-1);
    }

    //
    // We then stick a serialized version of PlanNodeTree into a PlanFragment
    //
    try {
      BuildDirectoryUtils.writeFile("statement-plans", name + "_json.txt", json);
      BuildDirectoryUtils.writeFile(
          "statement-plans", name + ".dot", nodeLists.get(0).toDOTString("name"));
    } catch (Exception e) {
      e.printStackTrace();
    }

    List<AbstractPlanNode> plannodes = new ArrayList<AbstractPlanNode>();
    for (PlanNodeList nodeList : nodeLists) {
      plannodes.add(nodeList.getRootPlanNode());
    }

    m_currentPlan = plan;
    return plannodes;
  }
Ejemplo n.º 15
0
  void compileDatabaseNode(DatabaseType database) throws VoltCompilerException {
    final ArrayList<String> programs = new ArrayList<String>();
    final ArrayList<String> schemas = new ArrayList<String>();
    final ArrayList<ProcedureDescriptor> procedures = new ArrayList<ProcedureDescriptor>();
    final ArrayList<Class<?>> classDependencies = new ArrayList<Class<?>>();
    final ArrayList<String[]> partitions = new ArrayList<String[]>();

    final String databaseName = database.getName();

    // schema does not verify that the database is named "database"
    if (databaseName.equals("database") == false) {
      final String msg =
          "VoltDB currently requires all database elements to be named "
              + "\"database\" (found: \""
              + databaseName
              + "\")";
      throw new VoltCompilerException(msg);
    }

    // create the database in the catalog
    m_catalog.execute("add /clusters[cluster] databases " + databaseName);
    Database db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);

    SnapshotType snapshotSettings = database.getSnapshot();
    if (snapshotSettings != null) {
      SnapshotSchedule schedule = db.getSnapshotschedule().add("default");
      String frequency = snapshotSettings.getFrequency();
      if (!frequency.endsWith("s") && !frequency.endsWith("m") && !frequency.endsWith("h")) {
        throw new VoltCompilerException(
            "Snapshot frequency "
                + frequency
                + " needs to end with time unit specified"
                + " that is one of [s, m, h] (seconds, minutes, hours)");
      }

      int frequencyInt = 0;
      String frequencySubstring = frequency.substring(0, frequency.length() - 1);
      try {
        frequencyInt = Integer.parseInt(frequencySubstring);
      } catch (Exception e) {
        throw new VoltCompilerException("Frequency " + frequencySubstring + " is not an integer ");
      }

      String prefix = snapshotSettings.getPrefix();
      if (prefix == null || prefix.isEmpty()) {
        throw new VoltCompilerException("Snapshot prefix " + prefix + " is not a valid prefix ");
      }

      if (prefix.contains("-") || prefix.contains(",")) {
        throw new VoltCompilerException("Snapshot prefix " + prefix + " cannot include , or - ");
      }

      String path = snapshotSettings.getPath();
      if (path == null || path.isEmpty()) {
        throw new VoltCompilerException("Snapshot path " + path + " is not a valid path ");
      }

      if (snapshotSettings.getRetain() == null) {
        throw new VoltCompilerException("Snapshot retain value not provided");
      }

      int retain = snapshotSettings.getRetain().intValue();
      if (retain < 1) {
        throw new VoltCompilerException(
            "Snapshot retain value " + retain + " is not a valid value. Must be 1 or greater.");
      }

      schedule.setFrequencyunit(frequency.substring(frequency.length() - 1, frequency.length()));
      schedule.setFrequencyvalue(frequencyInt);
      schedule.setPath(path);
      schedule.setPrefix(prefix);
      schedule.setRetain(retain);
    }

    // schemas/schema
    for (SchemasType.Schema schema : database.getSchemas().getSchema()) {
      LOG.l7dlog(
          Level.DEBUG,
          LogKeys.compiler_VoltCompiler_CatalogPath.name(),
          new Object[] {schema.getPath()},
          null);
      schemas.add(schema.getPath());
    }

    // groups/group.
    if (database.getGroups() != null) {
      for (GroupsType.Group group : database.getGroups().getGroup()) {
        org.voltdb.catalog.Group catGroup = db.getGroups().add(group.getName());
        catGroup.setAdhoc(group.isAdhoc());
        catGroup.setSysproc(group.isSysproc());
      }
    }

    // users/user
    if (database.getUsers() != null) {
      for (UsersType.User user : database.getUsers().getUser()) {
        org.voltdb.catalog.User catUser = db.getUsers().add(user.getName());
        catUser.setAdhoc(user.isAdhoc());
        catUser.setSysproc(user.isSysproc());
        byte passwordHash[] = extractPassword(user.getPassword());
        catUser.setShadowpassword(Encoder.hexEncode(passwordHash));

        // process the @groups comma separated list
        if (user.getGroups() != null) {
          String grouplist[] = user.getGroups().split(",");
          for (final String group : grouplist) {
            final GroupRef groupRef = catUser.getGroups().add(group);
            final Group catalogGroup = db.getGroups().get(group);
            if (catalogGroup != null) {
              groupRef.setGroup(catalogGroup);
            }
          }
        }
      }
    }

    // procedures/procedure
    for (ProceduresType.Procedure proc : database.getProcedures().getProcedure()) {
      procedures.add(getProcedure(proc));
    }

    // classdependencies/classdependency
    if (database.getClassdependencies() != null) {
      for (Classdependency dep : database.getClassdependencies().getClassdependency()) {
        classDependencies.add(getClassDependency(dep));
      }
    }

    // partitions/table
    if (database.getPartitions() != null) {
      for (org.voltdb.compiler.projectfile.PartitionsType.Partition table :
          database.getPartitions().getPartition()) {
        partitions.add(getPartition(table));
      }
    }

    String msg = "Database \"" + databaseName + "\" ";
    // TODO: schema allows 0 procedures. Testbase relies on this.
    if (procedures.size() == 0) {
      msg +=
          "needs at least one \"procedure\" element "
              + "(currently has "
              + String.valueOf(procedures.size())
              + ")";
      throw new VoltCompilerException(msg);
    }
    if (procedures.size() < 1) {
      msg += "is missing the \"procedures\" element";
      throw new VoltCompilerException(msg);
    }

    // shutdown and make a new hsqldb
    m_hsql = HSQLInterface.loadHsqldb();

    // Actually parse and handle all the programs
    for (final String programName : programs) {
      m_catalog.execute("add " + db.getPath() + " programs " + programName);
    }

    // Actually parse and handle all the DDL
    final DDLCompiler ddlcompiler = new DDLCompiler(this, m_hsql);

    for (final String schemaPath : schemas) {
      File schemaFile = null;

      if (schemaPath.contains(".jar!")) {
        String ddlText = null;
        try {
          ddlText = JarReader.readFileFromJarfile(schemaPath);
        } catch (final Exception e) {
          throw new VoltCompilerException(e);
        }
        schemaFile = VoltProjectBuilder.writeStringToTempFile(ddlText);
      } else {
        schemaFile = new File(schemaPath);
      }

      if (!schemaFile.isAbsolute()) {
        // Resolve schemaPath relative to the database definition xml file
        schemaFile = new File(new File(m_projectFileURL).getParent(), schemaPath);
      }

      // add the file object's path to the list of files for the jar
      m_ddlFilePaths.put(schemaFile.getName(), schemaFile.getPath());

      ddlcompiler.loadSchema(schemaFile.getAbsolutePath());
    }
    ddlcompiler.compileToCatalog(m_catalog, db);

    // Actually parse and handle all the partitions
    // this needs to happen before procedures are compiled
    msg = "In database \"" + databaseName + "\", ";
    final CatalogMap<Table> tables = db.getTables();
    for (final String[] partition : partitions) {
      final String tableName = partition[0];
      final String colName = partition[1];
      final Table t = tables.getIgnoreCase(tableName);
      if (t == null) {
        msg += "\"partition\" element has unknown \"table\" attribute '" + tableName + "'";
        throw new VoltCompilerException(msg);
      }
      final Column c = t.getColumns().getIgnoreCase(colName);
      // make sure the column exists
      if (c == null) {
        msg += "\"partition\" element has unknown \"column\" attribute '" + colName + "'";
        throw new VoltCompilerException(msg);
      }
      // make sure the column is marked not-nullable
      if (c.getNullable() == true) {
        msg +=
            "Partition column '"
                + tableName
                + "."
                + colName
                + "' is nullable. "
                + "Partition columns must be constrained \"NOT NULL\".";
        throw new VoltCompilerException(msg);
      }
      t.setPartitioncolumn(c);
      t.setIsreplicated(false);

      // Set the destination tables of associated views non-replicated.
      // If a view's source table is replicated, then a full scan of the
      // associated view is singled-sited. If the source is partitioned,
      // a full scan of the view must be distributed.
      final CatalogMap<MaterializedViewInfo> views = t.getViews();
      for (final MaterializedViewInfo mvi : views) {
        mvi.getDest().setIsreplicated(false);
      }
    }

    // add vertical partitions
    if (database.getVerticalpartitions() != null) {
      for (Verticalpartition vp : database.getVerticalpartitions().getVerticalpartition()) {
        try {
          addVerticalPartition(db, vp.getTable(), vp.getColumn(), vp.isIndexed());
        } catch (Exception ex) {
          throw new VoltCompilerException(
              "Failed to create vertical partition for " + vp.getTable(), ex);
        }
      }
    }

    // this should reorder the tables and partitions all alphabetically
    String catData = m_catalog.serialize();
    m_catalog = new Catalog();
    m_catalog.execute(catData);
    db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);

    // add database estimates info
    addDatabaseEstimatesInfo(m_estimates, db);
    addSystemProcsToCatalog(m_catalog, db);

    // Process and add exports and connectors to the catalog
    // Must do this before compiling procedures to deny updates
    // on append-only tables.
    if (database.getExports() != null) {
      // currently, only a single connector is allowed
      Connector conn = database.getExports().getConnector();
      compileConnector(conn, db);
    }

    // Actually parse and handle all the Procedures
    for (final ProcedureDescriptor procedureDescriptor : procedures) {
      final String procedureName = procedureDescriptor.m_className;
      m_currentFilename = procedureName.substring(procedureName.lastIndexOf('.') + 1);
      m_currentFilename += ".class";
      ProcedureCompiler.compile(this, m_hsql, m_estimates, m_catalog, db, procedureDescriptor);
    }

    // Add all the class dependencies to the output jar
    for (final Class<?> classDependency : classDependencies) {
      addClassToJar(classDependency, this);
    }

    m_hsql.close();
  }
Ejemplo n.º 16
0
 /** Initialize the catalog for one cluster */
 void temporaryCatalogInit() {
   m_catalog.execute("add / clusters cluster");
   m_catalog.getClusters().get("cluster").setSecurityenabled(false);
 }
Ejemplo n.º 17
0
 public CatalogContext deepCopy() {
   return new CatalogContext(catalog.deepCopy(), jarPath);
 }
Ejemplo n.º 18
0
  /**
   * @param projectFileURL URL of the project file.
   * @param clusterConfig Object containing desired physical cluster parameters
   * @param jarOutputPath The location to put the finished JAR to.
   * @param output Where to print status/errors to, usually stdout.
   * @param procInfoOverrides Optional overridden values for procedure annotations.
   */
  public boolean compile(
      final String projectFileURL,
      final ClusterConfig clusterConfig,
      final String jarOutputPath,
      final PrintStream output,
      final Map<String, ProcInfoData> procInfoOverrides) {
    m_hsql = null;
    m_projectFileURL = projectFileURL;
    m_jarOutputPath = jarOutputPath;
    m_outputStream = output;
    // use this map as default annotation values
    m_procInfoOverrides = procInfoOverrides;

    LOG.l7dlog(
        Level.DEBUG,
        LogKeys.compiler_VoltCompiler_LeaderAndHostCountAndSitesPerHost.name(),
        new Object[] {
          clusterConfig.getLeaderAddress(),
          clusterConfig.getHostCount(),
          clusterConfig.getSitesPerHost()
        },
        null);

    // do all the work to get the catalog
    final Catalog catalog = compileCatalog(projectFileURL, clusterConfig);
    if (catalog == null) {
      LOG.error(
          "VoltCompiler had " + m_errors.size() + " errors\n" + StringUtil.join("\n", m_errors));
      return (false);
    }

    // WRITE CATALOG TO JAR HERE
    final String catalogCommands = catalog.serialize();

    byte[] catalogBytes = null;
    try {
      catalogBytes = catalogCommands.getBytes("UTF-8");
    } catch (final UnsupportedEncodingException e1) {
      addErr("Can't encode the compiled catalog file correctly");
      return false;
    }

    // Create Dtxn.Coordinator configuration for cluster
    //        byte[] dtxnConfBytes = null;
    //        try {
    //            dtxnConfBytes = HStoreDtxnConf.toHStoreDtxnConf(catalog).getBytes("UTF-8");
    //        } catch (final Exception e1) {
    //            addErr("Can't encode the Dtxn.Coordinator configuration file correctly");
    //            return false;
    //        }

    try {
      //            m_jarBuilder.addEntry("dtxn.conf", dtxnConfBytes);
      m_jarBuilder.addEntry(CatalogUtil.CATALOG_FILENAME, catalogBytes);
      m_jarBuilder.addEntry("project.xml", new File(projectFileURL));
      for (final Entry<String, String> e : m_ddlFilePaths.entrySet())
        m_jarBuilder.addEntry(e.getKey(), new File(e.getValue()));
      m_jarBuilder.writeJarToDisk(jarOutputPath);
    } catch (final VoltCompilerException e) {
      return false;
    }

    assert (!hasErrors());

    if (hasErrors()) {
      return false;
    }

    return true;
  }