Example #1
0
  protected void setUp(AbstractProjectBuilder projectBuilder, boolean force) throws Exception {
    super.setUp();
    is_first = (is_first == null ? true : false);
    this.last_type = ProjectType.TEST;
    if (force == false) {
      catalog = project_catalogs.get(this.last_type);
      catalog_db = project_databases.get(this.last_type);
      p_estimator = project_p_estimators.get(this.last_type);
    }
    if (catalog == null || force) {
      String catalogJar = new File(projectBuilder.getJarName(true)).getAbsolutePath();
      try {
        boolean status = projectBuilder.compile(catalogJar);
        assert (status);
      } catch (Exception ex) {
        throw new RuntimeException(
            "Failed to create " + projectBuilder.getProjectName() + " catalog [" + catalogJar + "]",
            ex);
      }

      catalog = new Catalog();
      try {
        // read in the catalog
        String serializedCatalog =
            JarReader.readFileFromJarfile(catalogJar, CatalogUtil.CATALOG_FILENAME);
        // create the catalog (that will be passed to the ClientInterface
        catalog.execute(serializedCatalog);
      } catch (Exception ex) {
        throw new RuntimeException(
            "Failed to load " + projectBuilder.getProjectName() + " catalog [" + catalogJar + "]",
            ex);
      }

      this.init(this.last_type, catalog);
    }
  }
Example #2
0
  void compileDatabaseNode(DatabaseType database) throws VoltCompilerException {
    final ArrayList<String> programs = new ArrayList<String>();
    final ArrayList<String> schemas = new ArrayList<String>();
    final ArrayList<ProcedureDescriptor> procedures = new ArrayList<ProcedureDescriptor>();
    final ArrayList<Class<?>> classDependencies = new ArrayList<Class<?>>();
    final ArrayList<String[]> partitions = new ArrayList<String[]>();

    final String databaseName = database.getName();

    // schema does not verify that the database is named "database"
    if (databaseName.equals("database") == false) {
      final String msg =
          "VoltDB currently requires all database elements to be named "
              + "\"database\" (found: \""
              + databaseName
              + "\")";
      throw new VoltCompilerException(msg);
    }

    // create the database in the catalog
    m_catalog.execute("add /clusters[cluster] databases " + databaseName);
    Database db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);

    SnapshotType snapshotSettings = database.getSnapshot();
    if (snapshotSettings != null) {
      SnapshotSchedule schedule = db.getSnapshotschedule().add("default");
      String frequency = snapshotSettings.getFrequency();
      if (!frequency.endsWith("s") && !frequency.endsWith("m") && !frequency.endsWith("h")) {
        throw new VoltCompilerException(
            "Snapshot frequency "
                + frequency
                + " needs to end with time unit specified"
                + " that is one of [s, m, h] (seconds, minutes, hours)");
      }

      int frequencyInt = 0;
      String frequencySubstring = frequency.substring(0, frequency.length() - 1);
      try {
        frequencyInt = Integer.parseInt(frequencySubstring);
      } catch (Exception e) {
        throw new VoltCompilerException("Frequency " + frequencySubstring + " is not an integer ");
      }

      String prefix = snapshotSettings.getPrefix();
      if (prefix == null || prefix.isEmpty()) {
        throw new VoltCompilerException("Snapshot prefix " + prefix + " is not a valid prefix ");
      }

      if (prefix.contains("-") || prefix.contains(",")) {
        throw new VoltCompilerException("Snapshot prefix " + prefix + " cannot include , or - ");
      }

      String path = snapshotSettings.getPath();
      if (path == null || path.isEmpty()) {
        throw new VoltCompilerException("Snapshot path " + path + " is not a valid path ");
      }

      if (snapshotSettings.getRetain() == null) {
        throw new VoltCompilerException("Snapshot retain value not provided");
      }

      int retain = snapshotSettings.getRetain().intValue();
      if (retain < 1) {
        throw new VoltCompilerException(
            "Snapshot retain value " + retain + " is not a valid value. Must be 1 or greater.");
      }

      schedule.setFrequencyunit(frequency.substring(frequency.length() - 1, frequency.length()));
      schedule.setFrequencyvalue(frequencyInt);
      schedule.setPath(path);
      schedule.setPrefix(prefix);
      schedule.setRetain(retain);
    }

    // schemas/schema
    for (SchemasType.Schema schema : database.getSchemas().getSchema()) {
      LOG.l7dlog(
          Level.DEBUG,
          LogKeys.compiler_VoltCompiler_CatalogPath.name(),
          new Object[] {schema.getPath()},
          null);
      schemas.add(schema.getPath());
    }

    // groups/group.
    if (database.getGroups() != null) {
      for (GroupsType.Group group : database.getGroups().getGroup()) {
        org.voltdb.catalog.Group catGroup = db.getGroups().add(group.getName());
        catGroup.setAdhoc(group.isAdhoc());
        catGroup.setSysproc(group.isSysproc());
      }
    }

    // users/user
    if (database.getUsers() != null) {
      for (UsersType.User user : database.getUsers().getUser()) {
        org.voltdb.catalog.User catUser = db.getUsers().add(user.getName());
        catUser.setAdhoc(user.isAdhoc());
        catUser.setSysproc(user.isSysproc());
        byte passwordHash[] = extractPassword(user.getPassword());
        catUser.setShadowpassword(Encoder.hexEncode(passwordHash));

        // process the @groups comma separated list
        if (user.getGroups() != null) {
          String grouplist[] = user.getGroups().split(",");
          for (final String group : grouplist) {
            final GroupRef groupRef = catUser.getGroups().add(group);
            final Group catalogGroup = db.getGroups().get(group);
            if (catalogGroup != null) {
              groupRef.setGroup(catalogGroup);
            }
          }
        }
      }
    }

    // procedures/procedure
    for (ProceduresType.Procedure proc : database.getProcedures().getProcedure()) {
      procedures.add(getProcedure(proc));
    }

    // classdependencies/classdependency
    if (database.getClassdependencies() != null) {
      for (Classdependency dep : database.getClassdependencies().getClassdependency()) {
        classDependencies.add(getClassDependency(dep));
      }
    }

    // partitions/table
    if (database.getPartitions() != null) {
      for (org.voltdb.compiler.projectfile.PartitionsType.Partition table :
          database.getPartitions().getPartition()) {
        partitions.add(getPartition(table));
      }
    }

    String msg = "Database \"" + databaseName + "\" ";
    // TODO: schema allows 0 procedures. Testbase relies on this.
    if (procedures.size() == 0) {
      msg +=
          "needs at least one \"procedure\" element "
              + "(currently has "
              + String.valueOf(procedures.size())
              + ")";
      throw new VoltCompilerException(msg);
    }
    if (procedures.size() < 1) {
      msg += "is missing the \"procedures\" element";
      throw new VoltCompilerException(msg);
    }

    // shutdown and make a new hsqldb
    m_hsql = HSQLInterface.loadHsqldb();

    // Actually parse and handle all the programs
    for (final String programName : programs) {
      m_catalog.execute("add " + db.getPath() + " programs " + programName);
    }

    // Actually parse and handle all the DDL
    final DDLCompiler ddlcompiler = new DDLCompiler(this, m_hsql);

    for (final String schemaPath : schemas) {
      File schemaFile = null;

      if (schemaPath.contains(".jar!")) {
        String ddlText = null;
        try {
          ddlText = JarReader.readFileFromJarfile(schemaPath);
        } catch (final Exception e) {
          throw new VoltCompilerException(e);
        }
        schemaFile = VoltProjectBuilder.writeStringToTempFile(ddlText);
      } else {
        schemaFile = new File(schemaPath);
      }

      if (!schemaFile.isAbsolute()) {
        // Resolve schemaPath relative to the database definition xml file
        schemaFile = new File(new File(m_projectFileURL).getParent(), schemaPath);
      }

      // add the file object's path to the list of files for the jar
      m_ddlFilePaths.put(schemaFile.getName(), schemaFile.getPath());

      ddlcompiler.loadSchema(schemaFile.getAbsolutePath());
    }
    ddlcompiler.compileToCatalog(m_catalog, db);

    // Actually parse and handle all the partitions
    // this needs to happen before procedures are compiled
    msg = "In database \"" + databaseName + "\", ";
    final CatalogMap<Table> tables = db.getTables();
    for (final String[] partition : partitions) {
      final String tableName = partition[0];
      final String colName = partition[1];
      final Table t = tables.getIgnoreCase(tableName);
      if (t == null) {
        msg += "\"partition\" element has unknown \"table\" attribute '" + tableName + "'";
        throw new VoltCompilerException(msg);
      }
      final Column c = t.getColumns().getIgnoreCase(colName);
      // make sure the column exists
      if (c == null) {
        msg += "\"partition\" element has unknown \"column\" attribute '" + colName + "'";
        throw new VoltCompilerException(msg);
      }
      // make sure the column is marked not-nullable
      if (c.getNullable() == true) {
        msg +=
            "Partition column '"
                + tableName
                + "."
                + colName
                + "' is nullable. "
                + "Partition columns must be constrained \"NOT NULL\".";
        throw new VoltCompilerException(msg);
      }
      t.setPartitioncolumn(c);
      t.setIsreplicated(false);

      // Set the destination tables of associated views non-replicated.
      // If a view's source table is replicated, then a full scan of the
      // associated view is singled-sited. If the source is partitioned,
      // a full scan of the view must be distributed.
      final CatalogMap<MaterializedViewInfo> views = t.getViews();
      for (final MaterializedViewInfo mvi : views) {
        mvi.getDest().setIsreplicated(false);
      }
    }

    // add vertical partitions
    if (database.getVerticalpartitions() != null) {
      for (Verticalpartition vp : database.getVerticalpartitions().getVerticalpartition()) {
        try {
          addVerticalPartition(db, vp.getTable(), vp.getColumn(), vp.isIndexed());
        } catch (Exception ex) {
          throw new VoltCompilerException(
              "Failed to create vertical partition for " + vp.getTable(), ex);
        }
      }
    }

    // this should reorder the tables and partitions all alphabetically
    String catData = m_catalog.serialize();
    m_catalog = new Catalog();
    m_catalog.execute(catData);
    db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);

    // add database estimates info
    addDatabaseEstimatesInfo(m_estimates, db);
    addSystemProcsToCatalog(m_catalog, db);

    // Process and add exports and connectors to the catalog
    // Must do this before compiling procedures to deny updates
    // on append-only tables.
    if (database.getExports() != null) {
      // currently, only a single connector is allowed
      Connector conn = database.getExports().getConnector();
      compileConnector(conn, db);
    }

    // Actually parse and handle all the Procedures
    for (final ProcedureDescriptor procedureDescriptor : procedures) {
      final String procedureName = procedureDescriptor.m_className;
      m_currentFilename = procedureName.substring(procedureName.lastIndexOf('.') + 1);
      m_currentFilename += ".class";
      ProcedureCompiler.compile(this, m_hsql, m_estimates, m_catalog, db, procedureDescriptor);
    }

    // Add all the class dependencies to the output jar
    for (final Class<?> classDependency : classDependencies) {
      addClassToJar(classDependency, this);
    }

    m_hsql.close();
  }
Example #3
0
  public static final void addClassToJar(final Class<?> cls, final VoltCompiler compiler)
      throws VoltCompiler.VoltCompilerException {

    if (cachedAddedClasses.contains(cls)) {
      return;
    } else {
      cachedAddedClasses.add(cls);
    }

    for (final Class<?> nested : cls.getDeclaredClasses()) {
      addClassToJar(nested, compiler);
    }

    String packagePath = cls.getName();
    packagePath = packagePath.replace('.', '/');
    packagePath += ".class";

    String realName = cls.getName();
    realName = realName.substring(realName.lastIndexOf('.') + 1);
    realName += ".class";

    final URL absolutePath = cls.getResource(realName);
    File file = null;

    InputStream fis = null;
    int fileSize = 0;
    try {
      file = new File(URLDecoder.decode(absolutePath.getFile(), "UTF-8"));
      fis = new FileInputStream(file);
      assert (file.canRead());
      assert (file.isFile());
      fileSize = (int) file.length();
    } catch (final FileNotFoundException e) {
      try {
        final String contents = JarReader.readFileFromJarfile(absolutePath.getPath());
        fis = new StringInputStream(contents);
        fileSize = contents.length();
      } catch (final Exception e2) {
        final String msg = "Unable to locate classfile for " + realName;
        throw compiler.new VoltCompilerException(msg);
      }
    } catch (final UnsupportedEncodingException e) {
      e.printStackTrace();
      System.exit(-1);
    }

    assert (fileSize > 0);
    int readSize = 0;

    final byte[] fileBytes = new byte[fileSize];

    try {
      while (readSize < fileSize) {
        readSize = fis.read(fileBytes, readSize, fileSize - readSize);
      }
    } catch (final IOException e) {
      final String msg = "Unable to read (or completely read) classfile for " + realName;
      throw compiler.new VoltCompilerException(msg);
    }

    compiler.addEntryToJarOutput(packagePath, fileBytes);
  }