示例#1
0
  void compileDatabaseNode(DatabaseType database) throws VoltCompilerException {
    final ArrayList<String> programs = new ArrayList<String>();
    final ArrayList<String> schemas = new ArrayList<String>();
    final ArrayList<ProcedureDescriptor> procedures = new ArrayList<ProcedureDescriptor>();
    final ArrayList<Class<?>> classDependencies = new ArrayList<Class<?>>();
    final ArrayList<String[]> partitions = new ArrayList<String[]>();

    final String databaseName = database.getName();

    // schema does not verify that the database is named "database"
    if (databaseName.equals("database") == false) {
      final String msg =
          "VoltDB currently requires all database elements to be named "
              + "\"database\" (found: \""
              + databaseName
              + "\")";
      throw new VoltCompilerException(msg);
    }

    // create the database in the catalog
    m_catalog.execute("add /clusters[cluster] databases " + databaseName);
    Database db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);

    SnapshotType snapshotSettings = database.getSnapshot();
    if (snapshotSettings != null) {
      SnapshotSchedule schedule = db.getSnapshotschedule().add("default");
      String frequency = snapshotSettings.getFrequency();
      if (!frequency.endsWith("s") && !frequency.endsWith("m") && !frequency.endsWith("h")) {
        throw new VoltCompilerException(
            "Snapshot frequency "
                + frequency
                + " needs to end with time unit specified"
                + " that is one of [s, m, h] (seconds, minutes, hours)");
      }

      int frequencyInt = 0;
      String frequencySubstring = frequency.substring(0, frequency.length() - 1);
      try {
        frequencyInt = Integer.parseInt(frequencySubstring);
      } catch (Exception e) {
        throw new VoltCompilerException("Frequency " + frequencySubstring + " is not an integer ");
      }

      String prefix = snapshotSettings.getPrefix();
      if (prefix == null || prefix.isEmpty()) {
        throw new VoltCompilerException("Snapshot prefix " + prefix + " is not a valid prefix ");
      }

      if (prefix.contains("-") || prefix.contains(",")) {
        throw new VoltCompilerException("Snapshot prefix " + prefix + " cannot include , or - ");
      }

      String path = snapshotSettings.getPath();
      if (path == null || path.isEmpty()) {
        throw new VoltCompilerException("Snapshot path " + path + " is not a valid path ");
      }

      if (snapshotSettings.getRetain() == null) {
        throw new VoltCompilerException("Snapshot retain value not provided");
      }

      int retain = snapshotSettings.getRetain().intValue();
      if (retain < 1) {
        throw new VoltCompilerException(
            "Snapshot retain value " + retain + " is not a valid value. Must be 1 or greater.");
      }

      schedule.setFrequencyunit(frequency.substring(frequency.length() - 1, frequency.length()));
      schedule.setFrequencyvalue(frequencyInt);
      schedule.setPath(path);
      schedule.setPrefix(prefix);
      schedule.setRetain(retain);
    }

    // schemas/schema
    for (SchemasType.Schema schema : database.getSchemas().getSchema()) {
      LOG.l7dlog(
          Level.DEBUG,
          LogKeys.compiler_VoltCompiler_CatalogPath.name(),
          new Object[] {schema.getPath()},
          null);
      schemas.add(schema.getPath());
    }

    // groups/group.
    if (database.getGroups() != null) {
      for (GroupsType.Group group : database.getGroups().getGroup()) {
        org.voltdb.catalog.Group catGroup = db.getGroups().add(group.getName());
        catGroup.setAdhoc(group.isAdhoc());
        catGroup.setSysproc(group.isSysproc());
      }
    }

    // users/user
    if (database.getUsers() != null) {
      for (UsersType.User user : database.getUsers().getUser()) {
        org.voltdb.catalog.User catUser = db.getUsers().add(user.getName());
        catUser.setAdhoc(user.isAdhoc());
        catUser.setSysproc(user.isSysproc());
        byte passwordHash[] = extractPassword(user.getPassword());
        catUser.setShadowpassword(Encoder.hexEncode(passwordHash));

        // process the @groups comma separated list
        if (user.getGroups() != null) {
          String grouplist[] = user.getGroups().split(",");
          for (final String group : grouplist) {
            final GroupRef groupRef = catUser.getGroups().add(group);
            final Group catalogGroup = db.getGroups().get(group);
            if (catalogGroup != null) {
              groupRef.setGroup(catalogGroup);
            }
          }
        }
      }
    }

    // procedures/procedure
    for (ProceduresType.Procedure proc : database.getProcedures().getProcedure()) {
      procedures.add(getProcedure(proc));
    }

    // classdependencies/classdependency
    if (database.getClassdependencies() != null) {
      for (Classdependency dep : database.getClassdependencies().getClassdependency()) {
        classDependencies.add(getClassDependency(dep));
      }
    }

    // partitions/table
    if (database.getPartitions() != null) {
      for (org.voltdb.compiler.projectfile.PartitionsType.Partition table :
          database.getPartitions().getPartition()) {
        partitions.add(getPartition(table));
      }
    }

    String msg = "Database \"" + databaseName + "\" ";
    // TODO: schema allows 0 procedures. Testbase relies on this.
    if (procedures.size() == 0) {
      msg +=
          "needs at least one \"procedure\" element "
              + "(currently has "
              + String.valueOf(procedures.size())
              + ")";
      throw new VoltCompilerException(msg);
    }
    if (procedures.size() < 1) {
      msg += "is missing the \"procedures\" element";
      throw new VoltCompilerException(msg);
    }

    // shutdown and make a new hsqldb
    m_hsql = HSQLInterface.loadHsqldb();

    // Actually parse and handle all the programs
    for (final String programName : programs) {
      m_catalog.execute("add " + db.getPath() + " programs " + programName);
    }

    // Actually parse and handle all the DDL
    final DDLCompiler ddlcompiler = new DDLCompiler(this, m_hsql);

    for (final String schemaPath : schemas) {
      File schemaFile = null;

      if (schemaPath.contains(".jar!")) {
        String ddlText = null;
        try {
          ddlText = JarReader.readFileFromJarfile(schemaPath);
        } catch (final Exception e) {
          throw new VoltCompilerException(e);
        }
        schemaFile = VoltProjectBuilder.writeStringToTempFile(ddlText);
      } else {
        schemaFile = new File(schemaPath);
      }

      if (!schemaFile.isAbsolute()) {
        // Resolve schemaPath relative to the database definition xml file
        schemaFile = new File(new File(m_projectFileURL).getParent(), schemaPath);
      }

      // add the file object's path to the list of files for the jar
      m_ddlFilePaths.put(schemaFile.getName(), schemaFile.getPath());

      ddlcompiler.loadSchema(schemaFile.getAbsolutePath());
    }
    ddlcompiler.compileToCatalog(m_catalog, db);

    // Actually parse and handle all the partitions
    // this needs to happen before procedures are compiled
    msg = "In database \"" + databaseName + "\", ";
    final CatalogMap<Table> tables = db.getTables();
    for (final String[] partition : partitions) {
      final String tableName = partition[0];
      final String colName = partition[1];
      final Table t = tables.getIgnoreCase(tableName);
      if (t == null) {
        msg += "\"partition\" element has unknown \"table\" attribute '" + tableName + "'";
        throw new VoltCompilerException(msg);
      }
      final Column c = t.getColumns().getIgnoreCase(colName);
      // make sure the column exists
      if (c == null) {
        msg += "\"partition\" element has unknown \"column\" attribute '" + colName + "'";
        throw new VoltCompilerException(msg);
      }
      // make sure the column is marked not-nullable
      if (c.getNullable() == true) {
        msg +=
            "Partition column '"
                + tableName
                + "."
                + colName
                + "' is nullable. "
                + "Partition columns must be constrained \"NOT NULL\".";
        throw new VoltCompilerException(msg);
      }
      t.setPartitioncolumn(c);
      t.setIsreplicated(false);

      // Set the destination tables of associated views non-replicated.
      // If a view's source table is replicated, then a full scan of the
      // associated view is singled-sited. If the source is partitioned,
      // a full scan of the view must be distributed.
      final CatalogMap<MaterializedViewInfo> views = t.getViews();
      for (final MaterializedViewInfo mvi : views) {
        mvi.getDest().setIsreplicated(false);
      }
    }

    // add vertical partitions
    if (database.getVerticalpartitions() != null) {
      for (Verticalpartition vp : database.getVerticalpartitions().getVerticalpartition()) {
        try {
          addVerticalPartition(db, vp.getTable(), vp.getColumn(), vp.isIndexed());
        } catch (Exception ex) {
          throw new VoltCompilerException(
              "Failed to create vertical partition for " + vp.getTable(), ex);
        }
      }
    }

    // this should reorder the tables and partitions all alphabetically
    String catData = m_catalog.serialize();
    m_catalog = new Catalog();
    m_catalog.execute(catData);
    db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);

    // add database estimates info
    addDatabaseEstimatesInfo(m_estimates, db);
    addSystemProcsToCatalog(m_catalog, db);

    // Process and add exports and connectors to the catalog
    // Must do this before compiling procedures to deny updates
    // on append-only tables.
    if (database.getExports() != null) {
      // currently, only a single connector is allowed
      Connector conn = database.getExports().getConnector();
      compileConnector(conn, db);
    }

    // Actually parse and handle all the Procedures
    for (final ProcedureDescriptor procedureDescriptor : procedures) {
      final String procedureName = procedureDescriptor.m_className;
      m_currentFilename = procedureName.substring(procedureName.lastIndexOf('.') + 1);
      m_currentFilename += ".class";
      ProcedureCompiler.compile(this, m_hsql, m_estimates, m_catalog, db, procedureDescriptor);
    }

    // Add all the class dependencies to the output jar
    for (final Class<?> classDependency : classDependencies) {
      addClassToJar(classDependency, this);
    }

    m_hsql.close();
  }
示例#2
0
  protected void reflect() {
    // fill in the sql for single statement procs
    if (m_catProc.getHasjava() == false) {
      try {
        Map<String, Field> stmtMap =
            ProcedureCompiler.getValidSQLStmts(null, m_procedureName, m_procedure.getClass(), true);
        Field f = stmtMap.get(VoltDB.ANON_STMT_NAME);
        assert (f != null);
        SQLStmt stmt = (SQLStmt) f.get(m_procedure);
        Statement statement = m_catProc.getStatements().get(VoltDB.ANON_STMT_NAME);
        stmt.sqlText = statement.getSqltext().getBytes(VoltDB.UTF8ENCODING);
        m_cachedSingleStmt.stmt = stmt;

        int numParams = m_catProc.getParameters().size();
        m_paramTypes = new Class<?>[numParams];
        m_paramTypeIsPrimitive = new boolean[numParams];
        m_paramTypeIsArray = new boolean[numParams];
        m_paramTypeComponentType = new Class<?>[numParams];

        for (ProcParameter param : m_catProc.getParameters()) {
          VoltType type = VoltType.get((byte) param.getType());
          if (type == VoltType.INTEGER) {
            type = VoltType.BIGINT;
          } else if (type == VoltType.SMALLINT) {
            type = VoltType.BIGINT;
          } else if (type == VoltType.TINYINT) {
            type = VoltType.BIGINT;
          } else if (type == VoltType.NUMERIC) {
            type = VoltType.FLOAT;
          }

          m_paramTypes[param.getIndex()] = type.classFromType();
          m_paramTypeIsPrimitive[param.getIndex()] = m_paramTypes[param.getIndex()].isPrimitive();
          m_paramTypeIsArray[param.getIndex()] = param.getIsarray();
          assert (m_paramTypeIsArray[param.getIndex()] == false);
          m_paramTypeComponentType[param.getIndex()] = null;

          // rtb: what is broken (ambiguous?) that is being patched here?
          // hack to fixup varbinary support for statement procedures
          if (m_paramTypes[param.getIndex()] == byte[].class) {
            m_paramTypeComponentType[param.getIndex()] = byte.class;
            m_paramTypeIsArray[param.getIndex()] = true;
          }
        }
      } catch (Exception e) {
        // shouldn't throw anything outside of the compiler
        e.printStackTrace();
      }
    } else {
      // parse the java run method
      Method[] methods = m_procedure.getClass().getDeclaredMethods();

      for (final Method m : methods) {
        String name = m.getName();
        if (name.equals("run")) {
          if (Modifier.isPublic(m.getModifiers()) == false) continue;
          m_procMethod = m;
          m_paramTypes = m.getParameterTypes();
          int tempParamTypesLength = m_paramTypes.length;

          m_paramTypeIsPrimitive = new boolean[tempParamTypesLength];
          m_paramTypeIsArray = new boolean[tempParamTypesLength];
          m_paramTypeComponentType = new Class<?>[tempParamTypesLength];
          for (int ii = 0; ii < tempParamTypesLength; ii++) {
            m_paramTypeIsPrimitive[ii] = m_paramTypes[ii].isPrimitive();
            m_paramTypeIsArray[ii] = m_paramTypes[ii].isArray();
            m_paramTypeComponentType[ii] = m_paramTypes[ii].getComponentType();
          }
        }
      }

      if (m_procMethod == null) {
        throw new RuntimeException(
            "No \"run\" method found in: " + m_procedure.getClass().getName());
      }
    }

    // iterate through the fields and deal with sql statements
    Map<String, Field> stmtMap = null;
    try {
      stmtMap =
          ProcedureCompiler.getValidSQLStmts(null, m_procedureName, m_procedure.getClass(), true);
    } catch (Exception e1) {
      // shouldn't throw anything outside of the compiler
      e1.printStackTrace();
      return;
    }

    Field[] fields = new Field[stmtMap.size()];
    int index = 0;
    for (Field f : stmtMap.values()) {
      fields[index++] = f;
    }
    for (final Field f : fields) {
      String name = f.getName();
      Statement s = m_catProc.getStatements().get(name);
      if (s != null) {
        try {
          /*
           * Cache all the information we need about the statements in this stored
           * procedure locally instead of pulling them from the catalog on
           * a regular basis.
           */
          SQLStmt stmt = (SQLStmt) f.get(m_procedure);

          // done in a static method in an abstract class so users don't call it
          initSQLStmt(stmt, s);

        } catch (IllegalArgumentException e) {
          e.printStackTrace();
        } catch (IllegalAccessException e) {
          e.printStackTrace();
        }
        // LOG.fine("Found statement " + name);
      }
    }
  }
示例#3
0
  /** Add the system procedures to the catalog. */
  void addSystemProcsToCatalog(final Catalog catalog, final Database database)
      throws VoltCompilerException {
    assert (catalog != null);
    assert (database != null);

    // Table of sysproc metadata.
    final Object[][] procedures = {
      // SysProcedure Class                   readonly    everysite
      {LoadMultipartitionTable.class, false, true},
      {DatabaseDump.class, true, true},
      {RecomputeMarkovs.class, true, true},
      {Shutdown.class, false, true},
      {NoOp.class, true, false},
      {AdHoc.class, false, false},
      {GarbageCollection.class, true, true},
      {ExecutorStatus.class, true, false},
      {GetCatalog.class, true, false},
      {SnapshotSave.class, false, false},
      {SnapshotRestore.class, false, false},
      {SnapshotStatus.class, false, false},
      {SnapshotScan.class, false, false},
      {SnapshotDelete.class, false, false},

      //       {"org.voltdb.sysprocs.Quiesce",                      false,    false},
      //         {"org.voltdb.sysprocs.StartSampler",                 false,    false},
      //         {"org.voltdb.sysprocs.Statistics",                   true,     false},
      //         {"org.voltdb.sysprocs.SystemInformation",            true,     false},
      //         {"org.voltdb.sysprocs.UpdateApplicationCatalog",     false,    true},
      //         {"org.voltdb.sysprocs.UpdateLogging",                false,    true}

    };

    for (int ii = 0; ii < procedures.length; ++ii) {
      Class<?> procClass = (Class<?>) procedures[ii][0];
      boolean readonly = (Boolean) procedures[ii][1];
      boolean everysite = (Boolean) procedures[ii][2];

      // short name is "@ClassName" without package
      final String shortName = "@" + procClass.getSimpleName();

      // Make sure it's a VoltSystemProcedure
      if (ClassUtil.getSuperClasses(procClass).contains(VoltSystemProcedure.class) == false) {
        String msg =
            String.format(
                "Class %s does not extend %s",
                procClass.getCanonicalName(), VoltSystemProcedure.class.getSimpleName());
        throw new VoltCompilerException(msg);
      }

      // read annotations
      final ProcInfo info = procClass.getAnnotation(ProcInfo.class);
      if (info == null) {
        throw new VoltCompilerException("Sysproc " + shortName + " is missing annotation.");
      }

      // add an entry to the catalog
      final Procedure procedure = database.getProcedures().add(shortName);
      procedure.setId(this.getNextProcedureId());
      procedure.setClassname(procClass.getCanonicalName());
      procedure.setReadonly(readonly);
      procedure.setSystemproc(true);
      procedure.setHasjava(true);
      procedure.setSinglepartition(info.singlePartition());
      procedure.setEverysite(everysite);
      ProcedureCompiler.populateProcedureParameters(this, procClass, procedure);

      // Stored procedure sysproc classes are present in VoltDB.jar
      // and not duplicated in the catalog. This was decided
      // arbitrarily - no one had a strong opinion.
      //
      // VoltCompiler.addClassToJar(procClass, compiler);
    }
  }