void compileDatabaseNode(DatabaseType database) throws VoltCompilerException {
final ArrayList<String> programs = new ArrayList<String>();
final ArrayList<String> schemas = new ArrayList<String>();
final ArrayList<ProcedureDescriptor> procedures = new ArrayList<ProcedureDescriptor>();
final ArrayList<Class<?>> classDependencies = new ArrayList<Class<?>>();
final ArrayList<String[]> partitions = new ArrayList<String[]>();
final String databaseName = database.getName();
// schema does not verify that the database is named "database"
if (databaseName.equals("database") == false) {
final String msg =
"VoltDB currently requires all database elements to be named "
+ "\"database\" (found: \""
+ databaseName
+ "\")";
throw new VoltCompilerException(msg);
}
// create the database in the catalog
m_catalog.execute("add /clusters[cluster] databases " + databaseName);
Database db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);
SnapshotType snapshotSettings = database.getSnapshot();
if (snapshotSettings != null) {
SnapshotSchedule schedule = db.getSnapshotschedule().add("default");
String frequency = snapshotSettings.getFrequency();
if (!frequency.endsWith("s") && !frequency.endsWith("m") && !frequency.endsWith("h")) {
throw new VoltCompilerException(
"Snapshot frequency "
+ frequency
+ " needs to end with time unit specified"
+ " that is one of [s, m, h] (seconds, minutes, hours)");
}
int frequencyInt = 0;
String frequencySubstring = frequency.substring(0, frequency.length() - 1);
try {
frequencyInt = Integer.parseInt(frequencySubstring);
} catch (Exception e) {
throw new VoltCompilerException("Frequency " + frequencySubstring + " is not an integer ");
}
String prefix = snapshotSettings.getPrefix();
if (prefix == null || prefix.isEmpty()) {
throw new VoltCompilerException("Snapshot prefix " + prefix + " is not a valid prefix ");
}
if (prefix.contains("-") || prefix.contains(",")) {
throw new VoltCompilerException("Snapshot prefix " + prefix + " cannot include , or - ");
}
String path = snapshotSettings.getPath();
if (path == null || path.isEmpty()) {
throw new VoltCompilerException("Snapshot path " + path + " is not a valid path ");
}
if (snapshotSettings.getRetain() == null) {
throw new VoltCompilerException("Snapshot retain value not provided");
}
int retain = snapshotSettings.getRetain().intValue();
if (retain < 1) {
throw new VoltCompilerException(
"Snapshot retain value " + retain + " is not a valid value. Must be 1 or greater.");
}
schedule.setFrequencyunit(frequency.substring(frequency.length() - 1, frequency.length()));
schedule.setFrequencyvalue(frequencyInt);
schedule.setPath(path);
schedule.setPrefix(prefix);
schedule.setRetain(retain);
}
// schemas/schema
for (SchemasType.Schema schema : database.getSchemas().getSchema()) {
LOG.l7dlog(
Level.DEBUG,
LogKeys.compiler_VoltCompiler_CatalogPath.name(),
new Object[] {schema.getPath()},
null);
schemas.add(schema.getPath());
}
// groups/group.
if (database.getGroups() != null) {
for (GroupsType.Group group : database.getGroups().getGroup()) {
org.voltdb.catalog.Group catGroup = db.getGroups().add(group.getName());
catGroup.setAdhoc(group.isAdhoc());
catGroup.setSysproc(group.isSysproc());
}
}
// users/user
if (database.getUsers() != null) {
for (UsersType.User user : database.getUsers().getUser()) {
org.voltdb.catalog.User catUser = db.getUsers().add(user.getName());
catUser.setAdhoc(user.isAdhoc());
catUser.setSysproc(user.isSysproc());
byte passwordHash[] = extractPassword(user.getPassword());
catUser.setShadowpassword(Encoder.hexEncode(passwordHash));
// process the @groups comma separated list
if (user.getGroups() != null) {
String grouplist[] = user.getGroups().split(",");
for (final String group : grouplist) {
final GroupRef groupRef = catUser.getGroups().add(group);
final Group catalogGroup = db.getGroups().get(group);
if (catalogGroup != null) {
groupRef.setGroup(catalogGroup);
}
}
}
}
}
// procedures/procedure
for (ProceduresType.Procedure proc : database.getProcedures().getProcedure()) {
procedures.add(getProcedure(proc));
}
// classdependencies/classdependency
if (database.getClassdependencies() != null) {
for (Classdependency dep : database.getClassdependencies().getClassdependency()) {
classDependencies.add(getClassDependency(dep));
}
}
// partitions/table
if (database.getPartitions() != null) {
for (org.voltdb.compiler.projectfile.PartitionsType.Partition table :
database.getPartitions().getPartition()) {
partitions.add(getPartition(table));
}
}
String msg = "Database \"" + databaseName + "\" ";
// TODO: schema allows 0 procedures. Testbase relies on this.
if (procedures.size() == 0) {
msg +=
"needs at least one \"procedure\" element "
+ "(currently has "
+ String.valueOf(procedures.size())
+ ")";
throw new VoltCompilerException(msg);
}
if (procedures.size() < 1) {
msg += "is missing the \"procedures\" element";
throw new VoltCompilerException(msg);
}
// shutdown and make a new hsqldb
m_hsql = HSQLInterface.loadHsqldb();
// Actually parse and handle all the programs
for (final String programName : programs) {
m_catalog.execute("add " + db.getPath() + " programs " + programName);
}
// Actually parse and handle all the DDL
final DDLCompiler ddlcompiler = new DDLCompiler(this, m_hsql);
for (final String schemaPath : schemas) {
File schemaFile = null;
if (schemaPath.contains(".jar!")) {
String ddlText = null;
try {
ddlText = JarReader.readFileFromJarfile(schemaPath);
} catch (final Exception e) {
throw new VoltCompilerException(e);
}
schemaFile = VoltProjectBuilder.writeStringToTempFile(ddlText);
} else {
schemaFile = new File(schemaPath);
}
if (!schemaFile.isAbsolute()) {
// Resolve schemaPath relative to the database definition xml file
schemaFile = new File(new File(m_projectFileURL).getParent(), schemaPath);
}
// add the file object's path to the list of files for the jar
m_ddlFilePaths.put(schemaFile.getName(), schemaFile.getPath());
ddlcompiler.loadSchema(schemaFile.getAbsolutePath());
}
ddlcompiler.compileToCatalog(m_catalog, db);
// Actually parse and handle all the partitions
// this needs to happen before procedures are compiled
msg = "In database \"" + databaseName + "\", ";
final CatalogMap<Table> tables = db.getTables();
for (final String[] partition : partitions) {
final String tableName = partition[0];
final String colName = partition[1];
final Table t = tables.getIgnoreCase(tableName);
if (t == null) {
msg += "\"partition\" element has unknown \"table\" attribute '" + tableName + "'";
throw new VoltCompilerException(msg);
}
final Column c = t.getColumns().getIgnoreCase(colName);
// make sure the column exists
if (c == null) {
msg += "\"partition\" element has unknown \"column\" attribute '" + colName + "'";
throw new VoltCompilerException(msg);
}
// make sure the column is marked not-nullable
if (c.getNullable() == true) {
msg +=
"Partition column '"
+ tableName
+ "."
+ colName
+ "' is nullable. "
+ "Partition columns must be constrained \"NOT NULL\".";
throw new VoltCompilerException(msg);
}
t.setPartitioncolumn(c);
t.setIsreplicated(false);
// Set the destination tables of associated views non-replicated.
// If a view's source table is replicated, then a full scan of the
// associated view is singled-sited. If the source is partitioned,
// a full scan of the view must be distributed.
final CatalogMap<MaterializedViewInfo> views = t.getViews();
for (final MaterializedViewInfo mvi : views) {
mvi.getDest().setIsreplicated(false);
}
}
// add vertical partitions
if (database.getVerticalpartitions() != null) {
for (Verticalpartition vp : database.getVerticalpartitions().getVerticalpartition()) {
try {
addVerticalPartition(db, vp.getTable(), vp.getColumn(), vp.isIndexed());
} catch (Exception ex) {
throw new VoltCompilerException(
"Failed to create vertical partition for " + vp.getTable(), ex);
}
}
}
// this should reorder the tables and partitions all alphabetically
String catData = m_catalog.serialize();
m_catalog = new Catalog();
m_catalog.execute(catData);
db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);
// add database estimates info
addDatabaseEstimatesInfo(m_estimates, db);
addSystemProcsToCatalog(m_catalog, db);
// Process and add exports and connectors to the catalog
// Must do this before compiling procedures to deny updates
// on append-only tables.
if (database.getExports() != null) {
// currently, only a single connector is allowed
Connector conn = database.getExports().getConnector();
compileConnector(conn, db);
}
// Actually parse and handle all the Procedures
for (final ProcedureDescriptor procedureDescriptor : procedures) {
final String procedureName = procedureDescriptor.m_className;
m_currentFilename = procedureName.substring(procedureName.lastIndexOf('.') + 1);
m_currentFilename += ".class";
ProcedureCompiler.compile(this, m_hsql, m_estimates, m_catalog, db, procedureDescriptor);
}
// Add all the class dependencies to the output jar
for (final Class<?> classDependency : classDependencies) {
addClassToJar(classDependency, this);
}
m_hsql.close();
}
protected void reflect() {
// fill in the sql for single statement procs
if (m_catProc.getHasjava() == false) {
try {
Map<String, Field> stmtMap =
ProcedureCompiler.getValidSQLStmts(null, m_procedureName, m_procedure.getClass(), true);
Field f = stmtMap.get(VoltDB.ANON_STMT_NAME);
assert (f != null);
SQLStmt stmt = (SQLStmt) f.get(m_procedure);
Statement statement = m_catProc.getStatements().get(VoltDB.ANON_STMT_NAME);
stmt.sqlText = statement.getSqltext().getBytes(VoltDB.UTF8ENCODING);
m_cachedSingleStmt.stmt = stmt;
int numParams = m_catProc.getParameters().size();
m_paramTypes = new Class<?>[numParams];
m_paramTypeIsPrimitive = new boolean[numParams];
m_paramTypeIsArray = new boolean[numParams];
m_paramTypeComponentType = new Class<?>[numParams];
for (ProcParameter param : m_catProc.getParameters()) {
VoltType type = VoltType.get((byte) param.getType());
if (type == VoltType.INTEGER) {
type = VoltType.BIGINT;
} else if (type == VoltType.SMALLINT) {
type = VoltType.BIGINT;
} else if (type == VoltType.TINYINT) {
type = VoltType.BIGINT;
} else if (type == VoltType.NUMERIC) {
type = VoltType.FLOAT;
}
m_paramTypes[param.getIndex()] = type.classFromType();
m_paramTypeIsPrimitive[param.getIndex()] = m_paramTypes[param.getIndex()].isPrimitive();
m_paramTypeIsArray[param.getIndex()] = param.getIsarray();
assert (m_paramTypeIsArray[param.getIndex()] == false);
m_paramTypeComponentType[param.getIndex()] = null;
// rtb: what is broken (ambiguous?) that is being patched here?
// hack to fixup varbinary support for statement procedures
if (m_paramTypes[param.getIndex()] == byte[].class) {
m_paramTypeComponentType[param.getIndex()] = byte.class;
m_paramTypeIsArray[param.getIndex()] = true;
}
}
} catch (Exception e) {
// shouldn't throw anything outside of the compiler
e.printStackTrace();
}
} else {
// parse the java run method
Method[] methods = m_procedure.getClass().getDeclaredMethods();
for (final Method m : methods) {
String name = m.getName();
if (name.equals("run")) {
if (Modifier.isPublic(m.getModifiers()) == false) continue;
m_procMethod = m;
m_paramTypes = m.getParameterTypes();
int tempParamTypesLength = m_paramTypes.length;
m_paramTypeIsPrimitive = new boolean[tempParamTypesLength];
m_paramTypeIsArray = new boolean[tempParamTypesLength];
m_paramTypeComponentType = new Class<?>[tempParamTypesLength];
for (int ii = 0; ii < tempParamTypesLength; ii++) {
m_paramTypeIsPrimitive[ii] = m_paramTypes[ii].isPrimitive();
m_paramTypeIsArray[ii] = m_paramTypes[ii].isArray();
m_paramTypeComponentType[ii] = m_paramTypes[ii].getComponentType();
}
}
}
if (m_procMethod == null) {
throw new RuntimeException(
"No \"run\" method found in: " + m_procedure.getClass().getName());
}
}
// iterate through the fields and deal with sql statements
Map<String, Field> stmtMap = null;
try {
stmtMap =
ProcedureCompiler.getValidSQLStmts(null, m_procedureName, m_procedure.getClass(), true);
} catch (Exception e1) {
// shouldn't throw anything outside of the compiler
e1.printStackTrace();
return;
}
Field[] fields = new Field[stmtMap.size()];
int index = 0;
for (Field f : stmtMap.values()) {
fields[index++] = f;
}
for (final Field f : fields) {
String name = f.getName();
Statement s = m_catProc.getStatements().get(name);
if (s != null) {
try {
/*
* Cache all the information we need about the statements in this stored
* procedure locally instead of pulling them from the catalog on
* a regular basis.
*/
SQLStmt stmt = (SQLStmt) f.get(m_procedure);
// done in a static method in an abstract class so users don't call it
initSQLStmt(stmt, s);
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
}
// LOG.fine("Found statement " + name);
}
}
}
/** Add the system procedures to the catalog. */
void addSystemProcsToCatalog(final Catalog catalog, final Database database)
throws VoltCompilerException {
assert (catalog != null);
assert (database != null);
// Table of sysproc metadata.
final Object[][] procedures = {
// SysProcedure Class readonly everysite
{LoadMultipartitionTable.class, false, true},
{DatabaseDump.class, true, true},
{RecomputeMarkovs.class, true, true},
{Shutdown.class, false, true},
{NoOp.class, true, false},
{AdHoc.class, false, false},
{GarbageCollection.class, true, true},
{ExecutorStatus.class, true, false},
{GetCatalog.class, true, false},
{SnapshotSave.class, false, false},
{SnapshotRestore.class, false, false},
{SnapshotStatus.class, false, false},
{SnapshotScan.class, false, false},
{SnapshotDelete.class, false, false},
// {"org.voltdb.sysprocs.Quiesce", false, false},
// {"org.voltdb.sysprocs.StartSampler", false, false},
// {"org.voltdb.sysprocs.Statistics", true, false},
// {"org.voltdb.sysprocs.SystemInformation", true, false},
// {"org.voltdb.sysprocs.UpdateApplicationCatalog", false, true},
// {"org.voltdb.sysprocs.UpdateLogging", false, true}
};
for (int ii = 0; ii < procedures.length; ++ii) {
Class<?> procClass = (Class<?>) procedures[ii][0];
boolean readonly = (Boolean) procedures[ii][1];
boolean everysite = (Boolean) procedures[ii][2];
// short name is "@ClassName" without package
final String shortName = "@" + procClass.getSimpleName();
// Make sure it's a VoltSystemProcedure
if (ClassUtil.getSuperClasses(procClass).contains(VoltSystemProcedure.class) == false) {
String msg =
String.format(
"Class %s does not extend %s",
procClass.getCanonicalName(), VoltSystemProcedure.class.getSimpleName());
throw new VoltCompilerException(msg);
}
// read annotations
final ProcInfo info = procClass.getAnnotation(ProcInfo.class);
if (info == null) {
throw new VoltCompilerException("Sysproc " + shortName + " is missing annotation.");
}
// add an entry to the catalog
final Procedure procedure = database.getProcedures().add(shortName);
procedure.setId(this.getNextProcedureId());
procedure.setClassname(procClass.getCanonicalName());
procedure.setReadonly(readonly);
procedure.setSystemproc(true);
procedure.setHasjava(true);
procedure.setSinglepartition(info.singlePartition());
procedure.setEverysite(everysite);
ProcedureCompiler.populateProcedureParameters(this, procClass, procedure);
// Stored procedure sysproc classes are present in VoltDB.jar
// and not duplicated in the catalog. This was decided
// arbitrarily - no one had a strong opinion.
//
// VoltCompiler.addClassToJar(procClass, compiler);
}
}