public static MaterializedViewInfo addVerticalPartition(
final Table catalog_tbl, final Collection<Column> catalog_cols, final boolean createIndex)
throws Exception {
assert (catalog_cols.isEmpty() == false);
Database catalog_db = ((Database) catalog_tbl.getParent());
String viewName = getNextVerticalPartitionName(catalog_tbl, catalog_cols);
if (debug.get())
LOG.debug(
String.format(
"Adding Vertical Partition %s for %s: %s", viewName, catalog_tbl, catalog_cols));
// Create a new virtual table
Table virtual_tbl = catalog_db.getTables().get(viewName);
if (virtual_tbl == null) {
virtual_tbl = catalog_db.getTables().add(viewName);
}
virtual_tbl.setIsreplicated(true);
virtual_tbl.setMaterializer(catalog_tbl);
virtual_tbl.setSystable(true);
virtual_tbl.getColumns().clear();
// Create MaterializedView and link it to the virtual table
MaterializedViewInfo catalog_view = catalog_tbl.getViews().add(viewName);
catalog_view.setVerticalpartition(true);
catalog_view.setDest(virtual_tbl);
List<Column> indexColumns = new ArrayList<Column>();
Column partition_col = catalog_tbl.getPartitioncolumn();
if (partition_col instanceof VerticalPartitionColumn) {
partition_col = ((VerticalPartitionColumn) partition_col).getHorizontalColumn();
}
if (debug.get()) LOG.debug(catalog_tbl.getName() + " Partition Column: " + partition_col);
int i = 0;
assert (catalog_cols != null);
assert (catalog_cols.isEmpty() == false)
: "No vertical partitioning columns for " + catalog_view.fullName();
for (Column catalog_col : catalog_cols) {
// MaterializedView ColumnRef
ColumnRef catalog_ref = catalog_view.getGroupbycols().add(catalog_col.getName());
catalog_ref.setColumn(catalog_col);
catalog_ref.setIndex(i++);
// VirtualTable Column
Column virtual_col = virtual_tbl.getColumns().add(catalog_col.getName());
virtual_col.setDefaulttype(catalog_col.getDefaulttype());
virtual_col.setDefaultvalue(catalog_col.getDefaultvalue());
virtual_col.setIndex(catalog_col.getIndex());
virtual_col.setNullable(catalog_col.getNullable());
virtual_col.setSize(catalog_col.getSize());
virtual_col.setType(catalog_col.getType());
if (debug.get())
LOG.debug(String.format("Added VerticalPartition column %s", virtual_col.fullName()));
// If they want an index, then we'll make one based on every column except for the column
// that the table is partitioned on
if (createIndex) {
boolean include = true;
if (partition_col instanceof MultiColumn) {
include = (((MultiColumn) partition_col).contains(catalog_col) == false);
} else if (catalog_col.equals(partition_col)) {
include = false;
}
if (include) indexColumns.add(virtual_col);
}
} // FOR
if (createIndex) {
if (indexColumns.isEmpty()) {
Map<String, Object> m = new ListOrderedMap<String, Object>();
m.put("Partition Column", partition_col);
m.put("VP Table Columns", virtual_tbl.getColumns());
m.put("Passed-in Columns", CatalogUtil.debug(catalog_cols));
LOG.error("Failed to find index columns\n" + StringUtil.formatMaps(m));
throw new Exception(String.format("No columns selected for index on %s", viewName));
}
String idxName = "SYS_IDX_" + viewName;
Index virtual_idx = virtual_tbl.getIndexes().get(idxName);
if (virtual_idx == null) {
virtual_idx = virtual_tbl.getIndexes().add(idxName);
}
virtual_idx.getColumns().clear();
IndexType idxType =
(indexColumns.size() == 1 ? IndexType.HASH_TABLE : IndexType.BALANCED_TREE);
virtual_idx.setType(idxType.getValue());
i = 0;
for (Column catalog_col : indexColumns) {
ColumnRef cref = virtual_idx.getColumns().add(catalog_col.getTypeName());
cref.setColumn(catalog_col);
cref.setIndex(i++);
} // FOR
if (debug.get())
LOG.debug(
String.format(
"Created %s index '%s' for vertical partition '%s'", idxType, idxName, viewName));
}
return (catalog_view);
}
void compileDatabaseNode(DatabaseType database) throws VoltCompilerException {
final ArrayList<String> programs = new ArrayList<String>();
final ArrayList<String> schemas = new ArrayList<String>();
final ArrayList<ProcedureDescriptor> procedures = new ArrayList<ProcedureDescriptor>();
final ArrayList<Class<?>> classDependencies = new ArrayList<Class<?>>();
final ArrayList<String[]> partitions = new ArrayList<String[]>();
final String databaseName = database.getName();
// schema does not verify that the database is named "database"
if (databaseName.equals("database") == false) {
final String msg =
"VoltDB currently requires all database elements to be named "
+ "\"database\" (found: \""
+ databaseName
+ "\")";
throw new VoltCompilerException(msg);
}
// create the database in the catalog
m_catalog.execute("add /clusters[cluster] databases " + databaseName);
Database db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);
SnapshotType snapshotSettings = database.getSnapshot();
if (snapshotSettings != null) {
SnapshotSchedule schedule = db.getSnapshotschedule().add("default");
String frequency = snapshotSettings.getFrequency();
if (!frequency.endsWith("s") && !frequency.endsWith("m") && !frequency.endsWith("h")) {
throw new VoltCompilerException(
"Snapshot frequency "
+ frequency
+ " needs to end with time unit specified"
+ " that is one of [s, m, h] (seconds, minutes, hours)");
}
int frequencyInt = 0;
String frequencySubstring = frequency.substring(0, frequency.length() - 1);
try {
frequencyInt = Integer.parseInt(frequencySubstring);
} catch (Exception e) {
throw new VoltCompilerException("Frequency " + frequencySubstring + " is not an integer ");
}
String prefix = snapshotSettings.getPrefix();
if (prefix == null || prefix.isEmpty()) {
throw new VoltCompilerException("Snapshot prefix " + prefix + " is not a valid prefix ");
}
if (prefix.contains("-") || prefix.contains(",")) {
throw new VoltCompilerException("Snapshot prefix " + prefix + " cannot include , or - ");
}
String path = snapshotSettings.getPath();
if (path == null || path.isEmpty()) {
throw new VoltCompilerException("Snapshot path " + path + " is not a valid path ");
}
if (snapshotSettings.getRetain() == null) {
throw new VoltCompilerException("Snapshot retain value not provided");
}
int retain = snapshotSettings.getRetain().intValue();
if (retain < 1) {
throw new VoltCompilerException(
"Snapshot retain value " + retain + " is not a valid value. Must be 1 or greater.");
}
schedule.setFrequencyunit(frequency.substring(frequency.length() - 1, frequency.length()));
schedule.setFrequencyvalue(frequencyInt);
schedule.setPath(path);
schedule.setPrefix(prefix);
schedule.setRetain(retain);
}
// schemas/schema
for (SchemasType.Schema schema : database.getSchemas().getSchema()) {
LOG.l7dlog(
Level.DEBUG,
LogKeys.compiler_VoltCompiler_CatalogPath.name(),
new Object[] {schema.getPath()},
null);
schemas.add(schema.getPath());
}
// groups/group.
if (database.getGroups() != null) {
for (GroupsType.Group group : database.getGroups().getGroup()) {
org.voltdb.catalog.Group catGroup = db.getGroups().add(group.getName());
catGroup.setAdhoc(group.isAdhoc());
catGroup.setSysproc(group.isSysproc());
}
}
// users/user
if (database.getUsers() != null) {
for (UsersType.User user : database.getUsers().getUser()) {
org.voltdb.catalog.User catUser = db.getUsers().add(user.getName());
catUser.setAdhoc(user.isAdhoc());
catUser.setSysproc(user.isSysproc());
byte passwordHash[] = extractPassword(user.getPassword());
catUser.setShadowpassword(Encoder.hexEncode(passwordHash));
// process the @groups comma separated list
if (user.getGroups() != null) {
String grouplist[] = user.getGroups().split(",");
for (final String group : grouplist) {
final GroupRef groupRef = catUser.getGroups().add(group);
final Group catalogGroup = db.getGroups().get(group);
if (catalogGroup != null) {
groupRef.setGroup(catalogGroup);
}
}
}
}
}
// procedures/procedure
for (ProceduresType.Procedure proc : database.getProcedures().getProcedure()) {
procedures.add(getProcedure(proc));
}
// classdependencies/classdependency
if (database.getClassdependencies() != null) {
for (Classdependency dep : database.getClassdependencies().getClassdependency()) {
classDependencies.add(getClassDependency(dep));
}
}
// partitions/table
if (database.getPartitions() != null) {
for (org.voltdb.compiler.projectfile.PartitionsType.Partition table :
database.getPartitions().getPartition()) {
partitions.add(getPartition(table));
}
}
String msg = "Database \"" + databaseName + "\" ";
// TODO: schema allows 0 procedures. Testbase relies on this.
if (procedures.size() == 0) {
msg +=
"needs at least one \"procedure\" element "
+ "(currently has "
+ String.valueOf(procedures.size())
+ ")";
throw new VoltCompilerException(msg);
}
if (procedures.size() < 1) {
msg += "is missing the \"procedures\" element";
throw new VoltCompilerException(msg);
}
// shutdown and make a new hsqldb
m_hsql = HSQLInterface.loadHsqldb();
// Actually parse and handle all the programs
for (final String programName : programs) {
m_catalog.execute("add " + db.getPath() + " programs " + programName);
}
// Actually parse and handle all the DDL
final DDLCompiler ddlcompiler = new DDLCompiler(this, m_hsql);
for (final String schemaPath : schemas) {
File schemaFile = null;
if (schemaPath.contains(".jar!")) {
String ddlText = null;
try {
ddlText = JarReader.readFileFromJarfile(schemaPath);
} catch (final Exception e) {
throw new VoltCompilerException(e);
}
schemaFile = VoltProjectBuilder.writeStringToTempFile(ddlText);
} else {
schemaFile = new File(schemaPath);
}
if (!schemaFile.isAbsolute()) {
// Resolve schemaPath relative to the database definition xml file
schemaFile = new File(new File(m_projectFileURL).getParent(), schemaPath);
}
// add the file object's path to the list of files for the jar
m_ddlFilePaths.put(schemaFile.getName(), schemaFile.getPath());
ddlcompiler.loadSchema(schemaFile.getAbsolutePath());
}
ddlcompiler.compileToCatalog(m_catalog, db);
// Actually parse and handle all the partitions
// this needs to happen before procedures are compiled
msg = "In database \"" + databaseName + "\", ";
final CatalogMap<Table> tables = db.getTables();
for (final String[] partition : partitions) {
final String tableName = partition[0];
final String colName = partition[1];
final Table t = tables.getIgnoreCase(tableName);
if (t == null) {
msg += "\"partition\" element has unknown \"table\" attribute '" + tableName + "'";
throw new VoltCompilerException(msg);
}
final Column c = t.getColumns().getIgnoreCase(colName);
// make sure the column exists
if (c == null) {
msg += "\"partition\" element has unknown \"column\" attribute '" + colName + "'";
throw new VoltCompilerException(msg);
}
// make sure the column is marked not-nullable
if (c.getNullable() == true) {
msg +=
"Partition column '"
+ tableName
+ "."
+ colName
+ "' is nullable. "
+ "Partition columns must be constrained \"NOT NULL\".";
throw new VoltCompilerException(msg);
}
t.setPartitioncolumn(c);
t.setIsreplicated(false);
// Set the destination tables of associated views non-replicated.
// If a view's source table is replicated, then a full scan of the
// associated view is singled-sited. If the source is partitioned,
// a full scan of the view must be distributed.
final CatalogMap<MaterializedViewInfo> views = t.getViews();
for (final MaterializedViewInfo mvi : views) {
mvi.getDest().setIsreplicated(false);
}
}
// add vertical partitions
if (database.getVerticalpartitions() != null) {
for (Verticalpartition vp : database.getVerticalpartitions().getVerticalpartition()) {
try {
addVerticalPartition(db, vp.getTable(), vp.getColumn(), vp.isIndexed());
} catch (Exception ex) {
throw new VoltCompilerException(
"Failed to create vertical partition for " + vp.getTable(), ex);
}
}
}
// this should reorder the tables and partitions all alphabetically
String catData = m_catalog.serialize();
m_catalog = new Catalog();
m_catalog.execute(catData);
db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);
// add database estimates info
addDatabaseEstimatesInfo(m_estimates, db);
addSystemProcsToCatalog(m_catalog, db);
// Process and add exports and connectors to the catalog
// Must do this before compiling procedures to deny updates
// on append-only tables.
if (database.getExports() != null) {
// currently, only a single connector is allowed
Connector conn = database.getExports().getConnector();
compileConnector(conn, db);
}
// Actually parse and handle all the Procedures
for (final ProcedureDescriptor procedureDescriptor : procedures) {
final String procedureName = procedureDescriptor.m_className;
m_currentFilename = procedureName.substring(procedureName.lastIndexOf('.') + 1);
m_currentFilename += ".class";
ProcedureCompiler.compile(this, m_hsql, m_estimates, m_catalog, db, procedureDescriptor);
}
// Add all the class dependencies to the output jar
for (final Class<?> classDependency : classDependencies) {
addClassToJar(classDependency, this);
}
m_hsql.close();
}
