public static MaterializedViewInfo addVerticalPartition(
final Database catalog_db,
final String tableName,
final List<String> columnNames,
final boolean createIndex)
throws Exception {
Table catalog_tbl = catalog_db.getTables().get(tableName);
if (catalog_tbl == null) {
throw new Exception("Invalid vertical partition table '" + tableName + "'");
} else if (catalog_tbl.getIsreplicated()) {
throw new Exception(
"Cannot create vertical partition for replicated table '" + tableName + "'");
}
ArrayList<Column> catalog_cols = new ArrayList<Column>();
for (String columnName : columnNames) {
Column catalog_col = catalog_tbl.getColumns().get(columnName);
if (catalog_col == null) {
throw new Exception(
"Invalid vertical partition column '" + columnName + "' for table '" + tableName + "'");
} else if (catalog_cols.contains(catalog_col)) {
throw new Exception(
"Duplicate vertical partition column '"
+ columnName
+ "' for table '"
+ tableName
+ "'");
}
catalog_cols.add(catalog_col);
} // FOR
return (addVerticalPartition(catalog_tbl, catalog_cols, createIndex));
}
@Override
public boolean getIsReplicated() {
return m_table.getIsreplicated();
}
protected boolean createSetupInternal(
String file_path,
String file_nonce,
long txnId,
Map<Integer, Long> partitionTransactionIds,
JSONObject jsData,
SystemProcedureExecutionContext context,
String hostname,
final VoltTable result,
Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
SiteTracker tracker,
long timestamp)
throws IOException {
assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());
/*
* List of partitions to include if this snapshot is
* going to be deduped. Attempts to break up the work
* by seeding and RNG selecting
* a random replica to do the work. Will not work in failure
* cases, but we don't use dedupe when we want durability.
*/
List<Long> sitesToInclude = CSVSnapshotWritePlan.computeDedupedLocalSites(txnId, tracker);
// If there's no work to do on this host, just claim success and get out:
if (sitesToInclude.isEmpty() && !tracker.isFirstHost()) {
return false;
}
NativeSnapshotWritePlan.createFileBasedCompletionTasks(
file_path,
file_nonce,
txnId,
partitionTransactionIds,
context,
exportSequenceNumbers,
timestamp);
final List<Table> tables = SnapshotUtil.getTablesToSave(context.getDatabase());
final AtomicInteger numTables = new AtomicInteger(tables.size());
final SnapshotRegistry.Snapshot snapshotRecord =
SnapshotRegistry.startSnapshot(
txnId,
context.getHostId(),
file_path,
file_nonce,
SnapshotFormat.CSV,
tables.toArray(new Table[0]));
SnapshotDataTarget sdt = null;
boolean noTargetsCreated = true;
final ArrayList<SnapshotTableTask> partitionedSnapshotTasks =
new ArrayList<SnapshotTableTask>();
final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>();
for (final Table table : tables) {
/*
* For a deduped csv snapshot, only produce the replicated tables on the "leader"
* host.
*/
if (table.getIsreplicated() && !tracker.isFirstHost()) {
snapshotRecord.removeTable(table.getTypeName());
continue;
}
File saveFilePath = null;
saveFilePath =
SnapshotUtil.constructFileForTable(
table, file_path, file_nonce, SnapshotFormat.CSV, context.getHostId());
try {
sdt = new SimpleFileSnapshotDataTarget(saveFilePath, !table.getIsreplicated());
m_targets.add(sdt);
final Runnable onClose =
new TargetStatsClosure(sdt, table.getTypeName(), numTables, snapshotRecord);
sdt.setOnCloseHandler(onClose);
List<SnapshotDataFilter> filters = new ArrayList<SnapshotDataFilter>();
filters.add(new CSVSnapshotFilter(CatalogUtil.getVoltTable(table), ',', null));
final SnapshotTableTask task =
new SnapshotTableTask(
table.getRelativeIndex(),
sdt,
filters.toArray(new SnapshotDataFilter[filters.size()]),
table.getIsreplicated(),
table.getTypeName());
if (table.getIsreplicated()) {
replicatedSnapshotTasks.add(task);
} else {
partitionedSnapshotTasks.add(task);
}
noTargetsCreated = false;
result.addRow(context.getHostId(), hostname, table.getTypeName(), "SUCCESS", "");
} catch (IOException ex) {
handleTargetCreationError(
sdt, context, file_nonce, hostname, table.getTypeName(), ex, result);
}
}
if (noTargetsCreated) {
SnapshotRegistry.discardSnapshot(snapshotRecord);
}
// CSV snapshots do the partitioned work only on the specified sites for de-duping,
// but since we've pre-filtered the replicated task list to only contain entries on
// one node, we can go ahead and distribute them across all of the sites on that node.
placePartitionedTasks(partitionedSnapshotTasks, sitesToInclude);
placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
return noTargetsCreated;
}
public CatalogContext(Catalog catalog, File pathToCatalogJar) {
// check the heck out of the given params in this immutable class
assert (catalog != null);
if (catalog == null) {
throw new RuntimeException("Can't create CatalogContext with null catalog.");
}
this.jarPath = pathToCatalogJar;
this.catalog = catalog;
this.cluster = CatalogUtil.getCluster(this.catalog);
this.database = CatalogUtil.getDatabase(this.catalog);
this.hosts = this.cluster.getHosts();
this.sites = this.cluster.getSites();
if (this.jarPath != null) {
this.catalogClassLoader = new JarClassLoader(this.jarPath.getAbsolutePath());
this.paramMappings =
ParametersUtil.getParameterMappingsSetFromJar(this.database, this.jarPath);
} else {
this.catalogClassLoader = null;
this.paramMappings = null;
}
// ------------------------------------------------------------
// PROCEDURES
// ------------------------------------------------------------
this.procedures = database.getProcedures();
this.proceduresArray = new Procedure[this.procedures.size() + 1];
for (Procedure proc : this.procedures) {
this.proceduresArray[proc.getId()] = proc;
if (proc.getSystemproc()) {
this.sysProcedures.add(proc);
} else if (proc.getMapreduce()) {
this.mrProcedures.add(proc);
} else {
this.regularProcedures.add(proc);
}
} // FOR
authSystem = new AuthSystem(database, cluster.getSecurityenabled());
siteTracker = null; // new SiteTracker(cluster.getSites());
// count nodes
this.numberOfHosts = cluster.getHosts().size();
// count exec sites
this.numberOfSites = cluster.getSites().size();
// ------------------------------------------------------------
// PARTITIONS
// ------------------------------------------------------------
this.numberOfPartitions = cluster.getNum_partitions();
this.partitions = new Partition[this.numberOfPartitions];
this.partitionIdArray = new Integer[this.numberOfPartitions];
this.partitionSingletons = new PartitionSet[this.numberOfPartitions];
this.partitionSiteXref = new int[this.numberOfPartitions];
for (Partition part : CatalogUtil.getAllPartitions(catalog)) {
int p = part.getId();
this.partitions[p] = part;
this.partitionIdArray[p] = Integer.valueOf(p);
this.partitionSingletons[p] = new PartitionSet(p);
this.partitionIdCollection.add(this.partitionIdArray[p]);
this.partitionSiteXref[part.getId()] = ((Site) part.getParent()).getId();
} // FOR
// ------------------------------------------------------------
// TABLES
// ------------------------------------------------------------
for (Table tbl : database.getTables()) {
if (tbl.getSystable()) {
sysTables.add(tbl);
} else if (tbl.getMapreduce()) {
mapReduceTables.add(tbl);
} else if (tbl.getMaterializer() != null) {
viewTables.add(tbl);
} else {
dataTables.add(tbl);
if (tbl.getIsreplicated()) {
replicatedTables.add(tbl);
}
if (tbl.getEvictable()) {
evictableTables.add(tbl);
}
}
} // FOR
// PLANFRAGMENTS
this.initPlanFragments();
}
/**
* Get some embeddable HTML of some generic catalog/application stats that is drawn on the first
* page of the report.
*/
static String getStatsHTML(Database db, ArrayList<Feedback> warnings) {
StringBuilder sb = new StringBuilder();
sb.append("<table class='table table-condensed'>\n");
// count things
int indexes = 0, views = 0, statements = 0;
int partitionedTables = 0, replicatedTables = 0;
int partitionedProcs = 0, replicatedProcs = 0;
int readProcs = 0, writeProcs = 0;
for (Table t : db.getTables()) {
if (t.getMaterializer() != null) {
views++;
} else {
if (t.getIsreplicated()) {
replicatedTables++;
} else {
partitionedTables++;
}
}
indexes += t.getIndexes().size();
}
for (Procedure p : db.getProcedures()) {
// skip auto-generated crud procs
if (p.getDefaultproc()) {
continue;
}
if (p.getSinglepartition()) {
partitionedProcs++;
} else {
replicatedProcs++;
}
if (p.getReadonly()) {
readProcs++;
} else {
writeProcs++;
}
statements += p.getStatements().size();
}
// version
sb.append("<tr><td>Compiled by VoltDB Version</td><td>");
sb.append(VoltDB.instance().getVersionString()).append("</td></tr>\n");
// timestamp
sb.append("<tr><td>Compiled on</td><td>");
SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z");
sb.append(sdf.format(m_timestamp)).append("</td></tr>\n");
// tables
sb.append("<tr><td>Table Count</td><td>");
sb.append(
String.format(
"%d (%d partitioned / %d replicated)",
partitionedTables + replicatedTables, partitionedTables, replicatedTables));
sb.append("</td></tr>\n");
// views
sb.append("<tr><td>Materialized View Count</td><td>").append(views).append("</td></tr>\n");
// indexes
sb.append("<tr><td>Index Count</td><td>").append(indexes).append("</td></tr>\n");
// procedures
sb.append("<tr><td>Procedure Count</td><td>");
sb.append(
String.format(
"%d (%d partitioned / %d replicated) (%d read-only / %d read-write)",
partitionedProcs + replicatedProcs,
partitionedProcs,
replicatedProcs,
readProcs,
writeProcs));
sb.append("</td></tr>\n");
// statements
sb.append("<tr><td>SQL Statement Count</td><td>").append(statements).append("</td></tr>\n");
sb.append("</table>\n\n");
// warnings, add warning section if any
if (warnings.size() > 0) {
sb.append("<h4>Warnings</h4>");
sb.append("<table class='table table-condensed'>\n");
for (Feedback warning : warnings) {
String procName = warning.getFileName().replace(".class", "");
String nameLink = "";
// not a warning during compiling procedures, must from the schema
if (procName.compareToIgnoreCase("null") == 0) {
String schemaName = "";
String warningMsg = warning.getMessage().toLowerCase();
if (warningMsg.contains("table ")) {
int begin = warningMsg.indexOf("table ") + 6;
int end = (warningMsg.substring(begin)).indexOf(" ");
schemaName = warningMsg.substring(begin, begin + end);
}
nameLink = "<a href='#s-" + schemaName + "'>" + schemaName.toUpperCase() + "</a>";
} else {
nameLink = "<a href='#p-" + procName.toLowerCase() + "'>" + procName + "</a>";
}
sb.append("<tr><td>")
.append(nameLink)
.append("</td><td>")
.append(warning.getMessage())
.append("</td></tr>\n");
}
sb.append("").append("</table>\n").append("</td></tr>\n");
}
return sb.toString();
}
static String generateSchemaRow(Table table, boolean isExportTable) {
StringBuilder sb = new StringBuilder();
sb.append("<tr class='primaryrow'>");
// column 1: table name
String anchor = table.getTypeName().toLowerCase();
sb.append(
"<td style='white-space: nowrap;'><i id='s-"
+ anchor
+ "--icon' class='icon-chevron-right'></i> <a href='#' id='s-");
sb.append(anchor).append("' class='togglex'>");
sb.append(table.getTypeName());
sb.append("</a></td>");
// column 2: type
sb.append("<td>");
if (table.getMaterializer() != null) {
tag(sb, "info", "Materialized View");
} else {
if (isExportTable) {
tag(sb, "inverse", "Export Table");
} else {
tag(sb, null, "Table");
}
}
sb.append("</td>");
// column 3: partitioning
sb.append("<td style='whitespace: nowrap;'>");
if (table.getIsreplicated()) {
tag(sb, "warning", "Replicated");
} else {
tag(sb, "success", "Partitioned");
Column partitionCol = table.getPartitioncolumn();
if (partitionCol != null) {
sb.append("<small> on " + partitionCol.getName() + "</small>");
} else {
Table matSrc = table.getMaterializer();
if (matSrc != null) {
sb.append("<small> with " + matSrc.getTypeName() + "</small>");
}
}
}
sb.append("</td>");
// column 4: column count
sb.append("<td>");
sb.append(table.getColumns().size());
sb.append("</td>");
// column 5: index count
sb.append("<td>");
sb.append(table.getIndexes().size());
sb.append("</td>");
// column 6: has pkey
sb.append("<td>");
boolean found = false;
for (Constraint constraint : table.getConstraints()) {
if (ConstraintType.get(constraint.getType()) == ConstraintType.PRIMARY_KEY) {
found = true;
break;
}
}
if (found) {
tag(sb, "info", "Has-PKey");
} else {
tag(sb, null, "No-PKey");
}
sb.append("</td>");
// column 6: has tuple limit
sb.append("<td>");
if (table.getTuplelimit() != Integer.MAX_VALUE) {
tag(sb, "info", String.valueOf(table.getTuplelimit()));
} else {
tag(sb, null, "No-limit");
}
sb.append("</td>");
sb.append("</tr>\n");
// BUILD THE DROPDOWN FOR THE DDL / INDEXES DETAIL
sb.append(
"<tr class='tablesorter-childRow'><td class='invert' colspan='7' id='s-"
+ table.getTypeName().toLowerCase()
+ "--dropdown'>\n");
TableAnnotation annotation = (TableAnnotation) table.getAnnotation();
if (annotation != null) {
// output the DDL
if (annotation.ddl == null) {
sb.append("<p>MISSING DDL</p>\n");
} else {
String ddl = annotation.ddl;
sb.append("<p><pre>" + ddl + "</pre></p>\n");
}
// make sure procs appear in only one category
annotation.proceduresThatReadThis.removeAll(annotation.proceduresThatUpdateThis);
if (annotation.proceduresThatReadThis.size() > 0) {
sb.append("<p>Read-only by procedures: ");
List<String> procs = new ArrayList<String>();
for (Procedure proc : annotation.proceduresThatReadThis) {
procs.add("<a href='#p-" + proc.getTypeName() + "'>" + proc.getTypeName() + "</a>");
}
sb.append(StringUtils.join(procs, ", "));
sb.append("</p>");
}
if (annotation.proceduresThatUpdateThis.size() > 0) {
sb.append("<p>Read/Write by procedures: ");
List<String> procs = new ArrayList<String>();
for (Procedure proc : annotation.proceduresThatUpdateThis) {
procs.add("<a href='#p-" + proc.getTypeName() + "'>" + proc.getTypeName() + "</a>");
}
sb.append(StringUtils.join(procs, ", "));
sb.append("</p>");
}
}
if (table.getIndexes().size() > 0) {
sb.append(generateIndexesTable(table));
} else {
sb.append("<p>No indexes defined on table.</p>\n");
}
sb.append("</td></tr>\n");
return sb.toString();
}
