public static MaterializedViewInfo addVerticalPartition( final Database catalog_db, final String tableName, final List<String> columnNames, final boolean createIndex) throws Exception { Table catalog_tbl = catalog_db.getTables().get(tableName); if (catalog_tbl == null) { throw new Exception("Invalid vertical partition table '" + tableName + "'"); } else if (catalog_tbl.getIsreplicated()) { throw new Exception( "Cannot create vertical partition for replicated table '" + tableName + "'"); } ArrayList<Column> catalog_cols = new ArrayList<Column>(); for (String columnName : columnNames) { Column catalog_col = catalog_tbl.getColumns().get(columnName); if (catalog_col == null) { throw new Exception( "Invalid vertical partition column '" + columnName + "' for table '" + tableName + "'"); } else if (catalog_cols.contains(catalog_col)) { throw new Exception( "Duplicate vertical partition column '" + columnName + "' for table '" + tableName + "'"); } catalog_cols.add(catalog_col); } // FOR return (addVerticalPartition(catalog_tbl, catalog_cols, createIndex)); }
@Override public boolean getIsReplicated() { return m_table.getIsreplicated(); }
protected boolean createSetupInternal( String file_path, String file_nonce, long txnId, Map<Integer, Long> partitionTransactionIds, JSONObject jsData, SystemProcedureExecutionContext context, String hostname, final VoltTable result, Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers, SiteTracker tracker, long timestamp) throws IOException { assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty()); /* * List of partitions to include if this snapshot is * going to be deduped. Attempts to break up the work * by seeding and RNG selecting * a random replica to do the work. Will not work in failure * cases, but we don't use dedupe when we want durability. */ List<Long> sitesToInclude = CSVSnapshotWritePlan.computeDedupedLocalSites(txnId, tracker); // If there's no work to do on this host, just claim success and get out: if (sitesToInclude.isEmpty() && !tracker.isFirstHost()) { return false; } NativeSnapshotWritePlan.createFileBasedCompletionTasks( file_path, file_nonce, txnId, partitionTransactionIds, context, exportSequenceNumbers, timestamp); final List<Table> tables = SnapshotUtil.getTablesToSave(context.getDatabase()); final AtomicInteger numTables = new AtomicInteger(tables.size()); final SnapshotRegistry.Snapshot snapshotRecord = SnapshotRegistry.startSnapshot( txnId, context.getHostId(), file_path, file_nonce, SnapshotFormat.CSV, tables.toArray(new Table[0])); SnapshotDataTarget sdt = null; boolean noTargetsCreated = true; final ArrayList<SnapshotTableTask> partitionedSnapshotTasks = new ArrayList<SnapshotTableTask>(); final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>(); for (final Table table : tables) { /* * For a deduped csv snapshot, only produce the replicated tables on the "leader" * host. */ if (table.getIsreplicated() && !tracker.isFirstHost()) { snapshotRecord.removeTable(table.getTypeName()); continue; } File saveFilePath = null; saveFilePath = SnapshotUtil.constructFileForTable( table, file_path, file_nonce, SnapshotFormat.CSV, context.getHostId()); try { sdt = new SimpleFileSnapshotDataTarget(saveFilePath, !table.getIsreplicated()); m_targets.add(sdt); final Runnable onClose = new TargetStatsClosure(sdt, table.getTypeName(), numTables, snapshotRecord); sdt.setOnCloseHandler(onClose); List<SnapshotDataFilter> filters = new ArrayList<SnapshotDataFilter>(); filters.add(new CSVSnapshotFilter(CatalogUtil.getVoltTable(table), ',', null)); final SnapshotTableTask task = new SnapshotTableTask( table.getRelativeIndex(), sdt, filters.toArray(new SnapshotDataFilter[filters.size()]), table.getIsreplicated(), table.getTypeName()); if (table.getIsreplicated()) { replicatedSnapshotTasks.add(task); } else { partitionedSnapshotTasks.add(task); } noTargetsCreated = false; result.addRow(context.getHostId(), hostname, table.getTypeName(), "SUCCESS", ""); } catch (IOException ex) { handleTargetCreationError( sdt, context, file_nonce, hostname, table.getTypeName(), ex, result); } } if (noTargetsCreated) { SnapshotRegistry.discardSnapshot(snapshotRecord); } // CSV snapshots do the partitioned work only on the specified sites for de-duping, // but since we've pre-filtered the replicated task list to only contain entries on // one node, we can go ahead and distribute them across all of the sites on that node. placePartitionedTasks(partitionedSnapshotTasks, sitesToInclude); placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId())); return noTargetsCreated; }
public CatalogContext(Catalog catalog, File pathToCatalogJar) { // check the heck out of the given params in this immutable class assert (catalog != null); if (catalog == null) { throw new RuntimeException("Can't create CatalogContext with null catalog."); } this.jarPath = pathToCatalogJar; this.catalog = catalog; this.cluster = CatalogUtil.getCluster(this.catalog); this.database = CatalogUtil.getDatabase(this.catalog); this.hosts = this.cluster.getHosts(); this.sites = this.cluster.getSites(); if (this.jarPath != null) { this.catalogClassLoader = new JarClassLoader(this.jarPath.getAbsolutePath()); this.paramMappings = ParametersUtil.getParameterMappingsSetFromJar(this.database, this.jarPath); } else { this.catalogClassLoader = null; this.paramMappings = null; } // ------------------------------------------------------------ // PROCEDURES // ------------------------------------------------------------ this.procedures = database.getProcedures(); this.proceduresArray = new Procedure[this.procedures.size() + 1]; for (Procedure proc : this.procedures) { this.proceduresArray[proc.getId()] = proc; if (proc.getSystemproc()) { this.sysProcedures.add(proc); } else if (proc.getMapreduce()) { this.mrProcedures.add(proc); } else { this.regularProcedures.add(proc); } } // FOR authSystem = new AuthSystem(database, cluster.getSecurityenabled()); siteTracker = null; // new SiteTracker(cluster.getSites()); // count nodes this.numberOfHosts = cluster.getHosts().size(); // count exec sites this.numberOfSites = cluster.getSites().size(); // ------------------------------------------------------------ // PARTITIONS // ------------------------------------------------------------ this.numberOfPartitions = cluster.getNum_partitions(); this.partitions = new Partition[this.numberOfPartitions]; this.partitionIdArray = new Integer[this.numberOfPartitions]; this.partitionSingletons = new PartitionSet[this.numberOfPartitions]; this.partitionSiteXref = new int[this.numberOfPartitions]; for (Partition part : CatalogUtil.getAllPartitions(catalog)) { int p = part.getId(); this.partitions[p] = part; this.partitionIdArray[p] = Integer.valueOf(p); this.partitionSingletons[p] = new PartitionSet(p); this.partitionIdCollection.add(this.partitionIdArray[p]); this.partitionSiteXref[part.getId()] = ((Site) part.getParent()).getId(); } // FOR // ------------------------------------------------------------ // TABLES // ------------------------------------------------------------ for (Table tbl : database.getTables()) { if (tbl.getSystable()) { sysTables.add(tbl); } else if (tbl.getMapreduce()) { mapReduceTables.add(tbl); } else if (tbl.getMaterializer() != null) { viewTables.add(tbl); } else { dataTables.add(tbl); if (tbl.getIsreplicated()) { replicatedTables.add(tbl); } if (tbl.getEvictable()) { evictableTables.add(tbl); } } } // FOR // PLANFRAGMENTS this.initPlanFragments(); }
/** * Get some embeddable HTML of some generic catalog/application stats that is drawn on the first * page of the report. */ static String getStatsHTML(Database db, ArrayList<Feedback> warnings) { StringBuilder sb = new StringBuilder(); sb.append("<table class='table table-condensed'>\n"); // count things int indexes = 0, views = 0, statements = 0; int partitionedTables = 0, replicatedTables = 0; int partitionedProcs = 0, replicatedProcs = 0; int readProcs = 0, writeProcs = 0; for (Table t : db.getTables()) { if (t.getMaterializer() != null) { views++; } else { if (t.getIsreplicated()) { replicatedTables++; } else { partitionedTables++; } } indexes += t.getIndexes().size(); } for (Procedure p : db.getProcedures()) { // skip auto-generated crud procs if (p.getDefaultproc()) { continue; } if (p.getSinglepartition()) { partitionedProcs++; } else { replicatedProcs++; } if (p.getReadonly()) { readProcs++; } else { writeProcs++; } statements += p.getStatements().size(); } // version sb.append("<tr><td>Compiled by VoltDB Version</td><td>"); sb.append(VoltDB.instance().getVersionString()).append("</td></tr>\n"); // timestamp sb.append("<tr><td>Compiled on</td><td>"); SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z"); sb.append(sdf.format(m_timestamp)).append("</td></tr>\n"); // tables sb.append("<tr><td>Table Count</td><td>"); sb.append( String.format( "%d (%d partitioned / %d replicated)", partitionedTables + replicatedTables, partitionedTables, replicatedTables)); sb.append("</td></tr>\n"); // views sb.append("<tr><td>Materialized View Count</td><td>").append(views).append("</td></tr>\n"); // indexes sb.append("<tr><td>Index Count</td><td>").append(indexes).append("</td></tr>\n"); // procedures sb.append("<tr><td>Procedure Count</td><td>"); sb.append( String.format( "%d (%d partitioned / %d replicated) (%d read-only / %d read-write)", partitionedProcs + replicatedProcs, partitionedProcs, replicatedProcs, readProcs, writeProcs)); sb.append("</td></tr>\n"); // statements sb.append("<tr><td>SQL Statement Count</td><td>").append(statements).append("</td></tr>\n"); sb.append("</table>\n\n"); // warnings, add warning section if any if (warnings.size() > 0) { sb.append("<h4>Warnings</h4>"); sb.append("<table class='table table-condensed'>\n"); for (Feedback warning : warnings) { String procName = warning.getFileName().replace(".class", ""); String nameLink = ""; // not a warning during compiling procedures, must from the schema if (procName.compareToIgnoreCase("null") == 0) { String schemaName = ""; String warningMsg = warning.getMessage().toLowerCase(); if (warningMsg.contains("table ")) { int begin = warningMsg.indexOf("table ") + 6; int end = (warningMsg.substring(begin)).indexOf(" "); schemaName = warningMsg.substring(begin, begin + end); } nameLink = "<a href='#s-" + schemaName + "'>" + schemaName.toUpperCase() + "</a>"; } else { nameLink = "<a href='#p-" + procName.toLowerCase() + "'>" + procName + "</a>"; } sb.append("<tr><td>") .append(nameLink) .append("</td><td>") .append(warning.getMessage()) .append("</td></tr>\n"); } sb.append("").append("</table>\n").append("</td></tr>\n"); } return sb.toString(); }
static String generateSchemaRow(Table table, boolean isExportTable) { StringBuilder sb = new StringBuilder(); sb.append("<tr class='primaryrow'>"); // column 1: table name String anchor = table.getTypeName().toLowerCase(); sb.append( "<td style='white-space: nowrap;'><i id='s-" + anchor + "--icon' class='icon-chevron-right'></i> <a href='#' id='s-"); sb.append(anchor).append("' class='togglex'>"); sb.append(table.getTypeName()); sb.append("</a></td>"); // column 2: type sb.append("<td>"); if (table.getMaterializer() != null) { tag(sb, "info", "Materialized View"); } else { if (isExportTable) { tag(sb, "inverse", "Export Table"); } else { tag(sb, null, "Table"); } } sb.append("</td>"); // column 3: partitioning sb.append("<td style='whitespace: nowrap;'>"); if (table.getIsreplicated()) { tag(sb, "warning", "Replicated"); } else { tag(sb, "success", "Partitioned"); Column partitionCol = table.getPartitioncolumn(); if (partitionCol != null) { sb.append("<small> on " + partitionCol.getName() + "</small>"); } else { Table matSrc = table.getMaterializer(); if (matSrc != null) { sb.append("<small> with " + matSrc.getTypeName() + "</small>"); } } } sb.append("</td>"); // column 4: column count sb.append("<td>"); sb.append(table.getColumns().size()); sb.append("</td>"); // column 5: index count sb.append("<td>"); sb.append(table.getIndexes().size()); sb.append("</td>"); // column 6: has pkey sb.append("<td>"); boolean found = false; for (Constraint constraint : table.getConstraints()) { if (ConstraintType.get(constraint.getType()) == ConstraintType.PRIMARY_KEY) { found = true; break; } } if (found) { tag(sb, "info", "Has-PKey"); } else { tag(sb, null, "No-PKey"); } sb.append("</td>"); // column 6: has tuple limit sb.append("<td>"); if (table.getTuplelimit() != Integer.MAX_VALUE) { tag(sb, "info", String.valueOf(table.getTuplelimit())); } else { tag(sb, null, "No-limit"); } sb.append("</td>"); sb.append("</tr>\n"); // BUILD THE DROPDOWN FOR THE DDL / INDEXES DETAIL sb.append( "<tr class='tablesorter-childRow'><td class='invert' colspan='7' id='s-" + table.getTypeName().toLowerCase() + "--dropdown'>\n"); TableAnnotation annotation = (TableAnnotation) table.getAnnotation(); if (annotation != null) { // output the DDL if (annotation.ddl == null) { sb.append("<p>MISSING DDL</p>\n"); } else { String ddl = annotation.ddl; sb.append("<p><pre>" + ddl + "</pre></p>\n"); } // make sure procs appear in only one category annotation.proceduresThatReadThis.removeAll(annotation.proceduresThatUpdateThis); if (annotation.proceduresThatReadThis.size() > 0) { sb.append("<p>Read-only by procedures: "); List<String> procs = new ArrayList<String>(); for (Procedure proc : annotation.proceduresThatReadThis) { procs.add("<a href='#p-" + proc.getTypeName() + "'>" + proc.getTypeName() + "</a>"); } sb.append(StringUtils.join(procs, ", ")); sb.append("</p>"); } if (annotation.proceduresThatUpdateThis.size() > 0) { sb.append("<p>Read/Write by procedures: "); List<String> procs = new ArrayList<String>(); for (Procedure proc : annotation.proceduresThatUpdateThis) { procs.add("<a href='#p-" + proc.getTypeName() + "'>" + proc.getTypeName() + "</a>"); } sb.append(StringUtils.join(procs, ", ")); sb.append("</p>"); } } if (table.getIndexes().size() > 0) { sb.append(generateIndexesTable(table)); } else { sb.append("<p>No indexes defined on table.</p>\n"); } sb.append("</td></tr>\n"); return sb.toString(); }