/**
* Get a ascii-string-safe version of the binary value using a hex encoding.
*
* @return A hex-encoded string value representing the serialized objects.
*/
public String getHexEncodedBytes() {
buffer.b().flip();
byte bytes[] = new byte[buffer.b().remaining()];
buffer.b().get(bytes);
String hex = Encoder.hexEncode(bytes);
buffer.discard();
return hex;
}
public long run(byte[][] data1, String[] data2, byte[][] data3) {
for (byte[] b : data1) {
if (new String(b, Constants.UTF8ENCODING).equals("Hello") == false) {
throw new VoltAbortException("bad match a");
}
}
for (String s : data2) {
if (s.equals("Hello") == false) {
throw new VoltAbortException("bad match b");
}
}
if (Encoder.hexEncode(data3[0]).equalsIgnoreCase("AAbbff00") == false) {
throw new VoltAbortException("bad match d");
}
if (Encoder.hexEncode(data3[1]).equalsIgnoreCase("AAbbff0011") == false) {
throw new VoltAbortException("bad match e");
}
if (Encoder.hexEncode(data3[2]).equalsIgnoreCase("1234567890abcdef") == false) {
throw new VoltAbortException("bad match f");
}
return 0;
}
/** Generate the HTML catalog report from a newly compiled VoltDB catalog */
public static String report(Catalog catalog, ArrayList<Feedback> warnings) throws IOException {
// asynchronously get platform properties
new Thread() {
@Override
public void run() {
PlatformProperties.getPlatformProperties();
}
}.start();
URL url = Resources.getResource(ReportMaker.class, "template.html");
String contents = Resources.toString(url, Charsets.UTF_8);
Cluster cluster = catalog.getClusters().get("cluster");
assert (cluster != null);
Database db = cluster.getDatabases().get("database");
assert (db != null);
String statsData = getStatsHTML(db, warnings);
contents = contents.replace("##STATS##", statsData);
String schemaData = generateSchemaTable(db.getTables(), db.getConnectors());
contents = contents.replace("##SCHEMA##", schemaData);
String procData = generateProceduresTable(db.getProcedures());
contents = contents.replace("##PROCS##", procData);
DatabaseSizes sizes = CatalogSizing.getCatalogSizes(db);
String sizeData = generateSizeTable(sizes);
contents = contents.replace("##SIZES##", sizeData);
String sizeSummary = generateSizeSummary(sizes);
contents = contents.replace("##SIZESUMMARY##", sizeSummary);
String platformData = PlatformProperties.getPlatformProperties().toHTML();
contents = contents.replace("##PLATFORM##", platformData);
contents = contents.replace("##VERSION##", VoltDB.instance().getVersionString());
DateFormat df = new SimpleDateFormat("d MMM yyyy HH:mm:ss z");
contents = contents.replace("##TIMESTAMP##", df.format(m_timestamp));
String msg =
Encoder.hexEncode(VoltDB.instance().getVersionString() + "," + System.currentTimeMillis());
contents = contents.replace("get.py?a=KEY&", String.format("get.py?a=%s&", msg));
return contents;
}
@Test
public void testUpdateCatalog() throws IOException {
// only makes sense in pro (sysproc suite has a complementary test for community)
if (VoltDB.instance().getConfig().m_isEnterprise) {
String catalogHex = Encoder.hexEncode("blah");
ByteBuffer msg = createMsg("@UpdateApplicationCatalog", catalogHex, "blah");
ClientResponseImpl resp = m_ci.handleRead(msg, m_handler, m_cxn);
assertNull(resp);
ArgumentCaptor<LocalObjectMessage> captor = ArgumentCaptor.forClass(LocalObjectMessage.class);
verify(m_messenger)
.send(
eq(32L), // A fixed number set in setUpOnce()
captor.capture());
assertTrue(captor.getValue().payload instanceof CatalogChangeWork);
}
}
private static void validateTableColumnOfScalarVarbinary(
VoltTable vt, int col, String[] expected) {
assertNotNull(expected);
assertEquals(expected.length, vt.getRowCount());
int len = expected.length;
for (int i = 0; i < len; i++) {
assertTrue(vt.advanceRow());
byte[] actual = vt.getVarbinary(col);
if (expected[i] == null) {
assertTrue(vt.wasNull());
assertEquals(null, actual);
} else {
assertEquals(expected[i], Encoder.hexEncode(actual));
}
}
}
/**
* Make a printable, short string for a varbinary. String includes a CRC and the contents of the
* varbinary in hex. Contents longer than 13 chars are truncated and elipsized. Yes, "elipsized"
* is totally a word.
*
* <p>Example: "bin[crc:1298399436,value:0xABCDEF12345...]"
*
* @param bin The bytes to print out.
* @return A string representation that is printable and short.
*/
public static String varbinaryToPrintableString(byte[] bin) {
CRC32 crc = new CRC32();
StringBuilder sb = new StringBuilder();
sb.append("bin[crc:");
crc.update(bin);
sb.append(crc.getValue());
sb.append(",value:0x");
String hex = Encoder.hexEncode(bin);
if (hex.length() > 13) {
sb.append(hex.substring(0, 10));
sb.append("...");
} else {
sb.append(hex);
}
sb.append("]");
return sb.toString();
}
/**
* Compare two tables using the data inside them, rather than simply comparing the underlying
* buffers. This is slightly more tolerant of floating point issues than {@link
* VoltTable#hasSameContents(VoltTable)}. It's also much slower than comparing buffers.
*
* <p>This will also add a specific error message to the provided {@link StringBuilder} that
* explains how the tables are different, printing out values if needed.
*
* @param t1 {@link VoltTable} 1
* @param t2 {@link VoltTable} 2
* @param sb A {@link StringBuilder} to append the error message to.
* @return true if the tables are equal.
* @see TableHelper#deepEquals(VoltTable, VoltTable) deepEquals
*/
public static boolean deepEqualsWithErrorMsg(VoltTable t1, VoltTable t2, StringBuilder sb) {
// allow people to pass null without guarding everything with if statements
if (sb == null) {
sb = new StringBuilder();
}
// this behaves like an equals method should, but feels wrong here... alas...
if ((t1 == null) && (t2 == null)) {
return true;
}
// handle when one side is null
if (t1 == null) {
sb.append("t1 == NULL\n");
return false;
}
if (t2 == null) {
sb.append("t2 == NULL\n");
return false;
}
if (t1.getRowCount() != t2.getRowCount()) {
sb.append(String.format("Row count %d != %d\n", t1.getRowCount(), t2.getRowCount()));
return false;
}
if (t1.getColumnCount() != t2.getColumnCount()) {
sb.append(String.format("Col count %d != %d\n", t1.getColumnCount(), t2.getColumnCount()));
return false;
}
for (int col = 0; col < t1.getColumnCount(); col++) {
if (t1.getColumnType(col) != t2.getColumnType(col)) {
sb.append(
String.format(
"Column %d: type %s != %s\n",
col, t1.getColumnType(col).toString(), t2.getColumnType(col).toString()));
return false;
}
if (t1.getColumnName(col).equals(t2.getColumnName(col)) == false) {
sb.append(
String.format(
"Column %d: name %s != %s\n", col, t1.getColumnName(col), t2.getColumnName(col)));
return false;
}
}
t1.resetRowPosition();
t2.resetRowPosition();
for (int row = 0; row < t1.getRowCount(); row++) {
t1.advanceRow();
t2.advanceRow();
for (int col = 0; col < t1.getColumnCount(); col++) {
Object obj1 = t1.get(col, t1.getColumnType(col));
if (t1.wasNull()) {
obj1 = null;
}
Object obj2 = t2.get(col, t2.getColumnType(col));
if (t2.wasNull()) {
obj2 = null;
}
if ((obj1 == null) && (obj2 == null)) {
continue;
}
if ((obj1 == null) || (obj2 == null)) {
sb.append(
String.format(
"Row,Col-%d,%d of type %s: %s != %s\n",
row,
col,
t1.getColumnType(col).toString(),
String.valueOf(obj1),
String.valueOf(obj2)));
return false;
}
if (t1.getColumnType(col) == VoltType.VARBINARY) {
byte[] array1 = (byte[]) obj1;
byte[] array2 = (byte[]) obj2;
if (Arrays.equals(array1, array2) == false) {
sb.append(
String.format(
"Row,Col-%d,%d of type %s: %s != %s\n",
row,
col,
t1.getColumnType(col).toString(),
Encoder.hexEncode(array1),
Encoder.hexEncode(array2)));
return false;
}
} else {
if (obj1.equals(obj2) == false) {
sb.append(
String.format(
"Row,Col-%d,%d of type %s: %s != %s\n",
row, col, t1.getColumnType(col).toString(), obj1.toString(), obj2.toString()));
return false;
}
}
}
}
// true means we made it through the gaundlet and the tables are, fwiw, identical
return true;
}
void compileDatabaseNode(DatabaseType database) throws VoltCompilerException {
final ArrayList<String> programs = new ArrayList<String>();
final ArrayList<String> schemas = new ArrayList<String>();
final ArrayList<ProcedureDescriptor> procedures = new ArrayList<ProcedureDescriptor>();
final ArrayList<Class<?>> classDependencies = new ArrayList<Class<?>>();
final ArrayList<String[]> partitions = new ArrayList<String[]>();
final String databaseName = database.getName();
// schema does not verify that the database is named "database"
if (databaseName.equals("database") == false) {
final String msg =
"VoltDB currently requires all database elements to be named "
+ "\"database\" (found: \""
+ databaseName
+ "\")";
throw new VoltCompilerException(msg);
}
// create the database in the catalog
m_catalog.execute("add /clusters[cluster] databases " + databaseName);
Database db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);
SnapshotType snapshotSettings = database.getSnapshot();
if (snapshotSettings != null) {
SnapshotSchedule schedule = db.getSnapshotschedule().add("default");
String frequency = snapshotSettings.getFrequency();
if (!frequency.endsWith("s") && !frequency.endsWith("m") && !frequency.endsWith("h")) {
throw new VoltCompilerException(
"Snapshot frequency "
+ frequency
+ " needs to end with time unit specified"
+ " that is one of [s, m, h] (seconds, minutes, hours)");
}
int frequencyInt = 0;
String frequencySubstring = frequency.substring(0, frequency.length() - 1);
try {
frequencyInt = Integer.parseInt(frequencySubstring);
} catch (Exception e) {
throw new VoltCompilerException("Frequency " + frequencySubstring + " is not an integer ");
}
String prefix = snapshotSettings.getPrefix();
if (prefix == null || prefix.isEmpty()) {
throw new VoltCompilerException("Snapshot prefix " + prefix + " is not a valid prefix ");
}
if (prefix.contains("-") || prefix.contains(",")) {
throw new VoltCompilerException("Snapshot prefix " + prefix + " cannot include , or - ");
}
String path = snapshotSettings.getPath();
if (path == null || path.isEmpty()) {
throw new VoltCompilerException("Snapshot path " + path + " is not a valid path ");
}
if (snapshotSettings.getRetain() == null) {
throw new VoltCompilerException("Snapshot retain value not provided");
}
int retain = snapshotSettings.getRetain().intValue();
if (retain < 1) {
throw new VoltCompilerException(
"Snapshot retain value " + retain + " is not a valid value. Must be 1 or greater.");
}
schedule.setFrequencyunit(frequency.substring(frequency.length() - 1, frequency.length()));
schedule.setFrequencyvalue(frequencyInt);
schedule.setPath(path);
schedule.setPrefix(prefix);
schedule.setRetain(retain);
}
// schemas/schema
for (SchemasType.Schema schema : database.getSchemas().getSchema()) {
LOG.l7dlog(
Level.DEBUG,
LogKeys.compiler_VoltCompiler_CatalogPath.name(),
new Object[] {schema.getPath()},
null);
schemas.add(schema.getPath());
}
// groups/group.
if (database.getGroups() != null) {
for (GroupsType.Group group : database.getGroups().getGroup()) {
org.voltdb.catalog.Group catGroup = db.getGroups().add(group.getName());
catGroup.setAdhoc(group.isAdhoc());
catGroup.setSysproc(group.isSysproc());
}
}
// users/user
if (database.getUsers() != null) {
for (UsersType.User user : database.getUsers().getUser()) {
org.voltdb.catalog.User catUser = db.getUsers().add(user.getName());
catUser.setAdhoc(user.isAdhoc());
catUser.setSysproc(user.isSysproc());
byte passwordHash[] = extractPassword(user.getPassword());
catUser.setShadowpassword(Encoder.hexEncode(passwordHash));
// process the @groups comma separated list
if (user.getGroups() != null) {
String grouplist[] = user.getGroups().split(",");
for (final String group : grouplist) {
final GroupRef groupRef = catUser.getGroups().add(group);
final Group catalogGroup = db.getGroups().get(group);
if (catalogGroup != null) {
groupRef.setGroup(catalogGroup);
}
}
}
}
}
// procedures/procedure
for (ProceduresType.Procedure proc : database.getProcedures().getProcedure()) {
procedures.add(getProcedure(proc));
}
// classdependencies/classdependency
if (database.getClassdependencies() != null) {
for (Classdependency dep : database.getClassdependencies().getClassdependency()) {
classDependencies.add(getClassDependency(dep));
}
}
// partitions/table
if (database.getPartitions() != null) {
for (org.voltdb.compiler.projectfile.PartitionsType.Partition table :
database.getPartitions().getPartition()) {
partitions.add(getPartition(table));
}
}
String msg = "Database \"" + databaseName + "\" ";
// TODO: schema allows 0 procedures. Testbase relies on this.
if (procedures.size() == 0) {
msg +=
"needs at least one \"procedure\" element "
+ "(currently has "
+ String.valueOf(procedures.size())
+ ")";
throw new VoltCompilerException(msg);
}
if (procedures.size() < 1) {
msg += "is missing the \"procedures\" element";
throw new VoltCompilerException(msg);
}
// shutdown and make a new hsqldb
m_hsql = HSQLInterface.loadHsqldb();
// Actually parse and handle all the programs
for (final String programName : programs) {
m_catalog.execute("add " + db.getPath() + " programs " + programName);
}
// Actually parse and handle all the DDL
final DDLCompiler ddlcompiler = new DDLCompiler(this, m_hsql);
for (final String schemaPath : schemas) {
File schemaFile = null;
if (schemaPath.contains(".jar!")) {
String ddlText = null;
try {
ddlText = JarReader.readFileFromJarfile(schemaPath);
} catch (final Exception e) {
throw new VoltCompilerException(e);
}
schemaFile = VoltProjectBuilder.writeStringToTempFile(ddlText);
} else {
schemaFile = new File(schemaPath);
}
if (!schemaFile.isAbsolute()) {
// Resolve schemaPath relative to the database definition xml file
schemaFile = new File(new File(m_projectFileURL).getParent(), schemaPath);
}
// add the file object's path to the list of files for the jar
m_ddlFilePaths.put(schemaFile.getName(), schemaFile.getPath());
ddlcompiler.loadSchema(schemaFile.getAbsolutePath());
}
ddlcompiler.compileToCatalog(m_catalog, db);
// Actually parse and handle all the partitions
// this needs to happen before procedures are compiled
msg = "In database \"" + databaseName + "\", ";
final CatalogMap<Table> tables = db.getTables();
for (final String[] partition : partitions) {
final String tableName = partition[0];
final String colName = partition[1];
final Table t = tables.getIgnoreCase(tableName);
if (t == null) {
msg += "\"partition\" element has unknown \"table\" attribute '" + tableName + "'";
throw new VoltCompilerException(msg);
}
final Column c = t.getColumns().getIgnoreCase(colName);
// make sure the column exists
if (c == null) {
msg += "\"partition\" element has unknown \"column\" attribute '" + colName + "'";
throw new VoltCompilerException(msg);
}
// make sure the column is marked not-nullable
if (c.getNullable() == true) {
msg +=
"Partition column '"
+ tableName
+ "."
+ colName
+ "' is nullable. "
+ "Partition columns must be constrained \"NOT NULL\".";
throw new VoltCompilerException(msg);
}
t.setPartitioncolumn(c);
t.setIsreplicated(false);
// Set the destination tables of associated views non-replicated.
// If a view's source table is replicated, then a full scan of the
// associated view is singled-sited. If the source is partitioned,
// a full scan of the view must be distributed.
final CatalogMap<MaterializedViewInfo> views = t.getViews();
for (final MaterializedViewInfo mvi : views) {
mvi.getDest().setIsreplicated(false);
}
}
// add vertical partitions
if (database.getVerticalpartitions() != null) {
for (Verticalpartition vp : database.getVerticalpartitions().getVerticalpartition()) {
try {
addVerticalPartition(db, vp.getTable(), vp.getColumn(), vp.isIndexed());
} catch (Exception ex) {
throw new VoltCompilerException(
"Failed to create vertical partition for " + vp.getTable(), ex);
}
}
}
// this should reorder the tables and partitions all alphabetically
String catData = m_catalog.serialize();
m_catalog = new Catalog();
m_catalog.execute(catData);
db = m_catalog.getClusters().get("cluster").getDatabases().get(databaseName);
// add database estimates info
addDatabaseEstimatesInfo(m_estimates, db);
addSystemProcsToCatalog(m_catalog, db);
// Process and add exports and connectors to the catalog
// Must do this before compiling procedures to deny updates
// on append-only tables.
if (database.getExports() != null) {
// currently, only a single connector is allowed
Connector conn = database.getExports().getConnector();
compileConnector(conn, db);
}
// Actually parse and handle all the Procedures
for (final ProcedureDescriptor procedureDescriptor : procedures) {
final String procedureName = procedureDescriptor.m_className;
m_currentFilename = procedureName.substring(procedureName.lastIndexOf('.') + 1);
m_currentFilename += ".class";
ProcedureCompiler.compile(this, m_hsql, m_estimates, m_catalog, db, procedureDescriptor);
}
// Add all the class dependencies to the output jar
for (final Class<?> classDependency : classDependencies) {
addClassToJar(classDependency, this);
}
m_hsql.close();
}
VoltTable runSQLWithSubstitutions(
final SQLStmt stmt, ParameterSet params, byte[] paramJavaTypes) {
// HSQLProcedureWrapper does nothing smart. it just implements this interface with
// runStatement()
StringBuilder sqlOut = new StringBuilder(stmt.getText().length() * 2);
assert (paramJavaTypes != null);
int lastIndex = 0;
String sql = stmt.getText();
// if there's no ? in the statmemt, then zero out any auto-parameterization
int paramCount = StringUtils.countMatches(sql, "?");
if (paramCount == 0) {
params = ParameterSet.emptyParameterSet();
paramJavaTypes = new byte[0];
}
Object[] paramObjs = params.toArray();
for (int i = 0; i < paramObjs.length; i++) {
int nextIndex = sql.indexOf('?', lastIndex);
if (nextIndex == -1)
throw new RuntimeException("SQL Statement has more arguments than params.");
sqlOut.append(sql, lastIndex, nextIndex);
lastIndex = nextIndex + 1;
VoltType type = VoltType.get(paramJavaTypes[i]);
if (VoltType.isNullVoltType(paramObjs[i])) {
sqlOut.append("NULL");
} else if (paramObjs[i] instanceof TimestampType) {
if (type != VoltType.TIMESTAMP)
throw new RuntimeException("Inserting date into mismatched column type in HSQL.");
TimestampType d = (TimestampType) paramObjs[i];
// convert VoltDB's microsecond granularity to millis.
Timestamp t = new Timestamp(d.getTime() / 1000);
sqlOut.append('\'').append(t.toString()).append('\'');
} else if (paramObjs[i] instanceof byte[]) {
if (type == VoltType.STRING) {
// Convert from byte[] -> String; escape single quotes
try {
sqlOut.append(sqlEscape(new String((byte[]) paramObjs[i], "UTF-8")));
} catch (UnsupportedEncodingException e) {
// should NEVER HAPPEN
System.err.println("FATAL: Your JVM doens't support UTF-&");
System.exit(-1);
}
} else if (type == VoltType.VARBINARY) {
// Convert from byte[] -> String; using hex
sqlOut.append(sqlEscape(Encoder.hexEncode((byte[]) paramObjs[i])));
} else {
throw new RuntimeException(
"Inserting string/varbinary (bytes) into mismatched column type in HSQL.");
}
} else if (paramObjs[i] instanceof String) {
if (type != VoltType.STRING)
throw new RuntimeException("Inserting string into mismatched column type in HSQL.");
// Escape single quotes
sqlOut.append(sqlEscape((String) paramObjs[i]));
} else {
if (type == VoltType.TIMESTAMP) {
long t = Long.parseLong(paramObjs[i].toString());
TimestampType d = new TimestampType(t);
// convert VoltDB's microsecond granularity to millis
Timestamp ts = new Timestamp(d.getTime() * 1000);
sqlOut.append('\'').append(ts.toString()).append('\'');
} else sqlOut.append(paramObjs[i].toString());
}
}
sqlOut.append(sql, lastIndex, sql.length());
return runDML(sqlOut.toString());
}
