public static void main(String[] args) throws Exception {
VoltDB.setDefaultTimezone();
config = new SchemaChangeConfig();
config.parse("SchemaChangeClient", args);
ClientConfig clientConfig = new ClientConfig();
clientConfig.setProcedureCallTimeout(30 * 60 * 1000); // 30 min
client = ClientFactory.createClient(clientConfig);
String[] servers = config.servers.split(",");
for (String server : servers) {
server = server.trim();
client.createConnection(server);
}
// get the topo
topo = getCluterTopology(client);
// kick this off with a random schema
VoltTable t = catalogChange(null, true);
for (int i = 0; i < 50; i++) {
// make sure the table is full and mess around with it
loadTable(t);
for (int j = 0; j < 50; j++) {
String tableName = TableHelper.getTableName(t);
// deterministically sample some rows
VoltTable preT = sample(t);
// System.out.printf("First sample:\n%s\n", preT.toFormattedString());
// move to an entirely new table or migrated schema
t = catalogChange(t, (j == 0) && (rand.nextInt(5) == 0));
// if the table has been migrated, check the data
if (TableHelper.getTableName(t).equals(tableName)) {
VoltTable guessT = t.clone(4096 * 1024);
// System.out.printf("Empty clone:\n%s\n", guessT.toFormattedString());
TableHelper.migrateTable(preT, guessT);
// System.out.printf("Java migration:\n%s\n", guessT.toFormattedString());
// deterministically sample the same rows
VoltTable postT = sample(t);
// System.out.printf("Second sample:\n%s\n", postT.toFormattedString());
postT.resetRowPosition();
preT.resetRowPosition();
StringBuilder sb = new StringBuilder();
if (!TableHelper.deepEqualsWithErrorMsg(postT, guessT, sb)) {
System.err.println(sb.toString());
assert (false);
}
}
}
}
client.close();
}
private boolean checkConstraints(String procName, ClientResponse response) {
boolean isSatisfied = true;
int orig_position = -1;
// Check if all the tables in the result set satisfy the constraints.
for (int i = 0; isSatisfied && i < response.getResults().length; i++) {
Pair<String, Integer> key = Pair.of(procName, i);
if (!m_constraints.containsKey(key)) continue;
VoltTable table = response.getResults()[i];
orig_position = table.getActiveRowIndex();
table.resetRowPosition();
// Iterate through all rows and check if they satisfy the
// constraints.
while (isSatisfied && table.advanceRow()) {
isSatisfied = Verification.checkRow(m_constraints.get(key), table);
}
// Have to reset the position to its original position.
if (orig_position < 0) table.resetRowPosition();
else table.advanceToRow(orig_position);
}
if (!isSatisfied) System.err.println("Transaction " + procName + " failed check");
return isSatisfied;
}
private VoltTable evictData(Table catalog_tbl) throws Exception {
VoltTable results[] = this.ee.getStats(SysProcSelector.TABLE, this.locators, false, 0L);
assertEquals(1, results.length);
// System.err.println(VoltTableUtil.format(results));
while (results[0].advanceRow()) {
if (results[0].getString("TABLE_NAME").equalsIgnoreCase(catalog_tbl.getName()) == false)
continue;
for (String col : statsFields) {
int idx = results[0].getColumnIndex(col);
assertEquals(0, results[0].getLong(idx));
} // FOR
} // WHILE
// Now force the EE to evict our boys out
// We'll tell it to remove 1MB, which is guaranteed to include all of our tuples
VoltTable evictResult = this.ee.antiCacheEvictBlock(catalog_tbl, 1024 * 1024, 1);
// System.err.println("-------------------------------");
// System.err.println(VoltTableUtil.format(evictResult));
assertNotNull(evictResult);
assertEquals(1, evictResult.getRowCount());
assertNotSame(results[0].getColumnCount(), evictResult.getColumnCount());
evictResult.resetRowPosition();
boolean adv = evictResult.advanceRow();
assertTrue(adv);
return (evictResult);
}
protected void validateRowSeenAtAllPartitions(
VoltTable result, String columnName, String targetValue, boolean enforceUnique) {
result.resetRowPosition();
Set<Integer> partsSeen = new HashSet<>();
while (result.advanceRow()) {
String colValFromRow = result.getString(columnName);
if (targetValue.equalsIgnoreCase(colValFromRow)) {
int thisPartId = (int) result.getLong("PARTITION_ID");
if (enforceUnique) {
assertFalse(
"PARTITION_ID: "
+ thisPartId
+ " seen twice in table looking for "
+ targetValue
+ " in column "
+ columnName,
partsSeen.contains(thisPartId));
}
partsSeen.add(thisPartId);
}
}
// Remove the MPI in case it's in there
partsSeen.remove(MpInitiator.MP_INIT_PID);
assertEquals(PARTITIONS, partsSeen.size());
}
protected boolean validateRowSeenAtAllSites(
VoltTable result, String columnName, String targetValue, boolean enforceUnique) {
result.resetRowPosition();
Set<Long> sitesSeen = new HashSet<Long>();
while (result.advanceRow()) {
String colValFromRow = result.getString(columnName);
if (targetValue.equalsIgnoreCase(colValFromRow)) {
long hostId = result.getLong("HOST_ID");
long thisSiteId = result.getLong("SITE_ID");
thisSiteId |= hostId << 32;
if (enforceUnique) {
assertFalse(
"SITE_ID: "
+ thisSiteId
+ " seen twice in table looking for "
+ targetValue
+ " in column "
+ columnName,
sitesSeen.contains(thisSiteId));
}
sitesSeen.add(thisSiteId);
}
}
return (HOSTS * SITES) == sitesSeen.size();
}
/**
* Check the result of a query that has only an OFFSET and no LIMIT clause. This is done by
* executing the query with and without the offset clause, and then skipping past the offset rows
* in the expected table here on the client side.
*/
private static void doOffsetAndCheck(Client client, String stmt)
throws IOException, InterruptedException, ProcCallException {
String stmtNoOffset = stmt.substring(0, stmt.indexOf("OFFSET"));
VoltTable expectedTable = client.callProcedure("@AdHoc", stmtNoOffset).getResults()[0];
int rowCountBeforeOffset = expectedTable.getRowCount();
int[] offsets = {0, 1, 5, 10, 11, 15};
for (int offset : offsets) {
VoltTable actualTable = client.callProcedure("@AdHoc", stmt, offset).getResults()[0];
int expectedRowCount = Math.max(rowCountBeforeOffset - offset, 0);
assertEquals(
"Actual table has wrong number of rows: ", expectedRowCount, actualTable.getRowCount());
if (actualTable.getRowCount() == 0) continue;
// non-empty result.
// Advance expected table past offset
// then compare what's left.
actualTable.resetRowPosition();
for (int i = 0; i < offset; ++i) expectedTable.advanceRow();
while (actualTable.advanceRow() && expectedTable.advanceRow()) {
assertEquals(expectedTable.getLong(0), actualTable.getLong(0));
assertEquals(expectedTable.getLong(1), actualTable.getLong(1));
}
}
}
private int countHostsProvidingRows(
VoltTable result, Map<String, String> columnTargets, boolean enforceUnique) {
result.resetRowPosition();
Set<Long> hostsSeen = new HashSet<Long>();
while (result.advanceRow()) {
if (checkRowForMultipleTargets(result, columnTargets)) {
Long thisHostId = result.getLong("HOST_ID");
if (enforceUnique) {
StringBuilder message = new StringBuilder();
message.append("HOST_ID: " + thisHostId + " seen twice in table looking for ");
for (Entry<String, String> entry : columnTargets.entrySet()) {
message.append(entry.getValue() + " in column " + entry.getKey() + ";");
}
assertFalse(message.toString(), hostsSeen.contains(thisHostId));
}
hostsSeen.add(thisHostId);
}
}
// * Enable this to force a failure with diagnostics */ hostsSeen.add(123456789L);
// Before possibly failing an assert, prepare to report details of the non-conforming result.
m_recentAnalysis = null;
if (HOSTS != hostsSeen.size()) {
m_recentAnalysis = new StringBuilder();
m_recentAnalysis.append("Failure follows from these results:\n");
Set<Long> seenAgain = new HashSet<Long>();
result.resetRowPosition();
while (result.advanceRow()) {
Long thisHostId = result.getLong("HOST_ID");
String rowStatus = "Found a non-match";
if (checkRowForMultipleTargets(result, columnTargets)) {
if (seenAgain.add(thisHostId)) {
rowStatus = "Added a match";
} else {
rowStatus = "Duplicated a match";
}
}
m_recentAnalysis.append(rowStatus + " at host " + thisHostId + " for ");
for (String key : columnTargets.keySet()) {
m_recentAnalysis.append(key + " " + result.getString(key) + ";");
}
m_recentAnalysis.append("\n");
}
}
return hostsSeen.size();
}
/** Grab some random rows that aren't on the first EE page for the table. */
public static VoltTable sample(VoltTable t) throws Exception {
VoltTable t2 = t.clone(4096 * 1024);
ClientResponse cr =
client.callProcedure(
"@AdHoc",
String.format(
"select * from %s where pkey >= 100000 order by pkey limit 100;",
TableHelper.getTableName(t)));
assert (cr.getStatus() == ClientResponse.SUCCESS);
VoltTable result = cr.getResults()[0];
result.resetRowPosition();
while (result.advanceRow()) {
t2.add(result);
}
return t2;
}
static Topology getCluterTopology(Client client) throws Exception {
int hosts = -1;
int sitesPerHost = -1;
int k = -1;
VoltTable result = client.callProcedure("@SystemInformation", "DEPLOYMENT").getResults()[0];
result.resetRowPosition();
while (result.advanceRow()) {
String key = result.getString(0);
String value = result.getString(1);
if (key.equals("hostcount")) {
hosts = Integer.parseInt(value);
}
if (key.equals("sitesperhost")) {
sitesPerHost = Integer.parseInt(value);
}
if (key.equals("kfactor")) {
k = Integer.parseInt(value);
}
}
return new Topology(hosts, hosts * sitesPerHost, (hosts * sitesPerHost) / (k + 1));
}
/**
* Cache the current statistics.
*
* @param time
*/
private void statsTick(long time) {
/*
* grab the table statistics from ee and put it into the statistics
* agent.
*/
if (m_tableStats != null) {
CatalogMap<Table> tables = m_context.database.getTables();
int[] tableIds = new int[tables.size()];
int i = 0;
for (Table table : tables) {
tableIds[i++] = table.getRelativeIndex();
}
// data to aggregate
long tupleCount = 0;
int tupleDataMem = 0;
int tupleAllocatedMem = 0;
int indexMem = 0;
int stringMem = 0;
// update table stats
final VoltTable[] s1 = m_ee.getStats(SysProcSelector.TABLE, tableIds, false, time);
if (s1 != null) {
VoltTable stats = s1[0];
assert (stats != null);
// rollup the table memory stats for this site
while (stats.advanceRow()) {
tupleCount += stats.getLong(7);
tupleAllocatedMem += (int) stats.getLong(8);
tupleDataMem += (int) stats.getLong(9);
stringMem += (int) stats.getLong(10);
}
stats.resetRowPosition();
m_tableStats.setStatsTable(stats);
}
// update index stats
final VoltTable[] s2 = m_ee.getStats(SysProcSelector.INDEX, tableIds, false, time);
if ((s2 != null) && (s2.length > 0)) {
VoltTable stats = s2[0];
assert (stats != null);
// rollup the index memory stats for this site
while (stats.advanceRow()) {
indexMem += stats.getLong(10);
}
stats.resetRowPosition();
m_indexStats.setStatsTable(stats);
}
// update the rolled up memory statistics
if (m_memStats != null) {
m_memStats.eeUpdateMemStats(
m_siteId,
tupleCount,
tupleDataMem,
tupleAllocatedMem,
indexMem,
stringMem,
m_ee.getThreadLocalPoolAllocations());
}
}
}