protected Site getSite(int site_id) {
assertNotNull(catalog);
Cluster catalog_clus = this.getCluster();
Site catalog_site = catalog_clus.getSites().get("id", site_id);
assert (catalog_site != null) : "Failed to retrieve Site #" + site_id + " from catalog";
return (catalog_site);
}
@Override
public void loadTable(
long txnId, String clusterName, String databaseName, String tableName, VoltTable data)
throws VoltAbortException {
Cluster cluster = m_context.cluster;
if (cluster == null) {
throw new VoltAbortException("cluster '" + clusterName + "' does not exist");
}
Database db = cluster.getDatabases().get(databaseName);
if (db == null) {
throw new VoltAbortException(
"database '" + databaseName + "' does not exist in cluster " + clusterName);
}
Table table = db.getTables().getIgnoreCase(tableName);
if (table == null) {
throw new VoltAbortException(
"table '"
+ tableName
+ "' does not exist in database "
+ clusterName
+ "."
+ databaseName);
}
loadTable(txnId, table.getRelativeIndex(), data);
}
public MockVoltDB(int clientPort, int adminPort, int httpPort, int drPort) {
try {
JSONObject obj = new JSONObject();
JSONArray jsonArray = new JSONArray();
jsonArray.put("127.0.0.1");
obj.put("interfaces", jsonArray);
obj.put("clientPort", clientPort);
obj.put("adminPort", adminPort);
obj.put("httpPort", httpPort);
obj.put("drPort", drPort);
m_localMetadata = obj.toString(4);
m_catalog = new Catalog();
m_catalog.execute("add / clusters " + m_clusterName);
m_catalog.execute(
"add "
+ m_catalog.getClusters().get(m_clusterName).getPath()
+ " databases "
+ m_databaseName);
Cluster cluster = m_catalog.getClusters().get(m_clusterName);
// Set a sane default for TestMessaging (at least)
cluster.setHeartbeattimeout(10000);
assert (cluster != null);
try {
m_hostMessenger.start();
} catch (Exception e) {
throw new RuntimeException(e);
}
VoltZK.createPersistentZKNodes(m_hostMessenger.getZK());
m_hostMessenger
.getZK()
.create(
VoltZK.cluster_metadata + "/" + m_hostMessenger.getHostId(),
getLocalMetadata().getBytes("UTF-8"),
Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL);
m_hostMessenger.generateMailboxId(
m_hostMessenger.getHSIdForLocalSite(HostMessenger.STATS_SITE_ID));
m_statsAgent = new StatsAgent();
m_statsAgent.registerMailbox(
m_hostMessenger, m_hostMessenger.getHSIdForLocalSite(HostMessenger.STATS_SITE_ID));
for (MailboxType type : MailboxType.values()) {
m_mailboxMap.put(type, new LinkedList<MailboxNodeContent>());
}
m_mailboxMap
.get(MailboxType.StatsAgent)
.add(
new MailboxNodeContent(
m_hostMessenger.getHSIdForLocalSite(HostMessenger.STATS_SITE_ID), null));
m_siteTracker = new SiteTracker(m_hostId, m_mailboxMap);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
protected void setUp() throws Exception {
aide =
new PlannerTestAideDeCamp(
TestColumnTrimmingPlans.class.getResource("testplans-trimming-ddl.sql"),
"testtrimmingplans");
// Set all tables to non-replicated.
Cluster cluster = aide.getCatalog().getClusters().get("cluster");
CatalogMap<Table> tmap = cluster.getDatabases().get("database").getTables();
for (Table t : tmap) {
t.setIsreplicated(false);
}
}
/** Generate the HTML catalog report from a newly compiled VoltDB catalog */
public static String report(Catalog catalog, ArrayList<Feedback> warnings) throws IOException {
// asynchronously get platform properties
new Thread() {
@Override
public void run() {
PlatformProperties.getPlatformProperties();
}
}.start();
URL url = Resources.getResource(ReportMaker.class, "template.html");
String contents = Resources.toString(url, Charsets.UTF_8);
Cluster cluster = catalog.getClusters().get("cluster");
assert (cluster != null);
Database db = cluster.getDatabases().get("database");
assert (db != null);
String statsData = getStatsHTML(db, warnings);
contents = contents.replace("##STATS##", statsData);
String schemaData = generateSchemaTable(db.getTables(), db.getConnectors());
contents = contents.replace("##SCHEMA##", schemaData);
String procData = generateProceduresTable(db.getProcedures());
contents = contents.replace("##PROCS##", procData);
DatabaseSizes sizes = CatalogSizing.getCatalogSizes(db);
String sizeData = generateSizeTable(sizes);
contents = contents.replace("##SIZES##", sizeData);
String sizeSummary = generateSizeSummary(sizes);
contents = contents.replace("##SIZESUMMARY##", sizeSummary);
String platformData = PlatformProperties.getPlatformProperties().toHTML();
contents = contents.replace("##PLATFORM##", platformData);
contents = contents.replace("##VERSION##", VoltDB.instance().getVersionString());
DateFormat df = new SimpleDateFormat("d MMM yyyy HH:mm:ss z");
contents = contents.replace("##TIMESTAMP##", df.format(m_timestamp));
String msg =
Encoder.hexEncode(VoltDB.instance().getVersionString() + "," + System.currentTimeMillis());
contents = contents.replace("get.py?a=KEY&", String.format("get.py?a=%s&", msg));
return contents;
}
/**
* Add fake partitions to the loaded catalog Assuming that there is one partition per site
*
* @param num_partitions
*/
protected void addPartitions(int num_partitions) throws Exception {
// HACK! If we already have this many partitions in the catalog, then we won't recreate it
// This fixes problems where we need to reference the same catalog objects in multiple test
// cases
if (CatalogUtil.getNumberOfPartitions(catalog_db) != num_partitions) {
ClusterConfiguration cc = new ClusterConfiguration();
for (Integer i = 0; i < num_partitions; i++) {
cc.addPartition("localhost", 0, i);
// System.err.println("[" + i + "] " + Arrays.toString(triplets.lastElement()));
} // FOR
catalog = FixCatalog.addHostInfo(catalog, cc);
this.init(this.last_type, catalog);
}
Cluster cluster = CatalogUtil.getCluster(catalog_db);
assertEquals(num_partitions, cluster.getNum_partitions());
assertEquals(num_partitions, CatalogUtil.getNumberOfPartitions(cluster));
}
protected void initializeCluster(int num_hosts, int num_sites, int num_partitions)
throws Exception {
// HACK! If we already have this many partitions in the catalog, then we won't recreate it
// This fixes problems where we need to reference the same catalog objects in multiple test
// cases
if (CatalogUtil.getNumberOfHosts(catalog_db) != num_hosts
|| CatalogUtil.getNumberOfSites(catalog_db) != (num_hosts * num_sites)
|| CatalogUtil.getNumberOfPartitions(catalog_db)
!= (num_hosts * num_sites * num_partitions)) {
catalog = FixCatalog.addHostInfo(catalog, "localhost", num_hosts, num_sites, num_partitions);
this.init(this.last_type, catalog);
}
Cluster cluster = CatalogUtil.getCluster(catalog_db);
assertEquals(num_hosts, CatalogUtil.getNumberOfHosts(catalog_db));
assertEquals((num_hosts * num_sites), CatalogUtil.getNumberOfSites(catalog_db));
assertEquals(
(num_hosts * num_sites * num_partitions), CatalogUtil.getNumberOfPartitions(cluster));
assertEquals((num_hosts * num_sites * num_partitions), cluster.getNum_partitions());
}
/**
* Initialize planner with physical schema info and a reference to HSQLDB parser.
*
* @param sql Literal SQL statement to parse
* @param stmtName The name of the statement for logging/debugging
* @param procName The name of the proc for logging/debugging
* @param catalogCluster Catalog info about the physical layout of the cluster.
* @param catalogDb Catalog info about schema, metadata and procedures.
* @param partitioning Describes the specified and inferred partition context.
* @param HSQL HSQLInterface pointer used for parsing SQL into XML.
* @param estimates
* @param suppressDebugOutput
* @param maxTablesPerJoin
* @param costModel The current cost model to evaluate plans with.
* @param paramHints
* @param joinOrder
*/
public QueryPlanner(
String sql,
String stmtName,
String procName,
Cluster catalogCluster,
Database catalogDb,
StatementPartitioning partitioning,
HSQLInterface HSQL,
DatabaseEstimates estimates,
boolean suppressDebugOutput,
int maxTablesPerJoin,
AbstractCostModel costModel,
ScalarValueHints[] paramHints,
String joinOrder,
DeterminismMode detMode) {
assert (sql != null);
assert (stmtName != null);
assert (procName != null);
assert (HSQL != null);
assert (catalogCluster != null);
assert (catalogDb != null);
assert (costModel != null);
assert (catalogDb.getCatalog() == catalogCluster.getCatalog());
assert (detMode != null);
m_sql = sql;
m_stmtName = stmtName;
m_procName = procName;
m_HSQL = HSQL;
m_db = catalogDb;
m_cluster = catalogCluster;
m_estimates = estimates;
m_partitioning = partitioning;
m_maxTablesPerJoin = maxTablesPerJoin;
m_costModel = costModel;
m_paramHints = paramHints;
m_joinOrder = joinOrder;
m_detMode = detMode;
m_planSelector =
new PlanSelector(
m_cluster,
m_db,
m_estimates,
m_stmtName,
m_procName,
m_sql,
m_costModel,
m_paramHints,
m_detMode,
suppressDebugOutput);
m_isUpsert = false;
}
@Test
public void testLoader() {
VoltDB.Configuration configuration = new VoltDB.Configuration();
configuration.m_noLoadLibVOLTDB = true;
MockVoltDB mockvolt = new MockVoltDB();
VoltDB.ignoreCrash = true;
VoltDB.replaceVoltDBInstanceForTest(mockvolt);
mockvolt.m_noLoadLib = true;
assertFalse(EELibraryLoader.loadExecutionEngineLibrary(false));
assertFalse(VoltDB.wasCrashCalled);
boolean threw = false;
try {
assertFalse(EELibraryLoader.loadExecutionEngineLibrary(true));
} catch (AssertionError ae) {
threw = true;
}
assertTrue(threw);
assertTrue(VoltDB.wasCrashCalled);
VoltDB.wasCrashCalled = false;
VoltDB.initialize(configuration);
assertFalse(EELibraryLoader.loadExecutionEngineLibrary(true));
assertFalse(VoltDB.wasCrashCalled);
// Now test SUCCESS case
configuration = new VoltDB.Configuration();
VoltDBInterface mockitovolt = mock(VoltDBInterface.class);
VoltDBInterface realvolt = new RealVoltDB();
when(mockitovolt.getEELibraryVersionString()).thenReturn(realvolt.getEELibraryVersionString());
CatalogContext catContext = mock(CatalogContext.class);
Cluster cluster = mock(Cluster.class);
when(cluster.getVoltroot()).thenReturn(System.getProperty("java.io.tmpdir"));
when(catContext.getCluster()).thenReturn(cluster);
when(mockitovolt.getCatalogContext()).thenReturn(catContext);
VoltDB.replaceVoltDBInstanceForTest(mockitovolt);
VoltDB.initialize(configuration);
assertTrue(EELibraryLoader.loadExecutionEngineLibrary(true));
}
@SuppressWarnings("deprecation")
@Override
public void setUp() throws IOException, InterruptedException {
VoltDB.instance().readBuildInfo("Test");
// compile a catalog
String testDir = BuildDirectoryUtils.getBuildDirectoryPath();
String catalogJar = testDir + File.separator + JAR;
TPCCProjectBuilder pb = new TPCCProjectBuilder();
pb.addDefaultSchema();
pb.addDefaultPartitioning();
pb.addProcedures(MultiSiteSelect.class, InsertNewOrder.class);
pb.compile(catalogJar, 2, 0);
// load a catalog
byte[] bytes = CatalogUtil.toBytes(new File(catalogJar));
String serializedCatalog = CatalogUtil.loadCatalogFromJar(bytes, null);
// create the catalog (that will be passed to the ClientInterface
catalog = new Catalog();
catalog.execute(serializedCatalog);
// update the catalog with the data from the deployment file
String pathToDeployment = pb.getPathToDeployment();
assertTrue(CatalogUtil.compileDeploymentAndGetCRC(catalog, pathToDeployment, true) >= 0);
cluster = catalog.getClusters().get("cluster");
CatalogMap<Procedure> procedures = cluster.getDatabases().get("database").getProcedures();
Procedure insertProc = procedures.get("InsertNewOrder");
assert (insertProc != null);
selectProc = procedures.get("MultiSiteSelect");
assert (selectProc != null);
// Each EE needs its own thread for correct initialization.
final AtomicReference<ExecutionEngine> site1Reference = new AtomicReference<ExecutionEngine>();
final byte configBytes[] = LegacyHashinator.getConfigureBytes(2);
Thread site1Thread =
new Thread() {
@Override
public void run() {
site1Reference.set(
new ExecutionEngineJNI(
cluster.getRelativeIndex(),
1,
0,
0,
"",
100,
HashinatorType.LEGACY,
configBytes));
}
};
site1Thread.start();
site1Thread.join();
final AtomicReference<ExecutionEngine> site2Reference = new AtomicReference<ExecutionEngine>();
Thread site2Thread =
new Thread() {
@Override
public void run() {
site2Reference.set(
new ExecutionEngineJNI(
cluster.getRelativeIndex(),
2,
1,
0,
"",
100,
HashinatorType.LEGACY,
configBytes));
}
};
site2Thread.start();
site2Thread.join();
// create two EEs
site1 = new ExecutionSite(0); // site 0
ee1 = site1Reference.get();
ee1.loadCatalog(0, catalog.serialize());
site2 = new ExecutionSite(1); // site 1
ee2 = site2Reference.get();
ee2.loadCatalog(0, catalog.serialize());
// cache some plan fragments
selectStmt = selectProc.getStatements().get("selectAll");
assert (selectStmt != null);
int i = 0;
// this kinda assumes the right order
for (PlanFragment f : selectStmt.getFragments()) {
if (i == 0) selectTopFrag = f;
else selectBottomFrag = f;
i++;
}
assert (selectTopFrag != null);
assert (selectBottomFrag != null);
if (selectTopFrag.getHasdependencies() == false) {
PlanFragment temp = selectTopFrag;
selectTopFrag = selectBottomFrag;
selectBottomFrag = temp;
}
// get the insert frag
Statement insertStmt = insertProc.getStatements().get("insert");
assert (insertStmt != null);
for (PlanFragment f : insertStmt.getFragments()) insertFrag = f;
// populate plan cache
ActivePlanRepository.clear();
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(selectBottomFrag),
Encoder.base64Decode(selectBottomFrag.getPlannodetree()));
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(selectTopFrag),
Encoder.base64Decode(selectTopFrag.getPlannodetree()));
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(insertFrag),
Encoder.base64Decode(insertFrag.getPlannodetree()));
// insert some data
ParameterSet params = ParameterSet.fromArrayNoCopy(1L, 1L, 1L);
VoltTable[] results =
ee2.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
null,
new ParameterSet[] {params},
1,
0,
42,
Long.MAX_VALUE);
assert (results.length == 1);
assert (results[0].asScalarLong() == 1L);
params = ParameterSet.fromArrayNoCopy(2L, 2L, 2L);
results =
ee1.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
null,
new ParameterSet[] {params},
2,
1,
42,
Long.MAX_VALUE);
assert (results.length == 1);
assert (results[0].asScalarLong() == 1L);
}
/**
* Initialize TheHashinator
*
* @param catalog A pointer to the catalog data structure.
*/
public static void initialize(Catalog catalog) {
Cluster cluster = catalog.getClusters().get("cluster");
partitionCount = cluster.getNum_partitions();
}
public CatalogContext(Catalog catalog, File pathToCatalogJar) {
// check the heck out of the given params in this immutable class
assert (catalog != null);
if (catalog == null) {
throw new RuntimeException("Can't create CatalogContext with null catalog.");
}
this.jarPath = pathToCatalogJar;
this.catalog = catalog;
this.cluster = CatalogUtil.getCluster(this.catalog);
this.database = CatalogUtil.getDatabase(this.catalog);
this.hosts = this.cluster.getHosts();
this.sites = this.cluster.getSites();
if (this.jarPath != null) {
this.catalogClassLoader = new JarClassLoader(this.jarPath.getAbsolutePath());
this.paramMappings =
ParametersUtil.getParameterMappingsSetFromJar(this.database, this.jarPath);
} else {
this.catalogClassLoader = null;
this.paramMappings = null;
}
// ------------------------------------------------------------
// PROCEDURES
// ------------------------------------------------------------
this.procedures = database.getProcedures();
this.proceduresArray = new Procedure[this.procedures.size() + 1];
for (Procedure proc : this.procedures) {
this.proceduresArray[proc.getId()] = proc;
if (proc.getSystemproc()) {
this.sysProcedures.add(proc);
} else if (proc.getMapreduce()) {
this.mrProcedures.add(proc);
} else {
this.regularProcedures.add(proc);
}
} // FOR
authSystem = new AuthSystem(database, cluster.getSecurityenabled());
siteTracker = null; // new SiteTracker(cluster.getSites());
// count nodes
this.numberOfHosts = cluster.getHosts().size();
// count exec sites
this.numberOfSites = cluster.getSites().size();
// ------------------------------------------------------------
// PARTITIONS
// ------------------------------------------------------------
this.numberOfPartitions = cluster.getNum_partitions();
this.partitions = new Partition[this.numberOfPartitions];
this.partitionIdArray = new Integer[this.numberOfPartitions];
this.partitionSingletons = new PartitionSet[this.numberOfPartitions];
this.partitionSiteXref = new int[this.numberOfPartitions];
for (Partition part : CatalogUtil.getAllPartitions(catalog)) {
int p = part.getId();
this.partitions[p] = part;
this.partitionIdArray[p] = Integer.valueOf(p);
this.partitionSingletons[p] = new PartitionSet(p);
this.partitionIdCollection.add(this.partitionIdArray[p]);
this.partitionSiteXref[part.getId()] = ((Site) part.getParent()).getId();
} // FOR
// ------------------------------------------------------------
// TABLES
// ------------------------------------------------------------
for (Table tbl : database.getTables()) {
if (tbl.getSystable()) {
sysTables.add(tbl);
} else if (tbl.getMapreduce()) {
mapReduceTables.add(tbl);
} else if (tbl.getMaterializer() != null) {
viewTables.add(tbl);
} else {
dataTables.add(tbl);
if (tbl.getIsreplicated()) {
replicatedTables.add(tbl);
}
if (tbl.getEvictable()) {
evictableTables.add(tbl);
}
}
} // FOR
// PLANFRAGMENTS
this.initPlanFragments();
}