private static void buildCatalog() throws IOException {
// build a real catalog
File cat = File.createTempFile("temp-log-reinitiator", "catalog");
cat.deleteOnExit();
VoltProjectBuilder builder = new VoltProjectBuilder();
String schema = "create table A (i integer not null, primary key (i));";
builder.addLiteralSchema(schema);
builder.addPartitionInfo("A", "i");
builder.addStmtProcedure("hello", "select * from A where i = ?", "A.i: 0");
if (!builder.compile(cat.getAbsolutePath())) {
throw new IOException();
}
byte[] bytes = CatalogUtil.toBytes(cat);
String serializedCat = CatalogUtil.loadCatalogFromJar(bytes, null);
assertNotNull(serializedCat);
Catalog catalog = new Catalog();
catalog.execute(serializedCat);
String deploymentPath = builder.getPathToDeployment();
CatalogUtil.compileDeploymentAndGetCRC(catalog, deploymentPath, true);
m_context = new CatalogContext(0, 0, catalog, bytes, 0, 0, 0);
TheHashinator.initialize(LegacyHashinator.class, LegacyHashinator.getConfigureBytes(3));
}
/*
* Multi-partition/non-replicated fragment with collector and aggregator.
*/
void addFragmentPair(
int index,
PlanFragment collectorFragment,
PlanFragment aggregatorFragment,
ByteBuffer params) {
assert (index >= 0);
assert (index < m_batchSize);
assert (collectorFragment != null);
assert (aggregatorFragment != null);
assert (collectorFragment.getHasdependencies() == false);
assert (aggregatorFragment.getHasdependencies() == true);
// frags with no deps are usually collector frags that go to all partitions
long distributedFragId = CatalogUtil.getUniqueIdForFragment(collectorFragment);
long localFragId = CatalogUtil.getUniqueIdForFragment(aggregatorFragment);
// if any frag is transactional, update this check
if (aggregatorFragment.getNontransactional() == true) {
m_localFragsAreNonTransactional = true;
}
int outputDepId = m_txnState.getNextDependencyId() | DtxnConstants.MULTIPARTITION_DEPENDENCY;
m_depsForLocalTask[index] = outputDepId;
// Add local and distributed fragments.
m_localTask.addFragment(localFragId, m_depsToResume[index], params);
m_distributedTask.addFragment(distributedFragId, outputDepId, params);
}
@Override
public void run() {
CatalogAndIds catalogStuff = null;
do {
try {
catalogStuff = CatalogUtil.getCatalogFromZK(m_rvdb.getHostMessenger().getZK());
} catch (org.apache.zookeeper_voltpatches.KeeperException.NoNodeException e) {
} catch (Exception e) {
VoltDB.crashLocalVoltDB(
"System was interrupted while waiting for a catalog.", false, null);
}
} while (catalogStuff == null || catalogStuff.catalogBytes.length == 0);
String serializedCatalog = null;
byte[] catalogJarBytes = catalogStuff.catalogBytes;
try {
Pair<InMemoryJarfile, String> loadResults =
CatalogUtil.loadAndUpgradeCatalogFromJar(catalogStuff.catalogBytes);
serializedCatalog = CatalogUtil.getSerializedCatalogStringFromJar(loadResults.getFirst());
catalogJarBytes = loadResults.getFirst().getFullJarBytes();
} catch (IOException e) {
VoltDB.crashLocalVoltDB("Unable to load catalog", false, e);
}
if ((serializedCatalog == null) || (serializedCatalog.length() == 0))
VoltDB.crashLocalVoltDB("Catalog loading failure", false, null);
/* N.B. node recovery requires discovering the current catalog version. */
Catalog catalog = new Catalog();
catalog.execute(serializedCatalog);
serializedCatalog = null;
// note if this fails it will print an error first
// This is where we compile real catalog and create runtime
// catalog context. To validate deployment we compile and create
// a starter context which uses a placeholder catalog.
String result = CatalogUtil.compileDeployment(catalog, m_deployment, false);
if (result != null) {
hostLog.fatal(result);
VoltDB.crashLocalVoltDB(result);
}
try {
m_rvdb.m_catalogContext =
new CatalogContext(
catalogStuff.txnId,
catalogStuff.uniqueId,
catalog,
catalogJarBytes,
// Our starter catalog has set the deployment stuff, just yoink it out for now
m_rvdb.m_catalogContext.getDeploymentBytes(),
catalogStuff.version,
-1);
} catch (Exception e) {
VoltDB.crashLocalVoltDB("Error agreeing on starting catalog version", true, e);
}
}
@Override
public void run() {
// if I'm the leader, send out the catalog
if (m_rvdb.m_myHostId == m_rvdb.m_hostIdWithStartupCatalog) {
try {
// If no catalog was supplied provide an empty one.
if (m_rvdb.m_pathToStartupCatalog == null) {
try {
File emptyJarFile = CatalogUtil.createTemporaryEmptyCatalogJarFile();
if (emptyJarFile == null) {
VoltDB.crashLocalVoltDB("Failed to generate empty catalog.");
}
m_rvdb.m_pathToStartupCatalog = emptyJarFile.getAbsolutePath();
} catch (IOException e) {
VoltDB.crashLocalVoltDB(
"I/O exception while creating empty catalog jar file.", false, e);
}
}
// Get the catalog bytes and byte count.
byte[] catalogBytes = readCatalog(m_rvdb.m_pathToStartupCatalog);
// Export needs a cluster global unique id for the initial catalog version
long catalogUniqueId =
UniqueIdGenerator.makeIdFromComponents(
System.currentTimeMillis(), 0, MpInitiator.MP_INIT_PID);
hostLog.debug(String.format("Sending %d catalog bytes", catalogBytes.length));
long catalogTxnId;
catalogTxnId = TxnEgo.makeZero(MpInitiator.MP_INIT_PID).getTxnId();
// Need to get the deployment bytes from the starter catalog context
byte[] deploymentBytes = m_rvdb.getCatalogContext().getDeploymentBytes();
// publish the catalog bytes to ZK
CatalogUtil.updateCatalogToZK(
m_rvdb.getHostMessenger().getZK(),
0,
catalogTxnId,
catalogUniqueId,
catalogBytes,
deploymentBytes);
} catch (IOException e) {
VoltDB.crashGlobalVoltDB("Unable to distribute catalog.", false, e);
} catch (org.apache.zookeeper_voltpatches.KeeperException e) {
VoltDB.crashGlobalVoltDB("Unable to publish catalog.", false, e);
} catch (InterruptedException e) {
VoltDB.crashGlobalVoltDB("Interrupted while publishing catalog.", false, e);
}
}
}
@Override
public String getColumnName(int columnIndex) {
if (m_columns == null) {
m_columns = CatalogUtil.getSortedCatalogItems(m_table.getColumns(), "index");
}
return m_columns.get(columnIndex).getTypeName();
}
/*
* Replicated fragment.
*/
void addFragment(int index, PlanFragment frag, ByteBuffer params) {
assert (index >= 0);
assert (index < m_batchSize);
assert (frag != null);
assert (frag.getHasdependencies() == false);
// if any frag is transactional, update this check
if (frag.getNontransactional() == true) m_localFragsAreNonTransactional = true;
long localFragId = CatalogUtil.getUniqueIdForFragment(frag);
m_depsForLocalTask[index] = -1;
// Add the local fragment data.
m_localTask.addFragment(localFragId, m_depsToResume[index], params);
}
protected VoltTable[] executeQueriesInABatch(List<QueuedSQL> batch, boolean isFinalSQL) {
final int batchSize = batch.size();
VoltTable[] results = null;
if (batchSize == 0) {
return new VoltTable[] {};
}
// IF THIS IS HSQL, RUN THE QUERIES DIRECTLY IN HSQL
if (getHsqlBackendIfExists() != null) {
results = new VoltTable[batchSize];
int i = 0;
for (QueuedSQL qs : batch) {
List<StmtParameter> sparams;
if (qs.stmt.catStmt != null) {
CatalogMap<StmtParameter> sparamsMap = qs.stmt.catStmt.getParameters();
sparams = CatalogUtil.getSortedCatalogItems(sparamsMap, "index");
} else {
assert (qs.stmt.plan != null);
// TODO: For now extracted ad hoc SQL parameters are discarded
qs.params = new ParameterSet();
sparams = new ArrayList<StmtParameter>();
}
results[i++] =
getHsqlBackendIfExists().runSQLWithSubstitutions(qs.stmt, qs.params, sparams);
}
} else if (m_catProc.getSinglepartition()) {
results = fastPath(batch);
} else {
results = slowPath(batch, isFinalSQL);
}
// check expectations
int i = 0;
for (QueuedSQL qs : batch) {
Expectation.check(m_procedureName, qs.stmt.sqlText, i, qs.expectation, results[i]);
i++;
}
// clear the queued sql list for the next call
batch.clear();
return results;
}
public static void initSQLStmt(SQLStmt stmt, Statement catStmt) {
stmt.catStmt = catStmt;
stmt.numFragGUIDs = catStmt.getFragments().size();
PlanFragment fragments[] = new PlanFragment[stmt.numFragGUIDs];
stmt.fragGUIDs = new long[stmt.numFragGUIDs];
int i = 0;
for (PlanFragment frag : stmt.catStmt.getFragments()) {
fragments[i] = frag;
stmt.fragGUIDs[i] = CatalogUtil.getUniqueIdForFragment(frag);
i++;
}
stmt.numStatementParamJavaTypes = stmt.catStmt.getParameters().size();
// StmtParameter parameters[] = new StmtParameter[stmt.numStatementParamJavaTypes];
stmt.statementParamJavaTypes = new byte[stmt.numStatementParamJavaTypes];
for (StmtParameter param : stmt.catStmt.getParameters()) {
// parameters[i] = param;
stmt.statementParamJavaTypes[param.getIndex()] = (byte) param.getJavatype();
i++;
}
}
protected boolean createSetupInternal(
String file_path,
String file_nonce,
long txnId,
Map<Integer, Long> partitionTransactionIds,
JSONObject jsData,
SystemProcedureExecutionContext context,
String hostname,
final VoltTable result,
Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
SiteTracker tracker,
long timestamp)
throws IOException {
assert (SnapshotSiteProcessor.ExecutionSitesCurrentlySnapshotting.isEmpty());
/*
* List of partitions to include if this snapshot is
* going to be deduped. Attempts to break up the work
* by seeding and RNG selecting
* a random replica to do the work. Will not work in failure
* cases, but we don't use dedupe when we want durability.
*/
List<Long> sitesToInclude = CSVSnapshotWritePlan.computeDedupedLocalSites(txnId, tracker);
// If there's no work to do on this host, just claim success and get out:
if (sitesToInclude.isEmpty() && !tracker.isFirstHost()) {
return false;
}
NativeSnapshotWritePlan.createFileBasedCompletionTasks(
file_path,
file_nonce,
txnId,
partitionTransactionIds,
context,
exportSequenceNumbers,
timestamp);
final List<Table> tables = SnapshotUtil.getTablesToSave(context.getDatabase());
final AtomicInteger numTables = new AtomicInteger(tables.size());
final SnapshotRegistry.Snapshot snapshotRecord =
SnapshotRegistry.startSnapshot(
txnId,
context.getHostId(),
file_path,
file_nonce,
SnapshotFormat.CSV,
tables.toArray(new Table[0]));
SnapshotDataTarget sdt = null;
boolean noTargetsCreated = true;
final ArrayList<SnapshotTableTask> partitionedSnapshotTasks =
new ArrayList<SnapshotTableTask>();
final ArrayList<SnapshotTableTask> replicatedSnapshotTasks = new ArrayList<SnapshotTableTask>();
for (final Table table : tables) {
/*
* For a deduped csv snapshot, only produce the replicated tables on the "leader"
* host.
*/
if (table.getIsreplicated() && !tracker.isFirstHost()) {
snapshotRecord.removeTable(table.getTypeName());
continue;
}
File saveFilePath = null;
saveFilePath =
SnapshotUtil.constructFileForTable(
table, file_path, file_nonce, SnapshotFormat.CSV, context.getHostId());
try {
sdt = new SimpleFileSnapshotDataTarget(saveFilePath, !table.getIsreplicated());
m_targets.add(sdt);
final Runnable onClose =
new TargetStatsClosure(sdt, table.getTypeName(), numTables, snapshotRecord);
sdt.setOnCloseHandler(onClose);
List<SnapshotDataFilter> filters = new ArrayList<SnapshotDataFilter>();
filters.add(new CSVSnapshotFilter(CatalogUtil.getVoltTable(table), ',', null));
final SnapshotTableTask task =
new SnapshotTableTask(
table.getRelativeIndex(),
sdt,
filters.toArray(new SnapshotDataFilter[filters.size()]),
table.getIsreplicated(),
table.getTypeName());
if (table.getIsreplicated()) {
replicatedSnapshotTasks.add(task);
} else {
partitionedSnapshotTasks.add(task);
}
noTargetsCreated = false;
result.addRow(context.getHostId(), hostname, table.getTypeName(), "SUCCESS", "");
} catch (IOException ex) {
handleTargetCreationError(
sdt, context, file_nonce, hostname, table.getTypeName(), ex, result);
}
}
if (noTargetsCreated) {
SnapshotRegistry.discardSnapshot(snapshotRecord);
}
// CSV snapshots do the partitioned work only on the specified sites for de-duping,
// but since we've pre-filtered the replicated task list to only contain entries on
// one node, we can go ahead and distribute them across all of the sites on that node.
placePartitionedTasks(partitionedSnapshotTasks, sitesToInclude);
placeReplicatedTasks(replicatedSnapshotTasks, tracker.getSitesForHost(context.getHostId()));
return noTargetsCreated;
}
@Override
public void run() {
if (!m_isRejoin && !m_config.m_isRejoinTest && !m_rvdb.m_joining) {
String snapshotPath = null;
if (m_rvdb
.m_catalogContext
.cluster
.getDatabases()
.get("database")
.getSnapshotschedule()
.get("default")
!= null) {
snapshotPath =
m_rvdb
.m_catalogContext
.cluster
.getDatabases()
.get("database")
.getSnapshotschedule()
.get("default")
.getPath();
}
int[] allPartitions = new int[m_rvdb.m_configuredNumberOfPartitions];
for (int ii = 0; ii < allPartitions.length; ii++) {
allPartitions[ii] = ii;
}
org.voltdb.catalog.CommandLog cl =
m_rvdb.m_catalogContext.cluster.getLogconfig().get("log");
try {
m_rvdb.m_restoreAgent =
new RestoreAgent(
m_rvdb.m_messenger,
m_rvdb.getSnapshotCompletionMonitor(),
m_rvdb,
m_config.m_startAction,
cl.getEnabled(),
cl.getLogpath(),
cl.getInternalsnapshotpath(),
snapshotPath,
allPartitions,
CatalogUtil.getVoltDbRoot(m_deployment.getPaths()).getAbsolutePath());
} catch (IOException e) {
VoltDB.crashLocalVoltDB("Unable to construct the RestoreAgent", true, e);
}
m_rvdb.m_globalServiceElector.registerService(m_rvdb.m_restoreAgent);
m_rvdb.m_restoreAgent.setCatalogContext(m_rvdb.m_catalogContext);
// Generate plans and get (hostID, catalogPath) pair
Pair<Integer, String> catalog = m_rvdb.m_restoreAgent.findRestoreCatalog();
// if the restore agent found a catalog, set the following info
// so the right node can send it out to the others
if (catalog != null) {
// Make sure the catalog corresponds to the current server version.
// Prevent automatic upgrades by rejecting mismatched versions.
int hostId = catalog.getFirst().intValue();
String catalogPath = catalog.getSecond();
// Perform a version check when the catalog jar is available
// on the current host.
// Check that this host is the one providing the catalog.
if (m_rvdb.m_myHostId == hostId) {
try {
byte[] catalogBytes = readCatalog(catalogPath);
InMemoryJarfile inMemoryJar = CatalogUtil.loadInMemoryJarFile(catalogBytes);
// This call pre-checks and returns the build info/version.
String[] buildInfo = CatalogUtil.getBuildInfoFromJar(inMemoryJar);
String catalogVersion = buildInfo[0];
String serverVersion = m_rvdb.getVersionString();
if (!catalogVersion.equals(serverVersion)) {
VoltDB.crashLocalVoltDB(
String.format(
"Unable to load version %s catalog \"%s\" "
+ "from snapshot into a version %s server.",
catalogVersion, catalogPath, serverVersion),
false,
null);
}
} catch (IOException e) {
// Make it non-fatal with no check performed.
hostLog.warn(
String.format(
"Unable to load catalog for version check due to exception: %s.",
e.getMessage()));
}
}
hostLog.debug("Found catalog to load on host " + hostId + ": " + catalogPath);
m_rvdb.m_hostIdWithStartupCatalog = hostId;
assert (m_rvdb.m_hostIdWithStartupCatalog >= 0);
m_rvdb.m_pathToStartupCatalog = catalogPath;
assert (m_rvdb.m_pathToStartupCatalog != null);
}
}
}
@SuppressWarnings("deprecation")
@Override
public void setUp() throws IOException, InterruptedException {
VoltDB.instance().readBuildInfo("Test");
// compile a catalog
String testDir = BuildDirectoryUtils.getBuildDirectoryPath();
String catalogJar = testDir + File.separator + JAR;
TPCCProjectBuilder pb = new TPCCProjectBuilder();
pb.addDefaultSchema();
pb.addDefaultPartitioning();
pb.addProcedures(MultiSiteSelect.class, InsertNewOrder.class);
pb.compile(catalogJar, 2, 0);
// load a catalog
byte[] bytes = CatalogUtil.toBytes(new File(catalogJar));
String serializedCatalog = CatalogUtil.loadCatalogFromJar(bytes, null);
// create the catalog (that will be passed to the ClientInterface
catalog = new Catalog();
catalog.execute(serializedCatalog);
// update the catalog with the data from the deployment file
String pathToDeployment = pb.getPathToDeployment();
assertTrue(CatalogUtil.compileDeploymentAndGetCRC(catalog, pathToDeployment, true) >= 0);
cluster = catalog.getClusters().get("cluster");
CatalogMap<Procedure> procedures = cluster.getDatabases().get("database").getProcedures();
Procedure insertProc = procedures.get("InsertNewOrder");
assert (insertProc != null);
selectProc = procedures.get("MultiSiteSelect");
assert (selectProc != null);
// Each EE needs its own thread for correct initialization.
final AtomicReference<ExecutionEngine> site1Reference = new AtomicReference<ExecutionEngine>();
final byte configBytes[] = LegacyHashinator.getConfigureBytes(2);
Thread site1Thread =
new Thread() {
@Override
public void run() {
site1Reference.set(
new ExecutionEngineJNI(
cluster.getRelativeIndex(),
1,
0,
0,
"",
100,
HashinatorType.LEGACY,
configBytes));
}
};
site1Thread.start();
site1Thread.join();
final AtomicReference<ExecutionEngine> site2Reference = new AtomicReference<ExecutionEngine>();
Thread site2Thread =
new Thread() {
@Override
public void run() {
site2Reference.set(
new ExecutionEngineJNI(
cluster.getRelativeIndex(),
2,
1,
0,
"",
100,
HashinatorType.LEGACY,
configBytes));
}
};
site2Thread.start();
site2Thread.join();
// create two EEs
site1 = new ExecutionSite(0); // site 0
ee1 = site1Reference.get();
ee1.loadCatalog(0, catalog.serialize());
site2 = new ExecutionSite(1); // site 1
ee2 = site2Reference.get();
ee2.loadCatalog(0, catalog.serialize());
// cache some plan fragments
selectStmt = selectProc.getStatements().get("selectAll");
assert (selectStmt != null);
int i = 0;
// this kinda assumes the right order
for (PlanFragment f : selectStmt.getFragments()) {
if (i == 0) selectTopFrag = f;
else selectBottomFrag = f;
i++;
}
assert (selectTopFrag != null);
assert (selectBottomFrag != null);
if (selectTopFrag.getHasdependencies() == false) {
PlanFragment temp = selectTopFrag;
selectTopFrag = selectBottomFrag;
selectBottomFrag = temp;
}
// get the insert frag
Statement insertStmt = insertProc.getStatements().get("insert");
assert (insertStmt != null);
for (PlanFragment f : insertStmt.getFragments()) insertFrag = f;
// populate plan cache
ActivePlanRepository.clear();
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(selectBottomFrag),
Encoder.base64Decode(selectBottomFrag.getPlannodetree()));
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(selectTopFrag),
Encoder.base64Decode(selectTopFrag.getPlannodetree()));
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(insertFrag),
Encoder.base64Decode(insertFrag.getPlannodetree()));
// insert some data
ParameterSet params = ParameterSet.fromArrayNoCopy(1L, 1L, 1L);
VoltTable[] results =
ee2.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
null,
new ParameterSet[] {params},
1,
0,
42,
Long.MAX_VALUE);
assert (results.length == 1);
assert (results[0].asScalarLong() == 1L);
params = ParameterSet.fromArrayNoCopy(2L, 2L, 2L);
results =
ee1.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
null,
new ParameterSet[] {params},
2,
1,
42,
Long.MAX_VALUE);
assert (results.length == 1);
assert (results[0].asScalarLong() == 1L);
}
static String genrateStatementRow(Procedure procedure, Statement statement) {
StringBuilder sb = new StringBuilder();
sb.append(" <tr class='primaryrow2'>");
// name column
String anchor = (procedure.getTypeName() + "-" + statement.getTypeName()).toLowerCase();
sb.append(
"<td style='white-space: nowrap'><i id='p-"
+ anchor
+ "--icon' class='icon-chevron-right'></i> <a href='#' id='p-");
sb.append(anchor).append("' class='togglex'>");
sb.append(statement.getTypeName());
sb.append("</a></td>");
// sql column
sb.append("<td><tt>");
sb.append(statement.getSqltext());
sb.append("</td></tt>");
// params column
sb.append("<td>");
List<StmtParameter> params =
CatalogUtil.getSortedCatalogItems(statement.getParameters(), "index");
List<String> paramTypes = new ArrayList<String>();
for (StmtParameter param : params) {
paramTypes.add(VoltType.get((byte) param.getJavatype()).name());
}
if (paramTypes.size() == 0) {
sb.append("<i>None</i>");
}
sb.append(StringUtils.join(paramTypes, ", "));
sb.append("</td>");
// r/w column
sb.append("<td>");
if (statement.getReadonly()) {
tag(sb, "success", "Read");
} else {
tag(sb, "warning", "Write");
}
sb.append("</td>");
// attributes
sb.append("<td>");
if (!statement.getIscontentdeterministic() || !statement.getIsorderdeterministic()) {
tag(sb, "inverse", "Determinism");
}
if (statement.getSeqscancount() > 0) {
tag(sb, "important", "Scans");
}
sb.append("</td>");
sb.append("</tr>\n");
// BUILD THE DROPDOWN FOR THE PLAN/DETAIL TABLE
sb.append(
"<tr class='dropdown2'><td colspan='5' id='p-"
+ procedure.getTypeName().toLowerCase()
+ "-"
+ statement.getTypeName().toLowerCase()
+ "--dropdown'>\n");
sb.append("<div class='well well-small'><h4>Explain Plan:</h4>\n");
StatementAnnotation annotation = (StatementAnnotation) statement.getAnnotation();
if (annotation != null) {
String plan = annotation.explainPlan;
plan = plan.replace("\n", "<br/>");
plan = plan.replace(" ", " ");
for (Table t : annotation.tablesRead) {
String name = t.getTypeName().toUpperCase();
String link = "\"<a href='#s-" + t.getTypeName() + "'>" + name + "</a>\"";
plan = plan.replace("\"" + name + "\"", link);
}
for (Table t : annotation.tablesUpdated) {
String name = t.getTypeName().toUpperCase();
String link = "\"<a href='#s-" + t.getTypeName() + "'>" + name + "</a>\"";
plan = plan.replace("\"" + name + "\"", link);
}
for (Index i : annotation.indexesUsed) {
Table t = (Table) i.getParent();
String name = i.getTypeName().toUpperCase();
String link =
"\"<a href='#s-" + t.getTypeName() + "-" + i.getTypeName() + "'>" + name + "</a>\"";
plan = plan.replace("\"" + name + "\"", link);
}
sb.append("<tt>").append(plan).append("</tt>");
} else {
sb.append("<i>No SQL explain plan found.</i>\n");
}
sb.append("</div>\n");
sb.append("</td></tr>\n");
return sb.toString();
}
public ClientResponseImpl call(Object... paramListIn) {
// verify per-txn state has been reset
assert (m_statusCode == ClientResponse.UNINITIALIZED_APP_STATUS_CODE);
assert (m_statusString == null);
assert (m_cachedRNG == null);
// reset the hash of results
m_inputCRC.reset();
// use local var to avoid warnings about reassigning method argument
Object[] paramList = paramListIn;
ClientResponseImpl retval = null;
// assert no sql is queued
assert (m_batch.size() == 0);
try {
m_statsCollector.beginProcedure();
byte status = ClientResponse.SUCCESS;
VoltTable[] results = null;
// inject sysproc execution context as the first parameter.
if (isSystemProcedure()) {
final Object[] combinedParams = new Object[paramList.length + 1];
combinedParams[0] = m_systemProcedureContext;
for (int i = 0; i < paramList.length; ++i) combinedParams[i + 1] = paramList[i];
// swap the lists.
paramList = combinedParams;
}
if (paramList.length != m_paramTypes.length) {
m_statsCollector.endProcedure(false, true, null, null);
String msg =
"PROCEDURE "
+ m_procedureName
+ " EXPECTS "
+ String.valueOf(m_paramTypes.length)
+ " PARAMS, BUT RECEIVED "
+ String.valueOf(paramList.length);
status = ClientResponse.GRACEFUL_FAILURE;
return getErrorResponse(status, msg, null);
}
for (int i = 0; i < m_paramTypes.length; i++) {
try {
paramList[i] =
ParameterConverter.tryToMakeCompatible(
m_paramTypeIsPrimitive[i],
m_paramTypeIsArray[i],
m_paramTypes[i],
m_paramTypeComponentType[i],
paramList[i]);
} catch (Exception e) {
m_statsCollector.endProcedure(false, true, null, null);
String msg =
"PROCEDURE "
+ m_procedureName
+ " TYPE ERROR FOR PARAMETER "
+ i
+ ": "
+ e.toString();
status = ClientResponse.GRACEFUL_FAILURE;
return getErrorResponse(status, msg, null);
}
}
boolean error = false;
boolean abort = false;
// run a regular java class
if (m_catProc.getHasjava()) {
try {
if (log.isTraceEnabled()) {
log.trace(
"invoking... procMethod="
+ m_procMethod.getName()
+ ", class="
+ getClass().getName());
}
try {
Object rawResult = m_procMethod.invoke(m_procedure, paramList);
results = getResultsFromRawResults(rawResult);
} catch (IllegalAccessException e) {
// If reflection fails, invoke the same error handling that other exceptions do
throw new InvocationTargetException(e);
}
log.trace("invoked");
} catch (InvocationTargetException itex) {
// itex.printStackTrace();
Throwable ex = itex.getCause();
if (ex instanceof VoltAbortException && !(ex instanceof EEException)) {
abort = true;
} else {
error = true;
}
if (ex instanceof Error) {
m_statsCollector.endProcedure(false, true, null, null);
throw (Error) ex;
}
retval = getErrorResponse(ex);
}
}
// single statement only work
// (this could be made faster, but with less code re-use)
else {
assert (m_catProc.getStatements().size() == 1);
try {
m_cachedSingleStmt.params = getCleanParams(m_cachedSingleStmt.stmt, paramList);
if (getHsqlBackendIfExists() != null) {
// HSQL handling
CatalogMap<StmtParameter> sparamsMap = m_cachedSingleStmt.stmt.catStmt.getParameters();
List<StmtParameter> sparams = CatalogUtil.getSortedCatalogItems(sparamsMap, "index");
VoltTable table =
getHsqlBackendIfExists()
.runSQLWithSubstitutions(
m_cachedSingleStmt.stmt, m_cachedSingleStmt.params, sparams);
results = new VoltTable[] {table};
} else {
m_batch.add(m_cachedSingleStmt);
results = voltExecuteSQL(true);
}
} catch (SerializableException ex) {
retval = getErrorResponse(ex);
}
}
// Record statistics for procedure call.
StoredProcedureInvocation invoc = (m_txnState != null ? m_txnState.getInvocation() : null);
ParameterSet paramSet = (invoc != null ? invoc.getParams() : null);
m_statsCollector.endProcedure(abort, error, results, paramSet);
// don't leave empty handed
if (results == null) results = new VoltTable[0];
if (retval == null)
retval = new ClientResponseImpl(status, m_statusCode, m_statusString, results, null);
int hash = (int) m_inputCRC.getValue();
if ((retval.getStatus() == ClientResponse.SUCCESS) && (hash != 0)) {
retval.setHash(hash);
}
if ((m_txnState != null)
&& // may be null for tests
(m_txnState.getInvocation() != null)
&& (m_txnState.getInvocation().getType() == ProcedureInvocationType.REPLICATED)) {
retval.convertResultsToHashForDeterminism();
}
} finally {
// finally at the call(..) scope to ensure params can be
// garbage collected and that the queue will be empty for
// the next call
m_batch.clear();
// reset other per-txn state
m_txnState = null;
m_statusCode = ClientResponse.UNINITIALIZED_APP_STATUS_CODE;
m_statusString = null;
m_cachedRNG = null;
m_cachedSingleStmt.params = null;
m_cachedSingleStmt.expectation = null;
m_seenFinalBatch = false;
}
return retval;
}
static String genrateIndexRow(Table table, Index index) {
StringBuilder sb = new StringBuilder();
sb.append(" <tr class='primaryrow2'>");
// name column
String anchor = (table.getTypeName() + "-" + index.getTypeName()).toLowerCase();
sb.append(
"<td style='white-space: nowrap'><i id='s-"
+ anchor
+ "--icon' class='icon-chevron-right'></i> <a href='#' id='s-");
sb.append(anchor).append("' class='togglex'>");
sb.append(index.getTypeName());
sb.append("</a></td>");
// type column
sb.append("<td>");
sb.append(IndexType.get(index.getType()).toString());
sb.append("</td>");
// columns column
sb.append("<td>");
List<ColumnRef> cols = CatalogUtil.getSortedCatalogItems(index.getColumns(), "index");
List<String> columnNames = new ArrayList<String>();
for (ColumnRef colRef : cols) {
columnNames.add(colRef.getColumn().getTypeName());
}
sb.append(StringUtils.join(columnNames, ", "));
sb.append("</td>");
// uniqueness column
sb.append("<td>");
if (index.getAssumeunique()) {
tag(sb, "important", "AssumeUnique");
} else if (index.getUnique()) {
tag(sb, "important", "Unique");
} else {
tag(sb, "info", "Nonunique");
}
sb.append("</td>");
sb.append("</tr>\n");
// BUILD THE DROPDOWN FOR THE PLAN/DETAIL TABLE
sb.append(
"<tr class='dropdown2'><td colspan='5' id='s-"
+ table.getTypeName().toLowerCase()
+ "-"
+ index.getTypeName().toLowerCase()
+ "--dropdown'>\n");
IndexAnnotation annotation = (IndexAnnotation) index.getAnnotation();
if (annotation != null) {
if (annotation.proceduresThatUseThis.size() > 0) {
sb.append("<p>Used by procedures: ");
List<String> procs = new ArrayList<String>();
for (Procedure proc : annotation.proceduresThatUseThis) {
procs.add("<a href='#p-" + proc.getTypeName() + "'>" + proc.getTypeName() + "</a>");
}
sb.append(StringUtils.join(procs, ", "));
sb.append("</p>");
}
}
sb.append("</td></tr>\n");
return sb.toString();
}
static String generateProcedureRow(Procedure procedure) {
StringBuilder sb = new StringBuilder();
sb.append("<tr class='primaryrow'>");
// column 1: procedure name
String anchor = procedure.getTypeName().toLowerCase();
sb.append(
"<td style='white-space: nowrap'><i id='p-"
+ anchor
+ "--icon' class='icon-chevron-right'></i> <a href='#p-");
sb.append(anchor).append("' id='p-").append(anchor).append("' class='togglex'>");
sb.append(procedure.getTypeName());
sb.append("</a></td>");
// column 2: parameter types
sb.append("<td>");
List<ProcParameter> params =
CatalogUtil.getSortedCatalogItems(procedure.getParameters(), "index");
List<String> paramTypes = new ArrayList<String>();
for (ProcParameter param : params) {
String paramType = VoltType.get((byte) param.getType()).name();
if (param.getIsarray()) {
paramType += "[]";
}
paramTypes.add(paramType);
}
if (paramTypes.size() == 0) {
sb.append("<i>None</i>");
}
sb.append(StringUtils.join(paramTypes, ", "));
sb.append("</td>");
// column 3: partitioning
sb.append("<td>");
if (procedure.getSinglepartition()) {
tag(sb, "success", "Single");
} else {
tag(sb, "warning", "Multi");
}
sb.append("</td>");
// column 4: read/write
sb.append("<td>");
if (procedure.getReadonly()) {
tag(sb, "success", "Read");
} else {
tag(sb, "warning", "Write");
}
sb.append("</td>");
// column 5: access
sb.append("<td>");
List<String> groupNames = new ArrayList<String>();
for (GroupRef groupRef : procedure.getAuthgroups()) {
groupNames.add(groupRef.getGroup().getTypeName());
}
if (groupNames.size() == 0) {
sb.append("<i>None</i>");
}
sb.append(StringUtils.join(groupNames, ", "));
sb.append("</td>");
// column 6: attributes
sb.append("<td>");
if (procedure.getHasjava()) {
tag(sb, "info", "Java");
} else {
tag(sb, null, "Single-Stmt");
}
boolean isND = false;
int scanCount = 0;
for (Statement stmt : procedure.getStatements()) {
scanCount += stmt.getSeqscancount();
if (!stmt.getIscontentdeterministic() || !stmt.getIsorderdeterministic()) {
isND = false;
}
}
if (isND) {
tag(sb, "inverse", "Determinism");
}
if (scanCount > 0) {
tag(sb, "important", "Scans");
}
sb.append("</td>");
sb.append("</tr>\n");
// BUILD THE DROPDOWN FOR THE STATEMENT/DETAIL TABLE
sb.append(
"<tr class='tablesorter-childRow'><td class='invert' colspan='6' id='p-"
+ procedure.getTypeName().toLowerCase()
+ "--dropdown'>\n");
// output partitioning parameter info
if (procedure.getSinglepartition()) {
String pTable = procedure.getPartitioncolumn().getParent().getTypeName();
String pColumn = procedure.getPartitioncolumn().getTypeName();
int pIndex = procedure.getPartitionparameter();
sb.append(
String.format(
"<p>Partitioned on parameter %d which maps to column %s"
+ " of table <a class='invert' href='#s-%s'>%s</a>.</p>",
pIndex, pColumn, pTable, pTable));
}
// output what schema this interacts with
ProcedureAnnotation annotation = (ProcedureAnnotation) procedure.getAnnotation();
if (annotation != null) {
// make sure tables appear in only one category
annotation.tablesRead.removeAll(annotation.tablesUpdated);
if (annotation.tablesRead.size() > 0) {
sb.append("<p>Read-only access to tables: ");
List<String> tables = new ArrayList<String>();
for (Table table : annotation.tablesRead) {
tables.add("<a href='#s-" + table.getTypeName() + "'>" + table.getTypeName() + "</a>");
}
sb.append(StringUtils.join(tables, ", "));
sb.append("</p>");
}
if (annotation.tablesUpdated.size() > 0) {
sb.append("<p>Read/Write access to tables: ");
List<String> tables = new ArrayList<String>();
for (Table table : annotation.tablesUpdated) {
tables.add("<a href='#s-" + table.getTypeName() + "'>" + table.getTypeName() + "</a>");
}
sb.append(StringUtils.join(tables, ", "));
sb.append("</p>");
}
if (annotation.indexesUsed.size() > 0) {
sb.append("<p>Uses indexes: ");
List<String> indexes = new ArrayList<String>();
for (Index index : annotation.indexesUsed) {
Table table = (Table) index.getParent();
indexes.add(
"<a href='#s-"
+ table.getTypeName()
+ "-"
+ index.getTypeName()
+ "'>"
+ index.getTypeName()
+ "</a>");
}
sb.append(StringUtils.join(indexes, ", "));
sb.append("</p>");
}
}
sb.append(generateStatementsTable(procedure));
sb.append("</td></tr>\n");
return sb.toString();
}
static void compileSingleStmtProcedure(
VoltCompiler compiler,
HSQLInterface hsql,
DatabaseEstimates estimates,
Catalog catalog,
Database db,
ProcedureDescriptor procedureDescriptor)
throws VoltCompiler.VoltCompilerException {
final String className = procedureDescriptor.m_className;
if (className.indexOf('@') != -1) {
throw compiler.new VoltCompilerException("User procedure names can't contain \"@\".");
}
// get the short name of the class (no package if a user procedure)
// use the Table.<builtin> name (allowing the period) if builtin.
String shortName = className;
if (procedureDescriptor.m_builtInStmt == false) {
String[] parts = className.split("\\.");
shortName = parts[parts.length - 1];
}
// add an entry to the catalog (using the full className)
final Procedure procedure = db.getProcedures().add(shortName);
for (String groupName : procedureDescriptor.m_authGroups) {
final Group group = db.getGroups().get(groupName);
if (group == null) {
throw compiler
.new VoltCompilerException(
"Procedure "
+ className
+ " allows access by a role "
+ groupName
+ " that does not exist");
}
final GroupRef groupRef = procedure.getAuthgroups().add(groupName);
groupRef.setGroup(group);
}
procedure.setClassname(className);
// sysprocs don't use the procedure compiler
procedure.setSystemproc(false);
procedure.setDefaultproc(procedureDescriptor.m_builtInStmt);
procedure.setHasjava(false);
// get the annotation
// first try to get one that has been passed from the compiler
ProcInfoData info = compiler.getProcInfoOverride(shortName);
// then check for the usual one in the class itself
// and create a ProcInfo.Data instance for it
if (info == null) {
info = new ProcInfoData();
if (procedureDescriptor.m_partitionString != null) {
info.partitionInfo = procedureDescriptor.m_partitionString;
info.singlePartition = true;
}
}
assert (info != null);
// ADD THE STATEMENT
// add the statement to the catalog
Statement catalogStmt = procedure.getStatements().add(VoltDB.ANON_STMT_NAME);
// compile the statement
StatementPartitioning partitioning =
info.singlePartition ? StatementPartitioning.forceSP() : StatementPartitioning.forceMP();
// default to FASTER detmode because stmt procs can't feed read output into writes
StatementCompiler.compileFromSqlTextAndUpdateCatalog(
compiler,
hsql,
catalog,
db,
estimates,
catalogStmt,
procedureDescriptor.m_singleStmt,
procedureDescriptor.m_joinOrder,
DeterminismMode.FASTER,
partitioning);
// if the single stmt is not read only, then the proc is not read only
boolean procHasWriteStmts = (catalogStmt.getReadonly() == false);
// set the read onlyness of a proc
procedure.setReadonly(procHasWriteStmts == false);
int seqs = catalogStmt.getSeqscancount();
procedure.setHasseqscans(seqs > 0);
// set procedure parameter types
CatalogMap<ProcParameter> params = procedure.getParameters();
CatalogMap<StmtParameter> stmtParams = catalogStmt.getParameters();
// set the procedure parameter types from the statement parameter types
int paramCount = 0;
for (StmtParameter stmtParam : CatalogUtil.getSortedCatalogItems(stmtParams, "index")) {
// name each parameter "param1", "param2", etc...
ProcParameter procParam = params.add("param" + String.valueOf(paramCount));
procParam.setIndex(stmtParam.getIndex());
procParam.setIsarray(stmtParam.getIsarray());
procParam.setType(stmtParam.getJavatype());
paramCount++;
}
// parse the procinfo
procedure.setSinglepartition(info.singlePartition);
if (info.singlePartition) {
parsePartitionInfo(compiler, db, procedure, info.partitionInfo);
if (procedure.getPartitionparameter() >= params.size()) {
String msg =
"PartitionInfo parameter not a valid parameter for procedure: "
+ procedure.getClassname();
throw compiler.new VoltCompilerException(msg);
}
// TODO: The planner does not currently validate that a single-statement plan declared as
// single-partition correctly uses
// the designated parameter as a partitioning filter, maybe some day.
// In theory, the PartitioningForStatement would confirm the use of (only) a parameter as a
// partition key --
// or if the partition key was determined to be some other hard-coded constant (expression?)
// it might display a warning
// message that the passed parameter is assumed to be equal to that constant (expression).
} else {
if (partitioning.getCountOfIndependentlyPartitionedTables() == 1) {
AbstractExpression statementPartitionExpression =
partitioning.singlePartitioningExpressionForReport();
if (statementPartitionExpression != null) {
// The planner has uncovered an overlooked opportunity to run the statement SP.
String msg = null;
if (statementPartitionExpression instanceof ParameterValueExpression) {
msg =
"This procedure would benefit from setting the attribute 'partitioninfo="
+ partitioning.getFullColumnName()
+ ":"
+ ((ParameterValueExpression) statementPartitionExpression).getParameterIndex()
+ "'";
} else {
String valueDescription = null;
Object partitionValue = partitioning.getInferredPartitioningValue();
if (partitionValue == null) {
// Statement partitioned on a runtime constant. This is likely to be cryptic, but
// hopefully gets the idea across.
valueDescription = "of " + statementPartitionExpression.explain("");
} else {
valueDescription =
partitionValue.toString(); // A simple constant value COULD have been a parameter.
}
msg =
"This procedure would benefit from adding a parameter to be passed the value "
+ valueDescription
+ " and setting the attribute 'partitioninfo="
+ partitioning.getFullColumnName()
+ ":"
+ paramCount
+ "'";
}
compiler.addWarn(msg);
}
}
}
}
public void testMultiSiteSelectAll() {
ParameterSet params = ParameterSet.emptyParameterSet();
int outDepId = 1 | DtxnConstants.MULTIPARTITION_DEPENDENCY;
VoltTable dependency1 =
ee1.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(selectBottomFrag)},
null,
new ParameterSet[] {params},
3,
2,
42,
Long.MAX_VALUE)[0];
try {
System.out.println(dependency1.toString());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
assertTrue(dependency1 != null);
VoltTable dependency2 =
ee2.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(selectBottomFrag)},
null,
new ParameterSet[] {params},
3,
2,
42,
Long.MAX_VALUE)[0];
try {
System.out.println(dependency2.toString());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
assertTrue(dependency2 != null);
ee1.stashDependency(outDepId, dependency1);
ee1.stashDependency(outDepId, dependency2);
dependency1 =
ee1.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(selectTopFrag)},
new long[] {outDepId},
new ParameterSet[] {params},
3,
2,
42,
Long.MAX_VALUE)[0];
try {
System.out.println("Final Result");
System.out.println(dependency1.toString());
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
* Create a new data source.
*
* @param db
* @param tableName
* @param isReplicated
* @param partitionId
* @param HSId
* @param tableId
* @param catalogMap
*/
public ExportDataSource(
Runnable onDrain,
String db,
String tableName,
int partitionId,
long HSId,
String signature,
long generation,
CatalogMap<Column> catalogMap,
String overflowPath)
throws IOException {
m_generation = generation;
m_onDrain = onDrain;
m_database = db;
m_tableName = tableName;
String nonce = signature + "_" + HSId + "_" + partitionId;
m_committedBuffers = new StreamBlockQueue(overflowPath, nonce);
/*
* This is not the catalog relativeIndex(). This ID incorporates
* a catalog version and a table id so that it is constant across
* catalog updates that add or drop tables.
*/
m_signature = signature;
m_partitionId = partitionId;
m_HSId = HSId;
// Add the Export meta-data columns to the schema followed by the
// catalog columns for this table.
m_columnNames.add("VOLT_TRANSACTION_ID");
m_columnTypes.add(((int) VoltType.BIGINT.getValue()));
m_columnNames.add("VOLT_EXPORT_TIMESTAMP");
m_columnTypes.add(((int) VoltType.BIGINT.getValue()));
m_columnNames.add("VOLT_EXPORT_SEQUENCE_NUMBER");
m_columnTypes.add(((int) VoltType.BIGINT.getValue()));
m_columnNames.add("VOLT_PARTITION_ID");
m_columnTypes.add(((int) VoltType.BIGINT.getValue()));
m_columnNames.add("VOLT_SITE_ID");
m_columnTypes.add(((int) VoltType.BIGINT.getValue()));
m_columnNames.add("VOLT_EXPORT_OPERATION");
m_columnTypes.add(((int) VoltType.TINYINT.getValue()));
for (Column c : CatalogUtil.getSortedCatalogItems(catalogMap, "index")) {
m_columnNames.add(c.getName());
m_columnTypes.add(c.getType());
}
File adFile = new VoltFile(overflowPath, nonce + ".ad");
exportLog.info("Creating ad for " + nonce);
assert (!adFile.exists());
FastSerializer fs = new FastSerializer();
fs.writeLong(m_HSId);
fs.writeString(m_database);
writeAdvertisementTo(fs);
FileOutputStream fos = new FileOutputStream(adFile);
fos.write(fs.getBytes());
fos.getFD().sync();
fos.close();
// compute the number of bytes necessary to hold one bit per
// schema column
m_nullArrayLength = ((m_columnTypes.size() + 7) & -8) >> 3;
}