/**
* Create the completion node for the snapshot identified by the txnId. It assumes that all hosts
* will race to call this, so it doesn't fail if the node already exists.
*
* @param nonce Nonce of the snapshot
* @param txnId
* @param hostId The local host ID
* @param isTruncation Whether or not this is a truncation snapshot
* @param truncReqId Optional unique ID fed back to the monitor for identification
* @return true if the node is created successfully, false if the node already exists.
*/
public static ZKUtil.StringCallback createSnapshotCompletionNode(
String path, String nonce, long txnId, boolean isTruncation, String truncReqId) {
if (!(txnId > 0)) {
VoltDB.crashGlobalVoltDB("Txnid must be greather than 0", true, null);
}
byte nodeBytes[] = null;
try {
JSONStringer stringer = new JSONStringer();
stringer.object();
stringer.key("txnId").value(txnId);
stringer.key("isTruncation").value(isTruncation);
stringer.key("didSucceed").value(false);
stringer.key("hostCount").value(-1);
stringer.key("path").value(path);
stringer.key("nonce").value(nonce);
stringer.key("truncReqId").value(truncReqId);
stringer.key("exportSequenceNumbers").object().endObject();
stringer.endObject();
JSONObject jsonObj = new JSONObject(stringer.toString());
nodeBytes = jsonObj.toString(4).getBytes(Charsets.UTF_8);
} catch (Exception e) {
VoltDB.crashLocalVoltDB("Error serializing snapshot completion node JSON", true, e);
}
ZKUtil.StringCallback cb = new ZKUtil.StringCallback();
final String snapshotPath = VoltZK.completed_snapshots + "/" + txnId;
VoltDB.instance()
.getHostMessenger()
.getZK()
.create(snapshotPath, nodeBytes, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT, cb, null);
return cb;
}
@Override
public void run() {
CatalogAndIds catalogStuff = null;
do {
try {
catalogStuff = CatalogUtil.getCatalogFromZK(m_rvdb.getHostMessenger().getZK());
} catch (org.apache.zookeeper_voltpatches.KeeperException.NoNodeException e) {
} catch (Exception e) {
VoltDB.crashLocalVoltDB(
"System was interrupted while waiting for a catalog.", false, null);
}
} while (catalogStuff == null || catalogStuff.catalogBytes.length == 0);
String serializedCatalog = null;
byte[] catalogJarBytes = catalogStuff.catalogBytes;
try {
Pair<InMemoryJarfile, String> loadResults =
CatalogUtil.loadAndUpgradeCatalogFromJar(catalogStuff.catalogBytes);
serializedCatalog = CatalogUtil.getSerializedCatalogStringFromJar(loadResults.getFirst());
catalogJarBytes = loadResults.getFirst().getFullJarBytes();
} catch (IOException e) {
VoltDB.crashLocalVoltDB("Unable to load catalog", false, e);
}
if ((serializedCatalog == null) || (serializedCatalog.length() == 0))
VoltDB.crashLocalVoltDB("Catalog loading failure", false, null);
/* N.B. node recovery requires discovering the current catalog version. */
Catalog catalog = new Catalog();
catalog.execute(serializedCatalog);
serializedCatalog = null;
// note if this fails it will print an error first
// This is where we compile real catalog and create runtime
// catalog context. To validate deployment we compile and create
// a starter context which uses a placeholder catalog.
String result = CatalogUtil.compileDeployment(catalog, m_deployment, false);
if (result != null) {
hostLog.fatal(result);
VoltDB.crashLocalVoltDB(result);
}
try {
m_rvdb.m_catalogContext =
new CatalogContext(
catalogStuff.txnId,
catalogStuff.uniqueId,
catalog,
catalogJarBytes,
// Our starter catalog has set the deployment stuff, just yoink it out for now
m_rvdb.m_catalogContext.getDeploymentBytes(),
catalogStuff.version,
-1);
} catch (Exception e) {
VoltDB.crashLocalVoltDB("Error agreeing on starting catalog version", true, e);
}
}
@Override
public void run() {
// if I'm the leader, send out the catalog
if (m_rvdb.m_myHostId == m_rvdb.m_hostIdWithStartupCatalog) {
try {
// If no catalog was supplied provide an empty one.
if (m_rvdb.m_pathToStartupCatalog == null) {
try {
File emptyJarFile = CatalogUtil.createTemporaryEmptyCatalogJarFile();
if (emptyJarFile == null) {
VoltDB.crashLocalVoltDB("Failed to generate empty catalog.");
}
m_rvdb.m_pathToStartupCatalog = emptyJarFile.getAbsolutePath();
} catch (IOException e) {
VoltDB.crashLocalVoltDB(
"I/O exception while creating empty catalog jar file.", false, e);
}
}
// Get the catalog bytes and byte count.
byte[] catalogBytes = readCatalog(m_rvdb.m_pathToStartupCatalog);
// Export needs a cluster global unique id for the initial catalog version
long catalogUniqueId =
UniqueIdGenerator.makeIdFromComponents(
System.currentTimeMillis(), 0, MpInitiator.MP_INIT_PID);
hostLog.debug(String.format("Sending %d catalog bytes", catalogBytes.length));
long catalogTxnId;
catalogTxnId = TxnEgo.makeZero(MpInitiator.MP_INIT_PID).getTxnId();
// Need to get the deployment bytes from the starter catalog context
byte[] deploymentBytes = m_rvdb.getCatalogContext().getDeploymentBytes();
// publish the catalog bytes to ZK
CatalogUtil.updateCatalogToZK(
m_rvdb.getHostMessenger().getZK(),
0,
catalogTxnId,
catalogUniqueId,
catalogBytes,
deploymentBytes);
} catch (IOException e) {
VoltDB.crashGlobalVoltDB("Unable to distribute catalog.", false, e);
} catch (org.apache.zookeeper_voltpatches.KeeperException e) {
VoltDB.crashGlobalVoltDB("Unable to publish catalog.", false, e);
} catch (InterruptedException e) {
VoltDB.crashGlobalVoltDB("Interrupted while publishing catalog.", false, e);
}
}
}
@Override
public void run() {
try {
JSONStringer js = new JSONStringer();
js.object();
js.key("role").value(m_config.m_replicationRole.ordinal());
js.key("active").value(m_rvdb.getReplicationActive());
js.endObject();
ZooKeeper zk = m_rvdb.getHostMessenger().getZK();
// rejoining nodes figure out the replication role from other nodes
if (!m_isRejoin) {
try {
zk.create(
VoltZK.replicationconfig,
js.toString().getBytes("UTF-8"),
Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
} catch (KeeperException.NodeExistsException e) {
}
String discoveredReplicationConfig =
new String(zk.getData(VoltZK.replicationconfig, false, null), "UTF-8");
JSONObject discoveredjsObj = new JSONObject(discoveredReplicationConfig);
ReplicationRole discoveredRole =
ReplicationRole.get((byte) discoveredjsObj.getLong("role"));
if (!discoveredRole.equals(m_config.m_replicationRole)) {
VoltDB.crashGlobalVoltDB(
"Discovered replication role "
+ discoveredRole
+ " doesn't match locally specified replication role "
+ m_config.m_replicationRole,
true,
null);
}
// See if we should bring the server up in WAN replication mode
m_rvdb.setReplicationRole(discoveredRole);
} else {
String discoveredReplicationConfig =
new String(zk.getData(VoltZK.replicationconfig, false, null), "UTF-8");
JSONObject discoveredjsObj = new JSONObject(discoveredReplicationConfig);
ReplicationRole discoveredRole =
ReplicationRole.get((byte) discoveredjsObj.getLong("role"));
boolean replicationActive = discoveredjsObj.getBoolean("active");
// See if we should bring the server up in WAN replication mode
m_rvdb.setReplicationRole(discoveredRole);
m_rvdb.setReplicationActive(replicationActive);
}
} catch (Exception e) {
VoltDB.crashGlobalVoltDB("Error discovering replication role", false, e);
}
}
ProcedureRunner(
VoltProcedure procedure,
SiteProcedureConnection site,
SystemProcedureExecutionContext sysprocContext,
Procedure catProc,
CatalogSpecificPlanner csp) {
assert (m_inputCRC.getValue() == 0L);
if (procedure instanceof StmtProcedure) {
m_procedureName = catProc.getTypeName().intern();
} else {
m_procedureName = procedure.getClass().getSimpleName();
}
m_procedure = procedure;
m_isSysProc = procedure instanceof VoltSystemProcedure;
m_catProc = catProc;
m_site = site;
m_systemProcedureContext = sysprocContext;
m_csp = csp;
m_procedure.init(this);
m_statsCollector =
new ProcedureStatsCollector(
m_site.getCorrespondingSiteId(), m_site.getCorrespondingPartitionId(), m_catProc);
VoltDB.instance()
.getStatsAgent()
.registerStatsSource(
SysProcSelector.PROCEDURE, site.getCorrespondingSiteId(), m_statsCollector);
reflect();
}
@After
public void tearDown() throws Exception {
MockStatsSource.delay = 0;
StatsAgent.OPS_COLLECTION_TIMEOUT = 60 * 1000;
m_mvoltdb.shutdown(null);
VoltDB.replaceVoltDBInstanceForTest(null);
}
void shutdown() {
try {
m_leaderElector.shutdown();
} catch (Exception e) {
VoltDB.crashLocalVoltDB("Error shutting down GlobalServiceElector's LeaderElector", true, e);
}
}
/** Add a service to be notified if this node becomes the global leader */
synchronized void registerService(Promotable service) {
m_services.add(service);
if (m_isLeader) {
try {
service.acceptPromotion();
} catch (Exception e) {
VoltDB.crashLocalVoltDB("Unable to promote global service.", true, e);
}
}
}
@Override
public void run() {
boolean logEnabled = m_rvdb.m_catalogContext.cluster.getLogconfig().get("log").getEnabled();
if (logEnabled) {
if (m_config.m_isEnterprise) {
try {
Class<?> loggerClass =
MiscUtils.loadProClass("org.voltdb.CommandLogImpl", "Command logging", false);
if (loggerClass != null) {
m_rvdb.m_commandLog = (CommandLog) loggerClass.newInstance();
}
} catch (InstantiationException e) {
VoltDB.crashLocalVoltDB("Unable to instantiate command log", true, e);
} catch (IllegalAccessException e) {
VoltDB.crashLocalVoltDB("Unable to instantiate command log", true, e);
}
}
}
}
@Override
public synchronized void becomeLeader() {
hostLog.info("Host " + m_hostId + " promoted to be the global service provider");
m_isLeader = true;
for (Promotable service : m_services) {
try {
service.acceptPromotion();
} catch (Exception e) {
VoltDB.crashLocalVoltDB("Unable to promote global service.", true, e);
}
}
}
@Test
public void testLoader() {
VoltDB.Configuration configuration = new VoltDB.Configuration();
configuration.m_noLoadLibVOLTDB = true;
MockVoltDB mockvolt = new MockVoltDB();
VoltDB.ignoreCrash = true;
VoltDB.replaceVoltDBInstanceForTest(mockvolt);
mockvolt.m_noLoadLib = true;
assertFalse(EELibraryLoader.loadExecutionEngineLibrary(false));
assertFalse(VoltDB.wasCrashCalled);
boolean threw = false;
try {
assertFalse(EELibraryLoader.loadExecutionEngineLibrary(true));
} catch (AssertionError ae) {
threw = true;
}
assertTrue(threw);
assertTrue(VoltDB.wasCrashCalled);
VoltDB.wasCrashCalled = false;
VoltDB.initialize(configuration);
assertFalse(EELibraryLoader.loadExecutionEngineLibrary(true));
assertFalse(VoltDB.wasCrashCalled);
// Now test SUCCESS case
configuration = new VoltDB.Configuration();
VoltDBInterface mockitovolt = mock(VoltDBInterface.class);
VoltDBInterface realvolt = new RealVoltDB();
when(mockitovolt.getEELibraryVersionString()).thenReturn(realvolt.getEELibraryVersionString());
CatalogContext catContext = mock(CatalogContext.class);
Cluster cluster = mock(Cluster.class);
when(cluster.getVoltroot()).thenReturn(System.getProperty("java.io.tmpdir"));
when(catContext.getCluster()).thenReturn(cluster);
when(mockitovolt.getCatalogContext()).thenReturn(catContext);
VoltDB.replaceVoltDBInstanceForTest(mockitovolt);
VoltDB.initialize(configuration);
assertTrue(EELibraryLoader.loadExecutionEngineLibrary(true));
}
@Override
public void run() {
// Let the Export system read its configuration from the catalog.
try {
ExportManager.initialize(
m_rvdb.m_myHostId,
m_rvdb.m_catalogContext,
m_isRejoin,
m_rvdb.m_messenger,
m_rvdb.m_partitionsToSitesAtStartupForExportInit);
} catch (Throwable t) {
VoltDB.crashLocalVoltDB("Error setting up export", true, t);
}
}
@Before
public void setUp() throws Exception {
// Set up CI with the mock objects.
m_volt = mock(VoltDBInterface.class);
m_statsAgent = mock(StatsAgent.class);
m_sysinfoAgent = mock(SystemInformationAgent.class);
m_messenger = mock(HostMessenger.class);
m_handler = mock(ClientInputHandler.class);
m_cartographer = mock(Cartographer.class);
m_zk = mock(ZooKeeper.class);
m_cxn = mock(Connection.class);
/*
* Setup the mock objects so that they return expected objects in CI
* construction
*/
VoltDB.replaceVoltDBInstanceForTest(m_volt);
doReturn(m_statsAgent).when(m_volt).getStatsAgent();
doReturn(m_statsAgent).when(m_volt).getOpsAgent(OpsSelector.STATISTICS);
doReturn(m_sysinfoAgent).when(m_volt).getOpsAgent(OpsSelector.SYSTEMINFORMATION);
doReturn(mock(SnapshotCompletionMonitor.class)).when(m_volt).getSnapshotCompletionMonitor();
doReturn(m_messenger).when(m_volt).getHostMessenger();
doReturn(mock(VoltNetworkPool.class)).when(m_messenger).getNetwork();
doReturn(m_zk).when(m_messenger).getZK();
doReturn(mock(Configuration.class)).when(m_volt).getConfig();
doReturn(32L).when(m_messenger).getHSIdForLocalSite(HostMessenger.ASYNC_COMPILER_SITE_ID);
doAnswer(
new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) {
return null;
}
})
.when(m_cxn)
.queueTask(any(Runnable.class));
m_ci =
spy(
new ClientInterface(
null,
VoltDB.DEFAULT_PORT,
VoltDB.DEFAULT_ADMIN_PORT,
m_context,
m_messenger,
ReplicationRole.NONE,
m_cartographer,
m_allPartitions));
m_ci.bindAdapter(m_cxn);
// m_mb = m_ci.m_mailbox;
}
Inits(RealVoltDB rvdb, int threadCount) {
m_rvdb = rvdb;
m_config = rvdb.m_config;
// determine if this is a rejoining node
// (used for license check and later the actual rejoin)
if (m_config.m_startAction.doesRejoin()) {
m_isRejoin = true;
} else {
m_isRejoin = false;
}
m_threadCount = threadCount;
m_deployment = rvdb.m_catalogContext.getDeployment();
// find all the InitWork subclasses using reflection and load them up
Class<?>[] declaredClasses = Inits.class.getDeclaredClasses();
for (Class<?> cls : declaredClasses) {
// skip base classes and fake classes
if (cls == InitWork.class) continue;
if (cls == COMPLETION_WORK.class) continue;
if (InitWork.class.isAssignableFrom(cls)) {
InitWork instance = null;
try {
Constructor<?> constructor = cls.getDeclaredConstructor(Inits.class);
instance = (InitWork) constructor.newInstance(this);
} catch (Exception e) {
VoltDB.crashLocalVoltDB("Critical error loading class " + cls.getName(), true, e);
}
m_jobs.put(instance.getClass(), instance);
}
}
// make blockers and blockees symmetrical
for (InitWork iw : m_jobs.values()) {
for (Class<? extends InitWork> cls : iw.m_blockers) {
InitWork blocker = m_jobs.get(cls);
blocker.m_blockees.add(iw.getClass());
}
}
// collect initially ready jobs
List<Class<? extends InitWork>> toRemove = new ArrayList<Class<? extends InitWork>>();
for (Entry<Class<? extends InitWork>, InitWork> e : m_jobs.entrySet()) {
if (e.getValue().m_blockers.size() == 0) {
toRemove.add(e.getKey());
m_readyJobs.add(e.getValue());
}
}
}
/**
* Load the full subclass if it should, otherwise load the noop stub.
*
* @param partitionId partition id
* @param overflowDir
* @return Instance of PartitionDRGateway
*/
public static PartitionDRGateway getInstance(
int partitionId, NodeDRGateway nodeGateway, boolean isRejoin) {
final VoltDBInterface vdb = VoltDB.instance();
LicenseApi api = vdb.getLicenseApi();
final boolean licensedToDR = api.isDrReplicationAllowed();
// if this is a primary cluster in a DR-enabled scenario
// try to load the real version of this class
PartitionDRGateway pdrg = null;
if (licensedToDR && nodeGateway != null) {
pdrg = tryToLoadProVersion();
}
if (pdrg == null) {
pdrg = new PartitionDRGateway();
}
// init the instance and return
try {
pdrg.init(partitionId, nodeGateway, isRejoin);
} catch (IOException e) {
VoltDB.crashLocalVoltDB(e.getMessage(), false, e);
}
return pdrg;
}
@Override
public void run() {
try {
m_rvdb
.getAsyncCompilerAgent()
.createMailbox(
VoltDB.instance().getHostMessenger(),
m_rvdb
.getHostMessenger()
.getHSIdForLocalSite(HostMessenger.ASYNC_COMPILER_SITE_ID));
} catch (Exception e) {
hostLog.fatal(null, e);
System.exit(-1);
}
}
@Override
public void run() {
while (true) {
InitWork iw = null;
try {
iw = m_readyJobs.take();
} catch (InterruptedException e) {
VoltDB.crashLocalVoltDB(e.getMessage(), true, e);
}
if (iw instanceof COMPLETION_WORK) return;
// hostLog.info("Running InitWorker: " + iw.getClass().getName());
iw.run();
completeInitWork(iw);
}
}
@Override
public ListenableFuture<?> write(final Callable<BBContainer> tupleData, int tableId) {
final ListenableFuture<BBContainer> computedData =
VoltDB.instance().getComputationService().submit(tupleData);
return m_es.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
try {
final BBContainer data = computedData.get();
/*
* If a filter nulled out the buffer do nothing.
*/
if (data == null) return null;
if (m_writeFailed) {
data.discard();
return null;
}
try {
int totalWritten = 0;
final ByteBuffer dataBuf = data.b();
DefaultSnapshotDataTarget.enforceSnapshotRateLimit(dataBuf.remaining());
while (dataBuf.hasRemaining()) {
int written = m_fc.write(dataBuf);
if (written > 0) {
m_bytesWritten += written;
totalWritten += written;
}
}
if (m_bytesSinceLastSync.addAndGet(totalWritten) > m_bytesAllowedBeforeSync) {
m_fc.force(false);
m_bytesSinceLastSync.set(0);
}
} finally {
data.discard();
}
} catch (Throwable t) {
m_writeException = t;
m_writeFailed = true;
throw Throwables.propagate(t);
}
return null;
}
});
}
@Test
public void testUpdateCatalog() throws IOException {
// only makes sense in pro (sysproc suite has a complementary test for community)
if (VoltDB.instance().getConfig().m_isEnterprise) {
String catalogHex = Encoder.hexEncode("blah");
ByteBuffer msg = createMsg("@UpdateApplicationCatalog", catalogHex, "blah");
ClientResponseImpl resp = m_ci.handleRead(msg, m_handler, m_cxn);
assertNull(resp);
ArgumentCaptor<LocalObjectMessage> captor = ArgumentCaptor.forClass(LocalObjectMessage.class);
verify(m_messenger)
.send(
eq(32L), // A fixed number set in setUpOnce()
captor.capture());
assertTrue(captor.getValue().payload instanceof CatalogChangeWork);
}
}
@Test
public void testLoader() {
final VoltDB.Configuration configuration = new VoltDB.Configuration();
configuration.m_noLoadLibVOLTDB = true;
MockVoltDB mockvolt = new MockVoltDB();
VoltDB.replaceVoltDBInstanceForTest(mockvolt);
assert (EELibraryLoader.loadExecutionEngineLibrary(false));
// assertEquals(0, mockvolt.getCrashCount());
assert (EELibraryLoader.loadExecutionEngineLibrary(true));
// assertEquals(1, mockvolt.getCrashCount());
// VoltDB.initialize(configuration);
// assertFalse(EELibraryLoader.loadExecutionEngineLibrary(true));
// assertEquals(1, mockvolt.getCrashCount());
}
@Override
public void run() {
// If running commercial code (of value) and not rejoining, enforce licensing.
// Make the leader the only license enforcer.
boolean isLeader = (m_rvdb.m_myHostId == 0);
if (m_config.m_isEnterprise && isLeader && !m_isRejoin) {
if (!MiscUtils.validateLicense(
m_rvdb.getLicenseApi(),
m_deployment.getCluster().getHostcount(),
m_rvdb.getReplicationRole())) {
// validateLicense logs. Exit call is here for testability.
VoltDB.crashGlobalVoltDB("VoltDB license constraints are not met.", false, null);
}
}
}
public ParameterSet getParams() {
params.run();
try {
return params.get();
} catch (InterruptedException e) {
VoltDB.crashLocalVoltDB("Interrupted while deserializing a parameter set", false, e);
} catch (ExecutionException e) {
// Don't rethrow Errors as RuntimeExceptions because we will eat their
// delicious goodness later
if (e.getCause() != null && e.getCause() instanceof Error) {
throw (Error) e.getCause();
}
throw new RuntimeException(e);
}
return null;
}
public void testRejoinPropogateAdminMode() throws Exception {
// Reset the VoltFile prefix that may have been set by previous tests in this suite
org.voltdb.utils.VoltFile.resetSubrootForThisProcess();
VoltProjectBuilder builder = getBuilderForTest();
builder.setSecurityEnabled(true);
LocalCluster cluster =
new LocalCluster("rejoin.jar", 2, 3, 1, BackendTarget.NATIVE_EE_JNI, true);
boolean success = cluster.compileWithAdminMode(builder, 9998, false);
assertTrue(success);
MiscUtils.copyFile(
builder.getPathToDeployment(), Configuration.getPathToCatalogForTest("rejoin.xml"));
cluster.setHasLocalServer(false);
cluster.startUp();
ClientResponse response;
Client client;
client = ClientFactory.createClient(m_cconfig);
client.createConnection("localhost", 9997);
response = client.callProcedure("@Pause");
assertEquals(ClientResponse.SUCCESS, response.getStatus());
client.close();
cluster.shutDownSingleHost(0);
Thread.sleep(100);
VoltDB.Configuration config = new VoltDB.Configuration();
config.m_pathToCatalog = Configuration.getPathToCatalogForTest("rejoin.jar");
config.m_pathToDeployment = Configuration.getPathToCatalogForTest("rejoin.xml");
config.m_rejoinToHostAndPort = m_username + ":" + m_password + "@localhost:9996";
config.m_isRejoinTest = true;
ServerThread localServer = new ServerThread(config);
localServer.start();
localServer.waitForInitialization();
Thread.sleep(1000);
assertTrue(VoltDB.instance().getMode() == OperationMode.PAUSED);
localServer.shutdown();
cluster.shutDown();
}
public static HsqlBackend initializeHSQLBackend(long siteId, CatalogContext context) {
synchronized (backendLock) {
if (m_backend == null) {
try {
m_backend = new HsqlBackend(siteId);
final String binDDL = context.database.getSchema();
final String ddl = Encoder.decodeBase64AndDecompress(binDDL);
final String[] commands = ddl.split("\n");
for (String command : commands) {
String decoded_cmd = Encoder.hexDecodeToString(command);
decoded_cmd = decoded_cmd.trim();
if (decoded_cmd.length() == 0) {
continue;
}
m_backend.runDDL(decoded_cmd);
}
} catch (final Exception ex) {
hostLog.fatal("Unable to construct HSQL backend");
VoltDB.crashLocalVoltDB(ex.getMessage(), true, ex);
}
}
return m_backend;
}
}
/**
* Once participating host count is set, SnapshotCompletionMonitor can check this ZK node to
* determine whether the snapshot has finished or not.
*
* <p>This should only be called when all participants have responded. It is possible that some
* hosts finish taking snapshot before the coordinator logs the participating host count. In this
* case, the host count would have been decremented multiple times already. To make sure finished
* hosts are logged correctly, this method adds participating host count + 1 to the current host
* count.
*
* @param txnId The snapshot txnId
* @param participantCount The number of hosts participating in this snapshot
*/
public static void logParticipatingHostCount(long txnId, int participantCount) {
ZooKeeper zk = VoltDB.instance().getHostMessenger().getZK();
final String snapshotPath = VoltZK.completed_snapshots + "/" + txnId;
boolean success = false;
while (!success) {
Stat stat = new Stat();
byte data[] = null;
try {
data = zk.getData(snapshotPath, false, stat);
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.NONODE) {
// If snapshot creation failed for some reason, the node won't exist. ignore
return;
}
VoltDB.crashLocalVoltDB("Failed to get snapshot completion node", true, e);
} catch (InterruptedException e) {
VoltDB.crashLocalVoltDB("Interrupted getting snapshot completion node", true, e);
}
if (data == null) {
VoltDB.crashLocalVoltDB("Data should not be null if the node exists", false, null);
}
try {
JSONObject jsonObj = new JSONObject(new String(data, Charsets.UTF_8));
if (jsonObj.getLong("txnId") != txnId) {
VoltDB.crashLocalVoltDB("TxnId should match", false, null);
}
int hostCount = jsonObj.getInt("hostCount");
// +1 because hostCount was initialized to -1
jsonObj.put("hostCount", hostCount + participantCount + 1);
zk.setData(snapshotPath, jsonObj.toString(4).getBytes(Charsets.UTF_8), stat.getVersion());
} catch (KeeperException.BadVersionException e) {
continue;
} catch (Exception e) {
VoltDB.crashLocalVoltDB("This ZK call should never fail", true, e);
}
success = true;
}
}
@SuppressWarnings("deprecation")
@Override
public void setUp() throws IOException, InterruptedException {
VoltDB.instance().readBuildInfo("Test");
// compile a catalog
String testDir = BuildDirectoryUtils.getBuildDirectoryPath();
String catalogJar = testDir + File.separator + JAR;
TPCCProjectBuilder pb = new TPCCProjectBuilder();
pb.addDefaultSchema();
pb.addDefaultPartitioning();
pb.addProcedures(MultiSiteSelect.class, InsertNewOrder.class);
pb.compile(catalogJar, 2, 0);
// load a catalog
byte[] bytes = CatalogUtil.toBytes(new File(catalogJar));
String serializedCatalog = CatalogUtil.loadCatalogFromJar(bytes, null);
// create the catalog (that will be passed to the ClientInterface
catalog = new Catalog();
catalog.execute(serializedCatalog);
// update the catalog with the data from the deployment file
String pathToDeployment = pb.getPathToDeployment();
assertTrue(CatalogUtil.compileDeploymentAndGetCRC(catalog, pathToDeployment, true) >= 0);
cluster = catalog.getClusters().get("cluster");
CatalogMap<Procedure> procedures = cluster.getDatabases().get("database").getProcedures();
Procedure insertProc = procedures.get("InsertNewOrder");
assert (insertProc != null);
selectProc = procedures.get("MultiSiteSelect");
assert (selectProc != null);
// Each EE needs its own thread for correct initialization.
final AtomicReference<ExecutionEngine> site1Reference = new AtomicReference<ExecutionEngine>();
final byte configBytes[] = LegacyHashinator.getConfigureBytes(2);
Thread site1Thread =
new Thread() {
@Override
public void run() {
site1Reference.set(
new ExecutionEngineJNI(
cluster.getRelativeIndex(),
1,
0,
0,
"",
100,
HashinatorType.LEGACY,
configBytes));
}
};
site1Thread.start();
site1Thread.join();
final AtomicReference<ExecutionEngine> site2Reference = new AtomicReference<ExecutionEngine>();
Thread site2Thread =
new Thread() {
@Override
public void run() {
site2Reference.set(
new ExecutionEngineJNI(
cluster.getRelativeIndex(),
2,
1,
0,
"",
100,
HashinatorType.LEGACY,
configBytes));
}
};
site2Thread.start();
site2Thread.join();
// create two EEs
site1 = new ExecutionSite(0); // site 0
ee1 = site1Reference.get();
ee1.loadCatalog(0, catalog.serialize());
site2 = new ExecutionSite(1); // site 1
ee2 = site2Reference.get();
ee2.loadCatalog(0, catalog.serialize());
// cache some plan fragments
selectStmt = selectProc.getStatements().get("selectAll");
assert (selectStmt != null);
int i = 0;
// this kinda assumes the right order
for (PlanFragment f : selectStmt.getFragments()) {
if (i == 0) selectTopFrag = f;
else selectBottomFrag = f;
i++;
}
assert (selectTopFrag != null);
assert (selectBottomFrag != null);
if (selectTopFrag.getHasdependencies() == false) {
PlanFragment temp = selectTopFrag;
selectTopFrag = selectBottomFrag;
selectBottomFrag = temp;
}
// get the insert frag
Statement insertStmt = insertProc.getStatements().get("insert");
assert (insertStmt != null);
for (PlanFragment f : insertStmt.getFragments()) insertFrag = f;
// populate plan cache
ActivePlanRepository.clear();
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(selectBottomFrag),
Encoder.base64Decode(selectBottomFrag.getPlannodetree()));
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(selectTopFrag),
Encoder.base64Decode(selectTopFrag.getPlannodetree()));
ActivePlanRepository.addFragmentForTest(
CatalogUtil.getUniqueIdForFragment(insertFrag),
Encoder.base64Decode(insertFrag.getPlannodetree()));
// insert some data
ParameterSet params = ParameterSet.fromArrayNoCopy(1L, 1L, 1L);
VoltTable[] results =
ee2.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
null,
new ParameterSet[] {params},
1,
0,
42,
Long.MAX_VALUE);
assert (results.length == 1);
assert (results[0].asScalarLong() == 1L);
params = ParameterSet.fromArrayNoCopy(2L, 2L, 2L);
results =
ee1.executePlanFragments(
1,
new long[] {CatalogUtil.getUniqueIdForFragment(insertFrag)},
null,
new ParameterSet[] {params},
2,
1,
42,
Long.MAX_VALUE);
assert (results.length == 1);
assert (results[0].asScalarLong() == 1L);
}
@Override
public void run() {
if (!m_isRejoin && !m_config.m_isRejoinTest && !m_rvdb.m_joining) {
String snapshotPath = null;
if (m_rvdb
.m_catalogContext
.cluster
.getDatabases()
.get("database")
.getSnapshotschedule()
.get("default")
!= null) {
snapshotPath =
m_rvdb
.m_catalogContext
.cluster
.getDatabases()
.get("database")
.getSnapshotschedule()
.get("default")
.getPath();
}
int[] allPartitions = new int[m_rvdb.m_configuredNumberOfPartitions];
for (int ii = 0; ii < allPartitions.length; ii++) {
allPartitions[ii] = ii;
}
org.voltdb.catalog.CommandLog cl =
m_rvdb.m_catalogContext.cluster.getLogconfig().get("log");
try {
m_rvdb.m_restoreAgent =
new RestoreAgent(
m_rvdb.m_messenger,
m_rvdb.getSnapshotCompletionMonitor(),
m_rvdb,
m_config.m_startAction,
cl.getEnabled(),
cl.getLogpath(),
cl.getInternalsnapshotpath(),
snapshotPath,
allPartitions,
CatalogUtil.getVoltDbRoot(m_deployment.getPaths()).getAbsolutePath());
} catch (IOException e) {
VoltDB.crashLocalVoltDB("Unable to construct the RestoreAgent", true, e);
}
m_rvdb.m_globalServiceElector.registerService(m_rvdb.m_restoreAgent);
m_rvdb.m_restoreAgent.setCatalogContext(m_rvdb.m_catalogContext);
// Generate plans and get (hostID, catalogPath) pair
Pair<Integer, String> catalog = m_rvdb.m_restoreAgent.findRestoreCatalog();
// if the restore agent found a catalog, set the following info
// so the right node can send it out to the others
if (catalog != null) {
// Make sure the catalog corresponds to the current server version.
// Prevent automatic upgrades by rejecting mismatched versions.
int hostId = catalog.getFirst().intValue();
String catalogPath = catalog.getSecond();
// Perform a version check when the catalog jar is available
// on the current host.
// Check that this host is the one providing the catalog.
if (m_rvdb.m_myHostId == hostId) {
try {
byte[] catalogBytes = readCatalog(catalogPath);
InMemoryJarfile inMemoryJar = CatalogUtil.loadInMemoryJarFile(catalogBytes);
// This call pre-checks and returns the build info/version.
String[] buildInfo = CatalogUtil.getBuildInfoFromJar(inMemoryJar);
String catalogVersion = buildInfo[0];
String serverVersion = m_rvdb.getVersionString();
if (!catalogVersion.equals(serverVersion)) {
VoltDB.crashLocalVoltDB(
String.format(
"Unable to load version %s catalog \"%s\" "
+ "from snapshot into a version %s server.",
catalogVersion, catalogPath, serverVersion),
false,
null);
}
} catch (IOException e) {
// Make it non-fatal with no check performed.
hostLog.warn(
String.format(
"Unable to load catalog for version check due to exception: %s.",
e.getMessage()));
}
}
hostLog.debug("Found catalog to load on host " + hostId + ": " + catalogPath);
m_rvdb.m_hostIdWithStartupCatalog = hostId;
assert (m_rvdb.m_hostIdWithStartupCatalog >= 0);
m_rvdb.m_pathToStartupCatalog = catalogPath;
assert (m_rvdb.m_pathToStartupCatalog != null);
}
}
}
@Before
public void setUp() throws Exception {
m_mvoltdb = new MockVoltDB();
VoltDB.replaceVoltDBInstanceForTest(m_mvoltdb);
}
/**
* The only public method: do all the work to start a snapshot. Assumes that a snapshot is
* feasible, that the caller has validated it can be accomplished, that the caller knows this is a
* consistent or useful transaction point at which to snapshot.
*
* @param file_path
* @param file_nonce
* @param format
* @param block
* @param txnId
* @param data
* @param context
* @param hostname
* @return VoltTable describing the results of the snapshot attempt
*/
public VoltTable startSnapshotting(
final String file_path,
final String file_nonce,
final SnapshotFormat format,
final byte block,
final long multiPartTxnId,
final long partitionTxnId,
final long legacyPerPartitionTxnIds[],
final String data,
final SystemProcedureExecutionContext context,
final String hostname,
final HashinatorSnapshotData hashinatorData,
final long timestamp) {
TRACE_LOG.trace("Creating snapshot target and handing to EEs");
final VoltTable result = SnapshotUtil.constructNodeResultsTable();
final int numLocalSites =
context.getCluster().getDeployment().get("deployment").getSitesperhost();
// One site wins the race to create the snapshot targets, populating
// m_taskListsForSites for the other sites and creating an appropriate
// number of snapshot permits.
synchronized (SnapshotSiteProcessor.m_snapshotCreateLock) {
SnapshotSiteProcessor.m_snapshotCreateSetupBarrierActualAction.set(
new Runnable() {
@Override
public void run() {
Map<Integer, Long> partitionTransactionIds = new HashMap<Integer, Long>();
partitionTransactionIds = m_partitionLastSeenTransactionIds;
SNAP_LOG.debug("Last seen partition transaction ids " + partitionTransactionIds);
m_partitionLastSeenTransactionIds = new HashMap<Integer, Long>();
partitionTransactionIds.put(TxnEgo.getPartitionId(multiPartTxnId), multiPartTxnId);
/*
* Do a quick sanity check that the provided IDs
* don't conflict with currently active partitions. If they do
* it isn't fatal we can just skip it.
*/
for (long txnId : legacyPerPartitionTxnIds) {
final int legacyPartition = TxnEgo.getPartitionId(txnId);
if (partitionTransactionIds.containsKey(legacyPartition)) {
SNAP_LOG.warn(
"While saving a snapshot and propagating legacy "
+ "transaction ids found an id that matches currently active partition"
+ partitionTransactionIds.get(legacyPartition));
} else {
partitionTransactionIds.put(legacyPartition, txnId);
}
}
exportSequenceNumbers = SnapshotSiteProcessor.getExportSequenceNumbers();
createSetupIv2(
file_path,
file_nonce,
format,
multiPartTxnId,
partitionTransactionIds,
data,
context,
result,
exportSequenceNumbers,
context.getSiteTrackerForSnapshot(),
hashinatorData,
timestamp);
}
});
// Create a barrier to use with the current number of sites to wait for
// or if the barrier is already set up check if it is broken and reset if necessary
SnapshotSiteProcessor.readySnapshotSetupBarriers(numLocalSites);
// From within this EE, record the sequence numbers as of the start of the snapshot (now)
// so that the info can be put in the digest.
SnapshotSiteProcessor.populateExportSequenceNumbersForExecutionSite(context);
SNAP_LOG.debug(
"Registering transaction id "
+ partitionTxnId
+ " for "
+ TxnEgo.getPartitionId(partitionTxnId));
m_partitionLastSeenTransactionIds.put(TxnEgo.getPartitionId(partitionTxnId), partitionTxnId);
}
boolean runPostTasks = false;
VoltTable earlyResultTable = null;
try {
SnapshotSiteProcessor.m_snapshotCreateSetupBarrier.await();
try {
synchronized (m_createLock) {
SNAP_LOG.debug(
"Found tasks for HSIds: "
+ CoreUtils.hsIdCollectionToString(m_taskListsForHSIds.keySet()));
SNAP_LOG.debug("Looking for local HSID: " + CoreUtils.hsIdToString(context.getSiteId()));
Deque<SnapshotTableTask> taskList = m_taskListsForHSIds.remove(context.getSiteId());
// If createSetup failed, then the first site to reach here is going
// to send the results table generated by createSetup, and then empty out the table.
// All other sites to reach here will send the appropriate empty table.
// If createSetup was a success but the taskList is null, then we'll use the block
// switch to figure out what flavor of empty SnapshotSave result table to return.
if (!m_createSuccess.get()) {
// There shouldn't be any work for any site if we failed
assert (m_taskListsForHSIds.isEmpty());
VoltTable finalresult = m_createResult.get();
if (finalresult != null) {
m_createResult.set(null);
earlyResultTable = finalresult;
} else {
// We returned a non-empty NodeResultsTable with the failures in it,
// every other site needs to return a NodeResultsTable as well.
earlyResultTable = SnapshotUtil.constructNodeResultsTable();
}
} else if (taskList == null) {
SNAP_LOG.debug("No task for this site, block " + block);
// This node is participating in the snapshot but this site has nothing to do.
// Send back an appropriate empty table based on the block flag
if (block != 0) {
runPostTasks = true;
earlyResultTable = SnapshotUtil.constructPartitionResultsTable();
earlyResultTable.addRow(
context.getHostId(),
hostname,
CoreUtils.getSiteIdFromHSId(context.getSiteId()),
"SUCCESS",
"");
} else {
earlyResultTable = SnapshotUtil.constructNodeResultsTable();
}
} else {
context
.getSiteSnapshotConnection()
.initiateSnapshots(format, taskList, multiPartTxnId, exportSequenceNumbers);
}
if (m_deferredSetupFuture != null) {
// Add a listener to the deferred setup so that it can kick off the snapshot
// task once the setup is done.
m_deferredSetupFuture.addListener(
new Runnable() {
@Override
public void run() {
DeferredSnapshotSetup deferredSnapshotSetup = null;
try {
deferredSnapshotSetup = m_deferredSetupFuture.get();
} catch (Exception e) {
// it doesn't throw
}
assert deferredSnapshotSetup != null;
context
.getSiteSnapshotConnection()
.startSnapshotWithTargets(
deferredSnapshotSetup.getPlan().getSnapshotDataTargets());
}
},
CoreUtils.SAMETHREADEXECUTOR);
}
}
} finally {
SnapshotSiteProcessor.m_snapshotCreateFinishBarrier.await(120, TimeUnit.SECONDS);
}
} catch (TimeoutException e) {
VoltDB.crashLocalVoltDB(
"Timed out waiting 120 seconds for all threads to arrive and start snapshot", true, null);
} catch (InterruptedException e) {
result.addRow(context.getHostId(), hostname, "", "FAILURE", CoreUtils.throwableToString(e));
earlyResultTable = result;
} catch (BrokenBarrierException e) {
result.addRow(context.getHostId(), hostname, "", "FAILURE", CoreUtils.throwableToString(e));
earlyResultTable = result;
}
// If earlyResultTable is set, return here
if (earlyResultTable != null) {
if (runPostTasks) {
// Need to run post-snapshot tasks before finishing
SnapshotSiteProcessor.runPostSnapshotTasks(context);
}
return earlyResultTable;
}
if (block != 0) {
HashSet<Exception> failures = Sets.newHashSet();
String status = "SUCCESS";
String err = "";
try {
// For blocking snapshot, propogate the error from deferred setup back to the client
final DeferredSnapshotSetup deferredSnapshotSetup = m_deferredSetupFuture.get();
if (deferredSnapshotSetup != null && deferredSnapshotSetup.getError() != null) {
status = "FAILURE";
err = deferredSnapshotSetup.getError().toString();
failures.add(deferredSnapshotSetup.getError());
}
failures.addAll(context.getSiteSnapshotConnection().completeSnapshotWork());
SnapshotSiteProcessor.runPostSnapshotTasks(context);
} catch (Exception e) {
status = "FAILURE";
err = e.toString();
failures.add(e);
}
final VoltTable blockingResult = SnapshotUtil.constructPartitionResultsTable();
if (failures.isEmpty()) {
blockingResult.addRow(
context.getHostId(),
hostname,
CoreUtils.getSiteIdFromHSId(context.getSiteId()),
status,
err);
} else {
status = "FAILURE";
for (Exception e : failures) {
err = e.toString();
}
blockingResult.addRow(
context.getHostId(),
hostname,
CoreUtils.getSiteIdFromHSId(context.getSiteId()),
status,
err);
}
return blockingResult;
}
return result;
}
private void createSetupIv2(
final String file_path,
final String file_nonce,
SnapshotFormat format,
final long txnId,
final Map<Integer, Long> partitionTransactionIds,
String data,
final SystemProcedureExecutionContext context,
final VoltTable result,
Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers,
SiteTracker tracker,
HashinatorSnapshotData hashinatorData,
long timestamp) {
JSONObject jsData = null;
if (data != null && !data.isEmpty()) {
try {
jsData = new JSONObject(data);
} catch (JSONException e) {
SNAP_LOG.error(String.format("JSON exception on snapshot data \"%s\".", data), e);
}
}
SnapshotWritePlan plan;
if (format == SnapshotFormat.NATIVE) {
plan = new NativeSnapshotWritePlan();
} else if (format == SnapshotFormat.CSV) {
plan = new CSVSnapshotWritePlan();
} else if (format == SnapshotFormat.STREAM) {
plan = new StreamSnapshotWritePlan();
} else if (format == SnapshotFormat.INDEX) {
plan = new IndexSnapshotWritePlan();
} else {
throw new RuntimeException("BAD BAD BAD");
}
final Callable<Boolean> deferredSetup =
plan.createSetup(
file_path,
file_nonce,
txnId,
partitionTransactionIds,
jsData,
context,
result,
exportSequenceNumbers,
tracker,
hashinatorData,
timestamp);
m_deferredSetupFuture =
VoltDB.instance()
.submitSnapshotIOWork(
new DeferredSnapshotSetup(plan, deferredSetup, txnId, partitionTransactionIds));
synchronized (m_createLock) {
// Seems like this should be cleared out just in case
// Log if there is actually anything to clear since it is unexpected
if (!m_taskListsForHSIds.isEmpty()) {
SNAP_LOG.warn("Found lingering snapshot tasks while setting up a snapshot");
}
m_taskListsForHSIds.clear();
m_createSuccess.set(true);
m_createResult.set(result);
m_taskListsForHSIds.putAll(plan.getTaskListsForHSIds());
// HACK HACK HACK. If the task list is empty, this host has no work to do for
// this snapshot. We're going to create an empty list of tasks for one of the sites to do
// so that we'll have a SnapshotSiteProcessor which will do the logSnapshotCompleteToZK.
if (m_taskListsForHSIds.isEmpty()) {
SNAP_LOG.debug(
"Node had no snapshot work to do. Creating a null task to drive completion.");
m_taskListsForHSIds.put(context.getSiteId(), new ArrayDeque<SnapshotTableTask>());
}
SNAP_LOG.debug(
"Planned tasks: "
+ CoreUtils.hsIdCollectionToString(plan.getTaskListsForHSIds().keySet()));
SNAP_LOG.debug(
"Created tasks for HSIds: "
+ CoreUtils.hsIdCollectionToString(m_taskListsForHSIds.keySet()));
}
}
