@Override
public void run() {
initialize();
try {
while (m_shouldContinue) {
// Normal operation blocks the site thread on the sitetasker queue.
SiteTasker task = m_scheduler.take();
task.run(getSiteProcedureConnection());
}
} catch (OutOfMemoryError e) {
// Even though OOM should be caught by the Throwable section below,
// it sadly needs to be handled seperately. The goal here is to make
// sure VoltDB crashes.
String errmsg =
"Site: "
+ org.voltcore.utils.CoreUtils.hsIdToString(m_siteId)
+ " ran out of Java memory. "
+ "This node will shut down.";
VoltDB.crashLocalVoltDB(errmsg, true, e);
} catch (Throwable t) {
String errmsg =
"Site: "
+ org.voltcore.utils.CoreUtils.hsIdToString(m_siteId)
+ " encountered an "
+ "unexpected error and will die, taking this VoltDB node down.";
VoltDB.crashLocalVoltDB(errmsg, true, t);
}
shutdown();
}
public static void logFragmentTaskMessage(
FragmentTaskMessage ftask, long localHSId, long spHandle, boolean borrow) {
if (iv2log.isTraceEnabled()) {
String label = "rxFragMsg";
if (borrow) {
label = "rxBrrwMsg";
}
if (ftask.getSpHandle() != Long.MIN_VALUE && ftask.getSpHandle() != spHandle) {
iv2log.error(
"FragmentTaskMessage SP HANDLE conflict. Message: "
+ ftask.getSpHandle()
+ ", locally held: "
+ spHandle);
}
String logmsg = "%s %s from %s txnId %s spHandle %s trunc %s";
iv2log.trace(
String.format(
logmsg,
label,
CoreUtils.hsIdToString(localHSId),
CoreUtils.hsIdToString(ftask.m_sourceHSId),
txnIdToString(ftask.getTxnId()),
txnIdToString(spHandle),
txnIdToString(ftask.getTruncationHandle())));
}
}
/** Runs when the RejoinCoordinator decides this site should start rejoin. */
void doInitiation(RejoinMessage message) {
m_coordinatorHsId = message.m_sourceHSId;
m_streamSnapshotMb = VoltDB.instance().getHostMessenger().createMailbox();
m_rejoinSiteProcessor = new StreamSnapshotSink(m_streamSnapshotMb);
// MUST choose the leader as the source.
long sourceSite = m_mailbox.getMasterHsId(m_partitionId);
long hsId =
m_rejoinSiteProcessor.initialize(
message.getSnapshotSourceCount(), message.getSnapshotBufferPool());
REJOINLOG.debug(
m_whoami
+ "received INITIATION message. Doing rejoin"
+ ". Source site is: "
+ CoreUtils.hsIdToString(sourceSite)
+ " and destination rejoin processor is: "
+ CoreUtils.hsIdToString(hsId)
+ " and snapshot nonce is: "
+ message.getSnapshotNonce());
registerSnapshotMonitor(message.getSnapshotNonce());
// Tell the RejoinCoordinator everything it will need to know to get us our snapshot stream.
RejoinMessage initResp = new RejoinMessage(m_mailbox.getHSId(), sourceSite, hsId);
m_mailbox.send(m_coordinatorHsId, initResp);
// Start waiting for snapshot data
m_taskQueue.offer(this);
}
public static void logIv2InitiateTaskMessage(
Iv2InitiateTaskMessage itask, long localHSId, long txnid, long spHandle) {
if (iv2log.isTraceEnabled()) {
String logmsg = "rxInitMsg %s from %s ciHandle %s txnId %s spHandle %s trunc %s";
if (itask.getTxnId() != Long.MIN_VALUE && itask.getTxnId() != txnid) {
iv2log.error(
"Iv2InitiateTaskMessage TXN ID conflict. Message: "
+ itask.getTxnId()
+ ", locally held: "
+ txnid);
}
if (itask.getSpHandle() != Long.MIN_VALUE && itask.getSpHandle() != spHandle) {
iv2log.error(
"Iv2InitiateTaskMessage SP HANDLE conflict. Message: "
+ itask.getSpHandle()
+ ", locally held: "
+ spHandle);
}
iv2log.trace(
String.format(
logmsg,
CoreUtils.hsIdToString(localHSId),
CoreUtils.hsIdToString(itask.m_sourceHSId),
ClientInterfaceHandleManager.handleToString(itask.getClientInterfaceHandle()),
txnIdToString(txnid),
txnIdToString(spHandle),
txnIdToString(itask.getTruncationHandle())));
}
}
public static void logInitiatorRxMsg(VoltMessage msg, long localHSId) {
if (iv2log.isTraceEnabled()) {
if (msg instanceof InitiateResponseMessage) {
InitiateResponseMessage iresp = (InitiateResponseMessage) msg;
String logmsg = "rxInitRsp %s from %s ciHandle %s txnId %s spHandle %s status %s";
iv2log.trace(
String.format(
logmsg,
CoreUtils.hsIdToString(localHSId),
CoreUtils.hsIdToString(iresp.m_sourceHSId),
ClientInterfaceHandleManager.handleToString(iresp.getClientInterfaceHandle()),
txnIdToString(iresp.getTxnId()),
txnIdToString(iresp.getSpHandle()),
respStatusToString(iresp.getClientResponseData().getStatus())));
} else if (msg instanceof FragmentResponseMessage) {
FragmentResponseMessage fresp = (FragmentResponseMessage) msg;
String logmsg = "rxFragRsp %s from %s txnId %s spHandle %s status %s";
iv2log.trace(
String.format(
logmsg,
CoreUtils.hsIdToString(localHSId),
CoreUtils.hsIdToString(fresp.m_sourceHSId),
txnIdToString(fresp.getTxnId()),
txnIdToString(fresp.getSpHandle()),
fragStatusToString(fresp.getStatusCode())));
}
}
}
/** Process a new repair log response */
@Override
public void deliver(VoltMessage message) {
if (message instanceof Iv2RepairLogResponseMessage) {
Iv2RepairLogResponseMessage response = (Iv2RepairLogResponseMessage) message;
if (response.getRequestId() != m_requestId) {
tmLog.debug(
m_whoami
+ "rejecting stale repair response."
+ " Current request id is: "
+ m_requestId
+ " Received response for request id: "
+ response.getRequestId());
return;
}
ReplicaRepairStruct rrs = m_replicaRepairStructs.get(response.m_sourceHSId);
if (rrs.m_expectedResponses < 0) {
tmLog.debug(
m_whoami
+ "collecting "
+ response.getOfTotal()
+ " repair log entries from "
+ CoreUtils.hsIdToString(response.m_sourceHSId));
}
// Long.MAX_VALUE has rejoin semantics
if (response.getHandle() != Long.MAX_VALUE) {
m_maxSeenTxnId = Math.max(m_maxSeenTxnId, response.getHandle());
}
if (response.getPayload() != null) {
m_repairLogUnion.add(response);
if (tmLog.isTraceEnabled()) {
tmLog.trace(
m_whoami
+ " collected from "
+ CoreUtils.hsIdToString(response.m_sourceHSId)
+ ", message: "
+ response.getPayload());
}
}
if (rrs.update(response)) {
tmLog.debug(
m_whoami
+ "collected "
+ rrs.m_receivedResponses
+ " responses for "
+ rrs.m_expectedResponses
+ " repair log entries from "
+ CoreUtils.hsIdToString(response.m_sourceHSId));
if (areRepairLogsComplete()) {
repairSurvivors();
}
}
}
}
public static void logFinishTransaction(InitiateResponseMessage msg, long localHSId) {
if (iv2log.isTraceEnabled()) {
String logmsg = "finishTxn %s ciHandle %s initHSId %s status %s";
iv2log.trace(
String.format(
logmsg,
CoreUtils.hsIdToString(localHSId),
ClientInterfaceHandleManager.handleToString(msg.getClientInterfaceHandle()),
CoreUtils.hsIdToString(msg.getCoordinatorHSId()),
respStatusToString(msg.getClientResponseData().getStatus())));
}
}
public static void logCreateTransaction(Iv2InitiateTaskMessage msg) {
if (iv2log.isTraceEnabled()) {
String logmsg = "createTxn %s ciHandle %s initHSId %s proc %s";
iv2log.trace(
String.format(
logmsg,
CoreUtils.hsIdToString(msg.getInitiatorHSId()),
ClientInterfaceHandleManager.handleToString(msg.getClientInterfaceHandle()),
CoreUtils.hsIdToString(msg.getCoordinatorHSId()),
msg.getStoredProcedureInvocation().getProcName()));
}
}
public static void logIv2MultipartSentinel(
MultiPartitionParticipantMessage message, long localHSId, long txnId) {
if (iv2log.isTraceEnabled()) {
String logmsg = "rxSntlMsg %s from %s txnId %s";
iv2log.trace(
String.format(
logmsg,
CoreUtils.hsIdToString(localHSId),
CoreUtils.hsIdToString(message.m_sourceHSId),
txnIdToString(txnId)));
}
}
public static void logCompleteTransactionMessage(
CompleteTransactionMessage ctask, long localHSId) {
if (iv2log.isTraceEnabled()) {
String logmsg = "rxCompMsg %s from %s txnId %s %s %s";
iv2log.trace(
String.format(
logmsg,
CoreUtils.hsIdToString(localHSId),
CoreUtils.hsIdToString(ctask.m_sourceHSId),
txnIdToString(ctask.getTxnId()),
ctask.isRollback() ? "ROLLBACK" : "COMMIT",
ctask.isRestart() ? "RESTART" : ""));
}
}
public void handleDumpMessage() {
String who = CoreUtils.hsIdToString(m_mailbox.getHSId());
hostLog.warn("State dump for site: " + who);
hostLog.warn("" + who + ": partition: " + m_partitionId + ", isLeader: " + m_isLeader);
if (m_isLeader) {
hostLog.warn("" + who + ": replicas: " + CoreUtils.hsIdCollectionToString(m_replicaHSIds));
if (m_sendToHSIds.length > 0) {
m_mailbox.send(m_sendToHSIds, new DumpMessage());
}
}
hostLog.warn(
""
+ who
+ ": most recent SP handle: "
+ getCurrentTxnId()
+ " "
+ TxnEgo.txnIdToString(getCurrentTxnId()));
hostLog.warn(
""
+ who
+ ": outstanding txns: "
+ m_outstandingTxns.keySet()
+ " "
+ TxnEgo.txnIdCollectionToString(m_outstandingTxns.keySet()));
hostLog.warn("" + who + ": TransactionTaskQueue: " + m_pendingTasks.toString());
if (m_duplicateCounters.size() > 0) {
hostLog.warn("" + who + ": duplicate counters: ");
for (Entry<DuplicateCounterKey, DuplicateCounter> e : m_duplicateCounters.entrySet()) {
hostLog.warn("\t" + who + ": " + e.getKey().toString() + ": " + e.getValue().toString());
}
}
}
@Override
public void run() {
Thread.currentThread().setName("Iv2ExecutionSite: " + CoreUtils.hsIdToString(m_siteId));
initialize(m_startupConfig.m_serializedCatalog, m_startupConfig.m_timestamp);
m_startupConfig = null; // release the serializedCatalog bytes.
try {
while (m_shouldContinue) {
if (m_rejoinState == kStateRunning) {
// Normal operation blocks the site thread on the sitetasker queue.
SiteTasker task = m_scheduler.take();
if (task instanceof TransactionTask) {
m_currentTxnId = ((TransactionTask) task).getTxnId();
m_lastTxnTime = EstTime.currentTimeMillis();
}
task.run(getSiteProcedureConnection());
} else {
// Rejoin operation poll and try to do some catchup work. Tasks
// are responsible for logging any rejoin work they might have.
SiteTasker task = m_scheduler.poll();
if (task != null) {
task.runForRejoin(getSiteProcedureConnection(), m_rejoinTaskLog);
}
replayFromTaskLog();
}
}
} catch (OutOfMemoryError e) {
// Even though OOM should be caught by the Throwable section below,
// it sadly needs to be handled seperately. The goal here is to make
// sure VoltDB crashes.
String errmsg =
"Site: "
+ org.voltcore.utils.CoreUtils.hsIdToString(m_siteId)
+ " ran out of Java memory. "
+ "This node will shut down.";
VoltDB.crashLocalVoltDB(errmsg, true, e);
} catch (Throwable t) {
String errmsg =
"Site: "
+ org.voltcore.utils.CoreUtils.hsIdToString(m_siteId)
+ " encountered an "
+ "unexpected error and will die, taking this VoltDB node down.";
VoltDB.crashLocalVoltDB(errmsg, true, t);
}
shutdown();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("MpProcedureTask:");
sb.append(" TXN ID: ").append(TxnEgo.txnIdToString(getTxnId()));
sb.append(" SP HANDLE ID: ").append(TxnEgo.txnIdToString(getSpHandle()));
sb.append(" ON HSID: ").append(CoreUtils.hsIdToString(m_initiator.getHSId()));
return sb.toString();
}
public static void logTopology(long leaderHSId, List<Long> replicas, int partitionId) {
if (iv2log.isTraceEnabled()) {
String logmsg = "topology partition %d leader %s replicas (%s)";
iv2log.trace(
String.format(
logmsg,
partitionId,
CoreUtils.hsIdToString(leaderHSId),
CoreUtils.hsIdCollectionToString(replicas)));
}
}
/** Notify the coordinator that this site has received the first fragment message */
private void sendFirstFragResponse() {
if (JOINLOG.isDebugEnabled()) {
JOINLOG.debug(
"P"
+ m_partitionId
+ " sending first fragment response to coordinator "
+ CoreUtils.hsIdToString(m_coordinatorHsId));
}
RejoinMessage msg =
new RejoinMessage(m_mailbox.getHSId(), RejoinMessage.Type.FIRST_FRAGMENT_RECEIVED);
m_mailbox.send(m_coordinatorHsId, msg);
m_firstFragResponseSent = true;
}
@Override
public void run() {
REJOINLOG.debug(
m_whoami
+ "informing rejoinCoordinator "
+ CoreUtils.hsIdToString(m_coordinatorHsId)
+ " of REPLAY_FINISHED");
RejoinMessage replay_complete =
new RejoinMessage(m_mailbox.getHSId(), RejoinMessage.Type.REPLAY_FINISHED);
m_mailbox.send(m_coordinatorHsId, replay_complete);
m_currentlyRejoining.set(false);
SnapshotSaveAPI.recoveringSiteCount.decrementAndGet();
}
public long getBuddySiteForMPI(long hsid) {
int host = CoreUtils.getHostIdFromHSId(hsid);
// We'll be lazy and get the map we'd feed to SiteTracker's
// constructor, then go looking for a matching host ID.
List<MailboxNodeContent> sitesList = getMailboxNodeContentList();
for (MailboxNodeContent site : sitesList) {
if (site.partitionId != MpInitiator.MP_INIT_PID
&& host == CoreUtils.getHostIdFromHSId(site.HSId)) {
return site.HSId;
}
}
throw new RuntimeException(
"Unable to find a buddy initiator for MPI with HSID: " + CoreUtils.hsIdToString(hsid));
}
private long assignLeader(int partitionId, List<Long> children) {
// We used masterHostId = -1 as a way to force the leader choice to be
// the first replica in the list, if we don't have some other mechanism
// which has successfully overridden it.
int masterHostId = -1;
if (m_state.get() == AppointerState.CLUSTER_START) {
try {
// find master in topo
JSONArray parts = m_topo.getJSONArray("partitions");
for (int p = 0; p < parts.length(); p++) {
JSONObject aPartition = parts.getJSONObject(p);
int pid = aPartition.getInt("partition_id");
if (pid == partitionId) {
masterHostId = aPartition.getInt("master");
}
}
} catch (JSONException jse) {
tmLog.error("Failed to find master for partition " + partitionId + ", defaulting to 0");
jse.printStackTrace();
masterHostId = -1; // stupid default
}
} else {
// For now, if we're appointing a new leader as a result of a
// failure, just pick the first replica in the children list.
// Could eventually do something more complex here to try to keep a
// semi-balance, but it's unclear that this has much utility until
// we add rebalancing on rejoin as well.
masterHostId = -1;
}
long masterHSId = children.get(0);
for (Long child : children) {
if (CoreUtils.getHostIdFromHSId(child) == masterHostId) {
masterHSId = child;
break;
}
}
tmLog.info(
"Appointing HSId "
+ CoreUtils.hsIdToString(masterHSId)
+ " as leader for partition "
+ partitionId);
try {
m_iv2appointees.put(partitionId, masterHSId);
} catch (Exception e) {
VoltDB.crashLocalVoltDB("Unable to appoint new master for partition " + partitionId, true, e);
}
return masterHSId;
}
@Override
protected void updateStatsRow(Object rowKey, Object[] rowValues) {
long leader;
List<Long> sites = new ArrayList<Long>();
if (rowKey.equals(MpInitiator.MP_INIT_PID)) {
leader = getHSIdForMultiPartitionInitiator();
sites.add(leader);
} else {
leader = m_iv2Masters.pointInTimeCache().get((Integer) rowKey);
sites.addAll(getReplicasForPartition((Integer) rowKey));
}
rowValues[columnNameToIndex.get("Partition")] = rowKey;
rowValues[columnNameToIndex.get("Sites")] = CoreUtils.hsIdCollectionToString(sites);
rowValues[columnNameToIndex.get("Leader")] = CoreUtils.hsIdToString(leader);
}
public void dump(long hsId) {
final String who = CoreUtils.hsIdToString(hsId);
tmLog.info(
String.format(
"%s: REPLAY SEQUENCER DUMP, LAST POLLED FRAGMENT %d (%s), LAST SEEN TXNID %d (%s), %s%s",
who,
m_lastPolledFragmentTxnId,
TxnEgo.txnIdToString(m_lastPolledFragmentTxnId),
m_lastSeenTxnId,
TxnEgo.txnIdToString(m_lastSeenTxnId),
m_mpiEOLReached ? "MPI EOL, " : "",
m_mustDrain ? "MUST DRAIN" : ""));
for (Entry<Long, ReplayEntry> e : m_replayEntries.entrySet()) {
tmLog.info(String.format("%s: REPLAY ENTRY %s: %s", who, e.getKey(), e.getValue()));
}
}
/** Send missed-messages to survivors. */
public void repairSurvivors() {
// cancel() and repair() must be synchronized by the caller (the deliver lock,
// currently). If cancelled and the last repair message arrives, don't send
// out corrections!
if (this.m_promotionResult.isCancelled()) {
tmLog.debug(m_whoami + "Skipping repair message creation for cancelled Term.");
return;
}
int queued = 0;
tmLog.debug(m_whoami + "received all repair logs and is repairing surviving replicas.");
for (Iv2RepairLogResponseMessage li : m_repairLogUnion) {
List<Long> needsRepair = new ArrayList<Long>(5);
for (Entry<Long, ReplicaRepairStruct> entry : m_replicaRepairStructs.entrySet()) {
if (entry.getValue().needs(li.getHandle())) {
++queued;
tmLog.debug(
m_whoami
+ "repairing "
+ CoreUtils.hsIdToString(entry.getKey())
+ ". Max seen "
+ entry.getValue().m_maxSpHandleSeen
+ ". Repairing with "
+ li.getHandle());
needsRepair.add(entry.getKey());
}
}
if (!needsRepair.isEmpty()) {
if (tmLog.isTraceEnabled()) {
tmLog.trace(
m_whoami
+ "repairing: "
+ CoreUtils.hsIdCollectionToString(needsRepair)
+ " with message: "
+ li.getPayload());
}
m_mailbox.repairReplicasWith(needsRepair, li.getPayload());
}
}
tmLog.debug(m_whoami + "finished queuing " + queued + " replica repair messages.");
m_promotionResult.done(m_maxSeenTxnId);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("COMPLETE_TRANSACTION (FROM COORD: ");
sb.append(CoreUtils.hsIdToString(m_coordinatorHSId));
sb.append(") FOR TXN ");
sb.append(m_txnId);
sb.append("\n FLAGS: ").append(m_flags);
sb.append("\n HASH: " + String.valueOf(m_hash));
if (isRollback()) sb.append("\n THIS IS AN ROLLBACK REQUEST");
if (requiresAck()) sb.append("\n THIS MESSAGE REQUIRES AN ACK");
if (isRestart()) {
sb.append("\n THIS IS A TRANSACTION RESTART");
}
return sb.toString();
}
/**
* The only public method: do all the work to start a snapshot. Assumes that a snapshot is
* feasible, that the caller has validated it can be accomplished, that the caller knows this is a
* consistent or useful transaction point at which to snapshot.
*
* @param file_path
* @param file_nonce
* @param format
* @param block
* @param txnId
* @param data
* @param context
* @param hostname
* @return VoltTable describing the results of the snapshot attempt
*/
public VoltTable startSnapshotting(
final String file_path,
final String file_nonce,
final SnapshotFormat format,
final byte block,
final long multiPartTxnId,
final long partitionTxnId,
final long legacyPerPartitionTxnIds[],
final String data,
final SystemProcedureExecutionContext context,
final String hostname,
final HashinatorSnapshotData hashinatorData,
final long timestamp) {
TRACE_LOG.trace("Creating snapshot target and handing to EEs");
final VoltTable result = SnapshotUtil.constructNodeResultsTable();
final int numLocalSites =
context.getCluster().getDeployment().get("deployment").getSitesperhost();
// One site wins the race to create the snapshot targets, populating
// m_taskListsForSites for the other sites and creating an appropriate
// number of snapshot permits.
synchronized (SnapshotSiteProcessor.m_snapshotCreateLock) {
SnapshotSiteProcessor.m_snapshotCreateSetupBarrierActualAction.set(
new Runnable() {
@Override
public void run() {
Map<Integer, Long> partitionTransactionIds = new HashMap<Integer, Long>();
partitionTransactionIds = m_partitionLastSeenTransactionIds;
SNAP_LOG.debug("Last seen partition transaction ids " + partitionTransactionIds);
m_partitionLastSeenTransactionIds = new HashMap<Integer, Long>();
partitionTransactionIds.put(TxnEgo.getPartitionId(multiPartTxnId), multiPartTxnId);
/*
* Do a quick sanity check that the provided IDs
* don't conflict with currently active partitions. If they do
* it isn't fatal we can just skip it.
*/
for (long txnId : legacyPerPartitionTxnIds) {
final int legacyPartition = TxnEgo.getPartitionId(txnId);
if (partitionTransactionIds.containsKey(legacyPartition)) {
SNAP_LOG.warn(
"While saving a snapshot and propagating legacy "
+ "transaction ids found an id that matches currently active partition"
+ partitionTransactionIds.get(legacyPartition));
} else {
partitionTransactionIds.put(legacyPartition, txnId);
}
}
exportSequenceNumbers = SnapshotSiteProcessor.getExportSequenceNumbers();
createSetupIv2(
file_path,
file_nonce,
format,
multiPartTxnId,
partitionTransactionIds,
data,
context,
result,
exportSequenceNumbers,
context.getSiteTrackerForSnapshot(),
hashinatorData,
timestamp);
}
});
// Create a barrier to use with the current number of sites to wait for
// or if the barrier is already set up check if it is broken and reset if necessary
SnapshotSiteProcessor.readySnapshotSetupBarriers(numLocalSites);
// From within this EE, record the sequence numbers as of the start of the snapshot (now)
// so that the info can be put in the digest.
SnapshotSiteProcessor.populateExportSequenceNumbersForExecutionSite(context);
SNAP_LOG.debug(
"Registering transaction id "
+ partitionTxnId
+ " for "
+ TxnEgo.getPartitionId(partitionTxnId));
m_partitionLastSeenTransactionIds.put(TxnEgo.getPartitionId(partitionTxnId), partitionTxnId);
}
boolean runPostTasks = false;
VoltTable earlyResultTable = null;
try {
SnapshotSiteProcessor.m_snapshotCreateSetupBarrier.await();
try {
synchronized (m_createLock) {
SNAP_LOG.debug(
"Found tasks for HSIds: "
+ CoreUtils.hsIdCollectionToString(m_taskListsForHSIds.keySet()));
SNAP_LOG.debug("Looking for local HSID: " + CoreUtils.hsIdToString(context.getSiteId()));
Deque<SnapshotTableTask> taskList = m_taskListsForHSIds.remove(context.getSiteId());
// If createSetup failed, then the first site to reach here is going
// to send the results table generated by createSetup, and then empty out the table.
// All other sites to reach here will send the appropriate empty table.
// If createSetup was a success but the taskList is null, then we'll use the block
// switch to figure out what flavor of empty SnapshotSave result table to return.
if (!m_createSuccess.get()) {
// There shouldn't be any work for any site if we failed
assert (m_taskListsForHSIds.isEmpty());
VoltTable finalresult = m_createResult.get();
if (finalresult != null) {
m_createResult.set(null);
earlyResultTable = finalresult;
} else {
// We returned a non-empty NodeResultsTable with the failures in it,
// every other site needs to return a NodeResultsTable as well.
earlyResultTable = SnapshotUtil.constructNodeResultsTable();
}
} else if (taskList == null) {
SNAP_LOG.debug("No task for this site, block " + block);
// This node is participating in the snapshot but this site has nothing to do.
// Send back an appropriate empty table based on the block flag
if (block != 0) {
runPostTasks = true;
earlyResultTable = SnapshotUtil.constructPartitionResultsTable();
earlyResultTable.addRow(
context.getHostId(),
hostname,
CoreUtils.getSiteIdFromHSId(context.getSiteId()),
"SUCCESS",
"");
} else {
earlyResultTable = SnapshotUtil.constructNodeResultsTable();
}
} else {
context
.getSiteSnapshotConnection()
.initiateSnapshots(format, taskList, multiPartTxnId, exportSequenceNumbers);
}
if (m_deferredSetupFuture != null) {
// Add a listener to the deferred setup so that it can kick off the snapshot
// task once the setup is done.
m_deferredSetupFuture.addListener(
new Runnable() {
@Override
public void run() {
DeferredSnapshotSetup deferredSnapshotSetup = null;
try {
deferredSnapshotSetup = m_deferredSetupFuture.get();
} catch (Exception e) {
// it doesn't throw
}
assert deferredSnapshotSetup != null;
context
.getSiteSnapshotConnection()
.startSnapshotWithTargets(
deferredSnapshotSetup.getPlan().getSnapshotDataTargets());
}
},
CoreUtils.SAMETHREADEXECUTOR);
}
}
} finally {
SnapshotSiteProcessor.m_snapshotCreateFinishBarrier.await(120, TimeUnit.SECONDS);
}
} catch (TimeoutException e) {
VoltDB.crashLocalVoltDB(
"Timed out waiting 120 seconds for all threads to arrive and start snapshot", true, null);
} catch (InterruptedException e) {
result.addRow(context.getHostId(), hostname, "", "FAILURE", CoreUtils.throwableToString(e));
earlyResultTable = result;
} catch (BrokenBarrierException e) {
result.addRow(context.getHostId(), hostname, "", "FAILURE", CoreUtils.throwableToString(e));
earlyResultTable = result;
}
// If earlyResultTable is set, return here
if (earlyResultTable != null) {
if (runPostTasks) {
// Need to run post-snapshot tasks before finishing
SnapshotSiteProcessor.runPostSnapshotTasks(context);
}
return earlyResultTable;
}
if (block != 0) {
HashSet<Exception> failures = Sets.newHashSet();
String status = "SUCCESS";
String err = "";
try {
// For blocking snapshot, propogate the error from deferred setup back to the client
final DeferredSnapshotSetup deferredSnapshotSetup = m_deferredSetupFuture.get();
if (deferredSnapshotSetup != null && deferredSnapshotSetup.getError() != null) {
status = "FAILURE";
err = deferredSnapshotSetup.getError().toString();
failures.add(deferredSnapshotSetup.getError());
}
failures.addAll(context.getSiteSnapshotConnection().completeSnapshotWork());
SnapshotSiteProcessor.runPostSnapshotTasks(context);
} catch (Exception e) {
status = "FAILURE";
err = e.toString();
failures.add(e);
}
final VoltTable blockingResult = SnapshotUtil.constructPartitionResultsTable();
if (failures.isEmpty()) {
blockingResult.addRow(
context.getHostId(),
hostname,
CoreUtils.getSiteIdFromHSId(context.getSiteId()),
status,
err);
} else {
status = "FAILURE";
for (Exception e : failures) {
err = e.toString();
}
blockingResult.addRow(
context.getHostId(),
hostname,
CoreUtils.getSiteIdFromHSId(context.getSiteId()),
status,
err);
}
return blockingResult;
}
return result;
}
// Offer a new message to the repair log. This will truncate
// the repairLog if the message includes a truncation hint.
public void deliver(VoltMessage msg) {
if (!m_isLeader && msg instanceof Iv2InitiateTaskMessage) {
final Iv2InitiateTaskMessage m = (Iv2InitiateTaskMessage) msg;
// We can't repair read only SP transactions. Just don't log them to the repair log.
if (m.isReadOnly()) {
return;
}
m_lastSpHandle = m.getSpHandle();
truncate(m.getTruncationHandle(), IS_SP);
m_logSP.add(new Item(IS_SP, m, m.getSpHandle(), m.getTxnId()));
} else if (msg instanceof FragmentTaskMessage) {
final FragmentTaskMessage m = (FragmentTaskMessage) msg;
// We can't repair read only SP transactions. Just don't log them to the repair log.
if (m.isReadOnly()) {
return;
}
truncate(m.getTruncationHandle(), IS_MP);
// only log the first fragment of a procedure (and handle 1st case)
if (m.getTxnId() > m_lastMpHandle || m_lastMpHandle == Long.MAX_VALUE) {
m_logMP.add(new Item(IS_MP, m, m.getSpHandle(), m.getTxnId()));
m_lastMpHandle = m.getTxnId();
m_lastSpHandle = m.getSpHandle();
}
} else if (msg instanceof CompleteTransactionMessage) {
// a CompleteTransactionMessage which indicates restart is not the end of the
// transaction. We don't want to log it in the repair log.
CompleteTransactionMessage ctm = (CompleteTransactionMessage) msg;
// We can't repair read only SP transactions. Just don't log them to the repair log.
// Restart transaction do not need to be repaired here, don't log them as well.
if (ctm.isReadOnly() || ctm.isRestart()) {
return;
}
truncate(ctm.getTruncationHandle(), IS_MP);
m_logMP.add(new Item(IS_MP, ctm, ctm.getSpHandle(), ctm.getTxnId()));
// Restore will send a complete transaction message with a lower mp transaction id because
// the restore transaction precedes the loading of the right mp transaction id from the
// snapshot
// Hence Math.max
m_lastMpHandle = Math.max(m_lastMpHandle, ctm.getTxnId());
m_lastSpHandle = ctm.getSpHandle();
} else if (msg instanceof DumpMessage) {
String who = CoreUtils.hsIdToString(m_HSId);
tmLog.warn(
"Repair log dump for site: "
+ who
+ ", isLeader: "
+ m_isLeader
+ ", "
+ who
+ ": lastSpHandle: "
+ m_lastSpHandle
+ ", lastMpHandle: "
+ m_lastMpHandle);
for (Iv2RepairLogResponseMessage il : contents(0l, false)) {
tmLog.warn("[Repair log contents]" + who + ": msg: " + il);
}
} else if (msg instanceof RepairLogTruncationMessage) {
final RepairLogTruncationMessage truncateMsg = (RepairLogTruncationMessage) msg;
truncate(truncateMsg.getHandle(), IS_SP);
}
}
