@Override
public void run(SiteProcedureConnection siteConnection) {
hostLog.debug("STARTING: " + this);
if (!m_txnState.isReadOnly()) {
// the truncation point token SHOULD be part of m_txn. However, the
// legacy interaces don't work this way and IV2 hasn't changed this
// ownership yet. But truncateUndoLog is written assuming the right
// eventual encapsulation.
siteConnection.truncateUndoLog(
m_completeMsg.isRollback(),
m_txnState.getBeginUndoToken(),
m_txnState.m_spHandle,
m_txnState.getUndoLog());
}
if (!m_completeMsg.isRestart()) {
doCommonSPICompleteActions();
// Log invocation to DR
logToDR(siteConnection.getDRGateway());
hostLog.debug("COMPLETE: " + this);
} else {
// If we're going to restart the transaction, then reset the begin undo token so the
// first FragmentTask will set it correctly. Otherwise, don't set the Done state or
// flush the queue; we want the TransactionTaskQueue to stay blocked on this TXN ID
// for the restarted fragments.
m_txnState.setBeginUndoToken(Site.kInvalidUndoToken);
hostLog.debug("RESTART: " + this);
}
final CompleteTransactionResponseMessage resp =
new CompleteTransactionResponseMessage(m_completeMsg);
resp.m_sourceHSId = m_initiator.getHSId();
m_initiator.deliver(resp);
}
private void logToDR(PartitionDRGateway drGateway) {
// Log invocation to DR
if (drGateway != null
&& !m_txnState.isForReplay()
&& !m_txnState.isReadOnly()
&& !m_completeMsg.isRollback()) {
FragmentTaskMessage fragment = (FragmentTaskMessage) m_txnState.getNotice();
Iv2InitiateTaskMessage initiateTask = fragment.getInitiateTask();
assert (initiateTask != null);
if (initiateTask == null) {
hostLog.error(
"Unable to log MP transaction to DR because of missing InitiateTaskMessage, "
+ "fragment: "
+ fragment.toString());
}
StoredProcedureInvocation invocation =
initiateTask.getStoredProcedureInvocation().getShallowCopy();
drGateway.onSuccessfulMPCall(
m_txnState.m_spHandle,
m_txnState.txnId,
m_txnState.uniqueId,
m_completeMsg.getHash(),
invocation,
m_txnState.getResults());
}
}
public static void logCompleteTransactionMessage(
CompleteTransactionMessage ctask, long localHSId) {
if (iv2log.isTraceEnabled()) {
String logmsg = "rxCompMsg %s from %s txnId %s %s %s";
iv2log.trace(
String.format(
logmsg,
CoreUtils.hsIdToString(localHSId),
CoreUtils.hsIdToString(ctask.m_sourceHSId),
txnIdToString(ctask.getTxnId()),
ctask.isRollback() ? "ROLLBACK" : "COMMIT",
ctask.isRestart() ? "RESTART" : ""));
}
}
@Override
public void processCompleteTransaction(CompleteTransactionMessage complete) {
m_done = true;
if (complete.isRollback()) {
if (m_missingDependencies != null) {
m_missingDependencies.clear();
}
m_readyWorkUnits.clear();
m_needsRollback = true;
}
if (complete.requiresAck()) {
CompleteTransactionResponseMessage ctrm =
new CompleteTransactionResponseMessage(complete, m_hsId);
m_mbox.send(complete.getCoordinatorHSId(), ctrm);
}
}
@Override
public void runForRejoin(SiteProcedureConnection siteConnection, TaskLog taskLog)
throws IOException {
if (!m_txnState.isReadOnly() && !m_completeMsg.isRollback()) {
// ENG-5276: Need to set the last committed spHandle so that the rejoining site gets the
// accurate
// per-partition txnId set for the next snapshot. Normally, this is done through undo log
// truncation.
// Since the task is not run here, we need to set the last committed spHandle explicitly.
//
// How does this work?
// - Blocking rejoin with idle cluster: The spHandle is updated here with the spHandle of the
// stream
// snapshot that transfers the rejoin data. So the snapshot right after rejoin should have
// the spHandle
// passed here.
// - Live rejoin with idle cluster: Same as blocking rejoin.
// - Live rejoin with workload: Transactions will be logged and replayed afterward. The
// spHandle will be
// updated when they commit and truncate undo logs. So at the end of replay,
// the spHandle should have the latest value. If all replayed transactions rolled back,
// the spHandle is still guaranteed to be the spHandle of the stream snapshot that
// transfered the
// rejoin data, which is the correct value.
siteConnection.setSpHandleForSnapshotDigest(m_txnState.m_spHandle);
}
if (!m_completeMsg.isRestart()) {
// future: offer to siteConnection.IBS for replay.
doCommonSPICompleteActions();
}
if (!m_txnState.isReadOnly()) {
// We need to log the restarting message to the task log so we'll replay the whole
// stream faithfully
taskLog.logTask(m_completeMsg);
}
final CompleteTransactionResponseMessage resp =
new CompleteTransactionResponseMessage(m_completeMsg);
resp.setIsRecovering(true);
resp.m_sourceHSId = m_initiator.getHSId();
m_initiator.deliver(resp);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("CompleteTransactionTask:");
sb.append(" TXN ID: ").append(TxnEgo.txnIdToString(getTxnId()));
sb.append(" SP HANDLE: ").append(TxnEgo.txnIdToString(getSpHandle()));
sb.append(" UNDO TOKEN: ").append(m_txnState.getBeginUndoToken());
sb.append(" MSG: ").append(m_completeMsg.toString());
return sb.toString();
}
@Override
void completeInitiateTask(SiteProcedureConnection siteConnection) {
CompleteTransactionMessage complete =
new CompleteTransactionMessage(
m_initiator.getHSId(), // who is the "initiator" now??
m_initiator.getHSId(),
m_txnState.txnId,
m_txnState.isReadOnly(),
m_txnState.getHash(),
m_txnState.needsRollback(),
false, // really don't want to have ack the ack.
false,
m_msg.isForReplay());
complete.setTruncationHandle(m_msg.getTruncationHandle());
complete.setOriginalTxnId(m_msg.getOriginalTxnId());
m_initiator.send(com.google.common.primitives.Longs.toArray(m_initiatorHSIds), complete);
m_txnState.setDone();
m_queue.flush();
}
@Override
public void runFromTaskLog(SiteProcedureConnection siteConnection) {
if (!m_txnState.isReadOnly()) {
// the truncation point token SHOULD be part of m_txn. However, the
// legacy interaces don't work this way and IV2 hasn't changed this
// ownership yet. But truncateUndoLog is written assuming the right
// eventual encapsulation.
siteConnection.truncateUndoLog(
m_completeMsg.isRollback(),
m_txnState.getBeginUndoToken(),
m_txnState.m_spHandle,
m_txnState.getUndoLog());
}
if (!m_completeMsg.isRestart()) {
// this call does the right thing with a null TransactionTaskQueue
doCommonSPICompleteActions();
logToDR(siteConnection.getDRGateway());
} else {
m_txnState.setBeginUndoToken(Site.kInvalidUndoToken);
}
}
public void handleCompleteTransactionMessage(CompleteTransactionMessage message) {
if (m_isLeader) {
CompleteTransactionMessage replmsg = new CompleteTransactionMessage(message);
// Set the spHandle so that on repair the new master will set the max seen spHandle
// correctly
advanceTxnEgo();
replmsg.setSpHandle(getCurrentTxnId());
if (m_sendToHSIds.length > 0) {
m_mailbox.send(m_sendToHSIds, replmsg);
}
} else {
setMaxSeenTxnId(message.getSpHandle());
}
TransactionState txn = m_outstandingTxns.get(message.getTxnId());
// We can currently receive CompleteTransactionMessages for multipart procedures
// which only use the buddy site (replicated table read). Ignore them for
// now, fix that later.
if (txn != null) {
Iv2Trace.logCompleteTransactionMessage(message, m_mailbox.getHSId());
final CompleteTransactionTask task =
new CompleteTransactionTask(txn, m_pendingTasks, message, m_drGateway);
queueOrOfferMPTask(task);
// If this is a restart, then we need to leave the transaction state around
if (!message.isRestart()) {
m_outstandingTxns.remove(message.getTxnId());
}
}
}
@Override
public long getSpHandle() {
return m_completeMsg.getSpHandle();
}
/** Run is invoked by a run-loop to execute this transaction. */
@Override
public void run(SiteProcedureConnection siteConnection) {
hostLog.debug("STARTING: " + this);
// Cast up. Could avoid ugliness with Iv2TransactionClass baseclass
MpTransactionState txn = (MpTransactionState) m_txnState;
// Check for restarting sysprocs
String spName = txn.m_initiationMsg.getStoredProcedureName();
// certain system procs can and can't be restarted
// Right now this is adhoc and catalog update. Since these are treated specially
// in a few places (here, recovery, dr), maybe we should add another metadata
// property the sysproc registry about whether a proc can be restarted/recovered/dr-ed
if (m_isRestart
&& spName.startsWith("@")
&& !spName.startsWith("@AdHoc")
&& !spName.equals("@UpdateApplicationCatalog")) {
InitiateResponseMessage errorResp = new InitiateResponseMessage(txn.m_initiationMsg);
errorResp.setResults(
new ClientResponseImpl(
ClientResponse.UNEXPECTED_FAILURE,
new VoltTable[] {},
"Failure while running system procedure "
+ txn.m_initiationMsg.getStoredProcedureName()
+ ", and system procedures can not be restarted."));
txn.setNeedsRollback();
completeInitiateTask(siteConnection);
errorResp.m_sourceHSId = m_initiator.getHSId();
m_initiator.deliver(errorResp);
hostLog.debug("SYSPROCFAIL: " + this);
return;
}
// Let's ensure that we flush any previous attempts of this transaction
// at the masters we're going to try to use this time around.
if (m_isRestart) {
CompleteTransactionMessage restart =
new CompleteTransactionMessage(
m_initiator.getHSId(), // who is the "initiator" now??
m_initiator.getHSId(),
m_txnState.txnId,
m_txnState.isReadOnly(),
0,
true,
false, // really don't want to have ack the ack.
!m_txnState.isReadOnly(),
m_msg.isForReplay());
restart.setTruncationHandle(m_msg.getTruncationHandle());
restart.setOriginalTxnId(m_msg.getOriginalTxnId());
m_initiator.send(com.google.common.primitives.Longs.toArray(m_initiatorHSIds), restart);
}
final InitiateResponseMessage response =
processInitiateTask(txn.m_initiationMsg, siteConnection);
// We currently don't want to restart read-only MP transactions because:
// 1) We're not writing the Iv2InitiateTaskMessage to the first
// FragmentTaskMessage in read-only case in the name of some unmeasured
// performance impact,
// 2) We don't want to perturb command logging and/or DR this close to the 3.0 release
// 3) We don't guarantee the restarted results returned to the client
// anyway, so not restarting the read is currently harmless.
// We could actually restart this here, since we have the invocation, but let's be consistent?
int status = response.getClientResponseData().getStatus();
if (status != ClientResponse.TXN_RESTART
|| (status == ClientResponse.TXN_RESTART && m_msg.isReadOnly())) {
if (!response.shouldCommit()) {
txn.setNeedsRollback();
}
completeInitiateTask(siteConnection);
// Set the source HSId (ugh) to ourselves so we track the message path correctly
response.m_sourceHSId = m_initiator.getHSId();
m_initiator.deliver(response);
execLog.l7dlog(
Level.TRACE, LogKeys.org_voltdb_ExecutionSite_SendingCompletedWUToDtxn.name(), null);
hostLog.debug("COMPLETE: " + this);
} else {
restartTransaction();
hostLog.debug("RESTART: " + this);
}
}
// Offer a new message to the repair log. This will truncate
// the repairLog if the message includes a truncation hint.
public void deliver(VoltMessage msg) {
if (!m_isLeader && msg instanceof Iv2InitiateTaskMessage) {
final Iv2InitiateTaskMessage m = (Iv2InitiateTaskMessage) msg;
// We can't repair read only SP transactions. Just don't log them to the repair log.
if (m.isReadOnly()) {
return;
}
m_lastSpHandle = m.getSpHandle();
truncate(m.getTruncationHandle(), IS_SP);
m_logSP.add(new Item(IS_SP, m, m.getSpHandle(), m.getTxnId()));
} else if (msg instanceof FragmentTaskMessage) {
final FragmentTaskMessage m = (FragmentTaskMessage) msg;
// We can't repair read only SP transactions. Just don't log them to the repair log.
if (m.isReadOnly()) {
return;
}
truncate(m.getTruncationHandle(), IS_MP);
// only log the first fragment of a procedure (and handle 1st case)
if (m.getTxnId() > m_lastMpHandle || m_lastMpHandle == Long.MAX_VALUE) {
m_logMP.add(new Item(IS_MP, m, m.getSpHandle(), m.getTxnId()));
m_lastMpHandle = m.getTxnId();
m_lastSpHandle = m.getSpHandle();
}
} else if (msg instanceof CompleteTransactionMessage) {
// a CompleteTransactionMessage which indicates restart is not the end of the
// transaction. We don't want to log it in the repair log.
CompleteTransactionMessage ctm = (CompleteTransactionMessage) msg;
// We can't repair read only SP transactions. Just don't log them to the repair log.
// Restart transaction do not need to be repaired here, don't log them as well.
if (ctm.isReadOnly() || ctm.isRestart()) {
return;
}
truncate(ctm.getTruncationHandle(), IS_MP);
m_logMP.add(new Item(IS_MP, ctm, ctm.getSpHandle(), ctm.getTxnId()));
// Restore will send a complete transaction message with a lower mp transaction id because
// the restore transaction precedes the loading of the right mp transaction id from the
// snapshot
// Hence Math.max
m_lastMpHandle = Math.max(m_lastMpHandle, ctm.getTxnId());
m_lastSpHandle = ctm.getSpHandle();
} else if (msg instanceof DumpMessage) {
String who = CoreUtils.hsIdToString(m_HSId);
tmLog.warn(
"Repair log dump for site: "
+ who
+ ", isLeader: "
+ m_isLeader
+ ", "
+ who
+ ": lastSpHandle: "
+ m_lastSpHandle
+ ", lastMpHandle: "
+ m_lastMpHandle);
for (Iv2RepairLogResponseMessage il : contents(0l, false)) {
tmLog.warn("[Repair log contents]" + who + ": msg: " + il);
}
} else if (msg instanceof RepairLogTruncationMessage) {
final RepairLogTruncationMessage truncateMsg = (RepairLogTruncationMessage) msg;
truncate(truncateMsg.getHandle(), IS_SP);
}
}
void replayFromTaskLog() throws IOException {
// not yet time to catch-up.
if (m_rejoinState != kStateReplayingRejoin) {
return;
}
// replay 10:1 in favor of replay
for (int i = 0; i < 10; ++i) {
if (m_rejoinTaskLog.isEmpty()) {
break;
}
TransactionInfoBaseMessage tibm = m_rejoinTaskLog.getNextMessage();
if (tibm == null) {
break;
}
// Apply the readonly / sysproc filter. With Iv2 read optimizations,
// reads should not reach here; the cost of post-filtering shouldn't
// be particularly high (vs pre-filtering).
if (filter(tibm)) {
continue;
}
if (tibm instanceof Iv2InitiateTaskMessage) {
Iv2InitiateTaskMessage m = (Iv2InitiateTaskMessage) tibm;
SpProcedureTask t =
new SpProcedureTask(m_initiatorMailbox, m.getStoredProcedureName(), null, m, null);
t.runFromTaskLog(this);
} else if (tibm instanceof FragmentTaskMessage) {
FragmentTaskMessage m = (FragmentTaskMessage) tibm;
if (global_replay_mpTxn == null) {
global_replay_mpTxn = new ParticipantTransactionState(m.getTxnId(), m);
} else if (global_replay_mpTxn.txnId != m.getTxnId()) {
VoltDB.crashLocalVoltDB(
"Started a MP transaction during replay before completing " + " open transaction.",
false,
null);
}
FragmentTask t = new FragmentTask(m_initiatorMailbox, m, global_replay_mpTxn);
t.runFromTaskLog(this);
} else if (tibm instanceof CompleteTransactionMessage) {
// Needs improvement: completes for sysprocs aren't filterable as sysprocs.
// Only complete transactions that are open...
if (global_replay_mpTxn != null) {
CompleteTransactionMessage m = (CompleteTransactionMessage) tibm;
CompleteTransactionTask t =
new CompleteTransactionTask(global_replay_mpTxn, null, m, null);
if (!m.isRestart()) {
global_replay_mpTxn = null;
}
t.runFromTaskLog(this);
}
} else {
VoltDB.crashLocalVoltDB(
"Can not replay message type " + tibm + " during live rejoin. Unexpected error.",
false,
null);
}
}
// exit replay being careful not to exit in the middle of a multi-partititon
// transaction. The SPScheduler doesn't have a valid transaction state for a
// partially replayed MP txn and in case of rollback the scheduler's undo token
// is wrong. Run MP txns fully kStateRejoining or fully kStateRunning.
if (m_rejoinTaskLog.isEmpty() && global_replay_mpTxn == null) {
setReplayRejoinComplete();
}
}