Example #1
0
  @Override
  public void run() {
    initialize();

    try {
      while (m_shouldContinue) {
        // Normal operation blocks the site thread on the sitetasker queue.
        SiteTasker task = m_scheduler.take();
        task.run(getSiteProcedureConnection());
      }
    } catch (OutOfMemoryError e) {
      // Even though OOM should be caught by the Throwable section below,
      // it sadly needs to be handled seperately. The goal here is to make
      // sure VoltDB crashes.
      String errmsg =
          "Site: "
              + org.voltcore.utils.CoreUtils.hsIdToString(m_siteId)
              + " ran out of Java memory. "
              + "This node will shut down.";
      VoltDB.crashLocalVoltDB(errmsg, true, e);
    } catch (Throwable t) {
      String errmsg =
          "Site: "
              + org.voltcore.utils.CoreUtils.hsIdToString(m_siteId)
              + " encountered an "
              + "unexpected error and will die, taking this VoltDB node down.";
      VoltDB.crashLocalVoltDB(errmsg, true, t);
    }
    shutdown();
  }
Example #2
0
 public static void logFragmentTaskMessage(
     FragmentTaskMessage ftask, long localHSId, long spHandle, boolean borrow) {
   if (iv2log.isTraceEnabled()) {
     String label = "rxFragMsg";
     if (borrow) {
       label = "rxBrrwMsg";
     }
     if (ftask.getSpHandle() != Long.MIN_VALUE && ftask.getSpHandle() != spHandle) {
       iv2log.error(
           "FragmentTaskMessage SP HANDLE conflict.  Message: "
               + ftask.getSpHandle()
               + ", locally held: "
               + spHandle);
     }
     String logmsg = "%s %s from %s txnId %s spHandle %s trunc %s";
     iv2log.trace(
         String.format(
             logmsg,
             label,
             CoreUtils.hsIdToString(localHSId),
             CoreUtils.hsIdToString(ftask.m_sourceHSId),
             txnIdToString(ftask.getTxnId()),
             txnIdToString(spHandle),
             txnIdToString(ftask.getTruncationHandle())));
   }
 }
Example #3
0
  /** Runs when the RejoinCoordinator decides this site should start rejoin. */
  void doInitiation(RejoinMessage message) {
    m_coordinatorHsId = message.m_sourceHSId;
    m_streamSnapshotMb = VoltDB.instance().getHostMessenger().createMailbox();
    m_rejoinSiteProcessor = new StreamSnapshotSink(m_streamSnapshotMb);

    // MUST choose the leader as the source.
    long sourceSite = m_mailbox.getMasterHsId(m_partitionId);
    long hsId =
        m_rejoinSiteProcessor.initialize(
            message.getSnapshotSourceCount(), message.getSnapshotBufferPool());

    REJOINLOG.debug(
        m_whoami
            + "received INITIATION message. Doing rejoin"
            + ". Source site is: "
            + CoreUtils.hsIdToString(sourceSite)
            + " and destination rejoin processor is: "
            + CoreUtils.hsIdToString(hsId)
            + " and snapshot nonce is: "
            + message.getSnapshotNonce());

    registerSnapshotMonitor(message.getSnapshotNonce());
    // Tell the RejoinCoordinator everything it will need to know to get us our snapshot stream.
    RejoinMessage initResp = new RejoinMessage(m_mailbox.getHSId(), sourceSite, hsId);
    m_mailbox.send(m_coordinatorHsId, initResp);

    // Start waiting for snapshot data
    m_taskQueue.offer(this);
  }
Example #4
0
 public static void logIv2InitiateTaskMessage(
     Iv2InitiateTaskMessage itask, long localHSId, long txnid, long spHandle) {
   if (iv2log.isTraceEnabled()) {
     String logmsg = "rxInitMsg %s from %s ciHandle %s txnId %s spHandle %s trunc %s";
     if (itask.getTxnId() != Long.MIN_VALUE && itask.getTxnId() != txnid) {
       iv2log.error(
           "Iv2InitiateTaskMessage TXN ID conflict.  Message: "
               + itask.getTxnId()
               + ", locally held: "
               + txnid);
     }
     if (itask.getSpHandle() != Long.MIN_VALUE && itask.getSpHandle() != spHandle) {
       iv2log.error(
           "Iv2InitiateTaskMessage SP HANDLE conflict.  Message: "
               + itask.getSpHandle()
               + ", locally held: "
               + spHandle);
     }
     iv2log.trace(
         String.format(
             logmsg,
             CoreUtils.hsIdToString(localHSId),
             CoreUtils.hsIdToString(itask.m_sourceHSId),
             ClientInterfaceHandleManager.handleToString(itask.getClientInterfaceHandle()),
             txnIdToString(txnid),
             txnIdToString(spHandle),
             txnIdToString(itask.getTruncationHandle())));
   }
 }
Example #5
0
 public static void logInitiatorRxMsg(VoltMessage msg, long localHSId) {
   if (iv2log.isTraceEnabled()) {
     if (msg instanceof InitiateResponseMessage) {
       InitiateResponseMessage iresp = (InitiateResponseMessage) msg;
       String logmsg = "rxInitRsp %s from %s ciHandle %s txnId %s spHandle %s status %s";
       iv2log.trace(
           String.format(
               logmsg,
               CoreUtils.hsIdToString(localHSId),
               CoreUtils.hsIdToString(iresp.m_sourceHSId),
               ClientInterfaceHandleManager.handleToString(iresp.getClientInterfaceHandle()),
               txnIdToString(iresp.getTxnId()),
               txnIdToString(iresp.getSpHandle()),
               respStatusToString(iresp.getClientResponseData().getStatus())));
     } else if (msg instanceof FragmentResponseMessage) {
       FragmentResponseMessage fresp = (FragmentResponseMessage) msg;
       String logmsg = "rxFragRsp %s from %s txnId %s spHandle %s status %s";
       iv2log.trace(
           String.format(
               logmsg,
               CoreUtils.hsIdToString(localHSId),
               CoreUtils.hsIdToString(fresp.m_sourceHSId),
               txnIdToString(fresp.getTxnId()),
               txnIdToString(fresp.getSpHandle()),
               fragStatusToString(fresp.getStatusCode())));
     }
   }
 }
Example #6
0
 /** Process a new repair log response */
 @Override
 public void deliver(VoltMessage message) {
   if (message instanceof Iv2RepairLogResponseMessage) {
     Iv2RepairLogResponseMessage response = (Iv2RepairLogResponseMessage) message;
     if (response.getRequestId() != m_requestId) {
       tmLog.debug(
           m_whoami
               + "rejecting stale repair response."
               + " Current request id is: "
               + m_requestId
               + " Received response for request id: "
               + response.getRequestId());
       return;
     }
     ReplicaRepairStruct rrs = m_replicaRepairStructs.get(response.m_sourceHSId);
     if (rrs.m_expectedResponses < 0) {
       tmLog.debug(
           m_whoami
               + "collecting "
               + response.getOfTotal()
               + " repair log entries from "
               + CoreUtils.hsIdToString(response.m_sourceHSId));
     }
     // Long.MAX_VALUE has rejoin semantics
     if (response.getHandle() != Long.MAX_VALUE) {
       m_maxSeenTxnId = Math.max(m_maxSeenTxnId, response.getHandle());
     }
     if (response.getPayload() != null) {
       m_repairLogUnion.add(response);
       if (tmLog.isTraceEnabled()) {
         tmLog.trace(
             m_whoami
                 + " collected from "
                 + CoreUtils.hsIdToString(response.m_sourceHSId)
                 + ", message: "
                 + response.getPayload());
       }
     }
     if (rrs.update(response)) {
       tmLog.debug(
           m_whoami
               + "collected "
               + rrs.m_receivedResponses
               + " responses for "
               + rrs.m_expectedResponses
               + " repair log entries from "
               + CoreUtils.hsIdToString(response.m_sourceHSId));
       if (areRepairLogsComplete()) {
         repairSurvivors();
       }
     }
   }
 }
Example #7
0
 public static void logFinishTransaction(InitiateResponseMessage msg, long localHSId) {
   if (iv2log.isTraceEnabled()) {
     String logmsg = "finishTxn %s ciHandle %s initHSId %s status %s";
     iv2log.trace(
         String.format(
             logmsg,
             CoreUtils.hsIdToString(localHSId),
             ClientInterfaceHandleManager.handleToString(msg.getClientInterfaceHandle()),
             CoreUtils.hsIdToString(msg.getCoordinatorHSId()),
             respStatusToString(msg.getClientResponseData().getStatus())));
   }
 }
Example #8
0
 public static void logCreateTransaction(Iv2InitiateTaskMessage msg) {
   if (iv2log.isTraceEnabled()) {
     String logmsg = "createTxn %s ciHandle %s initHSId %s proc %s";
     iv2log.trace(
         String.format(
             logmsg,
             CoreUtils.hsIdToString(msg.getInitiatorHSId()),
             ClientInterfaceHandleManager.handleToString(msg.getClientInterfaceHandle()),
             CoreUtils.hsIdToString(msg.getCoordinatorHSId()),
             msg.getStoredProcedureInvocation().getProcName()));
   }
 }
Example #9
0
 public static void logIv2MultipartSentinel(
     MultiPartitionParticipantMessage message, long localHSId, long txnId) {
   if (iv2log.isTraceEnabled()) {
     String logmsg = "rxSntlMsg %s from %s txnId %s";
     iv2log.trace(
         String.format(
             logmsg,
             CoreUtils.hsIdToString(localHSId),
             CoreUtils.hsIdToString(message.m_sourceHSId),
             txnIdToString(txnId)));
   }
 }
Example #10
0
 public static void logCompleteTransactionMessage(
     CompleteTransactionMessage ctask, long localHSId) {
   if (iv2log.isTraceEnabled()) {
     String logmsg = "rxCompMsg %s from %s txnId %s %s %s";
     iv2log.trace(
         String.format(
             logmsg,
             CoreUtils.hsIdToString(localHSId),
             CoreUtils.hsIdToString(ctask.m_sourceHSId),
             txnIdToString(ctask.getTxnId()),
             ctask.isRollback() ? "ROLLBACK" : "COMMIT",
             ctask.isRestart() ? "RESTART" : ""));
   }
 }
Example #11
0
 public void handleDumpMessage() {
   String who = CoreUtils.hsIdToString(m_mailbox.getHSId());
   hostLog.warn("State dump for site: " + who);
   hostLog.warn("" + who + ": partition: " + m_partitionId + ", isLeader: " + m_isLeader);
   if (m_isLeader) {
     hostLog.warn("" + who + ": replicas: " + CoreUtils.hsIdCollectionToString(m_replicaHSIds));
     if (m_sendToHSIds.length > 0) {
       m_mailbox.send(m_sendToHSIds, new DumpMessage());
     }
   }
   hostLog.warn(
       ""
           + who
           + ": most recent SP handle: "
           + getCurrentTxnId()
           + " "
           + TxnEgo.txnIdToString(getCurrentTxnId()));
   hostLog.warn(
       ""
           + who
           + ": outstanding txns: "
           + m_outstandingTxns.keySet()
           + " "
           + TxnEgo.txnIdCollectionToString(m_outstandingTxns.keySet()));
   hostLog.warn("" + who + ": TransactionTaskQueue: " + m_pendingTasks.toString());
   if (m_duplicateCounters.size() > 0) {
     hostLog.warn("" + who + ": duplicate counters: ");
     for (Entry<DuplicateCounterKey, DuplicateCounter> e : m_duplicateCounters.entrySet()) {
       hostLog.warn("\t" + who + ": " + e.getKey().toString() + ": " + e.getValue().toString());
     }
   }
 }
Example #12
0
  @Override
  public void run() {
    Thread.currentThread().setName("Iv2ExecutionSite: " + CoreUtils.hsIdToString(m_siteId));
    initialize(m_startupConfig.m_serializedCatalog, m_startupConfig.m_timestamp);
    m_startupConfig = null; // release the serializedCatalog bytes.

    try {
      while (m_shouldContinue) {
        if (m_rejoinState == kStateRunning) {
          // Normal operation blocks the site thread on the sitetasker queue.
          SiteTasker task = m_scheduler.take();
          if (task instanceof TransactionTask) {
            m_currentTxnId = ((TransactionTask) task).getTxnId();
            m_lastTxnTime = EstTime.currentTimeMillis();
          }
          task.run(getSiteProcedureConnection());
        } else {
          // Rejoin operation poll and try to do some catchup work. Tasks
          // are responsible for logging any rejoin work they might have.
          SiteTasker task = m_scheduler.poll();
          if (task != null) {
            task.runForRejoin(getSiteProcedureConnection(), m_rejoinTaskLog);
          }
          replayFromTaskLog();
        }
      }
    } catch (OutOfMemoryError e) {
      // Even though OOM should be caught by the Throwable section below,
      // it sadly needs to be handled seperately. The goal here is to make
      // sure VoltDB crashes.
      String errmsg =
          "Site: "
              + org.voltcore.utils.CoreUtils.hsIdToString(m_siteId)
              + " ran out of Java memory. "
              + "This node will shut down.";
      VoltDB.crashLocalVoltDB(errmsg, true, e);
    } catch (Throwable t) {
      String errmsg =
          "Site: "
              + org.voltcore.utils.CoreUtils.hsIdToString(m_siteId)
              + " encountered an "
              + "unexpected error and will die, taking this VoltDB node down.";
      VoltDB.crashLocalVoltDB(errmsg, true, t);
    }
    shutdown();
  }
Example #13
0
 @Override
 public String toString() {
   StringBuilder sb = new StringBuilder();
   sb.append("MpProcedureTask:");
   sb.append("  TXN ID: ").append(TxnEgo.txnIdToString(getTxnId()));
   sb.append("  SP HANDLE ID: ").append(TxnEgo.txnIdToString(getSpHandle()));
   sb.append("  ON HSID: ").append(CoreUtils.hsIdToString(m_initiator.getHSId()));
   return sb.toString();
 }
Example #14
0
 public static void logTopology(long leaderHSId, List<Long> replicas, int partitionId) {
   if (iv2log.isTraceEnabled()) {
     String logmsg = "topology partition %d leader %s replicas (%s)";
     iv2log.trace(
         String.format(
             logmsg,
             partitionId,
             CoreUtils.hsIdToString(leaderHSId),
             CoreUtils.hsIdCollectionToString(replicas)));
   }
 }
Example #15
0
 /** Notify the coordinator that this site has received the first fragment message */
 private void sendFirstFragResponse() {
   if (JOINLOG.isDebugEnabled()) {
     JOINLOG.debug(
         "P"
             + m_partitionId
             + " sending first fragment response to coordinator "
             + CoreUtils.hsIdToString(m_coordinatorHsId));
   }
   RejoinMessage msg =
       new RejoinMessage(m_mailbox.getHSId(), RejoinMessage.Type.FIRST_FRAGMENT_RECEIVED);
   m_mailbox.send(m_coordinatorHsId, msg);
   m_firstFragResponseSent = true;
 }
Example #16
0
    @Override
    public void run() {
      REJOINLOG.debug(
          m_whoami
              + "informing rejoinCoordinator "
              + CoreUtils.hsIdToString(m_coordinatorHsId)
              + " of REPLAY_FINISHED");
      RejoinMessage replay_complete =
          new RejoinMessage(m_mailbox.getHSId(), RejoinMessage.Type.REPLAY_FINISHED);
      m_mailbox.send(m_coordinatorHsId, replay_complete);
      m_currentlyRejoining.set(false);

      SnapshotSaveAPI.recoveringSiteCount.decrementAndGet();
    }
Example #17
0
 public long getBuddySiteForMPI(long hsid) {
   int host = CoreUtils.getHostIdFromHSId(hsid);
   // We'll be lazy and get the map we'd feed to SiteTracker's
   // constructor, then go looking for a matching host ID.
   List<MailboxNodeContent> sitesList = getMailboxNodeContentList();
   for (MailboxNodeContent site : sitesList) {
     if (site.partitionId != MpInitiator.MP_INIT_PID
         && host == CoreUtils.getHostIdFromHSId(site.HSId)) {
       return site.HSId;
     }
   }
   throw new RuntimeException(
       "Unable to find a buddy initiator for MPI with HSID: " + CoreUtils.hsIdToString(hsid));
 }
Example #18
0
  private long assignLeader(int partitionId, List<Long> children) {
    // We used masterHostId = -1 as a way to force the leader choice to be
    // the first replica in the list, if we don't have some other mechanism
    // which has successfully overridden it.
    int masterHostId = -1;
    if (m_state.get() == AppointerState.CLUSTER_START) {
      try {
        // find master in topo
        JSONArray parts = m_topo.getJSONArray("partitions");
        for (int p = 0; p < parts.length(); p++) {
          JSONObject aPartition = parts.getJSONObject(p);
          int pid = aPartition.getInt("partition_id");
          if (pid == partitionId) {
            masterHostId = aPartition.getInt("master");
          }
        }
      } catch (JSONException jse) {
        tmLog.error("Failed to find master for partition " + partitionId + ", defaulting to 0");
        jse.printStackTrace();
        masterHostId = -1; // stupid default
      }
    } else {
      // For now, if we're appointing a new leader as a result of a
      // failure, just pick the first replica in the children list.
      // Could eventually do something more complex here to try to keep a
      // semi-balance, but it's unclear that this has much utility until
      // we add rebalancing on rejoin as well.
      masterHostId = -1;
    }

    long masterHSId = children.get(0);
    for (Long child : children) {
      if (CoreUtils.getHostIdFromHSId(child) == masterHostId) {
        masterHSId = child;
        break;
      }
    }
    tmLog.info(
        "Appointing HSId "
            + CoreUtils.hsIdToString(masterHSId)
            + " as leader for partition "
            + partitionId);
    try {
      m_iv2appointees.put(partitionId, masterHSId);
    } catch (Exception e) {
      VoltDB.crashLocalVoltDB("Unable to appoint new master for partition " + partitionId, true, e);
    }
    return masterHSId;
  }
Example #19
0
  @Override
  protected void updateStatsRow(Object rowKey, Object[] rowValues) {
    long leader;
    List<Long> sites = new ArrayList<Long>();
    if (rowKey.equals(MpInitiator.MP_INIT_PID)) {
      leader = getHSIdForMultiPartitionInitiator();
      sites.add(leader);
    } else {
      leader = m_iv2Masters.pointInTimeCache().get((Integer) rowKey);
      sites.addAll(getReplicasForPartition((Integer) rowKey));
    }

    rowValues[columnNameToIndex.get("Partition")] = rowKey;
    rowValues[columnNameToIndex.get("Sites")] = CoreUtils.hsIdCollectionToString(sites);
    rowValues[columnNameToIndex.get("Leader")] = CoreUtils.hsIdToString(leader);
  }
Example #20
0
 public void dump(long hsId) {
   final String who = CoreUtils.hsIdToString(hsId);
   tmLog.info(
       String.format(
           "%s: REPLAY SEQUENCER DUMP, LAST POLLED FRAGMENT %d (%s), LAST SEEN TXNID %d (%s), %s%s",
           who,
           m_lastPolledFragmentTxnId,
           TxnEgo.txnIdToString(m_lastPolledFragmentTxnId),
           m_lastSeenTxnId,
           TxnEgo.txnIdToString(m_lastSeenTxnId),
           m_mpiEOLReached ? "MPI EOL, " : "",
           m_mustDrain ? "MUST DRAIN" : ""));
   for (Entry<Long, ReplayEntry> e : m_replayEntries.entrySet()) {
     tmLog.info(String.format("%s: REPLAY ENTRY %s: %s", who, e.getKey(), e.getValue()));
   }
 }
Example #21
0
  /** Send missed-messages to survivors. */
  public void repairSurvivors() {
    // cancel() and repair() must be synchronized by the caller (the deliver lock,
    // currently). If cancelled and the last repair message arrives, don't send
    // out corrections!
    if (this.m_promotionResult.isCancelled()) {
      tmLog.debug(m_whoami + "Skipping repair message creation for cancelled Term.");
      return;
    }

    int queued = 0;
    tmLog.debug(m_whoami + "received all repair logs and is repairing surviving replicas.");
    for (Iv2RepairLogResponseMessage li : m_repairLogUnion) {
      List<Long> needsRepair = new ArrayList<Long>(5);
      for (Entry<Long, ReplicaRepairStruct> entry : m_replicaRepairStructs.entrySet()) {
        if (entry.getValue().needs(li.getHandle())) {
          ++queued;
          tmLog.debug(
              m_whoami
                  + "repairing "
                  + CoreUtils.hsIdToString(entry.getKey())
                  + ". Max seen "
                  + entry.getValue().m_maxSpHandleSeen
                  + ". Repairing with "
                  + li.getHandle());
          needsRepair.add(entry.getKey());
        }
      }
      if (!needsRepair.isEmpty()) {
        if (tmLog.isTraceEnabled()) {
          tmLog.trace(
              m_whoami
                  + "repairing: "
                  + CoreUtils.hsIdCollectionToString(needsRepair)
                  + " with message: "
                  + li.getPayload());
        }
        m_mailbox.repairReplicasWith(needsRepair, li.getPayload());
      }
    }
    tmLog.debug(m_whoami + "finished queuing " + queued + " replica repair messages.");

    m_promotionResult.done(m_maxSeenTxnId);
  }
  @Override
  public String toString() {
    StringBuilder sb = new StringBuilder();

    sb.append("COMPLETE_TRANSACTION (FROM COORD: ");
    sb.append(CoreUtils.hsIdToString(m_coordinatorHSId));
    sb.append(") FOR TXN ");
    sb.append(m_txnId);
    sb.append("\n  FLAGS: ").append(m_flags);

    sb.append("\n  HASH: " + String.valueOf(m_hash));

    if (isRollback()) sb.append("\n  THIS IS AN ROLLBACK REQUEST");

    if (requiresAck()) sb.append("\n  THIS MESSAGE REQUIRES AN ACK");

    if (isRestart()) {
      sb.append("\n  THIS IS A TRANSACTION RESTART");
    }

    return sb.toString();
  }
Example #23
0
  /**
   * The only public method: do all the work to start a snapshot. Assumes that a snapshot is
   * feasible, that the caller has validated it can be accomplished, that the caller knows this is a
   * consistent or useful transaction point at which to snapshot.
   *
   * @param file_path
   * @param file_nonce
   * @param format
   * @param block
   * @param txnId
   * @param data
   * @param context
   * @param hostname
   * @return VoltTable describing the results of the snapshot attempt
   */
  public VoltTable startSnapshotting(
      final String file_path,
      final String file_nonce,
      final SnapshotFormat format,
      final byte block,
      final long multiPartTxnId,
      final long partitionTxnId,
      final long legacyPerPartitionTxnIds[],
      final String data,
      final SystemProcedureExecutionContext context,
      final String hostname,
      final HashinatorSnapshotData hashinatorData,
      final long timestamp) {
    TRACE_LOG.trace("Creating snapshot target and handing to EEs");
    final VoltTable result = SnapshotUtil.constructNodeResultsTable();
    final int numLocalSites =
        context.getCluster().getDeployment().get("deployment").getSitesperhost();

    // One site wins the race to create the snapshot targets, populating
    // m_taskListsForSites for the other sites and creating an appropriate
    // number of snapshot permits.
    synchronized (SnapshotSiteProcessor.m_snapshotCreateLock) {
      SnapshotSiteProcessor.m_snapshotCreateSetupBarrierActualAction.set(
          new Runnable() {
            @Override
            public void run() {
              Map<Integer, Long> partitionTransactionIds = new HashMap<Integer, Long>();
              partitionTransactionIds = m_partitionLastSeenTransactionIds;
              SNAP_LOG.debug("Last seen partition transaction ids " + partitionTransactionIds);
              m_partitionLastSeenTransactionIds = new HashMap<Integer, Long>();
              partitionTransactionIds.put(TxnEgo.getPartitionId(multiPartTxnId), multiPartTxnId);

              /*
               * Do a quick sanity check that the provided IDs
               * don't conflict with currently active partitions. If they do
               * it isn't fatal we can just skip it.
               */
              for (long txnId : legacyPerPartitionTxnIds) {
                final int legacyPartition = TxnEgo.getPartitionId(txnId);
                if (partitionTransactionIds.containsKey(legacyPartition)) {
                  SNAP_LOG.warn(
                      "While saving a snapshot and propagating legacy "
                          + "transaction ids found an id that matches currently active partition"
                          + partitionTransactionIds.get(legacyPartition));
                } else {
                  partitionTransactionIds.put(legacyPartition, txnId);
                }
              }
              exportSequenceNumbers = SnapshotSiteProcessor.getExportSequenceNumbers();
              createSetupIv2(
                  file_path,
                  file_nonce,
                  format,
                  multiPartTxnId,
                  partitionTransactionIds,
                  data,
                  context,
                  result,
                  exportSequenceNumbers,
                  context.getSiteTrackerForSnapshot(),
                  hashinatorData,
                  timestamp);
            }
          });

      // Create a barrier to use with the current number of sites to wait for
      // or if the barrier is already set up check if it is broken and reset if necessary
      SnapshotSiteProcessor.readySnapshotSetupBarriers(numLocalSites);

      // From within this EE, record the sequence numbers as of the start of the snapshot (now)
      // so that the info can be put in the digest.
      SnapshotSiteProcessor.populateExportSequenceNumbersForExecutionSite(context);
      SNAP_LOG.debug(
          "Registering transaction id "
              + partitionTxnId
              + " for "
              + TxnEgo.getPartitionId(partitionTxnId));
      m_partitionLastSeenTransactionIds.put(TxnEgo.getPartitionId(partitionTxnId), partitionTxnId);
    }

    boolean runPostTasks = false;
    VoltTable earlyResultTable = null;
    try {
      SnapshotSiteProcessor.m_snapshotCreateSetupBarrier.await();
      try {
        synchronized (m_createLock) {
          SNAP_LOG.debug(
              "Found tasks for HSIds: "
                  + CoreUtils.hsIdCollectionToString(m_taskListsForHSIds.keySet()));
          SNAP_LOG.debug("Looking for local HSID: " + CoreUtils.hsIdToString(context.getSiteId()));
          Deque<SnapshotTableTask> taskList = m_taskListsForHSIds.remove(context.getSiteId());
          // If createSetup failed, then the first site to reach here is going
          // to send the results table generated by createSetup, and then empty out the table.
          // All other sites to reach here will send the appropriate empty table.
          // If createSetup was a success but the taskList is null, then we'll use the block
          // switch to figure out what flavor of empty SnapshotSave result table to return.
          if (!m_createSuccess.get()) {
            // There shouldn't be any work for any site if we failed
            assert (m_taskListsForHSIds.isEmpty());
            VoltTable finalresult = m_createResult.get();
            if (finalresult != null) {
              m_createResult.set(null);
              earlyResultTable = finalresult;
            } else {
              // We returned a non-empty NodeResultsTable with the failures in it,
              // every other site needs to return a NodeResultsTable as well.
              earlyResultTable = SnapshotUtil.constructNodeResultsTable();
            }
          } else if (taskList == null) {
            SNAP_LOG.debug("No task for this site, block " + block);
            // This node is participating in the snapshot but this site has nothing to do.
            // Send back an appropriate empty table based on the block flag
            if (block != 0) {
              runPostTasks = true;
              earlyResultTable = SnapshotUtil.constructPartitionResultsTable();
              earlyResultTable.addRow(
                  context.getHostId(),
                  hostname,
                  CoreUtils.getSiteIdFromHSId(context.getSiteId()),
                  "SUCCESS",
                  "");
            } else {
              earlyResultTable = SnapshotUtil.constructNodeResultsTable();
            }
          } else {
            context
                .getSiteSnapshotConnection()
                .initiateSnapshots(format, taskList, multiPartTxnId, exportSequenceNumbers);
          }

          if (m_deferredSetupFuture != null) {
            // Add a listener to the deferred setup so that it can kick off the snapshot
            // task once the setup is done.
            m_deferredSetupFuture.addListener(
                new Runnable() {
                  @Override
                  public void run() {
                    DeferredSnapshotSetup deferredSnapshotSetup = null;
                    try {
                      deferredSnapshotSetup = m_deferredSetupFuture.get();
                    } catch (Exception e) {
                      // it doesn't throw
                    }

                    assert deferredSnapshotSetup != null;
                    context
                        .getSiteSnapshotConnection()
                        .startSnapshotWithTargets(
                            deferredSnapshotSetup.getPlan().getSnapshotDataTargets());
                  }
                },
                CoreUtils.SAMETHREADEXECUTOR);
          }
        }
      } finally {
        SnapshotSiteProcessor.m_snapshotCreateFinishBarrier.await(120, TimeUnit.SECONDS);
      }
    } catch (TimeoutException e) {
      VoltDB.crashLocalVoltDB(
          "Timed out waiting 120 seconds for all threads to arrive and start snapshot", true, null);
    } catch (InterruptedException e) {
      result.addRow(context.getHostId(), hostname, "", "FAILURE", CoreUtils.throwableToString(e));
      earlyResultTable = result;
    } catch (BrokenBarrierException e) {
      result.addRow(context.getHostId(), hostname, "", "FAILURE", CoreUtils.throwableToString(e));
      earlyResultTable = result;
    }

    // If earlyResultTable is set, return here
    if (earlyResultTable != null) {
      if (runPostTasks) {
        // Need to run post-snapshot tasks before finishing
        SnapshotSiteProcessor.runPostSnapshotTasks(context);
      }
      return earlyResultTable;
    }

    if (block != 0) {
      HashSet<Exception> failures = Sets.newHashSet();
      String status = "SUCCESS";
      String err = "";
      try {
        // For blocking snapshot, propogate the error from deferred setup back to the client
        final DeferredSnapshotSetup deferredSnapshotSetup = m_deferredSetupFuture.get();
        if (deferredSnapshotSetup != null && deferredSnapshotSetup.getError() != null) {
          status = "FAILURE";
          err = deferredSnapshotSetup.getError().toString();
          failures.add(deferredSnapshotSetup.getError());
        }

        failures.addAll(context.getSiteSnapshotConnection().completeSnapshotWork());
        SnapshotSiteProcessor.runPostSnapshotTasks(context);
      } catch (Exception e) {
        status = "FAILURE";
        err = e.toString();
        failures.add(e);
      }
      final VoltTable blockingResult = SnapshotUtil.constructPartitionResultsTable();

      if (failures.isEmpty()) {
        blockingResult.addRow(
            context.getHostId(),
            hostname,
            CoreUtils.getSiteIdFromHSId(context.getSiteId()),
            status,
            err);
      } else {
        status = "FAILURE";
        for (Exception e : failures) {
          err = e.toString();
        }
        blockingResult.addRow(
            context.getHostId(),
            hostname,
            CoreUtils.getSiteIdFromHSId(context.getSiteId()),
            status,
            err);
      }
      return blockingResult;
    }

    return result;
  }
Example #24
0
  // Offer a new message to the repair log. This will truncate
  // the repairLog if the message includes a truncation hint.
  public void deliver(VoltMessage msg) {
    if (!m_isLeader && msg instanceof Iv2InitiateTaskMessage) {
      final Iv2InitiateTaskMessage m = (Iv2InitiateTaskMessage) msg;
      // We can't repair read only SP transactions. Just don't log them to the repair log.
      if (m.isReadOnly()) {
        return;
      }

      m_lastSpHandle = m.getSpHandle();
      truncate(m.getTruncationHandle(), IS_SP);
      m_logSP.add(new Item(IS_SP, m, m.getSpHandle(), m.getTxnId()));
    } else if (msg instanceof FragmentTaskMessage) {
      final FragmentTaskMessage m = (FragmentTaskMessage) msg;

      // We can't repair read only SP transactions. Just don't log them to the repair log.
      if (m.isReadOnly()) {
        return;
      }

      truncate(m.getTruncationHandle(), IS_MP);
      // only log the first fragment of a procedure (and handle 1st case)
      if (m.getTxnId() > m_lastMpHandle || m_lastMpHandle == Long.MAX_VALUE) {
        m_logMP.add(new Item(IS_MP, m, m.getSpHandle(), m.getTxnId()));
        m_lastMpHandle = m.getTxnId();
        m_lastSpHandle = m.getSpHandle();
      }
    } else if (msg instanceof CompleteTransactionMessage) {
      // a CompleteTransactionMessage which indicates restart is not the end of the
      // transaction.  We don't want to log it in the repair log.
      CompleteTransactionMessage ctm = (CompleteTransactionMessage) msg;
      // We can't repair read only SP transactions. Just don't log them to the repair log.
      // Restart transaction do not need to be repaired here, don't log them as well.
      if (ctm.isReadOnly() || ctm.isRestart()) {
        return;
      }

      truncate(ctm.getTruncationHandle(), IS_MP);
      m_logMP.add(new Item(IS_MP, ctm, ctm.getSpHandle(), ctm.getTxnId()));
      // Restore will send a complete transaction message with a lower mp transaction id because
      // the restore transaction precedes the loading of the right mp transaction id from the
      // snapshot
      // Hence Math.max
      m_lastMpHandle = Math.max(m_lastMpHandle, ctm.getTxnId());
      m_lastSpHandle = ctm.getSpHandle();
    } else if (msg instanceof DumpMessage) {
      String who = CoreUtils.hsIdToString(m_HSId);
      tmLog.warn(
          "Repair log dump for site: "
              + who
              + ", isLeader: "
              + m_isLeader
              + ", "
              + who
              + ": lastSpHandle: "
              + m_lastSpHandle
              + ", lastMpHandle: "
              + m_lastMpHandle);
      for (Iv2RepairLogResponseMessage il : contents(0l, false)) {
        tmLog.warn("[Repair log contents]" + who + ": msg: " + il);
      }
    } else if (msg instanceof RepairLogTruncationMessage) {
      final RepairLogTruncationMessage truncateMsg = (RepairLogTruncationMessage) msg;
      truncate(truncateMsg.getHandle(), IS_SP);
    }
  }