/**
   * This generates a map for use on the recalculation and recovery of pending maps after reloading
   * it
   *
   * @param queues
   * @param pendingNonTXPageCounter
   * @param txRecoverCounter
   * @return
   * @throws Exception
   */
  private Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>>
      generateMapsOnPendingCount(
          Map<Long, Queue> queues,
          List<PageCountPending> pendingNonTXPageCounter,
          Transaction txRecoverCounter)
          throws Exception {
    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap = new HashMap<>();
    for (PageCountPending pgCount : pendingNonTXPageCounter) {
      long queueID = pgCount.getQueueID();
      long pageID = pgCount.getPageID();

      // We first figure what Queue is based on the queue id
      Queue queue = queues.get(queueID);

      if (queue == null) {
        logger.debug(
            "removing pending page counter id = "
                + pgCount.getID()
                + " as queueID="
                + pgCount.getID()
                + " no longer exists");
        // this means the queue doesn't exist any longer, we will remove it from the storage
        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), pgCount.getID());
        txRecoverCounter.setContainsPersistent();
        continue;
      }

      // Level 1 on the structure, per address
      SimpleString address = queue.getAddress();

      Map<Long, Map<Long, List<PageCountPending>>> perPageMap = perAddressMap.get(address);

      if (perPageMap == null) {
        perPageMap = new HashMap<>();
        perAddressMap.put(address, perPageMap);
      }

      Map<Long, List<PageCountPending>> perQueueMap = perPageMap.get(pageID);

      if (perQueueMap == null) {
        perQueueMap = new HashMap<>();
        perPageMap.put(pageID, perQueueMap);
      }

      List<PageCountPending> pendingCounters = perQueueMap.get(queueID);

      if (pendingCounters == null) {
        pendingCounters = new LinkedList<>();
        perQueueMap.put(queueID, pendingCounters);
      }

      pendingCounters.add(pgCount);

      perQueueMap.put(queueID, pendingCounters);
    }
    return perAddressMap;
  }
Exemplo n.º 2
0
  private void installPageTransaction(final Transaction tx, final RouteContextList listCtx)
      throws Exception {
    FinishPageMessageOperation pgOper =
        (FinishPageMessageOperation) tx.getProperty(TransactionPropertyIndexes.PAGE_TRANSACTION);
    if (pgOper == null) {
      PageTransactionInfo pgTX = new PageTransactionInfoImpl(tx.getID());
      pagingManager.addTransaction(pgTX);
      pgOper = new FinishPageMessageOperation(pgTX, storageManager, pagingManager);
      tx.putProperty(TransactionPropertyIndexes.PAGE_TRANSACTION, pgOper);
      tx.addOperation(pgOper);
    }

    pgOper.addStore(this);
    pgOper.pageTransaction.increment(
        listCtx.getNumberOfDurableQueues(), listCtx.getNumberOfNonDurableQueues());

    return;
  }
Exemplo n.º 3
0
  @Override
  public boolean page(
      ServerMessage message,
      final Transaction tx,
      RouteContextList listCtx,
      final ReadLock managerLock)
      throws Exception {

    if (!running) {
      throw new IllegalStateException("PagingStore(" + getStoreName() + ") not initialized");
    }

    boolean full = isFull();

    if (addressFullMessagePolicy == AddressFullMessagePolicy.DROP
        || addressFullMessagePolicy == AddressFullMessagePolicy.FAIL) {
      if (full) {
        if (!printedDropMessagesWarning) {
          printedDropMessagesWarning = true;

          ActiveMQServerLogger.LOGGER.pageStoreDropMessages(storeName, sizeInBytes.get(), maxSize);
        }

        if (message.isLargeMessage()) {
          ((LargeServerMessage) message).deleteFile();
        }

        if (addressFullMessagePolicy == AddressFullMessagePolicy.FAIL) {
          throw ActiveMQMessageBundle.BUNDLE.addressIsFull(address.toString());
        }

        // Address is full, we just pretend we are paging, and drop the data
        return true;
      } else {
        return false;
      }
    } else if (addressFullMessagePolicy == AddressFullMessagePolicy.BLOCK) {
      return false;
    }

    // We need to ensure a read lock, as depage could change the paging state
    lock.readLock().lock();

    try {
      // First check done concurrently, to avoid synchronization and increase throughput
      if (!paging) {
        return false;
      }
    } finally {
      lock.readLock().unlock();
    }

    managerLock.lock();
    try {
      lock.writeLock().lock();

      try {
        if (!paging) {
          return false;
        }

        if (!message.isDurable()) {
          // The address should never be transient when paging (even for non-persistent messages
          // when paging)
          // This will force everything to be persisted
          message.forceAddress(address);
        }

        final long transactionID = tx == null ? -1 : tx.getID();
        PagedMessage pagedMessage =
            new PagedMessageImpl(message, routeQueues(tx, listCtx), transactionID);

        if (message.isLargeMessage()) {
          ((LargeServerMessage) message).setPaged();
        }

        int bytesToWrite = pagedMessage.getEncodeSize() + Page.SIZE_RECORD;

        if (currentPageSize.addAndGet(bytesToWrite) > pageSize
            && currentPage.getNumberOfMessages() > 0) {
          // Make sure nothing is currently validating or using currentPage
          openNewPage();
          currentPageSize.addAndGet(bytesToWrite);
        }

        if (tx != null) {
          installPageTransaction(tx, listCtx);
        }

        // the apply counter will make sure we write a record on journal
        // especially on the case for non transactional sends and paging
        // doing this will give us a possibility of recovering the page counters
        applyPageCounters(tx, getCurrentPage(), listCtx);

        currentPage.write(pagedMessage);

        if (tx == null && syncNonTransactional && message.isDurable()) {
          sync();
        }

        if (isTrace) {
          ActiveMQServerLogger.LOGGER.trace(
              "Paging message "
                  + pagedMessage
                  + " on pageStore "
                  + this.getStoreName()
                  + " pageId="
                  + currentPage.getPageId());
        }

        return true;
      } finally {
        lock.writeLock().unlock();
      }
    } finally {
      managerLock.unlock();
    }
  }
  /**
   * This method will recover the counters after failures making sure the page counter doesn't get
   * out of sync
   *
   * @param pendingNonTXPageCounter
   * @throws Exception
   */
  @Override
  public void recoverPendingPageCounters(List<PageCountPending> pendingNonTXPageCounter)
      throws Exception {
    // We need a structure of the following
    // Address -> PageID -> QueueID -> List<PageCountPending>
    // The following loop will sort the records according to the hierarchy we need

    Transaction txRecoverCounter = new TransactionImpl(storageManager);

    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap =
        generateMapsOnPendingCount(queues, pendingNonTXPageCounter, txRecoverCounter);

    for (SimpleString address : perAddressMap.keySet()) {
      PagingStore store = pagingManager.getPageStore(address);
      Map<Long, Map<Long, List<PageCountPending>>> perPageMap = perAddressMap.get(address);

      // We have already generated this before, so it can't be null
      assert (perPageMap != null);

      for (Long pageId : perPageMap.keySet()) {
        Map<Long, List<PageCountPending>> perQueue = perPageMap.get(pageId);

        // This can't be true!
        assert (perQueue != null);

        if (store.checkPageFileExists(pageId.intValue())) {
          // on this case we need to recalculate the records
          Page pg = store.createPage(pageId.intValue());
          pg.open();

          List<PagedMessage> pgMessages = pg.read(storageManager);
          Map<Long, AtomicInteger> countsPerQueueOnPage = new HashMap<>();

          for (PagedMessage pgd : pgMessages) {
            if (pgd.getTransactionID() <= 0) {
              for (long q : pgd.getQueueIDs()) {
                AtomicInteger countQ = countsPerQueueOnPage.get(q);
                if (countQ == null) {
                  countQ = new AtomicInteger(0);
                  countsPerQueueOnPage.put(q, countQ);
                }
                countQ.incrementAndGet();
              }
            }
          }

          for (Map.Entry<Long, List<PageCountPending>> entry : perQueue.entrySet()) {
            for (PageCountPending record : entry.getValue()) {
              logger.debug("Deleting pg tempCount " + record.getID());
              storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
            }

            PageSubscriptionCounter counter =
                store.getCursorProvider().getSubscription(entry.getKey()).getCounter();

            AtomicInteger value = countsPerQueueOnPage.get(entry.getKey());

            if (value == null) {
              logger.debug("Page " + entry.getKey() + " wasn't open, so we will just ignore");
            } else {
              logger.debug("Replacing counter " + value.get());
              counter.increment(txRecoverCounter, value.get());
            }
          }
        } else {
          // on this case the page file didn't exist, we just remove all the records since the page
          // is already gone
          logger.debug(
              "Page "
                  + pageId
                  + " didn't exist on address "
                  + address
                  + ", so we are just removing records");
          for (List<PageCountPending> records : perQueue.values()) {
            for (PageCountPending record : records) {
              logger.debug("Removing pending page counter " + record.getID());
              storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
              txRecoverCounter.setContainsPersistent();
            }
          }
        }
      }
    }

    txRecoverCounter.commit();
  }