@Test
  public void testCounter() throws Exception {
    ClientSessionFactory sf = createSessionFactory(sl);
    ClientSession session = sf.createSession();

    try {
      Queue queue =
          server.createQueue(new SimpleString("A1"), new SimpleString("A1"), null, true, false);

      PageSubscriptionCounter counter = locateCounter(queue);

      StorageManager storage = server.getStorageManager();

      Transaction tx = new TransactionImpl(server.getStorageManager());

      counter.increment(tx, 1);

      assertEquals(0, counter.getValue());

      tx.commit();

      storage.waitOnOperations();

      assertEquals(1, counter.getValue());
    } finally {
      sf.close();
      session.close();
    }
  }
  /**
   * This generates a map for use on the recalculation and recovery of pending maps after reloading
   * it
   *
   * @param queues
   * @param pendingNonTXPageCounter
   * @param txRecoverCounter
   * @return
   * @throws Exception
   */
  private Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>>
      generateMapsOnPendingCount(
          Map<Long, Queue> queues,
          List<PageCountPending> pendingNonTXPageCounter,
          Transaction txRecoverCounter)
          throws Exception {
    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap = new HashMap<>();
    for (PageCountPending pgCount : pendingNonTXPageCounter) {
      long queueID = pgCount.getQueueID();
      long pageID = pgCount.getPageID();

      // We first figure what Queue is based on the queue id
      Queue queue = queues.get(queueID);

      if (queue == null) {
        logger.debug(
            "removing pending page counter id = "
                + pgCount.getID()
                + " as queueID="
                + pgCount.getID()
                + " no longer exists");
        // this means the queue doesn't exist any longer, we will remove it from the storage
        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), pgCount.getID());
        txRecoverCounter.setContainsPersistent();
        continue;
      }

      // Level 1 on the structure, per address
      SimpleString address = queue.getAddress();

      Map<Long, Map<Long, List<PageCountPending>>> perPageMap = perAddressMap.get(address);

      if (perPageMap == null) {
        perPageMap = new HashMap<>();
        perAddressMap.put(address, perPageMap);
      }

      Map<Long, List<PageCountPending>> perQueueMap = perPageMap.get(pageID);

      if (perQueueMap == null) {
        perQueueMap = new HashMap<>();
        perPageMap.put(pageID, perQueueMap);
      }

      List<PageCountPending> pendingCounters = perQueueMap.get(queueID);

      if (pendingCounters == null) {
        pendingCounters = new LinkedList<>();
        perQueueMap.put(queueID, pendingCounters);
      }

      pendingCounters.add(pgCount);

      perQueueMap.put(queueID, pendingCounters);
    }
    return perAddressMap;
  }
  @Test
  public void testCleanupCounterNonPersistent() throws Exception {
    ClientSessionFactory sf = createSessionFactory(sl);
    ClientSession session = sf.createSession();

    try {
      Queue queue =
          server.createQueue(new SimpleString("A1"), new SimpleString("A1"), null, true, false);

      PageSubscriptionCounter counter = locateCounter(queue);

      ((PageSubscriptionCounterImpl) counter).setPersistent(false);

      StorageManager storage = server.getStorageManager();

      Transaction tx = new TransactionImpl(server.getStorageManager());

      for (int i = 0; i < 2100; i++) {

        counter.increment(tx, 1);

        if (i % 200 == 0) {
          tx.commit();

          storage.waitOnOperations();

          assertEquals(i + 1, counter.getValue());

          tx = new TransactionImpl(server.getStorageManager());
        }
      }

      tx.commit();

      storage.waitOnOperations();

      assertEquals(2100, counter.getValue());

      server.stop();

      server = newActiveMQServer();

      server.start();

      queue = server.locateQueue(new SimpleString("A1"));

      assertNotNull(queue);

      counter = locateCounter(queue);

      assertEquals(0, counter.getValue());

    } finally {
      sf.close();
      session.close();
    }
  }
  @Test
  public void testPrepareCounter() throws Exception {
    Xid xid = newXID();

    Queue queue =
        server.createQueue(new SimpleString("A1"), new SimpleString("A1"), null, true, false);

    PageSubscriptionCounter counter = locateCounter(queue);

    StorageManager storage = server.getStorageManager();

    Transaction tx = new TransactionImpl(xid, server.getStorageManager(), 300);

    for (int i = 0; i < 2000; i++) {
      counter.increment(tx, 1);
    }

    assertEquals(0, counter.getValue());

    tx.prepare();

    storage.waitOnOperations();

    assertEquals(0, counter.getValue());

    server.stop();

    server = newActiveMQServer();

    server.start();

    storage = server.getStorageManager();

    queue = server.locateQueue(new SimpleString("A1"));

    assertNotNull(queue);

    counter = locateCounter(queue);

    tx = server.getResourceManager().removeTransaction(xid);

    assertNotNull(tx);

    assertEquals(0, counter.getValue());

    tx.commit(false);

    storage.waitOnOperations();

    assertEquals(2000, counter.getValue());
  }
Ejemplo n.º 5
0
 private void storePageTX(final Transaction tx) throws Exception {
   if (!stored) {
     tx.setContainsPersistent();
     pageTransaction.store(storageManager, pagingManager, tx);
     stored = true;
   }
 }
Ejemplo n.º 6
0
  private void installPageTransaction(final Transaction tx, final RouteContextList listCtx)
      throws Exception {
    FinishPageMessageOperation pgOper =
        (FinishPageMessageOperation) tx.getProperty(TransactionPropertyIndexes.PAGE_TRANSACTION);
    if (pgOper == null) {
      PageTransactionInfo pgTX = new PageTransactionInfoImpl(tx.getID());
      pagingManager.addTransaction(pgTX);
      pgOper = new FinishPageMessageOperation(pgTX, storageManager, pagingManager);
      tx.putProperty(TransactionPropertyIndexes.PAGE_TRANSACTION, pgOper);
      tx.addOperation(pgOper);
    }

    pgOper.addStore(this);
    pgOper.pageTransaction.increment(
        listCtx.getNumberOfDurableQueues(), listCtx.getNumberOfNonDurableQueues());

    return;
  }
  @Test
  public void testRestartCounter() throws Exception {
    Queue queue =
        server.createQueue(new SimpleString("A1"), new SimpleString("A1"), null, true, false);

    PageSubscriptionCounter counter = locateCounter(queue);

    StorageManager storage = server.getStorageManager();

    Transaction tx = new TransactionImpl(server.getStorageManager());

    counter.increment(tx, 1);

    assertEquals(0, counter.getValue());

    tx.commit();

    storage.waitOnOperations();

    assertEquals(1, counter.getValue());

    sl.close();

    server.stop();

    server = newActiveMQServer();

    server.start();

    queue = server.locateQueue(new SimpleString("A1"));

    assertNotNull(queue);

    counter = locateCounter(queue);

    assertEquals(1, counter.getValue());
  }
  @Override
  public void handlePreparedTransaction(
      Transaction tx,
      List<MessageReference> referencesToAck,
      Xid xid,
      ResourceManager resourceManager)
      throws Exception {
    for (MessageReference ack : referencesToAck) {
      ack.getQueue().reacknowledge(tx, ack);
    }

    tx.setState(Transaction.State.PREPARED);

    resourceManager.putTransaction(xid, tx);
  }
Ejemplo n.º 9
0
  @Override
  public boolean page(
      ServerMessage message,
      final Transaction tx,
      RouteContextList listCtx,
      final ReadLock managerLock)
      throws Exception {

    if (!running) {
      throw new IllegalStateException("PagingStore(" + getStoreName() + ") not initialized");
    }

    boolean full = isFull();

    if (addressFullMessagePolicy == AddressFullMessagePolicy.DROP
        || addressFullMessagePolicy == AddressFullMessagePolicy.FAIL) {
      if (full) {
        if (!printedDropMessagesWarning) {
          printedDropMessagesWarning = true;

          ActiveMQServerLogger.LOGGER.pageStoreDropMessages(storeName, sizeInBytes.get(), maxSize);
        }

        if (message.isLargeMessage()) {
          ((LargeServerMessage) message).deleteFile();
        }

        if (addressFullMessagePolicy == AddressFullMessagePolicy.FAIL) {
          throw ActiveMQMessageBundle.BUNDLE.addressIsFull(address.toString());
        }

        // Address is full, we just pretend we are paging, and drop the data
        return true;
      } else {
        return false;
      }
    } else if (addressFullMessagePolicy == AddressFullMessagePolicy.BLOCK) {
      return false;
    }

    // We need to ensure a read lock, as depage could change the paging state
    lock.readLock().lock();

    try {
      // First check done concurrently, to avoid synchronization and increase throughput
      if (!paging) {
        return false;
      }
    } finally {
      lock.readLock().unlock();
    }

    managerLock.lock();
    try {
      lock.writeLock().lock();

      try {
        if (!paging) {
          return false;
        }

        if (!message.isDurable()) {
          // The address should never be transient when paging (even for non-persistent messages
          // when paging)
          // This will force everything to be persisted
          message.forceAddress(address);
        }

        final long transactionID = tx == null ? -1 : tx.getID();
        PagedMessage pagedMessage =
            new PagedMessageImpl(message, routeQueues(tx, listCtx), transactionID);

        if (message.isLargeMessage()) {
          ((LargeServerMessage) message).setPaged();
        }

        int bytesToWrite = pagedMessage.getEncodeSize() + Page.SIZE_RECORD;

        if (currentPageSize.addAndGet(bytesToWrite) > pageSize
            && currentPage.getNumberOfMessages() > 0) {
          // Make sure nothing is currently validating or using currentPage
          openNewPage();
          currentPageSize.addAndGet(bytesToWrite);
        }

        if (tx != null) {
          installPageTransaction(tx, listCtx);
        }

        // the apply counter will make sure we write a record on journal
        // especially on the case for non transactional sends and paging
        // doing this will give us a possibility of recovering the page counters
        applyPageCounters(tx, getCurrentPage(), listCtx);

        currentPage.write(pagedMessage);

        if (tx == null && syncNonTransactional && message.isDurable()) {
          sync();
        }

        if (isTrace) {
          ActiveMQServerLogger.LOGGER.trace(
              "Paging message "
                  + pagedMessage
                  + " on pageStore "
                  + this.getStoreName()
                  + " pageId="
                  + currentPage.getPageId());
        }

        return true;
      } finally {
        lock.writeLock().unlock();
      }
    } finally {
      managerLock.unlock();
    }
  }
  /**
   * This method will recover the counters after failures making sure the page counter doesn't get
   * out of sync
   *
   * @param pendingNonTXPageCounter
   * @throws Exception
   */
  @Override
  public void recoverPendingPageCounters(List<PageCountPending> pendingNonTXPageCounter)
      throws Exception {
    // We need a structure of the following
    // Address -> PageID -> QueueID -> List<PageCountPending>
    // The following loop will sort the records according to the hierarchy we need

    Transaction txRecoverCounter = new TransactionImpl(storageManager);

    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap =
        generateMapsOnPendingCount(queues, pendingNonTXPageCounter, txRecoverCounter);

    for (SimpleString address : perAddressMap.keySet()) {
      PagingStore store = pagingManager.getPageStore(address);
      Map<Long, Map<Long, List<PageCountPending>>> perPageMap = perAddressMap.get(address);

      // We have already generated this before, so it can't be null
      assert (perPageMap != null);

      for (Long pageId : perPageMap.keySet()) {
        Map<Long, List<PageCountPending>> perQueue = perPageMap.get(pageId);

        // This can't be true!
        assert (perQueue != null);

        if (store.checkPageFileExists(pageId.intValue())) {
          // on this case we need to recalculate the records
          Page pg = store.createPage(pageId.intValue());
          pg.open();

          List<PagedMessage> pgMessages = pg.read(storageManager);
          Map<Long, AtomicInteger> countsPerQueueOnPage = new HashMap<>();

          for (PagedMessage pgd : pgMessages) {
            if (pgd.getTransactionID() <= 0) {
              for (long q : pgd.getQueueIDs()) {
                AtomicInteger countQ = countsPerQueueOnPage.get(q);
                if (countQ == null) {
                  countQ = new AtomicInteger(0);
                  countsPerQueueOnPage.put(q, countQ);
                }
                countQ.incrementAndGet();
              }
            }
          }

          for (Map.Entry<Long, List<PageCountPending>> entry : perQueue.entrySet()) {
            for (PageCountPending record : entry.getValue()) {
              logger.debug("Deleting pg tempCount " + record.getID());
              storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
            }

            PageSubscriptionCounter counter =
                store.getCursorProvider().getSubscription(entry.getKey()).getCounter();

            AtomicInteger value = countsPerQueueOnPage.get(entry.getKey());

            if (value == null) {
              logger.debug("Page " + entry.getKey() + " wasn't open, so we will just ignore");
            } else {
              logger.debug("Replacing counter " + value.get());
              counter.increment(txRecoverCounter, value.get());
            }
          }
        } else {
          // on this case the page file didn't exist, we just remove all the records since the page
          // is already gone
          logger.debug(
              "Page "
                  + pageId
                  + " didn't exist on address "
                  + address
                  + ", so we are just removing records");
          for (List<PageCountPending> records : perQueue.values()) {
            for (PageCountPending record : records) {
              logger.debug("Removing pending page counter " + record.getID());
              storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
              txRecoverCounter.setContainsPersistent();
            }
          }
        }
      }
    }

    txRecoverCounter.commit();
  }