@Test
  public void testCounter() throws Exception {
    ClientSessionFactory sf = createSessionFactory(sl);
    ClientSession session = sf.createSession();

    try {
      Queue queue =
          server.createQueue(new SimpleString("A1"), new SimpleString("A1"), null, true, false);

      PageSubscriptionCounter counter = locateCounter(queue);

      StorageManager storage = server.getStorageManager();

      Transaction tx = new TransactionImpl(server.getStorageManager());

      counter.increment(tx, 1);

      assertEquals(0, counter.getValue());

      tx.commit();

      storage.waitOnOperations();

      assertEquals(1, counter.getValue());
    } finally {
      sf.close();
      session.close();
    }
  }
  @Test
  public void testCleanupCounterNonPersistent() throws Exception {
    ClientSessionFactory sf = createSessionFactory(sl);
    ClientSession session = sf.createSession();

    try {
      Queue queue =
          server.createQueue(new SimpleString("A1"), new SimpleString("A1"), null, true, false);

      PageSubscriptionCounter counter = locateCounter(queue);

      ((PageSubscriptionCounterImpl) counter).setPersistent(false);

      StorageManager storage = server.getStorageManager();

      Transaction tx = new TransactionImpl(server.getStorageManager());

      for (int i = 0; i < 2100; i++) {

        counter.increment(tx, 1);

        if (i % 200 == 0) {
          tx.commit();

          storage.waitOnOperations();

          assertEquals(i + 1, counter.getValue());

          tx = new TransactionImpl(server.getStorageManager());
        }
      }

      tx.commit();

      storage.waitOnOperations();

      assertEquals(2100, counter.getValue());

      server.stop();

      server = newActiveMQServer();

      server.start();

      queue = server.locateQueue(new SimpleString("A1"));

      assertNotNull(queue);

      counter = locateCounter(queue);

      assertEquals(0, counter.getValue());

    } finally {
      sf.close();
      session.close();
    }
  }
  @Test
  public void testPrepareCounter() throws Exception {
    Xid xid = newXID();

    Queue queue =
        server.createQueue(new SimpleString("A1"), new SimpleString("A1"), null, true, false);

    PageSubscriptionCounter counter = locateCounter(queue);

    StorageManager storage = server.getStorageManager();

    Transaction tx = new TransactionImpl(xid, server.getStorageManager(), 300);

    for (int i = 0; i < 2000; i++) {
      counter.increment(tx, 1);
    }

    assertEquals(0, counter.getValue());

    tx.prepare();

    storage.waitOnOperations();

    assertEquals(0, counter.getValue());

    server.stop();

    server = newActiveMQServer();

    server.start();

    storage = server.getStorageManager();

    queue = server.locateQueue(new SimpleString("A1"));

    assertNotNull(queue);

    counter = locateCounter(queue);

    tx = server.getResourceManager().removeTransaction(xid);

    assertNotNull(tx);

    assertEquals(0, counter.getValue());

    tx.commit(false);

    storage.waitOnOperations();

    assertEquals(2000, counter.getValue());
  }
  /** @throws Exception */
  protected void createStorage() throws Exception {

    if (storeType == StoreConfiguration.StoreType.DATABASE) {
      journal = createJDBCJournalStorageManager(createDefaultJDBCConfig(true));
    } else {
      journal = createJournalStorageManager(createDefaultInVMConfig());
    }

    journal.start();

    journal.loadBindingJournal(new ArrayList<QueueBindingInfo>(), new ArrayList<GroupingInfo>());

    journal.loadMessageJournal(
        new FakePostOffice(), null, null, null, null, null, null, new FakeJournalLoader());
  }
  @Override
  @After
  public void tearDown() throws Exception {
    Exception exception = null;

    if (journal != null) {
      try {
        journal.stop();
      } catch (Exception e) {
        exception = e;
      }

      journal = null;
    }

    if (jmsJournal != null) {
      try {
        jmsJournal.stop();
      } catch (Exception e) {
        if (exception != null) exception = e;
      }

      jmsJournal = null;
    }

    destroyTables(Arrays.asList(new String[] {"MESSAGE", "BINDINGS", "LARGE_MESSAGE"}));
    super.tearDown();
    if (exception != null) throw exception;
  }
  @Override
  public synchronized void stop() throws Exception {
    if (!started) {
      return;
    }

    // Channel may be null if there isn't a connection to a live server
    if (channel != null) {
      channel.close();
    }

    for (ReplicatedLargeMessage largeMessage : largeMessages.values()) {
      largeMessage.releaseResources();
    }
    largeMessages.clear();

    for (Entry<JournalContent, Map<Long, JournalSyncFile>> entry :
        filesReservedForSync.entrySet()) {
      for (JournalSyncFile filesReserved : entry.getValue().values()) {
        filesReserved.close();
      }
    }

    filesReservedForSync.clear();
    if (journals != null) {
      for (Journal j : journals) {
        if (j instanceof FileWrapperJournal) j.stop();
      }
    }

    for (ConcurrentMap<Integer, Page> map : pageIndex.values()) {
      for (Page page : map.values()) {
        try {
          page.sync();
          page.close();
        } catch (Exception e) {
          ActiveMQServerLogger.LOGGER.errorClosingPageOnReplication(e);
        }
      }
    }
    pageManager.stop();

    pageIndex.clear();
    final CountDownLatch latch = new CountDownLatch(1);
    executor.execute(
        new Runnable() {

          @Override
          public void run() {
            latch.countDown();
          }
        });
    latch.await(30, TimeUnit.SECONDS);

    // Storage needs to be the last to stop
    storageManager.stop();

    started = false;
  }
 @Override
 public void sync() throws Exception {
   if (syncTimer != null) {
     syncTimer.addSync(storageManager.getContext());
   } else {
     ioSync();
   }
 }
  /**
   * This generates a map for use on the recalculation and recovery of pending maps after reloading
   * it
   *
   * @param queues
   * @param pendingNonTXPageCounter
   * @param txRecoverCounter
   * @return
   * @throws Exception
   */
  private Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>>
      generateMapsOnPendingCount(
          Map<Long, Queue> queues,
          List<PageCountPending> pendingNonTXPageCounter,
          Transaction txRecoverCounter)
          throws Exception {
    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap = new HashMap<>();
    for (PageCountPending pgCount : pendingNonTXPageCounter) {
      long queueID = pgCount.getQueueID();
      long pageID = pgCount.getPageID();

      // We first figure what Queue is based on the queue id
      Queue queue = queues.get(queueID);

      if (queue == null) {
        logger.debug(
            "removing pending page counter id = "
                + pgCount.getID()
                + " as queueID="
                + pgCount.getID()
                + " no longer exists");
        // this means the queue doesn't exist any longer, we will remove it from the storage
        storageManager.deletePendingPageCounter(txRecoverCounter.getID(), pgCount.getID());
        txRecoverCounter.setContainsPersistent();
        continue;
      }

      // Level 1 on the structure, per address
      SimpleString address = queue.getAddress();

      Map<Long, Map<Long, List<PageCountPending>>> perPageMap = perAddressMap.get(address);

      if (perPageMap == null) {
        perPageMap = new HashMap<>();
        perAddressMap.put(address, perPageMap);
      }

      Map<Long, List<PageCountPending>> perQueueMap = perPageMap.get(pageID);

      if (perQueueMap == null) {
        perQueueMap = new HashMap<>();
        perPageMap.put(pageID, perQueueMap);
      }

      List<PageCountPending> pendingCounters = perQueueMap.get(queueID);

      if (pendingCounters == null) {
        pendingCounters = new LinkedList<>();
        perQueueMap.put(queueID, pendingCounters);
      }

      pendingCounters.add(pgCount);

      perQueueMap.put(queueID, pendingCounters);
    }
    return perAddressMap;
  }
  private void createLargeMessage(final long id, boolean liveToBackupSync) {
    ReplicatedLargeMessage msg;
    if (liveToBackupSync) {
      msg = new LargeServerMessageInSync(storageManager);
    } else {
      msg = storageManager.createLargeMessage();
    }

    msg.setDurable(true);
    msg.setMessageID(id);
    largeMessages.put(id, msg);
  }
 @Override
 public void handleNoMessageReferences(Map<Long, ServerMessage> messages) {
   for (ServerMessage msg : messages.values()) {
     if (msg.getRefCount() == 0) {
       ActiveMQServerLogger.LOGGER.journalUnreferencedMessage(msg.getMessageID());
       try {
         storageManager.deleteMessage(msg.getMessageID());
       } catch (Exception ignored) {
         ActiveMQServerLogger.LOGGER.journalErrorDeletingMessage(ignored, msg.getMessageID());
       }
     }
   }
 }
  @Override
  public synchronized void start() throws Exception {
    Configuration config = server.getConfiguration();
    try {
      storageManager = server.getStorageManager();
      storageManager.start();

      server.getManagementService().setStorageManager(storageManager);

      journalsHolder.put(JournalContent.BINDINGS, storageManager.getBindingsJournal());
      journalsHolder.put(JournalContent.MESSAGES, storageManager.getMessageJournal());

      for (JournalContent jc : EnumSet.allOf(JournalContent.class)) {
        filesReservedForSync.put(jc, new HashMap<Long, JournalSyncFile>());
        // We only need to load internal structures on the backup...
        journalLoadInformation[jc.typeByte] =
            journalsHolder.get(jc).loadSyncOnly(JournalState.SYNCING);
      }

      pageManager =
          new PagingManagerImpl(
              new PagingStoreFactoryNIO(
                  storageManager,
                  config.getPagingLocation(),
                  config.getJournalBufferSize_NIO(),
                  server.getScheduledPool(),
                  server.getExecutorFactory(),
                  config.isJournalSyncNonTransactional(),
                  criticalErrorListener),
              server.getAddressSettingsRepository());

      pageManager.start();

      started = true;
    } catch (Exception e) {
      if (server.isStarted()) throw e;
    }
  }
  @Test
  public void testRestartCounter() throws Exception {
    Queue queue =
        server.createQueue(new SimpleString("A1"), new SimpleString("A1"), null, true, false);

    PageSubscriptionCounter counter = locateCounter(queue);

    StorageManager storage = server.getStorageManager();

    Transaction tx = new TransactionImpl(server.getStorageManager());

    counter.increment(tx, 1);

    assertEquals(0, counter.getValue());

    tx.commit();

    storage.waitOnOperations();

    assertEquals(1, counter.getValue());

    sl.close();

    server.stop();

    server = newActiveMQServer();

    server.start();

    queue = server.locateQueue(new SimpleString("A1"));

    assertNotNull(queue);

    counter = locateCounter(queue);

    assertEquals(1, counter.getValue());
  }
示例#13
0
  @Override
  public String sendMessage(
      final Map<String, String> headers,
      final int type,
      final String body,
      final String userID,
      boolean durable,
      final String user,
      final String password)
      throws Exception {
    securityStore.check(
        queue.getAddress(),
        CheckType.SEND,
        new SecurityAuth() {
          @Override
          public String getUsername() {
            return user;
          }

          @Override
          public String getPassword() {
            return password;
          }

          @Override
          public RemotingConnection getRemotingConnection() {
            return null;
          }
        });
    ServerMessageImpl message = new ServerMessageImpl(storageManager.generateID(), 50);
    for (String header : headers.keySet()) {
      message.putStringProperty(new SimpleString(header), new SimpleString(headers.get(header)));
    }
    message.setType((byte) type);
    message.setDurable(durable);
    message.setTimestamp(System.currentTimeMillis());
    message.setUserID(new UUID(UUID.TYPE_TIME_BASED, UUID.stringToBytes(userID)));
    if (body != null) {
      message.getBodyBuffer().writeBytes(Base64.decode(body));
    }
    message.setAddress(queue.getAddress());
    postOffice.route(message, null, true);
    return "" + message.getMessageID();
  }
  /**
   * This method will recover the counters after failures making sure the page counter doesn't get
   * out of sync
   *
   * @param pendingNonTXPageCounter
   * @throws Exception
   */
  @Override
  public void recoverPendingPageCounters(List<PageCountPending> pendingNonTXPageCounter)
      throws Exception {
    // We need a structure of the following
    // Address -> PageID -> QueueID -> List<PageCountPending>
    // The following loop will sort the records according to the hierarchy we need

    Transaction txRecoverCounter = new TransactionImpl(storageManager);

    Map<SimpleString, Map<Long, Map<Long, List<PageCountPending>>>> perAddressMap =
        generateMapsOnPendingCount(queues, pendingNonTXPageCounter, txRecoverCounter);

    for (SimpleString address : perAddressMap.keySet()) {
      PagingStore store = pagingManager.getPageStore(address);
      Map<Long, Map<Long, List<PageCountPending>>> perPageMap = perAddressMap.get(address);

      // We have already generated this before, so it can't be null
      assert (perPageMap != null);

      for (Long pageId : perPageMap.keySet()) {
        Map<Long, List<PageCountPending>> perQueue = perPageMap.get(pageId);

        // This can't be true!
        assert (perQueue != null);

        if (store.checkPageFileExists(pageId.intValue())) {
          // on this case we need to recalculate the records
          Page pg = store.createPage(pageId.intValue());
          pg.open();

          List<PagedMessage> pgMessages = pg.read(storageManager);
          Map<Long, AtomicInteger> countsPerQueueOnPage = new HashMap<>();

          for (PagedMessage pgd : pgMessages) {
            if (pgd.getTransactionID() <= 0) {
              for (long q : pgd.getQueueIDs()) {
                AtomicInteger countQ = countsPerQueueOnPage.get(q);
                if (countQ == null) {
                  countQ = new AtomicInteger(0);
                  countsPerQueueOnPage.put(q, countQ);
                }
                countQ.incrementAndGet();
              }
            }
          }

          for (Map.Entry<Long, List<PageCountPending>> entry : perQueue.entrySet()) {
            for (PageCountPending record : entry.getValue()) {
              logger.debug("Deleting pg tempCount " + record.getID());
              storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
            }

            PageSubscriptionCounter counter =
                store.getCursorProvider().getSubscription(entry.getKey()).getCounter();

            AtomicInteger value = countsPerQueueOnPage.get(entry.getKey());

            if (value == null) {
              logger.debug("Page " + entry.getKey() + " wasn't open, so we will just ignore");
            } else {
              logger.debug("Replacing counter " + value.get());
              counter.increment(txRecoverCounter, value.get());
            }
          }
        } else {
          // on this case the page file didn't exist, we just remove all the records since the page
          // is already gone
          logger.debug(
              "Page "
                  + pageId
                  + " didn't exist on address "
                  + address
                  + ", so we are just removing records");
          for (List<PageCountPending> records : perQueue.values()) {
            for (PageCountPending record : records) {
              logger.debug("Removing pending page counter " + record.getID());
              storageManager.deletePendingPageCounter(txRecoverCounter.getID(), record.getID());
              txRecoverCounter.setContainsPersistent();
            }
          }
        }
      }
    }

    txRecoverCounter.commit();
  }
  @Override
  public void initQueues(
      Map<Long, QueueBindingInfo> queueBindingInfosMap, List<QueueBindingInfo> queueBindingInfos)
      throws Exception {
    int duplicateID = 0;
    for (QueueBindingInfo queueBindingInfo : queueBindingInfos) {
      queueBindingInfosMap.put(queueBindingInfo.getId(), queueBindingInfo);

      Filter filter = FilterImpl.createFilter(queueBindingInfo.getFilterString());

      boolean isTopicIdentification =
          filter != null
              && filter.getFilterString() != null
              && filter
                  .getFilterString()
                  .toString()
                  .equals(ActiveMQServerImpl.GENERIC_IGNORED_FILTER);

      if (postOffice.getBinding(queueBindingInfo.getQueueName()) != null) {

        if (isTopicIdentification) {
          long tx = storageManager.generateID();
          storageManager.deleteQueueBinding(tx, queueBindingInfo.getId());
          storageManager.commitBindings(tx);
          continue;
        } else {

          SimpleString newName = queueBindingInfo.getQueueName().concat("-" + (duplicateID++));
          ActiveMQServerLogger.LOGGER.queueDuplicatedRenaming(
              queueBindingInfo.getQueueName().toString(), newName.toString());
          queueBindingInfo.replaceQueueName(newName);
        }
      }

      PageSubscription subscription = null;

      if (!isTopicIdentification) {
        subscription =
            pagingManager
                .getPageStore(queueBindingInfo.getAddress())
                .getCursorProvider()
                .createSubscription(queueBindingInfo.getId(), filter, true);
      }

      Queue queue =
          queueFactory.createQueue(
              queueBindingInfo.getId(),
              queueBindingInfo.getAddress(),
              queueBindingInfo.getQueueName(),
              filter,
              subscription,
              queueBindingInfo.getUser(),
              true,
              false,
              queueBindingInfo.isAutoCreated());

      if (queueBindingInfo.isAutoCreated()) {
        queue.setConsumersRefCount(
            new AutoCreatedQueueManagerImpl(
                ((PostOfficeImpl) postOffice).getServer().getJMSQueueDeleter(),
                queueBindingInfo.getQueueName()));
      }

      Binding binding =
          new LocalQueueBinding(queueBindingInfo.getAddress(), queue, nodeManager.getNodeId());

      queues.put(queueBindingInfo.getId(), queue);

      postOffice.addBinding(binding);

      managementService.registerAddress(queueBindingInfo.getAddress());
      managementService.registerQueue(queue, queueBindingInfo.getAddress(), storageManager);
    }
  }