Ejemplo n.º 1
0
  public synchronized void reloadStores() throws Exception {
    List<PagingStore> reloadedStores = pagingStoreFactory.reloadStores(addressSettingsRepository);

    for (PagingStore store : reloadedStores) {
      store.start();
      stores.put(store.getStoreName(), store);
    }
  }
Ejemplo n.º 2
0
  public PageCache getPageCache(final long pageId) {
    try {
      boolean needToRead = false;
      PageCache cache = null;
      synchronized (softCache) {
        if (pageId > pagingStore.getCurrentWritingPage()) {
          return null;
        }

        cache = softCache.get(pageId);
        if (cache == null) {
          if (!pagingStore.checkPageFileExists((int) pageId)) {
            return null;
          }

          cache = createPageCache(pageId);
          needToRead = true;
          // anyone reading from this cache will have to wait reading to finish first
          // we also want only one thread reading this cache
          cache.lock();
          if (isTrace) {
            HornetQServerLogger.LOGGER.trace(
                "adding " + pageId + " into cursor = " + this.pagingStore.getAddress());
          }
          softCache.put(pageId, cache);
        }
      }

      // Reading is done outside of the synchronized block, however
      // the page stays locked until the entire reading is finished
      if (needToRead) {
        Page page = null;
        try {
          page = pagingStore.createPage((int) pageId);

          storageManager.beforePageRead();
          page.open();

          List<PagedMessage> pgdMessages = page.read(storageManager);
          cache.setMessages(pgdMessages.toArray(new PagedMessage[pgdMessages.size()]));
        } finally {
          try {
            if (page != null) {
              page.close();
            }
          } catch (Throwable ignored) {
          }
          storageManager.afterPageRead();
          cache.unlock();
        }
      }

      return cache;
    } catch (Exception e) {
      throw new RuntimeException(
          "Couldn't complete paging due to an IO Exception on Paging - " + e.getMessage(), e);
    }
  }
Ejemplo n.º 3
0
  public void requestProducerCredits(final SimpleString address, final int credits)
      throws Exception {
    PagingStore store = postOffice.getPagingManager().getPageStore(address);

    store.executeRunnableWhenMemoryAvailable(
        new Runnable() {
          public void run() {
            callback.sendProducerCreditsMessage(credits, address);
          }
        });
  }
Ejemplo n.º 4
0
  private synchronized PagingStore createPageStore(final SimpleString storeName) throws Exception {
    PagingStore store = stores.get(storeName);

    if (store == null) {
      store = newStore(storeName);

      store.start();

      stores.put(storeName, store);
    }

    return store;
  }
Ejemplo n.º 5
0
  public void requestProducerCredits(final SimpleString address, final int credits)
      throws Exception {
    PagingStore store = server.getPagingManager().getPageStore(address);

    if (!store.checkMemory(
        new Runnable() {
          public void run() {
            callback.sendProducerCreditsMessage(credits, address);
          }
        })) {
      callback.sendProducerCreditsFailMessage(credits, address);
    }
  }
Ejemplo n.º 6
0
  public int getNumberOfPages() throws Exception {
    clearIO();
    try {
      PagingStore pageStore = pagingManager.getPageStore(address);

      if (!pageStore.isPaging()) {
        return 0;
      } else {
        return pagingManager.getPageStore(address).getNumberOfPages();
      }
    } finally {
      blockOnIO();
    }
  }
Ejemplo n.º 7
0
  public synchronized void stop() throws Exception {
    if (!started) {
      return;
    }

    started = false;

    for (PagingStore store : stores.values()) {
      store.stop();
    }

    pagingStoreFactory.stop();

    totalMemoryBytes.set(0);
  }
Ejemplo n.º 8
0
 public void resumeDepages() {
   if (!started) {
     // If stop the server while depaging, the server may call a rollback,
     // the rollback may addSizes back and that would fire a globalDepage.
     // Because of that we must ignore any startGlobalDepage calls,
     // and this check needs to be done outside of the lock
     return;
   }
   synchronized (this) {
     for (PagingStore store : stores.values()) {
       if (store.isPaging()) {
         store.startDepaging();
       }
     }
   }
 }
Ejemplo n.º 9
0
  public void processReload() throws Exception {
    Collection<PageSubscription> cursorList = this.activeCursors.values();
    for (PageSubscription cursor : cursorList) {
      cursor.processReload();
    }

    if (!cursorList.isEmpty()) {
      // https://issues.jboss.org/browse/JBPAPP-10338 if you ack out of order,
      // the min page could be beyond the first page.
      // we have to reload any previously acked message
      long cursorsMinPage = checkMinPage(cursorList);

      // checkMinPage will return MaxValue if there aren't any pages or any cursors
      if (cursorsMinPage != Long.MAX_VALUE) {
        for (long startPage = pagingStore.getFirstPage(); startPage < cursorsMinPage; startPage++) {
          for (PageSubscription cursor : cursorList) {
            cursor.reloadPageInfo(startPage);
          }
        }
      }
    }

    cleanup();
  }
Ejemplo n.º 10
0
  @Test
  public void testSendPackets() throws Exception {
    setupServer(true);

    StorageManager storage = getStorage();

    manager = liveServer.getReplicationManager();
    waitForComponent(manager);

    Journal replicatedJournal = new ReplicatedJournal((byte) 1, new FakeJournal(), manager);

    replicatedJournal.appendPrepareRecord(1, new FakeData(), false);

    replicatedJournal.appendAddRecord(1, (byte) 1, new FakeData(), false);
    replicatedJournal.appendUpdateRecord(1, (byte) 2, new FakeData(), false);
    replicatedJournal.appendDeleteRecord(1, false);
    replicatedJournal.appendAddRecordTransactional(2, 2, (byte) 1, new FakeData());
    replicatedJournal.appendUpdateRecordTransactional(2, 2, (byte) 2, new FakeData());
    replicatedJournal.appendCommitRecord(2, false);

    replicatedJournal.appendDeleteRecordTransactional(3, 4, new FakeData());
    replicatedJournal.appendPrepareRecord(3, new FakeData(), false);
    replicatedJournal.appendRollbackRecord(3, false);

    blockOnReplication(storage, manager);

    Assert.assertTrue(
        "Expecting no active tokens:" + manager.getActiveTokens(),
        manager.getActiveTokens().isEmpty());

    ServerMessage msg = new ServerMessageImpl(1, 1024);

    SimpleString dummy = new SimpleString("dummy");
    msg.setAddress(dummy);

    replicatedJournal.appendAddRecordTransactional(23, 24, (byte) 1, new FakeData());

    PagedMessage pgmsg = new PagedMessageImpl(msg, new long[0]);
    manager.pageWrite(pgmsg, 1);
    manager.pageWrite(pgmsg, 2);
    manager.pageWrite(pgmsg, 3);
    manager.pageWrite(pgmsg, 4);

    blockOnReplication(storage, manager);

    PagingManager pagingManager =
        createPageManager(
            backupServer.getStorageManager(), backupServer.getConfiguration(),
            backupServer.getExecutorFactory(), backupServer.getAddressSettingsRepository());

    PagingStore store = pagingManager.getPageStore(dummy);
    store.start();
    Assert.assertEquals(4, store.getNumberOfPages());
    store.stop();

    manager.pageDeleted(dummy, 1);
    manager.pageDeleted(dummy, 2);
    manager.pageDeleted(dummy, 3);
    manager.pageDeleted(dummy, 4);
    manager.pageDeleted(dummy, 5);
    manager.pageDeleted(dummy, 6);

    blockOnReplication(storage, manager);

    ServerMessageImpl serverMsg = new ServerMessageImpl();
    serverMsg.setMessageID(500);
    serverMsg.setAddress(new SimpleString("tttt"));

    HornetQBuffer buffer = HornetQBuffers.dynamicBuffer(100);
    serverMsg.encodeHeadersAndProperties(buffer);

    manager.largeMessageBegin(500);

    manager.largeMessageWrite(500, new byte[1024]);

    manager.largeMessageDelete(Long.valueOf(500));

    blockOnReplication(storage, manager);

    store.start();

    Assert.assertEquals(0, store.getNumberOfPages());
  }
 /* Protected as we may let test cases to instrument the test */
 protected PageCacheImpl createPageCache(final long pageId) throws Exception {
   return new PageCacheImpl(pagingStore.createPage((int) pageId));
 }
  public void cleanup() {
    ArrayList<Page> depagedPages = new ArrayList<Page>();

    while (true) {
      if (pagingStore.lock(100)) {
        break;
      }
      if (!pagingStore.isStarted()) return;
    }

    synchronized (this) {
      try {
        if (!pagingStore.isStarted()) {
          return;
        }

        if (pagingStore.getNumberOfPages() == 0) {
          return;
        }

        if (log.isDebugEnabled()) {
          log.debug("Asserting cleanup for address " + this.pagingStore.getAddress());
        }

        ArrayList<PageSubscription> cursorList = new ArrayList<PageSubscription>();
        cursorList.addAll(activeCursors.values());

        long minPage = checkMinPage(cursorList);

        if (minPage == pagingStore.getCurrentWritingPage()
            && pagingStore.getCurrentPage().getNumberOfMessages() > 0) {
          boolean complete = true;

          for (PageSubscription cursor : cursorList) {
            if (!cursor.isComplete(minPage)) {
              if (log.isDebugEnabled()) {
                log.debug("Cursor " + cursor + " was considered incomplete at page " + minPage);
              }

              complete = false;
              break;
            } else {
              if (log.isDebugEnabled()) {
                log.debug("Cursor " + cursor + "was considered **complete** at page " + minPage);
              }
            }
          }

          if (!pagingStore.isStarted()) {
            return;
          }

          if (complete) {

            if (log.isDebugEnabled()) {
              log.debug(
                  "Address "
                      + pagingStore.getAddress()
                      + " is leaving page mode as all messages are consumed and acknowledged from the page store");
            }

            pagingStore.forceAnotherPage();

            Page currentPage = pagingStore.getCurrentPage();

            storePositions(cursorList, currentPage);

            pagingStore.stopPaging();

            // This has to be called after we stopped paging
            for (PageSubscription cursor : cursorList) {
              cursor.scheduleCleanupCheck();
            }
          }
        }

        for (long i = pagingStore.getFirstPage(); i < minPage; i++) {
          Page page = pagingStore.depage();
          if (page == null) {
            break;
          }
          depagedPages.add(page);
        }

        if (pagingStore.getNumberOfPages() == 0
            || pagingStore.getNumberOfPages() == 1
                && pagingStore.getCurrentPage().getNumberOfMessages() == 0) {
          pagingStore.stopPaging();
        } else {
          if (log.isTraceEnabled()) {
            log.trace(
                "Couldn't cleanup page on address "
                    + this.pagingStore.getAddress()
                    + " as numberOfPages == "
                    + pagingStore.getNumberOfPages()
                    + " and currentPage.numberOfMessages = "
                    + pagingStore.getCurrentPage().getNumberOfMessages());
          }
        }
      } catch (Exception ex) {
        log.warn("Couldn't complete cleanup on paging", ex);
        return;
      } finally {
        pagingStore.unlock();
      }
    }

    try {
      for (Page depagedPage : depagedPages) {
        PageCache cache;
        PagedMessage[] pgdMessages;
        synchronized (softCache) {
          cache = softCache.get((long) depagedPage.getPageId());
        }

        if (isTrace) {
          log.trace("Removing page " + depagedPage.getPageId() + " from page-cache");
        }

        if (cache == null) {
          // The page is not on cache any more
          // We need to read the page-file before deleting it
          // to make sure we remove any large-messages pending
          storageManager.beforePageRead();

          List<PagedMessage> pgdMessagesList = null;
          try {
            depagedPage.open();
            pgdMessagesList = depagedPage.read(storageManager);
          } finally {
            try {
              depagedPage.close();
            } catch (Exception e) {
            }

            storageManager.afterPageRead();
          }
          depagedPage.close();
          pgdMessages = pgdMessagesList.toArray(new PagedMessage[pgdMessagesList.size()]);
        } else {
          pgdMessages = cache.getMessages();
        }

        depagedPage.delete(pgdMessages);

        synchronized (softCache) {
          softCache.remove((long) depagedPage.getPageId());
        }
      }
    } catch (Exception ex) {
      log.warn("Couldn't complete cleanup on paging", ex);
      return;
    }
  }
Ejemplo n.º 13
0
 public void deletePageStore(final SimpleString storeName) throws Exception {
   PagingStore store = stores.remove(storeName);
   if (store != null) {
     store.stop();
   }
 }
Ejemplo n.º 14
0
  public void cleanup() {
    ArrayList<Page> depagedPages = new ArrayList<Page>();

    while (true) {
      if (pagingStore.lock(100)) {
        break;
      }
      if (!pagingStore.isStarted()) return;
    }

    synchronized (this) {
      try {
        if (!pagingStore.isStarted()) {
          return;
        }

        if (pagingStore.getNumberOfPages() == 0) {
          return;
        }

        if (HornetQServerLogger.LOGGER.isDebugEnabled()) {
          HornetQServerLogger.LOGGER.debug(
              "Asserting cleanup for address " + this.pagingStore.getAddress());
        }

        ArrayList<PageSubscription> cursorList = cloneSubscriptions();

        long minPage = checkMinPage(cursorList);

        // if the current page is being written...
        // on that case we need to move to verify it in a different way
        if (minPage == pagingStore.getCurrentWritingPage()
            && pagingStore.getCurrentPage().getNumberOfMessages() > 0) {
          boolean complete = true;

          for (PageSubscription cursor : cursorList) {
            if (!cursor.isComplete(minPage)) {
              if (HornetQServerLogger.LOGGER.isDebugEnabled()) {
                HornetQServerLogger.LOGGER.debug(
                    "Cursor " + cursor + " was considered incomplete at page " + minPage);
              }

              complete = false;
              break;
            } else {
              if (HornetQServerLogger.LOGGER.isDebugEnabled()) {
                HornetQServerLogger.LOGGER.debug(
                    "Cursor " + cursor + "was considered **complete** at page " + minPage);
              }
            }
          }

          if (!pagingStore.isStarted()) {
            return;
          }

          // All the pages on the cursor are complete.. so we will cleanup everything and store a
          // bookmark
          if (complete) {

            if (HornetQServerLogger.LOGGER.isDebugEnabled()) {
              HornetQServerLogger.LOGGER.debug(
                  "Address "
                      + pagingStore.getAddress()
                      + " is leaving page mode as all messages are consumed and acknowledged from the page store");
            }

            pagingStore.forceAnotherPage();

            Page currentPage = pagingStore.getCurrentPage();

            storeBookmark(cursorList, currentPage);

            pagingStore.stopPaging();
          }
        }

        for (long i = pagingStore.getFirstPage(); i < minPage; i++) {
          Page page = pagingStore.depage();
          if (page == null) {
            break;
          }
          depagedPages.add(page);
        }

        if (pagingStore.getNumberOfPages() == 0
            || pagingStore.getNumberOfPages() == 1
                && pagingStore.getCurrentPage().getNumberOfMessages() == 0) {
          pagingStore.stopPaging();
        } else {
          if (HornetQServerLogger.LOGGER.isTraceEnabled()) {
            HornetQServerLogger.LOGGER.trace(
                "Couldn't cleanup page on address "
                    + this.pagingStore.getAddress()
                    + " as numberOfPages == "
                    + pagingStore.getNumberOfPages()
                    + " and currentPage.numberOfMessages = "
                    + pagingStore.getCurrentPage().getNumberOfMessages());
          }
        }
      } catch (Exception ex) {
        HornetQServerLogger.LOGGER.problemCleaningPageAddress(ex, pagingStore.getAddress());
        return;
      } finally {
        pagingStore.unlock();
      }
    }

    try {
      for (Page depagedPage : depagedPages) {
        PageCache cache;
        PagedMessage[] pgdMessages;
        synchronized (softCache) {
          cache = softCache.get((long) depagedPage.getPageId());
        }

        if (isTrace) {
          HornetQServerLogger.LOGGER.trace(
              "Removing page " + depagedPage.getPageId() + " from page-cache");
        }

        if (cache == null) {
          // The page is not on cache any more
          // We need to read the page-file before deleting it
          // to make sure we remove any large-messages pending
          storageManager.beforePageRead();

          List<PagedMessage> pgdMessagesList = null;
          try {
            depagedPage.open();
            pgdMessagesList = depagedPage.read(storageManager);
          } finally {
            try {
              depagedPage.close();
            } catch (Exception e) {
            }

            storageManager.afterPageRead();
          }
          depagedPage.close();
          pgdMessages = pgdMessagesList.toArray(new PagedMessage[pgdMessagesList.size()]);
        } else {
          pgdMessages = cache.getMessages();
        }

        depagedPage.delete(pgdMessages);
        onDeletePage(depagedPage);

        synchronized (softCache) {
          softCache.remove((long) depagedPage.getPageId());
        }
      }
    } catch (Exception ex) {
      HornetQServerLogger.LOGGER.problemCleaningPageAddress(ex, pagingStore.getAddress());
      return;
    }
  }
Ejemplo n.º 15
0
  @Test
  public void testPagingOverCreatedDestinationQueues() throws Exception {

    Configuration config = createDefaultConfig();

    config.setJournalSyncNonTransactional(false);

    HornetQServer server =
        createServer(
            true,
            config,
            -1,
            -1,
            AddressFullMessagePolicy.BLOCK,
            new HashMap<String, AddressSettings>());

    JMSServerManagerImpl jmsServer = new JMSServerManagerImpl(server);
    InVMNamingContext context = new InVMNamingContext();
    jmsServer.setContext(context);
    jmsServer.start();

    server
        .getHornetQServerControl()
        .addAddressSettings(
            "jms.queue.Q1",
            "DLQ",
            "DLQ",
            -1,
            false,
            5,
            100 * 1024,
            10 * 1024,
            5,
            5,
            1,
            1000,
            0,
            false,
            "PAGE",
            -1,
            10,
            "KILL");

    jmsServer.createQueue(true, "Q1", null, true, "/queue/Q1");

    HornetQJMSConnectionFactory cf =
        (HornetQJMSConnectionFactory)
            HornetQJMSClient.createConnectionFactoryWithoutHA(
                JMSFactoryType.CF, new TransportConfiguration(INVM_CONNECTOR_FACTORY));

    conn = cf.createConnection();
    conn.setClientID("tst");
    Session sess = conn.createSession(false, Session.AUTO_ACKNOWLEDGE);
    javax.jms.Queue queue = (javax.jms.Queue) context.lookup("/queue/Q1");

    MessageProducer prod = sess.createProducer(queue);
    prod.setDeliveryMode(DeliveryMode.PERSISTENT);
    BytesMessage bmt = sess.createBytesMessage();

    bmt.writeBytes(new byte[1024]);

    for (int i = 0; i < 500; i++) {
      prod.send(bmt);
    }

    PagingStore store = server.getPagingManager().getPageStore(new SimpleString("jms.queue.Q1"));

    assertEquals(100 * 1024, store.getMaxSize());
    assertEquals(10 * 1024, store.getPageSizeBytes());
    assertEquals(AddressFullMessagePolicy.PAGE, store.getAddressFullMessagePolicy());

    jmsServer.stop();

    server =
        createServer(
            true,
            config,
            -1,
            -1,
            AddressFullMessagePolicy.BLOCK,
            new HashMap<String, AddressSettings>());

    jmsServer = new JMSServerManagerImpl(server);
    context = new InVMNamingContext();
    jmsServer.setContext(context);
    jmsServer.start();

    AddressSettings settings = server.getAddressSettingsRepository().getMatch("jms.queue.Q1");

    assertEquals(100 * 1024, settings.getMaxSizeBytes());
    assertEquals(10 * 1024, settings.getPageSizeBytes());
    assertEquals(AddressFullMessagePolicy.PAGE, settings.getAddressFullMessagePolicy());

    store = server.getPagingManager().getPageStore(new SimpleString("jms.queue.Q1"));
    assertEquals(100 * 1024, store.getMaxSize());
    assertEquals(10 * 1024, store.getPageSizeBytes());
    assertEquals(AddressFullMessagePolicy.PAGE, store.getAddressFullMessagePolicy());
  }
Ejemplo n.º 16
0
  @Test
  public void testPagingOverCreatedDestinationTopics() throws Exception {

    Configuration config = createDefaultConfig();

    config.setJournalSyncNonTransactional(false);

    HornetQServer server =
        createServer(true, config, PAGE_SIZE, -1, new HashMap<String, AddressSettings>());

    JMSServerManagerImpl jmsServer = new JMSServerManagerImpl(server);
    InVMNamingContext context = new InVMNamingContext();
    jmsServer.setContext(context);
    jmsServer.start();

    jmsServer.createTopic(true, "tt", "/topic/TT");

    server
        .getHornetQServerControl()
        .addAddressSettings(
            "jms.topic.TT",
            "DLQ",
            "DLQ",
            -1,
            false,
            5,
            1024 * 1024,
            1024 * 10,
            5,
            5,
            1,
            1000,
            0,
            false,
            "PAGE",
            -1,
            10,
            "KILL");

    HornetQJMSConnectionFactory cf =
        (HornetQJMSConnectionFactory)
            HornetQJMSClient.createConnectionFactoryWithoutHA(
                JMSFactoryType.CF, new TransportConfiguration(INVM_CONNECTOR_FACTORY));

    Connection conn = cf.createConnection();
    conn.setClientID("tst");
    Session sess = conn.createSession(false, Session.AUTO_ACKNOWLEDGE);
    Topic topic = (Topic) context.lookup("/topic/TT");
    sess.createDurableSubscriber(topic, "t1");

    MessageProducer prod = sess.createProducer(topic);
    prod.setDeliveryMode(DeliveryMode.PERSISTENT);
    TextMessage txt = sess.createTextMessage("TST");
    prod.send(txt);

    PagingStore store = server.getPagingManager().getPageStore(new SimpleString("jms.topic.TT"));

    assertEquals(1024 * 1024, store.getMaxSize());
    assertEquals(10 * 1024, store.getPageSizeBytes());

    jmsServer.stop();

    server = createServer(true, config, PAGE_SIZE, -1, new HashMap<String, AddressSettings>());

    jmsServer = new JMSServerManagerImpl(server);
    context = new InVMNamingContext();
    jmsServer.setContext(context);
    jmsServer.start();

    AddressSettings settings = server.getAddressSettingsRepository().getMatch("jms.topic.TT");

    assertEquals(1024 * 1024, settings.getMaxSizeBytes());
    assertEquals(10 * 1024, settings.getPageSizeBytes());
    assertEquals(AddressFullMessagePolicy.PAGE, settings.getAddressFullMessagePolicy());

    store = server.getPagingManager().getPageStore(new SimpleString("TT"));

    conn.close();

    server.stop();
  }