public synchronized void reloadStores() throws Exception {
List<PagingStore> reloadedStores = pagingStoreFactory.reloadStores(addressSettingsRepository);
for (PagingStore store : reloadedStores) {
store.start();
stores.put(store.getStoreName(), store);
}
}
public PageCache getPageCache(final long pageId) {
try {
boolean needToRead = false;
PageCache cache = null;
synchronized (softCache) {
if (pageId > pagingStore.getCurrentWritingPage()) {
return null;
}
cache = softCache.get(pageId);
if (cache == null) {
if (!pagingStore.checkPageFileExists((int) pageId)) {
return null;
}
cache = createPageCache(pageId);
needToRead = true;
// anyone reading from this cache will have to wait reading to finish first
// we also want only one thread reading this cache
cache.lock();
if (isTrace) {
HornetQServerLogger.LOGGER.trace(
"adding " + pageId + " into cursor = " + this.pagingStore.getAddress());
}
softCache.put(pageId, cache);
}
}
// Reading is done outside of the synchronized block, however
// the page stays locked until the entire reading is finished
if (needToRead) {
Page page = null;
try {
page = pagingStore.createPage((int) pageId);
storageManager.beforePageRead();
page.open();
List<PagedMessage> pgdMessages = page.read(storageManager);
cache.setMessages(pgdMessages.toArray(new PagedMessage[pgdMessages.size()]));
} finally {
try {
if (page != null) {
page.close();
}
} catch (Throwable ignored) {
}
storageManager.afterPageRead();
cache.unlock();
}
}
return cache;
} catch (Exception e) {
throw new RuntimeException(
"Couldn't complete paging due to an IO Exception on Paging - " + e.getMessage(), e);
}
}
public void requestProducerCredits(final SimpleString address, final int credits)
throws Exception {
PagingStore store = postOffice.getPagingManager().getPageStore(address);
store.executeRunnableWhenMemoryAvailable(
new Runnable() {
public void run() {
callback.sendProducerCreditsMessage(credits, address);
}
});
}
private synchronized PagingStore createPageStore(final SimpleString storeName) throws Exception {
PagingStore store = stores.get(storeName);
if (store == null) {
store = newStore(storeName);
store.start();
stores.put(storeName, store);
}
return store;
}
public void requestProducerCredits(final SimpleString address, final int credits)
throws Exception {
PagingStore store = server.getPagingManager().getPageStore(address);
if (!store.checkMemory(
new Runnable() {
public void run() {
callback.sendProducerCreditsMessage(credits, address);
}
})) {
callback.sendProducerCreditsFailMessage(credits, address);
}
}
public int getNumberOfPages() throws Exception {
clearIO();
try {
PagingStore pageStore = pagingManager.getPageStore(address);
if (!pageStore.isPaging()) {
return 0;
} else {
return pagingManager.getPageStore(address).getNumberOfPages();
}
} finally {
blockOnIO();
}
}
public synchronized void stop() throws Exception {
if (!started) {
return;
}
started = false;
for (PagingStore store : stores.values()) {
store.stop();
}
pagingStoreFactory.stop();
totalMemoryBytes.set(0);
}
public void resumeDepages() {
if (!started) {
// If stop the server while depaging, the server may call a rollback,
// the rollback may addSizes back and that would fire a globalDepage.
// Because of that we must ignore any startGlobalDepage calls,
// and this check needs to be done outside of the lock
return;
}
synchronized (this) {
for (PagingStore store : stores.values()) {
if (store.isPaging()) {
store.startDepaging();
}
}
}
}
public void processReload() throws Exception {
Collection<PageSubscription> cursorList = this.activeCursors.values();
for (PageSubscription cursor : cursorList) {
cursor.processReload();
}
if (!cursorList.isEmpty()) {
// https://issues.jboss.org/browse/JBPAPP-10338 if you ack out of order,
// the min page could be beyond the first page.
// we have to reload any previously acked message
long cursorsMinPage = checkMinPage(cursorList);
// checkMinPage will return MaxValue if there aren't any pages or any cursors
if (cursorsMinPage != Long.MAX_VALUE) {
for (long startPage = pagingStore.getFirstPage(); startPage < cursorsMinPage; startPage++) {
for (PageSubscription cursor : cursorList) {
cursor.reloadPageInfo(startPage);
}
}
}
}
cleanup();
}
@Test
public void testSendPackets() throws Exception {
setupServer(true);
StorageManager storage = getStorage();
manager = liveServer.getReplicationManager();
waitForComponent(manager);
Journal replicatedJournal = new ReplicatedJournal((byte) 1, new FakeJournal(), manager);
replicatedJournal.appendPrepareRecord(1, new FakeData(), false);
replicatedJournal.appendAddRecord(1, (byte) 1, new FakeData(), false);
replicatedJournal.appendUpdateRecord(1, (byte) 2, new FakeData(), false);
replicatedJournal.appendDeleteRecord(1, false);
replicatedJournal.appendAddRecordTransactional(2, 2, (byte) 1, new FakeData());
replicatedJournal.appendUpdateRecordTransactional(2, 2, (byte) 2, new FakeData());
replicatedJournal.appendCommitRecord(2, false);
replicatedJournal.appendDeleteRecordTransactional(3, 4, new FakeData());
replicatedJournal.appendPrepareRecord(3, new FakeData(), false);
replicatedJournal.appendRollbackRecord(3, false);
blockOnReplication(storage, manager);
Assert.assertTrue(
"Expecting no active tokens:" + manager.getActiveTokens(),
manager.getActiveTokens().isEmpty());
ServerMessage msg = new ServerMessageImpl(1, 1024);
SimpleString dummy = new SimpleString("dummy");
msg.setAddress(dummy);
replicatedJournal.appendAddRecordTransactional(23, 24, (byte) 1, new FakeData());
PagedMessage pgmsg = new PagedMessageImpl(msg, new long[0]);
manager.pageWrite(pgmsg, 1);
manager.pageWrite(pgmsg, 2);
manager.pageWrite(pgmsg, 3);
manager.pageWrite(pgmsg, 4);
blockOnReplication(storage, manager);
PagingManager pagingManager =
createPageManager(
backupServer.getStorageManager(), backupServer.getConfiguration(),
backupServer.getExecutorFactory(), backupServer.getAddressSettingsRepository());
PagingStore store = pagingManager.getPageStore(dummy);
store.start();
Assert.assertEquals(4, store.getNumberOfPages());
store.stop();
manager.pageDeleted(dummy, 1);
manager.pageDeleted(dummy, 2);
manager.pageDeleted(dummy, 3);
manager.pageDeleted(dummy, 4);
manager.pageDeleted(dummy, 5);
manager.pageDeleted(dummy, 6);
blockOnReplication(storage, manager);
ServerMessageImpl serverMsg = new ServerMessageImpl();
serverMsg.setMessageID(500);
serverMsg.setAddress(new SimpleString("tttt"));
HornetQBuffer buffer = HornetQBuffers.dynamicBuffer(100);
serverMsg.encodeHeadersAndProperties(buffer);
manager.largeMessageBegin(500);
manager.largeMessageWrite(500, new byte[1024]);
manager.largeMessageDelete(Long.valueOf(500));
blockOnReplication(storage, manager);
store.start();
Assert.assertEquals(0, store.getNumberOfPages());
}
/* Protected as we may let test cases to instrument the test */
protected PageCacheImpl createPageCache(final long pageId) throws Exception {
return new PageCacheImpl(pagingStore.createPage((int) pageId));
}
public void cleanup() {
ArrayList<Page> depagedPages = new ArrayList<Page>();
while (true) {
if (pagingStore.lock(100)) {
break;
}
if (!pagingStore.isStarted()) return;
}
synchronized (this) {
try {
if (!pagingStore.isStarted()) {
return;
}
if (pagingStore.getNumberOfPages() == 0) {
return;
}
if (log.isDebugEnabled()) {
log.debug("Asserting cleanup for address " + this.pagingStore.getAddress());
}
ArrayList<PageSubscription> cursorList = new ArrayList<PageSubscription>();
cursorList.addAll(activeCursors.values());
long minPage = checkMinPage(cursorList);
if (minPage == pagingStore.getCurrentWritingPage()
&& pagingStore.getCurrentPage().getNumberOfMessages() > 0) {
boolean complete = true;
for (PageSubscription cursor : cursorList) {
if (!cursor.isComplete(minPage)) {
if (log.isDebugEnabled()) {
log.debug("Cursor " + cursor + " was considered incomplete at page " + minPage);
}
complete = false;
break;
} else {
if (log.isDebugEnabled()) {
log.debug("Cursor " + cursor + "was considered **complete** at page " + minPage);
}
}
}
if (!pagingStore.isStarted()) {
return;
}
if (complete) {
if (log.isDebugEnabled()) {
log.debug(
"Address "
+ pagingStore.getAddress()
+ " is leaving page mode as all messages are consumed and acknowledged from the page store");
}
pagingStore.forceAnotherPage();
Page currentPage = pagingStore.getCurrentPage();
storePositions(cursorList, currentPage);
pagingStore.stopPaging();
// This has to be called after we stopped paging
for (PageSubscription cursor : cursorList) {
cursor.scheduleCleanupCheck();
}
}
}
for (long i = pagingStore.getFirstPage(); i < minPage; i++) {
Page page = pagingStore.depage();
if (page == null) {
break;
}
depagedPages.add(page);
}
if (pagingStore.getNumberOfPages() == 0
|| pagingStore.getNumberOfPages() == 1
&& pagingStore.getCurrentPage().getNumberOfMessages() == 0) {
pagingStore.stopPaging();
} else {
if (log.isTraceEnabled()) {
log.trace(
"Couldn't cleanup page on address "
+ this.pagingStore.getAddress()
+ " as numberOfPages == "
+ pagingStore.getNumberOfPages()
+ " and currentPage.numberOfMessages = "
+ pagingStore.getCurrentPage().getNumberOfMessages());
}
}
} catch (Exception ex) {
log.warn("Couldn't complete cleanup on paging", ex);
return;
} finally {
pagingStore.unlock();
}
}
try {
for (Page depagedPage : depagedPages) {
PageCache cache;
PagedMessage[] pgdMessages;
synchronized (softCache) {
cache = softCache.get((long) depagedPage.getPageId());
}
if (isTrace) {
log.trace("Removing page " + depagedPage.getPageId() + " from page-cache");
}
if (cache == null) {
// The page is not on cache any more
// We need to read the page-file before deleting it
// to make sure we remove any large-messages pending
storageManager.beforePageRead();
List<PagedMessage> pgdMessagesList = null;
try {
depagedPage.open();
pgdMessagesList = depagedPage.read(storageManager);
} finally {
try {
depagedPage.close();
} catch (Exception e) {
}
storageManager.afterPageRead();
}
depagedPage.close();
pgdMessages = pgdMessagesList.toArray(new PagedMessage[pgdMessagesList.size()]);
} else {
pgdMessages = cache.getMessages();
}
depagedPage.delete(pgdMessages);
synchronized (softCache) {
softCache.remove((long) depagedPage.getPageId());
}
}
} catch (Exception ex) {
log.warn("Couldn't complete cleanup on paging", ex);
return;
}
}
public void deletePageStore(final SimpleString storeName) throws Exception {
PagingStore store = stores.remove(storeName);
if (store != null) {
store.stop();
}
}
public void cleanup() {
ArrayList<Page> depagedPages = new ArrayList<Page>();
while (true) {
if (pagingStore.lock(100)) {
break;
}
if (!pagingStore.isStarted()) return;
}
synchronized (this) {
try {
if (!pagingStore.isStarted()) {
return;
}
if (pagingStore.getNumberOfPages() == 0) {
return;
}
if (HornetQServerLogger.LOGGER.isDebugEnabled()) {
HornetQServerLogger.LOGGER.debug(
"Asserting cleanup for address " + this.pagingStore.getAddress());
}
ArrayList<PageSubscription> cursorList = cloneSubscriptions();
long minPage = checkMinPage(cursorList);
// if the current page is being written...
// on that case we need to move to verify it in a different way
if (minPage == pagingStore.getCurrentWritingPage()
&& pagingStore.getCurrentPage().getNumberOfMessages() > 0) {
boolean complete = true;
for (PageSubscription cursor : cursorList) {
if (!cursor.isComplete(minPage)) {
if (HornetQServerLogger.LOGGER.isDebugEnabled()) {
HornetQServerLogger.LOGGER.debug(
"Cursor " + cursor + " was considered incomplete at page " + minPage);
}
complete = false;
break;
} else {
if (HornetQServerLogger.LOGGER.isDebugEnabled()) {
HornetQServerLogger.LOGGER.debug(
"Cursor " + cursor + "was considered **complete** at page " + minPage);
}
}
}
if (!pagingStore.isStarted()) {
return;
}
// All the pages on the cursor are complete.. so we will cleanup everything and store a
// bookmark
if (complete) {
if (HornetQServerLogger.LOGGER.isDebugEnabled()) {
HornetQServerLogger.LOGGER.debug(
"Address "
+ pagingStore.getAddress()
+ " is leaving page mode as all messages are consumed and acknowledged from the page store");
}
pagingStore.forceAnotherPage();
Page currentPage = pagingStore.getCurrentPage();
storeBookmark(cursorList, currentPage);
pagingStore.stopPaging();
}
}
for (long i = pagingStore.getFirstPage(); i < minPage; i++) {
Page page = pagingStore.depage();
if (page == null) {
break;
}
depagedPages.add(page);
}
if (pagingStore.getNumberOfPages() == 0
|| pagingStore.getNumberOfPages() == 1
&& pagingStore.getCurrentPage().getNumberOfMessages() == 0) {
pagingStore.stopPaging();
} else {
if (HornetQServerLogger.LOGGER.isTraceEnabled()) {
HornetQServerLogger.LOGGER.trace(
"Couldn't cleanup page on address "
+ this.pagingStore.getAddress()
+ " as numberOfPages == "
+ pagingStore.getNumberOfPages()
+ " and currentPage.numberOfMessages = "
+ pagingStore.getCurrentPage().getNumberOfMessages());
}
}
} catch (Exception ex) {
HornetQServerLogger.LOGGER.problemCleaningPageAddress(ex, pagingStore.getAddress());
return;
} finally {
pagingStore.unlock();
}
}
try {
for (Page depagedPage : depagedPages) {
PageCache cache;
PagedMessage[] pgdMessages;
synchronized (softCache) {
cache = softCache.get((long) depagedPage.getPageId());
}
if (isTrace) {
HornetQServerLogger.LOGGER.trace(
"Removing page " + depagedPage.getPageId() + " from page-cache");
}
if (cache == null) {
// The page is not on cache any more
// We need to read the page-file before deleting it
// to make sure we remove any large-messages pending
storageManager.beforePageRead();
List<PagedMessage> pgdMessagesList = null;
try {
depagedPage.open();
pgdMessagesList = depagedPage.read(storageManager);
} finally {
try {
depagedPage.close();
} catch (Exception e) {
}
storageManager.afterPageRead();
}
depagedPage.close();
pgdMessages = pgdMessagesList.toArray(new PagedMessage[pgdMessagesList.size()]);
} else {
pgdMessages = cache.getMessages();
}
depagedPage.delete(pgdMessages);
onDeletePage(depagedPage);
synchronized (softCache) {
softCache.remove((long) depagedPage.getPageId());
}
}
} catch (Exception ex) {
HornetQServerLogger.LOGGER.problemCleaningPageAddress(ex, pagingStore.getAddress());
return;
}
}
@Test
public void testPagingOverCreatedDestinationQueues() throws Exception {
Configuration config = createDefaultConfig();
config.setJournalSyncNonTransactional(false);
HornetQServer server =
createServer(
true,
config,
-1,
-1,
AddressFullMessagePolicy.BLOCK,
new HashMap<String, AddressSettings>());
JMSServerManagerImpl jmsServer = new JMSServerManagerImpl(server);
InVMNamingContext context = new InVMNamingContext();
jmsServer.setContext(context);
jmsServer.start();
server
.getHornetQServerControl()
.addAddressSettings(
"jms.queue.Q1",
"DLQ",
"DLQ",
-1,
false,
5,
100 * 1024,
10 * 1024,
5,
5,
1,
1000,
0,
false,
"PAGE",
-1,
10,
"KILL");
jmsServer.createQueue(true, "Q1", null, true, "/queue/Q1");
HornetQJMSConnectionFactory cf =
(HornetQJMSConnectionFactory)
HornetQJMSClient.createConnectionFactoryWithoutHA(
JMSFactoryType.CF, new TransportConfiguration(INVM_CONNECTOR_FACTORY));
conn = cf.createConnection();
conn.setClientID("tst");
Session sess = conn.createSession(false, Session.AUTO_ACKNOWLEDGE);
javax.jms.Queue queue = (javax.jms.Queue) context.lookup("/queue/Q1");
MessageProducer prod = sess.createProducer(queue);
prod.setDeliveryMode(DeliveryMode.PERSISTENT);
BytesMessage bmt = sess.createBytesMessage();
bmt.writeBytes(new byte[1024]);
for (int i = 0; i < 500; i++) {
prod.send(bmt);
}
PagingStore store = server.getPagingManager().getPageStore(new SimpleString("jms.queue.Q1"));
assertEquals(100 * 1024, store.getMaxSize());
assertEquals(10 * 1024, store.getPageSizeBytes());
assertEquals(AddressFullMessagePolicy.PAGE, store.getAddressFullMessagePolicy());
jmsServer.stop();
server =
createServer(
true,
config,
-1,
-1,
AddressFullMessagePolicy.BLOCK,
new HashMap<String, AddressSettings>());
jmsServer = new JMSServerManagerImpl(server);
context = new InVMNamingContext();
jmsServer.setContext(context);
jmsServer.start();
AddressSettings settings = server.getAddressSettingsRepository().getMatch("jms.queue.Q1");
assertEquals(100 * 1024, settings.getMaxSizeBytes());
assertEquals(10 * 1024, settings.getPageSizeBytes());
assertEquals(AddressFullMessagePolicy.PAGE, settings.getAddressFullMessagePolicy());
store = server.getPagingManager().getPageStore(new SimpleString("jms.queue.Q1"));
assertEquals(100 * 1024, store.getMaxSize());
assertEquals(10 * 1024, store.getPageSizeBytes());
assertEquals(AddressFullMessagePolicy.PAGE, store.getAddressFullMessagePolicy());
}
@Test
public void testPagingOverCreatedDestinationTopics() throws Exception {
Configuration config = createDefaultConfig();
config.setJournalSyncNonTransactional(false);
HornetQServer server =
createServer(true, config, PAGE_SIZE, -1, new HashMap<String, AddressSettings>());
JMSServerManagerImpl jmsServer = new JMSServerManagerImpl(server);
InVMNamingContext context = new InVMNamingContext();
jmsServer.setContext(context);
jmsServer.start();
jmsServer.createTopic(true, "tt", "/topic/TT");
server
.getHornetQServerControl()
.addAddressSettings(
"jms.topic.TT",
"DLQ",
"DLQ",
-1,
false,
5,
1024 * 1024,
1024 * 10,
5,
5,
1,
1000,
0,
false,
"PAGE",
-1,
10,
"KILL");
HornetQJMSConnectionFactory cf =
(HornetQJMSConnectionFactory)
HornetQJMSClient.createConnectionFactoryWithoutHA(
JMSFactoryType.CF, new TransportConfiguration(INVM_CONNECTOR_FACTORY));
Connection conn = cf.createConnection();
conn.setClientID("tst");
Session sess = conn.createSession(false, Session.AUTO_ACKNOWLEDGE);
Topic topic = (Topic) context.lookup("/topic/TT");
sess.createDurableSubscriber(topic, "t1");
MessageProducer prod = sess.createProducer(topic);
prod.setDeliveryMode(DeliveryMode.PERSISTENT);
TextMessage txt = sess.createTextMessage("TST");
prod.send(txt);
PagingStore store = server.getPagingManager().getPageStore(new SimpleString("jms.topic.TT"));
assertEquals(1024 * 1024, store.getMaxSize());
assertEquals(10 * 1024, store.getPageSizeBytes());
jmsServer.stop();
server = createServer(true, config, PAGE_SIZE, -1, new HashMap<String, AddressSettings>());
jmsServer = new JMSServerManagerImpl(server);
context = new InVMNamingContext();
jmsServer.setContext(context);
jmsServer.start();
AddressSettings settings = server.getAddressSettingsRepository().getMatch("jms.topic.TT");
assertEquals(1024 * 1024, settings.getMaxSizeBytes());
assertEquals(10 * 1024, settings.getPageSizeBytes());
assertEquals(AddressFullMessagePolicy.PAGE, settings.getAddressFullMessagePolicy());
store = server.getPagingManager().getPageStore(new SimpleString("TT"));
conn.close();
server.stop();
}