/**
* Tests that incremental metadata changes made on the master are visible (refreshed) when needed
* on the replicas. Incremental metadata changes occur when not all metadata is known to the DPL
* initially when the store is opened, and additional metadata is discovered as entities are
* written. This is scenario A-1 in the [#16655] SR.
*
* <p>This is not actually a schema upgrade test, but is conveniently tested here using the
* upgrade test framework.
*/
@Test
public void testIncrementalMetadataChanges() throws Exception {
open();
/* Master writes and reads Data entity. */
masterApp.writeData(0);
masterApp.readData(0);
/* Replicas read Data entity. */
replicaApp1.readData(0);
replicaApp2.readData(0);
/* Master writes DataA (subclass), causing a metadata update. */
masterApp.writeDataA(1);
masterApp.readDataA(1);
/* Replicas read DataA and must refresh metadata. */
replicaApp1.readDataA(1);
replicaApp2.readDataA(1);
/* Read Data again for good measure. */
masterApp.readData(0);
replicaApp1.readData(0);
replicaApp2.readData(0);
close();
}
/**
* Tests that when a replica having stale metadata is elected master, the first metadata update on
* the new master causes refresh of the stale metadata before the new metadata is written. This is
* scenario A-2 in the [#16655] SR.
*
* <p>This is not actually a schema upgrade test, but is conveniently tested here using the
* upgrade test framework.
*/
@Test
public void testElectedMasterWithStaleMetadata() throws Exception {
open();
/* Master writes and reads Data entity. */
masterApp.writeData(0);
masterApp.readData(0);
/* Replicas read Data entity. */
replicaApp1.readData(0);
replicaApp2.readData(0);
/* Master writes DataA (subclass), causing a metadata update. */
masterApp.writeDataA(1);
masterApp.readDataA(1);
/*
* Master is bounced (but not upgraded), replica1 switches roles with
* master.
*/
bounceMaster(0);
/*
* Master writes DataB, which requires a metadata change. Before this
* new metadata change, it must refresh metadata from disk to get the
* definition of DataA.
*/
masterApp.writeDataB(2);
/*
* Reading DataA would cause a ClassCastException if refresh did not
* occur above, because the format ID for DataA would be incorrect.
*/
masterApp.readDataA(1);
/* Read all again for good measure. */
masterApp.readData(0);
masterApp.readDataA(1);
masterApp.readDataB(2);
replicaApp1.readData(0);
replicaApp1.readDataA(1);
replicaApp1.readDataB(2);
replicaApp2.readData(0);
replicaApp2.readDataA(1);
replicaApp2.readDataB(2);
close();
}
/**
* Tests that a reasonable exception occurs when an upgraded node is elected Master *before* all
* other nodes have been upgraded. This is a user error, since the Master election should occur
* last, but it cannot always be avoided, for example, when an unexpected failover occurs during
* the upgrade process.
*
* <p>There are two cases: (1) when the non-upgraded Replica node is already running when an
* upgraded node becomes Master, and (2) when the non-upgraded node is brought up as a Replica in
* a group with an upgraded Master. However, implementation-wise case (1) becomes case (2),
* because in case (1) the Replica will attempt to refresh metadata, which is internally the same
* as bringing up the Replica from scratch. In both case we instantiate a new PersistCatalog
* internally, and run class evolution. This should result in an IncompatibleClassException.
*/
@Test
public void testPrematureUpgradedMaster() throws Exception {
open();
/* Master writes v0 entity, all nodes read. */
masterApp.writeData(0);
masterApp.readData(0);
replicaApp1.readData(0);
replicaApp2.readData(0);
/* Replica2 is upgraded to v1, then Master is upgraded to v1. */
bounceReplica2(1);
bounceMaster(1);
/* Replica2 and Replica1 were swapped when the Master was bounced. */
assertEquals(1, masterApp.getVersion());
assertEquals(1, replicaApp1.getVersion());
assertEquals(0, replicaApp2.getVersion());
/* Write a v1 entity on the Master. */
try {
masterApp.writeData(10);
fail();
} catch (DatabasePreemptedException expected) {
masterApp.close();
masterApp.open(masterEnv);
masterApp.writeData(10);
}
/* The upgraded replica can read the v1 entity. */
try {
replicaApp1.readData(10);
fail();
} catch (DatabasePreemptedException expected) {
replicaApp1.close();
replicaApp1.open(replicaEnv1);
replicaApp1.readData(10);
}
/* The non-upgraded replica will get IncompatibleClassException. */
try {
replicaApp2.readData(10);
fail();
} catch (DatabasePreemptedException expected) {
replicaApp2.close();
try {
replicaApp2.open(replicaEnv2);
fail();
} catch (IncompatibleClassException expected2) {
}
}
/* When finally upgraded, the replica can read the v1 entity. */
bounceReplica2(1);
replicaApp2.readData(10);
/* Read all for good measure. */
masterApp.readData(0);
masterApp.readData(10);
replicaApp1.readData(0);
replicaApp1.readData(10);
replicaApp2.readData(0);
replicaApp2.readData(10);
close();
}
/**
* Ensure that when the master is bounced, the first write will refresh metadata. The testUpgrade
* method, OTOH, ensures that metadata is refreshed when new indexes are requested.
*/
@Test
public void testRefreshAfterFirstWrite() throws Exception {
open();
/* Master writes v0 entity, all nodes read. */
masterApp.writeData(0);
masterApp.readData(0);
replicaApp1.readData(0);
replicaApp2.readData(0);
/* Replica1 is upgraded to v1, upgrades metadata in memory. */
bounceReplica1(1);
/* Upgraded replica1 reads v0 entities, can't get new index. */
try {
replicaApp1.readData(0);
fail();
} catch (IndexNotAvailableException e) {
}
/* Replica2 (not yet upgraded) reads v0 entity without errors. */
replicaApp2.readData(0);
/* Replica2 is upgraded to v1, upgrades metadata in memory. */
bounceReplica2(1);
/* Read again on master for good measure. */
masterApp.readData(0);
/* Master is upgraded to v1, replica1 switches roles with master. */
bounceMaster(1);
/* Metadata is refreshed on first write. */
try {
masterApp.writeData(10);
fail();
} catch (DatabasePreemptedException expected) {
masterApp.close();
masterApp.open(masterEnv);
masterApp.writeData(10);
}
/* Replicas also get DatabasePreemptedException on first read. */
try {
replicaApp1.readData(0);
fail();
} catch (DatabasePreemptedException expected) {
replicaApp1.close();
replicaApp1.open(replicaEnv1);
replicaApp1.readData(0);
}
try {
replicaApp2.readData(0);
fail();
} catch (DatabasePreemptedException expected) {
replicaApp2.close();
replicaApp2.open(replicaEnv2);
replicaApp2.readData(0);
}
/* All reads now work. */
masterApp.readData(0);
masterApp.readData(10);
replicaApp1.readData(0);
replicaApp1.readData(10);
replicaApp2.readData(0);
replicaApp2.readData(10);
close();
}
/** Tests scenarios B-1 and B-2 in the [#16655] SR. */
@Test
public void testUpgrade() throws Exception {
open();
/* Master writes and reads v0 entities. */
masterApp.writeData(0);
masterApp.writeDataA(1);
masterApp.writeDataB(2);
masterApp.readData(0);
masterApp.readDataA(1);
masterApp.readDataB(2);
/* Replicas read v0 entities. */
replicaApp1.readData(0);
replicaApp1.readDataA(1);
replicaApp1.readDataB(2);
replicaApp2.readData(0);
replicaApp2.readDataA(1);
replicaApp2.readDataB(2);
/* Replica1 is upgraded to v1, upgrades metadata in memory. */
bounceReplica1(1);
/* Upgraded replica1 reads v0 entities, can't get new index. */
try {
replicaApp1.readData(0);
fail();
} catch (IndexNotAvailableException e) {
}
try {
replicaApp1.readDataB(2);
fail();
} catch (IndexNotAvailableException e) {
}
/* Upgraded replica1 can't get index for new entity NewData2. */
try {
replicaApp1.readData2(14);
fail();
} catch (IndexNotAvailableException e) {
}
/* Replica1 can read v0 DataA, because it has no new indexes. */
replicaApp1.readDataA(1);
/* Replica2 (not yet upgraded) reads v0 entities without errors. */
replicaApp2.readData(0);
replicaApp2.readDataA(1);
replicaApp2.readDataB(2);
/* Replica2 is upgraded to v1, upgrades metadata in memory. */
bounceReplica2(1);
/* Upgraded replicas read v0 entities, can't get new index. */
try {
replicaApp1.readData(0);
fail();
} catch (IndexNotAvailableException e) {
}
try {
replicaApp1.readDataB(2);
fail();
} catch (IndexNotAvailableException e) {
}
try {
replicaApp2.readData(0);
fail();
} catch (IndexNotAvailableException e) {
}
try {
replicaApp2.readDataB(2);
fail();
} catch (IndexNotAvailableException e) {
}
/* Upgraded replicas can't get index for new entity NewData2. */
try {
replicaApp1.readData2(14);
fail();
} catch (IndexNotAvailableException e) {
}
try {
replicaApp2.readData2(14);
fail();
} catch (IndexNotAvailableException e) {
}
/* Upgraded replicas can read v0 DataA, it has no new indexes. */
replicaApp1.readDataA(1);
replicaApp2.readDataA(1);
/* Read again on master for good measure. */
masterApp.readData(0);
masterApp.readDataA(1);
masterApp.readDataB(2);
/* Master is upgraded to v1, replica1 switches roles with master. */
bounceMaster(1);
/* Metadata is refreshed when new indexes are requested. */
try {
masterApp.readData(0);
fail();
} catch (DatabasePreemptedException expected) {
masterApp.close();
masterApp.open(masterEnv);
masterApp.readData(0);
}
masterApp.readDataA(1);
masterApp.readDataB(2);
try {
replicaApp1.readData(0);
fail();
} catch (DatabasePreemptedException expected) {
replicaApp1.close();
replicaApp1.open(replicaEnv1);
replicaApp1.readData(0);
}
replicaApp1.readDataA(1);
replicaApp1.readDataB(2);
try {
replicaApp2.readData(0);
fail();
} catch (DatabasePreemptedException expected) {
replicaApp2.close();
replicaApp2.open(replicaEnv2);
replicaApp2.readData(0);
}
replicaApp2.readDataA(1);
replicaApp2.readDataB(2);
/* Master writes v1 entities. */
masterApp.writeData(10);
masterApp.writeDataA(11);
masterApp.writeDataB(12);
/* Master reads v0 and v1 entities. */
masterApp.readData(0);
masterApp.readData(10);
masterApp.readDataA(1);
masterApp.readDataA(11);
masterApp.readDataB(2);
masterApp.readDataB(12);
/* Replicas read v0 and v1 entities. */
replicaApp1.readData(0);
replicaApp1.readData(10);
replicaApp1.readDataA(1);
replicaApp1.readDataA(11);
replicaApp1.readDataB(2);
replicaApp1.readDataB(12);
replicaApp2.readData(0);
replicaApp2.readData(10);
replicaApp2.readDataA(1);
replicaApp2.readDataA(11);
replicaApp2.readDataB(2);
replicaApp2.readDataB(12);
/* Master writes new NewDataC subclass, all can read. */
masterApp.writeDataC(13);
masterApp.readDataC(13);
replicaApp1.readDataC(13);
replicaApp2.readDataC(13);
/* Master writes new NewData2 entity class, all can read. */
masterApp.writeData2(14);
masterApp.readData2(14);
replicaApp1.readData2(14);
replicaApp2.readData2(14);
close();
}
