/**
* Tests that Replication worker should clean the leadger under replication node of the ledger
* already deleted
*/
@Test(timeout = 3000)
public void testRWShouldCleanTheLedgerFromUnderReplicationIfLedgerAlreadyDeleted()
throws Exception {
LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
bkc.deleteLedger(lh.getId()); // Deleting the ledger
// Also mark ledger as in UnderReplication
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
} finally {
rw.shutdown();
}
}
/**
* Tests that ReplicationWorker should not have identified for postponing the replication if
* ledger is in open state and lastFragment is not in underReplication state. Note that RW should
* not fence such ledgers.
*/
@Test(timeout = 30000)
public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR() throws Exception {
LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int startNewBookie = startNewBookie();
// Reform ensemble...Making sure that last fragment is not in
// under-replication
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
LedgerManagerFactory mFactory =
LedgerManagerFactory.newLedgerManagerFactory(baseClientConf, zkc);
LedgerUnderreplicationManager underReplicationManager =
mFactory.newLedgerUnderreplicationManager();
rw.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
killAllBookies(lh, newBkAddr);
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
lh = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD);
// Ledger should be still in open state
assertTrue("Ledger must have been closed by RW", ClientUtil.isLedgerOpen(lh));
} finally {
rw.shutdown();
underReplicationManager.close();
}
}
/**
* Tests that replication worker should retry for replication until enough bookies available for
* replication
*/
@Test(timeout = 60000)
public void testRWShouldRetryUntilThereAreEnoughBksAvailableForReplication() throws Exception {
LedgerHandle lh = bkc.createLedger(1, 1, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
ServerConfiguration killedBookieConfig = killBookie(replicaToKill);
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
killAllBookies(lh, newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
int counter = 100;
while (counter-- > 0) {
assertTrue(
"Expecting that replication should not complete",
isLedgerInUnderReplication(lh.getId(), basePath));
Thread.sleep(100);
}
// restart killed bookie
bs.add(startBookie(killedBookieConfig));
bsConfs.add(killedBookieConfig);
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
} finally {
rw.shutdown();
}
}
/**
* Tests that replication worker should replicate the failed bookie fragments to target bookie
* given to the worker.
*/
@Test(timeout = 30000)
public void testRWShouldReplicateFragmentsToTargetBookie() throws Exception {
LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int startNewBookie = startNewBookie();
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
killAllBookies(lh, newBkAddr);
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
} finally {
rw.shutdown();
}
}
/** Test that the replication worker will shutdown if it lose its zookeeper session */
@Test(timeout = 30000)
public void testRWZKSessionLost() throws Exception {
ZooKeeper zk =
ZooKeeperClient.createConnectedZooKeeper(zkUtil.getZooKeeperConnectString(), 10000);
try {
ReplicationWorker rw = new ReplicationWorker(zk, baseConf, getBookie(0));
rw.start();
for (int i = 0; i < 10; i++) {
if (rw.isRunning()) {
break;
}
Thread.sleep(1000);
}
assertTrue("Replication worker should be running", rw.isRunning());
stopZKCluster();
for (int i = 0; i < 10; i++) {
if (!rw.isRunning()) {
break;
}
Thread.sleep(1000);
}
assertFalse("Replication worker should have shut down", rw.isRunning());
} finally {
zk.close();
}
}
/**
* Test that if the local bookie turns out to be readonly, then no point in running RW. So RW
* should shutdown.
*/
@Test(timeout = 20000)
public void testRWShutdownOnLocalBookieReadonlyTransition() throws Exception {
LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int newBkPort = startNewBookie();
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), newBkPort);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
BookieServer newBk = bs.get(bs.size() - 1);
bsConfs.get(bsConfs.size() - 1).setReadOnlyModeEnabled(true);
newBk.getBookie().doTransitionToReadOnlyMode();
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)
&& rw.isRunning()) {
Thread.sleep(100);
}
assertFalse("RW should shutdown if the bookie is readonly", rw.isRunning());
} finally {
rw.shutdown();
}
}
/**
* Tests that replication worker1 should take one fragment replication and other replication
* worker also should compete for the replication.
*/
@Test(timeout = 90000)
public void test2RWsShouldCompeteForReplicationOf2FragmentsAndCompleteReplication()
throws Exception {
LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
ServerConfiguration killedBookieConfig = killBookie(replicaToKill);
killAllBookies(lh, null);
// Starte RW1
int startNewBookie1 = startNewBookie();
BookieSocketAddress newBkAddr1 =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie1);
LOG.info("New Bookie addr :" + newBkAddr1);
ReplicationWorker rw1 = new ReplicationWorker(zkc, baseConf, newBkAddr1);
// Starte RW2
int startNewBookie2 = startNewBookie();
BookieSocketAddress newBkAddr2 =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie2);
LOG.info("New Bookie addr :" + newBkAddr2);
ZooKeeper zkc1 =
ZooKeeperClient.createConnectedZooKeeperClient(zkUtil.getZooKeeperConnectString(), 10000);
ReplicationWorker rw2 = new ReplicationWorker(zkc1, baseConf, newBkAddr2);
rw1.start();
rw2.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
int counter = 10;
while (counter-- > 0) {
assertTrue(
"Expecting that replication should not complete",
isLedgerInUnderReplication(lh.getId(), basePath));
Thread.sleep(100);
}
// restart killed bookie
bs.add(startBookie(killedBookieConfig));
bsConfs.add(killedBookieConfig);
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
} finally {
rw1.shutdown();
rw2.shutdown();
zkc1.close();
}
}
@Test(timeout = 60000)
public void testMultipleLedgerReplicationWithReplicationWorker() throws Exception {
// Ledger1
LedgerHandle lh1 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh1.addEntry(data);
}
BookieSocketAddress replicaToKillFromFirstLedger =
LedgerHandleAdapter.getLedgerMetadata(lh1).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKillFromFirstLedger);
// Ledger2
LedgerHandle lh2 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh2.addEntry(data);
}
BookieSocketAddress replicaToKillFromSecondLedger =
LedgerHandleAdapter.getLedgerMetadata(lh2).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKillFromSecondLedger);
// Kill ledger1
killBookie(replicaToKillFromFirstLedger);
lh1.close();
// Kill ledger2
killBookie(replicaToKillFromFirstLedger);
lh2.close();
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
// Mark ledger1 and 2 as underreplicated
underReplicationManager.markLedgerUnderreplicated(
lh1.getId(), replicaToKillFromFirstLedger.toString());
underReplicationManager.markLedgerUnderreplicated(
lh2.getId(), replicaToKillFromSecondLedger.toString());
while (isLedgerInUnderReplication(lh1.getId(), basePath)) {
Thread.sleep(100);
}
while (isLedgerInUnderReplication(lh2.getId(), basePath)) {
Thread.sleep(100);
}
killAllBookies(lh1, newBkAddr);
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh1, 0, 9);
verifyRecoveredLedgers(lh2, 0, 9);
} finally {
rw.shutdown();
}
}