/**
* Tests that ReplicationWorker should not have identified for postponing the replication if
* ledger is in open state and lastFragment is not in underReplication state. Note that RW should
* not fence such ledgers.
*/
@Test(timeout = 30000)
public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR() throws Exception {
LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int startNewBookie = startNewBookie();
// Reform ensemble...Making sure that last fragment is not in
// under-replication
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
LedgerManagerFactory mFactory =
LedgerManagerFactory.newLedgerManagerFactory(baseClientConf, zkc);
LedgerUnderreplicationManager underReplicationManager =
mFactory.newLedgerUnderreplicationManager();
rw.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
killAllBookies(lh, newBkAddr);
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
lh = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD);
// Ledger should be still in open state
assertTrue("Ledger must have been closed by RW", ClientUtil.isLedgerOpen(lh));
} finally {
rw.shutdown();
underReplicationManager.close();
}
}
/**
* Verify read last confirmed op, it shouldn't cause any deadlock as new bookie connection is
* being established and returning the connection notification in the same caller thread. It would
* be simulated by delaying the future.addlistener() in PerChannelBookieClient after the
* connection establishment. Now the future.addlistener() will notify back in the same thread and
* simultaneously invoke the pendingOp.operationComplete() event.
*
* <p>BOOKKEEPER-326
*/
@Test(timeout = 60000)
public void testReadLastConfirmedOp() throws Exception {
startNewBookie();
startNewBookie();
// Create a ledger
LedgerHandle beforelh =
bkc.createLedger(numBookies + 2, numBookies + 2, digestType, "".getBytes());
int numEntries = 10;
String tmp = "BookKeeper is cool!";
for (int i = 0; i < numEntries; i++) {
beforelh.addEntry(tmp.getBytes());
}
// shutdown first bookie server
killBookie(0);
startNewBookie();
// create new bookie client, and forcing to establish new
// PerChannelBookieClient connections for recovery flows.
BookKeeperTestClient bkc1 = new BookKeeperTestClient(baseClientConf);
// try to open ledger with recovery
LedgerHandle afterlh = bkc1.openLedger(beforelh.getId(), digestType, "".getBytes());
assertEquals("Entries got missed", beforelh.getLastAddPushed(), afterlh.getLastAddConfirmed());
bkc1.close();
}
/**
* Tests that Replication worker should clean the leadger under replication node of the ledger
* already deleted
*/
@Test(timeout = 3000)
public void testRWShouldCleanTheLedgerFromUnderReplicationIfLedgerAlreadyDeleted()
throws Exception {
LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
bkc.deleteLedger(lh.getId()); // Deleting the ledger
// Also mark ledger as in UnderReplication
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
} finally {
rw.shutdown();
}
}
@Test(timeout = 60000)
public void testBookieRecovery() throws Exception {
// Shutdown all but 1 bookie
bs.get(0).shutdown();
bs.get(1).shutdown();
bs.get(2).shutdown();
byte[] passwd = "blah".getBytes();
LedgerHandle lh = bkc.createLedger(1, 1, digestType, passwd);
int numEntries = 100;
for (int i = 0; i < numEntries; i++) {
byte[] data = ("" + i).getBytes();
lh.addEntry(data);
}
bs.get(3).shutdown();
BookieServer server = new BookieServer(bsConfs.get(3));
server.start();
bs.set(3, server);
assertEquals(numEntries - 1, lh.getLastAddConfirmed());
Enumeration<LedgerEntry> entries = lh.readEntries(0, lh.getLastAddConfirmed());
int numScanned = 0;
while (entries.hasMoreElements()) {
assertEquals(("" + numScanned), new String(entries.nextElement().getEntry()));
numScanned++;
}
assertEquals(numEntries, numScanned);
}
/**
* Tests that replication worker1 should take one fragment replication and other replication
* worker also should compete for the replication.
*/
@Test(timeout = 90000)
public void test2RWsShouldCompeteForReplicationOf2FragmentsAndCompleteReplication()
throws Exception {
LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
ServerConfiguration killedBookieConfig = killBookie(replicaToKill);
killAllBookies(lh, null);
// Starte RW1
int startNewBookie1 = startNewBookie();
BookieSocketAddress newBkAddr1 =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie1);
LOG.info("New Bookie addr :" + newBkAddr1);
ReplicationWorker rw1 = new ReplicationWorker(zkc, baseConf, newBkAddr1);
// Starte RW2
int startNewBookie2 = startNewBookie();
BookieSocketAddress newBkAddr2 =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie2);
LOG.info("New Bookie addr :" + newBkAddr2);
ZooKeeper zkc1 =
ZooKeeperClient.createConnectedZooKeeperClient(zkUtil.getZooKeeperConnectString(), 10000);
ReplicationWorker rw2 = new ReplicationWorker(zkc1, baseConf, newBkAddr2);
rw1.start();
rw2.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
int counter = 10;
while (counter-- > 0) {
assertTrue(
"Expecting that replication should not complete",
isLedgerInUnderReplication(lh.getId(), basePath));
Thread.sleep(100);
}
// restart killed bookie
bs.add(startBookie(killedBookieConfig));
bsConfs.add(killedBookieConfig);
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
} finally {
rw1.shutdown();
rw2.shutdown();
zkc1.close();
}
}
/**
* Tests that replication worker should replicate the failed bookie fragments to target bookie
* given to the worker.
*/
@Test(timeout = 30000)
public void testRWShouldReplicateFragmentsToTargetBookie() throws Exception {
LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int startNewBookie = startNewBookie();
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
killAllBookies(lh, newBkAddr);
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
} finally {
rw.shutdown();
}
}
/**
* Test that if the local bookie turns out to be readonly, then no point in running RW. So RW
* should shutdown.
*/
@Test(timeout = 20000)
public void testRWShutdownOnLocalBookieReadonlyTransition() throws Exception {
LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int newBkPort = startNewBookie();
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), newBkPort);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
BookieServer newBk = bs.get(bs.size() - 1);
bsConfs.get(bsConfs.size() - 1).setReadOnlyModeEnabled(true);
newBk.getBookie().doTransitionToReadOnlyMode();
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)
&& rw.isRunning()) {
Thread.sleep(100);
}
assertFalse("RW should shutdown if the bookie is readonly", rw.isRunning());
} finally {
rw.shutdown();
}
}
@Test(timeout = 60000)
public void testLedgerOpenAfterBKCrashed() throws Exception {
// Create a ledger
LedgerHandle beforelh = bkc.createLedger(numBookies, numBookies, digestType, "".getBytes());
int numEntries = 10;
String tmp = "BookKeeper is cool!";
for (int i = 0; i < numEntries; i++) {
beforelh.addEntry(tmp.getBytes());
}
// shutdown first bookie server
killBookie(0);
startNewBookie();
// try to open ledger no recovery
LedgerHandle afterlh = bkc.openLedger(beforelh.getId(), digestType, "".getBytes());
assertEquals(beforelh.getLastAddPushed(), afterlh.getLastAddConfirmed());
LedgerHandle beforelh2 = bkc.createLedger(numBookies, 1, digestType, "".getBytes());
for (int i = 0; i < numEntries; i++) {
beforelh2.addEntry(tmp.getBytes());
}
// shutdown first bookie server
killBookie(0);
// try to open ledger no recovery
try {
bkc.openLedger(beforelh2.getId(), digestType, "".getBytes());
fail("Should have thrown exception");
} catch (BKException.BKLedgerRecoveryException e) {
// correct behaviour
}
}
@Test(timeout = 60000)
public void testBadVersionOnTwoAllocators() throws Exception {
String allocationPath = "/allocation-bad-version";
zkc.get()
.create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
Stat stat = new Stat();
byte[] data = zkc.get().getData(allocationPath, false, stat);
Versioned<byte[]> allocationData =
new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));
SimpleLedgerAllocator allocator1 =
new SimpleLedgerAllocator(
allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
SimpleLedgerAllocator allocator2 =
new SimpleLedgerAllocator(
allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
allocator1.allocate();
// wait until allocated
ZKTransaction txn1 = newTxn();
LedgerHandle lh = FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER));
allocator2.allocate();
ZKTransaction txn2 = newTxn();
try {
FutureUtils.result(allocator2.tryObtain(txn2, NULL_LISTENER));
fail(
"Should fail allocating on second allocator as allocator1 is starting allocating something.");
} catch (ZKException zke) {
assertEquals(KeeperException.Code.BADVERSION, zke.getKeeperExceptionCode());
}
FutureUtils.result(txn1.execute());
Utils.close(allocator1);
Utils.close(allocator2);
long eid = lh.addEntry("hello world".getBytes());
lh.close();
LedgerHandle readLh =
bkc.get()
.openLedger(lh.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid);
int i = 0;
while (entries.hasMoreElements()) {
LedgerEntry entry = entries.nextElement();
assertEquals("hello world", new String(entry.getEntry(), UTF_8));
++i;
}
assertEquals(1, i);
}
@Test(timeout = 60000)
public void testCompactionSmallEntryLogs() throws Exception {
// create a ledger to write a few entries
LedgerHandle alh = bkc.createLedger(NUM_BOOKIES, NUM_BOOKIES, digestType, "".getBytes());
for (int i = 0; i < 3; i++) {
alh.addEntry(msg.getBytes());
}
alh.close();
// restart bookie to roll entry log files
restartBookies();
// prepare data
LedgerHandle[] lhs = prepareData(3, false);
for (LedgerHandle lh : lhs) {
lh.close();
}
// remove ledger2 and ledger3
bkc.deleteLedger(lhs[1].getId());
bkc.deleteLedger(lhs[2].getId());
LOG.info("Finished deleting the ledgers contains most entries.");
// restart bookies again to roll entry log files.
restartBookies();
Thread.sleep(baseConf.getMajorCompactionInterval() * 1000 + baseConf.getGcWaitTime());
// entry logs (0.log) should not be compacted
// entry logs ([1,2,3].log) should be compacted.
for (File ledgerDirectory : tmpDirs) {
assertTrue(
"Not Found entry log file ([0].log that should have been compacted in ledgerDirectory: "
+ ledgerDirectory,
TestUtils.hasLogFiles(ledgerDirectory, true, 0));
assertFalse(
"Found entry log file ([1,2,3].log that should have not been compacted in ledgerDirectory: "
+ ledgerDirectory,
TestUtils.hasLogFiles(ledgerDirectory, true, 1, 2, 3));
}
// even entry log files are removed, we still can access entries for ledger1
// since those entries has been compacted to new entry log
verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());
}
/**
* Tests that replication worker should retry for replication until enough bookies available for
* replication
*/
@Test(timeout = 60000)
public void testRWShouldRetryUntilThereAreEnoughBksAvailableForReplication() throws Exception {
LedgerHandle lh = bkc.createLedger(1, 1, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
ServerConfiguration killedBookieConfig = killBookie(replicaToKill);
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
killAllBookies(lh, newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
int counter = 100;
while (counter-- > 0) {
assertTrue(
"Expecting that replication should not complete",
isLedgerInUnderReplication(lh.getId(), basePath));
Thread.sleep(100);
}
// restart killed bookie
bs.add(startBookie(killedBookieConfig));
bsConfs.add(killedBookieConfig);
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
} finally {
rw.shutdown();
}
}
@Test(timeout = 60000)
public void testMultipleLedgerReplicationWithReplicationWorker() throws Exception {
// Ledger1
LedgerHandle lh1 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh1.addEntry(data);
}
BookieSocketAddress replicaToKillFromFirstLedger =
LedgerHandleAdapter.getLedgerMetadata(lh1).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKillFromFirstLedger);
// Ledger2
LedgerHandle lh2 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh2.addEntry(data);
}
BookieSocketAddress replicaToKillFromSecondLedger =
LedgerHandleAdapter.getLedgerMetadata(lh2).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKillFromSecondLedger);
// Kill ledger1
killBookie(replicaToKillFromFirstLedger);
lh1.close();
// Kill ledger2
killBookie(replicaToKillFromFirstLedger);
lh2.close();
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
// Mark ledger1 and 2 as underreplicated
underReplicationManager.markLedgerUnderreplicated(
lh1.getId(), replicaToKillFromFirstLedger.toString());
underReplicationManager.markLedgerUnderreplicated(
lh2.getId(), replicaToKillFromSecondLedger.toString());
while (isLedgerInUnderReplication(lh1.getId(), basePath)) {
Thread.sleep(100);
}
while (isLedgerInUnderReplication(lh2.getId(), basePath)) {
Thread.sleep(100);
}
killAllBookies(lh1, newBkAddr);
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh1, 0, 9);
verifyRecoveredLedgers(lh2, 0, 9);
} finally {
rw.shutdown();
}
}
@Test(timeout = 60000)
public void testSuccessAllocatorShouldDeleteUnusedledger() throws Exception {
String allocationPath = "/allocation-delete-unused-ledger";
zkc.get()
.create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
Stat stat = new Stat();
byte[] data = zkc.get().getData(allocationPath, false, stat);
Versioned<byte[]> allocationData =
new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));
SimpleLedgerAllocator allocator1 =
new SimpleLedgerAllocator(
allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
allocator1.allocate();
// wait until allocated
ZKTransaction txn1 = newTxn();
LedgerHandle lh1 = FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER));
// Second allocator kicks in
stat = new Stat();
data = zkc.get().getData(allocationPath, false, stat);
allocationData = new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));
SimpleLedgerAllocator allocator2 =
new SimpleLedgerAllocator(
allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
allocator2.allocate();
// wait until allocated
ZKTransaction txn2 = newTxn();
LedgerHandle lh2 = FutureUtils.result(allocator2.tryObtain(txn2, NULL_LISTENER));
// should fail to commit txn1 as version is changed by second allocator
try {
FutureUtils.result(txn1.execute());
fail(
"Should fail commit obtaining ledger handle from first allocator as allocator is modified by second allocator.");
} catch (ZKException ke) {
// as expected
}
FutureUtils.result(txn2.execute());
Utils.close(allocator1);
Utils.close(allocator2);
// ledger handle should be deleted
try {
lh1.close();
fail("LedgerHandle allocated by allocator1 should be deleted.");
} catch (BKException bke) {
// as expected
}
try {
bkc.get()
.openLedger(lh1.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
fail("LedgerHandle allocated by allocator1 should be deleted.");
} catch (BKException.BKNoSuchLedgerExistsException nslee) {
// as expected
}
long eid = lh2.addEntry("hello world".getBytes());
lh2.close();
LedgerHandle readLh =
bkc.get()
.openLedger(
lh2.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid);
int i = 0;
while (entries.hasMoreElements()) {
LedgerEntry entry = entries.nextElement();
assertEquals("hello world", new String(entry.getEntry(), UTF_8));
++i;
}
assertEquals(1, i);
}
