/**
* Tests that Replication worker should clean the leadger under replication node of the ledger
* already deleted
*/
@Test(timeout = 3000)
public void testRWShouldCleanTheLedgerFromUnderReplicationIfLedgerAlreadyDeleted()
throws Exception {
LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
killBookie(replicaToKill);
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
bkc.deleteLedger(lh.getId()); // Deleting the ledger
// Also mark ledger as in UnderReplication
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
} finally {
rw.shutdown();
}
}
@Test(timeout = 60000)
public void testMajorCompaction() throws Exception {
// prepare data
LedgerHandle[] lhs = prepareData(3, true);
for (LedgerHandle lh : lhs) {
lh.close();
}
// disable minor compaction
baseConf.setMinorCompactionThreshold(0.0f);
// restart bookies
restartBookies(baseConf);
// remove ledger1 and ledger3
bkc.deleteLedger(lhs[0].getId());
bkc.deleteLedger(lhs[2].getId());
LOG.info("Finished deleting the ledgers contains most entries.");
Thread.sleep(baseConf.getMajorCompactionInterval() * 1000 + baseConf.getGcWaitTime());
// entry logs ([0,1,2].log) should be compacted
for (File ledgerDirectory : tmpDirs) {
assertFalse(
"Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
+ ledgerDirectory,
TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
}
// even entry log files are removed, we still can access entries for ledger2
// since those entries has been compacted to new entry log
verifyLedger(lhs[1].getId(), 0, lhs[1].getLastAddConfirmed());
}
@Test(timeout = 60000)
public void testCompactionSmallEntryLogs() throws Exception {
// create a ledger to write a few entries
LedgerHandle alh = bkc.createLedger(NUM_BOOKIES, NUM_BOOKIES, digestType, "".getBytes());
for (int i = 0; i < 3; i++) {
alh.addEntry(msg.getBytes());
}
alh.close();
// restart bookie to roll entry log files
restartBookies();
// prepare data
LedgerHandle[] lhs = prepareData(3, false);
for (LedgerHandle lh : lhs) {
lh.close();
}
// remove ledger2 and ledger3
bkc.deleteLedger(lhs[1].getId());
bkc.deleteLedger(lhs[2].getId());
LOG.info("Finished deleting the ledgers contains most entries.");
// restart bookies again to roll entry log files.
restartBookies();
Thread.sleep(baseConf.getMajorCompactionInterval() * 1000 + baseConf.getGcWaitTime());
// entry logs (0.log) should not be compacted
// entry logs ([1,2,3].log) should be compacted.
for (File ledgerDirectory : tmpDirs) {
assertTrue(
"Not Found entry log file ([0].log that should have been compacted in ledgerDirectory: "
+ ledgerDirectory,
TestUtils.hasLogFiles(ledgerDirectory, true, 0));
assertFalse(
"Found entry log file ([1,2,3].log that should have not been compacted in ledgerDirectory: "
+ ledgerDirectory,
TestUtils.hasLogFiles(ledgerDirectory, true, 1, 2, 3));
}
// even entry log files are removed, we still can access entries for ledger1
// since those entries has been compacted to new entry log
verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed());
}
/**
* Tests that replication worker1 should take one fragment replication and other replication
* worker also should compete for the replication.
*/
@Test(timeout = 90000)
public void test2RWsShouldCompeteForReplicationOf2FragmentsAndCompleteReplication()
throws Exception {
LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
ServerConfiguration killedBookieConfig = killBookie(replicaToKill);
killAllBookies(lh, null);
// Starte RW1
int startNewBookie1 = startNewBookie();
BookieSocketAddress newBkAddr1 =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie1);
LOG.info("New Bookie addr :" + newBkAddr1);
ReplicationWorker rw1 = new ReplicationWorker(zkc, baseConf, newBkAddr1);
// Starte RW2
int startNewBookie2 = startNewBookie();
BookieSocketAddress newBkAddr2 =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie2);
LOG.info("New Bookie addr :" + newBkAddr2);
ZooKeeper zkc1 =
ZooKeeperClient.createConnectedZooKeeperClient(zkUtil.getZooKeeperConnectString(), 10000);
ReplicationWorker rw2 = new ReplicationWorker(zkc1, baseConf, newBkAddr2);
rw1.start();
rw2.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
int counter = 10;
while (counter-- > 0) {
assertTrue(
"Expecting that replication should not complete",
isLedgerInUnderReplication(lh.getId(), basePath));
Thread.sleep(100);
}
// restart killed bookie
bs.add(startBookie(killedBookieConfig));
bsConfs.add(killedBookieConfig);
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
} finally {
rw1.shutdown();
rw2.shutdown();
zkc1.close();
}
}
@Test(timeout = 60000)
public void testBadVersionOnTwoAllocators() throws Exception {
String allocationPath = "/allocation-bad-version";
zkc.get()
.create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
Stat stat = new Stat();
byte[] data = zkc.get().getData(allocationPath, false, stat);
Versioned<byte[]> allocationData =
new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));
SimpleLedgerAllocator allocator1 =
new SimpleLedgerAllocator(
allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
SimpleLedgerAllocator allocator2 =
new SimpleLedgerAllocator(
allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
allocator1.allocate();
// wait until allocated
ZKTransaction txn1 = newTxn();
LedgerHandle lh = FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER));
allocator2.allocate();
ZKTransaction txn2 = newTxn();
try {
FutureUtils.result(allocator2.tryObtain(txn2, NULL_LISTENER));
fail(
"Should fail allocating on second allocator as allocator1 is starting allocating something.");
} catch (ZKException zke) {
assertEquals(KeeperException.Code.BADVERSION, zke.getKeeperExceptionCode());
}
FutureUtils.result(txn1.execute());
Utils.close(allocator1);
Utils.close(allocator2);
long eid = lh.addEntry("hello world".getBytes());
lh.close();
LedgerHandle readLh =
bkc.get()
.openLedger(lh.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid);
int i = 0;
while (entries.hasMoreElements()) {
LedgerEntry entry = entries.nextElement();
assertEquals("hello world", new String(entry.getEntry(), UTF_8));
++i;
}
assertEquals(1, i);
}
/*
* (non-Javadoc)
*
* @see org.apache.bookkeeper.mledger.ManagedLedger#close()
*/
@Override
public synchronized void close() throws InterruptedException, ManagedLedgerException {
checkFenced();
for (LedgerHandle ledger : ledgerCache.asMap().values()) {
log.debug("Closing ledger: {}", ledger.getId());
try {
ledger.close();
} catch (BKException e) {
throw new ManagedLedgerException(e);
}
}
ledgerCache.invalidateAll();
log.info("Invalidated {} ledgers in cache", ledgerCache.size());
factory.close(this);
}
/**
* Tests that replication worker should retry for replication until enough bookies available for
* replication
*/
@Test(timeout = 60000)
public void testRWShouldRetryUntilThereAreEnoughBksAvailableForReplication() throws Exception {
LedgerHandle lh = bkc.createLedger(1, 1, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh.addEntry(data);
}
lh.close();
BookieSocketAddress replicaToKill =
LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKill);
ServerConfiguration killedBookieConfig = killBookie(replicaToKill);
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
killAllBookies(lh, newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
int counter = 100;
while (counter-- > 0) {
assertTrue(
"Expecting that replication should not complete",
isLedgerInUnderReplication(lh.getId(), basePath));
Thread.sleep(100);
}
// restart killed bookie
bs.add(startBookie(killedBookieConfig));
bsConfs.add(killedBookieConfig);
while (isLedgerInUnderReplication(lh.getId(), basePath)) {
Thread.sleep(100);
}
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh, 0, 9);
} finally {
rw.shutdown();
}
}
@Test(timeout = 60000)
public void testMajorCompactionAboveThreshold() throws Exception {
// prepare data
LedgerHandle[] lhs = prepareData(3, false);
for (LedgerHandle lh : lhs) {
lh.close();
}
// remove ledger1 and ledger2
bkc.deleteLedger(lhs[0].getId());
bkc.deleteLedger(lhs[1].getId());
LOG.info("Finished deleting the ledgers contains less entries.");
Thread.sleep(baseConf.getMajorCompactionInterval() * 1000 + baseConf.getGcWaitTime());
// entry logs ([0,1,2].log) should not be compacted
for (File ledgerDirectory : tmpDirs) {
assertTrue(
"Not Found entry log file ([1,2].log that should have been compacted in ledgerDirectory: "
+ ledgerDirectory,
TestUtils.hasLogFiles(ledgerDirectory, false, 0, 1, 2));
}
}
@Test(timeout = 60000)
public void testMultipleLedgerReplicationWithReplicationWorker() throws Exception {
// Ledger1
LedgerHandle lh1 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh1.addEntry(data);
}
BookieSocketAddress replicaToKillFromFirstLedger =
LedgerHandleAdapter.getLedgerMetadata(lh1).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKillFromFirstLedger);
// Ledger2
LedgerHandle lh2 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);
for (int i = 0; i < 10; i++) {
lh2.addEntry(data);
}
BookieSocketAddress replicaToKillFromSecondLedger =
LedgerHandleAdapter.getLedgerMetadata(lh2).getEnsembles().get(0L).get(0);
LOG.info("Killing Bookie", replicaToKillFromSecondLedger);
// Kill ledger1
killBookie(replicaToKillFromFirstLedger);
lh1.close();
// Kill ledger2
killBookie(replicaToKillFromFirstLedger);
lh2.close();
int startNewBookie = startNewBookie();
BookieSocketAddress newBkAddr =
new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
LOG.info("New Bookie addr :" + newBkAddr);
ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
rw.start();
try {
// Mark ledger1 and 2 as underreplicated
underReplicationManager.markLedgerUnderreplicated(
lh1.getId(), replicaToKillFromFirstLedger.toString());
underReplicationManager.markLedgerUnderreplicated(
lh2.getId(), replicaToKillFromSecondLedger.toString());
while (isLedgerInUnderReplication(lh1.getId(), basePath)) {
Thread.sleep(100);
}
while (isLedgerInUnderReplication(lh2.getId(), basePath)) {
Thread.sleep(100);
}
killAllBookies(lh1, newBkAddr);
// Should be able to read the entries from 0-9
verifyRecoveredLedgers(lh1, 0, 9);
verifyRecoveredLedgers(lh2, 0, 9);
} finally {
rw.shutdown();
}
}
@Test(timeout = 60000)
public void testSuccessAllocatorShouldDeleteUnusedledger() throws Exception {
String allocationPath = "/allocation-delete-unused-ledger";
zkc.get()
.create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
Stat stat = new Stat();
byte[] data = zkc.get().getData(allocationPath, false, stat);
Versioned<byte[]> allocationData =
new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));
SimpleLedgerAllocator allocator1 =
new SimpleLedgerAllocator(
allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
allocator1.allocate();
// wait until allocated
ZKTransaction txn1 = newTxn();
LedgerHandle lh1 = FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER));
// Second allocator kicks in
stat = new Stat();
data = zkc.get().getData(allocationPath, false, stat);
allocationData = new Versioned<byte[]>(data, new ZkVersion(stat.getVersion()));
SimpleLedgerAllocator allocator2 =
new SimpleLedgerAllocator(
allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc);
allocator2.allocate();
// wait until allocated
ZKTransaction txn2 = newTxn();
LedgerHandle lh2 = FutureUtils.result(allocator2.tryObtain(txn2, NULL_LISTENER));
// should fail to commit txn1 as version is changed by second allocator
try {
FutureUtils.result(txn1.execute());
fail(
"Should fail commit obtaining ledger handle from first allocator as allocator is modified by second allocator.");
} catch (ZKException ke) {
// as expected
}
FutureUtils.result(txn2.execute());
Utils.close(allocator1);
Utils.close(allocator2);
// ledger handle should be deleted
try {
lh1.close();
fail("LedgerHandle allocated by allocator1 should be deleted.");
} catch (BKException bke) {
// as expected
}
try {
bkc.get()
.openLedger(lh1.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
fail("LedgerHandle allocated by allocator1 should be deleted.");
} catch (BKException.BKNoSuchLedgerExistsException nslee) {
// as expected
}
long eid = lh2.addEntry("hello world".getBytes());
lh2.close();
LedgerHandle readLh =
bkc.get()
.openLedger(
lh2.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes());
Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid);
int i = 0;
while (entries.hasMoreElements()) {
LedgerEntry entry = entries.nextElement();
assertEquals("hello world", new String(entry.getEntry(), UTF_8));
++i;
}
assertEquals(1, i);
}
void auxTestReadWriteAsyncSingleClient(BookieServer bs) throws IOException {
SyncObj sync = new SyncObj();
try {
// Create a ledger
lh = bkc.createLedger(3, 2, digestType, ledgerPassword);
ledgerId = lh.getId();
LOG.info("Ledger ID: " + lh.getId());
for (int i = 0; i < numEntriesToWrite; i++) {
ByteBuffer entry = ByteBuffer.allocate(4);
entry.putInt(rng.nextInt(maxInt));
entry.position(0);
entries.add(entry.array());
entriesSize.add(entry.array().length);
lh.asyncAddEntry(entry.array(), this, sync);
}
LOG.info("Wrote " + numEntriesToWrite + " and now going to fail bookie.");
// Bookie fail
bs.shutdown();
// wait for all entries to be acknowledged
synchronized (sync) {
while (sync.counter < numEntriesToWrite) {
LOG.debug("Entries counter = " + sync.counter);
sync.wait(10000);
assertFalse("Failure occurred during write", sync.failureOccurred);
}
}
LOG.debug("*** WRITE COMPLETE ***");
// close ledger
lh.close();
// *** WRITING PART COMPLETE // READ PART BEGINS ***
// open ledger
bkc.close();
bkc = new BookKeeperTestClient(baseClientConf);
lh = bkc.openLedger(ledgerId, digestType, ledgerPassword);
LOG.debug("Number of entries written: " + (lh.getLastAddConfirmed() + 1));
assertTrue(
"Verifying number of entries written",
lh.getLastAddConfirmed() == (numEntriesToWrite - 1));
// read entries
lh.asyncReadEntries(0, numEntriesToWrite - 1, this, sync);
synchronized (sync) {
int i = 0;
sync.wait(10000);
assertFalse("Failure occurred during read", sync.failureOccurred);
assertTrue("Haven't received entries", sync.value);
}
LOG.debug("*** READ COMPLETE ***");
// at this point, Enumeration<LedgerEntry> ls is filled with the returned
// values
int i = 0;
while (sync.ls.hasMoreElements()) {
ByteBuffer origbb = ByteBuffer.wrap(entries.get(i));
Integer origEntry = origbb.getInt();
byte[] entry = sync.ls.nextElement().getEntry();
ByteBuffer result = ByteBuffer.wrap(entry);
Integer retrEntry = result.getInt();
LOG.debug("Retrieved entry: " + i);
assertTrue("Checking entry " + i + " for equality", origEntry.equals(retrEntry));
assertTrue(
"Checking entry " + i + " for size", entry.length == entriesSize.get(i).intValue());
i++;
}
assertTrue("Checking number of read entries", i == numEntriesToWrite);
LOG.info("Verified that entries are ok, and now closing ledger");
lh.close();
} catch (KeeperException e) {
LOG.error("Caught KeeperException", e);
fail(e.toString());
} catch (BKException e) {
LOG.error("Caught BKException", e);
fail(e.toString());
} catch (InterruptedException e) {
LOG.error("Caught InterruptedException", e);
fail(e.toString());
}
}
