/** * Tests that ReplicationWorker should not have identified for postponing the replication if * ledger is in open state and lastFragment is not in underReplication state. Note that RW should * not fence such ledgers. */ @Test(timeout = 30000) public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR() throws Exception { LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); for (int i = 0; i < 10; i++) { lh.addEntry(data); } BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0); LOG.info("Killing Bookie", replicaToKill); killBookie(replicaToKill); int startNewBookie = startNewBookie(); // Reform ensemble...Making sure that last fragment is not in // under-replication for (int i = 0; i < 10; i++) { lh.addEntry(data); } BookieSocketAddress newBkAddr = new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie); LOG.info("New Bookie addr :" + newBkAddr); ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr); LedgerManagerFactory mFactory = LedgerManagerFactory.newLedgerManagerFactory(baseClientConf, zkc); LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager(); rw.start(); try { underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString()); while (isLedgerInUnderReplication(lh.getId(), basePath)) { Thread.sleep(100); } killAllBookies(lh, newBkAddr); // Should be able to read the entries from 0-9 verifyRecoveredLedgers(lh, 0, 9); lh = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD); // Ledger should be still in open state assertTrue("Ledger must have been closed by RW", ClientUtil.isLedgerOpen(lh)); } finally { rw.shutdown(); underReplicationManager.close(); } }
/** * Verify read last confirmed op, it shouldn't cause any deadlock as new bookie connection is * being established and returning the connection notification in the same caller thread. It would * be simulated by delaying the future.addlistener() in PerChannelBookieClient after the * connection establishment. Now the future.addlistener() will notify back in the same thread and * simultaneously invoke the pendingOp.operationComplete() event. * * <p>BOOKKEEPER-326 */ @Test(timeout = 60000) public void testReadLastConfirmedOp() throws Exception { startNewBookie(); startNewBookie(); // Create a ledger LedgerHandle beforelh = bkc.createLedger(numBookies + 2, numBookies + 2, digestType, "".getBytes()); int numEntries = 10; String tmp = "BookKeeper is cool!"; for (int i = 0; i < numEntries; i++) { beforelh.addEntry(tmp.getBytes()); } // shutdown first bookie server killBookie(0); startNewBookie(); // create new bookie client, and forcing to establish new // PerChannelBookieClient connections for recovery flows. BookKeeperTestClient bkc1 = new BookKeeperTestClient(baseClientConf); // try to open ledger with recovery LedgerHandle afterlh = bkc1.openLedger(beforelh.getId(), digestType, "".getBytes()); assertEquals("Entries got missed", beforelh.getLastAddPushed(), afterlh.getLastAddConfirmed()); bkc1.close(); }
/** * Tests that Replication worker should clean the leadger under replication node of the ledger * already deleted */ @Test(timeout = 3000) public void testRWShouldCleanTheLedgerFromUnderReplicationIfLedgerAlreadyDeleted() throws Exception { LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD); for (int i = 0; i < 10; i++) { lh.addEntry(data); } lh.close(); BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0); LOG.info("Killing Bookie", replicaToKill); killBookie(replicaToKill); int startNewBookie = startNewBookie(); BookieSocketAddress newBkAddr = new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie); LOG.info("New Bookie addr :" + newBkAddr); ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr); rw.start(); try { bkc.deleteLedger(lh.getId()); // Deleting the ledger // Also mark ledger as in UnderReplication underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString()); while (isLedgerInUnderReplication(lh.getId(), basePath)) { Thread.sleep(100); } } finally { rw.shutdown(); } }
@Test(timeout = 60000) public void testBookieRecovery() throws Exception { // Shutdown all but 1 bookie bs.get(0).shutdown(); bs.get(1).shutdown(); bs.get(2).shutdown(); byte[] passwd = "blah".getBytes(); LedgerHandle lh = bkc.createLedger(1, 1, digestType, passwd); int numEntries = 100; for (int i = 0; i < numEntries; i++) { byte[] data = ("" + i).getBytes(); lh.addEntry(data); } bs.get(3).shutdown(); BookieServer server = new BookieServer(bsConfs.get(3)); server.start(); bs.set(3, server); assertEquals(numEntries - 1, lh.getLastAddConfirmed()); Enumeration<LedgerEntry> entries = lh.readEntries(0, lh.getLastAddConfirmed()); int numScanned = 0; while (entries.hasMoreElements()) { assertEquals(("" + numScanned), new String(entries.nextElement().getEntry())); numScanned++; } assertEquals(numEntries, numScanned); }
/** * Tests that replication worker1 should take one fragment replication and other replication * worker also should compete for the replication. */ @Test(timeout = 90000) public void test2RWsShouldCompeteForReplicationOf2FragmentsAndCompleteReplication() throws Exception { LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD); for (int i = 0; i < 10; i++) { lh.addEntry(data); } lh.close(); BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0); LOG.info("Killing Bookie", replicaToKill); ServerConfiguration killedBookieConfig = killBookie(replicaToKill); killAllBookies(lh, null); // Starte RW1 int startNewBookie1 = startNewBookie(); BookieSocketAddress newBkAddr1 = new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie1); LOG.info("New Bookie addr :" + newBkAddr1); ReplicationWorker rw1 = new ReplicationWorker(zkc, baseConf, newBkAddr1); // Starte RW2 int startNewBookie2 = startNewBookie(); BookieSocketAddress newBkAddr2 = new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie2); LOG.info("New Bookie addr :" + newBkAddr2); ZooKeeper zkc1 = ZooKeeperClient.createConnectedZooKeeperClient(zkUtil.getZooKeeperConnectString(), 10000); ReplicationWorker rw2 = new ReplicationWorker(zkc1, baseConf, newBkAddr2); rw1.start(); rw2.start(); try { underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString()); int counter = 10; while (counter-- > 0) { assertTrue( "Expecting that replication should not complete", isLedgerInUnderReplication(lh.getId(), basePath)); Thread.sleep(100); } // restart killed bookie bs.add(startBookie(killedBookieConfig)); bsConfs.add(killedBookieConfig); while (isLedgerInUnderReplication(lh.getId(), basePath)) { Thread.sleep(100); } // Should be able to read the entries from 0-9 verifyRecoveredLedgers(lh, 0, 9); } finally { rw1.shutdown(); rw2.shutdown(); zkc1.close(); } }
/** * Tests that replication worker should replicate the failed bookie fragments to target bookie * given to the worker. */ @Test(timeout = 30000) public void testRWShouldReplicateFragmentsToTargetBookie() throws Exception { LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); for (int i = 0; i < 10; i++) { lh.addEntry(data); } BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0); LOG.info("Killing Bookie", replicaToKill); killBookie(replicaToKill); int startNewBookie = startNewBookie(); for (int i = 0; i < 10; i++) { lh.addEntry(data); } BookieSocketAddress newBkAddr = new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie); LOG.info("New Bookie addr :" + newBkAddr); ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr); rw.start(); try { underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString()); while (isLedgerInUnderReplication(lh.getId(), basePath)) { Thread.sleep(100); } killAllBookies(lh, newBkAddr); // Should be able to read the entries from 0-9 verifyRecoveredLedgers(lh, 0, 9); } finally { rw.shutdown(); } }
/** * Test that if the local bookie turns out to be readonly, then no point in running RW. So RW * should shutdown. */ @Test(timeout = 20000) public void testRWShutdownOnLocalBookieReadonlyTransition() throws Exception { LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); for (int i = 0; i < 10; i++) { lh.addEntry(data); } BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0); LOG.info("Killing Bookie", replicaToKill); killBookie(replicaToKill); int newBkPort = startNewBookie(); for (int i = 0; i < 10; i++) { lh.addEntry(data); } BookieSocketAddress newBkAddr = new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), newBkPort); LOG.info("New Bookie addr :" + newBkAddr); ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr); rw.start(); try { BookieServer newBk = bs.get(bs.size() - 1); bsConfs.get(bsConfs.size() - 1).setReadOnlyModeEnabled(true); newBk.getBookie().doTransitionToReadOnlyMode(); underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString()); while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath) && rw.isRunning()) { Thread.sleep(100); } assertFalse("RW should shutdown if the bookie is readonly", rw.isRunning()); } finally { rw.shutdown(); } }
@Test(timeout = 60000) public void testLedgerOpenAfterBKCrashed() throws Exception { // Create a ledger LedgerHandle beforelh = bkc.createLedger(numBookies, numBookies, digestType, "".getBytes()); int numEntries = 10; String tmp = "BookKeeper is cool!"; for (int i = 0; i < numEntries; i++) { beforelh.addEntry(tmp.getBytes()); } // shutdown first bookie server killBookie(0); startNewBookie(); // try to open ledger no recovery LedgerHandle afterlh = bkc.openLedger(beforelh.getId(), digestType, "".getBytes()); assertEquals(beforelh.getLastAddPushed(), afterlh.getLastAddConfirmed()); LedgerHandle beforelh2 = bkc.createLedger(numBookies, 1, digestType, "".getBytes()); for (int i = 0; i < numEntries; i++) { beforelh2.addEntry(tmp.getBytes()); } // shutdown first bookie server killBookie(0); // try to open ledger no recovery try { bkc.openLedger(beforelh2.getId(), digestType, "".getBytes()); fail("Should have thrown exception"); } catch (BKException.BKLedgerRecoveryException e) { // correct behaviour } }
@Test(timeout = 60000) public void testBadVersionOnTwoAllocators() throws Exception { String allocationPath = "/allocation-bad-version"; zkc.get() .create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); Stat stat = new Stat(); byte[] data = zkc.get().getData(allocationPath, false, stat); Versioned<byte[]> allocationData = new Versioned<byte[]>(data, new ZkVersion(stat.getVersion())); SimpleLedgerAllocator allocator1 = new SimpleLedgerAllocator( allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc); SimpleLedgerAllocator allocator2 = new SimpleLedgerAllocator( allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc); allocator1.allocate(); // wait until allocated ZKTransaction txn1 = newTxn(); LedgerHandle lh = FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER)); allocator2.allocate(); ZKTransaction txn2 = newTxn(); try { FutureUtils.result(allocator2.tryObtain(txn2, NULL_LISTENER)); fail( "Should fail allocating on second allocator as allocator1 is starting allocating something."); } catch (ZKException zke) { assertEquals(KeeperException.Code.BADVERSION, zke.getKeeperExceptionCode()); } FutureUtils.result(txn1.execute()); Utils.close(allocator1); Utils.close(allocator2); long eid = lh.addEntry("hello world".getBytes()); lh.close(); LedgerHandle readLh = bkc.get() .openLedger(lh.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes()); Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid); int i = 0; while (entries.hasMoreElements()) { LedgerEntry entry = entries.nextElement(); assertEquals("hello world", new String(entry.getEntry(), UTF_8)); ++i; } assertEquals(1, i); }
@Test(timeout = 60000) public void testCompactionSmallEntryLogs() throws Exception { // create a ledger to write a few entries LedgerHandle alh = bkc.createLedger(NUM_BOOKIES, NUM_BOOKIES, digestType, "".getBytes()); for (int i = 0; i < 3; i++) { alh.addEntry(msg.getBytes()); } alh.close(); // restart bookie to roll entry log files restartBookies(); // prepare data LedgerHandle[] lhs = prepareData(3, false); for (LedgerHandle lh : lhs) { lh.close(); } // remove ledger2 and ledger3 bkc.deleteLedger(lhs[1].getId()); bkc.deleteLedger(lhs[2].getId()); LOG.info("Finished deleting the ledgers contains most entries."); // restart bookies again to roll entry log files. restartBookies(); Thread.sleep(baseConf.getMajorCompactionInterval() * 1000 + baseConf.getGcWaitTime()); // entry logs (0.log) should not be compacted // entry logs ([1,2,3].log) should be compacted. for (File ledgerDirectory : tmpDirs) { assertTrue( "Not Found entry log file ([0].log that should have been compacted in ledgerDirectory: " + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0)); assertFalse( "Found entry log file ([1,2,3].log that should have not been compacted in ledgerDirectory: " + ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 1, 2, 3)); } // even entry log files are removed, we still can access entries for ledger1 // since those entries has been compacted to new entry log verifyLedger(lhs[0].getId(), 0, lhs[0].getLastAddConfirmed()); }
/** * Tests that replication worker should retry for replication until enough bookies available for * replication */ @Test(timeout = 60000) public void testRWShouldRetryUntilThereAreEnoughBksAvailableForReplication() throws Exception { LedgerHandle lh = bkc.createLedger(1, 1, BookKeeper.DigestType.CRC32, TESTPASSWD); for (int i = 0; i < 10; i++) { lh.addEntry(data); } lh.close(); BookieSocketAddress replicaToKill = LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0); LOG.info("Killing Bookie", replicaToKill); ServerConfiguration killedBookieConfig = killBookie(replicaToKill); int startNewBookie = startNewBookie(); BookieSocketAddress newBkAddr = new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie); LOG.info("New Bookie addr :" + newBkAddr); killAllBookies(lh, newBkAddr); ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr); rw.start(); try { underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString()); int counter = 100; while (counter-- > 0) { assertTrue( "Expecting that replication should not complete", isLedgerInUnderReplication(lh.getId(), basePath)); Thread.sleep(100); } // restart killed bookie bs.add(startBookie(killedBookieConfig)); bsConfs.add(killedBookieConfig); while (isLedgerInUnderReplication(lh.getId(), basePath)) { Thread.sleep(100); } // Should be able to read the entries from 0-9 verifyRecoveredLedgers(lh, 0, 9); } finally { rw.shutdown(); } }
@Test(timeout = 60000) public void testMultipleLedgerReplicationWithReplicationWorker() throws Exception { // Ledger1 LedgerHandle lh1 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); for (int i = 0; i < 10; i++) { lh1.addEntry(data); } BookieSocketAddress replicaToKillFromFirstLedger = LedgerHandleAdapter.getLedgerMetadata(lh1).getEnsembles().get(0L).get(0); LOG.info("Killing Bookie", replicaToKillFromFirstLedger); // Ledger2 LedgerHandle lh2 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD); for (int i = 0; i < 10; i++) { lh2.addEntry(data); } BookieSocketAddress replicaToKillFromSecondLedger = LedgerHandleAdapter.getLedgerMetadata(lh2).getEnsembles().get(0L).get(0); LOG.info("Killing Bookie", replicaToKillFromSecondLedger); // Kill ledger1 killBookie(replicaToKillFromFirstLedger); lh1.close(); // Kill ledger2 killBookie(replicaToKillFromFirstLedger); lh2.close(); int startNewBookie = startNewBookie(); BookieSocketAddress newBkAddr = new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie); LOG.info("New Bookie addr :" + newBkAddr); ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr); rw.start(); try { // Mark ledger1 and 2 as underreplicated underReplicationManager.markLedgerUnderreplicated( lh1.getId(), replicaToKillFromFirstLedger.toString()); underReplicationManager.markLedgerUnderreplicated( lh2.getId(), replicaToKillFromSecondLedger.toString()); while (isLedgerInUnderReplication(lh1.getId(), basePath)) { Thread.sleep(100); } while (isLedgerInUnderReplication(lh2.getId(), basePath)) { Thread.sleep(100); } killAllBookies(lh1, newBkAddr); // Should be able to read the entries from 0-9 verifyRecoveredLedgers(lh1, 0, 9); verifyRecoveredLedgers(lh2, 0, 9); } finally { rw.shutdown(); } }
@Test(timeout = 60000) public void testSuccessAllocatorShouldDeleteUnusedledger() throws Exception { String allocationPath = "/allocation-delete-unused-ledger"; zkc.get() .create(allocationPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); Stat stat = new Stat(); byte[] data = zkc.get().getData(allocationPath, false, stat); Versioned<byte[]> allocationData = new Versioned<byte[]>(data, new ZkVersion(stat.getVersion())); SimpleLedgerAllocator allocator1 = new SimpleLedgerAllocator( allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc); allocator1.allocate(); // wait until allocated ZKTransaction txn1 = newTxn(); LedgerHandle lh1 = FutureUtils.result(allocator1.tryObtain(txn1, NULL_LISTENER)); // Second allocator kicks in stat = new Stat(); data = zkc.get().getData(allocationPath, false, stat); allocationData = new Versioned<byte[]>(data, new ZkVersion(stat.getVersion())); SimpleLedgerAllocator allocator2 = new SimpleLedgerAllocator( allocationPath, allocationData, newQuorumConfigProvider(dlConf), zkc, bkc); allocator2.allocate(); // wait until allocated ZKTransaction txn2 = newTxn(); LedgerHandle lh2 = FutureUtils.result(allocator2.tryObtain(txn2, NULL_LISTENER)); // should fail to commit txn1 as version is changed by second allocator try { FutureUtils.result(txn1.execute()); fail( "Should fail commit obtaining ledger handle from first allocator as allocator is modified by second allocator."); } catch (ZKException ke) { // as expected } FutureUtils.result(txn2.execute()); Utils.close(allocator1); Utils.close(allocator2); // ledger handle should be deleted try { lh1.close(); fail("LedgerHandle allocated by allocator1 should be deleted."); } catch (BKException bke) { // as expected } try { bkc.get() .openLedger(lh1.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes()); fail("LedgerHandle allocated by allocator1 should be deleted."); } catch (BKException.BKNoSuchLedgerExistsException nslee) { // as expected } long eid = lh2.addEntry("hello world".getBytes()); lh2.close(); LedgerHandle readLh = bkc.get() .openLedger( lh2.getId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes()); Enumeration<LedgerEntry> entries = readLh.readEntries(eid, eid); int i = 0; while (entries.hasMoreElements()) { LedgerEntry entry = entries.nextElement(); assertEquals("hello world", new String(entry.getEntry(), UTF_8)); ++i; } assertEquals(1, i); }