@Test(timeout = 60000) public void testAllWritesAreCompletedOnClosedLedger() throws Exception { for (int i = 0; i < 100; i++) { LOG.info("Iteration {}", i); List<AddCallbackFuture> futures = new ArrayList<AddCallbackFuture>(); LedgerHandle w = bkc.createLedger(DigestType.CRC32, new byte[0]); AddCallbackFuture f = new AddCallbackFuture(0L); w.asyncAddEntry("foobar".getBytes(UTF_8), f, null); f.get(); LedgerHandle r = bkc.openLedger(w.getId(), DigestType.CRC32, new byte[0]); for (int j = 0; j < 100; j++) { AddCallbackFuture f1 = new AddCallbackFuture(1L + j); w.asyncAddEntry("foobar".getBytes(), f1, null); futures.add(f1); } for (AddCallbackFuture f2 : futures) { try { f2.get(10, TimeUnit.SECONDS); } catch (ExecutionException ee) { // we don't care about errors } catch (TimeoutException te) { LOG.error("Error on waiting completing entry {} : ", f2.getExpectedEntryId(), te); fail("Should succeed on waiting completing entry " + f2.getExpectedEntryId()); } } } }
@Test(timeout = 60000) public void testLedgerCloseWithConsistentLength() throws Exception { ClientConfiguration conf = new ClientConfiguration(); conf.setZkServers(zkUtil.getZooKeeperConnectString()).setReadTimeout(1); BookKeeper bkc = new BookKeeper(conf); LedgerHandle lh = bkc.createLedger(6, 3, DigestType.CRC32, new byte[] {}); final CountDownLatch latch = new CountDownLatch(1); stopBKCluster(); final AtomicInteger i = new AtomicInteger(0xdeadbeef); AsyncCallback.AddCallback cb = new AsyncCallback.AddCallback() { @Override public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) { i.set(rc); latch.countDown(); } }; lh.asyncAddEntry("Test Entry".getBytes(), cb, null); latch.await(); assertEquals(i.get(), BKException.Code.NotEnoughBookiesException); assertEquals(0, lh.getLength()); assertEquals(LedgerHandle.INVALID_ENTRY_ID, lh.getLastAddConfirmed()); startBKCluster(); LedgerHandle newLh = bkc.openLedger(lh.getId(), DigestType.CRC32, new byte[] {}); assertEquals(0, newLh.getLength()); assertEquals(LedgerHandle.INVALID_ENTRY_ID, newLh.getLastAddConfirmed()); }
@Test(timeout = 60000) public void testLedgerCheckerShouldnotSelectInvalidLastFragments() throws Exception { int numEntries = 10; LedgerHandle lh = bkc.createLedger(3, 3, 3, digestType, "".getBytes()); // Add some entries before bookie failures for (int i = 0; i < numEntries; i++) { lh.addEntry("data".getBytes()); } numEntries = 4; // add n*ensemleSize+1 entries async after bookies // failed. verifyMetadataConsistency(numEntries, lh); LedgerChecker checker = new LedgerChecker(bkc); CheckerCallback cb = new CheckerCallback(); checker.checkLedger(lh, cb); Set<LedgerFragment> result = cb.waitAndGetResult(); assertEquals("No fragments should be selected", 0, result.size()); }
private Map<Integer, BookieSocketAddress> getReplacedBookiesByIndexes( LedgerHandle lh, List<BookieSocketAddress> ensemble, Set<Integer> bookieIndexesToRereplicate, Optional<Set<BookieSocketAddress>> excludedBookies) throws BKException.BKNotEnoughBookiesException { // target bookies to replicate Map<Integer, BookieSocketAddress> targetBookieAddresses = Maps.newHashMapWithExpectedSize(bookieIndexesToRereplicate.size()); // bookies to exclude for ensemble allocation Set<BookieSocketAddress> bookiesToExclude = Sets.newHashSet(); if (excludedBookies.isPresent()) { bookiesToExclude.addAll(excludedBookies.get()); } // excluding bookies that need to be replicated for (Integer bookieIndex : bookieIndexesToRereplicate) { BookieSocketAddress bookie = ensemble.get(bookieIndex); bookiesToExclude.add(bookie); } // allocate bookies for (Integer bookieIndex : bookieIndexesToRereplicate) { BookieSocketAddress oldBookie = ensemble.get(bookieIndex); BookieSocketAddress newBookie = bkc.getPlacementPolicy() .replaceBookie( lh.getLedgerMetadata().getEnsembleSize(), lh.getLedgerMetadata().getWriteQuorumSize(), lh.getLedgerMetadata().getAckQuorumSize(), ensemble, oldBookie, bookiesToExclude); targetBookieAddresses.put(bookieIndex, newBookie); bookiesToExclude.add(newBookie); } return targetBookieAddresses; }
private void verifyMetadataConsistency(int numEntries, LedgerHandle lh) throws Exception { final CountDownLatch addDoneLatch = new CountDownLatch(1); final CountDownLatch deadIOLatch = new CountDownLatch(1); final CountDownLatch recoverDoneLatch = new CountDownLatch(1); final CountDownLatch failedLatch = new CountDownLatch(1); // kill first bookie to replace with a unauthorize bookie BookieSocketAddress bookie = lh.getLedgerMetadata().currentEnsemble.get(0); ServerConfiguration conf = killBookie(bookie); // replace a unauthorize bookie startUnauthorizedBookie(conf, addDoneLatch); // kill second bookie to replace with a dead bookie bookie = lh.getLedgerMetadata().currentEnsemble.get(1); conf = killBookie(bookie); // replace a slow dead bookie startDeadBookie(conf, deadIOLatch); // tried to add entries for (int i = 0; i < numEntries; i++) { lh.asyncAddEntry( "data".getBytes(), new AddCallback() { @Override public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) { if (BKException.Code.OK != rc) { failedLatch.countDown(); deadIOLatch.countDown(); } if (0 == entryId) { try { recoverDoneLatch.await(); } catch (InterruptedException ie) { } } } }, null); } // add finished addDoneLatch.countDown(); // wait until entries failed due to UnauthorizedAccessException failedLatch.await(); // simulate the ownership of this ledger is transfer to another host LOG.info("Recover ledger {}.", lh.getId()); ClientConfiguration newConf = new ClientConfiguration(); newConf.addConfiguration(baseClientConf); BookKeeper newBkc = new BookKeeperTestClient(newConf.setReadTimeout(1)); LedgerHandle recoveredLh = newBkc.openLedger(lh.getId(), digestType, "".getBytes()); LOG.info("Recover ledger {} done.", lh.getId()); recoverDoneLatch.countDown(); // wait a bit until add operations failed from second bookie due to IOException TimeUnit.SECONDS.sleep(5); // open the ledger again to make sure we ge the right last confirmed. LedgerHandle newLh = newBkc.openLedger(lh.getId(), digestType, "".getBytes()); assertEquals( "Metadata should be consistent across different opened ledgers", recoveredLh.getLastAddConfirmed(), newLh.getLastAddConfirmed()); }
/** @return the metadata for the passed ledger handle */ public LedgerMetadata getLedgerMetadata(LedgerHandle lh) { return lh.getLedgerMetadata(); }
/** * Determines the last entry written to a ledger not closed properly due to a client crash * * @param passwd */ boolean recover(byte[] passwd) throws IOException, InterruptedException, BKException, KeeperException { /* * Create BookieClient objects and send a request to each one. */ for (InetSocketAddress s : bookies) { LOG.info(s); BookieClient client = new BookieClient(s, 3000); clients.add(client); client.readEntry(lId, -1, this, null); } /* * Wait until I have received enough responses */ synchronized (counter) { LOG.info("Counter: " + counter.get() + ", " + minimum + ", " + qMode); if (counter.get() < minimum) { LOG.info("Waiting..."); counter.wait(5000); } } /* * Obtain largest hint */ LedgerHandle lh = new LedgerHandle(self, lId, 0, qSize, qMode, passwd); for (InetSocketAddress addr : bookies) { lh.addBookieForReading(addr); } boolean notLegitimate = true; long readCounter = 0; while (notLegitimate) { readCounter = getNextHint(); if (readCounter > -1) { lh.setLast(readCounter); boolean hasMore = true; while (hasMore) { hasMore = false; LOG.debug("Recovering: " + lh.getLast()); LedgerSequence ls = lh.readEntries(lh.getLast(), lh.getLast()); LOG.debug("Received entry for: " + lh.getLast()); byte[] le = ls.nextElement().getEntry(); if (le != null) { if (notLegitimate) notLegitimate = false; lh.addEntry(le); hasMore = true; } } } else break; } /* * Write counter as the last entry of ledger */ if (!notLegitimate) { lh.setAddConfirmed(readCounter); lh.close(); return true; } else { lh.setLast(0); lh.close(); return false; } }