@Test(timeout = 60000)
public void testAllWritesAreCompletedOnClosedLedger() throws Exception {
for (int i = 0; i < 100; i++) {
LOG.info("Iteration {}", i);
List<AddCallbackFuture> futures = new ArrayList<AddCallbackFuture>();
LedgerHandle w = bkc.createLedger(DigestType.CRC32, new byte[0]);
AddCallbackFuture f = new AddCallbackFuture(0L);
w.asyncAddEntry("foobar".getBytes(UTF_8), f, null);
f.get();
LedgerHandle r = bkc.openLedger(w.getId(), DigestType.CRC32, new byte[0]);
for (int j = 0; j < 100; j++) {
AddCallbackFuture f1 = new AddCallbackFuture(1L + j);
w.asyncAddEntry("foobar".getBytes(), f1, null);
futures.add(f1);
}
for (AddCallbackFuture f2 : futures) {
try {
f2.get(10, TimeUnit.SECONDS);
} catch (ExecutionException ee) {
// we don't care about errors
} catch (TimeoutException te) {
LOG.error("Error on waiting completing entry {} : ", f2.getExpectedEntryId(), te);
fail("Should succeed on waiting completing entry " + f2.getExpectedEntryId());
}
}
}
}
@Test(timeout = 60000)
public void testLedgerCloseWithConsistentLength() throws Exception {
ClientConfiguration conf = new ClientConfiguration();
conf.setZkServers(zkUtil.getZooKeeperConnectString()).setReadTimeout(1);
BookKeeper bkc = new BookKeeper(conf);
LedgerHandle lh = bkc.createLedger(6, 3, DigestType.CRC32, new byte[] {});
final CountDownLatch latch = new CountDownLatch(1);
stopBKCluster();
final AtomicInteger i = new AtomicInteger(0xdeadbeef);
AsyncCallback.AddCallback cb =
new AsyncCallback.AddCallback() {
@Override
public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
i.set(rc);
latch.countDown();
}
};
lh.asyncAddEntry("Test Entry".getBytes(), cb, null);
latch.await();
assertEquals(i.get(), BKException.Code.NotEnoughBookiesException);
assertEquals(0, lh.getLength());
assertEquals(LedgerHandle.INVALID_ENTRY_ID, lh.getLastAddConfirmed());
startBKCluster();
LedgerHandle newLh = bkc.openLedger(lh.getId(), DigestType.CRC32, new byte[] {});
assertEquals(0, newLh.getLength());
assertEquals(LedgerHandle.INVALID_ENTRY_ID, newLh.getLastAddConfirmed());
}
@Test(timeout = 60000)
public void testLedgerCheckerShouldnotSelectInvalidLastFragments() throws Exception {
int numEntries = 10;
LedgerHandle lh = bkc.createLedger(3, 3, 3, digestType, "".getBytes());
// Add some entries before bookie failures
for (int i = 0; i < numEntries; i++) {
lh.addEntry("data".getBytes());
}
numEntries = 4; // add n*ensemleSize+1 entries async after bookies
// failed.
verifyMetadataConsistency(numEntries, lh);
LedgerChecker checker = new LedgerChecker(bkc);
CheckerCallback cb = new CheckerCallback();
checker.checkLedger(lh, cb);
Set<LedgerFragment> result = cb.waitAndGetResult();
assertEquals("No fragments should be selected", 0, result.size());
}
private Map<Integer, BookieSocketAddress> getReplacedBookiesByIndexes(
LedgerHandle lh,
List<BookieSocketAddress> ensemble,
Set<Integer> bookieIndexesToRereplicate,
Optional<Set<BookieSocketAddress>> excludedBookies)
throws BKException.BKNotEnoughBookiesException {
// target bookies to replicate
Map<Integer, BookieSocketAddress> targetBookieAddresses =
Maps.newHashMapWithExpectedSize(bookieIndexesToRereplicate.size());
// bookies to exclude for ensemble allocation
Set<BookieSocketAddress> bookiesToExclude = Sets.newHashSet();
if (excludedBookies.isPresent()) {
bookiesToExclude.addAll(excludedBookies.get());
}
// excluding bookies that need to be replicated
for (Integer bookieIndex : bookieIndexesToRereplicate) {
BookieSocketAddress bookie = ensemble.get(bookieIndex);
bookiesToExclude.add(bookie);
}
// allocate bookies
for (Integer bookieIndex : bookieIndexesToRereplicate) {
BookieSocketAddress oldBookie = ensemble.get(bookieIndex);
BookieSocketAddress newBookie =
bkc.getPlacementPolicy()
.replaceBookie(
lh.getLedgerMetadata().getEnsembleSize(),
lh.getLedgerMetadata().getWriteQuorumSize(),
lh.getLedgerMetadata().getAckQuorumSize(),
ensemble,
oldBookie,
bookiesToExclude);
targetBookieAddresses.put(bookieIndex, newBookie);
bookiesToExclude.add(newBookie);
}
return targetBookieAddresses;
}
private void verifyMetadataConsistency(int numEntries, LedgerHandle lh) throws Exception {
final CountDownLatch addDoneLatch = new CountDownLatch(1);
final CountDownLatch deadIOLatch = new CountDownLatch(1);
final CountDownLatch recoverDoneLatch = new CountDownLatch(1);
final CountDownLatch failedLatch = new CountDownLatch(1);
// kill first bookie to replace with a unauthorize bookie
BookieSocketAddress bookie = lh.getLedgerMetadata().currentEnsemble.get(0);
ServerConfiguration conf = killBookie(bookie);
// replace a unauthorize bookie
startUnauthorizedBookie(conf, addDoneLatch);
// kill second bookie to replace with a dead bookie
bookie = lh.getLedgerMetadata().currentEnsemble.get(1);
conf = killBookie(bookie);
// replace a slow dead bookie
startDeadBookie(conf, deadIOLatch);
// tried to add entries
for (int i = 0; i < numEntries; i++) {
lh.asyncAddEntry(
"data".getBytes(),
new AddCallback() {
@Override
public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
if (BKException.Code.OK != rc) {
failedLatch.countDown();
deadIOLatch.countDown();
}
if (0 == entryId) {
try {
recoverDoneLatch.await();
} catch (InterruptedException ie) {
}
}
}
},
null);
}
// add finished
addDoneLatch.countDown();
// wait until entries failed due to UnauthorizedAccessException
failedLatch.await();
// simulate the ownership of this ledger is transfer to another host
LOG.info("Recover ledger {}.", lh.getId());
ClientConfiguration newConf = new ClientConfiguration();
newConf.addConfiguration(baseClientConf);
BookKeeper newBkc = new BookKeeperTestClient(newConf.setReadTimeout(1));
LedgerHandle recoveredLh = newBkc.openLedger(lh.getId(), digestType, "".getBytes());
LOG.info("Recover ledger {} done.", lh.getId());
recoverDoneLatch.countDown();
// wait a bit until add operations failed from second bookie due to IOException
TimeUnit.SECONDS.sleep(5);
// open the ledger again to make sure we ge the right last confirmed.
LedgerHandle newLh = newBkc.openLedger(lh.getId(), digestType, "".getBytes());
assertEquals(
"Metadata should be consistent across different opened ledgers",
recoveredLh.getLastAddConfirmed(),
newLh.getLastAddConfirmed());
}
/** @return the metadata for the passed ledger handle */
public LedgerMetadata getLedgerMetadata(LedgerHandle lh) {
return lh.getLedgerMetadata();
}
/**
* Determines the last entry written to a ledger not closed properly due to a client crash
*
* @param passwd
*/
boolean recover(byte[] passwd)
throws IOException, InterruptedException, BKException, KeeperException {
/*
* Create BookieClient objects and send a request to each one.
*/
for (InetSocketAddress s : bookies) {
LOG.info(s);
BookieClient client = new BookieClient(s, 3000);
clients.add(client);
client.readEntry(lId, -1, this, null);
}
/*
* Wait until I have received enough responses
*/
synchronized (counter) {
LOG.info("Counter: " + counter.get() + ", " + minimum + ", " + qMode);
if (counter.get() < minimum) {
LOG.info("Waiting...");
counter.wait(5000);
}
}
/*
* Obtain largest hint
*/
LedgerHandle lh = new LedgerHandle(self, lId, 0, qSize, qMode, passwd);
for (InetSocketAddress addr : bookies) {
lh.addBookieForReading(addr);
}
boolean notLegitimate = true;
long readCounter = 0;
while (notLegitimate) {
readCounter = getNextHint();
if (readCounter > -1) {
lh.setLast(readCounter);
boolean hasMore = true;
while (hasMore) {
hasMore = false;
LOG.debug("Recovering: " + lh.getLast());
LedgerSequence ls = lh.readEntries(lh.getLast(), lh.getLast());
LOG.debug("Received entry for: " + lh.getLast());
byte[] le = ls.nextElement().getEntry();
if (le != null) {
if (notLegitimate) notLegitimate = false;
lh.addEntry(le);
hasMore = true;
}
}
} else break;
}
/*
* Write counter as the last entry of ledger
*/
if (!notLegitimate) {
lh.setAddConfirmed(readCounter);
lh.close();
return true;
} else {
lh.setLast(0);
lh.close();
return false;
}
}
