/**
   * Tests that replication worker1 should take one fragment replication and other replication
   * worker also should compete for the replication.
   */
  @Test(timeout = 90000)
  public void test2RWsShouldCompeteForReplicationOf2FragmentsAndCompleteReplication()
      throws Exception {
    LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);

    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }
    lh.close();
    BookieSocketAddress replicaToKill =
        LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
    LOG.info("Killing Bookie", replicaToKill);
    ServerConfiguration killedBookieConfig = killBookie(replicaToKill);

    killAllBookies(lh, null);
    // Starte RW1
    int startNewBookie1 = startNewBookie();
    BookieSocketAddress newBkAddr1 =
        new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie1);
    LOG.info("New Bookie addr :" + newBkAddr1);
    ReplicationWorker rw1 = new ReplicationWorker(zkc, baseConf, newBkAddr1);

    // Starte RW2
    int startNewBookie2 = startNewBookie();
    BookieSocketAddress newBkAddr2 =
        new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie2);
    LOG.info("New Bookie addr :" + newBkAddr2);
    ZooKeeper zkc1 =
        ZooKeeperClient.createConnectedZooKeeperClient(zkUtil.getZooKeeperConnectString(), 10000);
    ReplicationWorker rw2 = new ReplicationWorker(zkc1, baseConf, newBkAddr2);
    rw1.start();
    rw2.start();

    try {
      underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
      int counter = 10;
      while (counter-- > 0) {
        assertTrue(
            "Expecting that replication should not complete",
            isLedgerInUnderReplication(lh.getId(), basePath));
        Thread.sleep(100);
      }
      // restart killed bookie
      bs.add(startBookie(killedBookieConfig));
      bsConfs.add(killedBookieConfig);
      while (isLedgerInUnderReplication(lh.getId(), basePath)) {
        Thread.sleep(100);
      }
      // Should be able to read the entries from 0-9
      verifyRecoveredLedgers(lh, 0, 9);
    } finally {
      rw1.shutdown();
      rw2.shutdown();
      zkc1.close();
    }
  }
  /**
   * Tests that Replication worker should clean the leadger under replication node of the ledger
   * already deleted
   */
  @Test(timeout = 3000)
  public void testRWShouldCleanTheLedgerFromUnderReplicationIfLedgerAlreadyDeleted()
      throws Exception {
    LedgerHandle lh = bkc.createLedger(2, 2, BookKeeper.DigestType.CRC32, TESTPASSWD);

    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }
    lh.close();
    BookieSocketAddress replicaToKill =
        LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);

    int startNewBookie = startNewBookie();
    BookieSocketAddress newBkAddr =
        new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
    LOG.info("New Bookie addr :" + newBkAddr);
    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);
    rw.start();

    try {
      bkc.deleteLedger(lh.getId()); // Deleting the ledger
      // Also mark ledger as in UnderReplication
      underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
      while (isLedgerInUnderReplication(lh.getId(), basePath)) {
        Thread.sleep(100);
      }
    } finally {
      rw.shutdown();
    }
  }
  /**
   * Tests that ReplicationWorker should not have identified for postponing the replication if
   * ledger is in open state and lastFragment is not in underReplication state. Note that RW should
   * not fence such ledgers.
   */
  @Test(timeout = 30000)
  public void testRWShouldReplicateTheLedgersAfterTimeoutIfLastFragmentIsNotUR() throws Exception {
    LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);

    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill =
        LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);

    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);

    int startNewBookie = startNewBookie();

    // Reform ensemble...Making sure that last fragment is not in
    // under-replication
    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }

    BookieSocketAddress newBkAddr =
        new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
    LOG.info("New Bookie addr :" + newBkAddr);

    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);

    LedgerManagerFactory mFactory =
        LedgerManagerFactory.newLedgerManagerFactory(baseClientConf, zkc);
    LedgerUnderreplicationManager underReplicationManager =
        mFactory.newLedgerUnderreplicationManager();

    rw.start();
    try {

      underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
      while (isLedgerInUnderReplication(lh.getId(), basePath)) {
        Thread.sleep(100);
      }

      killAllBookies(lh, newBkAddr);

      // Should be able to read the entries from 0-9
      verifyRecoveredLedgers(lh, 0, 9);
      lh = bkc.openLedgerNoRecovery(lh.getId(), BookKeeper.DigestType.CRC32, TESTPASSWD);

      // Ledger should be still in open state
      assertTrue("Ledger must have been closed by RW", ClientUtil.isLedgerOpen(lh));
    } finally {
      rw.shutdown();
      underReplicationManager.close();
    }
  }
  /**
   * Tests that replication worker should retry for replication until enough bookies available for
   * replication
   */
  @Test(timeout = 60000)
  public void testRWShouldRetryUntilThereAreEnoughBksAvailableForReplication() throws Exception {
    LedgerHandle lh = bkc.createLedger(1, 1, BookKeeper.DigestType.CRC32, TESTPASSWD);

    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }
    lh.close();
    BookieSocketAddress replicaToKill =
        LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);
    LOG.info("Killing Bookie", replicaToKill);
    ServerConfiguration killedBookieConfig = killBookie(replicaToKill);

    int startNewBookie = startNewBookie();
    BookieSocketAddress newBkAddr =
        new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
    LOG.info("New Bookie addr :" + newBkAddr);

    killAllBookies(lh, newBkAddr);
    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);

    rw.start();
    try {
      underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
      int counter = 100;
      while (counter-- > 0) {
        assertTrue(
            "Expecting that replication should not complete",
            isLedgerInUnderReplication(lh.getId(), basePath));
        Thread.sleep(100);
      }
      // restart killed bookie
      bs.add(startBookie(killedBookieConfig));
      bsConfs.add(killedBookieConfig);
      while (isLedgerInUnderReplication(lh.getId(), basePath)) {
        Thread.sleep(100);
      }
      // Should be able to read the entries from 0-9
      verifyRecoveredLedgers(lh, 0, 9);
    } finally {
      rw.shutdown();
    }
  }
  /**
   * Tests that replication worker should replicate the failed bookie fragments to target bookie
   * given to the worker.
   */
  @Test(timeout = 30000)
  public void testRWShouldReplicateFragmentsToTargetBookie() throws Exception {
    LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);

    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill =
        LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);

    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);

    int startNewBookie = startNewBookie();
    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }

    BookieSocketAddress newBkAddr =
        new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
    LOG.info("New Bookie addr :" + newBkAddr);

    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);

    rw.start();
    try {

      underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());

      while (isLedgerInUnderReplication(lh.getId(), basePath)) {
        Thread.sleep(100);
      }

      killAllBookies(lh, newBkAddr);

      // Should be able to read the entries from 0-9
      verifyRecoveredLedgers(lh, 0, 9);
    } finally {
      rw.shutdown();
    }
  }
  /**
   * Test that if the local bookie turns out to be readonly, then no point in running RW. So RW
   * should shutdown.
   */
  @Test(timeout = 20000)
  public void testRWShutdownOnLocalBookieReadonlyTransition() throws Exception {
    LedgerHandle lh = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);

    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }
    BookieSocketAddress replicaToKill =
        LedgerHandleAdapter.getLedgerMetadata(lh).getEnsembles().get(0L).get(0);

    LOG.info("Killing Bookie", replicaToKill);
    killBookie(replicaToKill);

    int newBkPort = startNewBookie();
    for (int i = 0; i < 10; i++) {
      lh.addEntry(data);
    }

    BookieSocketAddress newBkAddr =
        new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), newBkPort);
    LOG.info("New Bookie addr :" + newBkAddr);

    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);

    rw.start();
    try {
      BookieServer newBk = bs.get(bs.size() - 1);
      bsConfs.get(bsConfs.size() - 1).setReadOnlyModeEnabled(true);
      newBk.getBookie().doTransitionToReadOnlyMode();
      underReplicationManager.markLedgerUnderreplicated(lh.getId(), replicaToKill.toString());
      while (ReplicationTestUtil.isLedgerInUnderReplication(zkc, lh.getId(), basePath)
          && rw.isRunning()) {
        Thread.sleep(100);
      }
      assertFalse("RW should shutdown if the bookie is readonly", rw.isRunning());
    } finally {
      rw.shutdown();
    }
  }
  @Test(timeout = 60000)
  public void testMultipleLedgerReplicationWithReplicationWorker() throws Exception {
    // Ledger1
    LedgerHandle lh1 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);

    for (int i = 0; i < 10; i++) {
      lh1.addEntry(data);
    }
    BookieSocketAddress replicaToKillFromFirstLedger =
        LedgerHandleAdapter.getLedgerMetadata(lh1).getEnsembles().get(0L).get(0);

    LOG.info("Killing Bookie", replicaToKillFromFirstLedger);

    // Ledger2
    LedgerHandle lh2 = bkc.createLedger(3, 3, BookKeeper.DigestType.CRC32, TESTPASSWD);

    for (int i = 0; i < 10; i++) {
      lh2.addEntry(data);
    }
    BookieSocketAddress replicaToKillFromSecondLedger =
        LedgerHandleAdapter.getLedgerMetadata(lh2).getEnsembles().get(0L).get(0);

    LOG.info("Killing Bookie", replicaToKillFromSecondLedger);

    // Kill ledger1
    killBookie(replicaToKillFromFirstLedger);
    lh1.close();
    // Kill ledger2
    killBookie(replicaToKillFromFirstLedger);
    lh2.close();

    int startNewBookie = startNewBookie();

    BookieSocketAddress newBkAddr =
        new BookieSocketAddress(InetAddress.getLocalHost().getHostAddress(), startNewBookie);
    LOG.info("New Bookie addr :" + newBkAddr);

    ReplicationWorker rw = new ReplicationWorker(zkc, baseConf, newBkAddr);

    rw.start();
    try {

      // Mark ledger1 and 2 as underreplicated
      underReplicationManager.markLedgerUnderreplicated(
          lh1.getId(), replicaToKillFromFirstLedger.toString());
      underReplicationManager.markLedgerUnderreplicated(
          lh2.getId(), replicaToKillFromSecondLedger.toString());

      while (isLedgerInUnderReplication(lh1.getId(), basePath)) {
        Thread.sleep(100);
      }

      while (isLedgerInUnderReplication(lh2.getId(), basePath)) {
        Thread.sleep(100);
      }

      killAllBookies(lh1, newBkAddr);

      // Should be able to read the entries from 0-9
      verifyRecoveredLedgers(lh1, 0, 9);
      verifyRecoveredLedgers(lh2, 0, 9);
    } finally {
      rw.shutdown();
    }
  }