Esempio n. 1
0
  @Test(timeout = 60000)
  public void testLedgerCloseWithConsistentLength() throws Exception {
    ClientConfiguration conf = new ClientConfiguration();
    conf.setZkServers(zkUtil.getZooKeeperConnectString()).setReadTimeout(1);

    BookKeeper bkc = new BookKeeper(conf);
    LedgerHandle lh = bkc.createLedger(6, 3, DigestType.CRC32, new byte[] {});
    final CountDownLatch latch = new CountDownLatch(1);
    stopBKCluster();
    final AtomicInteger i = new AtomicInteger(0xdeadbeef);
    AsyncCallback.AddCallback cb =
        new AsyncCallback.AddCallback() {
          @Override
          public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
            i.set(rc);
            latch.countDown();
          }
        };
    lh.asyncAddEntry("Test Entry".getBytes(), cb, null);
    latch.await();
    assertEquals(i.get(), BKException.Code.NotEnoughBookiesException);
    assertEquals(0, lh.getLength());
    assertEquals(LedgerHandle.INVALID_ENTRY_ID, lh.getLastAddConfirmed());
    startBKCluster();
    LedgerHandle newLh = bkc.openLedger(lh.getId(), DigestType.CRC32, new byte[] {});
    assertEquals(0, newLh.getLength());
    assertEquals(LedgerHandle.INVALID_ENTRY_ID, newLh.getLastAddConfirmed());
  }
Esempio n. 2
0
  private void verifyMetadataConsistency(int numEntries, LedgerHandle lh) throws Exception {
    final CountDownLatch addDoneLatch = new CountDownLatch(1);
    final CountDownLatch deadIOLatch = new CountDownLatch(1);
    final CountDownLatch recoverDoneLatch = new CountDownLatch(1);
    final CountDownLatch failedLatch = new CountDownLatch(1);
    // kill first bookie to replace with a unauthorize bookie
    BookieSocketAddress bookie = lh.getLedgerMetadata().currentEnsemble.get(0);
    ServerConfiguration conf = killBookie(bookie);
    // replace a unauthorize bookie
    startUnauthorizedBookie(conf, addDoneLatch);
    // kill second bookie to replace with a dead bookie
    bookie = lh.getLedgerMetadata().currentEnsemble.get(1);
    conf = killBookie(bookie);
    // replace a slow dead bookie
    startDeadBookie(conf, deadIOLatch);

    // tried to add entries
    for (int i = 0; i < numEntries; i++) {
      lh.asyncAddEntry(
          "data".getBytes(),
          new AddCallback() {
            @Override
            public void addComplete(int rc, LedgerHandle lh, long entryId, Object ctx) {
              if (BKException.Code.OK != rc) {
                failedLatch.countDown();
                deadIOLatch.countDown();
              }
              if (0 == entryId) {
                try {
                  recoverDoneLatch.await();
                } catch (InterruptedException ie) {
                }
              }
            }
          },
          null);
    }
    // add finished
    addDoneLatch.countDown();
    // wait until entries failed due to UnauthorizedAccessException
    failedLatch.await();
    // simulate the ownership of this ledger is transfer to another host
    LOG.info("Recover ledger {}.", lh.getId());
    ClientConfiguration newConf = new ClientConfiguration();
    newConf.addConfiguration(baseClientConf);
    BookKeeper newBkc = new BookKeeperTestClient(newConf.setReadTimeout(1));
    LedgerHandle recoveredLh = newBkc.openLedger(lh.getId(), digestType, "".getBytes());
    LOG.info("Recover ledger {} done.", lh.getId());
    recoverDoneLatch.countDown();
    // wait a bit until add operations failed from second bookie due to IOException
    TimeUnit.SECONDS.sleep(5);
    // open the ledger again to make sure we ge the right last confirmed.
    LedgerHandle newLh = newBkc.openLedger(lh.getId(), digestType, "".getBytes());
    assertEquals(
        "Metadata should be consistent across different opened ledgers",
        recoveredLh.getLastAddConfirmed(),
        newLh.getLastAddConfirmed());
  }
Esempio n. 3
0
  /**
   * Hedwig Console
   *
   * @param args arguments
   * @throws IOException
   * @throws InterruptedException
   */
  public HedwigConsole(String[] args) throws IOException, InterruptedException {
    // Setup Terminal
    terminal = Terminal.setupTerminal();
    HedwigCommands.init();
    cl.parseOptions(args);

    if (cl.getCommand() == null) {
      inConsole = true;
    } else {
      inConsole = false;
    }

    org.apache.bookkeeper.conf.ClientConfiguration bkClientConf =
        new org.apache.bookkeeper.conf.ClientConfiguration();
    ServerConfiguration hubServerConf = new ServerConfiguration();
    String serverCfgFile = cl.getOption("server-cfg");
    if (serverCfgFile != null) {
      try {
        hubServerConf.loadConf(new File(serverCfgFile).toURI().toURL());
      } catch (ConfigurationException e) {
        throw new IOException(e);
      }
      try {
        bkClientConf.loadConf(new File(serverCfgFile).toURI().toURL());
      } catch (ConfigurationException e) {
        throw new IOException(e);
      }
    }

    ClientConfiguration hubClientCfg = new ClientConfiguration();
    String clientCfgFile = cl.getOption("client-cfg");
    if (clientCfgFile != null) {
      try {
        hubClientCfg.loadConf(new File(clientCfgFile).toURI().toURL());
      } catch (ConfigurationException e) {
        throw new IOException(e);
      }
    }

    printMessage("Connecting to zookeeper/bookkeeper using HedwigAdmin");
    try {
      admin = new HedwigAdmin(bkClientConf, hubServerConf);
      admin.getZkHandle().register(new MyWatcher());
    } catch (Exception e) {
      throw new IOException(e);
    }

    printMessage("Connecting to default hub server " + hubClientCfg.getDefaultServerHost());
    hubClient = new HedwigClient(hubClientCfg);
    publisher = hubClient.getPublisher();
    subscriber = hubClient.getSubscriber();
    subscriber.addSubscriptionListener(new ConsoleSubscriptionListener());

    // other parameters
    myRegion = hubServerConf.getMyRegion();
  }
Esempio n. 4
0
  /**
   * Constructor that takes in a configuration object so we know how to connect to ZooKeeper to
   * retrieve information about the BookKeeper cluster. We need this before we can do any type of
   * admin operations on the BookKeeper cluster.
   *
   * @param conf Client Configuration Object
   * @throws IOException throws this exception if there is an error instantiating the ZooKeeper
   *     client.
   * @throws InterruptedException Throws this exception if there is an error instantiating the
   *     BookKeeper client.
   * @throws KeeperException Throws this exception if there is an error instantiating the BookKeeper
   *     client.
   */
  public BookKeeperAdmin(ClientConfiguration conf)
      throws IOException, InterruptedException, KeeperException {
    // Create the ZooKeeper client instance
    zk =
        ZooKeeperClient.newBuilder()
            .connectString(conf.getZkServers())
            .sessionTimeoutMs(conf.getZkTimeout())
            .requestRateLimit(conf.getZkRequestRateLimit())
            .build();
    ownsZK = true;

    // Create the BookKeeper client instance
    bkc = new BookKeeper(conf, zk);
    ownsBK = true;

    this.lfr = new LedgerFragmentReplicator(bkc);
  }
Esempio n. 5
0
  /**
   * Format the BookKeeper metadata in zookeeper
   *
   * @param isInteractive Whether format should ask prompt for confirmation if old data exists or
   *     not.
   * @param force If non interactive and force is true, then old data will be removed without
   *     prompt.
   * @return Returns true if format succeeds else false.
   */
  public static boolean format(ClientConfiguration conf, boolean isInteractive, boolean force)
      throws Exception {
    ZooKeeper zkc =
        ZooKeeperClient.createConnectedZooKeeperClient(
            conf.getZkServers(),
            conf.getZkTimeout(),
            new BoundExponentialBackoffRetryPolicy(
                conf.getZkTimeout(), 3 * conf.getZkTimeout(), Integer.MAX_VALUE));
    BookKeeper bkc = null;
    try {
      boolean ledgerRootExists = null != zkc.exists(conf.getZkLedgersRootPath(), false);
      boolean availableNodeExists = null != zkc.exists(conf.getZkAvailableBookiesPath(), false);

      // Create ledgers root node if not exists
      if (!ledgerRootExists) {
        zkc.create(
            conf.getZkLedgersRootPath(),
            "".getBytes(UTF_8),
            Ids.OPEN_ACL_UNSAFE,
            CreateMode.PERSISTENT);
      }
      // create available bookies node if not exists
      if (!availableNodeExists) {
        zkc.create(
            conf.getZkAvailableBookiesPath(),
            "".getBytes(UTF_8),
            Ids.OPEN_ACL_UNSAFE,
            CreateMode.PERSISTENT);
      }

      // If old data was there then confirm with admin.
      if (ledgerRootExists) {
        boolean confirm = false;
        if (!isInteractive) {
          // If non interactive and force is set, then delete old
          // data.
          if (force) {
            confirm = true;
          } else {
            confirm = false;
          }
        } else {
          // Confirm with the admin.
          confirm = IOUtils.confirmPrompt("Are you sure to format bookkeeper metadata ?");
        }
        if (!confirm) {
          LOG.error("BookKeeper metadata Format aborted!!");
          return false;
        }
      }
      bkc = new BookKeeper(conf, zkc);
      // Format all ledger metadata layout
      bkc.ledgerManagerFactory.format(conf, zkc);

      // Clear the cookies
      try {
        ZKUtil.deleteRecursive(zkc, conf.getZkLedgersRootPath() + "/cookies");
      } catch (KeeperException.NoNodeException e) {
        LOG.debug("cookies node not exists in zookeeper to delete");
      }

      // Clear the INSTANCEID
      try {
        zkc.delete(conf.getZkLedgersRootPath() + "/" + INSTANCEID, -1);
      } catch (KeeperException.NoNodeException e) {
        LOG.debug("INSTANCEID not exists in zookeeper to delete");
      }

      // create INSTANCEID
      String instanceId = UUID.randomUUID().toString();
      zkc.create(
          conf.getZkLedgersRootPath() + "/" + INSTANCEID,
          instanceId.getBytes(UTF_8),
          Ids.OPEN_ACL_UNSAFE,
          CreateMode.PERSISTENT);

      LOG.info("Successfully formatted BookKeeper metadata");
    } finally {
      if (null != bkc) {
        bkc.close();
      }
      if (null != zkc) {
        zkc.close();
      }
    }
    return true;
  }