@Test(timeout = 30000)
  public void testInfo() {
    HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
    MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl(master);
    assertEquals(master.getSplitPlanCount(), info.getSplitPlanCount(), 0);
    assertEquals(master.getMergePlanCount(), info.getMergePlanCount(), 0);
    assertEquals(master.getAverageLoad(), info.getAverageLoad(), 0);
    assertEquals(master.getClusterId(), info.getClusterId());
    assertEquals(master.getMasterActiveTime(), info.getActiveTime());
    assertEquals(master.getMasterStartTime(), info.getStartTime());
    assertEquals(master.getMasterCoprocessors().length, info.getCoprocessors().length);
    assertEquals(
        master.getServerManager().getOnlineServersList().size(), info.getNumRegionServers());
    assertEquals(5, info.getNumRegionServers());

    String zkServers = info.getZookeeperQuorum();
    assertEquals(zkServers.split(",").length, TEST_UTIL.getZkCluster().getZooKeeperServerNum());

    final int index = 3;
    LOG.info("Stopping " + TEST_UTIL.getMiniHBaseCluster().getRegionServer(index));
    TEST_UTIL.getMiniHBaseCluster().stopRegionServer(index, false);
    TEST_UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
    // We stopped the regionserver but could take a while for the master to notice it so hang here
    // until it does... then move forward to see if metrics wrapper notices.
    while (TEST_UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServers().size()
        != 4) {
      Threads.sleep(10);
    }
    assertEquals(4, info.getNumRegionServers());
    assertEquals(1, info.getNumDeadRegionServers());
    assertEquals(1, info.getNumWALFiles());
  }
Example #2
0
  private void testTruncateTable(final TableName tableName, boolean preserveSplits)
      throws IOException {
    byte[][] splitKeys = new byte[2][];
    splitKeys[0] = Bytes.toBytes(4);
    splitKeys[1] = Bytes.toBytes(8);

    // Create & Fill the table
    HTable table = TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY, splitKeys);
    try {
      TEST_UTIL.loadNumericRows(table, HConstants.CATALOG_FAMILY, 0, 10);
      assertEquals(10, TEST_UTIL.countRows(table));
    } finally {
      table.close();
    }
    assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());

    // Truncate & Verify
    this.admin.disableTable(tableName);
    this.admin.truncateTable(tableName, preserveSplits);
    table = new HTable(TEST_UTIL.getConfiguration(), tableName);
    try {
      assertEquals(0, TEST_UTIL.countRows(table));
    } finally {
      table.close();
    }
    if (preserveSplits) {
      assertEquals(3, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
    } else {
      assertEquals(1, TEST_UTIL.getHBaseCluster().getRegions(tableName).size());
    }
  }
  /**
   * Test snapshotting a table that is online without flushing
   *
   * @throws Exception
   */
  @Test
  public void testSkipFlushTableSnapshot() throws Exception {
    // make sure we don't fail on listing snapshots
    SnapshotTestingUtils.assertNoSnapshots(admin);

    // put some stuff in the table
    Table table = UTIL.getConnection().getTable(TABLE_NAME);
    UTIL.loadTable(table, TEST_FAM);
    UTIL.flush(TABLE_NAME);

    LOG.debug("FS state before snapshot:");
    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

    // take a snapshot of the enabled table
    String snapshotString = "skipFlushTableSnapshot";
    byte[] snapshot = Bytes.toBytes(snapshotString);
    admin.snapshot(snapshotString, TABLE_NAME, SnapshotType.SKIPFLUSH);
    LOG.debug("Snapshot completed.");

    // make sure we have the snapshot
    List<SnapshotDescription> snapshots =
        SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);

    // make sure its a valid snapshot
    LOG.debug("FS state after snapshot:");
    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

    SnapshotTestingUtils.confirmSnapshotValid(
        UTIL, ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);

    admin.deleteSnapshot(snapshot);
    snapshots = admin.listSnapshots();
    SnapshotTestingUtils.assertNoSnapshots(admin);
  }
  /**
   * Test simple flush snapshotting a table that is online
   *
   * @throws Exception
   */
  @Test
  public void testFlushTableSnapshotWithProcedure() throws Exception {
    // make sure we don't fail on listing snapshots
    SnapshotTestingUtils.assertNoSnapshots(admin);

    // put some stuff in the table
    SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);

    LOG.debug("FS state before snapshot:");
    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

    // take a snapshot of the enabled table
    String snapshotString = "offlineTableSnapshot";
    byte[] snapshot = Bytes.toBytes(snapshotString);
    Map<String, String> props = new HashMap<String, String>();
    props.put("table", TABLE_NAME.getNameAsString());
    admin.execProcedure(
        SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, snapshotString, props);

    LOG.debug("Snapshot completed.");

    // make sure we have the snapshot
    List<SnapshotDescription> snapshots =
        SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);

    // make sure its a valid snapshot
    LOG.debug("FS state after snapshot:");
    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

    SnapshotTestingUtils.confirmSnapshotValid(
        UTIL, ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
  }
  @Before
  public void setup() throws Exception {
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
    assertTrue("expected executor to be running", procExec.isRunning());

    nonceGroup =
        MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
    nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
  }
  /*
   * Take a snapshot of a table, add metadata, and verify that this only
   * affects one table
   * @param online - Whether the table is online or not during the snapshot
   */
  private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();

    // Create a table
    Admin admin = UTIL.getHBaseAdmin();
    final long startTime = System.currentTimeMillis();
    final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
    Table original = UTIL.createTable(localTableName, TEST_FAM);
    UTIL.loadTable(original, TEST_FAM);

    final String snapshotNameAsString = "snapshot_" + localTableName;

    // Create a snapshot
    SnapshotTestingUtils.createSnapshotAndValidate(
        admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);

    if (!online) {
      admin.enableTable(localTableName);
    }
    TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);

    // Clone the snapshot
    byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
    admin.cloneSnapshot(snapshotName, cloneTableName);

    // Add a new column family to the original table
    byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
    HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);

    admin.disableTable(localTableName);
    admin.addColumnFamily(localTableName, hcd);

    // Verify that it is not in the snapshot
    admin.enableTable(localTableName);

    // get a description of the cloned table
    // get a list of its families
    // assert that the family is there
    HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
    HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);

    Assert.assertTrue(
        "The original family was not found. There is something wrong. ",
        originalTableDescriptor.hasFamily(TEST_FAM));
    Assert.assertTrue(
        "The original family was not found in the clone. There is something wrong. ",
        clonedTableDescriptor.hasFamily(TEST_FAM));

    Assert.assertTrue(
        "The new family was not found. ", originalTableDescriptor.hasFamily(TEST_FAM_2));
    Assert.assertTrue(
        "The new family was not found. ", !clonedTableDescriptor.hasFamily(TEST_FAM_2));
  }
  /*
   * Take a snapshot of a table, do a split, and verify that this only affects one table
   * @param online - Whether the table is online or not during the snapshot
   */
  private void runTestRegionOperationsIndependent(boolean online) throws Exception {
    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();

    // Create a table
    Admin admin = UTIL.getHBaseAdmin();
    final long startTime = System.currentTimeMillis();
    final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
    Table original = UTIL.createTable(localTableName, TEST_FAM);
    UTIL.loadTable(original, TEST_FAM);
    final int loadedTableCount = UTIL.countRows(original);
    System.out.println("Original table has: " + loadedTableCount + " rows");

    final String snapshotNameAsString = "snapshot_" + localTableName;

    // Create a snapshot
    SnapshotTestingUtils.createSnapshotAndValidate(
        admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);

    if (!online) {
      admin.enableTable(localTableName);
    }

    TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);

    // Clone the snapshot
    byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
    admin.cloneSnapshot(snapshotName, cloneTableName);

    // Verify that region information is the same pre-split
    ((ClusterConnection) UTIL.getConnection()).clearRegionCache();
    List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);

    final int originalRegionCount = originalTableHRegions.size();
    final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
    Assert.assertEquals(
        "The number of regions in the cloned table is different than in the original table.",
        originalRegionCount,
        cloneTableRegionCount);

    // Split a region on the parent table
    admin.splitRegion(originalTableHRegions.get(0).getRegionName());
    waitOnSplit(UTIL.getConnection(), original, originalRegionCount);

    // Verify that the cloned table region is not split
    final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
    Assert.assertEquals(
        "The number of regions in the cloned table changed though none of its regions were split.",
        cloneTableRegionCount,
        cloneTableRegionCount2);
  }
  /** Basic end-to-end test of simple-flush-based snapshots */
  @Test
  public void testFlushCreateListDestroy() throws Exception {
    LOG.debug("------- Starting Snapshot test -------------");
    // make sure we don't fail on listing snapshots
    SnapshotTestingUtils.assertNoSnapshots(admin);
    // load the table so we have some data
    SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);

    String snapshotName = "flushSnapshotCreateListDestroy";
    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
    SnapshotTestingUtils.createSnapshotAndValidate(
        admin, TABLE_NAME, Bytes.toString(TEST_FAM), snapshotName, rootDir, fs, true);
  }
 @Test(timeout = 60000)
 public void testExceptionDuringInitialization() throws Exception {
   Configuration conf = TEST_UTIL.getConfiguration();
   conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast.
   conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
   conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
   TEST_UTIL.startMiniCluster(2);
   try {
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     // Trigger one regionserver to fail as if it came up with a coprocessor
     // that fails during initialization
     final HRegionServer regionServer = cluster.getRegionServer(0);
     conf.set(
         CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
         FailedInitializationObserver.class.getName());
     regionServer
         .getRegionServerCoprocessorHost()
         .loadSystemCoprocessors(conf, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
     TEST_UTIL.waitFor(
         10000,
         1000,
         new Predicate<Exception>() {
           @Override
           public boolean evaluate() throws Exception {
             return regionServer.isAborted();
           }
         });
   } finally {
     TEST_UTIL.shutdownMiniCluster();
   }
 }
  @Test(timeout = 60000)
  public void testDeleteDeletedTable() throws Exception {
    final TableName tableName = TableName.valueOf("testDeleteDeletedTable");
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table (that exists)
    long procId1 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
    // delete the table (that will no longer exist)
    long procId2 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName),
            nonceGroup + 1,
            nonce + 1);

    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId1);
    ProcedureTestingUtility.waitProcedure(procExec, procId2);

    // First delete should succeed
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");

    // Second delete should fail with TableNotFound
    ProcedureResult result = procExec.getResult(procId2);
    assertTrue(result.isFailed());
    LOG.debug("Delete failed with exception: " + result.getException());
    assertTrue(result.getException().getCause() instanceof TableNotFoundException);
  }
  /**
   * This test assumes a master crash/failure during the meta migration process and attempts to
   * continue the meta migration process when a new master takes over. When a master dies during the
   * meta migration we will have some rows of META.CatalogFamily updated with PB serialization and
   * some still hanging with writable serialization. When the backup master/ or fresh start of
   * master attempts the migration it will encounter some rows of META already updated with new HRI
   * and some still legacy. This test will simulate this scenario and validates that the migration
   * process can safely skip the updated rows and migrate any pending rows at startup.
   *
   * @throws Exception
   */
  @Test
  public void testMasterCrashDuringMetaMigration() throws Exception {
    final byte[] FAMILY = Bytes.toBytes("family");
    HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration");
    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
    htd.addFamily(hcd);
    Configuration conf = TEST_UTIL.getConfiguration();
    // Create 10 New regions.
    createMultiRegionsWithPBSerialization(conf, htd.getName(), 10);
    // Create 10 Legacy regions.
    createMultiRegionsWithWritableSerialization(conf, htd.getName(), 10);
    CatalogTracker ct = TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
    // Erase the current version of root meta for this test.
    undoVersionInRoot(ct);

    MetaReader.fullScanMetaAndPrint(ct);
    LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");

    long numMigratedRows =
        MetaMigrationConvertingToPB.updateMetaIfNecessary(TEST_UTIL.getHBaseCluster().getMaster());
    assertEquals(numMigratedRows, 10);

    // Assert that the flag in ROOT is updated to reflect the correct status
    boolean metaUpdated =
        MetaMigrationConvertingToPB.isMetaTableUpdated(
            TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker());
    assertEquals(true, metaUpdated);

    verifyMetaRowsAreUpdated(ct);

    LOG.info("END testMasterCrashDuringMetaMigration");
  }
  @Test(timeout = 60000)
  public void testRecoveryAndDoubleExecution() throws Exception {
    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");

    // create the table
    byte[][] splitKeys = null;
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(
            getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
    UTIL.getHBaseAdmin().disableTable(tableName);

    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    // Start the Delete procedure && kill the executor
    long procId =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);

    // Restart the executor and execute the step twice
    // NOTE: the 6 (number of DeleteTableState steps) is hardcoded,
    //       so you have to look at this test at least once when you add a new step.
    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
        procExec, procId, 6, DeleteTableState.values());

    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
  }
  @Test(timeout = 60000)
  public void testDoubleDeletedTableWithSameNonce() throws Exception {
    final TableName tableName = TableName.valueOf("testDoubleDeletedTableWithSameNonce");
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table (that exists)
    long procId1 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
    // delete the table (that will no longer exist)
    long procId2 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);

    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId1);
    ProcedureTestingUtility.waitProcedure(procExec, procId2);

    // First delete should succeed
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");

    // Second delete should not fail, because it is the same delete
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
    assertTrue(procId1 == procId2);
  }
  @Test
  public void testMetaMigration() throws Exception {
    LOG.info("Starting testMetaMigration");
    final byte[] FAMILY = Bytes.toBytes("family");
    HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
    htd.addFamily(hcd);
    Configuration conf = TEST_UTIL.getConfiguration();
    byte[][] regionNames =
        new byte[][] {
          HConstants.EMPTY_START_ROW, Bytes.toBytes("region_a"), Bytes.toBytes("region_b")
        };
    createMultiRegionsWithWritableSerialization(conf, htd.getName(), regionNames);
    CatalogTracker ct = TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
    // Erase the current version of root meta for this test.
    undoVersionInRoot(ct);
    MetaReader.fullScanMetaAndPrint(ct);
    LOG.info("Meta Print completed.testMetaMigration");

    long numMigratedRows =
        MetaMigrationConvertingToPB.updateMeta(TEST_UTIL.getHBaseCluster().getMaster());
    MetaReader.fullScanMetaAndPrint(ct);

    // Should be one entry only and it should be for the table we just added.
    assertEquals(regionNames.length, numMigratedRows);

    // Assert that the flag in ROOT is updated to reflect the correct status
    boolean metaUpdated =
        MetaMigrationConvertingToPB.isMetaTableUpdated(
            TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker());
    assertEquals(true, metaUpdated);
    verifyMetaRowsAreUpdated(ct);
  }
 @BeforeClass
 public static void beforeAllTests() throws Exception {
   // Start a cluster
   TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
   MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
   master = cluster.getMaster();
   master.balanceSwitch(false);
   admin = TEST_UTIL.getHBaseAdmin();
 }
  /*
   * Take a snapshot of a table, add data, and verify that deleting the snapshot does not affect
   * either table.
   * @param online - Whether the table is online or not during the snapshot
   */
  private void runTestSnapshotDeleteIndependent(boolean online) throws Exception {
    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();

    final Admin admin = UTIL.getHBaseAdmin();
    final long startTime = System.currentTimeMillis();
    final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);

    try (Table original = UTIL.createTable(localTableName, TEST_FAM)) {
      UTIL.loadTable(original, TEST_FAM);
    }

    // Take a snapshot
    final String snapshotNameAsString = "snapshot_" + localTableName;
    byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);

    SnapshotTestingUtils.createSnapshotAndValidate(
        admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);

    if (!online) {
      admin.enableTable(localTableName);
    }
    TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
    admin.cloneSnapshot(snapshotName, cloneTableName);

    // Ensure the original table does not reference the HFiles anymore
    admin.majorCompact(localTableName);

    // Deleting the snapshot used to break the cloned table by deleting in-use HFiles
    admin.deleteSnapshot(snapshotName);

    // Wait for cleaner run and DFS heartbeats so that anything that is deletable is fully deleted
    Thread.sleep(10000);

    try (Table original = UTIL.getConnection().getTable(localTableName)) {
      try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
        // Verify that all regions of both tables are readable
        final int origTableRowCount = UTIL.countRows(original);
        final int clonedTableRowCount = UTIL.countRows(clonedTable);
        Assert.assertEquals(origTableRowCount, clonedTableRowCount);
      }
    }
  }
Example #17
0
 @BeforeClass
 public static void startCluster() throws Exception {
   LOG.info("Starting cluster");
   TEST_UTIL = new HBaseTestingUtility();
   TEST_UTIL.startMiniCluster(1, 1, 1, null, MyMaster.class, null);
   cluster = TEST_UTIL.getHBaseCluster();
   LOG.info("Waiting for active/ready master");
   cluster.waitForActiveAndReadyMaster();
   master = cluster.getMaster();
 }
 @Test
 public void test() throws IOException, InterruptedException {
   testUtil
       .getHBaseAdmin()
       .createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()).build());
   Table table = testUtil.createTable(tableName, families);
   table.put(
       new Put(Bytes.toBytes("k")).addColumn(family, Bytes.toBytes("q"), Bytes.toBytes("v")));
   MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
   List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
   Region region = null;
   for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
     HRegionServer hrs = rsts.get(i).getRegionServer();
     for (Region r : hrs.getOnlineRegions(tableName)) {
       region = r;
       break;
     }
   }
   assertNotNull(region);
   Thread.sleep(2000);
   RegionStoreSequenceIds ids =
       testUtil
           .getHBaseCluster()
           .getMaster()
           .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
   assertEquals(HConstants.NO_SEQNUM, ids.getLastFlushedSequenceId());
   // This will be the sequenceid just before that of the earliest edit in memstore.
   long storeSequenceId = ids.getStoreSequenceId(0).getSequenceId();
   assertTrue(storeSequenceId > 0);
   testUtil.getHBaseAdmin().flush(tableName);
   Thread.sleep(2000);
   ids =
       testUtil
           .getHBaseCluster()
           .getMaster()
           .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
   assertTrue(
       ids.getLastFlushedSequenceId() + " > " + storeSequenceId,
       ids.getLastFlushedSequenceId() > storeSequenceId);
   assertEquals(ids.getLastFlushedSequenceId(), ids.getStoreSequenceId(0).getSequenceId());
   table.close();
 }
Example #19
0
 private void deleteTable(final TableName tableName) throws Exception {
   // NOTE: We need a latch because admin is not sync,
   // so the postOp coprocessor method may be called after the admin operation returned.
   MasterSyncObserver observer =
       (MasterSyncObserver)
           UTIL.getHBaseCluster()
               .getMaster()
               .getMasterCoprocessorHost()
               .findCoprocessor(MasterSyncObserver.class.getName());
   ADMIN.deleteTable(tableName);
   observer.tableDeletionLatch.await();
 }
Example #20
0
 @Test
 public void testMoveRegionWhenNotInitialized() {
   MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
   HMaster m = cluster.getMaster();
   try {
     m.setInitialized(false); // fake it, set back later
     HRegionInfo meta = HRegionInfo.FIRST_META_REGIONINFO;
     m.move(meta.getEncodedNameAsBytes(), null);
     fail("Region should not be moved since master is not initialized");
   } catch (IOException ioe) {
     assertTrue(ioe instanceof PleaseHoldException);
   } finally {
     m.setInitialized(true);
   }
 }
Example #21
0
 @After
 public void cleanup() throws Exception, KeeperException {
   for (HTableDescriptor table : ADMIN.listTables()) {
     ADMIN.disableTable(table.getTableName());
     deleteTable(table.getTableName());
   }
   for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
     if (ns.getName().startsWith(prefix)) {
       ADMIN.deleteNamespace(ns.getName());
     }
   }
   assertTrue(
       "Quota manager not enabled",
       UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().isQuotaEnabled());
 }
  private void testSimpleDelete(final TableName tableName, byte[][] splitKeys) throws Exception {
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(
            getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    long procId =
        ProcedureTestingUtility.submitAndWait(
            procExec, new DeleteTableProcedure(procExec.getEnvironment(), tableName));
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
  }
  public static void stopMasterAndAssignMeta(HBaseTestingUtility HTU)
      throws IOException, InterruptedException {
    // Stop master
    HMaster master = HTU.getHBaseCluster().getMaster();
    ServerName masterAddr = master.getServerName();
    master.stopMaster();

    Log.info("Waiting until master thread exits");
    while (HTU.getHBaseCluster().getMasterThread() != null
        && HTU.getHBaseCluster().getMasterThread().isAlive()) {
      Threads.sleep(100);
    }

    HRegionServer.TEST_SKIP_REPORTING_TRANSITION = true;
    // Master is down, so is the meta. We need to assign it somewhere
    // so that regions can be assigned during the mocking phase.
    HRegionServer hrs = HTU.getHBaseCluster().getLiveRegionServerThreads().get(0).getRegionServer();
    ZooKeeperWatcher zkw = hrs.getZooKeeper();
    MetaTableLocator mtl = new MetaTableLocator();
    ServerName sn = mtl.getMetaRegionLocation(zkw);
    if (sn != null && !masterAddr.equals(sn)) {
      return;
    }

    ProtobufUtil.openRegion(
        hrs.getRSRpcServices(), hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
    while (true) {
      sn = mtl.getMetaRegionLocation(zkw);
      if (sn != null
          && sn.equals(hrs.getServerName())
          && hrs.onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
        break;
      }
      Thread.sleep(100);
    }
  }
Example #24
0
 @Test(timeout = 300000)
 public void testCreateTable() throws IOException {
   HTableDescriptor[] tables = admin.listTables();
   int numTables = tables.length;
   TEST_UTIL.createTable(TableName.valueOf("testCreateTable"), HConstants.CATALOG_FAMILY).close();
   tables = this.admin.listTables();
   assertEquals(numTables + 1, tables.length);
   assertTrue(
       "Table must be enabled.",
       TEST_UTIL
           .getHBaseCluster()
           .getMaster()
           .getAssignmentManager()
           .getTableStateManager()
           .isTableState(
               TableName.valueOf("testCreateTable"), ZooKeeperProtos.Table.State.ENABLED));
 }
  @BeforeClass
  public static void beforeClass() throws Exception {
    // enable store file refreshing
    HTU.getConfiguration()
        .setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, REFRESH_PERIOD);

    HTU.getConfiguration().setFloat("hbase.regionserver.logroll.multiplier", 0.0001f);
    HTU.getConfiguration().setInt("replication.source.size.capacity", 10240);
    HTU.getConfiguration().setLong("replication.source.sleepforretries", 100);
    HTU.getConfiguration().setInt("hbase.regionserver.maxlogs", 2);
    HTU.getConfiguration().setLong("hbase.master.logcleaner.ttl", 10);
    HTU.getConfiguration().setInt("zookeeper.recovery.retry", 1);
    HTU.getConfiguration().setInt("zookeeper.recovery.retry.intervalmill", 10);

    HTU.startMiniCluster(NB_SERVERS);
    HTU.getHBaseCluster().startMaster();
  }
  /** Atomic bulk load. */
  @Test
  public void testAtomicBulkLoad() throws Exception {
    TableName TABLE_NAME = TableName.valueOf("atomicBulkLoad");

    int millisToRun = 30000;
    int numScanners = 50;

    UTIL.startMiniCluster(1);
    try {
      WAL log = UTIL.getHBaseCluster().getRegionServer(0).getWAL(null);
      FindBulkHBaseListener listener = new FindBulkHBaseListener();
      log.registerWALActionsListener(listener);
      runAtomicBulkloadTest(TABLE_NAME, millisToRun, numScanners);
      assertThat(listener.isFound(), is(true));
    } finally {
      UTIL.shutdownMiniCluster();
    }
  }
  @Test
  public void testWholesomeMerge() throws Exception {
    LOG.info("Starting testWholesomeMerge");
    final TableName tableName = TableName.valueOf("testWholesomeMerge");

    // Create table and load data.
    Table table = createTableAndLoadData(master, tableName);
    // Merge 1st and 2nd region
    mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1, INITIAL_REGION_NUM - 1);

    // Merge 2nd and 3th region
    PairOfSameType<HRegionInfo> mergedRegions =
        mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2, INITIAL_REGION_NUM - 2);

    verifyRowCount(table, ROWSIZE);

    // Randomly choose one of the two merged regions
    HRegionInfo hri =
        RandomUtils.nextBoolean() ? mergedRegions.getFirst() : mergedRegions.getSecond();
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    AssignmentManager am = cluster.getMaster().getAssignmentManager();
    RegionStates regionStates = am.getRegionStates();
    long start = EnvironmentEdgeManager.currentTime();
    while (!regionStates.isRegionInState(hri, State.MERGED)) {
      assertFalse(
          "Timed out in waiting one merged region to be in state MERGED",
          EnvironmentEdgeManager.currentTime() - start > 60000);
      Thread.sleep(500);
    }

    // We should not be able to assign it again
    am.assign(hri, true);
    assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri));
    assertTrue(regionStates.isRegionInState(hri, State.MERGED));

    // We should not be able to unassign it either
    am.unassign(hri, null);
    assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri));
    assertTrue(regionStates.isRegionInState(hri, State.MERGED));

    table.close();
  }
Example #28
0
  @Test(timeout = 60000)
  public void testServerListener() throws IOException, InterruptedException {
    ServerManager serverManager = TEST_UTIL.getHBaseCluster().getMaster().getServerManager();

    DummyServerListener listener = new DummyServerListener();
    serverManager.registerListener(listener);
    try {
      MiniHBaseCluster miniCluster = TEST_UTIL.getMiniHBaseCluster();

      // Start a new Region Server
      miniCluster.startRegionServer();
      listener.awaitModifications(1);
      assertEquals(1, listener.getAddedCount());
      assertEquals(0, listener.getRemovedCount());

      // Start another Region Server
      listener.reset();
      miniCluster.startRegionServer();
      listener.awaitModifications(1);
      assertEquals(1, listener.getAddedCount());
      assertEquals(0, listener.getRemovedCount());

      int nrs = miniCluster.getRegionServerThreads().size();

      // Stop a Region Server
      listener.reset();
      miniCluster.stopRegionServer(nrs - 1);
      listener.awaitModifications(1);
      assertEquals(0, listener.getAddedCount());
      assertEquals(1, listener.getRemovedCount());

      // Stop another Region Server
      listener.reset();
      miniCluster.stopRegionServer(nrs - 2);
      listener.awaitModifications(1);
      assertEquals(0, listener.getAddedCount());
      assertEquals(1, listener.getRemovedCount());
    } finally {
      serverManager.unregisterListener(listener);
    }
  }
Example #29
0
  @Test
  @SuppressWarnings("deprecation")
  public void testMasterOpsWhileSplitting() throws Exception {
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    HMaster m = cluster.getMaster();

    try (Table ht = TEST_UTIL.createTable(TABLENAME, FAMILYNAME)) {
      assertTrue(m.getTableStateManager().isTableState(TABLENAME, TableState.State.ENABLED));
      TEST_UTIL.loadTable(ht, FAMILYNAME, false);
    }

    List<Pair<HRegionInfo, ServerName>> tableRegions =
        MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME);
    LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
    assertEquals(1, tableRegions.size());
    assertArrayEquals(HConstants.EMPTY_START_ROW, tableRegions.get(0).getFirst().getStartKey());
    assertArrayEquals(HConstants.EMPTY_END_ROW, tableRegions.get(0).getFirst().getEndKey());

    // Now trigger a split and stop when the split is in progress
    LOG.info("Splitting table");
    TEST_UTIL.getAdmin().split(TABLENAME);
    LOG.info("Waiting for split result to be about to open");
    RegionStates regionStates = m.getAssignmentManager().getRegionStates();
    while (regionStates.getRegionsOfTable(TABLENAME).size() <= 1) {
      Thread.sleep(100);
    }
    LOG.info("Making sure we can call getTableRegions while opening");
    tableRegions =
        MetaTableAccessor.getTableRegionsAndLocations(m.getConnection(), TABLENAME, false);

    LOG.info("Regions: " + Joiner.on(',').join(tableRegions));
    // We have three regions because one is split-in-progress
    assertEquals(3, tableRegions.size());
    LOG.info("Making sure we can call getTableRegionClosest while opening");
    Pair<HRegionInfo, ServerName> pair = m.getTableRegionForRow(TABLENAME, Bytes.toBytes("cde"));
    LOG.info("Result is: " + pair);
    Pair<HRegionInfo, ServerName> tableRegionFromName =
        MetaTableAccessor.getRegion(m.getConnection(), pair.getFirst().getRegionName());
    assertEquals(tableRegionFromName.getFirst(), pair.getFirst());
  }
  @Test
  public void testAsyncFlushSnapshot() throws Exception {
    HBaseProtos.SnapshotDescription snapshot =
        HBaseProtos.SnapshotDescription.newBuilder()
            .setName("asyncSnapshot")
            .setTable(TABLE_NAME.getNameAsString())
            .setType(HBaseProtos.SnapshotDescription.Type.FLUSH)
            .build();

    // take the snapshot async
    admin.takeSnapshotAsync(
        new SnapshotDescription("asyncSnapshot", TABLE_NAME, SnapshotType.FLUSH));

    // constantly loop, looking for the snapshot to complete
    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
    SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
    LOG.info(" === Async Snapshot Completed ===");
    UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

    // make sure we get the snapshot
    SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
  }