예제 #1
0
  @Test(timeout = 120000)
  public void testChangeTable() throws Exception {
    HTableDescriptor hdt = HTU.createTableDescriptor("testChangeTable");
    hdt.setRegionReplication(NB_SERVERS);
    hdt.addCoprocessor(SlowMeCopro.class.getName());
    Table table = HTU.createTable(hdt, new byte[][] {f}, HTU.getConfiguration());

    // basic test: it should work.
    Put p = new Put(row);
    p.add(f, row, row);
    table.put(p);

    Get g = new Get(row);
    Result r = table.get(g);
    Assert.assertFalse(r.isStale());

    // Add a CF, it should work.
    HTableDescriptor bHdt = HTU.getHBaseAdmin().getTableDescriptor(hdt.getTableName());
    HColumnDescriptor hcd = new HColumnDescriptor(row);
    hdt.addFamily(hcd);
    HTU.getHBaseAdmin().disableTable(hdt.getTableName());
    HTU.getHBaseAdmin().modifyTable(hdt.getTableName(), hdt);
    HTU.getHBaseAdmin().enableTable(hdt.getTableName());
    HTableDescriptor nHdt = HTU.getHBaseAdmin().getTableDescriptor(hdt.getTableName());
    Assert.assertEquals(
        "fams=" + Arrays.toString(nHdt.getColumnFamilies()),
        bHdt.getColumnFamilies().length + 1,
        nHdt.getColumnFamilies().length);

    p = new Put(row);
    p.add(row, row, row);
    table.put(p);

    g = new Get(row);
    r = table.get(g);
    Assert.assertFalse(r.isStale());

    try {
      SlowMeCopro.cdl.set(new CountDownLatch(1));
      g = new Get(row);
      g.setConsistency(Consistency.TIMELINE);
      r = table.get(g);
      Assert.assertTrue(r.isStale());
    } finally {
      SlowMeCopro.cdl.get().countDown();
      SlowMeCopro.sleepTime.set(0);
    }

    Admin admin = HTU.getHBaseAdmin();
    nHdt = admin.getTableDescriptor(hdt.getTableName());
    Assert.assertEquals(
        "fams=" + Arrays.toString(nHdt.getColumnFamilies()),
        bHdt.getColumnFamilies().length + 1,
        nHdt.getColumnFamilies().length);

    admin.disableTable(hdt.getTableName());
    admin.deleteTable(hdt.getTableName());
    admin.close();
  }
  @Test
  public void testTTL() throws Exception {
    TableName tableName = TableName.valueOf("testTTL");
    if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
      TEST_UTIL.deleteTable(tableName);
    }
    HTableDescriptor desc = new HTableDescriptor(tableName);
    HColumnDescriptor hcd = new HColumnDescriptor(F).setMaxVersions(10).setTimeToLive(1);
    desc.addFamily(hcd);
    TEST_UTIL.getHBaseAdmin().createTable(desc);
    Table t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
    long now = EnvironmentEdgeManager.currentTime();
    ManualEnvironmentEdge me = new ManualEnvironmentEdge();
    me.setValue(now);
    EnvironmentEdgeManagerTestHelper.injectEdge(me);
    // 2s in the past
    long ts = now - 2000;
    // Set the TTL override to 3s
    Put p = new Put(R);
    p.setAttribute("ttl", new byte[] {});
    p.add(F, tableName.getName(), Bytes.toBytes(3000L));
    t.put(p);

    p = new Put(R);
    p.add(F, Q, ts, Q);
    t.put(p);
    p = new Put(R);
    p.add(F, Q, ts + 1, Q);
    t.put(p);

    // these two should be expired but for the override
    // (their ts was 2s in the past)
    Get g = new Get(R);
    g.setMaxVersions(10);
    Result r = t.get(g);
    // still there?
    assertEquals(2, r.size());

    TEST_UTIL.flush(tableName);
    TEST_UTIL.compact(tableName, true);

    g = new Get(R);
    g.setMaxVersions(10);
    r = t.get(g);
    // still there?
    assertEquals(2, r.size());

    // roll time forward 2s.
    me.setValue(now + 2000);
    // now verify that data eventually does expire
    g = new Get(R);
    g.setMaxVersions(10);
    r = t.get(g);
    // should be gone now
    assertEquals(0, r.size());
    t.close();
  }
  @Before
  public void setUp() throws Exception {
    TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(namespace).build());
    try (Table table =
        TEST_UTIL.createTable(
            tableName, new String[] {Bytes.toString(TEST_FAMILY), Bytes.toString(TEST_FAMILY_2)})) {
      TEST_UTIL.waitTableEnabled(tableName);

      List<Put> puts = new ArrayList<Put>(5);
      Put put_1 = new Put(TEST_ROW);
      put_1.addColumn(TEST_FAMILY, Q1, value1);

      Put put_2 = new Put(TEST_ROW_2);
      put_2.addColumn(TEST_FAMILY, Q2, value2);

      Put put_3 = new Put(TEST_ROW_3);
      put_3.addColumn(TEST_FAMILY_2, Q1, value1);

      puts.add(put_1);
      puts.add(put_2);
      puts.add(put_3);

      table.put(puts);
    }

    assertEquals(1, AccessControlLists.getTablePermissions(conf, tableName).size());
    try {
      assertEquals(
          1, AccessControlClient.getUserPermissions(connection, tableName.toString()).size());
    } catch (Throwable e) {
      LOG.error("Error during call of AccessControlClient.getUserPermissions. ", e);
    }
    // setupOperations();
  }
예제 #4
0
  @Test(timeout = 60000)
  public void testDoubleDeletedTableWithSameNonce() throws Exception {
    final TableName tableName = TableName.valueOf("testDoubleDeletedTableWithSameNonce");
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table (that exists)
    long procId1 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
    // delete the table (that will no longer exist)
    long procId2 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);

    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId1);
    ProcedureTestingUtility.waitProcedure(procExec, procId2);

    // First delete should succeed
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");

    // Second delete should not fail, because it is the same delete
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
    assertTrue(procId1 == procId2);
  }
예제 #5
0
  @Test(timeout = 30000)
  public void testCreateDeleteTable() throws IOException {
    // Create table then get the single region for our new table.
    HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable");
    hdt.setRegionReplication(NB_SERVERS);
    hdt.addCoprocessor(SlowMeCopro.class.getName());
    Table table = HTU.createTable(hdt, new byte[][] {f}, HTU.getConfiguration());

    Put p = new Put(row);
    p.add(f, row, row);
    table.put(p);

    Get g = new Get(row);
    Result r = table.get(g);
    Assert.assertFalse(r.isStale());

    try {
      // But if we ask for stale we will get it
      SlowMeCopro.cdl.set(new CountDownLatch(1));
      g = new Get(row);
      g.setConsistency(Consistency.TIMELINE);
      r = table.get(g);
      Assert.assertTrue(r.isStale());
      SlowMeCopro.cdl.get().countDown();
    } finally {
      SlowMeCopro.cdl.get().countDown();
      SlowMeCopro.sleepTime.set(0);
    }

    HTU.getHBaseAdmin().disableTable(hdt.getTableName());
    HTU.deleteTable(hdt.getTableName());
  }
  @Test
  public void testPreWALRestoreSkip() throws Exception {
    LOG.info(TestRegionObserverInterface.class.getName() + ".testPreWALRestoreSkip");
    TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED);
    HTable table = util.createTable(tableName, new byte[][] {A, B, C});

    JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
    ServerName sn2 = rs1.getRegionServer().getServerName();
    String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();

    util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
    while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
      Thread.sleep(100);
    }

    Put put = new Put(ROW);
    put.add(A, A, A);
    put.add(B, B, B);
    put.add(C, C, C);
    table.put(put);
    table.flushCommits();

    cluster.killRegionServer(rs1.getRegionServer().getServerName());
    Threads.sleep(20000); // just to be sure that the kill has fully started.
    util.waitUntilAllRegionsAssigned(tableName);

    verifyMethodResult(
        SimpleRegionObserver.class,
        new String[] {"getCtPreWALRestore", "getCtPostWALRestore"},
        tableName,
        new Integer[] {0, 0});

    util.deleteTable(tableName);
    table.close();
  }
예제 #7
0
  @Test(timeout = 60000)
  public void testRecoveryAndDoubleExecution() throws Exception {
    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");

    // create the table
    byte[][] splitKeys = null;
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(
            getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
    UTIL.getHBaseAdmin().disableTable(tableName);

    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    // Start the Delete procedure && kill the executor
    long procId =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);

    // Restart the executor and execute the step twice
    // NOTE: the 6 (number of DeleteTableState steps) is hardcoded,
    //       so you have to look at this test at least once when you add a new step.
    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
        procExec, procId, 6, DeleteTableState.values());

    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
  }
예제 #8
0
  @Test(timeout = 60000)
  public void testDeleteDeletedTable() throws Exception {
    final TableName tableName = TableName.valueOf("testDeleteDeletedTable");
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table (that exists)
    long procId1 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
    // delete the table (that will no longer exist)
    long procId2 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName),
            nonceGroup + 1,
            nonce + 1);

    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId1);
    ProcedureTestingUtility.waitProcedure(procExec, procId2);

    // First delete should succeed
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");

    // Second delete should fail with TableNotFound
    ProcedureResult result = procExec.getResult(procId2);
    assertTrue(result.isFailed());
    LOG.debug("Delete failed with exception: " + result.getException());
    assertTrue(result.getException().getCause() instanceof TableNotFoundException);
  }
 /**
  * Setup a clean table before we start mucking with it.
  *
  * @throws IOException
  * @throws InterruptedException
  * @throws KeeperException
  */
 private Table setupTable(TableName tablename) throws Exception {
   HTableDescriptor desc = new HTableDescriptor(tablename);
   HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
   desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
   TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
   return this.connection.getTable(tablename);
 }
예제 #10
0
 @After
 public void tearDown() throws Exception {
   ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
   for (HTableDescriptor htd : UTIL.getHBaseAdmin().listTables()) {
     LOG.info("Tear down, remove table=" + htd.getTableName());
     UTIL.deleteTable(htd.getTableName());
   }
 }
  private void runTest(
      String testName,
      HTableDescriptor htd,
      BloomType bloomType,
      boolean preCreateTable,
      byte[][] tableSplitKeys,
      byte[][][] hfileRanges)
      throws Exception {
    Path dir = util.getDataTestDirOnTestFS(testName);
    FileSystem fs = util.getTestFileSystem();
    dir = dir.makeQualified(fs);
    Path familyDir = new Path(dir, Bytes.toString(FAMILY));

    int hfileIdx = 0;
    for (byte[][] range : hfileRanges) {
      byte[] from = range[0];
      byte[] to = range[1];
      HFileTestUtil.createHFile(
          util.getConfiguration(),
          fs,
          new Path(familyDir, "hfile_" + hfileIdx++),
          FAMILY,
          QUALIFIER,
          from,
          to,
          1000);
    }
    int expectedRows = hfileIdx * 1000;

    if (preCreateTable) {
      util.getHBaseAdmin().createTable(htd, tableSplitKeys);
    }

    final TableName tableName = htd.getTableName();
    LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration());
    String[] args = {dir.toString(), tableName.toString()};
    loader.run(args);

    Table table = new HTable(util.getConfiguration(), tableName);
    try {
      assertEquals(expectedRows, util.countRows(table));
    } finally {
      table.close();
    }

    // verify staging folder has been cleaned up
    Path stagingBasePath = SecureBulkLoadUtil.getBaseStagingDir(util.getConfiguration());
    if (fs.exists(stagingBasePath)) {
      FileStatus[] files = fs.listStatus(stagingBasePath);
      for (FileStatus file : files) {
        assertTrue(
            "Folder=" + file.getPath() + " is not cleaned up.",
            file.getPath().getName() != "DONOTERASE");
      }
    }

    util.deleteTable(tableName);
  }
 @BeforeClass
 public static void beforeAllTests() throws Exception {
   // Start a cluster
   TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
   MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
   master = cluster.getMaster();
   master.balanceSwitch(false);
   admin = TEST_UTIL.getHBaseAdmin();
 }
 @After
 public void tearDown() throws Exception {
   // Clean the _acl_ table
   TEST_UTIL.deleteTable(tableName);
   TEST_UTIL.getHBaseAdmin().deleteNamespace(namespace);
   // Verify all table/namespace permissions are erased
   assertEquals(0, AccessControlLists.getTablePermissions(conf, tableName).size());
   assertEquals(0, AccessControlLists.getNamespacePermissions(conf, namespace).size());
 }
  @Test
  public void testRecovery() throws Exception {
    LOG.info(TestRegionObserverInterface.class.getName() + ".testRecovery");
    TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRecovery");
    HTable table = util.createTable(tableName, new byte[][] {A, B, C});
    try {
      JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
      ServerName sn2 = rs1.getRegionServer().getServerName();
      String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();

      util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
      while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
        Thread.sleep(100);
      }

      Put put = new Put(ROW);
      put.add(A, A, A);
      put.add(B, B, B);
      put.add(C, C, C);
      table.put(put);

      verifyMethodResult(
          SimpleRegionObserver.class,
          new String[] {
            "hadPreGet",
            "hadPostGet",
            "hadPrePut",
            "hadPostPut",
            "hadPreBatchMutate",
            "hadPostBatchMutate",
            "hadDelete"
          },
          tableName,
          new Boolean[] {false, false, true, true, true, true, false});

      verifyMethodResult(
          SimpleRegionObserver.class,
          new String[] {"getCtPreWALRestore", "getCtPostWALRestore", "getCtPrePut", "getCtPostPut"},
          tableName,
          new Integer[] {0, 0, 1, 1});

      cluster.killRegionServer(rs1.getRegionServer().getServerName());
      Threads.sleep(1000); // Let the kill soak in.
      util.waitUntilAllRegionsAssigned(tableName);
      LOG.info("All regions assigned");

      verifyMethodResult(
          SimpleRegionObserver.class,
          new String[] {"getCtPrePut", "getCtPostPut"},
          tableName,
          new Integer[] {0, 0});
    } finally {
      util.deleteTable(tableName);
      table.close();
    }
  }
 @BeforeClass
 public static void setupBeforeClass() throws Exception {
   testUtil = new HBaseTestingUtility();
   testUtil.startMiniCluster();
   conf = testUtil.getConfiguration();
   hBaseAdmin = testUtil.getHBaseAdmin();
   txStateStorage = new InMemoryTransactionStateStorage();
   txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
   txManager.startAndWait();
 }
 @Test
 public void test() throws IOException, InterruptedException {
   testUtil
       .getHBaseAdmin()
       .createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()).build());
   Table table = testUtil.createTable(tableName, families);
   table.put(
       new Put(Bytes.toBytes("k")).addColumn(family, Bytes.toBytes("q"), Bytes.toBytes("v")));
   MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
   List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
   Region region = null;
   for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
     HRegionServer hrs = rsts.get(i).getRegionServer();
     for (Region r : hrs.getOnlineRegions(tableName)) {
       region = r;
       break;
     }
   }
   assertNotNull(region);
   Thread.sleep(2000);
   RegionStoreSequenceIds ids =
       testUtil
           .getHBaseCluster()
           .getMaster()
           .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
   assertEquals(HConstants.NO_SEQNUM, ids.getLastFlushedSequenceId());
   // This will be the sequenceid just before that of the earliest edit in memstore.
   long storeSequenceId = ids.getStoreSequenceId(0).getSequenceId();
   assertTrue(storeSequenceId > 0);
   testUtil.getHBaseAdmin().flush(tableName);
   Thread.sleep(2000);
   ids =
       testUtil
           .getHBaseCluster()
           .getMaster()
           .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
   assertTrue(
       ids.getLastFlushedSequenceId() + " > " + storeSequenceId,
       ids.getLastFlushedSequenceId() > storeSequenceId);
   assertEquals(ids.getLastFlushedSequenceId(), ids.getStoreSequenceId(0).getSequenceId());
   table.close();
 }
  /*
   * Take a snapshot of a table, add metadata, and verify that this only
   * affects one table
   * @param online - Whether the table is online or not during the snapshot
   */
  private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();

    // Create a table
    Admin admin = UTIL.getHBaseAdmin();
    final long startTime = System.currentTimeMillis();
    final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
    Table original = UTIL.createTable(localTableName, TEST_FAM);
    UTIL.loadTable(original, TEST_FAM);

    final String snapshotNameAsString = "snapshot_" + localTableName;

    // Create a snapshot
    SnapshotTestingUtils.createSnapshotAndValidate(
        admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);

    if (!online) {
      admin.enableTable(localTableName);
    }
    TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);

    // Clone the snapshot
    byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
    admin.cloneSnapshot(snapshotName, cloneTableName);

    // Add a new column family to the original table
    byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
    HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);

    admin.disableTable(localTableName);
    admin.addColumnFamily(localTableName, hcd);

    // Verify that it is not in the snapshot
    admin.enableTable(localTableName);

    // get a description of the cloned table
    // get a list of its families
    // assert that the family is there
    HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
    HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);

    Assert.assertTrue(
        "The original family was not found. There is something wrong. ",
        originalTableDescriptor.hasFamily(TEST_FAM));
    Assert.assertTrue(
        "The original family was not found in the clone. There is something wrong. ",
        clonedTableDescriptor.hasFamily(TEST_FAM));

    Assert.assertTrue(
        "The new family was not found. ", originalTableDescriptor.hasFamily(TEST_FAM_2));
    Assert.assertTrue(
        "The new family was not found. ", !clonedTableDescriptor.hasFamily(TEST_FAM_2));
  }
예제 #18
0
  @Before
  public void setUpBefore() throws Exception {
    TEST_UTIL = new HBaseTestingUtility();
    TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
    TEST_UTIL.startMiniCluster(3);
    conf = TEST_UTIL.getConfiguration();
    this.connection = ConnectionFactory.createConnection(conf);
    assertEquals(0, TEST_UTIL.getHBaseAdmin().listTables().length);

    // setup the table
    table = TableName.valueOf(TABLE_BASE + "-" + tableIdx);
    tableIdx++;
    htbl = setupTable(table);
    populateTable(htbl);
    assertEquals(5, scanMeta());
    LOG.info("Table " + table + " has " + tableRowCount(conf, table) + " entries.");
    assertEquals(16, tableRowCount(conf, table));
    TEST_UTIL.getHBaseAdmin().disableTable(table);
    assertEquals(1, TEST_UTIL.getHBaseAdmin().listTables().length);
  }
 private PairOfSameType<HRegionInfo> requestMergeRegion(
     HMaster master, TableName tablename, int regionAnum, int regionBnum) throws Exception {
   List<Pair<HRegionInfo, ServerName>> tableRegions =
       MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename);
   HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
   HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
   TEST_UTIL
       .getHBaseAdmin()
       .mergeRegions(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false);
   return new PairOfSameType<HRegionInfo>(regionA, regionB);
 }
예제 #20
0
 /** Create the Mob Table. */
 public static void createMobTable(
     final HBaseTestingUtility util,
     final TableName tableName,
     int regionReplication,
     final byte[]... families)
     throws IOException, InterruptedException {
   HTableDescriptor htd = new HTableDescriptor(tableName);
   htd.setRegionReplication(regionReplication);
   for (byte[] family : families) {
     HColumnDescriptor hcd = new HColumnDescriptor(family);
     hcd.setMobEnabled(true);
     hcd.setMobThreshold(0L);
     htd.addFamily(hcd);
   }
   byte[][] splitKeys = SnapshotTestingUtils.getSplitKeys();
   util.getHBaseAdmin().createTable(htd, splitKeys);
   SnapshotTestingUtils.waitForTableToBeOnline(util, tableName);
   assertEquals(
       (splitKeys.length + 1) * regionReplication,
       util.getHBaseAdmin().getTableRegions(tableName).size());
 }
 private static void initialize(Configuration conf) {
   conf = HBaseConfiguration.create(conf);
   conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
   try {
     admin = TEST_UTIL.getHBaseAdmin();
   } catch (MasterNotRunningException e) {
     assertNull("Master is not running", e);
   } catch (ZooKeeperConnectionException e) {
     assertNull("Cannot connect to ZooKeeper", e);
   } catch (IOException e) {
     assertNull("IOException", e);
   }
 }
 @Test
 public void testCreateInstance_success_shouldNotCreateHBaseNamespace() throws Exception {
   // given
   HBaseAdmin hBaseAdmin = utility.getHBaseAdmin();
   int numberOfNamespacesBeforeServiceCreation = hBaseAdmin.listNamespaceDescriptors().length;
   // when
   ServiceInstance instance = getServiceInstance("instanceId", "fake-bare-plan");
   instanceService.createServiceInstance(getCreateInstanceRequest(instance));
   // then
   int numberOfNamespacesAfterServiceCreation = hBaseAdmin.listNamespaceDescriptors().length;
   assertThat(
       numberOfNamespacesBeforeServiceCreation, equalTo(numberOfNamespacesAfterServiceCreation));
 }
  /*
   * Take a snapshot of a table, do a split, and verify that this only affects one table
   * @param online - Whether the table is online or not during the snapshot
   */
  private void runTestRegionOperationsIndependent(boolean online) throws Exception {
    FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();

    // Create a table
    Admin admin = UTIL.getHBaseAdmin();
    final long startTime = System.currentTimeMillis();
    final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
    Table original = UTIL.createTable(localTableName, TEST_FAM);
    UTIL.loadTable(original, TEST_FAM);
    final int loadedTableCount = UTIL.countRows(original);
    System.out.println("Original table has: " + loadedTableCount + " rows");

    final String snapshotNameAsString = "snapshot_" + localTableName;

    // Create a snapshot
    SnapshotTestingUtils.createSnapshotAndValidate(
        admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);

    if (!online) {
      admin.enableTable(localTableName);
    }

    TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);

    // Clone the snapshot
    byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
    admin.cloneSnapshot(snapshotName, cloneTableName);

    // Verify that region information is the same pre-split
    ((ClusterConnection) UTIL.getConnection()).clearRegionCache();
    List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);

    final int originalRegionCount = originalTableHRegions.size();
    final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
    Assert.assertEquals(
        "The number of regions in the cloned table is different than in the original table.",
        originalRegionCount,
        cloneTableRegionCount);

    // Split a region on the parent table
    admin.splitRegion(originalTableHRegions.get(0).getRegionName());
    waitOnSplit(UTIL.getConnection(), original, originalRegionCount);

    // Verify that the cloned table region is not split
    final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
    Assert.assertEquals(
        "The number of regions in the cloned table changed though none of its regions were split.",
        cloneTableRegionCount,
        cloneTableRegionCount2);
  }
 @Test
 public void testTableExists() throws IOException {
   final String name = "testTableExists";
   final byte[] nameBytes = Bytes.toBytes(name);
   assertFalse(MetaReader.tableExists(ct, name));
   UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
   assertTrue(MetaReader.tableExists(ct, name));
   HBaseAdmin admin = UTIL.getHBaseAdmin();
   admin.disableTable(name);
   admin.deleteTable(name);
   assertFalse(MetaReader.tableExists(ct, name));
   assertTrue(MetaReader.tableExists(ct, Bytes.toString(HConstants.META_TABLE_NAME)));
   assertTrue(MetaReader.tableExists(ct, Bytes.toString(HConstants.ROOT_TABLE_NAME)));
 }
예제 #25
0
  /**
   * Creates a table with given table name and specified number of column families if the table does
   * not already exist.
   */
  private void setupTable(TableName table, int cfs) throws IOException {
    try {
      LOG.info("Creating table " + table);
      HTableDescriptor htd = new HTableDescriptor(table);
      htd.addCoprocessor(MyObserver.class.getName());
      MyObserver.sleepDuration = this.sleepDuration;
      for (int i = 0; i < 10; i++) {
        htd.addFamily(new HColumnDescriptor(family(i)));
      }

      UTIL.getHBaseAdmin().createTable(htd);
    } catch (TableExistsException tee) {
      LOG.info("Table " + table + " already exists");
    }
  }
예제 #26
0
  private void testSimpleDelete(final TableName tableName, byte[][] splitKeys) throws Exception {
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(
            getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    long procId =
        ProcedureTestingUtility.submitAndWait(
            procExec, new DeleteTableProcedure(procExec.getEnvironment(), tableName));
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
  }
  @Test
  public void testBaseCases() throws Exception {
    TableName tableName = TableName.valueOf("baseCases");
    if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
      TEST_UTIL.deleteTable(tableName);
    }
    Table t = TEST_UTIL.createTable(tableName, F, 1);
    // set the version override to 2
    Put p = new Put(R);
    p.setAttribute("versions", new byte[] {});
    p.add(F, tableName.getName(), Bytes.toBytes(2));
    t.put(p);

    long now = EnvironmentEdgeManager.currentTime();

    // insert 2 versions
    p = new Put(R);
    p.add(F, Q, now, Q);
    t.put(p);
    p = new Put(R);
    p.add(F, Q, now + 1, Q);
    t.put(p);
    Get g = new Get(R);
    g.setMaxVersions(10);
    Result r = t.get(g);
    assertEquals(2, r.size());

    TEST_UTIL.flush(tableName);
    TEST_UTIL.compact(tableName, true);

    // both version are still visible even after a flush/compaction
    g = new Get(R);
    g.setMaxVersions(10);
    r = t.get(g);
    assertEquals(2, r.size());

    // insert a 3rd version
    p = new Put(R);
    p.add(F, Q, now + 2, Q);
    t.put(p);
    g = new Get(R);
    g.setMaxVersions(10);
    r = t.get(g);
    // still only two version visible
    assertEquals(2, r.size());

    t.close();
  }
예제 #28
0
  @BeforeClass
  public static void setUpOnce() throws Exception {
    miniCluster = System.getProperty("cluster.type").equals("mini");
    securedCluster = System.getProperty("cluster.secured").equals("true");
    System.out.println("realCluster - " + !miniCluster);
    System.out.println("securedCluster - " + securedCluster);

    Util.setLoggingThreshold("ERROR");

    if (miniCluster) {
      if (hbase == null) {
        hbase = new HBaseTestingUtility();
        conf = hbase.getConfiguration();
        conf.set("zookeeper.session.timeout", "3600000");
        conf.set("dfs.client.socket-timeout", "3600000");

        if (securedCluster) {
          hbase.startMiniCluster(RS_COUNT);
          hbase.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME, 30000L);
          admin = new HBaseAdminWrapper(conf);
        } else {
          hbase.startMiniCluster(RS_COUNT);
          admin = hbase.getHBaseAdmin();
        }
      }
    } else {
      if (admin == null) {
        final String argsFileName =
            securedCluster
                ? "../../testClusterRealSecured.args"
                : "../../testClusterRealNonSecured.args";
        if (!Util.isFile(argsFileName)) {
          throw new IllegalStateException(
              "You have to define args file " + argsFileName + " for tests.");
        }

        String[] testArgs = {argsFileName};
        Args args = new TestArgs(testArgs);
        admin = HBaseClient.getAdmin(args);
        conf = admin.getConfiguration();
        RS_COUNT = getServerNameList().size();
      }
    }
    previousBalancerRunning = admin.setBalancerRunning(false, true);
    hConnection = HConnectionManager.createConnection(conf);

    USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
  }
예제 #29
0
  @Test
  public void testTableCreateAndDeletePB() throws IOException, JAXBException {
    String schemaPath = "/" + TABLE2 + "/schema";
    TableSchemaModel model;
    Response response;

    Admin admin = TEST_UTIL.getHBaseAdmin();
    assertFalse(admin.tableExists(TableName.valueOf(TABLE2)));

    // create the table
    model = testTableSchemaModel.buildTestModel(TABLE2);
    testTableSchemaModel.checkModel(model, TABLE2);
    response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
    assertEquals(response.getCode(), 201);

    // recall the same put operation but in read-only mode
    conf.set("hbase.rest.readonly", "true");
    response = client.put(schemaPath, Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
    assertEquals(response.getCode(), 403);

    // retrieve the schema and validate it
    response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF);
    assertEquals(response.getCode(), 200);
    assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
    model = new TableSchemaModel();
    model.getObjectFromMessage(response.getBody());
    testTableSchemaModel.checkModel(model, TABLE2);

    // retrieve the schema and validate it with alternate pbuf type
    response = client.get(schemaPath, Constants.MIMETYPE_PROTOBUF_IETF);
    assertEquals(response.getCode(), 200);
    assertEquals(Constants.MIMETYPE_PROTOBUF_IETF, response.getHeader("content-type"));
    model = new TableSchemaModel();
    model.getObjectFromMessage(response.getBody());
    testTableSchemaModel.checkModel(model, TABLE2);

    // test delete schema operation is forbidden in read-only mode
    response = client.delete(schemaPath);
    assertEquals(response.getCode(), 403);

    // return read-only setting back to default
    conf.set("hbase.rest.readonly", "false");

    // delete the table and make sure HBase concurs
    response = client.delete(schemaPath);
    assertEquals(response.getCode(), 200);
    assertFalse(admin.tableExists(TableName.valueOf(TABLE2)));
  }
예제 #30
0
  @SuppressWarnings("deprecation")
  @Before
  public void setUp() throws Exception {
    HTableDescriptor desc = new HTableDescriptor(tableName);
    HColumnDescriptor hcd = new HColumnDescriptor(family);
    hcd.setMobEnabled(true);
    hcd.setMobThreshold(3L);
    hcd.setMaxVersions(4);
    desc.addFamily(hcd);

    admin = TEST_UTIL.getHBaseAdmin();
    admin.createTable(desc);
    table =
        ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
            .getBufferedMutator(TableName.valueOf(tableName));
  }