@Test
 public void testDeleteTable() throws Exception {
   String namespace = prefix + "_dummy";
   NamespaceDescriptor nspDesc =
       NamespaceDescriptor.create(namespace)
           .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "100")
           .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "3")
           .build();
   ADMIN.createNamespace(nspDesc);
   assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(namespace));
   NamespaceTableAndRegionInfo stateInfo = getNamespaceState(nspDesc.getName());
   assertNotNull("Namespace state found null for " + namespace, stateInfo);
   HTableDescriptor tableDescOne =
       new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table1"));
   HTableDescriptor tableDescTwo =
       new HTableDescriptor(TableName.valueOf(namespace + TableName.NAMESPACE_DELIM + "table2"));
   ADMIN.createTable(tableDescOne);
   ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
   stateInfo = getNamespaceState(nspDesc.getName());
   assertNotNull("Namespace state found to be null.", stateInfo);
   assertEquals(2, stateInfo.getTables().size());
   assertEquals(5, stateInfo.getRegionCountOfTable(tableDescTwo.getTableName()));
   assertEquals(6, stateInfo.getRegionCount());
   ADMIN.disableTable(tableDescOne.getTableName());
   deleteTable(tableDescOne.getTableName());
   stateInfo = getNamespaceState(nspDesc.getName());
   assertNotNull("Namespace state found to be null.", stateInfo);
   assertEquals(5, stateInfo.getRegionCount());
   assertEquals(1, stateInfo.getTables().size());
   ADMIN.disableTable(tableDescTwo.getTableName());
   deleteTable(tableDescTwo.getTableName());
   ADMIN.deleteNamespace(namespace);
   stateInfo = getNamespaceState(namespace);
   assertNull("Namespace state not found to be null.", stateInfo);
 }
  @Test(timeout = 60000)
  public void testRollbackAndDoubleExecution() throws Exception {
    final NamespaceDescriptor nsd =
        NamespaceDescriptor.create("testRollbackAndDoubleExecution").build();
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    // Start the CreateNamespace procedure && kill the executor
    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));

    int numberOfSteps = 0; // failing at pre operation
    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);

    // Validate the non-existence of namespace
    try {
      NamespaceDescriptor nsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
      assertNull(nsDescriptor);
    } catch (NamespaceNotFoundException nsnfe) {
      // Expected
      LOG.info("The namespace " + nsd.getName() + " is not created.");
    }
  }
  @Override
  protected boolean acquireLock(final MasterProcedureEnv env) {
    if (!env.getMasterServices().isInitialized()) {
      // Namespace manager might not be ready if master is not fully initialized,
      // return false to reject user namespace creation; return true for default
      // and system namespace creation (this is part of master initialization).
      if (nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE)
          || nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE)) {
        return true;
      }

      return false;
    }
    return getTableNamespaceManager(env).acquireExclusiveLock();
  }
 @After
 public void cleanup() throws Exception, KeeperException {
   for (HTableDescriptor table : ADMIN.listTables()) {
     ADMIN.disableTable(table.getTableName());
     deleteTable(table.getTableName());
   }
   for (NamespaceDescriptor ns : ADMIN.listNamespaceDescriptors()) {
     if (ns.getName().startsWith(prefix)) {
       ADMIN.deleteNamespace(ns.getName());
     }
   }
   assertTrue(
       "Quota manager not enabled",
       UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().isQuotaEnabled());
 }
 @Override
 public void toStringClassDetails(StringBuilder sb) {
   sb.append(getClass().getSimpleName());
   sb.append(" (Namespace=");
   sb.append(nsDescriptor.getName());
   sb.append(")");
 }
  @Test(timeout = 60000)
  public void testCreateSameNamespaceTwice() throws Exception {
    final NamespaceDescriptor nsd =
        NamespaceDescriptor.create("testCreateSameNamespaceTwice").build();
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    long procId1 =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId1);
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);

    // Create the namespace that exists
    long procId2 =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId2);

    // Second create should fail with NamespaceExistException
    ProcedureInfo result = procExec.getResult(procId2);
    assertTrue(result.isFailed());
    LOG.debug("Create namespace failed with exception: " + result.getExceptionFullMessage());
    assertTrue(
        ProcedureTestingUtility.getExceptionCause(result) instanceof NamespaceExistException);
  }
  @Before
  public void setUp() throws Exception {
    TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(namespace).build());
    try (Table table =
        TEST_UTIL.createTable(
            tableName, new String[] {Bytes.toString(TEST_FAMILY), Bytes.toString(TEST_FAMILY_2)})) {
      TEST_UTIL.waitTableEnabled(tableName);

      List<Put> puts = new ArrayList<Put>(5);
      Put put_1 = new Put(TEST_ROW);
      put_1.addColumn(TEST_FAMILY, Q1, value1);

      Put put_2 = new Put(TEST_ROW_2);
      put_2.addColumn(TEST_FAMILY, Q2, value2);

      Put put_3 = new Put(TEST_ROW_3);
      put_3.addColumn(TEST_FAMILY_2, Q1, value1);

      puts.add(put_1);
      puts.add(put_2);
      puts.add(put_3);

      table.put(puts);
    }

    assertEquals(1, AccessControlLists.getTablePermissions(conf, tableName).size());
    try {
      assertEquals(
          1, AccessControlClient.getUserPermissions(connection, tableName.toString()).size());
    } catch (Throwable e) {
      LOG.error("Error during call of AccessControlClient.getUserPermissions. ", e);
    }
    // setupOperations();
  }
  @Test(timeout = 180000)
  public void testRestoreSnapshot() throws Exception {
    String nsp = prefix + "_testRestoreSnapshot";
    NamespaceDescriptor nspDesc =
        NamespaceDescriptor.create(nsp)
            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
            .build();
    ADMIN.createNamespace(nspDesc);
    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
    HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);

    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());

    String snapshot = "snapshot_testRestoreSnapshot";
    ADMIN.snapshot(snapshot, tableName1);

    List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
    Collections.sort(regions);

    ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
    Thread.sleep(2000);
    assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());

    ADMIN.disableTable(tableName1);
    ADMIN.restoreSnapshot(snapshot);

    assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount());

    ADMIN.enableTable(tableName1);
    ADMIN.deleteSnapshot(snapshot);
  }
 /**
  * Undo the insert.
  *
  * @param env MasterProcedureEnv
  * @throws IOException
  */
 private void rollbackInsertIntoNSTable(final MasterProcedureEnv env) throws IOException {
   try {
     DeleteNamespaceProcedure.deleteFromNSTable(env, nsDescriptor.getName());
   } catch (Exception e) {
     // Ignore exception
     LOG.debug("Rollback of insertIntoNSTable throws exception: " + e);
   }
 }
 /**
  * undo create directory
  *
  * @param env MasterProcedureEnv
  * @throws IOException
  */
 private void rollbackCreateDirectory(final MasterProcedureEnv env) throws IOException {
   try {
     DeleteNamespaceProcedure.deleteDirectory(env, nsDescriptor.getName());
   } catch (Exception e) {
     // Ignore exception
     LOG.debug("Rollback of createDirectory throws exception: " + e);
   }
 }
 /**
  * remove quota for the namespace if exists
  *
  * @param env MasterProcedureEnv
  * @throws IOException
  */
 private void rollbackSetNamespaceQuota(final MasterProcedureEnv env) throws IOException {
   try {
     DeleteNamespaceProcedure.removeNamespaceQuota(env, nsDescriptor.getName());
   } catch (Exception e) {
     // Ignore exception
     LOG.debug("Rollback of setNamespaceQuota throws exception: " + e);
   }
 }
 /**
  * rollback Zookeeper update.
  *
  * @param env MasterProcedureEnv
  * @throws IOException
  */
 private void rollbackZKNamespaceManagerChange(final MasterProcedureEnv env) throws IOException {
   try {
     DeleteNamespaceProcedure.removeFromZKNamespaceManager(env, nsDescriptor.getName());
   } catch (Exception e) {
     // Ignore exception
     LOG.debug("Rollback of updateZKNamespaceManager throws exception: " + e);
   }
 }
  @Test(timeout = 60000)
  public void testCreateNamespaceWithInvalidTableCount() throws Exception {
    final NamespaceDescriptor nsd =
        NamespaceDescriptor.create("testCreateNamespaceWithInvalidTableCount").build();
    final String nsKey = "hbase.namespace.quota.maxtables";
    final String nsValue = "-1";
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    nsd.setConfiguration(nsKey, nsValue);

    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId);
    ProcedureInfo result = procExec.getResult(procId);
    assertTrue(result.isFailed());
    LOG.debug("Create namespace failed with exception: " + result.getExceptionFullMessage());
    assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof ConstraintException);
  }
Beispiel #14
0
  @Test(timeout = 180000)
  public void testRestoreSnapshotQuotaExceed() throws Exception {
    String nsp = prefix + "_testRestoreSnapshotQuotaExceed";
    NamespaceDescriptor nspDesc =
        NamespaceDescriptor.create(nsp)
            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10")
            .build();
    ADMIN.createNamespace(nspDesc);
    NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp);
    assertNotNull("Namespace descriptor found null.", ndesc);
    TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
    HTableDescriptor tableDescOne = new HTableDescriptor(tableName1);
    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);

    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
    assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount());

    String snapshot = "snapshot_testRestoreSnapshotQuotaExceed";
    ADMIN.snapshot(snapshot, tableName1);

    List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1);
    Collections.sort(regions);

    ADMIN.split(tableName1, Bytes.toBytes("JJJ"));
    Thread.sleep(2000);
    assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount());

    ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2");
    ADMIN.modifyNamespace(ndesc);

    ADMIN.disableTable(tableName1);
    try {
      ADMIN.restoreSnapshot(snapshot);
      fail(
          "Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin"
              + " wraps IOException into RestoreSnapshotException");
    } catch (RestoreSnapshotException ignore) {
    }
    ADMIN.enableTable(tableName1);
    ADMIN.deleteSnapshot(snapshot);
  }
Beispiel #15
0
  /*
   * Create a table and make sure that the table creation fails after adding this table entry into
   * namespace quota cache. Now correct the failure and recreate the table with same name.
   * HBASE-13394
   */
  @Test(timeout = 180000)
  public void testRecreateTableWithSameNameAfterFirstTimeFailure() throws Exception {
    String nsp1 = prefix + "_testRecreateTable";
    NamespaceDescriptor nspDesc =
        NamespaceDescriptor.create(nsp1)
            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
            .build();
    ADMIN.createNamespace(nspDesc);
    final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
    byte[] columnFamily = Bytes.toBytes("info");
    HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
    tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
    MasterSyncObserver.throwExceptionInPreCreateTableHandler = true;
    try {
      try {
        ADMIN.createTable(tableDescOne);
        fail("Table " + tableOne.toString() + "creation should fail.");
      } catch (Exception exp) {
        LOG.error(exp);
      }
      assertFalse(ADMIN.tableExists(tableOne));

      NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp1);
      assertEquals(
          "First table creation failed in namespace so number of tables in namespace "
              + "should be 0.",
          0,
          nstate.getTables().size());

      MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
      try {
        ADMIN.createTable(tableDescOne);
      } catch (Exception e) {
        fail("Table " + tableOne.toString() + "creation should succeed.");
        LOG.error(e);
      }
      assertTrue(ADMIN.tableExists(tableOne));
      nstate = getNamespaceState(nsp1);
      assertEquals(
          "First table was created successfully so table size in namespace should " + "be one now.",
          1,
          nstate.getTables().size());
    } finally {
      MasterSyncObserver.throwExceptionInPreCreateTableHandler = false;
      if (ADMIN.tableExists(tableOne)) {
        ADMIN.disableTable(tableOne);
        deleteTable(tableOne);
      }
      ADMIN.deleteNamespace(nsp1);
    }
  }
  @Test(timeout = 60000)
  public void testCreateNamespace() throws Exception {
    final NamespaceDescriptor nsd = NamespaceDescriptor.create("testCreateNamespace").build();
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId);
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);

    validateNamespaceCreated(nsd);
  }
  @Override
  protected Flow executeFromState(final MasterProcedureEnv env, final CreateNamespaceState state)
      throws InterruptedException {
    if (isTraceEnabled()) {
      LOG.trace(this + " execute state=" + state);
    }

    try {
      switch (state) {
        case CREATE_NAMESPACE_PREPARE:
          prepareCreate(env);
          setNextState(CreateNamespaceState.CREATE_NAMESPACE_CREATE_DIRECTORY);
          break;
        case CREATE_NAMESPACE_CREATE_DIRECTORY:
          createDirectory(env, nsDescriptor);
          setNextState(CreateNamespaceState.CREATE_NAMESPACE_INSERT_INTO_NS_TABLE);
          break;
        case CREATE_NAMESPACE_INSERT_INTO_NS_TABLE:
          insertIntoNSTable(env, nsDescriptor);
          setNextState(CreateNamespaceState.CREATE_NAMESPACE_UPDATE_ZK);
          break;
        case CREATE_NAMESPACE_UPDATE_ZK:
          updateZKNamespaceManager(env, nsDescriptor);
          setNextState(CreateNamespaceState.CREATE_NAMESPACE_SET_NAMESPACE_QUOTA);
          break;
        case CREATE_NAMESPACE_SET_NAMESPACE_QUOTA:
          setNamespaceQuota(env, nsDescriptor);
          return Flow.NO_MORE_STATE;
        default:
          throw new UnsupportedOperationException(this + " unhandled state=" + state);
      }
    } catch (IOException e) {
      if (isRollbackSupported(state)) {
        setFailure("master-create-namespace", e);
      } else {
        LOG.warn(
            "Retriable error trying to create namespace="
                + nsDescriptor.getName()
                + " (in state="
                + state
                + ")",
            e);
      }
    }
    return Flow.HAS_MORE_STATE;
  }
Beispiel #18
0
 @Test(expected = QuotaExceededException.class, timeout = 30000)
 public void testExceedTableQuotaInNamespace() throws Exception {
   String nsp = prefix + "_testExceedTableQuotaInNamespace";
   NamespaceDescriptor nspDesc =
       NamespaceDescriptor.create(nsp)
           .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
           .build();
   ADMIN.createNamespace(nspDesc);
   assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
   assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
   HTableDescriptor tableDescOne =
       new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
   HTableDescriptor tableDescTwo =
       new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
   ADMIN.createTable(tableDescOne);
   ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
 }
Beispiel #19
0
 @Test(expected = QuotaExceededException.class, timeout = 30000)
 public void testCloneSnapshotQuotaExceed() throws Exception {
   String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot";
   NamespaceDescriptor nspDesc =
       NamespaceDescriptor.create(nsp)
           .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1")
           .build();
   ADMIN.createNamespace(nspDesc);
   assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
   TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
   TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
   HTableDescriptor tableDescOne = new HTableDescriptor(tableName);
   ADMIN.createTable(tableDescOne);
   String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot";
   ADMIN.snapshot(snapshot, tableName);
   ADMIN.cloneSnapshot(snapshot, cloneTableName);
   ADMIN.deleteSnapshot(snapshot);
 }
Beispiel #20
0
 @Test(timeout = 60000)
 public void testTableOperations() throws Exception {
   String nsp = prefix + "_np2";
   NamespaceDescriptor nspDesc =
       NamespaceDescriptor.create(nsp)
           .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
           .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
           .build();
   ADMIN.createNamespace(nspDesc);
   assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
   assertEquals(ADMIN.listNamespaceDescriptors().length, 3);
   HTableDescriptor tableDescOne =
       new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
   HTableDescriptor tableDescTwo =
       new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
   HTableDescriptor tableDescThree =
       new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table3"));
   ADMIN.createTable(tableDescOne);
   boolean constraintViolated = false;
   try {
     ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 5);
   } catch (Exception exp) {
     assertTrue(exp instanceof IOException);
     constraintViolated = true;
   } finally {
     assertTrue(
         "Constraint not violated for table " + tableDescTwo.getTableName(), constraintViolated);
   }
   ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
   NamespaceTableAndRegionInfo nspState = getQuotaManager().getState(nsp);
   assertNotNull(nspState);
   assertTrue(nspState.getTables().size() == 2);
   assertTrue(nspState.getRegionCount() == 5);
   constraintViolated = false;
   try {
     ADMIN.createTable(tableDescThree);
   } catch (Exception exp) {
     assertTrue(exp instanceof IOException);
     constraintViolated = true;
   } finally {
     assertTrue(
         "Constraint not violated for table " + tableDescThree.getTableName(), constraintViolated);
   }
 }
 @Test
 public void test() throws IOException, InterruptedException {
   testUtil
       .getHBaseAdmin()
       .createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()).build());
   Table table = testUtil.createTable(tableName, families);
   table.put(
       new Put(Bytes.toBytes("k")).addColumn(family, Bytes.toBytes("q"), Bytes.toBytes("v")));
   MiniHBaseCluster cluster = testUtil.getMiniHBaseCluster();
   List<JVMClusterUtil.RegionServerThread> rsts = cluster.getRegionServerThreads();
   Region region = null;
   for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
     HRegionServer hrs = rsts.get(i).getRegionServer();
     for (Region r : hrs.getOnlineRegions(tableName)) {
       region = r;
       break;
     }
   }
   assertNotNull(region);
   Thread.sleep(2000);
   RegionStoreSequenceIds ids =
       testUtil
           .getHBaseCluster()
           .getMaster()
           .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
   assertEquals(HConstants.NO_SEQNUM, ids.getLastFlushedSequenceId());
   // This will be the sequenceid just before that of the earliest edit in memstore.
   long storeSequenceId = ids.getStoreSequenceId(0).getSequenceId();
   assertTrue(storeSequenceId > 0);
   testUtil.getHBaseAdmin().flush(tableName);
   Thread.sleep(2000);
   ids =
       testUtil
           .getHBaseCluster()
           .getMaster()
           .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes());
   assertTrue(
       ids.getLastFlushedSequenceId() + " > " + storeSequenceId,
       ids.getLastFlushedSequenceId() > storeSequenceId);
   assertEquals(ids.getLastFlushedSequenceId(), ids.getStoreSequenceId(0).getSequenceId());
   table.close();
 }
  @Test(timeout = 60000)
  public void testRecoveryAndDoubleExecution() throws Exception {
    final NamespaceDescriptor nsd =
        NamespaceDescriptor.create("testRecoveryAndDoubleExecution").build();
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    // Start the CreateNamespace procedure && kill the executor
    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));

    // Restart the executor and execute the step twice
    int numberOfSteps = CreateNamespaceState.values().length;
    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps);

    // Validate the creation of namespace
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
    validateNamespaceCreated(nsd);
  }
  public static void main(String[] args) throws IOException, InterruptedException {
    Configuration conf = HBaseConfiguration.create();
    HBaseHelper helper = HBaseHelper.getHelper(conf);
    helper.dropTable("testtable");
    Connection connection = ConnectionFactory.createConnection(conf);
    Admin admin = connection.getAdmin();

    // vv CreateTableWithNamespaceExample
    /*[*/ NamespaceDescriptor namespace = NamespaceDescriptor.create("testspace").build();
    admin.createNamespace(namespace); /*]*/

    TableName tableName = TableName.valueOf("testspace", "testtable");
    HTableDescriptor desc = new HTableDescriptor(tableName);

    HColumnDescriptor coldef = new HColumnDescriptor(Bytes.toBytes("colfam1"));
    desc.addFamily(coldef);

    admin.createTable(desc);
    // ^^ CreateTableWithNamespaceExample

    boolean avail = admin.isTableAvailable(tableName);
    System.out.println("Table available: " + avail);
  }
 @Override
 protected void rollbackState(final MasterProcedureEnv env, final CreateNamespaceState state)
     throws IOException {
   if (isTraceEnabled()) {
     LOG.trace(this + " rollback state=" + state);
   }
   try {
     switch (state) {
       case CREATE_NAMESPACE_SET_NAMESPACE_QUOTA:
         rollbackSetNamespaceQuota(env);
         break;
       case CREATE_NAMESPACE_UPDATE_ZK:
         rollbackZKNamespaceManagerChange(env);
         break;
       case CREATE_NAMESPACE_INSERT_INTO_NS_TABLE:
         rollbackInsertIntoNSTable(env);
         break;
       case CREATE_NAMESPACE_CREATE_DIRECTORY:
         rollbackCreateDirectory(env);
         break;
       case CREATE_NAMESPACE_PREPARE:
         break; // nothing to do
       default:
         throw new UnsupportedOperationException(this + " unhandled state=" + state);
     }
   } catch (IOException e) {
     // This will be retried. Unless there is a bug in the code,
     // this should be just a "temporary error" (e.g. network down)
     LOG.warn(
         "Failed rollback attempt step "
             + state
             + " for creating the namespace "
             + nsDescriptor.getName(),
         e);
     throw e;
   }
 }
Beispiel #25
0
 @Test
 public void testStatePreserve() throws Exception {
   final String nsp1 = prefix + "_testStatePreserve";
   NamespaceDescriptor nspDesc =
       NamespaceDescriptor.create(nsp1)
           .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
           .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "10")
           .build();
   ADMIN.createNamespace(nspDesc);
   TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
   TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
   TableName tableThree = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table3");
   HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
   HTableDescriptor tableDescTwo = new HTableDescriptor(tableTwo);
   HTableDescriptor tableDescThree = new HTableDescriptor(tableThree);
   ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
   ADMIN.createTable(tableDescTwo, Bytes.toBytes("1"), Bytes.toBytes("1000"), 3);
   ADMIN.createTable(tableDescThree, Bytes.toBytes("1"), Bytes.toBytes("1000"), 4);
   ADMIN.disableTable(tableThree);
   deleteTable(tableThree);
   // wait for chore to complete
   UTIL.waitFor(
       1000,
       new Waiter.Predicate<Exception>() {
         @Override
         public boolean evaluate() throws Exception {
           return (getNamespaceState(nsp1).getTables().size() == 2);
         }
       });
   NamespaceTableAndRegionInfo before = getNamespaceState(nsp1);
   restartMaster();
   NamespaceTableAndRegionInfo after = getNamespaceState(nsp1);
   assertEquals(
       "Expected: " + before.getTables() + " Found: " + after.getTables(),
       before.getTables().size(),
       after.getTables().size());
 }
Beispiel #26
0
  @Test(timeout = 180000)
  public void testCloneSnapshot() throws Exception {
    String nsp = prefix + "_testCloneSnapshot";
    NamespaceDescriptor nspDesc =
        NamespaceDescriptor.create(nsp)
            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20")
            .build();
    ADMIN.createNamespace(nspDesc);
    assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
    TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
    TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");
    HTableDescriptor tableDescOne = new HTableDescriptor(tableName);

    ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
    String snapshot = "snapshot_testCloneSnapshot";
    ADMIN.snapshot(snapshot, tableName);
    ADMIN.cloneSnapshot(snapshot, cloneTableName);

    int tableLength;
    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
      tableLength = locator.getStartKeys().length;
    }
    assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);

    try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
      tableLength = locator.getStartKeys().length;
    }
    assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);

    NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
    assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
    assertEquals("Total regions count should be.", 8, nstate.getRegionCount());

    ADMIN.deleteSnapshot(snapshot);
  }
Beispiel #27
0
  @Ignore("Hangs on occasion waiting on countdown latch")
  @Test
  public void testRegionOperations() throws Exception {
    String nsp1 = prefix + "_regiontest";
    NamespaceDescriptor nspDesc =
        NamespaceDescriptor.create(nsp1)
            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2")
            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
            .build();
    ADMIN.createNamespace(nspDesc);
    boolean constraintViolated = false;
    final TableName tableOne = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table1");
    byte[] columnFamily = Bytes.toBytes("info");
    HTableDescriptor tableDescOne = new HTableDescriptor(tableOne);
    tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
    NamespaceTableAndRegionInfo stateInfo;
    try {
      ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("1000"), 7);
    } catch (Exception exp) {
      assertTrue(exp instanceof DoNotRetryIOException);
      LOG.info(exp);
      constraintViolated = true;
    } finally {
      assertTrue(constraintViolated);
    }
    assertFalse(ADMIN.tableExists(tableOne));
    // This call will pass.
    ADMIN.createTable(tableDescOne);
    Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
    Table htable = connection.getTable(tableOne);
    UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000);
    ADMIN.flush(tableOne);
    stateInfo = getNamespaceState(nsp1);
    assertEquals(1, stateInfo.getTables().size());
    assertEquals(1, stateInfo.getRegionCount());
    restartMaster();
    ADMIN.split(tableOne, Bytes.toBytes("500"));
    HRegion actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
    CustomObserver observer =
        (CustomObserver)
            actualRegion.getCoprocessorHost().findCoprocessor(CustomObserver.class.getName());
    assertNotNull(observer);
    observer.postSplit.await();
    assertEquals(2, ADMIN.getTableRegions(tableOne).size());
    actualRegion = UTIL.getHBaseCluster().getRegions(tableOne).get(0);
    observer =
        (CustomObserver)
            actualRegion.getCoprocessorHost().findCoprocessor(CustomObserver.class.getName());
    assertNotNull(observer);
    ADMIN.split(
        tableOne,
        getSplitKey(
            actualRegion.getRegionInfo().getStartKey(), actualRegion.getRegionInfo().getEndKey()));
    observer.postSplit.await();
    // Make sure no regions have been added.
    List<HRegionInfo> hris = ADMIN.getTableRegions(tableOne);
    assertEquals(2, hris.size());
    assertTrue("split completed", observer.preSplitBeforePONR.getCount() == 1);

    htable.close();
  }
 private void validateNamespaceCreated(NamespaceDescriptor nsd) throws IOException {
   NamespaceDescriptor createdNsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
   assertNotNull(createdNsDescriptor);
 }
Beispiel #29
0
  @BeforeClass
  public static void beforeClass() throws Exception {
    conf = UTIL.getConfiguration();
    enableSecurity(conf);

    SUPERUSER = User.createUserForTesting(conf, "admin", new String[] {"supergroup"});
    // Users with global permissions
    USER_GLOBAL_ADMIN = User.createUserForTesting(conf, "global_admin", new String[0]);
    USER_GLOBAL_CREATE = User.createUserForTesting(conf, "global_create", new String[0]);
    USER_GLOBAL_WRITE = User.createUserForTesting(conf, "global_write", new String[0]);
    USER_GLOBAL_READ = User.createUserForTesting(conf, "global_read", new String[0]);
    USER_GLOBAL_EXEC = User.createUserForTesting(conf, "global_exec", new String[0]);

    USER_NS_ADMIN = User.createUserForTesting(conf, "namespace_admin", new String[0]);
    USER_NS_CREATE = User.createUserForTesting(conf, "namespace_create", new String[0]);
    USER_NS_WRITE = User.createUserForTesting(conf, "namespace_write", new String[0]);
    USER_NS_READ = User.createUserForTesting(conf, "namespace_read", new String[0]);
    USER_NS_EXEC = User.createUserForTesting(conf, "namespace_exec", new String[0]);

    USER_TABLE_CREATE = User.createUserForTesting(conf, "table_create", new String[0]);
    USER_TABLE_WRITE = User.createUserForTesting(conf, "table_write", new String[0]);

    USER_GROUP_ADMIN =
        User.createUserForTesting(conf, "user_group_admin", new String[] {GROUP_ADMIN});
    USER_GROUP_NS_ADMIN =
        User.createUserForTesting(conf, "user_group_ns_admin", new String[] {GROUP_NS_ADMIN});
    USER_GROUP_CREATE =
        User.createUserForTesting(conf, "user_group_create", new String[] {GROUP_CREATE});
    USER_GROUP_READ = User.createUserForTesting(conf, "user_group_read", new String[] {GROUP_READ});
    USER_GROUP_WRITE =
        User.createUserForTesting(conf, "user_group_write", new String[] {GROUP_WRITE});
    // TODO: other table perms

    UTIL.startMiniCluster();
    // Wait for the ACL table to become available
    UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME.getName(), 30 * 1000);

    ACCESS_CONTROLLER =
        (AccessController)
            UTIL.getMiniHBaseCluster()
                .getMaster()
                .getRegionServerCoprocessorHost()
                .findCoprocessor(AccessController.class.getName());

    UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(TEST_NAMESPACE).build());
    UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(TEST_NAMESPACE2).build());

    // grants on global
    grantGlobal(UTIL, USER_GLOBAL_ADMIN.getShortName(), Permission.Action.ADMIN);
    grantGlobal(UTIL, USER_GLOBAL_CREATE.getShortName(), Permission.Action.CREATE);
    grantGlobal(UTIL, USER_GLOBAL_WRITE.getShortName(), Permission.Action.WRITE);
    grantGlobal(UTIL, USER_GLOBAL_READ.getShortName(), Permission.Action.READ);
    grantGlobal(UTIL, USER_GLOBAL_EXEC.getShortName(), Permission.Action.EXEC);

    // grants on namespace
    grantOnNamespace(UTIL, USER_NS_ADMIN.getShortName(), TEST_NAMESPACE, Permission.Action.ADMIN);
    grantOnNamespace(UTIL, USER_NS_CREATE.getShortName(), TEST_NAMESPACE, Permission.Action.CREATE);
    grantOnNamespace(UTIL, USER_NS_WRITE.getShortName(), TEST_NAMESPACE, Permission.Action.WRITE);
    grantOnNamespace(UTIL, USER_NS_READ.getShortName(), TEST_NAMESPACE, Permission.Action.READ);
    grantOnNamespace(UTIL, USER_NS_EXEC.getShortName(), TEST_NAMESPACE, Permission.Action.EXEC);
    grantOnNamespace(UTIL, toGroupEntry(GROUP_NS_ADMIN), TEST_NAMESPACE, Permission.Action.ADMIN);

    grantOnNamespace(UTIL, USER_NS_ADMIN.getShortName(), TEST_NAMESPACE2, Permission.Action.ADMIN);

    grantGlobal(UTIL, toGroupEntry(GROUP_ADMIN), Permission.Action.ADMIN);
    grantGlobal(UTIL, toGroupEntry(GROUP_CREATE), Permission.Action.CREATE);
    grantGlobal(UTIL, toGroupEntry(GROUP_READ), Permission.Action.READ);
    grantGlobal(UTIL, toGroupEntry(GROUP_WRITE), Permission.Action.WRITE);
  }
Beispiel #30
0
  @Test
  public void testRegionMerge() throws Exception {
    String nsp1 = prefix + "_regiontest";
    NamespaceDescriptor nspDesc =
        NamespaceDescriptor.create(nsp1)
            .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "3")
            .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
            .build();
    ADMIN.createNamespace(nspDesc);
    final TableName tableTwo = TableName.valueOf(nsp1 + TableName.NAMESPACE_DELIM + "table2");
    byte[] columnFamily = Bytes.toBytes("info");
    HTableDescriptor tableDescOne = new HTableDescriptor(tableTwo);
    tableDescOne.addFamily(new HColumnDescriptor(columnFamily));
    final int initialRegions = 3;
    ADMIN.createTable(tableDescOne, Bytes.toBytes("1"), Bytes.toBytes("2000"), initialRegions);
    Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
    try (Table table = connection.getTable(tableTwo)) {
      UTIL.loadNumericRows(table, Bytes.toBytes("info"), 1000, 1999);
    }
    ADMIN.flush(tableTwo);
    List<HRegionInfo> hris = ADMIN.getTableRegions(tableTwo);
    Collections.sort(hris);
    // merge the two regions
    final Set<String> encodedRegionNamesToMerge =
        Sets.newHashSet(hris.get(0).getEncodedName(), hris.get(1).getEncodedName());
    ADMIN.mergeRegions(
        hris.get(0).getEncodedNameAsBytes(), hris.get(1).getEncodedNameAsBytes(), false);
    UTIL.waitFor(
        10000,
        100,
        new Waiter.ExplainingPredicate<Exception>() {

          @Override
          public boolean evaluate() throws Exception {
            RegionStates regionStates =
                UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
            for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
              if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
                return false;
              }
              if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
                return false;
              }
            }
            return true;
          }

          @Override
          public String explainFailure() throws Exception {
            RegionStates regionStates =
                UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
            for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
              if (encodedRegionNamesToMerge.contains(hri.getEncodedName())) {
                return hri + " which is expected to be merged is still online";
              }
              if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
                return hri + " is still in not opened";
              }
            }
            return "Unknown";
          }
        });
    hris = ADMIN.getTableRegions(tableTwo);
    assertEquals(initialRegions - 1, hris.size());
    Collections.sort(hris);

    final HRegionInfo hriToSplit = hris.get(1);
    ADMIN.split(tableTwo, Bytes.toBytes("500"));

    UTIL.waitFor(
        10000,
        100,
        new Waiter.ExplainingPredicate<Exception>() {

          @Override
          public boolean evaluate() throws Exception {
            RegionStates regionStates =
                UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
            for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
              if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
                return false;
              }
              if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
                return false;
              }
            }
            return true;
          }

          @Override
          public String explainFailure() throws Exception {
            RegionStates regionStates =
                UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
            for (HRegionInfo hri : ADMIN.getTableRegions(tableTwo)) {
              if (hri.getEncodedName().equals(hriToSplit.getEncodedName())) {
                return hriToSplit + " which is expected to be split is still online";
              }
              if (!regionStates.isRegionInState(hri, RegionState.State.OPEN)) {
                return hri + " is still in not opened";
              }
            }
            return "Unknown";
          }
        });
    hris = ADMIN.getTableRegions(tableTwo);
    assertEquals(initialRegions, hris.size());
    Collections.sort(hris);

    // fail region merge through Coprocessor hook
    MiniHBaseCluster cluster = UTIL.getHBaseCluster();
    HRegionServer regionServer = cluster.getRegionServer(0);
    RegionServerCoprocessorHost cpHost = regionServer.getRegionServerCoprocessorHost();
    Coprocessor coprocessor = cpHost.findCoprocessor(CPRegionServerObserver.class.getName());
    CPRegionServerObserver regionServerObserver = (CPRegionServerObserver) coprocessor;
    regionServerObserver.failMerge(true);
    regionServerObserver.triggered = false;

    ADMIN.mergeRegions(
        hris.get(1).getEncodedNameAsBytes(), hris.get(2).getEncodedNameAsBytes(), false);
    regionServerObserver.waitUtilTriggered();
    hris = ADMIN.getTableRegions(tableTwo);
    assertEquals(initialRegions, hris.size());
    Collections.sort(hris);
    // verify that we cannot split
    HRegionInfo hriToSplit2 = hris.get(1);
    ADMIN.split(
        tableTwo,
        TableInputFormatBase.getSplitKey(hriToSplit2.getStartKey(), hriToSplit2.getEndKey(), true));
    Thread.sleep(2000);
    assertEquals(initialRegions, ADMIN.getTableRegions(tableTwo).size());
  }