@Test(timeout = 60000)
  public void testDoubleDeletedTableWithSameNonce() throws Exception {
    final TableName tableName = TableName.valueOf("testDoubleDeletedTableWithSameNonce");
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table (that exists)
    long procId1 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
    // delete the table (that will no longer exist)
    long procId2 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);

    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId1);
    ProcedureTestingUtility.waitProcedure(procExec, procId2);

    // First delete should succeed
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");

    // Second delete should not fail, because it is the same delete
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId2);
    assertTrue(procId1 == procId2);
  }
  @Test(timeout = 60000)
  public void testRecoveryAndDoubleExecution() throws Exception {
    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution");

    // create the table
    byte[][] splitKeys = null;
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(
            getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
    UTIL.getHBaseAdmin().disableTable(tableName);

    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    // Start the Delete procedure && kill the executor
    long procId =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);

    // Restart the executor and execute the step twice
    // NOTE: the 6 (number of DeleteTableState steps) is hardcoded,
    //       so you have to look at this test at least once when you add a new step.
    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
        procExec, procId, 6, DeleteTableState.values());

    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
  }
  @Test(timeout = 60000)
  public void testRollbackAndDoubleExecutionOffline() throws Exception {
    final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
    final String familyName = "cf2";
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    // create the table
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
    UTIL.getAdmin().disableTable(tableName);

    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
    boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
    htd.setCompactionEnabled(newCompactionEnableOption);
    htd.addFamily(new HColumnDescriptor(familyName));
    htd.setRegionReplication(3);

    // Start the Modify procedure && kill the executor
    long procId =
        procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), htd));

    // Restart the executor and rollback the step twice
    int numberOfSteps = 1; // failing at pre operation
    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);

    // cf2 should not be present
    MasterProcedureTestingUtility.validateTableCreation(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "cf1");
  }
  @Test(timeout = 60000)
  public void testRollbackAndDoubleExecution() throws Exception {
    final NamespaceDescriptor nsd =
        NamespaceDescriptor.create("testRollbackAndDoubleExecution").build();
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    // Start the CreateNamespace procedure && kill the executor
    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));

    int numberOfSteps = 0; // failing at pre operation
    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);

    // Validate the non-existence of namespace
    try {
      NamespaceDescriptor nsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName());
      assertNull(nsDescriptor);
    } catch (NamespaceNotFoundException nsnfe) {
      // Expected
      LOG.info("The namespace " + nsd.getName() + " is not created.");
    }
  }
  @Before
  public void setup() throws Exception {
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false);
    assertTrue("expected executor to be running", procExec.isRunning());

    nonceGroup =
        MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster());
    nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster());
  }
 /** Remove the procedures that are marked as finished */
 private synchronized void cleanupCompletedRestoreInMap() {
   ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
   Iterator<Map.Entry<TableName, Long>> it = restoreTableToProcIdMap.entrySet().iterator();
   while (it.hasNext()) {
     Map.Entry<TableName, Long> entry = it.next();
     Long procId = entry.getValue();
     if (procExec.isRunning() && procExec.isFinished(procId)) {
       it.remove();
     }
   }
 }
  @Test(timeout = 60000, expected = TableNotFoundException.class)
  public void testDeleteNotExistentTable() throws Exception {
    final TableName tableName = TableName.valueOf("testDeleteNotExistentTable");

    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    ProcedurePrepareLatch latch = new ProcedurePrepareLatch.CompatibilityLatch();
    long procId =
        ProcedureTestingUtility.submitAndWait(
            procExec, new DeleteTableProcedure(procExec.getEnvironment(), tableName, latch));
    latch.await();
  }
 /**
  * Verify if the restore of the specified table is in progress.
  *
  * @param tableName table under restore
  * @return <tt>true</tt> if there is a restore in progress of the specified table.
  */
 private synchronized boolean isRestoringTable(final TableName tableName) {
   Long procId = this.restoreTableToProcIdMap.get(tableName);
   if (procId == null) {
     return false;
   }
   ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
   if (procExec.isRunning() && !procExec.isFinished(procId)) {
     return true;
   } else {
     this.restoreTableToProcIdMap.remove(tableName);
     return false;
   }
 }
  @Test(timeout = 60000)
  public void testCreateNamespace() throws Exception {
    final NamespaceDescriptor nsd = NamespaceDescriptor.create("testCreateNamespace").build();
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId);
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);

    validateNamespaceCreated(nsd);
  }
  private void testSimpleDelete(final TableName tableName, byte[][] splitKeys) throws Exception {
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(
            getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
    long procId =
        ProcedureTestingUtility.submitAndWait(
            procExec, new DeleteTableProcedure(procExec.getEnvironment(), tableName));
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f1", "f2");
  }
  @Test(timeout = 60000)
  public void testCreateSystemNamespace() throws Exception {
    final NamespaceDescriptor nsd =
        UTIL.getAdmin().getNamespaceDescriptor(NamespaceDescriptor.SYSTEM_NAMESPACE.getName());
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId);
    ProcedureInfo result = procExec.getResult(procId);
    assertTrue(result.isFailed());
    LOG.debug("Create namespace failed with exception: " + result.getExceptionFullMessage());
    assertTrue(
        ProcedureTestingUtility.getExceptionCause(result) instanceof NamespaceExistException);
  }
  @Test(timeout = 60000)
  public void testCreateNamespaceWithInvalidTableCount() throws Exception {
    final NamespaceDescriptor nsd =
        NamespaceDescriptor.create("testCreateNamespaceWithInvalidTableCount").build();
    final String nsKey = "hbase.namespace.quota.maxtables";
    final String nsValue = "-1";
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    nsd.setConfiguration(nsKey, nsValue);

    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId);
    ProcedureInfo result = procExec.getResult(procId);
    assertTrue(result.isFailed());
    LOG.debug("Create namespace failed with exception: " + result.getExceptionFullMessage());
    assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof ConstraintException);
  }
  @Test(timeout = 60000)
  public void testModifyTableDeleteCF() throws Exception {
    final TableName tableName = TableName.valueOf("testModifyTableDeleteCF");
    final String cf1 = "cf1";
    final String cf2 = "cf2";
    final String cf3 = "cf3";
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2, cf3);
    HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertEquals(3, currentHtd.getFamiliesKeys().size());

    // Test 1: Modify the table descriptor
    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
    htd.removeFamily(cf2.getBytes());

    long procId =
        ProcedureTestingUtility.submitAndWait(
            procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));

    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertEquals(2, currentHtd.getFamiliesKeys().size());
    assertFalse(currentHtd.hasFamily(cf2.getBytes()));

    // Test 2: Modify the table descriptor offline
    UTIL.getAdmin().disableTable(tableName);
    ProcedureTestingUtility.waitNoProcedureRunning(procExec);

    HTableDescriptor htd2 = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
    htd2.removeFamily(cf3.getBytes());
    // Disable Sanity check
    htd2.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());

    long procId2 =
        ProcedureTestingUtility.submitAndWait(
            procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd2));
    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));

    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertEquals(1, currentHtd.getFamiliesKeys().size());
    assertFalse(currentHtd.hasFamily(cf3.getBytes()));

    // Removing the last family will fail
    HTableDescriptor htd3 = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
    htd3.removeFamily(cf1.getBytes());
    long procId3 =
        ProcedureTestingUtility.submitAndWait(
            procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd3));
    final ProcedureInfo result = procExec.getResult(procId3);
    assertEquals(true, result.isFailed());
    Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
    assertTrue(
        "expected DoNotRetryIOException, got " + cause, cause instanceof DoNotRetryIOException);
    assertEquals(1, currentHtd.getFamiliesKeys().size());
    assertTrue(currentHtd.hasFamily(cf1.getBytes()));
  }
  @Test(timeout = 60000)
  public void testRecoveryAndDoubleExecution() throws Exception {
    final NamespaceDescriptor nsd =
        NamespaceDescriptor.create("testRecoveryAndDoubleExecution").build();
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    // Start the CreateNamespace procedure && kill the executor
    long procId =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));

    // Restart the executor and execute the step twice
    int numberOfSteps = CreateNamespaceState.values().length;
    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps);

    // Validate the creation of namespace
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
    validateNamespaceCreated(nsd);
  }
  @Test(timeout = 60000)
  public void testRecoveryAndDoubleExecutionOffline() throws Exception {
    final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
    final String cf2 = "cf2";
    final String cf3 = "cf3";
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    // create the table
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1", cf3);
    UTIL.getAdmin().disableTable(tableName);

    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    // Modify multiple properties of the table.
    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
    boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
    htd.setCompactionEnabled(newCompactionEnableOption);
    htd.addFamily(new HColumnDescriptor(cf2));
    htd.removeFamily(cf3.getBytes());
    htd.setRegionReplication(3);

    // Start the Modify procedure && kill the executor
    long procId =
        procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), htd));

    // Restart the executor and execute the step twice
    int numberOfSteps = ModifyTableState.values().length;
    MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps);

    // Validate descriptor
    HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
    assertEquals(2, currentHtd.getFamiliesKeys().size());

    // cf2 should be added cf3 should be removed
    MasterProcedureTestingUtility.validateTableCreation(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, false, "cf1", cf2);
  }
  @Test(timeout = 60000)
  public void testCreateSameNamespaceTwice() throws Exception {
    final NamespaceDescriptor nsd =
        NamespaceDescriptor.create("testCreateSameNamespaceTwice").build();
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    long procId1 =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId1);
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);

    // Create the namespace that exists
    long procId2 =
        procExec.submitProcedure(new CreateNamespaceProcedure(procExec.getEnvironment(), nsd));
    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId2);

    // Second create should fail with NamespaceExistException
    ProcedureInfo result = procExec.getResult(procId2);
    assertTrue(result.isFailed());
    LOG.debug("Create namespace failed with exception: " + result.getExceptionFullMessage());
    assertTrue(
        ProcedureTestingUtility.getExceptionCause(result) instanceof NamespaceExistException);
  }
  @Test(timeout = 60000)
  public void testDeleteDeletedTable() throws Exception {
    final TableName tableName = TableName.valueOf("testDeleteDeletedTable");
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f");
    UTIL.getHBaseAdmin().disableTable(tableName);

    // delete the table (that exists)
    long procId1 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName), nonceGroup, nonce);
    // delete the table (that will no longer exist)
    long procId2 =
        procExec.submitProcedure(
            new DeleteTableProcedure(procExec.getEnvironment(), tableName),
            nonceGroup + 1,
            nonce + 1);

    // Wait the completion
    ProcedureTestingUtility.waitProcedure(procExec, procId1);
    ProcedureTestingUtility.waitProcedure(procExec, procId2);

    // First delete should succeed
    ProcedureTestingUtility.assertProcNotFailed(procExec, procId1);
    MasterProcedureTestingUtility.validateTableDeletion(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "f");

    // Second delete should fail with TableNotFound
    ProcedureResult result = procExec.getResult(procId2);
    assertTrue(result.isFailed());
    LOG.debug("Delete failed with exception: " + result.getException());
    assertTrue(result.getException().getCause() instanceof TableNotFoundException);
  }
  @Test(timeout = 60000)
  public void testModifyTableAddCF() throws Exception {
    final TableName tableName = TableName.valueOf("testModifyTableAddCF");
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
    HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertEquals(1, currentHtd.getFamiliesKeys().size());

    // Test 1: Modify the table descriptor online
    String cf2 = "cf2";
    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
    htd.addFamily(new HColumnDescriptor(cf2));

    long procId =
        ProcedureTestingUtility.submitAndWait(
            procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));

    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertEquals(2, currentHtd.getFamiliesKeys().size());
    assertTrue(currentHtd.hasFamily(cf2.getBytes()));

    // Test 2: Modify the table descriptor offline
    UTIL.getAdmin().disableTable(tableName);
    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    String cf3 = "cf3";
    HTableDescriptor htd2 = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
    htd2.addFamily(new HColumnDescriptor(cf3));

    long procId2 =
        ProcedureTestingUtility.submitAndWait(
            procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd2));
    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));

    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertTrue(currentHtd.hasFamily(cf3.getBytes()));
    assertEquals(3, currentHtd.getFamiliesKeys().size());
  }
  @Test(timeout = 60000)
  public void testModifyTable() throws Exception {
    final TableName tableName = TableName.valueOf("testModifyTable");
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf");
    UTIL.getAdmin().disableTable(tableName);

    // Modify the table descriptor
    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));

    // Test 1: Modify 1 property
    long newMaxFileSize = htd.getMaxFileSize() * 2;
    htd.setMaxFileSize(newMaxFileSize);
    htd.setRegionReplication(3);

    long procId1 =
        ProcedureTestingUtility.submitAndWait(
            procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));

    HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertEquals(newMaxFileSize, currentHtd.getMaxFileSize());

    // Test 2: Modify multiple properties
    boolean newReadOnlyOption = htd.isReadOnly() ? false : true;
    long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2;
    htd.setReadOnly(newReadOnlyOption);
    htd.setMemStoreFlushSize(newMemStoreFlushSize);

    long procId2 =
        ProcedureTestingUtility.submitAndWait(
            procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
    ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));

    currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
    assertEquals(newReadOnlyOption, currentHtd.isReadOnly());
    assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize());
  }