@Test(timeout = 60000)
public void testModifyTableDeleteCF() throws Exception {
final TableName tableName = TableName.valueOf("testModifyTableDeleteCF");
final String cf1 = "cf1";
final String cf2 = "cf2";
final String cf3 = "cf3";
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
MasterProcedureTestingUtility.createTable(procExec, tableName, null, cf1, cf2, cf3);
HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(3, currentHtd.getFamiliesKeys().size());
// Test 1: Modify the table descriptor
HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
htd.removeFamily(cf2.getBytes());
long procId =
ProcedureTestingUtility.submitAndWait(
procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(2, currentHtd.getFamiliesKeys().size());
assertFalse(currentHtd.hasFamily(cf2.getBytes()));
// Test 2: Modify the table descriptor offline
UTIL.getAdmin().disableTable(tableName);
ProcedureTestingUtility.waitNoProcedureRunning(procExec);
HTableDescriptor htd2 = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
htd2.removeFamily(cf3.getBytes());
// Disable Sanity check
htd2.setConfiguration("hbase.table.sanity.checks", Boolean.FALSE.toString());
long procId2 =
ProcedureTestingUtility.submitAndWait(
procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd2));
ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(1, currentHtd.getFamiliesKeys().size());
assertFalse(currentHtd.hasFamily(cf3.getBytes()));
// Removing the last family will fail
HTableDescriptor htd3 = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
htd3.removeFamily(cf1.getBytes());
long procId3 =
ProcedureTestingUtility.submitAndWait(
procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd3));
final ProcedureInfo result = procExec.getResult(procId3);
assertEquals(true, result.isFailed());
Throwable cause = ProcedureTestingUtility.getExceptionCause(result);
assertTrue(
"expected DoNotRetryIOException, got " + cause, cause instanceof DoNotRetryIOException);
assertEquals(1, currentHtd.getFamiliesKeys().size());
assertTrue(currentHtd.hasFamily(cf1.getBytes()));
}
/*
* Take a snapshot of a table, add metadata, and verify that this only
* affects one table
* @param online - Whether the table is online or not during the snapshot
*/
private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
// Create a table
Admin admin = UTIL.getHBaseAdmin();
final long startTime = System.currentTimeMillis();
final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
Table original = UTIL.createTable(localTableName, TEST_FAM);
UTIL.loadTable(original, TEST_FAM);
final String snapshotNameAsString = "snapshot_" + localTableName;
// Create a snapshot
SnapshotTestingUtils.createSnapshotAndValidate(
admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);
if (!online) {
admin.enableTable(localTableName);
}
TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
// Clone the snapshot
byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
admin.cloneSnapshot(snapshotName, cloneTableName);
// Add a new column family to the original table
byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
admin.disableTable(localTableName);
admin.addColumnFamily(localTableName, hcd);
// Verify that it is not in the snapshot
admin.enableTable(localTableName);
// get a description of the cloned table
// get a list of its families
// assert that the family is there
HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
Assert.assertTrue(
"The original family was not found. There is something wrong. ",
originalTableDescriptor.hasFamily(TEST_FAM));
Assert.assertTrue(
"The original family was not found in the clone. There is something wrong. ",
clonedTableDescriptor.hasFamily(TEST_FAM));
Assert.assertTrue(
"The new family was not found. ", originalTableDescriptor.hasFamily(TEST_FAM_2));
Assert.assertTrue(
"The new family was not found. ", !clonedTableDescriptor.hasFamily(TEST_FAM_2));
}
@Test(timeout = 60000)
public void testModifyTableAddCF() throws Exception {
final TableName tableName = TableName.valueOf("testModifyTableAddCF");
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(1, currentHtd.getFamiliesKeys().size());
// Test 1: Modify the table descriptor online
String cf2 = "cf2";
HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
htd.addFamily(new HColumnDescriptor(cf2));
long procId =
ProcedureTestingUtility.submitAndWait(
procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(2, currentHtd.getFamiliesKeys().size());
assertTrue(currentHtd.hasFamily(cf2.getBytes()));
// Test 2: Modify the table descriptor offline
UTIL.getAdmin().disableTable(tableName);
ProcedureTestingUtility.waitNoProcedureRunning(procExec);
String cf3 = "cf3";
HTableDescriptor htd2 = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
htd2.addFamily(new HColumnDescriptor(cf3));
long procId2 =
ProcedureTestingUtility.submitAndWait(
procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd2));
ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertTrue(currentHtd.hasFamily(cf3.getBytes()));
assertEquals(3, currentHtd.getFamiliesKeys().size());
}
@Test(timeout = 60000)
public void testRecoveryAndDoubleExecutionOnline() throws Exception {
final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
final String cf2 = "cf2";
final String cf3 = "cf3";
final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
// create the table
HRegionInfo[] regions =
MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1", cf3);
ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
// Modify multiple properties of the table.
HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
htd.setCompactionEnabled(newCompactionEnableOption);
htd.addFamily(new HColumnDescriptor(cf2));
htd.removeFamily(cf3.getBytes());
// Start the Modify procedure && kill the executor
long procId =
procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), htd));
// Restart the executor and execute the step twice
int numberOfSteps = ModifyTableState.values().length;
MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps);
// Validate descriptor
HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName);
assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
assertEquals(2, currentHtd.getFamiliesKeys().size());
assertTrue(currentHtd.hasFamily(cf2.getBytes()));
assertFalse(currentHtd.hasFamily(cf3.getBytes()));
// cf2 should be added cf3 should be removed
MasterProcedureTestingUtility.validateTableCreation(
UTIL.getHBaseCluster().getMaster(), tableName, regions, "cf1", cf2);
}
private void verifyColumns() {
if (AbstractRecordReader.isStarQuery(columns)) {
return;
}
for (SchemaPath column : columns) {
if (!(column.equals(ROW_KEY_PATH)
|| hTableDesc.hasFamily(HBaseUtils.getBytes(column.getRootSegment().getPath())))) {
DrillRuntimeException.format(
"The column family '%s' does not exist in HBase table: %s .",
column.getRootSegment().getPath(), hTableDesc.getNameAsString());
}
}
}
/**
* Action before any real action of modifying column family.
*
* @param env MasterProcedureEnv
* @throws IOException
*/
private void prepareModify(final MasterProcedureEnv env) throws IOException {
// Checks whether the table is allowed to be modified.
MasterDDLOperationHelper.checkTableModifiable(env, tableName);
unmodifiedHTableDescriptor = env.getMasterServices().getTableDescriptors().get(tableName);
if (unmodifiedHTableDescriptor == null) {
throw new IOException("HTableDescriptor missing for " + tableName);
}
if (!unmodifiedHTableDescriptor.hasFamily(cfDescriptor.getName())) {
throw new InvalidFamilyOperationException(
"Family '" + getColumnFamilyName() + "' does not exist, so it cannot be modified");
}
}
/**
* Modify Column of a table
* @param tableName
* @param hcd HColumnDesciptor
* @return Modified HTableDescriptor with the column modified.
* @throws IOException
*/
public HTableDescriptor modifyColumn(byte[] tableName, HColumnDescriptor hcd)
throws IOException {
LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
+ " HCD = " + hcd.toString());
HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
byte [] familyName = hcd.getName();
if(!htd.hasFamily(familyName)) {
throw new InvalidFamilyOperationException("Family '" +
Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
}
htd.addFamily(hcd);
this.services.getTableDescriptors().add(htd);
return htd;
}
/** Add the column family to the file system */
private void updateTableDescriptor(final MasterProcedureEnv env) throws IOException {
// Update table descriptor
LOG.info("AddColumn. Table = " + tableName + " HCD = " + cfDescriptor.toString());
HTableDescriptor htd = env.getMasterServices().getTableDescriptors().get(tableName);
if (htd.hasFamily(cfDescriptor.getName())) {
// It is possible to reach this situation, as we could already add the column family
// to table descriptor, but the master failover happens before we complete this state.
// We should be able to handle running this function multiple times without causing problem.
return;
}
htd.addFamily(cfDescriptor);
env.getMasterServices().getTableDescriptors().add(htd);
}
/**
* Restore the table descriptor back to pre-add
*
* @param env MasterProcedureEnv
* @throws IOException
*/
private void restoreTableDescriptor(final MasterProcedureEnv env) throws IOException {
HTableDescriptor htd = env.getMasterServices().getTableDescriptors().get(tableName);
if (htd.hasFamily(cfDescriptor.getName())) {
// Remove the column family from file system and update the table descriptor to
// the before-add-column-family-state
MasterDDLOperationHelper.deleteColumnFamilyFromFileSystem(
env,
tableName,
getRegionInfoList(env),
cfDescriptor.getName(),
cfDescriptor.isMobEnabled());
env.getMasterServices().getTableDescriptors().add(unmodifiedHTableDescriptor);
// Make sure regions are opened after table descriptor is updated.
reOpenAllRegionsIfTableIsOnline(env);
}
}
/**
* Action before any real action of adding column family.
*
* @param env MasterProcedureEnv
* @throws IOException
*/
private void prepareAdd(final MasterProcedureEnv env) throws IOException {
// Checks whether the table is allowed to be modified.
MasterDDLOperationHelper.checkTableModifiable(env, tableName);
// In order to update the descriptor, we need to retrieve the old descriptor for comparison.
unmodifiedHTableDescriptor = env.getMasterServices().getTableDescriptors().get(tableName);
if (unmodifiedHTableDescriptor == null) {
throw new IOException("HTableDescriptor missing for " + tableName);
}
if (unmodifiedHTableDescriptor.hasFamily(cfDescriptor.getName())) {
throw new InvalidFamilyOperationException(
"Column family '"
+ getColumnFamilyName()
+ "' in table '"
+ tableName
+ "' already exists so cannot be added");
}
}