/*
* Take a snapshot of a table, add metadata, and verify that this only
* affects one table
* @param online - Whether the table is online or not during the snapshot
*/
private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
// Create a table
Admin admin = UTIL.getHBaseAdmin();
final long startTime = System.currentTimeMillis();
final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
Table original = UTIL.createTable(localTableName, TEST_FAM);
UTIL.loadTable(original, TEST_FAM);
final String snapshotNameAsString = "snapshot_" + localTableName;
// Create a snapshot
SnapshotTestingUtils.createSnapshotAndValidate(
admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);
if (!online) {
admin.enableTable(localTableName);
}
TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
// Clone the snapshot
byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
admin.cloneSnapshot(snapshotName, cloneTableName);
// Add a new column family to the original table
byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
admin.disableTable(localTableName);
admin.addColumnFamily(localTableName, hcd);
// Verify that it is not in the snapshot
admin.enableTable(localTableName);
// get a description of the cloned table
// get a list of its families
// assert that the family is there
HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
Assert.assertTrue(
"The original family was not found. There is something wrong. ",
originalTableDescriptor.hasFamily(TEST_FAM));
Assert.assertTrue(
"The original family was not found in the clone. There is something wrong. ",
clonedTableDescriptor.hasFamily(TEST_FAM));
Assert.assertTrue(
"The new family was not found. ", originalTableDescriptor.hasFamily(TEST_FAM_2));
Assert.assertTrue(
"The new family was not found. ", !clonedTableDescriptor.hasFamily(TEST_FAM_2));
}
/*
* Take a snapshot of a table, do a split, and verify that this only affects one table
* @param online - Whether the table is online or not during the snapshot
*/
private void runTestRegionOperationsIndependent(boolean online) throws Exception {
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
// Create a table
Admin admin = UTIL.getHBaseAdmin();
final long startTime = System.currentTimeMillis();
final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
Table original = UTIL.createTable(localTableName, TEST_FAM);
UTIL.loadTable(original, TEST_FAM);
final int loadedTableCount = UTIL.countRows(original);
System.out.println("Original table has: " + loadedTableCount + " rows");
final String snapshotNameAsString = "snapshot_" + localTableName;
// Create a snapshot
SnapshotTestingUtils.createSnapshotAndValidate(
admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);
if (!online) {
admin.enableTable(localTableName);
}
TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
// Clone the snapshot
byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
admin.cloneSnapshot(snapshotName, cloneTableName);
// Verify that region information is the same pre-split
((ClusterConnection) UTIL.getConnection()).clearRegionCache();
List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
final int originalRegionCount = originalTableHRegions.size();
final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
Assert.assertEquals(
"The number of regions in the cloned table is different than in the original table.",
originalRegionCount,
cloneTableRegionCount);
// Split a region on the parent table
admin.splitRegion(originalTableHRegions.get(0).getRegionName());
waitOnSplit(UTIL.getConnection(), original, originalRegionCount);
// Verify that the cloned table region is not split
final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
Assert.assertEquals(
"The number of regions in the cloned table changed though none of its regions were split.",
cloneTableRegionCount,
cloneTableRegionCount2);
}
@Override
void perform() throws IOException {
HTableDescriptor selected = selectTable(disabledTables);
if (selected == null) {
return;
}
Admin admin = connection.getAdmin();
try {
TableName tableName = selected.getTableName();
LOG.info("Enabling table :" + selected);
admin.enableTable(tableName);
Assert.assertTrue(
"Table: " + selected + " was not enabled", admin.isTableEnabled(tableName));
HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
enabledTables.put(tableName, freshTableDesc);
LOG.info("Enabled table :" + freshTableDesc);
} catch (Exception e) {
LOG.warn("Caught exception in action: " + this.getClass());
// TODO workaround
// loose restriction for TableNotDisabledException/TableNotEnabledException thrown in sync
// operations 1) when enable/disable starts, the table state is changed to
// ENABLING/DISABLING (ZK node in 1.x), which will be further changed to ENABLED/DISABLED
// once the operation completes 2) if master failover happens in the middle of the
// enable/disable operation, the new master will try to recover the tables in
// ENABLING/DISABLING state, as programmed in
// AssignmentManager#recoverTableInEnablingState() and
// AssignmentManager#recoverTableInDisablingState()
// 3) after the new master initialization completes, the procedure tries to re-do the
// enable/disable operation, which was already done. Ignore those exceptions before
// change of behaviors of AssignmentManager in presence of PV2
if (e instanceof TableNotDisabledException) {
LOG.warn("Caught TableNotDisabledException in action: " + this.getClass());
e.printStackTrace();
} else {
throw e;
}
} finally {
admin.close();
}
verifyTables();
}
/*
* Take a snapshot of a table, add data, and verify that deleting the snapshot does not affect
* either table.
* @param online - Whether the table is online or not during the snapshot
*/
private void runTestSnapshotDeleteIndependent(boolean online) throws Exception {
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
final Admin admin = UTIL.getHBaseAdmin();
final long startTime = System.currentTimeMillis();
final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
try (Table original = UTIL.createTable(localTableName, TEST_FAM)) {
UTIL.loadTable(original, TEST_FAM);
}
// Take a snapshot
final String snapshotNameAsString = "snapshot_" + localTableName;
byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
SnapshotTestingUtils.createSnapshotAndValidate(
admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);
if (!online) {
admin.enableTable(localTableName);
}
TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
admin.cloneSnapshot(snapshotName, cloneTableName);
// Ensure the original table does not reference the HFiles anymore
admin.majorCompact(localTableName);
// Deleting the snapshot used to break the cloned table by deleting in-use HFiles
admin.deleteSnapshot(snapshotName);
// Wait for cleaner run and DFS heartbeats so that anything that is deletable is fully deleted
Thread.sleep(10000);
try (Table original = UTIL.getConnection().getTable(localTableName)) {
try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
// Verify that all regions of both tables are readable
final int origTableRowCount = UTIL.countRows(original);
final int clonedTableRowCount = UTIL.countRows(clonedTable);
Assert.assertEquals(origTableRowCount, clonedTableRowCount);
}
}
}
/**
* Test retain assignment on enableTable.
*
* @throws IOException
*/
@Test(timeout = 300000)
public void testEnableTableRetainAssignment() throws IOException {
final TableName tableName = TableName.valueOf("testEnableTableAssignment");
byte[][] splitKeys = {
new byte[] {1, 1, 1},
new byte[] {2, 2, 2},
new byte[] {3, 3, 3},
new byte[] {4, 4, 4},
new byte[] {5, 5, 5},
new byte[] {6, 6, 6},
new byte[] {7, 7, 7},
new byte[] {8, 8, 8},
new byte[] {9, 9, 9}
};
int expectedRegions = splitKeys.length + 1;
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, splitKeys);
HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
assertEquals(
"Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
expectedRegions,
regions.size());
// Disable table.
admin.disableTable(tableName);
// Enable table, use retain assignment to assign regions.
admin.enableTable(tableName);
Map<HRegionInfo, ServerName> regions2 = ht.getRegionLocations();
// Check the assignment.
assertEquals(regions.size(), regions2.size());
for (Map.Entry<HRegionInfo, ServerName> entry : regions.entrySet()) {
assertEquals(regions2.get(entry.getKey()), entry.getValue());
}
}
/*
* Take a snapshot of a table, add data, and verify that this only
* affects one table
* @param online - Whether the table is online or not during the snapshot
*/
private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
Admin admin = UTIL.getHBaseAdmin();
final long startTime = System.currentTimeMillis();
final TableName localTableName = TableName.valueOf(STRING_TABLE_NAME + startTime);
try (Table original = UTIL.createTable(localTableName, TEST_FAM)) {
UTIL.loadTable(original, TEST_FAM);
final int origTableRowCount = UTIL.countRows(original);
// Take a snapshot
final String snapshotNameAsString = "snapshot_" + localTableName;
byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
SnapshotTestingUtils.createSnapshotAndValidate(
admin, localTableName, TEST_FAM_STR, snapshotNameAsString, rootDir, fs, online);
if (!online) {
admin.enableTable(localTableName);
}
TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
admin.cloneSnapshot(snapshotName, cloneTableName);
try (Table clonedTable = UTIL.getConnection().getTable(cloneTableName)) {
final int clonedTableRowCount = UTIL.countRows(clonedTable);
Assert.assertEquals(
"The line counts of original and cloned tables do not match after clone. ",
origTableRowCount,
clonedTableRowCount);
// Attempt to add data to the test
final String rowKey = "new-row-" + System.currentTimeMillis();
Put p = new Put(Bytes.toBytes(rowKey));
p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
original.put(p);
// Verify that it is not present in the original table
Assert.assertEquals(
"The row count of the original table was not modified by the put",
origTableRowCount + 1,
UTIL.countRows(original));
Assert.assertEquals(
"The row count of the cloned table changed as a result of addition to the original",
clonedTableRowCount,
UTIL.countRows(clonedTable));
p = new Put(Bytes.toBytes(rowKey));
p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
clonedTable.put(p);
// Verify that the new family is not in the restored table's description
Assert.assertEquals(
"The row count of the original table was modified by the put to the clone",
origTableRowCount + 1,
UTIL.countRows(original));
Assert.assertEquals(
"The row count of the cloned table was not modified by the put",
clonedTableRowCount + 1,
UTIL.countRows(clonedTable));
}
}
}