@Test
public void testInvalidColumnFamily() throws IOException, InterruptedException {
byte[] table = Bytes.toBytes("testInvalidColumnFamily");
byte[] family = Bytes.toBytes("family");
byte[] fakecf = Bytes.toBytes("fakecf");
boolean caughtMinorCompact = false;
boolean caughtMajorCompact = false;
Table ht = null;
try {
ht = TEST_UTIL.createTable(table, family);
HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
try {
admin.compact(table, fakecf);
} catch (IOException ioe) {
caughtMinorCompact = true;
}
try {
admin.majorCompact(table, fakecf);
} catch (IOException ioe) {
caughtMajorCompact = true;
}
} finally {
if (ht != null) {
TEST_UTIL.deleteTable(table);
}
assertTrue(caughtMinorCompact);
assertTrue(caughtMajorCompact);
}
}
/**
* This test is to test the scenario happened in HBASE-6901. All files are bulk loaded and
* excluded from minor compaction. Without the fix of HBASE-6901, an
* ArrayIndexOutOfBoundsException will be thrown.
*/
@Ignore("Flakey: See HBASE-9051")
@Test
public void testExcludeAllFromMinorCompaction() throws Exception {
Configuration conf = util.getConfiguration();
conf.setInt("hbase.hstore.compaction.min", 2);
generateRandomStartKeys(5);
try {
util.startMiniCluster();
final FileSystem fs = util.getDFSCluster().getFileSystem();
HBaseAdmin admin = new HBaseAdmin(conf);
HTable table = util.createTable(TABLE_NAME, FAMILIES);
assertEquals("Should start with empty table", 0, util.countRows(table));
// deep inspection: get the StoreFile dir
final Path storePath =
HStore.getStoreHomedir(
FSUtils.getTableDir(FSUtils.getRootDir(conf), TABLE_NAME),
admin.getTableRegions(TABLE_NAME).get(0),
FAMILIES[0]);
assertEquals(0, fs.listStatus(storePath).length);
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", true);
util.startMiniMapReduceCluster();
for (int i = 0; i < 2; i++) {
Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
runIncrementalPELoad(conf, table, testDir);
// Perform the actual load
new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table);
}
// Ensure data shows up
int expectedRows = 2 * NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT;
assertEquals(
"LoadIncrementalHFiles should put expected data in table",
expectedRows,
util.countRows(table));
// should have a second StoreFile now
assertEquals(2, fs.listStatus(storePath).length);
// minor compactions shouldn't get rid of the file
admin.compact(TABLE_NAME.getName());
try {
quickPoll(
new Callable<Boolean>() {
public Boolean call() throws Exception {
return fs.listStatus(storePath).length == 1;
}
},
5000);
throw new IOException("SF# = " + fs.listStatus(storePath).length);
} catch (AssertionError ae) {
// this is expected behavior
}
// a major compaction should work though
admin.majorCompact(TABLE_NAME.getName());
quickPoll(
new Callable<Boolean>() {
public Boolean call() throws Exception {
return fs.listStatus(storePath).length == 1;
}
},
5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
/**
* Load data to a table, flush it to disk, trigger compaction, confirm the compaction state is
* right and wait till it is done.
*
* @param tableName
* @param flushes
* @param expectedState
* @param singleFamily otherwise, run compaction on all cfs
* @throws IOException
* @throws InterruptedException
*/
private void compaction(
final String tableName,
final int flushes,
final CompactionState expectedState,
boolean singleFamily)
throws IOException, InterruptedException {
// Create a table with regions
TableName table = TableName.valueOf(tableName);
byte[] family = Bytes.toBytes("family");
byte[][] families = {
family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))
};
Table ht = null;
try {
ht = TEST_UTIL.createTable(table, families);
loadData(ht, families, 3000, flushes);
HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
List<HRegion> regions = rs.getOnlineRegions(table);
int countBefore = countStoreFilesInFamilies(regions, families);
int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
assertTrue(countBefore > 0); // there should be some data files
HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
if (expectedState == CompactionState.MINOR) {
if (singleFamily) {
admin.compact(table.getName(), family);
} else {
admin.compact(table.getName());
}
} else {
if (singleFamily) {
admin.majorCompact(table.getName(), family);
} else {
admin.majorCompact(table.getName());
}
}
long curt = System.currentTimeMillis();
long waitTime = 5000;
long endt = curt + waitTime;
CompactionState state = admin.getCompactionState(table.getName());
while (state == CompactionState.NONE && curt < endt) {
Thread.sleep(10);
state = admin.getCompactionState(table.getName());
curt = System.currentTimeMillis();
}
// Now, should have the right compaction state,
// otherwise, the compaction should have already been done
if (expectedState != state) {
for (HRegion region : regions) {
state = region.getCompactionState();
assertEquals(CompactionState.NONE, state);
}
} else {
// Wait until the compaction is done
state = admin.getCompactionState(table.getName());
while (state != CompactionState.NONE && curt < endt) {
Thread.sleep(10);
state = admin.getCompactionState(table.getName());
}
// Now, compaction should be done.
assertEquals(CompactionState.NONE, state);
}
int countAfter = countStoreFilesInFamilies(regions, families);
int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
assertTrue(countAfter < countBefore);
if (!singleFamily) {
if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
else assertTrue(families.length < countAfter);
} else {
int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
// assert only change was to single column family
assertTrue(singleFamDiff == (countBefore - countAfter));
if (expectedState == CompactionState.MAJOR) {
assertTrue(1 == countAfterSingleFamily);
} else {
assertTrue(1 < countAfterSingleFamily);
}
}
} finally {
if (ht != null) {
TEST_UTIL.deleteTable(table);
}
}
}