private Table createTableAndLoadData( HMaster master, TableName tablename, int numRegions, int replication) throws Exception { assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions); byte[][] splitRows = new byte[numRegions - 1][]; for (int i = 0; i < splitRows.length; i++) { splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions]; } Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows); if (replication > 1) { HBaseTestingUtility.setReplicas(admin, tablename, replication); } loadData(table); verifyRowCount(table, ROWSIZE); // sleep here is an ugly hack to allow region transitions to finish long timeout = System.currentTimeMillis() + waitTime; List<Pair<HRegionInfo, ServerName>> tableRegions; while (System.currentTimeMillis() < timeout) { tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename); if (tableRegions.size() == numRegions * replication) break; Thread.sleep(250); } tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename); LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions)); assertEquals(numRegions * replication, tableRegions.size()); return table; }
@Test public void testMergeWithReplicas() throws Exception { final TableName tableName = TableName.valueOf("testMergeWithReplicas"); // Create table and load data. createTableAndLoadData(master, tableName, 5, 2); List<Pair<HRegionInfo, ServerName>> initialRegionToServers = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tableName); // Merge 1st and 2nd region PairOfSameType<HRegionInfo> mergedRegions = mergeRegionsAndVerifyRegionNum(master, tableName, 0, 2, 5 * 2 - 2); List<Pair<HRegionInfo, ServerName>> currentRegionToServers = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tableName); List<HRegionInfo> initialRegions = new ArrayList<HRegionInfo>(); for (Pair<HRegionInfo, ServerName> p : initialRegionToServers) { initialRegions.add(p.getFirst()); } List<HRegionInfo> currentRegions = new ArrayList<HRegionInfo>(); for (Pair<HRegionInfo, ServerName> p : currentRegionToServers) { currentRegions.add(p.getFirst()); } assertTrue(initialRegions.contains(mergedRegions.getFirst())); // this is the first region assertTrue( initialRegions.contains( RegionReplicaUtil.getRegionInfoForReplica( mergedRegions.getFirst(), 1))); // this is the replica of the first region assertTrue(initialRegions.contains(mergedRegions.getSecond())); // this is the second region assertTrue( initialRegions.contains( RegionReplicaUtil.getRegionInfoForReplica( mergedRegions.getSecond(), 1))); // this is the replica of the second region assertTrue(!initialRegions.contains(currentRegions.get(0))); // this is the new region assertTrue( !initialRegions.contains( RegionReplicaUtil.getRegionInfoForReplica( currentRegions.get(0), 1))); // replica of the new region assertTrue( currentRegions.contains( RegionReplicaUtil.getRegionInfoForReplica( currentRegions.get(0), 1))); // replica of the new region assertTrue( !currentRegions.contains( RegionReplicaUtil.getRegionInfoForReplica( mergedRegions.getFirst(), 1))); // replica of the merged region assertTrue( !currentRegions.contains( RegionReplicaUtil.getRegionInfoForReplica( mergedRegions.getSecond(), 1))); // replica of the merged region }
private PairOfSameType<HRegionInfo> requestMergeRegion( HMaster master, TableName tablename, int regionAnum, int regionBnum) throws Exception { List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename); HRegionInfo regionA = tableRegions.get(regionAnum).getFirst(); HRegionInfo regionB = tableRegions.get(regionBnum).getFirst(); TEST_UTIL .getHBaseAdmin() .mergeRegions(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false); return new PairOfSameType<HRegionInfo>(regionA, regionB); }
private void waitAndVerifyRegionNum(HMaster master, TableName tablename, int expectedRegionNum) throws Exception { List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta; List<HRegionInfo> tableRegionsInMaster; long timeout = System.currentTimeMillis() + waitTime; while (System.currentTimeMillis() < timeout) { tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename); tableRegionsInMaster = master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename); if (tableRegionsInMeta.size() == expectedRegionNum && tableRegionsInMaster.size() == expectedRegionNum) { break; } Thread.sleep(250); } tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tablename); LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta)); assertEquals(expectedRegionNum, tableRegionsInMeta.size()); }
@SuppressWarnings("deprecation") @Test public void testCleanMergeReference() throws Exception { LOG.info("Starting testCleanMergeReference"); admin.enableCatalogJanitor(false); try { final TableName tableName = TableName.valueOf("testCleanMergeReference"); // Create table and load data. Table table = createTableAndLoadData(master, tableName); // Merge 1st and 2nd region mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1, INITIAL_REGION_NUM - 1); verifyRowCount(table, ROWSIZE); table.close(); List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getConnection(), tableName); HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst(); HTableDescriptor tableDescritor = master.getTableDescriptors().get(tableName); Result mergedRegionResult = MetaTableAccessor.getRegionResult( master.getConnection(), mergedRegionInfo.getRegionName()); // contains merge reference in META assertTrue( mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null); assertTrue( mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER) != null); // merging regions' directory are in the file system all the same PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(mergedRegionResult); HRegionInfo regionA = p.getFirst(); HRegionInfo regionB = p.getSecond(); FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path rootDir = master.getMasterFileSystem().getRootDir(); Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable()); Path regionAdir = new Path(tabledir, regionA.getEncodedName()); Path regionBdir = new Path(tabledir, regionB.getEncodedName()); assertTrue(fs.exists(regionAdir)); assertTrue(fs.exists(regionBdir)); admin.compactRegion(mergedRegionInfo.getRegionName()); // wait until merged region doesn't have reference file long timeout = System.currentTimeMillis() + waitTime; HRegionFileSystem hrfs = new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo); while (System.currentTimeMillis() < timeout) { if (!hrfs.hasReferences(tableDescritor)) { break; } Thread.sleep(50); } assertFalse(hrfs.hasReferences(tableDescritor)); // run CatalogJanitor to clean merge references in hbase:meta and archive the // files of merging regions int cleaned = admin.runCatalogScan(); assertTrue(cleaned > 0); assertFalse(fs.exists(regionAdir)); assertFalse(fs.exists(regionBdir)); mergedRegionResult = MetaTableAccessor.getRegionResult( master.getConnection(), mergedRegionInfo.getRegionName()); assertFalse( mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null); assertFalse( mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER) != null); } finally { admin.enableCatalogJanitor(true); } }