/** * Assert that getSplitEditFilesSorted returns files in expected order and that it skips * moved-aside files. * * @throws IOException */ @Test public void testGetSplitEditFilesSorted() throws IOException { FileSystem fs = FileSystem.get(util.getConfiguration()); Path regiondir = util.getDataTestDir("regiondir"); fs.delete(regiondir, true); fs.mkdirs(regiondir); Path recoverededits = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); String first = WALSplitter.formatRecoveredEditsFileName(-1); createFile(fs, recoverededits, first); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(0)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(1)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(11)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(2)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(50)); String last = WALSplitter.formatRecoveredEditsFileName(Long.MAX_VALUE); createFile(fs, recoverededits, last); createFile( fs, recoverededits, Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis()); final Configuration walConf = new Configuration(util.getConfiguration()); FSUtils.setRootDir(walConf, regiondir); (new WALFactory(walConf, null, "dummyLogName")).getWAL(new byte[] {}, null); NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir); assertEquals(7, files.size()); assertEquals(files.pollFirst().getName(), first); assertEquals(files.pollLast().getName(), last); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(0)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(1)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(2)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(11)); }
/** * Assert that getSplitEditFilesSorted returns files in expected order and that it skips * moved-aside files. * * @throws IOException */ @Test public void testGetSplitEditFilesSorted() throws IOException { FileSystem fs = FileSystem.get(util.getConfiguration()); Path regiondir = util.getDataTestDir("regiondir"); fs.delete(regiondir, true); fs.mkdirs(regiondir); Path recoverededits = HLogUtil.getRegionDirRecoveredEditsDir(regiondir); String first = HLogSplitter.formatRecoveredEditsFileName(-1); createFile(fs, recoverededits, first); createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(0)); createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(1)); createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(11)); createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(2)); createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(50)); String last = HLogSplitter.formatRecoveredEditsFileName(Long.MAX_VALUE); createFile(fs, recoverededits, last); createFile( fs, recoverededits, Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis()); HLogFactory.createHLog(fs, regiondir, "dummyLogName", util.getConfiguration()); NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regiondir); assertEquals(7, files.size()); assertEquals(files.pollFirst().getName(), first); assertEquals(files.pollLast().getName(), last); assertEquals(files.pollFirst().getName(), HLogSplitter.formatRecoveredEditsFileName(0)); assertEquals(files.pollFirst().getName(), HLogSplitter.formatRecoveredEditsFileName(1)); assertEquals(files.pollFirst().getName(), HLogSplitter.formatRecoveredEditsFileName(2)); assertEquals(files.pollFirst().getName(), HLogSplitter.formatRecoveredEditsFileName(11)); }
/* * Add to each of the regions in .META. a value. Key is the startrow of the * region (except its 'aaa' for first region). Actual value is the row name. * @param expected * @return * @throws IOException */ private static int addToEachStartKey(final int expected) throws IOException { HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); HTable meta = new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME); int rows = 0; Scan scan = new Scan(); scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); ResultScanner s = meta.getScanner(scan); for (Result r = null; (r = s.next()) != null; ) { byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); if (b == null || b.length <= 0) break; HRegionInfo hri = Writables.getHRegionInfo(b); // If start key, add 'aaa'. byte[] row = getStartKey(hri); Put p = new Put(row); p.setWriteToWAL(false); p.add(getTestFamily(), getTestQualifier(), row); t.put(p); rows++; } s.close(); Assert.assertEquals(expected, rows); t.close(); meta.close(); return rows; }
/** * Start up a mini cluster and put a small table of many empty regions into it. * * @throws Exception */ @BeforeClass public static void beforeAllTests() throws Exception { TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true); TEST_UTIL.startMiniCluster(2); // Create a table of three families. This will assign a region. TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES); HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily()); TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions); addToEachStartKey(countOfRegions); t.close(); }
@Test public void testReadingArchiveDirectoryFromFS() throws IOException { FileSystem fs = FileSystem.get(UTIL.getConfiguration()); try { new FSTableDescriptors(fs, FSUtils.getRootDir(UTIL.getConfiguration())) .get(HConstants.HFILE_ARCHIVE_DIRECTORY); fail("Shouldn't be able to read a table descriptor for the archive directory."); } catch (IOException e) { LOG.debug( "Correctly got error when reading a table descriptor from the archive directory: " + e.getMessage()); } }
private CatalogTracker constructAndStartCatalogTracker(final HConnection c) throws IOException, InterruptedException { CatalogTracker ct = new CatalogTracker(this.watcher, UTIL.getConfiguration(), c, this.abortable); ct.start(); return ct; }
/** * A set up method to start the test cluster. AggregateProtocolImpl is registered and will be * loaded during region startup. * * @throws Exception */ @BeforeClass public static void setupBeforeClass() throws Exception { conf.set( CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "org.apache.hadoop.hbase.coprocessor.AggregateImplementation"); util.startMiniCluster(2); HTable table = util.createTable(TEST_TABLE, TEST_FAMILY); util.createMultiRegions( util.getConfiguration(), table, TEST_FAMILY, new byte[][] {HConstants.EMPTY_BYTE_ARRAY, ROWS[rowSeperator1], ROWS[rowSeperator2]}); /** * The testtable has one CQ which is always populated and one variable CQ for each row rowkey1: * CF:CQ CF:CQ1 rowKey2: CF:CQ CF:CQ2 */ for (int i = 0; i < ROWSIZE; i++) { Put put = new Put(ROWS[i]); put.setWriteToWAL(false); Long l = new Long(i); put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(l)); table.put(put); Put p2 = new Put(ROWS[i]); put.setWriteToWAL(false); p2.add(TEST_FAMILY, Bytes.add(TEST_MULTI_CQ, Bytes.toBytes(l)), Bytes.toBytes(l * 10)); table.put(p2); } table.close(); }
/** @throws java.lang.Exception */ @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniZKCluster(); Configuration conf = TEST_UTIL.getConfiguration(); conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT); admin = new ReplicationAdmin(conf); }
@Test public void testNoSuchTable() throws IOException { final String name = "testNoSuchTable"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, rootdir); assertNull("There shouldn't be any HTD for this table", htds.get("NoSuchTable")); }
public static void verifyMobRowCount( final HBaseTestingUtility util, final TableName tableName, long expectedRows) throws IOException { Table table = ConnectionFactory.createConnection(util.getConfiguration()).getTable(tableName); try { assertEquals(expectedRows, countMobRows(table)); } finally { table.close(); } }
@Test public void testReadingHTDFromFS() throws IOException { final String name = "testReadingHTDFromFS"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); HTableDescriptor htd = new HTableDescriptor(name); Path rootdir = UTIL.getDataTestDir(name); createHTDInFS(fs, rootdir, htd); HTableDescriptor htd2 = FSTableDescriptors.getTableDescriptor(fs, rootdir, htd.getNameAsString()); assertTrue(htd.equals(htd2)); }
@Test public void testUpdates() throws IOException { final String name = "testUpdates"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, rootdir); HTableDescriptor htd = new HTableDescriptor(name); htds.add(htd); htds.add(htd); htds.add(htd); }
@BeforeClass public static void setUpBeforeClass() throws Exception { conf = TEST_UTIL.getConfiguration(); TEST_UTIL.startMiniCluster(); REST_TEST_UTIL.startServletContainer(conf); client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); context = JAXBContext.newInstance( CellModel.class, CellSetModel.class, RowModel.class, ScannerModel.class); marshaller = context.createMarshaller(); unmarshaller = context.createUnmarshaller(); Admin admin = TEST_UTIL.getHBaseAdmin(); if (admin.tableExists(TABLE)) { return; } HTableDescriptor htd = new HTableDescriptor(TABLE); htd.addFamily(new HColumnDescriptor(CFA)); htd.addFamily(new HColumnDescriptor(CFB)); admin.createTable(htd); expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0); expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5); }
/* * @return Count of rows in TABLENAME * @throws IOException */ private static int count() throws IOException { HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME); int rows = 0; Scan scan = new Scan(); ResultScanner s = t.getScanner(scan); for (Result r = null; (r = s.next()) != null; ) { rows++; } s.close(); LOG.info("Counted=" + rows); t.close(); return rows; }
@Test public void testHTableDescriptors() throws IOException, InterruptedException { final String name = "testHTableDescriptors"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any debris laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); final int count = 10; // Write out table infos. for (int i = 0; i < count; i++) { HTableDescriptor htd = new HTableDescriptor(name + i); createHTDInFS(fs, rootdir, htd); } FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) { @Override public HTableDescriptor get(byte[] tablename) throws TableExistsException, FileNotFoundException, IOException { LOG.info(Bytes.toString(tablename) + ", cachehits=" + this.cachehits); return super.get(tablename); } }; for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } // Update the table infos for (int i = 0; i < count; i++) { HTableDescriptor htd = new HTableDescriptor(name + i); htd.addFamily(new HColumnDescriptor("" + i)); FSTableDescriptors.updateHTableDescriptor(fs, rootdir, htd); } // Wait a while so mod time we write is for sure different. Thread.sleep(100); for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } for (int i = 0; i < count; i++) { assertTrue(htds.get(Bytes.toBytes(name + i)) != null); } assertEquals(count * 4, htds.invocations); assertTrue( "expected=" + (count * 2) + ", actual=" + htds.cachehits, htds.cachehits >= (count * 2)); assertTrue(htds.get(HConstants.ROOT_TABLE_NAME) != null); assertEquals(htds.invocations, count * 4 + 1); assertTrue( "expected=" + ((count * 2) + 1) + ", actual=" + htds.cachehits, htds.cachehits >= ((count * 2) + 1)); }
@Test public void createTableTest() throws IOException, InterruptedException { String testName = "createTableTest"; String nsName = prefix + "_" + testName; LOG.info(testName); HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(nsName + ":my_table")); HColumnDescriptor colDesc = new HColumnDescriptor("my_cf"); desc.addFamily(colDesc); try { admin.createTable(desc); fail("Expected no namespace exists exception"); } catch (NamespaceNotFoundException ex) { } // create table and in new namespace admin.createNamespace(NamespaceDescriptor.create(nsName).build()); admin.createTable(desc); TEST_UTIL.waitTableAvailable(desc.getTableName().getName(), 10000); FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); assertTrue( fs.exists( new Path( master.getMasterFileSystem().getRootDir(), new Path( HConstants.BASE_NAMESPACE_DIR, new Path(nsName, desc.getTableName().getQualifierAsString()))))); assertEquals(1, admin.listTables().length); // verify non-empty namespace can't be removed try { admin.deleteNamespace(nsName); fail("Expected non-empty namespace constraint exception"); } catch (Exception ex) { LOG.info("Caught expected exception: " + ex); } // sanity check try to write and read from table Table table = TEST_UTIL.getConnection().getTable(desc.getTableName()); Put p = new Put(Bytes.toBytes("row1")); p.add(Bytes.toBytes("my_cf"), Bytes.toBytes("my_col"), Bytes.toBytes("value1")); table.put(p); // flush and read from disk to make sure directory changes are working admin.flush(desc.getTableName()); Get g = new Get(Bytes.toBytes("row1")); assertTrue(table.exists(g)); // normal case of removing namespace TEST_UTIL.deleteTable(desc.getTableName()); admin.deleteNamespace(nsName); }
private void testVerifyMetaRegionLocationWithException(Exception ex) throws IOException, InterruptedException, KeeperException { // Mock an HRegionInterface. final HRegionInterface implementation = Mockito.mock(HRegionInterface.class); HConnection connection = mockConnection(implementation); // If a 'get' is called on mocked interface, throw connection refused. Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenThrow(ex); // Now start up the catalogtracker with our doctored Connection. final CatalogTracker ct = constructAndStartCatalogTracker(connection); RootLocationEditor.setRootLocation(this.watcher, SN); long timeout = UTIL.getConfiguration().getLong("hbase.catalog.verification.timeout", 1000); Assert.assertFalse(ct.verifyMetaRegionLocation(timeout)); }
@After public void after() { try { // Clean out root location or later tests will be confused... they presume // start fresh in zk. RootLocationEditor.deleteRootLocation(this.watcher); } catch (KeeperException e) { LOG.warn("Unable to delete root location", e); } // Clear out our doctored connection or could mess up subsequent tests. HConnectionManager.deleteConnection(UTIL.getConfiguration()); this.watcher.close(); }
@BeforeClass public static void setUpOnce() throws Exception { miniCluster = System.getProperty("cluster.type").equals("mini"); securedCluster = System.getProperty("cluster.secured").equals("true"); System.out.println("realCluster - " + !miniCluster); System.out.println("securedCluster - " + securedCluster); Util.setLoggingThreshold("ERROR"); if (miniCluster) { if (hbase == null) { hbase = new HBaseTestingUtility(); conf = hbase.getConfiguration(); conf.set("zookeeper.session.timeout", "3600000"); conf.set("dfs.client.socket-timeout", "3600000"); if (securedCluster) { hbase.startMiniCluster(RS_COUNT); hbase.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME, 30000L); admin = new HBaseAdminWrapper(conf); } else { hbase.startMiniCluster(RS_COUNT); admin = hbase.getHBaseAdmin(); } } } else { if (admin == null) { final String argsFileName = securedCluster ? "../../testClusterRealSecured.args" : "../../testClusterRealNonSecured.args"; if (!Util.isFile(argsFileName)) { throw new IllegalStateException( "You have to define args file " + argsFileName + " for tests."); } String[] testArgs = {argsFileName}; Args args = new TestArgs(testArgs); admin = HBaseClient.getAdmin(args); conf = admin.getConfiguration(); RS_COUNT = getServerNameList().size(); } } previousBalancerRunning = admin.setBalancerRunning(false, true); hConnection = HConnectionManager.createConnection(conf); USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]); }
@Test public void testCreateAndUpdate() throws IOException { Path testdir = UTIL.getDataTestDir("testCreateAndUpdate"); HTableDescriptor htd = new HTableDescriptor("testCreate"); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); assertTrue(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); assertFalse(FSTableDescriptors.createTableDescriptor(fs, testdir, htd)); FileStatus[] statuses = fs.listStatus(testdir); assertTrue("statuses.length=" + statuses.length, statuses.length == 1); for (int i = 0; i < 10; i++) { FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); } statuses = fs.listStatus(testdir); assertTrue(statuses.length == 1); Path tmpTableDir = new Path(FSUtils.getTablePath(testdir, htd.getName()), ".tmp"); statuses = fs.listStatus(tmpTableDir); assertTrue(statuses.length == 0); }
@Test public void testSequenceidAdvancesOnTableInfo() throws IOException { Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo"); HTableDescriptor htd = new HTableDescriptor("testSequenceidAdvancesOnTableInfo"); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path p0 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); int i0 = FSTableDescriptors.getTableInfoSequenceid(p0); Path p1 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); // Assert we cleaned up the old file. assertTrue(!fs.exists(p0)); int i1 = FSTableDescriptors.getTableInfoSequenceid(p1); assertTrue(i1 == i0 + 1); Path p2 = FSTableDescriptors.updateHTableDescriptor(fs, testdir, htd); // Assert we cleaned up the old file. assertTrue(!fs.exists(p1)); int i2 = FSTableDescriptors.getTableInfoSequenceid(p2); assertTrue(i2 == i1 + 1); }
@Before public void before() throws IOException { this.abortable = new Abortable() { @Override public void abort(String why, Throwable e) { LOG.info(why, e); } @Override public boolean isAborted() { return false; } }; this.watcher = new ZooKeeperWatcher( UTIL.getConfiguration(), this.getClass().getSimpleName(), this.abortable, true); }
/** * @param implementation An {@link HRegionInterface} instance; you'll likely want to pass a mocked * HRS; can be null. * @return Mock up a connection that returns a {@link org.apache.hadoop.conf.Configuration} when * {@link HConnection#getConfiguration()} is called, a 'location' when {@link * HConnection#getRegionLocation(byte[], byte[], boolean)} is called, and that returns the * passed {@link HRegionInterface} instance when {@link * HConnection#getHRegionConnection(String, int)} is called (Be sure call {@link * HConnectionManager#deleteConnection(org.apache.hadoop.conf.Configuration)} when done with * this mocked Connection. * @throws IOException */ private HConnection mockConnection(final HRegionInterface implementation) throws IOException { HConnection connection = HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration()); Mockito.doNothing().when(connection).close(); // Make it so we return any old location when asked. final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN.getHostname(), SN.getPort()); Mockito.when( connection.getRegionLocation( (byte[]) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean())) .thenReturn(anyLocation); Mockito.when(connection.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())) .thenReturn(anyLocation); if (implementation != null) { // If a call to getHRegionConnection, return this implementation. Mockito.when(connection.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())) .thenReturn(implementation); } return connection; }
@Test public void testRPCException() throws Exception { HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); TEST_UTIL.startMiniZKCluster(); Configuration conf = TEST_UTIL.getConfiguration(); conf.set(HConstants.MASTER_PORT, "0"); HMaster hm = new HMaster(conf); ServerName sm = hm.getServerName(); InetSocketAddress isa = new InetSocketAddress(sm.getHostname(), sm.getPort()); int i = 0; // retry the RPC a few times; we have seen SocketTimeoutExceptions if we // try to connect too soon. Retry on SocketTimeoutException. while (i < 20) { try { MasterMonitorProtocol inf = (MasterMonitorProtocol) HBaseClientRPC.getProxy(MasterMonitorProtocol.class, isa, conf, 100 * 10); inf.isMasterRunning(null, IsMasterRunningRequest.getDefaultInstance()); fail(); } catch (ServiceException ex) { IOException ie = ProtobufUtil.getRemoteException(ex); if (!(ie instanceof SocketTimeoutException)) { if (ie.getMessage() .startsWith( "org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet")) { return; } } else { System.err.println("Got SocketTimeoutException. Will retry. "); } } catch (Throwable t) { fail("Unexpected throwable: " + t); } Thread.sleep(100); i++; } fail(); }
/** * Create a Mob table. * * @param util * @param tableName * @param families * @return An HTable instance for the created table. * @throws IOException */ public static Table createMobTable( final HBaseTestingUtility util, final TableName tableName, final byte[]... families) throws IOException { HTableDescriptor htd = new HTableDescriptor(tableName); for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family); // Disable blooms (they are on by default as of 0.95) but we disable them // here because // tests have hard coded counts of what to expect in block cache, etc., // and blooms being // on is interfering. hcd.setBloomFilterType(BloomType.NONE); hcd.setMobEnabled(true); hcd.setMobThreshold(0L); htd.addFamily(hcd); } util.getHBaseAdmin().createTable(htd); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait // until they are assigned util.waitUntilAllRegionsAssigned(htd.getTableName()); return ConnectionFactory.createConnection(util.getConfiguration()).getTable(htd.getTableName()); }
/** Spin up a cluster with a bunch of regions on it. */ @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(NB_SLAVES); TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster(); TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); final List<String> families = new ArrayList<String>(1); families.add("family"); TEST_UTIL.createRandomTable("table", families, 1, 0, 0, COUNT_OF_REGIONS, 0); // Ensure a stable env TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, false); boolean ready = false; while (!ready) { waitForAllRegionsOnline(); // Assert that every regionserver has some regions on it. int i = 0; ready = true; while (i < NB_SLAVES && ready) { HRegionServer hrs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i); if (ProtobufUtil.getOnlineRegions(hrs).isEmpty()) { ready = false; } i++; } if (!ready) { TEST_UTIL.getHBaseAdmin().setBalancerRunning(true, true); Assert.assertTrue("Can't start a balance!", TEST_UTIL.getHBaseAdmin().balancer()); TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, false); Thread.sleep(100); } } }
@Test public void testAssignmentManagerDoesntUseDrainingServer() throws Exception { AssignmentManager am; Configuration conf = TEST_UTIL.getConfiguration(); final HMaster master = Mockito.mock(HMaster.class); final Server server = Mockito.mock(Server.class); final ServerManager serverManager = Mockito.mock(ServerManager.class); final ServerName SERVERNAME_A = ServerName.valueOf("mockserver_a.org", 1000, 8000); final ServerName SERVERNAME_B = ServerName.valueOf("mockserver_b.org", 1001, 8000); LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf); final HRegionInfo REGIONINFO = new HRegionInfo( TableName.valueOf("table_test"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "zkWatcher-Test", abortable, true); Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>(); onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD); Mockito.when(server.getConfiguration()).thenReturn(conf); Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1")); Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher); CoordinatedStateManager cp = new ZkCoordinatedStateManager(); cp.initialize(server); cp.start(); Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp); Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers); Mockito.when(serverManager.getOnlineServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList(null)) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); for (ServerName sn : onlineServers.keySet()) { Mockito.when(serverManager.isServerOnline(sn)).thenReturn(true); Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1)).thenReturn(true); Mockito.when(serverManager.sendRegionClose(sn, REGIONINFO, -1, null, false)).thenReturn(true); Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, new ArrayList<ServerName>())) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.sendRegionOpen(sn, REGIONINFO, -1, null)) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.addServerToDrainList(sn)).thenReturn(true); } Mockito.when(master.getServerManager()).thenReturn(serverManager); am = new AssignmentManager( server, serverManager, balancer, startupMasterExecutor("mockExecutorService"), null, null); Mockito.when(master.getAssignmentManager()).thenReturn(am); Mockito.when(master.getZooKeeper()).thenReturn(zkWatcher); am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_A)); zkWatcher.registerListenerFirst(am); addServerToDrainedList(SERVERNAME_A, onlineServers, serverManager); am.assign(REGIONINFO, true); setRegionOpenedOnZK(zkWatcher, SERVERNAME_A, REGIONINFO); setRegionOpenedOnZK(zkWatcher, SERVERNAME_B, REGIONINFO); am.waitForAssignment(REGIONINFO); assertTrue(am.getRegionStates().isRegionOnline(REGIONINFO)); assertNotEquals(am.getRegionStates().getRegionServerOfRegion(REGIONINFO), SERVERNAME_A); }
@BeforeClass public static void beforeClass() throws Exception { TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true); TEST_UTIL.startMiniZKCluster(); }
@Test public void testAssignmentManagerDoesntUseDrainedServerWithBulkAssign() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(conf); AssignmentManager am; final HMaster master = Mockito.mock(HMaster.class); final Server server = Mockito.mock(Server.class); final ServerManager serverManager = Mockito.mock(ServerManager.class); final ServerName SERVERNAME_A = ServerName.valueOf("mockserverbulk_a.org", 1000, 8000); final ServerName SERVERNAME_B = ServerName.valueOf("mockserverbulk_b.org", 1001, 8000); final ServerName SERVERNAME_C = ServerName.valueOf("mockserverbulk_c.org", 1002, 8000); final ServerName SERVERNAME_D = ServerName.valueOf("mockserverbulk_d.org", 1003, 8000); final ServerName SERVERNAME_E = ServerName.valueOf("mockserverbulk_e.org", 1004, 8000); final Map<HRegionInfo, ServerName> bulk = new HashMap<HRegionInfo, ServerName>(); Set<ServerName> bunchServersAssigned = new HashSet<ServerName>(); HRegionInfo REGIONINFO_A = new HRegionInfo( TableName.valueOf("table_A"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_B = new HRegionInfo( TableName.valueOf("table_B"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_C = new HRegionInfo( TableName.valueOf("table_C"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_D = new HRegionInfo( TableName.valueOf("table_D"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); HRegionInfo REGIONINFO_E = new HRegionInfo( TableName.valueOf("table_E"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); Map<ServerName, ServerLoad> onlineServers = new HashMap<ServerName, ServerLoad>(); List<ServerName> drainedServers = new ArrayList<ServerName>(); onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_C, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_D, ServerLoad.EMPTY_SERVERLOAD); onlineServers.put(SERVERNAME_E, ServerLoad.EMPTY_SERVERLOAD); bulk.put(REGIONINFO_A, SERVERNAME_A); bulk.put(REGIONINFO_B, SERVERNAME_B); bulk.put(REGIONINFO_C, SERVERNAME_C); bulk.put(REGIONINFO_D, SERVERNAME_D); bulk.put(REGIONINFO_E, SERVERNAME_E); ZooKeeperWatcher zkWatcher = new ZooKeeperWatcher( TEST_UTIL.getConfiguration(), "zkWatcher-BulkAssignTest", abortable, true); Mockito.when(server.getConfiguration()).thenReturn(conf); Mockito.when(server.getServerName()).thenReturn(ServerName.valueOf("masterMock,1,1")); Mockito.when(server.getZooKeeper()).thenReturn(zkWatcher); CoordinatedStateManager cp = new ZkCoordinatedStateManager(); cp.initialize(server); cp.start(); Mockito.when(server.getCoordinatedStateManager()).thenReturn(cp); Mockito.when(serverManager.getOnlineServers()).thenReturn(onlineServers); Mockito.when(serverManager.getOnlineServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList()) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); Mockito.when(serverManager.createDestinationServersList(null)) .thenReturn(new ArrayList<ServerName>(onlineServers.keySet())); for (Entry<HRegionInfo, ServerName> entry : bulk.entrySet()) { Mockito.when(serverManager.isServerOnline(entry.getValue())).thenReturn(true); Mockito.when(serverManager.sendRegionClose(entry.getValue(), entry.getKey(), -1)) .thenReturn(true); Mockito.when(serverManager.sendRegionOpen(entry.getValue(), entry.getKey(), -1, null)) .thenReturn(RegionOpeningState.OPENED); Mockito.when(serverManager.addServerToDrainList(entry.getValue())).thenReturn(true); } Mockito.when(master.getServerManager()).thenReturn(serverManager); drainedServers.add(SERVERNAME_A); drainedServers.add(SERVERNAME_B); drainedServers.add(SERVERNAME_C); drainedServers.add(SERVERNAME_D); am = new AssignmentManager( server, serverManager, balancer, startupMasterExecutor("mockExecutorServiceBulk"), null, null); Mockito.when(master.getAssignmentManager()).thenReturn(am); zkWatcher.registerListener(am); for (ServerName drained : drainedServers) { addServerToDrainedList(drained, onlineServers, serverManager); } am.assign(bulk); Map<String, RegionState> regionsInTransition = am.getRegionStates().getRegionsInTransition(); for (Entry<String, RegionState> entry : regionsInTransition.entrySet()) { setRegionOpenedOnZK( zkWatcher, entry.getValue().getServerName(), entry.getValue().getRegion()); } am.waitForAssignment(REGIONINFO_A); am.waitForAssignment(REGIONINFO_B); am.waitForAssignment(REGIONINFO_C); am.waitForAssignment(REGIONINFO_D); am.waitForAssignment(REGIONINFO_E); Map<HRegionInfo, ServerName> regionAssignments = am.getRegionStates().getRegionAssignments(); for (Entry<HRegionInfo, ServerName> entry : regionAssignments.entrySet()) { LOG.info( "Region Assignment: " + entry.getKey().getRegionNameAsString() + " Server: " + entry.getValue()); bunchServersAssigned.add(entry.getValue()); } for (ServerName sn : drainedServers) { assertFalse(bunchServersAssigned.contains(sn)); } }
@Test public void testLogCleaning() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); // set TTL long ttl = 10000; conf.setLong("hbase.master.logcleaner.ttl", ttl); conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT); Replication.decorateMasterConfiguration(conf); Server server = new DummyServer(); ReplicationQueues repQueues = ReplicationFactory.getReplicationQueues(server.getZooKeeper(), conf, server); repQueues.init(server.getServerName().toString()); final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HREGION_OLDLOGDIR_NAME); String fakeMachineName = URLEncoder.encode(server.getServerName().toString(), "UTF8"); final FileSystem fs = FileSystem.get(conf); // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files long now = System.currentTimeMillis(); fs.delete(oldLogDir, true); fs.mkdirs(oldLogDir); // Case 1: 2 invalid files, which would be deleted directly fs.createNewFile(new Path(oldLogDir, "a")); fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a")); // Case 2: 1 "recent" file, not even deletable for the first log cleaner // (TimeToLiveLogCleaner), so we are not going down the chain System.out.println("Now is: " + now); for (int i = 1; i < 31; i++) { // Case 3: old files which would be deletable for the first log cleaner // (TimeToLiveLogCleaner), and also for the second (ReplicationLogCleaner) Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i)); fs.createNewFile(fileName); // Case 4: put 3 old log files in ZK indicating that they are scheduled // for replication so these files would pass the first log cleaner // (TimeToLiveLogCleaner) but would be rejected by the second // (ReplicationLogCleaner) if (i % (30 / 3) == 1) { repQueues.addLog(fakeMachineName, fileName.getName()); System.out.println("Replication log file: " + fileName); } } // sleep for sometime to get newer modifcation time Thread.sleep(ttl); fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now)); // Case 2: 1 newer file, not even deletable for the first log cleaner // (TimeToLiveLogCleaner), so we are not going down the chain fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000))); for (FileStatus stat : fs.listStatus(oldLogDir)) { System.out.println(stat.getPath().toString()); } assertEquals(34, fs.listStatus(oldLogDir).length); LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir); cleaner.chore(); // We end up with the current log file, a newer one and the 3 old log // files which are scheduled for replication TEST_UTIL.waitFor( 1000, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return 5 == fs.listStatus(oldLogDir).length; } }); for (FileStatus file : fs.listStatus(oldLogDir)) { System.out.println("Kept log files: " + file.getPath().getName()); } }