@Test(timeout = 300000)
public void testCyclicReplication() throws Exception {
LOG.info("testCyclicReplication");
utility1.startMiniCluster();
utility2.startMiniCluster();
utility3.startMiniCluster();
ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
ReplicationAdmin admin3 = new ReplicationAdmin(conf3);
new HBaseAdmin(conf1).createTable(table);
new HBaseAdmin(conf2).createTable(table);
new HBaseAdmin(conf3).createTable(table);
HTable htable1 = new HTable(conf1, tableName);
htable1.setWriteBufferSize(1024);
HTable htable2 = new HTable(conf2, tableName);
htable2.setWriteBufferSize(1024);
HTable htable3 = new HTable(conf3, tableName);
htable3.setWriteBufferSize(1024);
admin1.addPeer("1", clusterKey2);
admin2.addPeer("1", clusterKey3);
admin3.addPeer("1", clusterKey1);
// put "row" and wait 'til it got around
putAndWait(row, famName, htable1, htable3);
// it should have passed through table2
check(row, famName, htable2);
putAndWait(row1, famName, htable2, htable1);
check(row, famName, htable3);
putAndWait(row2, famName, htable3, htable2);
check(row, famName, htable1);
deleteAndWait(row, htable1, htable3);
deleteAndWait(row1, htable2, htable1);
deleteAndWait(row2, htable3, htable2);
assertEquals("Puts were replicated back ", 3, getCount(htable1, put));
assertEquals("Puts were replicated back ", 3, getCount(htable2, put));
assertEquals("Puts were replicated back ", 3, getCount(htable3, put));
assertEquals("Deletes were replicated back ", 3, getCount(htable1, delete));
assertEquals("Deletes were replicated back ", 3, getCount(htable2, delete));
assertEquals("Deletes were replicated back ", 3, getCount(htable3, delete));
utility3.shutdownMiniCluster();
utility2.shutdownMiniCluster();
utility1.shutdownMiniCluster();
}
@BeforeClass
public static void setUpOnce() throws Exception {
miniCluster = System.getProperty("cluster.type").equals("mini");
securedCluster = System.getProperty("cluster.secured").equals("true");
System.out.println("realCluster - " + !miniCluster);
System.out.println("securedCluster - " + securedCluster);
Util.setLoggingThreshold("ERROR");
if (miniCluster) {
if (hbase == null) {
hbase = new HBaseTestingUtility();
conf = hbase.getConfiguration();
conf.set("zookeeper.session.timeout", "3600000");
conf.set("dfs.client.socket-timeout", "3600000");
if (securedCluster) {
hbase.startMiniCluster(RS_COUNT);
hbase.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME, 30000L);
admin = new HBaseAdminWrapper(conf);
} else {
hbase.startMiniCluster(RS_COUNT);
admin = hbase.getHBaseAdmin();
}
}
} else {
if (admin == null) {
final String argsFileName =
securedCluster
? "../../testClusterRealSecured.args"
: "../../testClusterRealNonSecured.args";
if (!Util.isFile(argsFileName)) {
throw new IllegalStateException(
"You have to define args file " + argsFileName + " for tests.");
}
String[] testArgs = {argsFileName};
Args args = new TestArgs(testArgs);
admin = HBaseClient.getAdmin(args);
conf = admin.getConfiguration();
RS_COUNT = getServerNameList().size();
}
}
previousBalancerRunning = admin.setBalancerRunning(false, true);
hConnection = HConnectionManager.createConnection(conf);
USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
TEST_UTIL.startMiniCluster(3);
}
@Before
public void before() throws Exception {
testingUtility = new HBaseTestingUtility();
testingUtility.startMiniZKCluster();
testingUtility.startMiniCluster(1);
hTable = testingUtility.createTable(TABLE, CF);
}
/**
* A set up method to start the test cluster. AggregateProtocolImpl is registered and will be
* loaded during region startup.
*
* @throws Exception
*/
@BeforeClass
public static void setupBeforeClass() throws Exception {
conf.set(
CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
"org.apache.hadoop.hbase.coprocessor.AggregateImplementation");
util.startMiniCluster(2);
HTable table = util.createTable(TEST_TABLE, TEST_FAMILY);
util.createMultiRegions(
util.getConfiguration(),
table,
TEST_FAMILY,
new byte[][] {HConstants.EMPTY_BYTE_ARRAY, ROWS[rowSeperator1], ROWS[rowSeperator2]});
/**
* The testtable has one CQ which is always populated and one variable CQ for each row rowkey1:
* CF:CQ CF:CQ1 rowKey2: CF:CQ CF:CQ2
*/
for (int i = 0; i < ROWSIZE; i++) {
Put put = new Put(ROWS[i]);
put.setWriteToWAL(false);
Long l = new Long(i);
put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(l));
table.put(put);
Put p2 = new Put(ROWS[i]);
put.setWriteToWAL(false);
p2.add(TEST_FAMILY, Bytes.add(TEST_MULTI_CQ, Bytes.toBytes(l)), Bytes.toBytes(l * 10));
table.put(p2);
}
table.close();
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TestHelper.setupLogging();
TEST_UTIL.startMiniCluster(1);
IndexManager.createIndexMetaTable(TEST_UTIL.getConfiguration());
}
@Test(timeout = 60000)
public void testExceptionDuringInitialization() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast.
conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
TEST_UTIL.startMiniCluster(2);
try {
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
// Trigger one regionserver to fail as if it came up with a coprocessor
// that fails during initialization
final HRegionServer regionServer = cluster.getRegionServer(0);
conf.set(
CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
FailedInitializationObserver.class.getName());
regionServer
.getRegionServerCoprocessorHost()
.loadSystemCoprocessors(conf, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
TEST_UTIL.waitFor(
10000,
1000,
new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
return regionServer.isAborted();
}
});
} finally {
TEST_UTIL.shutdownMiniCluster();
}
}
@BeforeClass
public static void beforeAllTests() throws Exception {
// we will retry operations when PleaseHoldException is thrown
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
// Start a cluster of two regionservers.
TEST_UTIL.startMiniCluster(2);
admin = TEST_UTIL.getAdmin();
}
/** @throws java.lang.Exception */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniCluster();
TEST_UTIL.startMiniMapReduceCluster();
Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
writeRows(table);
table.close();
}
@BeforeClass
public static void provisionCluster() throws Exception {
conf = util.getConfiguration();
conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName());
conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName());
util.startMiniCluster();
util.startMiniMapReduceCluster();
}
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
HTable table =
UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY});
UTIL.createMultiRegions(table, INPUT_FAMILY);
UTIL.loadTable(table, INPUT_FAMILY);
UTIL.startMiniMapReduceCluster();
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf = TEST_UTIL.getConfiguration();
TEST_UTIL.startMiniCluster();
REST_TEST_UTIL.startServletContainer(conf);
client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
testTableSchemaModel = new TestTableSchemaModel();
context = JAXBContext.newInstance(ColumnSchemaModel.class, TableSchemaModel.class);
}
@BeforeClass
public static void beforeAllTests() throws Exception {
// Start a cluster
TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
master = cluster.getMaster();
master.balanceSwitch(false);
admin = TEST_UTIL.getHBaseAdmin();
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
TEST_UTIL.startMiniCluster(3);
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
util.getConfiguration()
.setInt(
LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, MAX_FILES_PER_REGION_PER_FAMILY);
util.startMiniCluster();
setupNamespace();
}
@BeforeClass
public static void startCluster() throws Exception {
LOG.info("Starting cluster");
TEST_UTIL = new HBaseTestingUtility();
TEST_UTIL.startMiniCluster(1, 1, 1, null, MyMaster.class, null);
cluster = TEST_UTIL.getHBaseCluster();
LOG.info("Waiting for active/ready master");
cluster.waitForActiveAndReadyMaster();
master = cluster.getMaster();
}
@BeforeClass
public static void setupBeforeClass() throws Exception {
Configuration conf = util.getConfiguration();
conf.setStrings(
CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RowProcessorEndpoint.class.getName());
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
conf.setLong("hbase.hregion.row.processor.timeout", 1000L);
conf.setLong(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH, 2048);
util.startMiniCluster();
}
@BeforeClass
public static void setupBeforeClass() throws Exception {
testUtil = new HBaseTestingUtility();
testUtil.startMiniCluster();
conf = testUtil.getConfiguration();
hBaseAdmin = testUtil.getHBaseAdmin();
txStateStorage = new InMemoryTransactionStateStorage();
txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
txManager.startAndWait();
}
/** @throws java.lang.Exception */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
TEST_UTIL
.getConfiguration()
.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
TEST_UTIL.startMiniCluster(3);
SINK = new ReplicationSink(new Configuration(TEST_UTIL.getConfiguration()), STOPPABLE);
table1 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAME1);
table2 = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAME2);
}
/**
* Start up a mini cluster and put a small table of many empty regions into it.
*
* @throws Exception
*/
@BeforeClass
public static void beforeAllTests() throws Exception {
TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
TEST_UTIL.startMiniCluster(2);
// Create a table of three families. This will assign a region.
TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
addToEachStartKey(countOfRegions);
}
@BeforeClass
public static void setupBeforeClass() throws Exception {
// set configure to indicate which cp should be loaded
Configuration conf = util.getConfiguration();
conf.setBoolean("hbase.master.distributed.log.replay", true);
conf.set(
CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
"org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver");
util.startMiniCluster();
cluster = util.getMiniHBaseCluster();
}
@BeforeClass
public static void beforeClass() throws Exception {
// Up the handlers; this test needs more than usual.
UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
UTIL.startMiniCluster();
Table table =
UTIL.createMultiRegionTable(
MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY});
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
}
/**
* Add a row to a table in each cluster, check it's replicated, delete it, check's gone Also check
* the puts and deletes are not replicated back to the originating cluster.
*/
@Test(timeout = 300000)
public void testSimplePutDelete() throws Exception {
LOG.info("testSimplePutDelete");
utility1.startMiniCluster();
utility2.startMiniCluster();
ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
new HBaseAdmin(conf1).createTable(table);
new HBaseAdmin(conf2).createTable(table);
HTable htable1 = new HTable(conf1, tableName);
htable1.setWriteBufferSize(1024);
HTable htable2 = new HTable(conf2, tableName);
htable2.setWriteBufferSize(1024);
// set M-M
admin1.addPeer("1", clusterKey2);
admin2.addPeer("1", clusterKey1);
// add rows to both clusters,
// make sure they are both replication
putAndWait(row, famName, htable1, htable2);
putAndWait(row1, famName, htable2, htable1);
// make sure "row" did not get replicated back.
assertEquals("Puts were replicated back ", 2, getCount(htable1, put));
// delete "row" and wait
deleteAndWait(row, htable1, htable2);
// make the 2nd cluster replicated back
assertEquals("Puts were replicated back ", 2, getCount(htable2, put));
deleteAndWait(row1, htable2, htable1);
assertEquals("Deletes were replicated back ", 2, getCount(htable1, delete));
utility2.shutdownMiniCluster();
utility1.shutdownMiniCluster();
}
@Test(timeout = 60000)
public void testExceptionFromCoprocessorDuringPut() throws Exception {
// set configure to indicate which cp should be loaded
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast.
conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, BuggyRegionObserver.class.getName());
conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
TEST_UTIL.startMiniCluster(2);
try {
// When we try to write to TEST_TABLE, the buggy coprocessor will
// cause a NullPointerException, which will cause the regionserver (which
// hosts the region we attempted to write to) to abort.
final byte[] TEST_FAMILY = Bytes.toBytes("aaa");
Table table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, TEST_FAMILY);
TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME);
// Note which regionServer will abort (after put is attempted).
final HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME);
try {
final byte[] ROW = Bytes.toBytes("aaa");
Put put = new Put(ROW);
put.add(TEST_FAMILY, ROW, ROW);
table.put(put);
} catch (IOException e) {
// The region server is going to be aborted.
// We may get an exception if we retry,
// which is not guaranteed.
}
// Wait 10 seconds for the regionserver to abort: expected result is that
// it will abort.
boolean aborted = false;
for (int i = 0; i < 10; i++) {
aborted = regionServer.isAborted();
if (aborted) {
break;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
fail("InterruptedException while waiting for regionserver " + "zk node to be deleted.");
}
}
Assert.assertTrue("The region server should have aborted", aborted);
table.close();
} finally {
TEST_UTIL.shutdownMiniCluster();
}
}
@BeforeClass
public static void before() throws Exception {
HTU.startMiniCluster(NB_SERVERS);
final TableName tableName = TableName.valueOf(TestRegionServerNoMaster.class.getSimpleName());
// Create table then get the single region for our new table.
table = HTU.createTable(tableName, HConstants.CATALOG_FAMILY);
Put p = new Put(row);
p.add(HConstants.CATALOG_FAMILY, row, row);
table.put(p);
hri = table.getRegionLocation(row, false).getRegionInfo();
regionName = hri.getRegionName();
stopMasterAndAssignMeta(HTU);
}
@BeforeClass
public static void before() throws Exception {
UTIL.getConfiguration()
.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
UTIL.getConfiguration()
.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
Configuration conf = UTIL.getConfiguration();
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
conf.setClass(
"hbase.coprocessor.regionserver.classes",
CPRegionServerObserver.class,
RegionServerObserver.class);
UTIL.startMiniCluster(1, 1);
waitForQuotaEnabled();
ADMIN = UTIL.getHBaseAdmin();
}
@BeforeClass
public static void beforeClass() throws Exception {
// enable store file refreshing
HTU.getConfiguration()
.setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, REFRESH_PERIOD);
HTU.getConfiguration().setFloat("hbase.regionserver.logroll.multiplier", 0.0001f);
HTU.getConfiguration().setInt("replication.source.size.capacity", 10240);
HTU.getConfiguration().setLong("replication.source.sleepforretries", 100);
HTU.getConfiguration().setInt("hbase.regionserver.maxlogs", 2);
HTU.getConfiguration().setLong("hbase.master.logcleaner.ttl", 10);
HTU.getConfiguration().setInt("zookeeper.recovery.retry", 1);
HTU.getConfiguration().setInt("zookeeper.recovery.retry.intervalmill", 10);
HTU.startMiniCluster(NB_SERVERS);
HTU.getHBaseCluster().startMaster();
}
@BeforeClass
public static void setupBeforeClass() throws Exception {
conf = TEST_UTIL.getConfiguration();
// Enable security
enableSecurity(conf);
// Verify enableSecurity sets up what we require
verifyConfiguration(conf);
TEST_UTIL.startMiniCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitUntilAllRegionsAssigned(AccessControlLists.ACL_TABLE_NAME);
TESTGROUP1_USER1 =
User.createUserForTesting(conf, "testgroup1_user1", new String[] {TESTGROUP_1});
TESTGROUP2_USER1 =
User.createUserForTesting(conf, "testgroup2_user2", new String[] {TESTGROUP_2});
connection = ConnectionFactory.createConnection(conf);
}
/** Atomic bulk load. */
@Test
public void testAtomicBulkLoad() throws Exception {
TableName TABLE_NAME = TableName.valueOf("atomicBulkLoad");
int millisToRun = 30000;
int numScanners = 50;
UTIL.startMiniCluster(1);
try {
WAL log = UTIL.getHBaseCluster().getRegionServer(0).getWAL(null);
FindBulkHBaseListener listener = new FindBulkHBaseListener();
log.registerWALActionsListener(listener);
runAtomicBulkloadTest(TABLE_NAME, millisToRun, numScanners);
assertThat(listener.isFound(), is(true));
} finally {
UTIL.shutdownMiniCluster();
}
}
@Test(timeout = 300000)
public void testClusterRestart() throws Exception {
UTIL.startMiniCluster(3);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
Threads.sleep(1);
}
LOG.info("\n\nCreating tables");
for (byte[] TABLE : TABLES) {
UTIL.createTable(TABLE, FAMILY);
}
for (byte[] TABLE : TABLES) {
UTIL.waitTableEnabled(TABLE);
}
List<HRegionInfo> allRegions = MetaScanner.listAllRegions(UTIL.getConfiguration(), true);
assertEquals(4, allRegions.size());
LOG.info("\n\nShutting down cluster");
UTIL.shutdownMiniHBaseCluster();
LOG.info("\n\nSleeping a bit");
Thread.sleep(2000);
LOG.info("\n\nStarting cluster the second time");
UTIL.restartHBaseCluster(3);
// Need to use a new 'Configuration' so we make a new HConnection.
// Otherwise we're reusing an HConnection that has gone stale because
// the shutdown of the cluster also called shut of the connection.
allRegions = MetaScanner.listAllRegions(new Configuration(UTIL.getConfiguration()), true);
assertEquals(4, allRegions.size());
LOG.info("\n\nWaiting for tables to be available");
for (byte[] TABLE : TABLES) {
try {
UTIL.createTable(TABLE, FAMILY);
assertTrue("Able to create table that should already exist", false);
} catch (TableExistsException tee) {
LOG.info("Table already exists as expected");
}
UTIL.waitTableAvailable(TABLE);
}
}
