@Test(timeout = 300000)
  public void testCyclicReplication() throws Exception {
    LOG.info("testCyclicReplication");
    utility1.startMiniCluster();
    utility2.startMiniCluster();
    utility3.startMiniCluster();
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
    ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
    ReplicationAdmin admin3 = new ReplicationAdmin(conf3);

    new HBaseAdmin(conf1).createTable(table);
    new HBaseAdmin(conf2).createTable(table);
    new HBaseAdmin(conf3).createTable(table);
    HTable htable1 = new HTable(conf1, tableName);
    htable1.setWriteBufferSize(1024);
    HTable htable2 = new HTable(conf2, tableName);
    htable2.setWriteBufferSize(1024);
    HTable htable3 = new HTable(conf3, tableName);
    htable3.setWriteBufferSize(1024);

    admin1.addPeer("1", clusterKey2);
    admin2.addPeer("1", clusterKey3);
    admin3.addPeer("1", clusterKey1);

    // put "row" and wait 'til it got around
    putAndWait(row, famName, htable1, htable3);
    // it should have passed through table2
    check(row, famName, htable2);

    putAndWait(row1, famName, htable2, htable1);
    check(row, famName, htable3);
    putAndWait(row2, famName, htable3, htable2);
    check(row, famName, htable1);

    deleteAndWait(row, htable1, htable3);
    deleteAndWait(row1, htable2, htable1);
    deleteAndWait(row2, htable3, htable2);

    assertEquals("Puts were replicated back ", 3, getCount(htable1, put));
    assertEquals("Puts were replicated back ", 3, getCount(htable2, put));
    assertEquals("Puts were replicated back ", 3, getCount(htable3, put));
    assertEquals("Deletes were replicated back ", 3, getCount(htable1, delete));
    assertEquals("Deletes were replicated back ", 3, getCount(htable2, delete));
    assertEquals("Deletes were replicated back ", 3, getCount(htable3, delete));
    utility3.shutdownMiniCluster();
    utility2.shutdownMiniCluster();
    utility1.shutdownMiniCluster();
  }
Esempio n. 2
0
  @BeforeClass
  public static void setUpOnce() throws Exception {
    miniCluster = System.getProperty("cluster.type").equals("mini");
    securedCluster = System.getProperty("cluster.secured").equals("true");
    System.out.println("realCluster - " + !miniCluster);
    System.out.println("securedCluster - " + securedCluster);

    Util.setLoggingThreshold("ERROR");

    if (miniCluster) {
      if (hbase == null) {
        hbase = new HBaseTestingUtility();
        conf = hbase.getConfiguration();
        conf.set("zookeeper.session.timeout", "3600000");
        conf.set("dfs.client.socket-timeout", "3600000");

        if (securedCluster) {
          hbase.startMiniCluster(RS_COUNT);
          hbase.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME, 30000L);
          admin = new HBaseAdminWrapper(conf);
        } else {
          hbase.startMiniCluster(RS_COUNT);
          admin = hbase.getHBaseAdmin();
        }
      }
    } else {
      if (admin == null) {
        final String argsFileName =
            securedCluster
                ? "../../testClusterRealSecured.args"
                : "../../testClusterRealNonSecured.args";
        if (!Util.isFile(argsFileName)) {
          throw new IllegalStateException(
              "You have to define args file " + argsFileName + " for tests.");
        }

        String[] testArgs = {argsFileName};
        Args args = new TestArgs(testArgs);
        admin = HBaseClient.getAdmin(args);
        conf = admin.getConfiguration();
        RS_COUNT = getServerNameList().size();
      }
    }
    previousBalancerRunning = admin.setBalancerRunning(false, true);
    hConnection = HConnectionManager.createConnection(conf);

    USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
  }
Esempio n. 3
0
 @BeforeClass
 public static void setUpBeforeClass() throws Exception {
   TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
   TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
   TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
   TEST_UTIL.startMiniCluster(3);
 }
 @Before
 public void before() throws Exception {
   testingUtility = new HBaseTestingUtility();
   testingUtility.startMiniZKCluster();
   testingUtility.startMiniCluster(1);
   hTable = testingUtility.createTable(TABLE, CF);
 }
  /**
   * A set up method to start the test cluster. AggregateProtocolImpl is registered and will be
   * loaded during region startup.
   *
   * @throws Exception
   */
  @BeforeClass
  public static void setupBeforeClass() throws Exception {

    conf.set(
        CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
        "org.apache.hadoop.hbase.coprocessor.AggregateImplementation");

    util.startMiniCluster(2);
    HTable table = util.createTable(TEST_TABLE, TEST_FAMILY);
    util.createMultiRegions(
        util.getConfiguration(),
        table,
        TEST_FAMILY,
        new byte[][] {HConstants.EMPTY_BYTE_ARRAY, ROWS[rowSeperator1], ROWS[rowSeperator2]});
    /**
     * The testtable has one CQ which is always populated and one variable CQ for each row rowkey1:
     * CF:CQ CF:CQ1 rowKey2: CF:CQ CF:CQ2
     */
    for (int i = 0; i < ROWSIZE; i++) {
      Put put = new Put(ROWS[i]);
      put.setWriteToWAL(false);
      Long l = new Long(i);
      put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(l));
      table.put(put);
      Put p2 = new Put(ROWS[i]);
      put.setWriteToWAL(false);
      p2.add(TEST_FAMILY, Bytes.add(TEST_MULTI_CQ, Bytes.toBytes(l)), Bytes.toBytes(l * 10));
      table.put(p2);
    }
    table.close();
  }
  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    TestHelper.setupLogging();
    TEST_UTIL.startMiniCluster(1);

    IndexManager.createIndexMetaTable(TEST_UTIL.getConfiguration());
  }
 @Test(timeout = 60000)
 public void testExceptionDuringInitialization() throws Exception {
   Configuration conf = TEST_UTIL.getConfiguration();
   conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast.
   conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
   conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
   TEST_UTIL.startMiniCluster(2);
   try {
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
     // Trigger one regionserver to fail as if it came up with a coprocessor
     // that fails during initialization
     final HRegionServer regionServer = cluster.getRegionServer(0);
     conf.set(
         CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
         FailedInitializationObserver.class.getName());
     regionServer
         .getRegionServerCoprocessorHost()
         .loadSystemCoprocessors(conf, CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
     TEST_UTIL.waitFor(
         10000,
         1000,
         new Predicate<Exception>() {
           @Override
           public boolean evaluate() throws Exception {
             return regionServer.isAborted();
           }
         });
   } finally {
     TEST_UTIL.shutdownMiniCluster();
   }
 }
Esempio n. 8
0
 @BeforeClass
 public static void beforeAllTests() throws Exception {
   // we will retry operations when PleaseHoldException is thrown
   TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
   // Start a cluster of two regionservers.
   TEST_UTIL.startMiniCluster(2);
   admin = TEST_UTIL.getAdmin();
 }
Esempio n. 9
0
 /** @throws java.lang.Exception */
 @BeforeClass
 public static void setUpBeforeClass() throws Exception {
   TEST_UTIL.startMiniCluster();
   TEST_UTIL.startMiniMapReduceCluster();
   Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
   writeRows(table);
   table.close();
 }
 @BeforeClass
 public static void provisionCluster() throws Exception {
   conf = util.getConfiguration();
   conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName());
   conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName());
   util.startMiniCluster();
   util.startMiniMapReduceCluster();
 }
Esempio n. 11
0
 @BeforeClass
 public static void beforeClass() throws Exception {
   UTIL.startMiniCluster();
   HTable table =
       UTIL.createTable(MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY});
   UTIL.createMultiRegions(table, INPUT_FAMILY);
   UTIL.loadTable(table, INPUT_FAMILY);
   UTIL.startMiniMapReduceCluster();
 }
Esempio n. 12
0
 @BeforeClass
 public static void setUpBeforeClass() throws Exception {
   conf = TEST_UTIL.getConfiguration();
   TEST_UTIL.startMiniCluster();
   REST_TEST_UTIL.startServletContainer(conf);
   client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort()));
   testTableSchemaModel = new TestTableSchemaModel();
   context = JAXBContext.newInstance(ColumnSchemaModel.class, TableSchemaModel.class);
 }
 @BeforeClass
 public static void beforeAllTests() throws Exception {
   // Start a cluster
   TEST_UTIL.startMiniCluster(1, NB_SERVERS, null, MyMaster.class, null);
   MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
   master = cluster.getMaster();
   master.balanceSwitch(false);
   admin = TEST_UTIL.getHBaseAdmin();
 }
Esempio n. 14
0
 @BeforeClass
 public static void setUpBeforeClass() throws Exception {
   TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
   TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
   TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
   TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
   TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
   TEST_UTIL.startMiniCluster(3);
 }
  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
    util.getConfiguration()
        .setInt(
            LoadIncrementalHFiles.MAX_FILES_PER_REGION_PER_FAMILY, MAX_FILES_PER_REGION_PER_FAMILY);
    util.startMiniCluster();

    setupNamespace();
  }
Esempio n. 16
0
 @BeforeClass
 public static void startCluster() throws Exception {
   LOG.info("Starting cluster");
   TEST_UTIL = new HBaseTestingUtility();
   TEST_UTIL.startMiniCluster(1, 1, 1, null, MyMaster.class, null);
   cluster = TEST_UTIL.getHBaseCluster();
   LOG.info("Waiting for active/ready master");
   cluster.waitForActiveAndReadyMaster();
   master = cluster.getMaster();
 }
 @BeforeClass
 public static void setupBeforeClass() throws Exception {
   Configuration conf = util.getConfiguration();
   conf.setStrings(
       CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, RowProcessorEndpoint.class.getName());
   conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
   conf.setLong("hbase.hregion.row.processor.timeout", 1000L);
   conf.setLong(RpcScheduler.IPC_SERVER_MAX_CALLQUEUE_LENGTH, 2048);
   util.startMiniCluster();
 }
 @BeforeClass
 public static void setupBeforeClass() throws Exception {
   testUtil = new HBaseTestingUtility();
   testUtil.startMiniCluster();
   conf = testUtil.getConfiguration();
   hBaseAdmin = testUtil.getHBaseAdmin();
   txStateStorage = new InMemoryTransactionStateStorage();
   txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
   txManager.startAndWait();
 }
Esempio n. 19
0
 /** @throws java.lang.Exception */
 @BeforeClass
 public static void setUpBeforeClass() throws Exception {
   TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
   TEST_UTIL
       .getConfiguration()
       .setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
   TEST_UTIL.startMiniCluster(3);
   SINK = new ReplicationSink(new Configuration(TEST_UTIL.getConfiguration()), STOPPABLE);
   table1 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAME1);
   table2 = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAME2);
 }
 /**
  * Start up a mini cluster and put a small table of many empty regions into it.
  *
  * @throws Exception
  */
 @BeforeClass
 public static void beforeAllTests() throws Exception {
   TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
   TEST_UTIL.startMiniCluster(2);
   // Create a table of three families.  This will assign a region.
   TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
   HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
   int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
   TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
   addToEachStartKey(countOfRegions);
 }
  @BeforeClass
  public static void setupBeforeClass() throws Exception {
    // set configure to indicate which cp should be loaded
    Configuration conf = util.getConfiguration();
    conf.setBoolean("hbase.master.distributed.log.replay", true);
    conf.set(
        CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
        "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver");

    util.startMiniCluster();
    cluster = util.getMiniHBaseCluster();
  }
 @BeforeClass
 public static void beforeClass() throws Exception {
   // Up the handlers; this test needs more than usual.
   UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
   UTIL.startMiniCluster();
   Table table =
       UTIL.createMultiRegionTable(
           MULTI_REGION_TABLE_NAME, new byte[][] {INPUT_FAMILY, OUTPUT_FAMILY});
   UTIL.loadTable(table, INPUT_FAMILY, false);
   UTIL.startMiniMapReduceCluster();
   UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
 }
  /**
   * Add a row to a table in each cluster, check it's replicated, delete it, check's gone Also check
   * the puts and deletes are not replicated back to the originating cluster.
   */
  @Test(timeout = 300000)
  public void testSimplePutDelete() throws Exception {
    LOG.info("testSimplePutDelete");
    utility1.startMiniCluster();
    utility2.startMiniCluster();

    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
    ReplicationAdmin admin2 = new ReplicationAdmin(conf2);

    new HBaseAdmin(conf1).createTable(table);
    new HBaseAdmin(conf2).createTable(table);
    HTable htable1 = new HTable(conf1, tableName);
    htable1.setWriteBufferSize(1024);
    HTable htable2 = new HTable(conf2, tableName);
    htable2.setWriteBufferSize(1024);

    // set M-M
    admin1.addPeer("1", clusterKey2);
    admin2.addPeer("1", clusterKey1);

    // add rows to both clusters,
    // make sure they are both replication
    putAndWait(row, famName, htable1, htable2);
    putAndWait(row1, famName, htable2, htable1);

    // make sure "row" did not get replicated back.
    assertEquals("Puts were replicated back ", 2, getCount(htable1, put));

    // delete "row" and wait
    deleteAndWait(row, htable1, htable2);

    // make the 2nd cluster replicated back
    assertEquals("Puts were replicated back ", 2, getCount(htable2, put));

    deleteAndWait(row1, htable2, htable1);

    assertEquals("Deletes were replicated back ", 2, getCount(htable1, delete));
    utility2.shutdownMiniCluster();
    utility1.shutdownMiniCluster();
  }
  @Test(timeout = 60000)
  public void testExceptionFromCoprocessorDuringPut() throws Exception {
    // set configure to indicate which cp should be loaded
    Configuration conf = TEST_UTIL.getConfiguration();
    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); // Let's fail fast.
    conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, BuggyRegionObserver.class.getName());
    conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
    TEST_UTIL.startMiniCluster(2);
    try {
      // When we try to write to TEST_TABLE, the buggy coprocessor will
      // cause a NullPointerException, which will cause the regionserver (which
      // hosts the region we attempted to write to) to abort.
      final byte[] TEST_FAMILY = Bytes.toBytes("aaa");

      Table table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, TEST_FAMILY);
      TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME);

      // Note which regionServer will abort (after put is attempted).
      final HRegionServer regionServer = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME);

      try {
        final byte[] ROW = Bytes.toBytes("aaa");
        Put put = new Put(ROW);
        put.add(TEST_FAMILY, ROW, ROW);
        table.put(put);
      } catch (IOException e) {
        // The region server is going to be aborted.
        // We may get an exception if we retry,
        // which is not guaranteed.
      }

      // Wait 10 seconds for the regionserver to abort: expected result is that
      // it will abort.
      boolean aborted = false;
      for (int i = 0; i < 10; i++) {
        aborted = regionServer.isAborted();
        if (aborted) {
          break;
        }
        try {
          Thread.sleep(1000);
        } catch (InterruptedException e) {
          fail("InterruptedException while waiting for regionserver " + "zk node to be deleted.");
        }
      }
      Assert.assertTrue("The region server should have aborted", aborted);
      table.close();
    } finally {
      TEST_UTIL.shutdownMiniCluster();
    }
  }
Esempio n. 25
0
  @BeforeClass
  public static void before() throws Exception {
    HTU.startMiniCluster(NB_SERVERS);
    final TableName tableName = TableName.valueOf(TestRegionServerNoMaster.class.getSimpleName());

    // Create table then get the single region for our new table.
    table = HTU.createTable(tableName, HConstants.CATALOG_FAMILY);
    Put p = new Put(row);
    p.add(HConstants.CATALOG_FAMILY, row, row);
    table.put(p);

    hri = table.getRegionLocation(row, false).getRegionInfo();
    regionName = hri.getRegionName();

    stopMasterAndAssignMeta(HTU);
  }
Esempio n. 26
0
 @BeforeClass
 public static void before() throws Exception {
   UTIL.getConfiguration()
       .set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
   UTIL.getConfiguration()
       .set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MasterSyncObserver.class.getName());
   Configuration conf = UTIL.getConfiguration();
   conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
   conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
   conf.setClass(
       "hbase.coprocessor.regionserver.classes",
       CPRegionServerObserver.class,
       RegionServerObserver.class);
   UTIL.startMiniCluster(1, 1);
   waitForQuotaEnabled();
   ADMIN = UTIL.getHBaseAdmin();
 }
Esempio n. 27
0
  @BeforeClass
  public static void beforeClass() throws Exception {
    // enable store file refreshing
    HTU.getConfiguration()
        .setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, REFRESH_PERIOD);

    HTU.getConfiguration().setFloat("hbase.regionserver.logroll.multiplier", 0.0001f);
    HTU.getConfiguration().setInt("replication.source.size.capacity", 10240);
    HTU.getConfiguration().setLong("replication.source.sleepforretries", 100);
    HTU.getConfiguration().setInt("hbase.regionserver.maxlogs", 2);
    HTU.getConfiguration().setLong("hbase.master.logcleaner.ttl", 10);
    HTU.getConfiguration().setInt("zookeeper.recovery.retry", 1);
    HTU.getConfiguration().setInt("zookeeper.recovery.retry.intervalmill", 10);

    HTU.startMiniCluster(NB_SERVERS);
    HTU.getHBaseCluster().startMaster();
  }
  @BeforeClass
  public static void setupBeforeClass() throws Exception {
    conf = TEST_UTIL.getConfiguration();
    // Enable security
    enableSecurity(conf);
    // Verify enableSecurity sets up what we require
    verifyConfiguration(conf);
    TEST_UTIL.startMiniCluster();
    // Wait for the ACL table to become available
    TEST_UTIL.waitUntilAllRegionsAssigned(AccessControlLists.ACL_TABLE_NAME);

    TESTGROUP1_USER1 =
        User.createUserForTesting(conf, "testgroup1_user1", new String[] {TESTGROUP_1});
    TESTGROUP2_USER1 =
        User.createUserForTesting(conf, "testgroup2_user2", new String[] {TESTGROUP_2});

    connection = ConnectionFactory.createConnection(conf);
  }
  /** Atomic bulk load. */
  @Test
  public void testAtomicBulkLoad() throws Exception {
    TableName TABLE_NAME = TableName.valueOf("atomicBulkLoad");

    int millisToRun = 30000;
    int numScanners = 50;

    UTIL.startMiniCluster(1);
    try {
      WAL log = UTIL.getHBaseCluster().getRegionServer(0).getWAL(null);
      FindBulkHBaseListener listener = new FindBulkHBaseListener();
      log.registerWALActionsListener(listener);
      runAtomicBulkloadTest(TABLE_NAME, millisToRun, numScanners);
      assertThat(listener.isFound(), is(true));
    } finally {
      UTIL.shutdownMiniCluster();
    }
  }
Esempio n. 30
0
  @Test(timeout = 300000)
  public void testClusterRestart() throws Exception {
    UTIL.startMiniCluster(3);
    while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
      Threads.sleep(1);
    }
    LOG.info("\n\nCreating tables");
    for (byte[] TABLE : TABLES) {
      UTIL.createTable(TABLE, FAMILY);
    }
    for (byte[] TABLE : TABLES) {
      UTIL.waitTableEnabled(TABLE);
    }

    List<HRegionInfo> allRegions = MetaScanner.listAllRegions(UTIL.getConfiguration(), true);
    assertEquals(4, allRegions.size());

    LOG.info("\n\nShutting down cluster");
    UTIL.shutdownMiniHBaseCluster();

    LOG.info("\n\nSleeping a bit");
    Thread.sleep(2000);

    LOG.info("\n\nStarting cluster the second time");
    UTIL.restartHBaseCluster(3);

    // Need to use a new 'Configuration' so we make a new HConnection.
    // Otherwise we're reusing an HConnection that has gone stale because
    // the shutdown of the cluster also called shut of the connection.
    allRegions = MetaScanner.listAllRegions(new Configuration(UTIL.getConfiguration()), true);
    assertEquals(4, allRegions.size());
    LOG.info("\n\nWaiting for tables to be available");
    for (byte[] TABLE : TABLES) {
      try {
        UTIL.createTable(TABLE, FAMILY);
        assertTrue("Able to create table that should already exist", false);
      } catch (TableExistsException tee) {
        LOG.info("Table already exists as expected");
      }
      UTIL.waitTableAvailable(TABLE);
    }
  }