private List<HRegionServer> getOnlineRegionServers() {
List<HRegionServer> list = new ArrayList<HRegionServer>();
for (JVMClusterUtil.RegionServerThread rst : cluster.getRegionServerThreads()) {
if (rst.getRegionServer().isOnline()) {
list.add(rst.getRegionServer());
}
}
return list;
}
@Test
public void testHostRank() throws Exception {
if (System.getProperty("prop.mapred.job.tracker") != null) {
if (LOG.isInfoEnabled())
LOG.info("testHBaseInputOutput: Ignore this test if not local mode.");
return;
}
File jarTest = new File(System.getProperty("prop.jarLocation"));
if (!jarTest.exists()) {
fail(
"Could not find Giraph jar at "
+ "location specified by 'prop.jarLocation'. "
+ "Make sure you built the main Giraph artifact?.");
}
MiniHBaseCluster cluster = null;
MiniZooKeeperCluster zkCluster = null;
FileSystem fs = null;
try {
// using the restart method allows us to avoid having the hbase
// root directory overwritten by /home/$username
zkCluster = testUtil.startMiniZKCluster();
testUtil.restartHBaseCluster(2);
cluster = testUtil.getMiniHBaseCluster();
final byte[] OL_BYTES = Bytes.toBytes("ol");
final byte[] S_BYTES = Bytes.toBytes("s");
final byte[] METADATA_BYTES = Bytes.toBytes("mtdt");
final byte[] HR_BYTES = Bytes.toBytes("_hr_");
final byte[] TAB = Bytes.toBytes(TABLE_NAME);
Configuration conf = cluster.getConfiguration();
HTableDescriptor desc = new HTableDescriptor(TAB);
desc.addFamily(new HColumnDescriptor(OL_BYTES));
desc.addFamily(new HColumnDescriptor(S_BYTES));
desc.addFamily(new HColumnDescriptor(METADATA_BYTES));
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
if (hbaseAdmin.isTableAvailable(TABLE_NAME)) {
hbaseAdmin.disableTable(TABLE_NAME);
hbaseAdmin.deleteTable(TABLE_NAME);
}
hbaseAdmin.createTable(desc);
/**
* Enter the initial data (a,b), (b,c), (a,c) a = 1.0 - google b = 1.0 - yahoo c = 1.0 - bing
*/
HTable table = new HTable(conf, TABLE_NAME);
Put p1 = new Put(Bytes.toBytes("com.google.www"));
p1.add(OL_BYTES, Bytes.toBytes("www.yahoo.com"), Bytes.toBytes("ab"));
Put p2 = new Put(Bytes.toBytes("com.google.www"));
p2.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("ac"));
p2.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("invalid1"));
p2.add(OL_BYTES, Bytes.toBytes("www.google.com"), Bytes.toBytes("invalid2"));
Put p3 = new Put(Bytes.toBytes("com.yahoo.www"));
p3.add(OL_BYTES, Bytes.toBytes("www.bing.com"), Bytes.toBytes("bc"));
// p3.add(OL_BYTES, Bytes.toBytes(""), Bytes.toBytes("invalid4"));
Put p4 = new Put(Bytes.toBytes("com.bing.www"));
// TODO: Handle below case. use apache isValid method.
p4.add(OL_BYTES, Bytes.toBytes("http://invalidurl"), Bytes.toBytes("invalid5"));
p4.add(S_BYTES, S_BYTES, Bytes.toBytes(10.0d));
Put p5 = new Put(Bytes.toBytes("dummy"));
p5.add(S_BYTES, S_BYTES, Bytes.toBytes(10.0d));
table.put(p1);
table.put(p2);
table.put(p3);
table.put(p4);
table.put(p5);
// Set Giraph configuration
// now operate over HBase using Vertex I/O formats
conf.set(TableInputFormat.INPUT_TABLE, TABLE_NAME);
conf.set(TableOutputFormat.OUTPUT_TABLE, TABLE_NAME);
// Start the giraph job
GiraphJob giraphJob = new GiraphJob(conf, BspCase.getCallingMethodName());
GiraphConfiguration giraphConf = giraphJob.getConfiguration();
giraphConf.setZooKeeperConfiguration(cluster.getMaster().getZooKeeper().getQuorum());
setupConfiguration(giraphJob);
giraphConf.setComputationClass(LinkRankComputation.class);
giraphConf.setMasterComputeClass(LinkRankVertexMasterCompute.class);
giraphConf.setOutEdgesClass(ByteArrayEdges.class);
giraphConf.setVertexInputFormatClass(Nutch2HostInputFormat.class);
giraphConf.setVertexOutputFormatClass(Nutch2HostOutputFormat.class);
giraphConf.setInt("giraph.linkRank.superstepCount", 10);
giraphConf.setInt("giraph.linkRank.scale", 10);
giraphConf.set("giraph.linkRank.family", "mtdt");
giraphConf.set("giraph.linkRank.qualifier", "_hr_");
giraphConf.setVertexInputFilterClass(HostRankVertexFilter.class);
assertTrue(giraphJob.run(true));
if (LOG.isInfoEnabled()) LOG.info("Giraph job successful. Checking output qualifier.");
/** Check the results * */
Result result;
String key;
byte[] calculatedScoreByte;
HashMap expectedValues = new HashMap<String, Double>();
expectedValues.put("com.google.www", 1.3515060339386287d);
expectedValues.put("com.yahoo.www", 4.144902009567587d);
expectedValues.put("com.bing.www", 9.063893290511482d);
for (Object keyObject : expectedValues.keySet()) {
key = keyObject.toString();
result = table.get(new Get(key.getBytes()));
calculatedScoreByte = result.getValue(METADATA_BYTES, HR_BYTES);
assertNotNull(calculatedScoreByte);
assertTrue(calculatedScoreByte.length > 0);
Assert.assertEquals(
"Scores are not the same",
(Double) expectedValues.get(key),
Bytes.toDouble(calculatedScoreByte),
DELTA);
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
if (zkCluster != null) {
zkCluster.shutdown();
}
// clean test files
if (fs != null) {
fs.delete(hbaseRootdir);
}
}
}
@Test(timeout = 300000)
public void testMultiSlaveReplication() throws Exception {
LOG.info("testCyclicReplication");
MiniHBaseCluster master = utility1.startMiniCluster();
utility2.startMiniCluster();
utility3.startMiniCluster();
ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
new HBaseAdmin(conf1).createTable(table);
new HBaseAdmin(conf2).createTable(table);
new HBaseAdmin(conf3).createTable(table);
HTable htable1 = new HTable(conf1, tableName);
htable1.setWriteBufferSize(1024);
HTable htable2 = new HTable(conf2, tableName);
htable2.setWriteBufferSize(1024);
HTable htable3 = new HTable(conf3, tableName);
htable3.setWriteBufferSize(1024);
admin1.addPeer("1", utility2.getClusterKey());
// put "row" and wait 'til it got around, then delete
putAndWait(row, famName, htable1, htable2);
deleteAndWait(row, htable1, htable2);
// check it wasn't replication to cluster 3
checkRow(row, 0, htable3);
putAndWait(row2, famName, htable1, htable2);
// now roll the region server's logs
new HBaseAdmin(conf1).rollHLogWriter(master.getRegionServer(0).getServerName().toString());
// after the log was rolled put a new row
putAndWait(row3, famName, htable1, htable2);
admin1.addPeer("2", utility3.getClusterKey());
// put a row, check it was replicated to all clusters
putAndWait(row1, famName, htable1, htable2, htable3);
// delete and verify
deleteAndWait(row1, htable1, htable2, htable3);
// make sure row2 did not get replicated after
// cluster 3 was added
checkRow(row2, 0, htable3);
// row3 will get replicated, because it was in the
// latest log
checkRow(row3, 1, htable3);
Put p = new Put(row);
p.add(famName, row, row);
htable1.put(p);
// now roll the logs again
new HBaseAdmin(conf1).rollHLogWriter(master.getRegionServer(0).getServerName().toString());
// cleanup "row2", also conveniently use this to wait replication
// to finish
deleteAndWait(row2, htable1, htable2, htable3);
// Even if the log was rolled in the middle of the replication
// "row" is still replication.
checkRow(row, 1, htable2, htable3);
// cleanup the rest
deleteAndWait(row, htable1, htable2, htable3);
deleteAndWait(row3, htable1, htable2, htable3);
utility3.shutdownMiniCluster();
utility2.shutdownMiniCluster();
utility1.shutdownMiniCluster();
}
/**
* Test the global mem store size in the region server is equal to sum of each region's mem store
* size
*
* @throws Exception
*/
@Test
public void testGlobalMemStore() throws Exception {
// Start the cluster
LOG.info("Starting cluster");
Configuration conf = HBaseConfiguration.create();
TEST_UTIL = new HBaseTestingUtility(conf);
TEST_UTIL.startMiniCluster(1, regionServerNum);
cluster = TEST_UTIL.getHBaseCluster();
LOG.info("Waiting for active/ready master");
cluster.waitForActiveAndReadyMaster();
// Create a table with regions
TableName table = TableName.valueOf("TestGlobalMemStoreSize");
byte[] family = Bytes.toBytes("family");
LOG.info("Creating table with " + regionNum + " regions");
Table ht = TEST_UTIL.createMultiRegionTable(table, family, regionNum);
int numRegions = -1;
try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(table)) {
numRegions = r.getStartKeys().length;
}
assertEquals(regionNum, numRegions);
waitForAllRegionsAssigned();
for (HRegionServer server : getOnlineRegionServers()) {
long globalMemStoreSize = 0;
for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server.getRSRpcServices())) {
globalMemStoreSize +=
server.getFromOnlineRegions(regionInfo.getEncodedName()).getMemstoreSize();
}
assertEquals(server.getRegionServerAccounting().getGlobalMemstoreSize(), globalMemStoreSize);
}
// check the global memstore size after flush
int i = 0;
for (HRegionServer server : getOnlineRegionServers()) {
LOG.info(
"Starting flushes on "
+ server.getServerName()
+ ", size="
+ server.getRegionServerAccounting().getGlobalMemstoreSize());
for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server.getRSRpcServices())) {
Region r = server.getFromOnlineRegions(regionInfo.getEncodedName());
flush(r, server);
}
LOG.info("Post flush on " + server.getServerName());
long now = System.currentTimeMillis();
long timeout = now + 1000;
while (server.getRegionServerAccounting().getGlobalMemstoreSize() != 0
&& timeout < System.currentTimeMillis()) {
Threads.sleep(10);
}
long size = server.getRegionServerAccounting().getGlobalMemstoreSize();
if (size > 0) {
// If size > 0, see if its because the meta region got edits while
// our test was running....
for (HRegionInfo regionInfo : ProtobufUtil.getOnlineRegions(server.getRSRpcServices())) {
Region r = server.getFromOnlineRegions(regionInfo.getEncodedName());
long l = r.getMemstoreSize();
if (l > 0) {
// Only meta could have edits at this stage. Give it another flush
// clear them.
assertTrue(regionInfo.isMetaRegion());
LOG.info(r.toString() + " " + l + ", reflushing");
r.flush(true);
}
}
}
size = server.getRegionServerAccounting().getGlobalMemstoreSize();
assertEquals("Server=" + server.getServerName() + ", i=" + i++, 0, size);
}
ht.close();
TEST_UTIL.shutdownMiniCluster();
}
