public void updateRecord(MetaRecord rec) {
try {
long ts = System.currentTimeMillis();
putRecord(rec, ts);
String node = ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(rec.getId()));
// setData会异步触发所有机器(包括本机)上的H2MetaTableTracker.nodeDataChanged
// 然后触发下列调用:
// =>org.h2.engine.Database.updateDatabaseObject(int)
// =>org.h2.engine.Database.update(Session, DbObject)
// =>org.h2.engine.Database.addMeta0(Session, DbObject, boolean)
// =>又到此方法
// 所以会造成循环
synchronized (this) { // 避免setData后立刻触发nodeDataChanged，此时IdVersion还未更新
ZKUtil.setData(watcher, node, Bytes.toBytes(ts));
// setData后watch不见了，所以要继续watch，监听其他人对此node的修改
// ZKUtil.watchAndCheckExists(watcher, node);
Stat stat = new Stat();
ZKUtil.getDataAndWatch(watcher, node, stat);
// 这里记录下id的最新版本，触发nodeDataChanged时再检查一下是否版本一样，
// 如果不大于这里的版本那么就不再执行updateDatabaseObject操作
tracker.updateIdVersion(rec.getId(), stat.getVersion());
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private void waitForNewProcedures() {
// watch for new procedues that we need to start subprocedures for
LOG.debug("Looking for new procedures under znode:'" + zkController.getAcquiredBarrier() + "'");
List<String> runningProcedures = null;
try {
runningProcedures =
ZKUtil.listChildrenAndWatchForNewChildren(
zkController.getWatcher(), zkController.getAcquiredBarrier());
if (runningProcedures == null) {
LOG.debug("No running procedures.");
return;
}
} catch (KeeperException e) {
member.controllerConnectionFailure(
"General failure when watching for new procedures", e, null);
}
if (runningProcedures == null) {
LOG.debug("No running procedures.");
return;
}
for (String procName : runningProcedures) {
// then read in the procedure information
String path = ZKUtil.joinZNode(zkController.getAcquiredBarrier(), procName);
startNewSubprocedure(path);
}
}
@Override
public Configuration getPeerConf(String peerId) throws ReplicationException {
String znode = ZKUtil.joinZNode(this.peersZNode, peerId);
byte[] data = null;
try {
data = ZKUtil.getData(this.zookeeper, znode);
} catch (KeeperException e) {
throw new ReplicationException("Error getting configuration for peer with id=" + peerId, e);
}
if (data == null) {
LOG.error("Could not get configuration for peer because it doesn't exist. peerId=" + peerId);
return null;
}
String otherClusterKey = "";
try {
otherClusterKey = parsePeerFrom(data);
} catch (DeserializationException e) {
LOG.warn(
"Failed to parse cluster key from peerId="
+ peerId
+ ", specifically the content from the following znode: "
+ znode);
return null;
}
Configuration otherConf = new Configuration(this.conf);
try {
ZKUtil.applyClusterKeyToConf(otherConf, otherClusterKey);
} catch (IOException e) {
LOG.error("Can't get peer configuration for peerId=" + peerId + " because:", e);
return null;
}
return otherConf;
}
@Override
public void addPeer(String id, String clusterKey, String tableCFs) throws ReplicationException {
try {
if (peerExists(id)) {
throw new IllegalArgumentException(
"Cannot add a peer with id=" + id + " because that id already exists.");
}
ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
ZKUtil.createAndWatch(
this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id), toByteArray(clusterKey));
// There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the
// peer-state znode. This happens while adding a peer.
// The peer state data is set as "ENABLED" by default.
ZKUtil.createNodeIfNotExistsAndWatch(
this.zookeeper, getPeerStateNode(id), ENABLED_ZNODE_BYTES);
// A peer is enabled by default
String tableCFsStr = (tableCFs == null) ? "" : tableCFs;
ZKUtil.createNodeIfNotExistsAndWatch(
this.zookeeper, getTableCFsNode(id), Bytes.toBytes(tableCFsStr));
} catch (KeeperException e) {
throw new ReplicationException(
"Could not add peer with id=" + id + ", clusterKey=" + clusterKey, e);
}
}
/**
* This attempts to create an acquired state znode for the procedure (snapshot name).
*
* <p>It then looks for the reached znode to trigger in-barrier execution. If not present we have
* a watcher, if present then trigger the in-barrier action.
*/
@Override
public void sendMemberAcquired(Subprocedure sub) throws IOException {
String procName = sub.getName();
try {
LOG.debug(
"Member: '"
+ memberName
+ "' joining acquired barrier for procedure ("
+ procName
+ ") in zk");
String acquiredZNode =
ZKUtil.joinZNode(
ZKProcedureUtil.getAcquireBarrierNode(zkController, procName), memberName);
ZKUtil.createAndFailSilent(zkController.getWatcher(), acquiredZNode);
// watch for the complete node for this snapshot
String reachedBarrier = zkController.getReachedBarrierNode(procName);
LOG.debug("Watch for global barrier reached:" + reachedBarrier);
if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), reachedBarrier)) {
receivedReachedGlobalBarrier(reachedBarrier);
}
} catch (KeeperException e) {
member.controllerConnectionFailure(
"Failed to acquire barrier for procedure: " + procName + " and member: " + memberName,
e,
procName);
}
}
@Test
public void TestMap() throws Exception {
String prefix = "0000";
final String fileName = "19691231f2cd014ea28f42788214560a21a44cef";
final String mobFilePath = prefix + fileName;
ImmutableBytesWritable r = new ImmutableBytesWritable(Bytes.toBytes("r"));
final KeyValue[] kvList = new KeyValue[1];
kvList[0] =
new KeyValue(
Bytes.toBytes("row"),
Bytes.toBytes("family"),
Bytes.toBytes("column"),
Bytes.toBytes(mobFilePath));
Result columns = mock(Result.class);
when(columns.rawCells()).thenReturn(kvList);
Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable());
TableName tn = TableName.valueOf("testSweepMapper");
TableName lockName = MobUtils.getTableLockName(tn);
String znode = ZKUtil.joinZNode(zkw.tableLockZNode, lockName.getNameAsString());
configuration.set(SweepJob.SWEEP_JOB_ID, "1");
configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode);
ServerName serverName = SweepJob.getCurrentServerName(configuration);
configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString());
TableLockManager tableLockManager =
TableLockManager.createTableLockManager(configuration, zkw, serverName);
TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool");
lock.acquire();
try {
Mapper<ImmutableBytesWritable, Result, Text, KeyValue>.Context ctx =
mock(Mapper.Context.class);
when(ctx.getConfiguration()).thenReturn(configuration);
SweepMapper map = new SweepMapper();
doAnswer(
new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocation) throws Throwable {
Text text = (Text) invocation.getArguments()[0];
KeyValue kv = (KeyValue) invocation.getArguments()[1];
assertEquals(Bytes.toString(text.getBytes(), 0, text.getLength()), fileName);
assertEquals(0, Bytes.compareTo(kv.getKey(), kvList[0].getKey()));
return null;
}
})
.when(ctx)
.write(any(Text.class), any(KeyValue.class));
map.map(r, columns, ctx);
} finally {
lock.release();
}
}
private static String getPgPortEphemeralNodePath(ServerName sn, int port, boolean isMaster) {
String znode =
(isMaster ? "M" : "S")
+ ":"
+ sn.getHostAndPort()
+ Addressing.HOSTNAME_PORT_SEPARATOR
+ port;
return ZKUtil.joinZNode(ZooKeeperAdmin.PG_SERVER_NODE, znode);
}
public void addRecord(MetaRecord rec) {
try {
putRecord(rec, System.currentTimeMillis());
String node = ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(rec.getId()));
ZKUtil.createAndWatch(watcher, node, EMPTY_BYTE_ARRAY);
tracker.updateIdVersion(rec.getId(), 0);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public LeaderElector(ZooKeeperWatcher watcher, String serverName) {
setDaemon(true);
setName("ZKSecretWatcher-leaderElector");
zkLeader =
new ZKLeaderManager(
watcher,
ZKUtil.joinZNode(zkWatcher.getRootKeyZNode(), "keymaster"),
Bytes.toBytes(serverName),
this);
}
@Test
public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception {
// this test is for when distributed log replay is enabled
if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return;
LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization");
HMaster master = UTIL.getMiniHBaseCluster().getMaster();
MasterFileSystem fs = master.getMasterFileSystem();
String failedRegion = "failedRegoin1";
String staleRegion = "staleRegion";
ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1");
ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1");
String walPath =
"/hbase/data/.logs/" + inRecoveryServerName.getServerName() + "-splitting/test";
// Create a ZKW to use in the test
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
zkw.getRecoverableZooKeeper()
.create(
ZKSplitLog.getEncodedNodeName(zkw, walPath),
new SplitLogTask.Owned(inRecoveryServerName, fs.getLogRecoveryMode()).toByteArray(),
Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion);
ZKUtil.createWithParents(zkw, staleRegionPath);
String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion);
inRecoveringRegionPath =
ZKUtil.joinZNode(inRecoveringRegionPath, inRecoveryServerName.getServerName());
ZKUtil.createWithParents(zkw, inRecoveringRegionPath);
Set<ServerName> servers = new HashSet<ServerName>();
servers.add(previouselyFaildServerName);
fs.removeStaleRecoveringRegionsFromZK(servers);
// verification
assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1);
assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1);
ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode);
ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode);
zkw.close();
}
@Override
public void removePeer(String id) throws ReplicationException {
try {
if (!peerExists(id)) {
throw new IllegalArgumentException(
"Cannot remove peer with id=" + id + " because that id does not exist.");
}
ZKUtil.deleteNodeRecursively(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id));
} catch (KeeperException e) {
throw new ReplicationException("Could not remove peer with id=" + id, e);
}
}
private static HRegionServer setDrainingServer(final HRegionServer hrs) throws KeeperException {
LOG.info(
"Making "
+ hrs.getServerName()
+ " the draining server; "
+ "it has "
+ hrs.getNumberOfOnlineRegions()
+ " online regions");
ZooKeeperWatcher zkw = hrs.getZooKeeper();
String hrsDrainingZnode = ZKUtil.joinZNode(zkw.drainingZNode, hrs.getServerName().toString());
ZKUtil.createWithParents(zkw, hrsDrainingZnode);
return hrs;
}
public void removeRecord(int id) {
try {
// System.out.println("removeRecord id: " + id);
// new Error().printStackTrace();
Delete delete = new Delete(Bytes.toBytes(id));
table.delete(delete);
ZKUtil.deleteNodeFailSilent(
watcher, ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(id)));
tracker.removeId(id);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public void loadMetaRecords(List<MetaRecord> records) throws Exception {
MetaRecord rec;
for (Result r : table.getScanner(new Scan())) {
if (r.isEmpty()) continue;
rec = getMetaRecord(r);
records.add(rec);
if (!tracker.contains(rec.getId()))
ZKUtil.createNodeIfNotExistsAndWatch(
watcher,
ZKUtil.joinZNode(H2MetaTableTracker.NODE_NAME, Integer.toString(rec.getId())),
EMPTY_BYTE_ARRAY);
}
}
private void watchForAbortedProcedures() {
LOG.debug("Checking for aborted procedures on node: '" + zkController.getAbortZnode() + "'");
try {
// this is the list of the currently aborted procedues
for (String node :
ZKUtil.listChildrenAndWatchForNewChildren(
zkController.getWatcher(), zkController.getAbortZnode())) {
String abortNode = ZKUtil.joinZNode(zkController.getAbortZnode(), node);
abort(abortNode);
}
} catch (KeeperException e) {
member.controllerConnectionFailure(
"Failed to list children for abort node:" + zkController.getAbortZnode(), e, null);
}
}
/** This acts as the ack for a completed procedure */
@Override
public void sendMemberCompleted(Subprocedure sub, byte[] data) throws IOException {
String procName = sub.getName();
LOG.debug(
"Marking procedure '" + procName + "' completed for member '" + memberName + "' in zk");
String joinPath = ZKUtil.joinZNode(zkController.getReachedBarrierNode(procName), memberName);
// ProtobufUtil.prependPBMagic does not take care of null
if (data == null) {
data = new byte[0];
}
try {
ZKUtil.createAndFailSilent(
zkController.getWatcher(), joinPath, ProtobufUtil.prependPBMagic(data));
} catch (KeeperException e) {
member.controllerConnectionFailure(
"Failed to post zk node:" + joinPath + " to join procedure barrier.", e, procName);
}
}
@Override
public Map<String, String> getAllPeerClusterKeys() {
Map<String, String> peers = new TreeMap<String, String>();
List<String> ids = null;
try {
ids = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
for (String id : ids) {
byte[] bytes = ZKUtil.getData(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id));
String clusterKey = null;
try {
clusterKey = parsePeerFrom(bytes);
} catch (DeserializationException de) {
LOG.warn("Failed parse of clusterid=" + id + " znode content, continuing.");
continue;
}
peers.put(id, clusterKey);
}
} catch (KeeperException e) {
this.abortable.abort("Cannot get the list of peers ", e);
}
return peers;
}
public boolean shouldAddCheckerMaster() {
ZooKeeperWatcher zk = super.getZooKeeper();
String numberN = ZKUtil.joinZNode(zk.baseZNode, CCIndexConstants.CheckNumNode);
try {
if (ZKUtil.checkExists(zk, numberN) != -1) {
ZKUtil.createSetData(zk, numberN, Bytes.toBytes(1));
} else {
int num = Bytes.toInt(ZKUtil.getData(zk, numberN));
if (num < this.checkMasterN) {
ZKUtil.setData(zk, numberN, Bytes.toBytes(num + 1));
return true;
} else {
return false;
}
}
} catch (KeeperException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return false;
}
/**
* Wait for tasks to become available at /hbase/splitlog zknode. Grab a task one at a time. This
* policy puts an upper-limit on the number of simultaneous log splitting that could be happening
* in a cluster.
*
* <p>Synchronization using {@link #taskReadyLock} ensures that it will try to grab every task
* that has been put up
*
* @throws InterruptedException
*/
@Override
public void taskLoop() throws InterruptedException {
while (!shouldStop) {
int seq_start = taskReadySeq;
List<String> paths = null;
paths = getTaskList();
if (paths == null) {
LOG.warn(
"Could not get tasks, did someone remove "
+ watcher.splitLogZNode
+ " ... worker thread exiting.");
return;
}
// pick meta wal firstly
int offset = (int) (Math.random() * paths.size());
for (int i = 0; i < paths.size(); i++) {
if (DefaultWALProvider.isMetaFile(paths.get(i))) {
offset = i;
break;
}
}
int numTasks = paths.size();
for (int i = 0; i < numTasks; i++) {
int idx = (i + offset) % paths.size();
// don't call ZKSplitLog.getNodeName() because that will lead to
// double encoding of the path name
if (this.calculateAvailableSplitters(numTasks) > 0) {
grabTask(ZKUtil.joinZNode(watcher.splitLogZNode, paths.get(idx)));
} else {
LOG.debug(
"Current region server "
+ server.getServerName()
+ " has "
+ this.tasksInProgress.get()
+ " tasks in progress and can't take more.");
break;
}
if (shouldStop) {
return;
}
}
SplitLogCounters.tot_wkr_task_grabing.incrementAndGet();
synchronized (taskReadyLock) {
while (seq_start == taskReadySeq) {
taskReadyLock.wait(checkInterval);
if (server != null) {
// check to see if we have stale recovering regions in our internal memory state
Map<String, HRegion> recoveringRegions = server.getRecoveringRegions();
if (!recoveringRegions.isEmpty()) {
// Make a local copy to prevent ConcurrentModificationException when other threads
// modify recoveringRegions
List<String> tmpCopy = new ArrayList<String>(recoveringRegions.keySet());
int listSize = tmpCopy.size();
for (int i = 0; i < listSize; i++) {
String region = tmpCopy.get(i);
String nodePath = ZKUtil.joinZNode(watcher.recoveringRegionsZNode, region);
try {
if (ZKUtil.checkExists(watcher, nodePath) == -1) {
HRegion r = recoveringRegions.remove(region);
if (r != null) {
r.setRecovering(false);
}
LOG.debug("Mark recovering region:" + region + " up.");
} else {
// current check is a defensive(or redundant) mechanism to prevent us from
// having stale recovering regions in our internal RS memory state while
// zookeeper(source of truth) says differently. We stop at the first good one
// because we should not have a single instance such as this in normal case so
// check the first one is good enough.
break;
}
} catch (KeeperException e) {
// ignore zookeeper error
LOG.debug("Got a zookeeper when trying to open a recovering region", e);
break;
}
}
}
}
}
}
}
}
private String getTableCFsNode(String id) {
return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.tableCFsNodeName));
}
@Test
public void testRun() throws Exception {
TableName tn = TableName.valueOf(tableName);
byte[] mobValueBytes = new byte[100];
// get the path where mob files lie in
Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, family);
Put put = new Put(Bytes.toBytes(row));
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes);
Put put2 = new Put(Bytes.toBytes(row + "ignore"));
put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes);
table.mutate(put);
table.mutate(put2);
table.flush();
admin.flush(tn);
FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
// check the generation of a mob file
assertEquals(1, fileStatuses.length);
String mobFile1 = fileStatuses[0].getPath().getName();
Configuration configuration = new Configuration(TEST_UTIL.getConfiguration());
configuration.setFloat(MobConstants.MOB_SWEEP_TOOL_COMPACTION_RATIO, 0.6f);
configuration.setStrings(TableInputFormat.INPUT_TABLE, tableName);
configuration.setStrings(TableInputFormat.SCAN_COLUMN_FAMILY, family);
configuration.setStrings(SweepJob.WORKING_VISITED_DIR_KEY, "jobWorkingNamesDir");
configuration.setStrings(SweepJob.WORKING_FILES_DIR_KEY, "compactionFileDir");
configuration.setStrings(
CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, JavaSerialization.class.getName());
configuration.set(SweepJob.WORKING_VISITED_DIR_KEY, "compactionVisitedDir");
configuration.setLong(
MobConstants.MOB_SWEEP_TOOL_COMPACTION_START_DATE,
System.currentTimeMillis() + 24 * 3600 * 1000);
ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable());
TableName lockName = MobUtils.getTableLockName(tn);
String znode = ZKUtil.joinZNode(zkw.tableLockZNode, lockName.getNameAsString());
configuration.set(SweepJob.SWEEP_JOB_ID, "1");
configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode);
ServerName serverName = SweepJob.getCurrentServerName(configuration);
configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString());
TableLockManager tableLockManager =
TableLockManager.createTableLockManager(configuration, zkw, serverName);
TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool");
lock.acquire();
try {
// use the same counter when mocking
Counter counter = new GenericCounter();
Reducer<Text, KeyValue, Writable, Writable>.Context ctx = mock(Reducer.Context.class);
when(ctx.getConfiguration()).thenReturn(configuration);
when(ctx.getCounter(Matchers.any(SweepCounter.class))).thenReturn(counter);
when(ctx.nextKey()).thenReturn(true).thenReturn(false);
when(ctx.getCurrentKey()).thenReturn(new Text(mobFile1));
byte[] refBytes = Bytes.toBytes(mobFile1);
long valueLength = refBytes.length;
byte[] newValue = Bytes.add(Bytes.toBytes(valueLength), refBytes);
KeyValue kv2 =
new KeyValue(
Bytes.toBytes(row),
Bytes.toBytes(family),
Bytes.toBytes(qf),
1,
KeyValue.Type.Put,
newValue);
List<KeyValue> list = new ArrayList<KeyValue>();
list.add(kv2);
when(ctx.getValues()).thenReturn(list);
SweepReducer reducer = new SweepReducer();
reducer.run(ctx);
} finally {
lock.release();
}
FileStatus[] filsStatuses2 = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
String mobFile2 = filsStatuses2[0].getPath().getName();
// new mob file is generated, old one has been archived
assertEquals(1, filsStatuses2.length);
assertEquals(false, mobFile2.equalsIgnoreCase(mobFile1));
// test sequence file
String workingPath = configuration.get(SweepJob.WORKING_VISITED_DIR_KEY);
FileStatus[] statuses = TEST_UTIL.getTestFileSystem().listStatus(new Path(workingPath));
Set<String> files = new TreeSet<String>();
for (FileStatus st : statuses) {
files.addAll(
getKeyFromSequenceFile(TEST_UTIL.getTestFileSystem(), st.getPath(), configuration));
}
assertEquals(1, files.size());
assertEquals(true, files.contains(mobFile1));
}
private String getPeerStateNode(String id) {
return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.peerStateNodeName));
}
private static HRegionServer unsetDrainingServer(final HRegionServer hrs) throws KeeperException {
ZooKeeperWatcher zkw = hrs.getZooKeeper();
String hrsDrainingZnode = ZKUtil.joinZNode(zkw.drainingZNode, hrs.getServerName().toString());
ZKUtil.deleteNode(zkw, hrsDrainingZnode);
return hrs;
}