/**
* Rebuild the point-in-time snapshot of the children objects and set watches on new
* children. @Param event may be null on the first initialization.
*/
private void processParentEvent(WatchedEvent event) throws Exception {
// get current children snapshot and reset this watch.
Set<String> children = new TreeSet<String>(m_zk.getChildren(m_rootNode, m_parentWatch));
// intersect to get newChildren and update m_lastChildren to the current set.
Set<String> newChildren = new HashSet<String>(children);
newChildren.removeAll(m_lastChildren);
m_lastChildren = children;
List<ByteArrayCallback> callbacks = new ArrayList<ByteArrayCallback>();
for (String child : children) {
ByteArrayCallback cb = new ByteArrayCallback();
// set watches on new children.
if (newChildren.contains(child)) {
m_zk.getData(ZKUtil.joinZKPath(m_rootNode, child), m_childWatch, cb, null);
} else {
m_zk.getData(ZKUtil.joinZKPath(m_rootNode, child), false, cb, null);
}
callbacks.add(cb);
}
HashMap<String, JSONObject> cache = new HashMap<String, JSONObject>();
for (ByteArrayCallback callback : callbacks) {
try {
byte payload[] = callback.getData();
JSONObject jsObj = new JSONObject(new String(payload, "UTF-8"));
cache.put(callback.getPath(), jsObj);
} catch (KeeperException.NoNodeException e) {
// child may have been deleted between the parent trigger and getData.
}
}
m_publicCache.set(ImmutableMap.copyOf(cache));
}
@Override
public void run() {
try {
JSONStringer js = new JSONStringer();
js.object();
js.key("role").value(m_config.m_replicationRole.ordinal());
js.key("active").value(m_rvdb.getReplicationActive());
js.endObject();
ZooKeeper zk = m_rvdb.getHostMessenger().getZK();
// rejoining nodes figure out the replication role from other nodes
if (!m_isRejoin) {
try {
zk.create(
VoltZK.replicationconfig,
js.toString().getBytes("UTF-8"),
Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
} catch (KeeperException.NodeExistsException e) {
}
String discoveredReplicationConfig =
new String(zk.getData(VoltZK.replicationconfig, false, null), "UTF-8");
JSONObject discoveredjsObj = new JSONObject(discoveredReplicationConfig);
ReplicationRole discoveredRole =
ReplicationRole.get((byte) discoveredjsObj.getLong("role"));
if (!discoveredRole.equals(m_config.m_replicationRole)) {
VoltDB.crashGlobalVoltDB(
"Discovered replication role "
+ discoveredRole
+ " doesn't match locally specified replication role "
+ m_config.m_replicationRole,
true,
null);
}
// See if we should bring the server up in WAN replication mode
m_rvdb.setReplicationRole(discoveredRole);
} else {
String discoveredReplicationConfig =
new String(zk.getData(VoltZK.replicationconfig, false, null), "UTF-8");
JSONObject discoveredjsObj = new JSONObject(discoveredReplicationConfig);
ReplicationRole discoveredRole =
ReplicationRole.get((byte) discoveredjsObj.getLong("role"));
boolean replicationActive = discoveredjsObj.getBoolean("active");
// See if we should bring the server up in WAN replication mode
m_rvdb.setReplicationRole(discoveredRole);
m_rvdb.setReplicationActive(replicationActive);
}
} catch (Exception e) {
VoltDB.crashGlobalVoltDB("Error discovering replication role", false, e);
}
}
/*
* Inherit the per partition txnid from the long since gone
* partition that existed in the past
*/
private long[] fetchPerPartitionTxnId() {
ZooKeeper zk = VoltDB.instance().getHostMessenger().getZK();
byte partitionTxnIdsBytes[] = null;
try {
partitionTxnIdsBytes = zk.getData(VoltZK.perPartitionTxnIds, false, null);
} catch (KeeperException.NoNodeException e) {
return null;
} // Can be no node if the cluster was never restored
catch (Exception e) {
VoltDB.crashLocalVoltDB("Error retrieving per partition txn ids", true, e);
}
ByteBuffer buf = ByteBuffer.wrap(partitionTxnIdsBytes);
int count = buf.getInt();
Long partitionTxnId = null;
long partitionTxnIds[] = new long[count];
for (int ii = 0; ii < count; ii++) {
long txnId = buf.getLong();
partitionTxnIds[ii] = txnId;
int partitionId = TxnEgo.getPartitionId(txnId);
if (partitionId == m_partitionId) {
partitionTxnId = txnId;
continue;
}
}
if (partitionTxnId != null) {
return partitionTxnIds;
}
return null;
}
/**
* Update a modified child and republish a new snapshot. This may indicate a deleted child or a
* child with modified data.
*/
private void processChildEvent(WatchedEvent event) throws Exception {
HashMap<String, JSONObject> cacheCopy = new HashMap<String, JSONObject>(m_publicCache.get());
ByteArrayCallback cb = new ByteArrayCallback();
m_zk.getData(event.getPath(), m_childWatch, cb, null);
try {
byte payload[] = cb.getData();
JSONObject jsObj = new JSONObject(new String(payload, "UTF-8"));
cacheCopy.put(cb.getPath(), jsObj);
} catch (KeeperException.NoNodeException e) {
cacheCopy.remove(event.getPath());
}
m_publicCache.set(ImmutableMap.copyOf(cacheCopy));
}
private Set<Integer> readPriorKnownLiveNodes() {
Set<Integer> nodes = new HashSet<Integer>();
try {
byte[] data = m_zk.getData(VoltZK.lastKnownLiveNodes, false, null);
String jsonString = new String(data, "UTF-8");
tmLog.debug("Read prior known live nodes: " + jsonString);
JSONObject jsObj = new JSONObject(jsonString);
JSONArray jsonNodes = jsObj.getJSONArray("liveNodes");
for (int ii = 0; ii < jsonNodes.length(); ii++) {
nodes.add(jsonNodes.getInt(ii));
}
} catch (Exception e) {
VoltDB.crashLocalVoltDB(
"Unable to read prior known live nodes at ZK path: " + VoltZK.lastKnownLiveNodes,
true,
e);
}
return nodes;
}
/**
* Once participating host count is set, SnapshotCompletionMonitor can check this ZK node to
* determine whether the snapshot has finished or not.
*
* <p>This should only be called when all participants have responded. It is possible that some
* hosts finish taking snapshot before the coordinator logs the participating host count. In this
* case, the host count would have been decremented multiple times already. To make sure finished
* hosts are logged correctly, this method adds participating host count + 1 to the current host
* count.
*
* @param txnId The snapshot txnId
* @param participantCount The number of hosts participating in this snapshot
*/
public static void logParticipatingHostCount(long txnId, int participantCount) {
ZooKeeper zk = VoltDB.instance().getHostMessenger().getZK();
final String snapshotPath = VoltZK.completed_snapshots + "/" + txnId;
boolean success = false;
while (!success) {
Stat stat = new Stat();
byte data[] = null;
try {
data = zk.getData(snapshotPath, false, stat);
} catch (KeeperException e) {
if (e.code() == KeeperException.Code.NONODE) {
// If snapshot creation failed for some reason, the node won't exist. ignore
return;
}
VoltDB.crashLocalVoltDB("Failed to get snapshot completion node", true, e);
} catch (InterruptedException e) {
VoltDB.crashLocalVoltDB("Interrupted getting snapshot completion node", true, e);
}
if (data == null) {
VoltDB.crashLocalVoltDB("Data should not be null if the node exists", false, null);
}
try {
JSONObject jsonObj = new JSONObject(new String(data, Charsets.UTF_8));
if (jsonObj.getLong("txnId") != txnId) {
VoltDB.crashLocalVoltDB("TxnId should match", false, null);
}
int hostCount = jsonObj.getInt("hostCount");
// +1 because hostCount was initialized to -1
jsonObj.put("hostCount", hostCount + participantCount + 1);
zk.setData(snapshotPath, jsonObj.toString(4).getBytes(Charsets.UTF_8), stat.getVersion());
} catch (KeeperException.BadVersionException e) {
continue;
} catch (Exception e) {
VoltDB.crashLocalVoltDB("This ZK call should never fail", true, e);
}
success = true;
}
}
public static Pair<byte[], Integer> retrieveChunksAsBytes(
ZooKeeper zk, String path, String prefix, boolean getCRC) throws Exception {
TreeSet<String> chunks = new TreeSet<String>();
while (true) {
boolean allUploadsComplete = true;
if (!chunks.contains(path + "/" + prefix + "_complete")) {
allUploadsComplete = false;
}
if (allUploadsComplete) {
break;
}
chunks = new TreeSet<String>(zk.getChildren(path, false));
for (String chunk : chunks) {
for (int ii = 0; ii < chunks.size(); ii++) {
if (chunk.startsWith(path + "/" + prefix)) {
chunks.add(chunk);
}
}
}
}
byte resultBuffers[][] = new byte[chunks.size() - 1][];
int ii = 0;
PureJavaCrc32 crc = getCRC ? new PureJavaCrc32() : null;
for (String chunk : chunks) {
if (chunk.endsWith("_complete")) continue;
resultBuffers[ii] = zk.getData(chunk, false, null);
if (crc != null) {
crc.update(resultBuffers[ii]);
}
ii++;
}
return Pair.of(decompressBytes(resultBuffers), crc != null ? (int) crc.getValue() : null);
}