private void getDeadMembersFromCoordinator() {
if (!deadMembersKnown) {
try {
DistributedReplicantManager drm = partition.getDistributedReplicantManager();
List<ClusterNode> nodes = drm.lookupReplicantsNodes(getServiceHAName());
ClusterNode coord = (nodes != null && nodes.size() > 0 ? nodes.get(0) : null);
if (coord != null && coord.equals(partition.getClusterNode()) == false) {
Object rsp =
partition.callMethodOnNode(
getServiceHAName(), "getDiscrepancies", NULL_ARGS, NULL_TYPES, 60000, coord);
if (rsp instanceof RemoteDiscrepancies) {
handleRemoteDiscrepancies((RemoteDiscrepancies) rsp);
deadMembersKnown = true;
} else {
log.error("No valid response from coordinator: " + rsp);
}
}
} catch (Throwable e) {
if (e instanceof Error) {
throw (Error) e;
}
log.error("Caught exception pulling dead member records from coordinator", e);
}
}
}
/**
* Takes a set of discrepancies provided by another node and adds any missing entries to our
* discrepancy set, *after* adjusting the TimestampDiscrepancy objects to incorporate our
* discrepancy with the node that provided the set.
*/
private void convertRemoteDiscrepanciesToLocalTime(
Map<Server, TimestampDiscrepancy> remoteDiscrepancies,
TimestampDiscrepancy senderDiscrepancy) {
for (Map.Entry<Server, TimestampDiscrepancy> entry : remoteDiscrepancies.entrySet()) {
Server key = entry.getKey();
if (discrepancies.get(key) == null) {
// A node we didn't know about
discrepancies.put(
new Server(key, senderDiscrepancy),
new TimestampDiscrepancy(entry.getValue(), senderDiscrepancy));
ClusterNode node = key.getNode();
nodesByName.put(node.getName(), node);
}
}
}
@Override
public int hashCode() {
return node.hashCode();
}
private synchronized void statusCheck() {
if (statusCheckRequired
|| (System.currentTimeMillis() - lastStatusCheck > minStatusCheckFrequency)) {
try {
long requestSent = System.currentTimeMillis();
@SuppressWarnings("unchecked")
List rsps =
partition.callMethodOnCluster(
getServiceHAName(), "getLocalTimestamp", NULL_ARGS, NULL_TYPES, true);
long responseReceived = System.currentTimeMillis();
long mcastTime = responseReceived - requestSent;
Map<ClusterNode, TimestampDiscrepancy> rspBySender =
new HashMap<ClusterNode, TimestampDiscrepancy>();
if (rsps != null) {
for (Object rsp : rsps) {
if (rsp instanceof TimestampResponse) {
TimestampResponse tr = (TimestampResponse) rsp;
rspBySender.put(
tr.getResponder(),
new TimestampDiscrepancy(tr.getTimestamp(), requestSent, responseReceived));
} else if (rsp != null) {
log.warn("Unknown status check response " + rsp);
}
}
}
if (mcastTime > 250) {
// Multicasting the RPC introduced a high possible error;
// see if multiple unicast is better
List<ClusterNode> nodes =
partition.getDistributedReplicantManager().lookupReplicantsNodes(getServiceHAName());
for (ClusterNode node : nodes) {
if (node.equals(this.partition.getClusterNode())) {
continue;
}
try {
long singleRequestSent = System.currentTimeMillis();
Object rsp =
partition.callMethodOnNode(
getServiceHAName(),
"getLocalTimestamp",
NULL_ARGS,
NULL_TYPES,
mcastTime,
node);
long singleResponseReceived = System.currentTimeMillis();
long elapsed = singleResponseReceived - singleRequestSent;
if (elapsed < mcastTime) // better result than multicast
{
if (rsp instanceof TimestampResponse) {
TimestampResponse tr = (TimestampResponse) rsp;
rspBySender.put(
tr.getResponder(),
new TimestampDiscrepancy(
tr.getTimestamp(), singleRequestSent, singleResponseReceived));
} else if (rsp != null) {
log.warn("Unknown status check response " + rsp);
}
}
} catch (Throwable e) {
if (e instanceof Error) {
throw (Error) e;
}
log.error("Caught exception requesting timestamp from node " + node, e);
}
}
}
synchronized (discrepancies) {
synchronized (liveServers) {
for (Map.Entry<ClusterNode, TimestampDiscrepancy> entry : rspBySender.entrySet()) {
Server s = new Server(entry.getKey());
TimestampDiscrepancy latest = entry.getValue();
TimestampDiscrepancy existing = discrepancies.get(s);
if (existing == null
|| latest.getDiscrepancyRange() <= existing.getDiscrepancyRange()
|| liveServers.contains(s) == false) {
updateTimestampDiscrepancy(s, latest, true);
} else {
// We already have an entry for this live server with a
// narrower range that we'd prefer to keep
// If the new entry doesn't fit within the parameters
// of the old, we have to replace the old
if (existing.getMinDiscrepancy() < latest.getMinDiscrepancy()
|| existing.getMaxDiscrepancy() > latest.getMaxDiscrepancy()) {
updateTimestampDiscrepancy(s, latest, true);
} else {
// Re-store existing, but with the new key
updateTimestampDiscrepancy(s, existing, true);
}
}
}
}
}
statusCheckRequired = false;
lastStatusCheck = System.currentTimeMillis();
} catch (Exception e) {
log.error("Caught exception in status check", e);
}
}
getDeadMembersFromCoordinator();
}