void notifyFailureDetector(Map<InetAddress, EndpointState> remoteEpStateMap) {
IFailureDetector fd = FailureDetector.instance;
for (Entry<InetAddress, EndpointState> entry : remoteEpStateMap.entrySet()) {
InetAddress endpoint = entry.getKey();
EndpointState remoteEndpointState = entry.getValue();
EndpointState localEndpointState = endpointStateMap_.get(endpoint);
/*
* If the local endpoint state exists then report to the FD only
* if the versions workout.
*/
if (localEndpointState != null) {
int localGeneration = localEndpointState.getHeartBeatState().generation_;
int remoteGeneration = remoteEndpointState.getHeartBeatState().generation_;
if (remoteGeneration > localGeneration) {
fd.report(endpoint);
continue;
}
if (remoteGeneration == localGeneration) {
int localVersion = getMaxEndpointStateVersion(localEndpointState);
// int localVersion = localEndpointState.getHeartBeatState().getHeartBeatVersion();
int remoteVersion = remoteEndpointState.getHeartBeatState().getHeartBeatVersion();
if (remoteVersion > localVersion) {
fd.report(endpoint);
}
}
}
}
}
/**
* The gossip digest is built based on randomization rather than just looping through the
* collection of live endpoints.
*
* @param gDigests list of Gossip Digests.
*/
void makeRandomGossipDigest(List<GossipDigest> gDigests) {
/* Add the local endpoint state */
EndpointState epState = endpointStateMap_.get(localEndpoint_);
int generation = epState.getHeartBeatState().getGeneration();
int maxVersion = getMaxEndpointStateVersion(epState);
gDigests.add(new GossipDigest(localEndpoint_, generation, maxVersion));
List<InetAddress> endpoints = new ArrayList<InetAddress>(endpointStateMap_.keySet());
Collections.shuffle(endpoints, random_);
for (InetAddress endpoint : endpoints) {
epState = endpointStateMap_.get(endpoint);
if (epState != null) {
generation = epState.getHeartBeatState().getGeneration();
maxVersion = getMaxEndpointStateVersion(epState);
gDigests.add(new GossipDigest(endpoint, generation, maxVersion));
} else {
gDigests.add(new GossipDigest(endpoint, 0, 0));
}
}
/* FOR DEBUG ONLY - remove later */
StringBuilder sb = new StringBuilder();
for (GossipDigest gDigest : gDigests) {
sb.append(gDigest);
sb.append(" ");
}
if (logger_.isTraceEnabled()) logger_.trace("Gossip Digests are : " + sb.toString());
}
private void appendEndpointState(StringBuilder sb, EndpointState endpointState) {
sb.append(" generation:")
.append(endpointState.getHeartBeatState().getGeneration())
.append("\n");
sb.append(" heartbeat:")
.append(endpointState.getHeartBeatState().getHeartBeatVersion())
.append("\n");
for (Map.Entry<ApplicationState, VersionedValue> state :
endpointState.applicationState.entrySet()) {
if (state.getKey() == ApplicationState.TOKENS) continue;
sb.append(" ")
.append(state.getKey())
.append(":")
.append(state.getValue().value)
.append("\n");
}
}
void applyApplicationStateLocally(
InetAddress addr, EndpointState localStatePtr, EndpointState remoteStatePtr) {
Map<ApplicationState, VersionedValue> localAppStateMap = localStatePtr.getApplicationStateMap();
for (Entry<ApplicationState, VersionedValue> remoteEntry :
remoteStatePtr.getApplicationStateMap().entrySet()) {
ApplicationState remoteKey = remoteEntry.getKey();
VersionedValue remoteValue = remoteEntry.getValue();
VersionedValue localValue = localAppStateMap.get(remoteKey);
/* If state doesn't exist locally for this key then just apply it */
if (localValue == null) {
localStatePtr.addApplicationState(remoteKey, remoteValue);
doNotifications(addr, remoteKey, remoteValue);
continue;
}
int remoteGeneration = remoteStatePtr.getHeartBeatState().getGeneration();
int localGeneration = localStatePtr.getHeartBeatState().getGeneration();
assert remoteGeneration
>= localGeneration; // SystemTable makes sure we never generate a smaller generation on
// start
/* If the remoteGeneration is greater than localGeneration then apply state blindly */
if (remoteGeneration > localGeneration) {
localStatePtr.addApplicationState(remoteKey, remoteValue);
doNotifications(addr, remoteKey, remoteValue);
continue;
}
/* If the generations are the same then apply state if the remote version is greater than local version. */
if (remoteGeneration == localGeneration) {
int remoteVersion = remoteValue.version;
int localVersion = localValue.version;
if (remoteVersion > localVersion) {
localStatePtr.addApplicationState(remoteKey, remoteValue);
doNotifications(addr, remoteKey, remoteValue);
}
}
}
}
void applyHeartBeatStateLocally(
InetAddress addr, EndpointState localState, EndpointState remoteState) {
HeartBeatState localHbState = localState.getHeartBeatState();
HeartBeatState remoteHbState = remoteState.getHeartBeatState();
if (remoteHbState.getGeneration() > localHbState.getGeneration()) {
if (logger_.isTraceEnabled())
logger_.trace(
"Updating heartbeat state generation to "
+ remoteHbState.getGeneration()
+ " from "
+ localHbState.getGeneration()
+ " for "
+ addr);
localState.setHeartBeatState(remoteHbState);
}
if (localHbState.getGeneration() == remoteHbState.getGeneration()) {
if (remoteHbState.getHeartBeatVersion() > localHbState.getHeartBeatVersion()) {
int oldVersion = localHbState.getHeartBeatVersion();
localState.setHeartBeatState(remoteHbState);
if (logger_.isTraceEnabled())
logger_.trace(
"Updating heartbeat state version to "
+ localState.getHeartBeatState().getHeartBeatVersion()
+ " from "
+ oldVersion
+ " for "
+ addr
+ " ...");
} else {
if (logger_.isTraceEnabled())
logger_.trace(
"Ignoring lower version "
+ remoteHbState.getHeartBeatVersion()
+ " for "
+ addr
+ " which is lower than "
+ localHbState.getHeartBeatVersion());
}
}
}
EndpointState getStateForVersionBiggerThan(InetAddress forEndpoint, int version) {
EndpointState epState = endpointStateMap_.get(forEndpoint);
EndpointState reqdEndpointState = null;
if (epState != null) {
/*
* Here we try to include the Heart Beat state only if it is
* greater than the version passed in. It might happen that
* the heart beat version maybe lesser than the version passed
* in and some application state has a version that is greater
* than the version passed in. In this case we also send the old
* heart beat and throw it away on the receiver if it is redundant.
*/
int localHbVersion = epState.getHeartBeatState().getHeartBeatVersion();
if (localHbVersion > version) {
reqdEndpointState = new EndpointState(epState.getHeartBeatState());
if (logger_.isTraceEnabled())
logger_.trace(
"local heartbeat version "
+ localHbVersion
+ " greater than "
+ version
+ " for "
+ forEndpoint);
}
/* Accumulate all application states whose versions are greater than "version" variable */
for (Entry<ApplicationState, VersionedValue> entry :
epState.getApplicationStateMap().entrySet()) {
VersionedValue value = entry.getValue();
if (value.version > version) {
if (reqdEndpointState == null) {
reqdEndpointState = new EndpointState(epState.getHeartBeatState());
}
final ApplicationState key = entry.getKey();
if (logger_.isTraceEnabled()) logger_.trace("Adding state " + key + ": " + value.value);
reqdEndpointState.addApplicationState(key, value);
}
}
}
return reqdEndpointState;
}
/*
This method is used to figure the state that the Gossiper has but Gossipee doesn't. The delta digests
and the delta state are built up.
*/
void examineGossiper(
List<GossipDigest> gDigestList,
List<GossipDigest> deltaGossipDigestList,
Map<InetAddress, EndpointState> deltaEpStateMap) {
for (GossipDigest gDigest : gDigestList) {
int remoteGeneration = gDigest.getGeneration();
int maxRemoteVersion = gDigest.getMaxVersion();
/* Get state associated with the end point in digest */
EndpointState epStatePtr = endpointStateMap_.get(gDigest.getEndpoint());
/*
Here we need to fire a GossipDigestAckMessage. If we have some data associated with this endpoint locally
then we follow the "if" path of the logic. If we have absolutely nothing for this endpoint we need to
request all the data for this endpoint.
*/
if (epStatePtr != null) {
int localGeneration = epStatePtr.getHeartBeatState().getGeneration();
/* get the max version of all keys in the state associated with this endpoint */
int maxLocalVersion = getMaxEndpointStateVersion(epStatePtr);
if (remoteGeneration == localGeneration && maxRemoteVersion == maxLocalVersion) continue;
if (remoteGeneration > localGeneration) {
/* we request everything from the gossiper */
requestAll(gDigest, deltaGossipDigestList, remoteGeneration);
}
if (remoteGeneration < localGeneration) {
/* send all data with generation = localgeneration and version > 0 */
sendAll(gDigest, deltaEpStateMap, 0);
}
if (remoteGeneration == localGeneration) {
/*
If the max remote version is greater then we request the remote endpoint send us all the data
for this endpoint with version greater than the max version number we have locally for this
endpoint.
If the max remote version is lesser, then we send all the data we have locally for this endpoint
with version greater than the max remote version.
*/
if (maxRemoteVersion > maxLocalVersion) {
deltaGossipDigestList.add(
new GossipDigest(gDigest.getEndpoint(), remoteGeneration, maxLocalVersion));
}
if (maxRemoteVersion < maxLocalVersion) {
/* send all data with generation = localgeneration and version > maxRemoteVersion */
sendAll(gDigest, deltaEpStateMap, maxRemoteVersion);
}
}
} else {
/* We are here since we have no data for this endpoint locally so request everything. */
requestAll(gDigest, deltaGossipDigestList, remoteGeneration);
}
}
}
void applyStateLocally(Map<InetAddress, EndpointState> epStateMap) {
for (Entry<InetAddress, EndpointState> entry : epStateMap.entrySet()) {
InetAddress ep = entry.getKey();
if (ep.equals(localEndpoint_)) continue;
EndpointState localEpStatePtr = endpointStateMap_.get(ep);
EndpointState remoteState = entry.getValue();
/*
If state does not exist just add it. If it does then add it only if the version
of the remote copy is greater than the local copy.
*/
if (localEpStatePtr != null) {
int localGeneration = localEpStatePtr.getHeartBeatState().getGeneration();
int remoteGeneration = remoteState.getHeartBeatState().getGeneration();
if (remoteGeneration > localGeneration) {
handleGenerationChange(ep, remoteState);
} else if (remoteGeneration == localGeneration) {
/* manage the membership state */
int localMaxVersion = getMaxEndpointStateVersion(localEpStatePtr);
int remoteMaxVersion = getMaxEndpointStateVersion(remoteState);
if (remoteMaxVersion > localMaxVersion) {
markAlive(ep, localEpStatePtr);
applyHeartBeatStateLocally(ep, localEpStatePtr, remoteState);
/* apply ApplicationState */
applyApplicationStateLocally(ep, localEpStatePtr, remoteState);
}
} else {
if (logger_.isTraceEnabled())
logger_.trace(
"Ignoring remote generation " + remoteGeneration + " < " + localGeneration);
}
} else {
handleNewJoin(ep, remoteState);
}
}
}
/** determine which endpoint started up earlier */
public int compareEndpointStartup(InetAddress addr1, InetAddress addr2) {
EndpointState ep1 = getEndpointStateForEndpoint(addr1);
EndpointState ep2 = getEndpointStateForEndpoint(addr2);
assert ep1 != null && ep2 != null;
return ep1.getHeartBeatState().getGeneration() - ep2.getHeartBeatState().getGeneration();
}
int getMaxEndpointStateVersion(EndpointState epState) {
int maxVersion = epState.getHeartBeatState().getHeartBeatVersion();
for (VersionedValue value : epState.getApplicationStateMap().values())
maxVersion = Math.max(maxVersion, value.version);
return maxVersion;
}