/**
* This method is part of IFailureDetectionEventListener interface. This is invoked by the Failure
* Detector when it convicts an end point.
*
* <p>param @ endpoint end point that is convicted.
*/
public void convict(InetAddress endpoint) {
EndpointState epState = endpointStateMap_.get(endpoint);
if (epState.isAlive()) {
logger_.info("InetAddress {} is now dead.", endpoint);
isAlive(endpoint, epState, false);
}
}
void notifyFailureDetector(Map<InetAddress, EndpointState> remoteEpStateMap) {
IFailureDetector fd = FailureDetector.instance;
for (Entry<InetAddress, EndpointState> entry : remoteEpStateMap.entrySet()) {
InetAddress endpoint = entry.getKey();
EndpointState remoteEndpointState = entry.getValue();
EndpointState localEndpointState = endpointStateMap_.get(endpoint);
/*
* If the local endpoint state exists then report to the FD only
* if the versions workout.
*/
if (localEndpointState != null) {
int localGeneration = localEndpointState.getHeartBeatState().generation_;
int remoteGeneration = remoteEndpointState.getHeartBeatState().generation_;
if (remoteGeneration > localGeneration) {
fd.report(endpoint);
continue;
}
if (remoteGeneration == localGeneration) {
int localVersion = getMaxEndpointStateVersion(localEndpointState);
// int localVersion = localEndpointState.getHeartBeatState().getHeartBeatVersion();
int remoteVersion = remoteEndpointState.getHeartBeatState().getHeartBeatVersion();
if (remoteVersion > localVersion) {
fd.report(endpoint);
}
}
}
}
}
/** Start the gossiper with the generation # retrieved from the System table */
public void start(InetAddress localEndpoint, int generationNbr) {
localEndpoint_ = localEndpoint;
/* Get the seeds from the config and initialize them. */
Set<InetAddress> seedHosts = DatabaseDescriptor.getSeeds();
for (InetAddress seed : seedHosts) {
if (seed.equals(localEndpoint)) continue;
seeds_.add(seed);
}
/* initialize the heartbeat state for this localEndpoint */
EndpointState localState = endpointStateMap_.get(localEndpoint_);
if (localState == null) {
HeartBeatState hbState = new HeartBeatState(generationNbr);
localState = new EndpointState(hbState);
localState.isAlive(true);
localState.isAGossiper(true);
endpointStateMap_.put(localEndpoint_, localState);
}
// notify snitches that Gossiper is about to start
DatabaseDescriptor.getEndpointSnitch().gossiperStarting();
scheduledGossipTask =
StorageService.scheduledTasks.scheduleWithFixedDelay(
new GossipTask(),
Gossiper.intervalInMillis_,
Gossiper.intervalInMillis_,
TimeUnit.MILLISECONDS);
}
/**
* The gossip digest is built based on randomization rather than just looping through the
* collection of live endpoints.
*
* @param gDigests list of Gossip Digests.
*/
void makeRandomGossipDigest(List<GossipDigest> gDigests) {
/* Add the local endpoint state */
EndpointState epState = endpointStateMap_.get(localEndpoint_);
int generation = epState.getHeartBeatState().getGeneration();
int maxVersion = getMaxEndpointStateVersion(epState);
gDigests.add(new GossipDigest(localEndpoint_, generation, maxVersion));
List<InetAddress> endpoints = new ArrayList<InetAddress>(endpointStateMap_.keySet());
Collections.shuffle(endpoints, random_);
for (InetAddress endpoint : endpoints) {
epState = endpointStateMap_.get(endpoint);
if (epState != null) {
generation = epState.getHeartBeatState().getGeneration();
maxVersion = getMaxEndpointStateVersion(epState);
gDigests.add(new GossipDigest(endpoint, generation, maxVersion));
} else {
gDigests.add(new GossipDigest(endpoint, 0, 0));
}
}
/* FOR DEBUG ONLY - remove later */
StringBuilder sb = new StringBuilder();
for (GossipDigest gDigest : gDigests) {
sb.append(gDigest);
sb.append(" ");
}
if (logger_.isTraceEnabled()) logger_.trace("Gossip Digests are : " + sb.toString());
}
/** Add an endpoint we knew about previously, but whose state is unknown */
public void addSavedEndpoint(InetAddress ep) {
EndpointState epState = endpointStateMap_.get(ep);
if (epState == null) {
epState = new EndpointState(new HeartBeatState(0));
epState.isAlive(false);
epState.isAGossiper(true);
epState.setHasToken(true);
endpointStateMap_.put(ep, epState);
unreachableEndpoints_.put(ep, System.currentTimeMillis());
}
}
/** This should *only* be used for testing purposes. */
public void initializeNodeUnsafe(InetAddress addr, int generationNbr) {
/* initialize the heartbeat state for this localEndpoint */
EndpointState localState = endpointStateMap_.get(addr);
if (localState == null) {
HeartBeatState hbState = new HeartBeatState(generationNbr);
localState = new EndpointState(hbState);
localState.isAlive(true);
localState.isAGossiper(true);
endpointStateMap_.put(addr, localState);
}
}
/*
This method is used to figure the state that the Gossiper has but Gossipee doesn't. The delta digests
and the delta state are built up.
*/
void examineGossiper(
List<GossipDigest> gDigestList,
List<GossipDigest> deltaGossipDigestList,
Map<InetAddress, EndpointState> deltaEpStateMap) {
for (GossipDigest gDigest : gDigestList) {
int remoteGeneration = gDigest.getGeneration();
int maxRemoteVersion = gDigest.getMaxVersion();
/* Get state associated with the end point in digest */
EndpointState epStatePtr = endpointStateMap_.get(gDigest.getEndpoint());
/*
Here we need to fire a GossipDigestAckMessage. If we have some data associated with this endpoint locally
then we follow the "if" path of the logic. If we have absolutely nothing for this endpoint we need to
request all the data for this endpoint.
*/
if (epStatePtr != null) {
int localGeneration = epStatePtr.getHeartBeatState().getGeneration();
/* get the max version of all keys in the state associated with this endpoint */
int maxLocalVersion = getMaxEndpointStateVersion(epStatePtr);
if (remoteGeneration == localGeneration && maxRemoteVersion == maxLocalVersion) continue;
if (remoteGeneration > localGeneration) {
/* we request everything from the gossiper */
requestAll(gDigest, deltaGossipDigestList, remoteGeneration);
}
if (remoteGeneration < localGeneration) {
/* send all data with generation = localgeneration and version > 0 */
sendAll(gDigest, deltaEpStateMap, 0);
}
if (remoteGeneration == localGeneration) {
/*
If the max remote version is greater then we request the remote endpoint send us all the data
for this endpoint with version greater than the max version number we have locally for this
endpoint.
If the max remote version is lesser, then we send all the data we have locally for this endpoint
with version greater than the max remote version.
*/
if (maxRemoteVersion > maxLocalVersion) {
deltaGossipDigestList.add(
new GossipDigest(gDigest.getEndpoint(), remoteGeneration, maxLocalVersion));
}
if (maxRemoteVersion < maxLocalVersion) {
/* send all data with generation = localgeneration and version > maxRemoteVersion */
sendAll(gDigest, deltaEpStateMap, maxRemoteVersion);
}
}
} else {
/* We are here since we have no data for this endpoint locally so request everything. */
requestAll(gDigest, deltaGossipDigestList, remoteGeneration);
}
}
}
void markAlive(InetAddress addr, EndpointState localState) {
if (logger_.isTraceEnabled()) logger_.trace("marking as alive {}", addr);
if (!localState.isAlive()) {
isAlive(addr, localState, true);
logger_.info("InetAddress {} is now UP", addr);
}
}
void isAlive(InetAddress addr, EndpointState epState, boolean value) {
epState.isAlive(value);
if (value) {
liveEndpoints_.add(addr);
unreachableEndpoints_.remove(addr);
for (IEndpointStateChangeSubscriber subscriber : subscribers_)
subscriber.onAlive(addr, epState);
} else {
liveEndpoints_.remove(addr);
unreachableEndpoints_.put(addr, System.currentTimeMillis());
for (IEndpointStateChangeSubscriber subscriber : subscribers_)
subscriber.onDead(addr, epState);
}
if (epState.isAGossiper()) return;
epState.isAGossiper(true);
}
void applyHeartBeatStateLocally(
InetAddress addr, EndpointState localState, EndpointState remoteState) {
HeartBeatState localHbState = localState.getHeartBeatState();
HeartBeatState remoteHbState = remoteState.getHeartBeatState();
if (remoteHbState.getGeneration() > localHbState.getGeneration()) {
if (logger_.isTraceEnabled())
logger_.trace(
"Updating heartbeat state generation to "
+ remoteHbState.getGeneration()
+ " from "
+ localHbState.getGeneration()
+ " for "
+ addr);
localState.setHeartBeatState(remoteHbState);
}
if (localHbState.getGeneration() == remoteHbState.getGeneration()) {
if (remoteHbState.getHeartBeatVersion() > localHbState.getHeartBeatVersion()) {
int oldVersion = localHbState.getHeartBeatVersion();
localState.setHeartBeatState(remoteHbState);
if (logger_.isTraceEnabled())
logger_.trace(
"Updating heartbeat state version to "
+ localState.getHeartBeatState().getHeartBeatVersion()
+ " from "
+ oldVersion
+ " for "
+ addr
+ " ...");
} else {
if (logger_.isTraceEnabled())
logger_.trace(
"Ignoring lower version "
+ remoteHbState.getHeartBeatVersion()
+ " for "
+ addr
+ " which is lower than "
+ localHbState.getHeartBeatVersion());
}
}
}
void applyStateLocally(Map<InetAddress, EndpointState> epStateMap) {
for (Entry<InetAddress, EndpointState> entry : epStateMap.entrySet()) {
InetAddress ep = entry.getKey();
if (ep.equals(localEndpoint_)) continue;
EndpointState localEpStatePtr = endpointStateMap_.get(ep);
EndpointState remoteState = entry.getValue();
/*
If state does not exist just add it. If it does then add it only if the version
of the remote copy is greater than the local copy.
*/
if (localEpStatePtr != null) {
int localGeneration = localEpStatePtr.getHeartBeatState().getGeneration();
int remoteGeneration = remoteState.getHeartBeatState().getGeneration();
if (remoteGeneration > localGeneration) {
handleGenerationChange(ep, remoteState);
} else if (remoteGeneration == localGeneration) {
/* manage the membership state */
int localMaxVersion = getMaxEndpointStateVersion(localEpStatePtr);
int remoteMaxVersion = getMaxEndpointStateVersion(remoteState);
if (remoteMaxVersion > localMaxVersion) {
markAlive(ep, localEpStatePtr);
applyHeartBeatStateLocally(ep, localEpStatePtr, remoteState);
/* apply ApplicationState */
applyApplicationStateLocally(ep, localEpStatePtr, remoteState);
}
} else {
if (logger_.isTraceEnabled())
logger_.trace(
"Ignoring remote generation " + remoteGeneration + " < " + localGeneration);
}
} else {
handleNewJoin(ep, remoteState);
}
}
}
void doStatusCheck() {
long now = System.currentTimeMillis();
Set<InetAddress> eps = endpointStateMap_.keySet();
for (InetAddress endpoint : eps) {
if (endpoint.equals(localEndpoint_)) continue;
FailureDetector.instance.interpret(endpoint);
EndpointState epState = endpointStateMap_.get(endpoint);
if (epState != null) {
long duration = now - epState.getUpdateTimestamp();
if (StorageService.instance.getTokenMetadata().isMember(endpoint))
epState.setHasToken(true);
// check if this is a fat client. fat clients are removed automatically from
// gosip after FatClientTimeout
if (!epState.getHasToken()
&& !epState.isAlive()
&& !justRemovedEndpoints_.containsKey(endpoint)
&& (duration > FatClientTimeout_)) {
logger_.info(
"FatClient "
+ endpoint
+ " has been silent for "
+ FatClientTimeout_
+ "ms, removing from gossip");
removeEndpoint(
endpoint); // will put it in justRemovedEndpoints to respect quarantine delay
evictFromMembership(endpoint); // can get rid of the state immediately
}
if (!epState.isAlive() && (duration > aVeryLongTime_)) {
evictFromMembership(endpoint);
}
}
}
if (!justRemovedEndpoints_.isEmpty()) {
Map<InetAddress, Long> copy = new HashMap<InetAddress, Long>(justRemovedEndpoints_);
for (Map.Entry<InetAddress, Long> entry : copy.entrySet()) {
if ((now - entry.getValue()) > QUARANTINE_DELAY) {
if (logger_.isDebugEnabled())
logger_.debug(
QUARANTINE_DELAY + " elapsed, " + entry.getKey() + " gossip quarantine over");
justRemovedEndpoints_.remove(entry.getKey());
}
}
}
}
void applyApplicationStateLocally(
InetAddress addr, EndpointState localStatePtr, EndpointState remoteStatePtr) {
Map<ApplicationState, VersionedValue> localAppStateMap = localStatePtr.getApplicationStateMap();
for (Entry<ApplicationState, VersionedValue> remoteEntry :
remoteStatePtr.getApplicationStateMap().entrySet()) {
ApplicationState remoteKey = remoteEntry.getKey();
VersionedValue remoteValue = remoteEntry.getValue();
VersionedValue localValue = localAppStateMap.get(remoteKey);
/* If state doesn't exist locally for this key then just apply it */
if (localValue == null) {
localStatePtr.addApplicationState(remoteKey, remoteValue);
doNotifications(addr, remoteKey, remoteValue);
continue;
}
int remoteGeneration = remoteStatePtr.getHeartBeatState().getGeneration();
int localGeneration = localStatePtr.getHeartBeatState().getGeneration();
assert remoteGeneration
>= localGeneration; // SystemTable makes sure we never generate a smaller generation on
// start
/* If the remoteGeneration is greater than localGeneration then apply state blindly */
if (remoteGeneration > localGeneration) {
localStatePtr.addApplicationState(remoteKey, remoteValue);
doNotifications(addr, remoteKey, remoteValue);
continue;
}
/* If the generations are the same then apply state if the remote version is greater than local version. */
if (remoteGeneration == localGeneration) {
int remoteVersion = remoteValue.version;
int localVersion = localValue.version;
if (remoteVersion > localVersion) {
localStatePtr.addApplicationState(remoteKey, remoteValue);
doNotifications(addr, remoteKey, remoteValue);
}
}
}
}
EndpointState getStateForVersionBiggerThan(InetAddress forEndpoint, int version) {
EndpointState epState = endpointStateMap_.get(forEndpoint);
EndpointState reqdEndpointState = null;
if (epState != null) {
/*
* Here we try to include the Heart Beat state only if it is
* greater than the version passed in. It might happen that
* the heart beat version maybe lesser than the version passed
* in and some application state has a version that is greater
* than the version passed in. In this case we also send the old
* heart beat and throw it away on the receiver if it is redundant.
*/
int localHbVersion = epState.getHeartBeatState().getHeartBeatVersion();
if (localHbVersion > version) {
reqdEndpointState = new EndpointState(epState.getHeartBeatState());
if (logger_.isTraceEnabled())
logger_.trace(
"local heartbeat version "
+ localHbVersion
+ " greater than "
+ version
+ " for "
+ forEndpoint);
}
/* Accumulate all application states whose versions are greater than "version" variable */
for (Entry<ApplicationState, VersionedValue> entry :
epState.getApplicationStateMap().entrySet()) {
VersionedValue value = entry.getValue();
if (value.version > version) {
if (reqdEndpointState == null) {
reqdEndpointState = new EndpointState(epState.getHeartBeatState());
}
final ApplicationState key = entry.getKey();
if (logger_.isTraceEnabled()) logger_.trace("Adding state " + key + ": " + value.value);
reqdEndpointState.addApplicationState(key, value);
}
}
}
return reqdEndpointState;
}
public void onJoin(InetAddress endpoint, EndpointState epState) {
VersionedValue localValue = epState.getApplicationState(ApplicationState.LOAD);
if (localValue != null) {
onChange(endpoint, ApplicationState.LOAD, localValue);
}
}
public void addLocalApplicationState(ApplicationState state, VersionedValue value) {
EndpointState epState = endpointStateMap_.get(localEndpoint_);
assert epState != null;
epState.addApplicationState(state, value);
}
int getMaxEndpointStateVersion(EndpointState epState) {
int maxVersion = epState.getHeartBeatState().getHeartBeatVersion();
for (VersionedValue value : epState.getApplicationStateMap().values())
maxVersion = Math.max(maxVersion, value.version);
return maxVersion;
}
/** determine which endpoint started up earlier */
public int compareEndpointStartup(InetAddress addr1, InetAddress addr2) {
EndpointState ep1 = getEndpointStateForEndpoint(addr1);
EndpointState ep2 = getEndpointStateForEndpoint(addr2);
assert ep1 != null && ep2 != null;
return ep1.getHeartBeatState().getGeneration() - ep2.getHeartBeatState().getGeneration();
}