/** Compares our trees, and triggers repairs for any ranges that mismatch. */
public void run() {
InetAddress local = FBUtilities.getLocalAddress();
// restore partitioners (in case we were serialized)
if (ltree.partitioner() == null) ltree.partitioner(StorageService.getPartitioner());
if (rtree.partitioner() == null) rtree.partitioner(StorageService.getPartitioner());
// compare trees, and collect differences
differences.addAll(MerkleTree.difference(ltree, rtree));
// choose a repair method based on the significance of the difference
String format =
"Endpoints " + local + " and " + remote + " %s for " + cfname + " on " + range;
if (differences.isEmpty()) {
logger.info(String.format(format, "are consistent"));
completed(remote, cfname);
return;
}
// non-0 difference: perform streaming repair
logger.info(String.format(format, "have " + differences.size() + " range(s) out of sync"));
try {
performStreamingRepair();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** Return all of the neighbors with whom we share the provided range. */
static Set<InetAddress> getNeighbors(String table, Range<Token> toRepair) {
StorageService ss = StorageService.instance;
Map<Range<Token>, List<InetAddress>> replicaSets = ss.getRangeToAddressMap(table);
Range<Token> rangeSuperSet = null;
for (Range<Token> range : ss.getLocalRanges(table)) {
if (range.contains(toRepair)) {
rangeSuperSet = range;
break;
} else if (range.intersects(toRepair)) {
throw new IllegalArgumentException(
"Requested range intersects a local range but is not fully contained in one; this would lead to imprecise repair");
}
}
if (rangeSuperSet == null || !replicaSets.containsKey(toRepair)) return Collections.emptySet();
Set<InetAddress> neighbors = new HashSet<InetAddress>(replicaSets.get(rangeSuperSet));
neighbors.remove(FBUtilities.getBroadcastAddress());
// Excluding all node with version <= 0.7 since they don't know how to
// create a correct merkle tree (they build it over the full range)
Iterator<InetAddress> iter = neighbors.iterator();
while (iter.hasNext()) {
InetAddress endpoint = iter.next();
if (Gossiper.instance.getVersion(endpoint) <= MessagingService.VERSION_07) {
logger.info(
"Excluding "
+ endpoint
+ " from repair because it is on version 0.7 or sooner. You should consider updating this node before running repair again.");
iter.remove();
}
}
return neighbors;
}
private MerkleTree.RowHash rowHash(AbstractCompactedRow row) {
validated++;
// MerkleTree uses XOR internally, so we want lots of output bits here
MessageDigest digest = FBUtilities.newMessageDigest("SHA-256");
row.update(digest);
return new MerkleTree.RowHash(row.key.token, digest.digest());
}
/** Send merkle tree request to every involved neighbor. */
public void sendTreeRequests() {
requestedEndpoints.addAll(endpoints);
requestedEndpoints.add(FBUtilities.getLocalAddress());
// send requests to all nodes
for (InetAddress endpoint : requestedEndpoints)
AntiEntropyService.instance.request(getName(), endpoint, range, tablename, cfname);
}
public Token midpoint(Token ltoken, Token rtoken) {
// the symbolic MINIMUM token should act as ZERO: the empty bit array
BigInteger left = ltoken.equals(MINIMUM) ? ZERO : ((BigIntegerToken) ltoken).token;
BigInteger right = rtoken.equals(MINIMUM) ? ZERO : ((BigIntegerToken) rtoken).token;
Pair<BigInteger, Boolean> midpair = FBUtilities.midpoint(left, right, 127);
// discard the remainder
return new BigIntegerToken(midpair.left);
}
public RepairSession(
Range<Token> range, String tablename, boolean isSequential, String... cfnames) {
this(
UUIDGen.makeType1UUIDFromHost(FBUtilities.getBroadcastAddress()).toString(),
range,
tablename,
isSequential,
cfnames);
}
/**
* Submit differencers for running. All tree *must* have been received before this is called.
*/
public void submitDifferencers() {
assert requestedEndpoints.size() == 0;
// Right now, we only difference local host against each other. CASSANDRA-2610 will fix
// that.
// In the meantime ugly special casing will work good enough.
MerkleTree localTree = trees.get(FBUtilities.getLocalAddress());
assert localTree != null;
for (Map.Entry<InetAddress, MerkleTree> entry : trees.entrySet()) {
if (entry.getKey().equals(FBUtilities.getLocalAddress())) continue;
Differencer differencer =
new Differencer(cfname, entry.getKey(), entry.getValue(), localTree);
syncJobs.add(entry.getKey());
logger.debug("Queueing comparison " + differencer);
StageManager.getStage(Stage.ANTI_ENTROPY).execute(differencer);
}
trees.clear(); // allows gc to do its thing
}
/**
* Get system file descriptor from FileDescriptor object.
*
* @param descriptor - FileDescriptor objec to get fd from
* @return file descriptor, -1 or error
*/
public static int getfd(FileDescriptor descriptor) {
Field field = FBUtilities.getProtectedField(descriptor.getClass(), "fd");
if (field == null) return -1;
try {
return field.getInt(descriptor);
} catch (Exception e) {
logger.warn("unable to read fd field from FileDescriptor");
}
return -1;
}
static Message makeVerb(TreeRequest request, int version) {
try {
FastByteArrayOutputStream bos = new FastByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(bos);
SERIALIZER.serialize(request, dos, version);
return new Message(
FBUtilities.getBroadcastAddress(),
StorageService.Verb.TREE_REQUEST,
bos.toByteArray(),
version);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private boolean hasAcceptableAddressSpace() {
// Check is invalid on Windows
if (FBUtilities.isWindows()) return true;
try {
long fileMax = sigar.getResourceLimit().getVirtualMemoryMax();
if (fileMax == EXPECTED_AS) {
return true;
} else {
return false;
}
} catch (SigarException sigarException) {
logger.warn(
"Could not determine if VirtualMemoryMax was acceptable. Error message: {}",
sigarException);
return false;
}
}
/** Send merkle tree request to every involved neighbor. */
public void sendTreeRequests() {
// send requests to all nodes
List<InetAddress> allEndpoints = new ArrayList<InetAddress>(endpoints);
allEndpoints.add(FBUtilities.getBroadcastAddress());
if (isSequential) makeSnapshots(endpoints);
for (InetAddress endpoint : allEndpoints)
treeRequests.add(
new TreeRequest(getName(), endpoint, range, new CFPair(tablename, cfname)));
logger.info(
String.format(
"[repair #%s] requesting merkle trees for %s (to %s)",
getName(), cfname, allEndpoints));
treeRequests.start();
requestsSent.signalAll();
}
/**
* Responds to the node that requested the given valid tree.
*
* @param validator A locally generated validator
* @param local localhost (parameterized for testing)
*/
void respond(Validator validator, InetAddress local) {
MessagingService ms = MessagingService.instance();
try {
Message message = TreeResponseVerbHandler.makeVerb(local, validator);
if (!validator.request.endpoint.equals(FBUtilities.getBroadcastAddress()))
logger.info(
String.format(
"[repair #%s] Sending completed merkle tree to %s for %s",
validator.request.sessionid, validator.request.endpoint, validator.request.cf));
ms.sendOneWay(message, validator.request.endpoint);
} catch (Exception e) {
logger.error(
String.format(
"[repair #%s] Error sending completed merkle tree to %s for %s ",
validator.request.sessionid, validator.request.endpoint, validator.request.cf),
e);
}
}
/** Return all of the neighbors with whom we share data. */
static Set<InetAddress> getNeighbors(String table, Range range) {
StorageService ss = StorageService.instance;
Map<Range, List<InetAddress>> replicaSets = ss.getRangeToAddressMap(table);
if (!replicaSets.containsKey(range)) return Collections.emptySet();
Set<InetAddress> neighbors = new HashSet<InetAddress>(replicaSets.get(range));
neighbors.remove(FBUtilities.getLocalAddress());
// Excluding all node with version <= 0.7 since they don't know how to
// create a correct merkle tree (they build it over the full range)
Iterator<InetAddress> iter = neighbors.iterator();
while (iter.hasNext()) {
InetAddress endpoint = iter.next();
if (Gossiper.instance.getVersion(endpoint) <= MessagingService.VERSION_07) {
logger.info(
"Excluding "
+ endpoint
+ " from repair because it is on version 0.7 or sooner. You should consider updating this node before running repair again.");
iter.remove();
}
}
return neighbors;
}
/**
* The primitive with which all hashing should be accomplished: hashes a left and right value
* together.
*/
static byte[] binaryHash(final byte[] left, final byte[] right) {
return FBUtilities.xor(left, right);
}
public static void assertHashEquals(String message, final byte[] left, final byte[] right) {
String lstring = left == null ? "null" : FBUtilities.bytesToHex(left);
String rstring = right == null ? "null" : FBUtilities.bytesToHex(right);
assertEquals(message, lstring, rstring);
}
private String repairedNodes() {
StringBuilder sb = new StringBuilder();
sb.append(FBUtilities.getBroadcastAddress());
for (InetAddress ep : endpoints) sb.append(", ").append(ep);
return sb.toString();
}
public static String toString(byte[] hash) {
if (hash == null) return "null";
return "[" + FBUtilities.bytesToHex(hash) + "]";
}
/**
* Called after the validation lifecycle to respond with the now valid tree. Runs in
* Stage.ANTIENTROPY.
*/
public void run() {
// respond to the request that triggered this validation
AntiEntropyService.instance.respond(this, FBUtilities.getBroadcastAddress());
}