protected void closeInternal() {
if (client != null) {
TTransport transport = client.getOutputProtocol().getTransport();
if (transport.isOpen()) transport.close();
}
}
// This method exists to verify a particularly nasty bug where cassandra doesn't have a
// consistent ring across all of it's nodes. One node will think it owns more than the others
// think it does and they will not send writes to it, but it will respond to requests
// acting like it does.
protected static void sanityCheckRingConsistency(
Set<String> currentHosts,
int port,
String keyspace,
boolean isSsl,
boolean safetyDisabled,
int socketTimeoutMillis,
int socketQueryTimeoutMillis) {
Multimap<Set<TokenRange>, String> tokenRangesToHost = HashMultimap.create();
for (String host : currentHosts) {
Cassandra.Client client = null;
try {
client =
CassandraClientPoolingContainer.getClientInternal(
host, port, isSsl, socketTimeoutMillis, socketQueryTimeoutMillis);
try {
client.describe_keyspace(keyspace);
} catch (NotFoundException e) {
log.warn(
"Tried to check ring consistency for node "
+ host
+ " before keyspace was fully setup; aborting check for now.",
e);
return;
}
tokenRangesToHost.put(ImmutableSet.copyOf(client.describe_ring(keyspace)), host);
} catch (Exception e) {
log.warn("failed to get ring info from host: {}", host, e);
} finally {
if (client != null) {
client.getOutputProtocol().getTransport().close();
}
}
}
if (tokenRangesToHost.isEmpty()) {
log.error(
"Failed to get ring info for entire Cassandra cluster ("
+ keyspace
+ "); ring could not be checked for consistency.");
return;
}
if (tokenRangesToHost.keySet().size() == 1) {
return;
}
RuntimeException e =
new IllegalStateException(
"Hosts have differing ring descriptions. This can lead to inconsistent reads and lost data. ");
log.error("QA-86204 " + e.getMessage() + tokenRangesToHost, e);
if (tokenRangesToHost.size() > 2) {
for (Entry<Set<TokenRange>, Collection<String>> entry :
tokenRangesToHost.asMap().entrySet()) {
if (entry.getValue().size() == 1) {
log.error(
"Host: "
+ entry.getValue().iterator().next()
+ " disagrees with the other nodes about the ring state.");
}
}
}
if (tokenRangesToHost.keySet().size() == 2) {
ImmutableList<Set<TokenRange>> sets = ImmutableList.copyOf(tokenRangesToHost.keySet());
Set<TokenRange> set1 = sets.get(0);
Set<TokenRange> set2 = sets.get(1);
log.error(
"Hosts are split. group1: "
+ tokenRangesToHost.get(set1)
+ " group2: "
+ tokenRangesToHost.get(set2));
}
logErrorOrThrow(e.getMessage(), safetyDisabled);
}
