private void receiveMessages() throws IOException {
// handshake (true) endpoint versions
DataOutputStream out = new DataOutputStream(socket.getOutputStream());
// if this version is < the MS version the other node is trying
// to connect with, the other node will disconnect
out.writeInt(MessagingService.current_version);
out.flush();
DataInputStream in = new DataInputStream(socket.getInputStream());
int maxVersion = in.readInt();
// outbound side will reconnect if necessary to upgrade version
assert version <= MessagingService.current_version;
from = CompactEndpointSerializationHelper.deserialize(in);
// record the (true) version of the endpoint
MessagingService.instance().setVersion(from, maxVersion);
logger.debug(
"Set version for {} to {} (will use {})",
from,
maxVersion,
MessagingService.instance().getVersion(from));
if (compressed) {
logger.debug("Upgrading incoming connection to be compressed");
in = new DataInputStream(new SnappyInputStream(socket.getInputStream()));
} else {
in = new DataInputStream(new BufferedInputStream(socket.getInputStream(), 4096));
}
while (true) {
MessagingService.validateMagic(in.readInt());
receiveMessage(in, version);
}
}
/**
* Responds to the node that requested the given valid tree.
*
* @param validator A locally generated validator
* @param local localhost (parameterized for testing)
*/
void respond(Validator validator, InetAddress local) {
MessagingService ms = MessagingService.instance();
try {
Message message = TreeResponseVerbHandler.makeVerb(local, validator);
logger.info("Sending AEService tree for " + validator.request);
ms.sendOneWay(message, validator.request.endpoint);
} catch (Exception e) {
logger.error("Could not send valid tree for request " + validator.request, e);
}
}
private void receiveMessages() throws IOException {
// handshake (true) endpoint versions
DataOutputStream out = new DataOutputStream(socket.getOutputStream());
out.writeInt(MessagingService.current_version);
out.flush();
DataInputStream in = new DataInputStream(socket.getInputStream());
int maxVersion = in.readInt();
from = CompactEndpointSerializationHelper.deserialize(in);
// record the (true) version of the endpoint
MessagingService.instance().setVersion(from, maxVersion);
logger.debug(
"Set version for {} to {} (will use {})",
from,
maxVersion,
MessagingService.instance().getVersion(from));
if (compressed) {
logger.debug("Upgrading incoming connection to be compressed");
if (version < MessagingService.VERSION_21) {
in = new DataInputStream(new SnappyInputStream(socket.getInputStream()));
} else {
LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
Checksum checksum =
XXHashFactory.fastestInstance()
.newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED)
.asChecksum();
in =
new DataInputStream(
new LZ4BlockInputStream(socket.getInputStream(), decompressor, checksum));
}
} else {
in = new DataInputStream(new BufferedInputStream(socket.getInputStream(), BUFFER_SIZE));
}
if (version > MessagingService.current_version) {
// save the endpoint so gossip will reconnect to it
Gossiper.instance.addSavedEndpoint(from);
logger.info("Received messages from newer protocol version {}. Ignoring", version);
return;
}
// outbound side will reconnect if necessary to upgrade version
while (true) {
MessagingService.validateMagic(in.readInt());
receiveMessage(in, version);
}
}
public void makeSnapshots(Collection<InetAddress> endpoints) {
try {
snapshotLatch = new CountDownLatch(endpoints.size());
IAsyncCallback callback =
new IAsyncCallback() {
@Override
public boolean isLatencyForSnitch() {
return false;
}
@Override
public void response(Message msg) {
RepairJob.this.snapshotLatch.countDown();
}
};
for (InetAddress endpoint : endpoints)
MessagingService.instance()
.sendRR(
new SnapshotCommand(tablename, cfname, sessionName, false), endpoint, callback);
snapshotLatch.await();
snapshotLatch = null;
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
/** Requests a tree from the given node, and returns the request that was sent. */
TreeRequest request(
String sessionid, InetAddress remote, Range range, String ksname, String cfname) {
TreeRequest request = new TreeRequest(sessionid, remote, range, new CFPair(ksname, cfname));
MessagingService.instance()
.sendOneWay(
TreeRequestVerbHandler.makeVerb(request, Gossiper.instance.getVersion(remote)), remote);
return request;
}
public void doVerb(MessageIn message, int id) {
logger.debug("Received migration request from {}.", message.from);
MessageOut<Collection<RowMutation>> response =
new MessageOut<>(
MessagingService.Verb.INTERNAL_RESPONSE,
SystemKeyspace.serializeSchema(),
MigrationManager.MigrationsSerializer.instance);
MessagingService.instance().sendReply(response, id, message.from);
}
/*
Use this version for fire and forget style messaging.
*/
public void sendOneWay(Message message, EndPoint to) {
// do local deliveries
if (message.getFrom().equals(to)) {
MessagingService.receive(message);
return;
}
Runnable tcpWriteEvent = new MessageSerializationTask(message, to);
messageSerializerExecutor_.execute(tcpWriteEvent);
}
/**
* Returns true if the chosen target was also a seed. False otherwise
*
* @param message message to sent
* @param epSet a set of endpoint from which a random endpoint is chosen.
* @return true if the chosen endpoint is also a seed.
*/
boolean sendGossip(Message message, Set<InetAddress> epSet) {
int size = epSet.size();
/* Generate a random number from 0 -> size */
List<InetAddress> liveEndpoints = new ArrayList<InetAddress>(epSet);
int index = (size == 1) ? 0 : random_.nextInt(size);
InetAddress to = liveEndpoints.get(index);
if (logger_.isTraceEnabled()) logger_.trace("Sending a GossipDigestSynMessage to {} ...", to);
MessagingService.instance().sendOneWay(message, to);
return seeds_.contains(to);
}
/**
* Responds to the node that requested the given valid tree.
*
* @param validator A locally generated validator
* @param local localhost (parameterized for testing)
*/
void respond(Validator validator, InetAddress local) {
MessagingService ms = MessagingService.instance();
try {
Message message = TreeResponseVerbHandler.makeVerb(local, validator);
if (!validator.request.endpoint.equals(FBUtilities.getBroadcastAddress()))
logger.info(
String.format(
"[repair #%s] Sending completed merkle tree to %s for %s",
validator.request.sessionid, validator.request.endpoint, validator.request.cf));
ms.sendOneWay(message, validator.request.endpoint);
} catch (Exception e) {
logger.error(
String.format(
"[repair #%s] Error sending completed merkle tree to %s for %s ",
validator.request.sessionid, validator.request.endpoint, validator.request.cf),
e);
}
}
public void run() {
try {
String verb = message_.getVerb();
IVerbHandler verbHandler = MessagingService.getMessagingInstance().getVerbHandler(verb);
if (verbHandler != null) {
verbHandler.doVerb(message_);
}
} catch (Throwable th) {
logger_.warn(LogUtil.throwableToString(th));
}
}
@Test
public void testTruncateRead() throws IOException {
if (EXECUTE_WRITES) testTruncateWrite();
DataInputStream in = getInput("db.Truncation.bin");
assert Truncation.serializer.deserialize(in, getVersion()) != null;
assert TruncateResponse.serializer.deserialize(in, getVersion()) != null;
assert TruncateResponse.serializer.deserialize(in, getVersion()) != null;
assert MessageIn.read(in, getVersion(), -1) != null;
// set up some fake callbacks so deserialization knows that what it's deserializing is a
// TruncateResponse
MessagingService.instance()
.setCallbackForTests(1, new CallbackInfo(null, null, TruncateResponse.serializer));
MessagingService.instance()
.setCallbackForTests(2, new CallbackInfo(null, null, TruncateResponse.serializer));
assert MessageIn.read(in, getVersion(), 1) != null;
assert MessageIn.read(in, getVersion(), 2) != null;
in.close();
}
public void serialize(PrepareMessage message, DataOutputPlus out, int version)
throws IOException {
out.writeInt(message.cfIds.size());
for (UUID cfId : message.cfIds) UUIDSerializer.serializer.serialize(cfId, out, version);
UUIDSerializer.serializer.serialize(message.parentRepairSession, out, version);
out.writeInt(message.ranges.size());
for (Range<Token> r : message.ranges) {
MessagingService.validatePartitioner(r);
Range.tokenSerializer.serialize(r, out, version);
}
out.writeBoolean(message.isIncremental);
}
public void run() {
Message message = null;
try {
message = Message.serializer().deserialize(new DataInputStream(bytes));
} catch (IOException e) {
throw new RuntimeException(e);
}
if (message != null) {
message = SinkManager.processServerMessageSink(message);
MessagingService.receive(message);
}
}
public void reduce(
Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter)
throws IOException {
ColumnFamily columnFamily;
String keyspace = "Keyspace1";
String cfName = "Super1";
Message message;
List<ColumnFamily> columnFamilies;
columnFamilies = new LinkedList<ColumnFamily>();
String line;
/* Create a column family */
columnFamily = ColumnFamily.create(keyspace, cfName);
while (values.hasNext()) {
// Split the value (line based on your own delimiter)
line = values.next().toString();
String[] fields = line.split("\1");
String SuperColumnName = fields[1];
String ColumnName = fields[2];
String ColumnValue = fields[3];
int timestamp = 0;
columnFamily.addColumn(
new QueryPath(
cfName, ByteBufferUtil.bytes(SuperColumnName), ByteBufferUtil.bytes(ColumnName)),
ByteBufferUtil.bytes(ColumnValue),
timestamp);
}
columnFamilies.add(columnFamily);
/* Get serialized message to send to cluster */
message = createMessage(keyspace, key.getBytes(), cfName, columnFamilies);
List<IAsyncResult> results = new ArrayList<IAsyncResult>();
for (InetAddress endpoint :
StorageService.instance.getNaturalEndpoints(keyspace, ByteBufferUtil.bytes(key))) {
/* Send message to end point */
results.add(MessagingService.instance().sendRR(message, endpoint));
}
/* wait for acks */
for (IAsyncResult result : results) {
try {
result.get(DatabaseDescriptor.getRpcTimeout(), TimeUnit.MILLISECONDS);
} catch (TimeoutException e) {
// you should probably add retry logic here
throw new RuntimeException(e);
}
}
output.collect(key, new Text(" inserted into Cassandra node(s)"));
}
@Before
public void setup() throws IOException, ConfigurationException {
tmd.clearUnsafe();
// create a ring of 5 nodes
Util.createInitialRing(ss, partitioner, endpointTokens, keyTokens, hosts, hostIds, 6);
MessagingService.instance().listen(FBUtilities.getBroadcastAddress());
Gossiper.instance.start(1);
removalhost = hosts.get(5);
hosts.remove(removalhost);
removalId = hostIds.get(5);
hostIds.remove(removalId);
}
public PrepareMessage deserialize(DataInput in, int version) throws IOException {
int cfIdCount = in.readInt();
List<UUID> cfIds = new ArrayList<>(cfIdCount);
for (int i = 0; i < cfIdCount; i++)
cfIds.add(UUIDSerializer.serializer.deserialize(in, version));
UUID parentRepairSession = UUIDSerializer.serializer.deserialize(in, version);
int rangeCount = in.readInt();
List<Range<Token>> ranges = new ArrayList<>(rangeCount);
for (int i = 0; i < rangeCount; i++)
ranges.add(
(Range<Token>)
Range.tokenSerializer.deserialize(
in, MessagingService.globalPartitioner(), version));
boolean isIncremental = in.readBoolean();
return new PrepareMessage(parentRepairSession, cfIds, ranges, isIncremental, isGlobal);
}
public void sendUdpOneWay(Message message, EndPoint to) {
EndPoint from = message.getFrom();
if (message.getFrom().equals(to)) {
MessagingService.receive(message);
return;
}
UdpConnection connection = null;
try {
connection = new UdpConnection();
connection.init();
connection.write(message, to);
} catch (IOException e) {
logger_.warn(LogUtil.throwableToString(e));
} finally {
if (connection != null) connection.close();
}
}
@Test
public void testRemoveHostId() throws InterruptedException {
ReplicationSink rSink = new ReplicationSink();
SinkManager.add(rSink);
// start removal in background and send replication confirmations
final AtomicBoolean success = new AtomicBoolean(false);
Thread remover =
new Thread() {
public void run() {
try {
ss.removeNode(removalId.toString());
} catch (Exception e) {
System.err.println(e);
e.printStackTrace();
return;
}
success.set(true);
}
};
remover.start();
Thread.sleep(1000); // make sure removal is waiting for confirmation
assertTrue(tmd.isLeaving(removalhost));
assertEquals(1, tmd.getLeavingEndpoints().size());
for (InetAddress host : hosts) {
MessageOut msg =
new MessageOut(
host,
MessagingService.Verb.REPLICATION_FINISHED,
null,
null,
Collections.<String, byte[]>emptyMap());
MessagingService.instance().sendRR(msg, FBUtilities.getBroadcastAddress());
}
remover.join();
assertTrue(success.get());
assertTrue(tmd.getLeavingEndpoints().isEmpty());
}
public void run() {
try {
// wait on messaging service to start listening
MessagingService.instance().waitUntilListening();
/* Update the local heartbeat counter. */
endpointStateMap_.get(localEndpoint_).getHeartBeatState().updateHeartBeat();
List<GossipDigest> gDigests = new ArrayList<GossipDigest>();
Gossiper.instance.makeRandomGossipDigest(gDigests);
if (gDigests.size() > 0) {
Message message = makeGossipDigestSynMessage(gDigests);
/* Gossip to some random live member */
boolean gossipedToSeed = doGossipToLiveMember(message);
/* Gossip to some unreachable member with some probability to check if he is back up */
doGossipToUnreachableMember(message);
/* Gossip to a seed if we did not do so above, or we have seen less nodes
than there are seeds. This prevents partitions where each group of nodes
is only gossiping to a subset of the seeds.
The most straightforward check would be to check that all the seeds have been
verified either as live or unreachable. To avoid that computation each round,
we reason that:
either all the live nodes are seeds, in which case non-seeds that come online
will introduce themselves to a member of the ring by definition,
or there is at least one non-seed node in the list, in which case eventually
someone will gossip to it, and then do a gossip to a random seed from the
gossipedToSeed check.
See CASSANDRA-150 for more exposition. */
if (!gossipedToSeed || liveEndpoints_.size() < seeds_.size()) doGossipToSeed(message);
if (logger_.isTraceEnabled()) logger_.trace("Performing status check ...");
doStatusCheck();
}
} catch (Exception e) {
logger_.error("Gossip error", e);
}
}
private InetAddress receiveMessage(DataInputStream input, int version) throws IOException {
int id;
if (version < MessagingService.VERSION_20) id = Integer.parseInt(input.readUTF());
else id = input.readInt();
long timestamp = System.currentTimeMillis();
// make sure to readInt, even if cross_node_to is not enabled
int partial = input.readInt();
if (DatabaseDescriptor.hasCrossNodeTimeout())
timestamp = (timestamp & 0xFFFFFFFF00000000L) | (((partial & 0xFFFFFFFFL) << 2) >> 2);
MessageIn message = MessageIn.read(input, version, id);
if (message == null) {
// callback expired; nothing to do
return null;
}
if (version <= MessagingService.current_version) {
MessagingService.instance().receive(message, id, timestamp);
} else {
logger.debug("Received connection from newer protocol version {}. Ignoring message", version);
}
return message.from;
}
@After
public void tearDown() {
SinkManager.clear();
MessagingService.instance().clearCallbacksUnsafe();
MessagingService.instance().shutdown();
}
