@Test
public void testCnxManagerTimeout() throws Exception {
Random rand = new Random();
byte b = (byte) rand.nextInt();
int deadPort = PortAssignment.unique();
String deadAddress = new String("10.1.1." + b);
LOG.info("This is the dead address I'm trying: " + deadAddress);
peers.put(
Long.valueOf(2),
new QuorumServer(
2,
new InetSocketAddress(deadAddress, deadPort),
new InetSocketAddress(deadAddress, PortAssignment.unique())));
peerTmpdir[2] = ClientBase.createTmpDir();
QuorumPeer peer =
new QuorumPeer(peers, peerTmpdir[1], peerTmpdir[1], peerClientPort[1], 3, 1, 1000, 2, 2);
QuorumCnxManager cnxManager = new QuorumCnxManager(peer);
QuorumCnxManager.Listener listener = cnxManager.listener;
if (listener != null) {
listener.start();
} else {
LOG.error("Null listener when initializing cnx manager");
}
long begin = System.currentTimeMillis();
cnxManager.toSend(new Long(2), createMsg(ServerState.LOOKING.ordinal(), 1, -1, 1));
long end = System.currentTimeMillis();
if ((end - begin) > 6000) fail("Waited more than necessary");
}
@Test
public void testCnxManager() throws Exception {
CnxManagerThread thread = new CnxManagerThread();
thread.start();
QuorumPeer peer =
new QuorumPeer(peers, peerTmpdir[1], peerTmpdir[1], peerClientPort[1], 3, 1, 1000, 2, 2);
QuorumCnxManager cnxManager = new QuorumCnxManager(peer);
QuorumCnxManager.Listener listener = cnxManager.listener;
if (listener != null) {
listener.start();
} else {
LOG.error("Null listener when initializing cnx manager");
}
cnxManager.toSend(new Long(0), createMsg(ServerState.LOOKING.ordinal(), 1, -1, 1));
Message m = null;
int numRetries = 1;
while ((m == null) && (numRetries++ <= THRESHOLD)) {
m = cnxManager.recvQueue.poll(3000, TimeUnit.MILLISECONDS);
if (m == null) cnxManager.connectAll();
}
assertTrue("Exceeded number of retries", numRetries <= THRESHOLD);
thread.join(5000);
if (thread.isAlive()) {
fail("Thread didn't join");
} else {
if (thread.failed) fail("Did not receive expected message");
}
}
/**
* Tests a bug in QuorumCnxManager that causes a spin lock when a negative value is sent. This
* test checks if the connection is being closed upon a message with negative length.
*
* @throws Exception
*/
@Test
public void testCnxManagerSpinLock() throws Exception {
QuorumPeer peer =
new QuorumPeer(peers, peerTmpdir[1], peerTmpdir[1], peerClientPort[1], 3, 1, 1000, 2, 2);
QuorumCnxManager cnxManager = new QuorumCnxManager(peer);
QuorumCnxManager.Listener listener = cnxManager.listener;
if (listener != null) {
listener.start();
} else {
LOG.error("Null listener when initializing cnx manager");
}
int port = peers.get(peer.getId()).electionAddr.getPort();
LOG.info("Election port: " + port);
InetSocketAddress addr = new InetSocketAddress(port);
Thread.sleep(1000);
SocketChannel sc = SocketChannel.open();
sc.socket().connect(peers.get(new Long(1)).electionAddr, 5000);
/*
* Write id first then negative length.
*/
byte[] msgBytes = new byte[8];
ByteBuffer msgBuffer = ByteBuffer.wrap(msgBytes);
msgBuffer.putLong(new Long(2));
msgBuffer.position(0);
sc.write(msgBuffer);
msgBuffer = ByteBuffer.wrap(new byte[4]);
msgBuffer.putInt(-20);
msgBuffer.position(0);
sc.write(msgBuffer);
Thread.sleep(1000);
try {
/*
* Write a number of times until it
* detects that the socket is broken.
*/
for (int i = 0; i < 100; i++) {
msgBuffer.position(0);
sc.write(msgBuffer);
}
fail("Socket has not been closed");
} catch (Exception e) {
LOG.info("Socket has been closed as expected");
}
peer.shutdown();
cnxManager.halt();
}
public void shutdown() {
stop = true;
LOG.debug("Shutting down connection manager");
manager.halt();
LOG.debug("Shutting down messenger");
messenger.halt();
LOG.debug("FLE is down");
}
public String _verifyThreadCount(ArrayList<QuorumPeer> peerList, long ecnt) {
for (int myid = 0; myid < peerList.size(); myid++) {
QuorumPeer peer = peerList.get(myid);
QuorumCnxManager cnxManager = peer.getQuorumCnxManager();
long cnt = cnxManager.getThreadCount();
if (cnt != ecnt) {
return new String(
new Date()
+ " Incorrect number of Worker threads for sid="
+ myid
+ " expected "
+ ecnt
+ " found "
+ cnt);
}
}
return null;
}
public void run() {
try {
QuorumPeer peer =
new QuorumPeer(
peers, peerTmpdir[0], peerTmpdir[0], peerClientPort[0], 3, 0, 1000, 2, 2);
QuorumCnxManager cnxManager = new QuorumCnxManager(peer);
QuorumCnxManager.Listener listener = cnxManager.listener;
if (listener != null) {
listener.start();
} else {
LOG.error("Null listener when initializing cnx manager");
}
long sid = 1;
cnxManager.toSend(sid, createMsg(ServerState.LOOKING.ordinal(), 0, -1, 1));
Message m = null;
int numRetries = 1;
while ((m == null) && (numRetries++ <= THRESHOLD)) {
m = cnxManager.recvQueue.poll(3000, TimeUnit.MILLISECONDS);
if (m == null) cnxManager.connectAll();
}
if (numRetries > THRESHOLD) {
failed = true;
return;
}
cnxManager.testInitiateConnection(sid);
m = cnxManager.recvQueue.poll(3000, TimeUnit.MILLISECONDS);
if (m == null) {
failed = true;
return;
}
} catch (Exception e) {
LOG.error("Exception while running mock thread", e);
fail("Unexpected exception");
}
}
/** 寻找对应的leader */
public Vote lookForLeader() throws InterruptedException {
try {
self.jmxLeaderElectionBean = new LeaderElectionBean();
MBeanRegistry.getInstance().register(self.jmxLeaderElectionBean, self.jmxLocalPeerBean);
} catch (Exception e) {
LOG.warn("Failed to register with JMX", e);
self.jmxLeaderElectionBean = null;
}
if (self.start_fle == 0) {
self.start_fle = System.currentTimeMillis();
}
try {
HashMap<Long, Vote> recvset = new HashMap<Long, Vote>();
HashMap<Long, Vote> outofelection = new HashMap<Long, Vote>();
int notTimeout = finalizeWait;
synchronized (this) {
logicalclock++;
updateProposal(getInitId(), getInitLastLoggedZxid(), getPeerEpoch());
}
LOG.info(
"New election. My id = "
+ self.getId()
+ ", proposed zxid=0x"
+ Long.toHexString(proposedZxid));
sendNotifications();
/*
* Loop in which we exchange notifications until we find a leader
*/
while ((self.getPeerState() == ServerState.LOOKING) && (!stop)) {
/*
* Remove next notification from queue, times out after 2 times
* the termination time
*/
Notification n = recvqueue.poll(notTimeout, TimeUnit.MILLISECONDS);
/*
* Sends more notifications if haven't received enough.
* Otherwise processes new notification.
*/
if (n == null) {
if (manager.haveDelivered()) {
sendNotifications();
} else {
manager.connectAll();
}
/*
* Exponential backoff
*/
int tmpTimeOut = notTimeout * 2;
notTimeout =
(tmpTimeOut < maxNotificationInterval ? tmpTimeOut : maxNotificationInterval);
LOG.info("Notification time out: " + notTimeout);
} else if (self.getVotingView().containsKey(n.sid)) {
/*
* Only proceed if the vote comes from a replica in the
* voting view.
*/
switch (n.state) {
case LOOKING:
// If notification > current, replace and send messages
// out
if (n.electionEpoch > logicalclock) {
logicalclock = n.electionEpoch;
recvset.clear();
if (totalOrderPredicate(
n.leader,
n.zxid,
n.peerEpoch,
getInitId(),
getInitLastLoggedZxid(),
getPeerEpoch())) {
updateProposal(n.leader, n.zxid, n.peerEpoch);
} else {
updateProposal(getInitId(), getInitLastLoggedZxid(), getPeerEpoch());
}
sendNotifications();
} else if (n.electionEpoch < logicalclock) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Notification election epoch is smaller than logicalclock. n.electionEpoch = 0x"
+ Long.toHexString(n.electionEpoch)
+ ", logicalclock=0x"
+ Long.toHexString(logicalclock));
}
break;
} else if (totalOrderPredicate(
n.leader, n.zxid, n.peerEpoch, proposedLeader, proposedZxid, proposedEpoch)) {
updateProposal(n.leader, n.zxid, n.peerEpoch);
sendNotifications();
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Adding vote: from="
+ n.sid
+ ", proposed leader="
+ n.leader
+ ", proposed zxid=0x"
+ Long.toHexString(n.zxid)
+ ", proposed election epoch=0x"
+ Long.toHexString(n.electionEpoch));
}
recvset.put(n.sid, new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch));
if (termPredicate(
recvset, new Vote(proposedLeader, proposedZxid, logicalclock, proposedEpoch))) {
// Verify if there is any change in the proposed
// leader
while ((n = recvqueue.poll(finalizeWait, TimeUnit.MILLISECONDS)) != null) {
if (totalOrderPredicate(
n.leader, n.zxid, n.peerEpoch, proposedLeader, proposedZxid, proposedEpoch)) {
recvqueue.put(n);
break;
}
}
/*
* This predicate is true once we don't read any new
* relevant message from the reception queue
*/
if (n == null) {
self.setPeerState(
(proposedLeader == self.getId()) ? ServerState.LEADING : learningState());
Vote endVote =
new Vote(proposedLeader, proposedZxid, logicalclock, proposedEpoch);
leaveInstance(endVote);
return endVote;
}
}
break;
case OBSERVING:
LOG.debug("Notification from observer: " + n.sid);
break;
case FOLLOWING:
case LEADING:
/*
* Consider all notifications from the same epoch
* together.
*/
if (n.electionEpoch == logicalclock) {
recvset.put(n.sid, new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch));
if (ooePredicate(recvset, outofelection, n)) {
self.setPeerState(
(n.leader == self.getId()) ? ServerState.LEADING : learningState());
Vote endVote = new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch);
leaveInstance(endVote);
return endVote;
}
}
/*
* Before joining an established ensemble, verify a
* majority is following the same leader.
*/
outofelection.put(
n.sid,
new Vote(n.version, n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state));
if (ooePredicate(outofelection, outofelection, n)) {
synchronized (this) {
logicalclock = n.electionEpoch;
self.setPeerState(
(n.leader == self.getId()) ? ServerState.LEADING : learningState());
}
Vote endVote = new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch);
leaveInstance(endVote);
return endVote;
}
break;
default:
LOG.warn("Notification state unrecognized: {} (n.state), {} (n.sid)", n.state, n.sid);
break;
}
} else {
LOG.warn("Ignoring notification from non-cluster member " + n.sid);
}
}
return null;
} finally {
try {
if (self.jmxLeaderElectionBean != null) {
MBeanRegistry.getInstance().unregister(self.jmxLeaderElectionBean);
}
} catch (Exception e) {
LOG.warn("Failed to unregister with JMX", e);
}
self.jmxLeaderElectionBean = null;
}
}
/**
* Called by run() once there is a new message to send.
*
* @param m message to send
*/
void process(ToSend m) {
ByteBuffer requestBuffer =
buildMsg(m.state.ordinal(), m.leader, m.zxid, m.electionEpoch, m.peerEpoch);
manager.toSend(m.sid, requestBuffer);
}
public void run() {
Message response;
while (!stop) {
// Sleeps on receive
try {
response = manager.pollRecvQueue(3000, TimeUnit.MILLISECONDS);
if (response == null) continue;
/*
* If it is from an observer, respond right away. Note
* that the following predicate assumes that if a server
* is not a follower, then it must be an observer. If we
* ever have any other type of learner in the future,
* we'll have to change the way we check for observers.
*/
if (!self.getVotingView().containsKey(response.sid)) {
Vote current = self.getCurrentVote();
ToSend notmsg =
new ToSend(
ToSend.mType.notification,
current.getId(),
current.getZxid(),
logicalclock,
self.getPeerState(),
response.sid,
current.getPeerEpoch());
sendqueue.offer(notmsg);
} else {
// Receive new message
if (LOG.isDebugEnabled()) {
LOG.debug("Receive new notification message. My id = " + self.getId());
}
/*
* We check for 28 bytes for backward compatibility
*/
if (response.buffer.capacity() < 28) {
LOG.error("Got a short response: " + response.buffer.capacity());
continue;
}
boolean backCompatibility = (response.buffer.capacity() == 28);
response.buffer.clear();
// Instantiate Notification and set its attributes
Notification n = new Notification();
// State of peer that sent this message
QuorumPeer.ServerState ackstate = QuorumPeer.ServerState.LOOKING;
switch (response.buffer.getInt()) {
case 0:
ackstate = QuorumPeer.ServerState.LOOKING;
break;
case 1:
ackstate = QuorumPeer.ServerState.FOLLOWING;
break;
case 2:
ackstate = QuorumPeer.ServerState.LEADING;
break;
case 3:
ackstate = QuorumPeer.ServerState.OBSERVING;
break;
default:
continue;
}
n.leader = response.buffer.getLong();
n.zxid = response.buffer.getLong();
n.electionEpoch = response.buffer.getLong();
n.state = ackstate;
n.sid = response.sid;
if (!backCompatibility) {
n.peerEpoch = response.buffer.getLong();
} else {
if (LOG.isInfoEnabled()) {
LOG.info("Backward compatibility mode, server id=" + n.sid);
}
n.peerEpoch = ZxidUtils.getEpochFromZxid(n.zxid);
}
/*
* Version added in 3.4.6
*/
n.version = (response.buffer.remaining() >= 4) ? response.buffer.getInt() : 0x0;
/*
* Print notification info
*/
if (LOG.isInfoEnabled()) {
printNotification(n);
}
/*
* If this server is looking, then send proposed
* leader
*/
if (self.getPeerState() == QuorumPeer.ServerState.LOOKING) {
recvqueue.offer(n);
/*
* Send a notification back if the peer that
* sent this message is also looking and its
* logical clock is lagging behind.
*/
if ((ackstate == QuorumPeer.ServerState.LOOKING)
&& (n.electionEpoch < logicalclock)) {
Vote v = getVote();
ToSend notmsg =
new ToSend(
ToSend.mType.notification,
v.getId(),
v.getZxid(),
logicalclock,
self.getPeerState(),
response.sid,
v.getPeerEpoch());
sendqueue.offer(notmsg);
}
} else {
/*
* If this server is not looking, but the one
* that sent the ack is looking, then send back
* what it believes to be the leader.
*/
Vote current = self.getCurrentVote();
if (ackstate == QuorumPeer.ServerState.LOOKING) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Sending new notification. My id = "
+ self.getId()
+ " recipient="
+ response.sid
+ " zxid=0x"
+ Long.toHexString(current.getZxid())
+ " leader="
+ current.getId());
}
ToSend notmsg;
if (n.version > 0x0) {
notmsg =
new ToSend(
ToSend.mType.notification,
current.getId(),
current.getZxid(),
current.getElectionEpoch(),
self.getPeerState(),
response.sid,
current.getPeerEpoch());
} else {
Vote bcVote = self.getBCVote();
notmsg =
new ToSend(
ToSend.mType.notification,
bcVote.getId(),
bcVote.getZxid(),
bcVote.getElectionEpoch(),
self.getPeerState(),
response.sid,
bcVote.getPeerEpoch());
}
sendqueue.offer(notmsg);
}
}
}
} catch (InterruptedException e) {
System.out.println(
"Interrupted Exception while waiting for new message" + e.toString());
}
}
LOG.info("WorkerReceiver is down");
}