@Test
public void testGetManagementPlaneSyncStateInfersTimedOutNodeAsFailed() throws Exception {
persister.delta(
ManagementPlaneSyncRecordDeltaImpl.builder()
.node(newManagerMemento(ownNodeId, ManagementNodeState.STANDBY, tickerCurrentMillis()))
.node(newManagerMemento("node1", ManagementNodeState.MASTER, tickerCurrentMillis()))
.setMaster("node1")
.build());
manager.start(HighAvailabilityMode.AUTO);
ManagementPlaneSyncRecord state = manager.getManagementPlaneSyncState();
assertEquals(state.getManagementNodes().get("node1").getStatus(), ManagementNodeState.MASTER);
assertEquals(
state.getManagementNodes().get(ownNodeId).getStatus(), ManagementNodeState.STANDBY);
// Simulate passage of time; ticker used by this HA-manager so it will "correctly" publish
// its own heartbeat with the new time; but node1's record is now out-of-date.
tickerAdvance(Duration.seconds(31));
ManagementPlaneSyncRecord state2 = manager.getManagementPlaneSyncState();
assertEquals(state2.getManagementNodes().get("node1").getStatus(), ManagementNodeState.FAILED);
assertNotEquals(
state.getManagementNodes().get(ownNodeId).getStatus(), ManagementNodeState.FAILED);
}
@Test(groups = "Integration") // because one second wait in succeedsContinually
public void testDoesNotPromoteIfMasterTimeoutNotExpired() throws Exception {
persister.delta(
ManagementPlaneSyncRecordDeltaImpl.builder()
.node(newManagerMemento(ownNodeId, ManagementNodeState.STANDBY, tickerCurrentMillis()))
.node(newManagerMemento("node1", ManagementNodeState.MASTER, tickerCurrentMillis()))
.setMaster("node1")
.build());
manager.start(HighAvailabilityMode.AUTO);
tickerAdvance(Duration.seconds(29));
// Expect not to be notified, as 29s < 30s timeout (it's a fake clock so won't hit 30, even
// waiting 1s below)
Asserts.succeedsContinually(
new Runnable() {
@Override
public void run() {
assertTrue(
promotionListener.callTimestamps.isEmpty(),
"calls=" + promotionListener.callTimestamps);
}
});
}
public void testGetManagementPlaneStatus() throws Exception {
// with the name zzzzz the mgr created here should never be promoted by the alphabetical
// strategy!
tickerAdvance(Duration.FIVE_SECONDS);
persister.delta(
ManagementPlaneSyncRecordDeltaImpl.builder()
.node(
newManagerMemento(
"zzzzzzz_node1", ManagementNodeState.STANDBY, tickerCurrentMillis()))
.build());
long zzzTime = tickerCurrentMillis();
tickerAdvance(Duration.FIVE_SECONDS);
manager.start(HighAvailabilityMode.AUTO);
ManagementPlaneSyncRecord memento = manager.getManagementPlaneSyncState();
// Note can assert timestamp because not "real" time; it's using our own Ticker
assertEquals(memento.getMasterNodeId(), ownNodeId);
assertEquals(
memento.getManagementNodes().keySet(), ImmutableSet.of(ownNodeId, "zzzzzzz_node1"));
assertEquals(memento.getManagementNodes().get(ownNodeId).getNodeId(), ownNodeId);
assertEquals(
memento.getManagementNodes().get(ownNodeId).getStatus(), ManagementNodeState.MASTER);
assertEquals(
memento.getManagementNodes().get(ownNodeId).getTimestampUtc(), tickerCurrentMillis());
assertEquals(memento.getManagementNodes().get("zzzzzzz_node1").getNodeId(), "zzzzzzz_node1");
assertEquals(
memento.getManagementNodes().get("zzzzzzz_node1").getStatus(), ManagementNodeState.STANDBY);
assertEquals(memento.getManagementNodes().get("zzzzzzz_node1").getTimestampUtc(), zzzTime);
}
// Can get a log.error about our management node's heartbeat being out of date. Caused by
// poller first writing a heartbeat record, and then the clock being incremented. But the
// next poll fixes it.
public void testPromotes() throws Exception {
persister.delta(
ManagementPlaneSyncRecordDeltaImpl.builder()
.node(newManagerMemento(ownNodeId, ManagementNodeState.STANDBY, tickerCurrentMillis()))
.node(newManagerMemento("node1", ManagementNodeState.MASTER, tickerCurrentMillis()))
.setMaster("node1")
.build());
manager.start(HighAvailabilityMode.AUTO);
// Simulate passage of time; ticker used by this HA-manager so it will "correctly" publish
// its own heartbeat with the new time; but node1's record is now out-of-date.
tickerAdvance(Duration.seconds(31));
// Expect to be notified of our promotion, as the only other node
promotionListener.assertCalledEventually();
}
