@Test
public void testGetManagementPlaneSyncStateInfersTimedOutNodeAsFailed() throws Exception {
persister.delta(
ManagementPlaneSyncRecordDeltaImpl.builder()
.node(newManagerMemento(ownNodeId, ManagementNodeState.STANDBY, tickerCurrentMillis()))
.node(newManagerMemento("node1", ManagementNodeState.MASTER, tickerCurrentMillis()))
.setMaster("node1")
.build());
manager.start(HighAvailabilityMode.AUTO);
ManagementPlaneSyncRecord state = manager.getManagementPlaneSyncState();
assertEquals(state.getManagementNodes().get("node1").getStatus(), ManagementNodeState.MASTER);
assertEquals(
state.getManagementNodes().get(ownNodeId).getStatus(), ManagementNodeState.STANDBY);
// Simulate passage of time; ticker used by this HA-manager so it will "correctly" publish
// its own heartbeat with the new time; but node1's record is now out-of-date.
tickerAdvance(Duration.seconds(31));
ManagementPlaneSyncRecord state2 = manager.getManagementPlaneSyncState();
assertEquals(state2.getManagementNodes().get("node1").getStatus(), ManagementNodeState.FAILED);
assertNotEquals(
state.getManagementNodes().get(ownNodeId).getStatus(), ManagementNodeState.FAILED);
}
public void testGetManagementPlaneStatus() throws Exception {
// with the name zzzzz the mgr created here should never be promoted by the alphabetical
// strategy!
tickerAdvance(Duration.FIVE_SECONDS);
persister.delta(
ManagementPlaneSyncRecordDeltaImpl.builder()
.node(
newManagerMemento(
"zzzzzzz_node1", ManagementNodeState.STANDBY, tickerCurrentMillis()))
.build());
long zzzTime = tickerCurrentMillis();
tickerAdvance(Duration.FIVE_SECONDS);
manager.start(HighAvailabilityMode.AUTO);
ManagementPlaneSyncRecord memento = manager.getManagementPlaneSyncState();
// Note can assert timestamp because not "real" time; it's using our own Ticker
assertEquals(memento.getMasterNodeId(), ownNodeId);
assertEquals(
memento.getManagementNodes().keySet(), ImmutableSet.of(ownNodeId, "zzzzzzz_node1"));
assertEquals(memento.getManagementNodes().get(ownNodeId).getNodeId(), ownNodeId);
assertEquals(
memento.getManagementNodes().get(ownNodeId).getStatus(), ManagementNodeState.MASTER);
assertEquals(
memento.getManagementNodes().get(ownNodeId).getTimestampUtc(), tickerCurrentMillis());
assertEquals(memento.getManagementNodes().get("zzzzzzz_node1").getNodeId(), "zzzzzzz_node1");
assertEquals(
memento.getManagementNodes().get("zzzzzzz_node1").getStatus(), ManagementNodeState.STANDBY);
assertEquals(memento.getManagementNodes().get("zzzzzzz_node1").getTimestampUtc(), zzzTime);
}
// The web-console could still be polling (e.g. if have just restarted brooklyn), before the
// persister is set.
// Must not throw NPE, but instead return something sensible (e.g. an empty state record).
@Test
public void testGetManagementPlaneSyncStateDoesNotThrowNpeBeforePersisterSet() throws Exception {
HighAvailabilityManagerImpl manager2 =
new HighAvailabilityManagerImpl(managementContext)
.setPollPeriod(Duration.millis(10))
.setHeartbeatTimeout(Duration.THIRTY_SECONDS)
.setPromotionListener(promotionListener)
.setTicker(ticker);
try {
ManagementPlaneSyncRecord state = manager2.getManagementPlaneSyncState();
assertNotNull(state);
} finally {
manager2.stop();
}
}
@Test(groups = "Integration") // because one second wait in succeedsContinually
public void testDoesNotPromoteIfMasterTimeoutNotExpired() throws Exception {
persister.delta(
ManagementPlaneSyncRecordDeltaImpl.builder()
.node(newManagerMemento(ownNodeId, ManagementNodeState.STANDBY, tickerCurrentMillis()))
.node(newManagerMemento("node1", ManagementNodeState.MASTER, tickerCurrentMillis()))
.setMaster("node1")
.build());
manager.start(HighAvailabilityMode.AUTO);
tickerAdvance(Duration.seconds(29));
// Expect not to be notified, as 29s < 30s timeout (it's a fake clock so won't hit 30, even
// waiting 1s below)
Asserts.succeedsContinually(
new Runnable() {
@Override
public void run() {
assertTrue(
promotionListener.callTimestamps.isEmpty(),
"calls=" + promotionListener.callTimestamps);
}
});
}
// Can get a log.error about our management node's heartbeat being out of date. Caused by
// poller first writing a heartbeat record, and then the clock being incremented. But the
// next poll fixes it.
public void testPromotes() throws Exception {
persister.delta(
ManagementPlaneSyncRecordDeltaImpl.builder()
.node(newManagerMemento(ownNodeId, ManagementNodeState.STANDBY, tickerCurrentMillis()))
.node(newManagerMemento("node1", ManagementNodeState.MASTER, tickerCurrentMillis()))
.setMaster("node1")
.build());
manager.start(HighAvailabilityMode.AUTO);
// Simulate passage of time; ticker used by this HA-manager so it will "correctly" publish
// its own heartbeat with the new time; but node1's record is now out-of-date.
tickerAdvance(Duration.seconds(31));
// Expect to be notified of our promotion, as the only other node
promotionListener.assertCalledEventually();
}
@AfterMethod(alwaysRun = true)
public void tearDown() throws Exception {
if (manager != null) manager.stop();
if (managementContext != null) Entities.destroyAll(managementContext);
if (objectStore != null) objectStore.deleteCompletely();
}
