public void loadSlavesAndRacksFromMaster(MesosMasterStateObject state) {
Map<String, SingularitySlave> activeSlavesById =
slaveManager.getObjectsByIdForState(MachineState.ACTIVE);
Map<String, SingularityRack> activeRacksById =
rackManager.getObjectsByIdForState(MachineState.ACTIVE);
Map<String, SingularityRack> remainingActiveRacks = Maps.newHashMap(activeRacksById);
int slaves = 0;
int racks = 0;
for (MesosMasterSlaveObject slaveJsonObject : state.getSlaves()) {
String slaveId = slaveJsonObject.getId();
String rackId = slaveAndRackHelper.getRackId(slaveJsonObject.getAttributes());
String host = slaveAndRackHelper.getMaybeTruncatedHost(slaveJsonObject.getHostname());
if (activeSlavesById.containsKey(slaveId)) {
activeSlavesById.remove(slaveId);
} else {
SingularitySlave newSlave = new SingularitySlave(slaveId, host, rackId);
if (check(newSlave, slaveManager) == CheckResult.NEW) {
slaves++;
}
}
if (activeRacksById.containsKey(rackId)) {
remainingActiveRacks.remove(rackId);
} else {
SingularityRack rack = new SingularityRack(rackId);
if (check(rack, rackManager) == CheckResult.NEW) {
racks++;
}
}
}
for (SingularitySlave leftOverSlave : activeSlavesById.values()) {
slaveManager.changeState(
leftOverSlave, MachineState.MISSING_ON_STARTUP, Optional.<String>absent());
}
for (SingularityRack leftOverRack : remainingActiveRacks.values()) {
rackManager.changeState(
leftOverRack, MachineState.MISSING_ON_STARTUP, Optional.<String>absent());
}
LOG.info(
"Found {} new racks ({} missing) and {} new slaves ({} missing)",
racks,
remainingActiveRacks.size(),
slaves,
activeSlavesById.size());
}
public void slaveLost(SlaveID slaveIdObj) {
final String slaveId = slaveIdObj.getValue();
Optional<SingularitySlave> slave = slaveManager.getObject(slaveId);
if (slave.isPresent()) {
slaveManager.changeState(slave.get(), MachineState.DEAD, Optional.<String>absent());
checkRackAfterSlaveLoss(slave.get());
} else {
LOG.warn("Lost a slave {}, but didn't know about it", slaveId);
}
}
@Timed
public void checkStateAfterFinishedTask(
SingularityTaskId taskId, String slaveId, SingularitySchedulerStateCache stateCache) {
Optional<SingularitySlave> slave = slaveManager.getObject(slaveId);
if (!slave.isPresent()) {
final String message =
String.format("Couldn't find slave with id %s for task %s", slaveId, taskId);
LOG.warn(message);
exceptionNotifier.notify(
message, ImmutableMap.of("slaveId", slaveId, "taskId", taskId.toString()));
return;
}
if (slave.get().getCurrentState().getState() == MachineState.DECOMMISSIONING) {
if (!hasTaskLeftOnSlave(taskId, slaveId, stateCache)) {
slaveManager.changeState(
slave.get(), MachineState.DECOMMISSIONED, slave.get().getCurrentState().getUser());
}
}
Optional<SingularityRack> rack = rackManager.getObject(slave.get().getRackId());
if (!rack.isPresent()) {
final String message =
String.format(
"Couldn't find rack with id %s for task %s", slave.get().getRackId(), taskId);
LOG.warn(message);
exceptionNotifier.notify(
message, ImmutableMap.of("rackId", slave.get().getRackId(), "taskId", taskId.toString()));
return;
}
if (rack.get().getCurrentState().getState() == MachineState.DECOMMISSIONING) {
if (!hasTaskLeftOnRack(taskId, stateCache)) {
rackManager.changeState(
rack.get(), MachineState.DECOMMISSIONED, rack.get().getCurrentState().getUser());
}
}
}