public void loadSlavesAndRacksFromMaster(MesosMasterStateObject state) {
Map<String, SingularitySlave> activeSlavesById =
slaveManager.getObjectsByIdForState(MachineState.ACTIVE);
Map<String, SingularityRack> activeRacksById =
rackManager.getObjectsByIdForState(MachineState.ACTIVE);
Map<String, SingularityRack> remainingActiveRacks = Maps.newHashMap(activeRacksById);
int slaves = 0;
int racks = 0;
for (MesosMasterSlaveObject slaveJsonObject : state.getSlaves()) {
String slaveId = slaveJsonObject.getId();
String rackId = slaveAndRackHelper.getRackId(slaveJsonObject.getAttributes());
String host = slaveAndRackHelper.getMaybeTruncatedHost(slaveJsonObject.getHostname());
if (activeSlavesById.containsKey(slaveId)) {
activeSlavesById.remove(slaveId);
} else {
SingularitySlave newSlave = new SingularitySlave(slaveId, host, rackId);
if (check(newSlave, slaveManager) == CheckResult.NEW) {
slaves++;
}
}
if (activeRacksById.containsKey(rackId)) {
remainingActiveRacks.remove(rackId);
} else {
SingularityRack rack = new SingularityRack(rackId);
if (check(rack, rackManager) == CheckResult.NEW) {
racks++;
}
}
}
for (SingularitySlave leftOverSlave : activeSlavesById.values()) {
slaveManager.changeState(
leftOverSlave, MachineState.MISSING_ON_STARTUP, Optional.<String>absent());
}
for (SingularityRack leftOverRack : remainingActiveRacks.values()) {
rackManager.changeState(
leftOverRack, MachineState.MISSING_ON_STARTUP, Optional.<String>absent());
}
LOG.info(
"Found {} new racks ({} missing) and {} new slaves ({} missing)",
racks,
remainingActiveRacks.size(),
slaves,
activeSlavesById.size());
}
@Timed
public void checkOffer(Offer offer) {
final String slaveId = offer.getSlaveId().getValue();
final String rackId = slaveAndRackHelper.getRackIdOrDefault(offer);
final String host = slaveAndRackHelper.getMaybeTruncatedHost(offer);
final SingularitySlave slave = new SingularitySlave(slaveId, host, rackId);
if (check(slave, slaveManager) == CheckResult.NEW) {
LOG.info("Offer revealed a new slave {}", slave);
}
final SingularityRack rack = new SingularityRack(rackId);
if (check(rack, rackManager) == CheckResult.NEW) {
LOG.info("Offer revealed a new rack {}", rack);
}
}
public SlaveMatchState doesOfferMatch(
Protos.Offer offer,
SingularityTaskRequest taskRequest,
SingularitySchedulerStateCache stateCache) {
final String host = offer.getHostname();
final String rackId = slaveAndRackHelper.getRackIdOrDefault(offer);
final String slaveId = offer.getSlaveId().getValue();
final MachineState currentSlaveState =
stateCache.getSlave(slaveId).get().getCurrentState().getState();
if (currentSlaveState == MachineState.FROZEN) {
return SlaveMatchState.SLAVE_FROZEN;
}
if (currentSlaveState.isDecommissioning()) {
return SlaveMatchState.SLAVE_DECOMMISSIONING;
}
final MachineState currentRackState =
stateCache.getRack(rackId).get().getCurrentState().getState();
if (currentRackState == MachineState.FROZEN) {
return SlaveMatchState.RACK_FROZEN;
}
if (currentRackState.isDecommissioning()) {
return SlaveMatchState.RACK_DECOMMISSIONING;
}
if (!taskRequest.getRequest().getRackAffinity().or(Collections.<String>emptyList()).isEmpty()) {
if (!taskRequest.getRequest().getRackAffinity().get().contains(rackId)) {
LOG.trace(
"Task {} requires a rack in {} (current rack {})",
taskRequest.getPendingTask().getPendingTaskId(),
taskRequest.getRequest().getRackAffinity().get(),
rackId);
return SlaveMatchState.RACK_AFFINITY_NOT_MATCHING;
}
}
final SlavePlacement slavePlacement =
taskRequest.getRequest().getSlavePlacement().or(configuration.getDefaultSlavePlacement());
if (!taskRequest.getRequest().isRackSensitive() && slavePlacement == SlavePlacement.GREEDY) {
return SlaveMatchState.NOT_RACK_OR_SLAVE_PARTICULAR;
}
final int numDesiredInstances = taskRequest.getRequest().getInstancesSafe();
double numOnRack = 0;
double numOnSlave = 0;
double numCleaningOnSlave = 0;
double numOtherDeploysOnSlave = 0;
final String sanitizedHost = JavaUtils.getReplaceHyphensWithUnderscores(host);
final String sanitizedRackId = JavaUtils.getReplaceHyphensWithUnderscores(rackId);
Collection<SingularityTaskId> cleaningTasks = stateCache.getCleaningTasks();
for (SingularityTaskId taskId :
SingularityTaskId.matchingAndNotIn(
stateCache.getActiveTaskIds(),
taskRequest.getRequest().getId(),
Collections.<SingularityTaskId>emptyList())) {
// TODO consider using executorIds
if (taskId.getSanitizedHost().equals(sanitizedHost)) {
if (taskRequest.getDeploy().getId().equals(taskId.getDeployId())) {
if (cleaningTasks.contains(taskId)) {
numCleaningOnSlave++;
} else {
numOnSlave++;
}
} else {
numOtherDeploysOnSlave++;
}
}
if (taskId.getSanitizedRackId().equals(sanitizedRackId)
&& !cleaningTasks.contains(taskId)
&& taskRequest.getDeploy().getId().equals(taskId.getDeployId())) {
numOnRack++;
}
}
if (taskRequest.getRequest().isRackSensitive()) {
final double numPerRack = numDesiredInstances / (double) stateCache.getNumActiveRacks();
final boolean isRackOk = numOnRack < numPerRack;
if (!isRackOk) {
LOG.trace(
"Rejecting RackSensitive task {} from slave {} ({}) due to numOnRack {} and cleaningOnSlave {}",
taskRequest.getRequest().getId(),
slaveId,
host,
numOnRack,
numCleaningOnSlave);
return SlaveMatchState.RACK_SATURATED;
}
}
switch (slavePlacement) {
case SEPARATE:
case SEPARATE_BY_DEPLOY:
if (numOnSlave > 0 || numCleaningOnSlave > 0) {
LOG.trace(
"Rejecting SEPARATE task {} from slave {} ({}) due to numOnSlave {} numCleaningOnSlave {}",
taskRequest.getRequest().getId(),
slaveId,
host,
numOnSlave,
numCleaningOnSlave);
return SlaveMatchState.SLAVE_SATURATED;
}
break;
case SEPARATE_BY_REQUEST:
if (numOnSlave > 0 || numCleaningOnSlave > 0 || numOtherDeploysOnSlave > 0) {
LOG.trace(
"Rejecting SEPARATE task {} from slave {} ({}) due to numOnSlave {} numCleaningOnSlave {} numOtherDeploysOnSlave {}",
taskRequest.getRequest().getId(),
slaveId,
host,
numOnSlave,
numCleaningOnSlave,
numOtherDeploysOnSlave);
return SlaveMatchState.SLAVE_SATURATED;
}
break;
case OPTIMISTIC:
final double numPerSlave = numDesiredInstances / (double) stateCache.getNumActiveSlaves();
final boolean isSlaveOk = numOnSlave < numPerSlave;
if (!isSlaveOk) {
LOG.trace(
"Rejecting OPTIMISTIC task {} from slave {} ({}) due to numOnSlave {}",
taskRequest.getRequest().getId(),
slaveId,
host,
numOnSlave);
return SlaveMatchState.SLAVE_SATURATED;
}
break;
case GREEDY:
}
return SlaveMatchState.OK;
}
