@Test
public void testAssignmentClearedOnError() throws Exception {
expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(OFFER));
offerManager.launchTask(MESOS_OFFER.getId(), TASK_INFO);
expectLastCall().andThrow(new OfferManager.LaunchException("expected"));
expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT);
expect(filter.filter(UNUSED, RESOURCE_REQUEST)).andReturn(ImmutableSet.of());
expect(
stateManager.assignTask(
storeProvider,
Tasks.id(TASK),
MESOS_OFFER.getHostname(),
MESOS_OFFER.getSlaveId(),
ImmutableMap.of(PORT_NAME, PORT)))
.andReturn(TASK.getAssignedTask());
expect(
stateManager.changeState(
storeProvider, Tasks.id(TASK), Optional.of(PENDING), LOST, LAUNCH_FAILED_MSG))
.andReturn(StateChangeResult.SUCCESS);
expect(taskFactory.createFrom(TASK.getAssignedTask(), MESOS_OFFER.getSlaveId()))
.andReturn(TASK_INFO);
control.replay();
assertFalse(
assigner.maybeAssign(
storeProvider,
new ResourceRequest(TASK.getAssignedTask().getTask(), EMPTY),
TaskGroupKey.from(TASK.getAssignedTask().getTask()),
Tasks.id(TASK),
NO_RESERVATION));
}
@Test
public void testAssignNoVetoes() throws Exception {
expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(OFFER));
offerManager.launchTask(MESOS_OFFER.getId(), TASK_INFO);
expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT);
expect(filter.filter(UNUSED, RESOURCE_REQUEST)).andReturn(ImmutableSet.of());
expect(
stateManager.assignTask(
storeProvider,
Tasks.id(TASK),
MESOS_OFFER.getHostname(),
MESOS_OFFER.getSlaveId(),
ImmutableMap.of(PORT_NAME, PORT)))
.andReturn(TASK.getAssignedTask());
expect(taskFactory.createFrom(TASK.getAssignedTask(), MESOS_OFFER.getSlaveId()))
.andReturn(TASK_INFO);
control.replay();
assertTrue(
assigner.maybeAssign(
storeProvider,
new ResourceRequest(TASK.getAssignedTask().getTask(), EMPTY),
TaskGroupKey.from(TASK.getAssignedTask().getTask()),
Tasks.id(TASK),
ImmutableMap.of(SLAVE_ID, GROUP_KEY)));
}
@Test
public void testAssignerDoesNotReturnOnFirstMismatch() throws Exception {
// Ensures scheduling loop does not terminate prematurely when the first mismatch is identified.
HostOffer mismatched =
new HostOffer(
Offer.newBuilder()
.setId(OfferID.newBuilder().setValue("offerId0"))
.setFrameworkId(FrameworkID.newBuilder().setValue("frameworkId"))
.setSlaveId(SlaveID.newBuilder().setValue("slaveId0"))
.setHostname("hostName0")
.addResources(
Resource.newBuilder()
.setName("ports")
.setType(Type.RANGES)
.setRanges(
Ranges.newBuilder()
.addRange(Range.newBuilder().setBegin(PORT).setEnd(PORT))))
.build(),
IHostAttributes.build(new HostAttributes()));
expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(mismatched, OFFER));
expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT).times(2);
expect(
filter.filter(
new UnusedResource(
Resources.from(mismatched.getOffer()).slot(), mismatched.getAttributes()),
new ResourceRequest(TASK.getAssignedTask().getTask(), EMPTY)))
.andReturn(ImmutableSet.of(Veto.constraintMismatch("constraint mismatch")));
offerManager.banOffer(mismatched.getOffer().getId(), GROUP_KEY);
expect(
filter.filter(
new UnusedResource(Resources.from(OFFER.getOffer()).slot(), OFFER.getAttributes()),
new ResourceRequest(TASK.getAssignedTask().getTask(), EMPTY)))
.andReturn(ImmutableSet.of());
expect(
stateManager.assignTask(
storeProvider,
Tasks.id(TASK),
OFFER.getOffer().getHostname(),
OFFER.getOffer().getSlaveId(),
ImmutableMap.of(PORT_NAME, PORT)))
.andReturn(TASK.getAssignedTask());
expect(taskFactory.createFrom(TASK.getAssignedTask(), OFFER.getOffer().getSlaveId()))
.andReturn(TASK_INFO);
offerManager.launchTask(OFFER.getOffer().getId(), TASK_INFO);
control.replay();
assertTrue(
assigner.maybeAssign(
storeProvider,
new ResourceRequest(TASK.getAssignedTask().getTask(), EMPTY),
TaskGroupKey.from(TASK.getAssignedTask().getTask()),
Tasks.id(TASK),
ImmutableMap.of(SLAVE_ID, GROUP_KEY)));
}
@Override
public ResourceSlot apply(PreemptionVictim victim) {
ResourceSlot slot = victim.getResourceSlot();
if (tierManager.getTier(victim.getConfig()).isRevocable()) {
// Revocable task CPU cannot be used for preemption purposes as it's a compressible
// resource. We can still use RAM, DISK and PORTS as they are not compressible.
slot = new ResourceSlot(0.0, slot.getRam(), slot.getDisk(), slot.getNumPorts());
}
return slot.add(executorSettings.getExecutorOverhead());
}
@Test
public void testTaskWithReservedSlaveLandsElsewhere() throws Exception {
// Ensures slave/task reservation relationship is only enforced in slave->task direction
// and permissive in task->slave direction. In other words, a task with a slave reservation
// should still be tried against other unreserved slaves.
HostOffer offer =
new HostOffer(
Offer.newBuilder()
.setId(OfferID.newBuilder().setValue("offerId0"))
.setFrameworkId(FrameworkID.newBuilder().setValue("frameworkId"))
.setSlaveId(SlaveID.newBuilder().setValue("slaveId0"))
.setHostname("hostName0")
.addResources(
Resource.newBuilder()
.setName("ports")
.setType(Type.RANGES)
.setRanges(
Ranges.newBuilder()
.addRange(Range.newBuilder().setBegin(PORT).setEnd(PORT))))
.build(),
IHostAttributes.build(new HostAttributes()));
expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(offer, OFFER));
expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT);
expect(filter.filter(UNUSED, RESOURCE_REQUEST)).andReturn(ImmutableSet.of());
expect(
stateManager.assignTask(
storeProvider,
Tasks.id(TASK),
offer.getOffer().getHostname(),
offer.getOffer().getSlaveId(),
ImmutableMap.of(PORT_NAME, PORT)))
.andReturn(TASK.getAssignedTask());
expect(taskFactory.createFrom(TASK.getAssignedTask(), offer.getOffer().getSlaveId()))
.andReturn(TASK_INFO);
offerManager.launchTask(offer.getOffer().getId(), TASK_INFO);
control.replay();
assertTrue(
assigner.maybeAssign(
storeProvider,
new ResourceRequest(TASK.getAssignedTask().getTask(), EMPTY),
TaskGroupKey.from(TASK.getAssignedTask().getTask()),
Tasks.id(TASK),
ImmutableMap.of(SLAVE_ID, GROUP_KEY)));
}
@Test
public void testAssignVetoesWithNoStaticBan() throws Exception {
expect(offerManager.getOffers(GROUP_KEY)).andReturn(ImmutableSet.of(OFFER));
expect(tierManager.getTier(TASK.getAssignedTask().getTask())).andReturn(DEFAULT);
expect(filter.filter(UNUSED, RESOURCE_REQUEST))
.andReturn(ImmutableSet.of(Veto.unsatisfiedLimit("limit")));
control.replay();
assertFalse(
assigner.maybeAssign(
storeProvider,
new ResourceRequest(TASK.getAssignedTask().getTask(), EMPTY),
TaskGroupKey.from(TASK.getAssignedTask().getTask()),
Tasks.id(TASK),
NO_RESERVATION));
}
/**
* Check validity of and populates defaults in a task configuration. This will return a deep copy
* of the provided task configuration with default configuration values applied, and configuration
* map values sanitized and applied to their respective struct fields.
*
* @param config Task config to validate and populate.
* @return A reference to the modified {@code config} (for chaining).
* @throws TaskDescriptionException If the task is invalid.
*/
public ITaskConfig validateAndPopulate(ITaskConfig config) throws TaskDescriptionException {
TaskConfig builder = config.newBuilder();
if (!builder.isSetRequestedPorts()) {
builder.setRequestedPorts(ImmutableSet.of());
}
if (config.isSetTier() && !UserProvidedStrings.isGoodIdentifier(config.getTier())) {
throw new TaskDescriptionException("Tier contains illegal characters: " + config.getTier());
}
try {
tierManager.getTier(config);
} catch (IllegalArgumentException e) {
throw new TaskDescriptionException(e.getMessage(), e);
}
if (!JobKeys.isValid(config.getJob())) {
// Job key is set but invalid
throw new TaskDescriptionException("Job key " + config.getJob() + " is invalid.");
}
// A task must either have an executor configuration or specify a Docker container.
if (!builder.isSetExecutorConfig()
&& !(builder.isSetContainer() && builder.getContainer().isSetDocker())) {
throw new TaskDescriptionException(NO_EXECUTOR_OR_CONTAINER);
}
// Docker containers don't require executors, validate the rest
if (builder.isSetExecutorConfig()) {
if (!builder.getExecutorConfig().isSetName()) {
throw new TaskDescriptionException(INVALID_EXECUTOR_CONFIG);
}
executorSettings
.getExecutorConfig(builder.getExecutorConfig().getName())
.orElseThrow(
() ->
new TaskDescriptionException(
"Configuration for executor '"
+ builder.getExecutorConfig().getName()
+ "' doesn't exist."));
}
// Maximize the usefulness of any thrown error message by checking required fields first.
for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) {
validator.validate(builder);
}
IConstraint constraint = getDedicatedConstraint(config);
if (constraint != null) {
if (!isValueConstraint(constraint.getConstraint())) {
throw new TaskDescriptionException("A dedicated constraint must be of value type.");
}
IValueConstraint valueConstraint = constraint.getConstraint().getValue();
if (valueConstraint.getValues().size() != 1) {
throw new TaskDescriptionException("A dedicated constraint must have exactly one value");
}
String dedicatedRole = getRole(valueConstraint);
if (!("*".equals(dedicatedRole) || config.getJob().getRole().equals(dedicatedRole))) {
throw new TaskDescriptionException(
"Only " + dedicatedRole + " may use hosts dedicated for that role.");
}
}
Optional<Container._Fields> containerType;
if (config.isSetContainer()) {
IContainer containerConfig = config.getContainer();
containerType = Optional.of(containerConfig.getSetField());
if (containerConfig.isSetDocker()) {
if (!containerConfig.getDocker().isSetImage()) {
throw new TaskDescriptionException("A container must specify an image.");
}
if (containerConfig.getDocker().getParameters().isEmpty()) {
for (Map.Entry<String, String> e : settings.defaultDockerParameters.entries()) {
builder
.getContainer()
.getDocker()
.addToParameters(new DockerParameter(e.getKey(), e.getValue()));
}
} else {
if (!settings.allowDockerParameters) {
throw new TaskDescriptionException(NO_DOCKER_PARAMETERS);
}
}
if (settings.requireDockerUseExecutor && !config.isSetExecutorConfig()) {
throw new TaskDescriptionException(EXECUTOR_REQUIRED_WITH_DOCKER);
}
}
} else {
// Default to mesos container type if unset.
containerType = Optional.of(Container._Fields.MESOS);
}
if (!containerType.isPresent()) {
throw new TaskDescriptionException("A job must have a container type.");
}
if (!settings.allowedContainerTypes.contains(containerType.get())) {
throw new TaskDescriptionException(
"This scheduler is not configured to allow the container type "
+ containerType.get().toString());
}
thriftBackfill.backfillTask(builder);
String types =
config
.getResources()
.stream()
.collect(Collectors.groupingBy(e -> ResourceType.fromResource(e)))
.entrySet()
.stream()
.filter(e -> !e.getKey().isMultipleAllowed() && e.getValue().size() > 1)
.map(r -> r.getKey().getAuroraName())
.sorted()
.collect(Collectors.joining(", "));
if (!Strings.isNullOrEmpty(types)) {
throw new TaskDescriptionException("Multiple resource values are not supported for " + types);
}
if (!settings.allowGpuResource
&& config
.getResources()
.stream()
.filter(r -> ResourceType.fromResource(r).equals(GPUS))
.findAny()
.isPresent()) {
throw new TaskDescriptionException("GPU resource support is disabled in this cluster.");
}
if (!settings.enableMesosFetcher && !config.getMesosFetcherUris().isEmpty()) {
throw new TaskDescriptionException(MESOS_FETCHER_DISABLED);
}
if (config.getContainer().isSetMesos()) {
IMesosContainer container = config.getContainer().getMesos();
if (!settings.allowContainerVolumes && !container.getVolumes().isEmpty()) {
throw new TaskDescriptionException(NO_CONTAINER_VOLUMES);
}
}
maybeFillLinks(builder);
return ITaskConfig.build(builder);
}
