/**
* Check validity of and populates defaults in a job configuration. This will return a deep copy
* of the provided job configuration with default configuration values applied, and configuration
* map values sanitized and applied to their respective struct fields.
*
* @param job Job to validate and populate.
* @return A deep copy of {@code job} that has been populated.
* @throws TaskDescriptionException If the job configuration is invalid.
*/
public IJobConfiguration validateAndPopulate(IJobConfiguration job)
throws TaskDescriptionException {
requireNonNull(job);
if (!job.isSetTaskConfig()) {
throw new TaskDescriptionException("Job configuration must have taskConfig set.");
}
if (job.getInstanceCount() <= 0) {
throw new TaskDescriptionException("Instance count must be positive.");
}
JobConfiguration builder = job.newBuilder();
if (!JobKeys.isValid(job.getKey())) {
throw new TaskDescriptionException("Job key " + job.getKey() + " is invalid.");
}
if (job.isSetOwner() && !UserProvidedStrings.isGoodIdentifier(job.getOwner().getUser())) {
throw new TaskDescriptionException(
"Job user contains illegal characters: " + job.getOwner().getUser());
}
builder.setTaskConfig(
validateAndPopulate(ITaskConfig.build(builder.getTaskConfig())).newBuilder());
// Only one of [service=true, cron_schedule] may be set.
if (!Strings.isNullOrEmpty(job.getCronSchedule()) && builder.getTaskConfig().isIsService()) {
throw new TaskDescriptionException(
"A service task may not be run on a cron schedule: " + builder);
}
return IJobConfiguration.build(builder);
}
/**
* Check validity of and populates defaults in a task configuration. This will return a deep copy
* of the provided task configuration with default configuration values applied, and configuration
* map values sanitized and applied to their respective struct fields.
*
* @param config Task config to validate and populate.
* @return A reference to the modified {@code config} (for chaining).
* @throws TaskDescriptionException If the task is invalid.
*/
public ITaskConfig validateAndPopulate(ITaskConfig config) throws TaskDescriptionException {
TaskConfig builder = config.newBuilder();
if (!builder.isSetRequestedPorts()) {
builder.setRequestedPorts(ImmutableSet.of());
}
if (config.isSetTier() && !UserProvidedStrings.isGoodIdentifier(config.getTier())) {
throw new TaskDescriptionException("Tier contains illegal characters: " + config.getTier());
}
try {
tierManager.getTier(config);
} catch (IllegalArgumentException e) {
throw new TaskDescriptionException(e.getMessage(), e);
}
if (!JobKeys.isValid(config.getJob())) {
// Job key is set but invalid
throw new TaskDescriptionException("Job key " + config.getJob() + " is invalid.");
}
// A task must either have an executor configuration or specify a Docker container.
if (!builder.isSetExecutorConfig()
&& !(builder.isSetContainer() && builder.getContainer().isSetDocker())) {
throw new TaskDescriptionException(NO_EXECUTOR_OR_CONTAINER);
}
// Docker containers don't require executors, validate the rest
if (builder.isSetExecutorConfig()) {
if (!builder.getExecutorConfig().isSetName()) {
throw new TaskDescriptionException(INVALID_EXECUTOR_CONFIG);
}
executorSettings
.getExecutorConfig(builder.getExecutorConfig().getName())
.orElseThrow(
() ->
new TaskDescriptionException(
"Configuration for executor '"
+ builder.getExecutorConfig().getName()
+ "' doesn't exist."));
}
// Maximize the usefulness of any thrown error message by checking required fields first.
for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) {
validator.validate(builder);
}
IConstraint constraint = getDedicatedConstraint(config);
if (constraint != null) {
if (!isValueConstraint(constraint.getConstraint())) {
throw new TaskDescriptionException("A dedicated constraint must be of value type.");
}
IValueConstraint valueConstraint = constraint.getConstraint().getValue();
if (valueConstraint.getValues().size() != 1) {
throw new TaskDescriptionException("A dedicated constraint must have exactly one value");
}
String dedicatedRole = getRole(valueConstraint);
if (!("*".equals(dedicatedRole) || config.getJob().getRole().equals(dedicatedRole))) {
throw new TaskDescriptionException(
"Only " + dedicatedRole + " may use hosts dedicated for that role.");
}
}
Optional<Container._Fields> containerType;
if (config.isSetContainer()) {
IContainer containerConfig = config.getContainer();
containerType = Optional.of(containerConfig.getSetField());
if (containerConfig.isSetDocker()) {
if (!containerConfig.getDocker().isSetImage()) {
throw new TaskDescriptionException("A container must specify an image.");
}
if (containerConfig.getDocker().getParameters().isEmpty()) {
for (Map.Entry<String, String> e : settings.defaultDockerParameters.entries()) {
builder
.getContainer()
.getDocker()
.addToParameters(new DockerParameter(e.getKey(), e.getValue()));
}
} else {
if (!settings.allowDockerParameters) {
throw new TaskDescriptionException(NO_DOCKER_PARAMETERS);
}
}
if (settings.requireDockerUseExecutor && !config.isSetExecutorConfig()) {
throw new TaskDescriptionException(EXECUTOR_REQUIRED_WITH_DOCKER);
}
}
} else {
// Default to mesos container type if unset.
containerType = Optional.of(Container._Fields.MESOS);
}
if (!containerType.isPresent()) {
throw new TaskDescriptionException("A job must have a container type.");
}
if (!settings.allowedContainerTypes.contains(containerType.get())) {
throw new TaskDescriptionException(
"This scheduler is not configured to allow the container type "
+ containerType.get().toString());
}
thriftBackfill.backfillTask(builder);
String types =
config
.getResources()
.stream()
.collect(Collectors.groupingBy(e -> ResourceType.fromResource(e)))
.entrySet()
.stream()
.filter(e -> !e.getKey().isMultipleAllowed() && e.getValue().size() > 1)
.map(r -> r.getKey().getAuroraName())
.sorted()
.collect(Collectors.joining(", "));
if (!Strings.isNullOrEmpty(types)) {
throw new TaskDescriptionException("Multiple resource values are not supported for " + types);
}
if (!settings.allowGpuResource
&& config
.getResources()
.stream()
.filter(r -> ResourceType.fromResource(r).equals(GPUS))
.findAny()
.isPresent()) {
throw new TaskDescriptionException("GPU resource support is disabled in this cluster.");
}
if (!settings.enableMesosFetcher && !config.getMesosFetcherUris().isEmpty()) {
throw new TaskDescriptionException(MESOS_FETCHER_DISABLED);
}
if (config.getContainer().isSetMesos()) {
IMesosContainer container = config.getContainer().getMesos();
if (!settings.allowContainerVolumes && !container.getVolumes().isEmpty()) {
throw new TaskDescriptionException(NO_CONTAINER_VOLUMES);
}
}
maybeFillLinks(builder);
return ITaskConfig.build(builder);
}