public void validate(TaskConfig task) throws TaskDescriptionException {
if (!task.isSet(field)) {
throw new TaskDescriptionException("Field " + field.getFieldName() + " is required.");
}
@SuppressWarnings("unchecked")
T value = (T) task.getFieldValue(field);
validator.validate(value);
}
private static void maybeFillLinks(TaskConfig task) {
if (task.getTaskLinksSize() == 0) {
ImmutableMap.Builder<String, String> links = ImmutableMap.builder();
if (task.getRequestedPorts().contains("health")) {
links.put("health", "http://%host%:%port:health%");
}
if (task.getRequestedPorts().contains("http")) {
links.put("http", "http://%host%:%port:http%");
}
task.setTaskLinks(links.build());
}
}
private static void maybeFillLinks(TaskConfig task) {
if (task.getTaskLinksSize() == 0) {
ImmutableMap.Builder<String, String> links = ImmutableMap.builder();
for (IResource resource : ResourceManager.getTaskResources(ITaskConfig.build(task), PORTS)) {
if (resource.getNamedPort().equals("health")) {
links.put("health", "http://%host%:%port:health%");
} else if (resource.getNamedPort().equals("http")) {
links.put("http", "http://%host%:%port:http%");
}
}
task.setTaskLinks(links.build());
}
}
@Test
public void testGetRoleSummary() throws Exception {
final String BAZ_ROLE = "baz_role";
final Identity BAZ_ROLE_IDENTITY = new Identity(BAZ_ROLE, USER);
JobConfiguration cronJobOne =
makeJob()
.setCronSchedule("1 * * * *")
.setKey(JOB_KEY.newBuilder())
.setTaskConfig(nonProductionTask());
JobConfiguration cronJobTwo =
makeJob()
.setCronSchedule("2 * * * *")
.setKey(JOB_KEY.newBuilder().setName("cronJob2"))
.setTaskConfig(nonProductionTask());
JobConfiguration cronJobThree =
makeJob()
.setCronSchedule("3 * * * *")
.setKey(JOB_KEY.newBuilder().setRole(BAZ_ROLE))
.setTaskConfig(nonProductionTask())
.setOwner(BAZ_ROLE_IDENTITY);
Set<JobConfiguration> crons = ImmutableSet.of(cronJobOne, cronJobTwo, cronJobThree);
TaskConfig immediateTaskConfig =
defaultTask(false)
.setJob(JOB_KEY.newBuilder().setName("immediate"))
.setJobName("immediate")
.setOwner(ROLE_IDENTITY);
IScheduledTask task1 =
IScheduledTask.build(
new ScheduledTask().setAssignedTask(new AssignedTask().setTask(immediateTaskConfig)));
IScheduledTask task2 =
IScheduledTask.build(
new ScheduledTask()
.setAssignedTask(new AssignedTask().setTask(immediateTaskConfig.setNumCpus(2))));
TaskConfig immediateTaskConfigTwo =
defaultTask(false)
.setJob(
JOB_KEY.newBuilder().setRole(BAZ_ROLE_IDENTITY.getRole()).setName("immediateTwo"))
.setJobName("immediateTwo")
.setOwner(BAZ_ROLE_IDENTITY);
IScheduledTask task3 =
IScheduledTask.build(
new ScheduledTask()
.setAssignedTask(new AssignedTask().setTask(immediateTaskConfigTwo)));
TaskConfig immediateTaskConfigThree =
defaultTask(false)
.setJob(
JOB_KEY.newBuilder().setRole(BAZ_ROLE_IDENTITY.getRole()).setName("immediateThree"))
.setJobName("immediateThree")
.setOwner(BAZ_ROLE_IDENTITY);
IScheduledTask task4 =
IScheduledTask.build(
new ScheduledTask()
.setAssignedTask(new AssignedTask().setTask(immediateTaskConfigThree)));
expect(storageUtil.taskStore.getJobKeys())
.andReturn(
FluentIterable.from(ImmutableSet.of(task1, task2, task3, task4))
.transform(Tasks::getJob)
.toSet());
expect(storageUtil.jobStore.fetchJobs()).andReturn(IJobConfiguration.setFromBuilders(crons));
RoleSummaryResult expectedResult = new RoleSummaryResult();
expectedResult.addToSummaries(
new RoleSummary().setRole(ROLE).setCronJobCount(2).setJobCount(1));
expectedResult.addToSummaries(
new RoleSummary().setRole(BAZ_ROLE).setCronJobCount(1).setJobCount(2));
control.replay();
Response response = assertOkResponse(thrift.getRoleSummary());
assertEquals(expectedResult, response.getResult().getRoleSummaryResult());
}
@Test
public void testGetJobs() throws Exception {
TaskConfig ownedCronJobTask =
nonProductionTask()
.setJobName(JOB_KEY.getName())
.setOwner(ROLE_IDENTITY)
.setEnvironment(JOB_KEY.getEnvironment());
JobConfiguration ownedCronJob =
makeJob().setCronSchedule(CRON_SCHEDULE).setTaskConfig(ownedCronJobTask);
IScheduledTask ownedCronJobScheduledTask =
IScheduledTask.build(
new ScheduledTask()
.setAssignedTask(new AssignedTask().setTask(ownedCronJobTask))
.setStatus(ScheduleStatus.ASSIGNED));
Identity otherOwner = new Identity("other", "other");
JobConfiguration unownedCronJob =
makeJob()
.setOwner(otherOwner)
.setCronSchedule(CRON_SCHEDULE)
.setKey(JOB_KEY.newBuilder().setRole("other"))
.setTaskConfig(ownedCronJobTask.deepCopy().setOwner(otherOwner));
TaskConfig ownedImmediateTaskInfo =
defaultTask(false)
.setJob(JOB_KEY.newBuilder().setName("immediate"))
.setJobName("immediate")
.setOwner(ROLE_IDENTITY);
Set<JobConfiguration> ownedCronJobOnly = ImmutableSet.of(ownedCronJob);
Set<JobConfiguration> unownedCronJobOnly = ImmutableSet.of(unownedCronJob);
Set<JobConfiguration> bothCronJobs = ImmutableSet.of(ownedCronJob, unownedCronJob);
IScheduledTask ownedImmediateTask =
IScheduledTask.build(
new ScheduledTask()
.setAssignedTask(new AssignedTask().setTask(ownedImmediateTaskInfo))
.setStatus(ScheduleStatus.ASSIGNED));
JobConfiguration ownedImmediateJob =
new JobConfiguration()
.setKey(JOB_KEY.newBuilder().setName("immediate"))
.setOwner(ROLE_IDENTITY)
.setInstanceCount(1)
.setTaskConfig(ownedImmediateTaskInfo);
Query.Builder query = Query.roleScoped(ROLE).active();
storageUtil.expectTaskFetch(query);
expect(storageUtil.jobStore.fetchJobs())
.andReturn(IJobConfiguration.setFromBuilders(ownedCronJobOnly));
storageUtil.expectTaskFetch(query);
expect(storageUtil.jobStore.fetchJobs())
.andReturn(IJobConfiguration.setFromBuilders(bothCronJobs));
storageUtil.expectTaskFetch(query, ownedImmediateTask);
expect(storageUtil.jobStore.fetchJobs())
.andReturn(IJobConfiguration.setFromBuilders(unownedCronJobOnly));
expect(storageUtil.jobStore.fetchJobs()).andReturn(ImmutableSet.of());
storageUtil.expectTaskFetch(query);
// Handle the case where a cron job has a running task (same JobKey present in both stores).
storageUtil.expectTaskFetch(query, ownedCronJobScheduledTask);
expect(storageUtil.jobStore.fetchJobs())
.andReturn(IJobConfiguration.setFromBuilders(ImmutableSet.of(ownedCronJob)));
control.replay();
assertJobsEqual(
ownedCronJob,
Iterables.getOnlyElement(thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs()));
assertJobsEqual(
ownedCronJob,
Iterables.getOnlyElement(thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs()));
Set<JobConfiguration> queryResult3 =
thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs();
assertJobsEqual(ownedImmediateJob, Iterables.getOnlyElement(queryResult3));
assertEquals(
ITaskConfig.build(ownedImmediateTaskInfo),
ITaskConfig.build(Iterables.getOnlyElement(queryResult3).getTaskConfig()));
assertTrue(thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs().isEmpty());
assertJobsEqual(
ownedCronJob,
Iterables.getOnlyElement(thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs()));
}
@Test
public void testGetJobSummary() throws Exception {
long nextCronRunMs = 100;
TaskConfig ownedCronJobTask =
nonProductionTask()
.setJob(JOB_KEY.newBuilder())
.setJobName(JOB_KEY.getName())
.setOwner(ROLE_IDENTITY)
.setEnvironment(JOB_KEY.getEnvironment());
JobConfiguration ownedCronJob =
makeJob().setCronSchedule(CRON_SCHEDULE).setTaskConfig(ownedCronJobTask);
IScheduledTask ownedCronJobScheduledTask =
IScheduledTask.build(
new ScheduledTask()
.setAssignedTask(new AssignedTask().setTask(ownedCronJobTask))
.setStatus(ScheduleStatus.ASSIGNED));
Identity otherOwner = new Identity("other", "other");
JobConfiguration unownedCronJob =
makeJob()
.setOwner(otherOwner)
.setCronSchedule(CRON_SCHEDULE)
.setKey(JOB_KEY.newBuilder().setRole("other"))
.setTaskConfig(ownedCronJobTask.deepCopy().setOwner(otherOwner));
TaskConfig ownedImmediateTaskInfo =
defaultTask(false)
.setJob(JOB_KEY.newBuilder().setName("immediate"))
.setJobName("immediate")
.setOwner(ROLE_IDENTITY);
Set<JobConfiguration> ownedCronJobOnly = ImmutableSet.of(ownedCronJob);
Set<JobSummary> ownedCronJobSummaryOnly =
ImmutableSet.of(
new JobSummary()
.setJob(ownedCronJob)
.setStats(new JobStats())
.setNextCronRunMs(nextCronRunMs));
Set<JobSummary> ownedCronJobSummaryWithRunningTask =
ImmutableSet.of(
new JobSummary()
.setJob(ownedCronJob)
.setStats(new JobStats().setActiveTaskCount(1))
.setNextCronRunMs(nextCronRunMs));
Set<JobConfiguration> unownedCronJobOnly = ImmutableSet.of(unownedCronJob);
Set<JobConfiguration> bothCronJobs = ImmutableSet.of(ownedCronJob, unownedCronJob);
IScheduledTask ownedImmediateTask =
IScheduledTask.build(
new ScheduledTask()
.setAssignedTask(new AssignedTask().setTask(ownedImmediateTaskInfo))
.setStatus(ScheduleStatus.ASSIGNED));
JobConfiguration ownedImmediateJob =
new JobConfiguration()
.setKey(JOB_KEY.newBuilder().setName("immediate"))
.setOwner(ROLE_IDENTITY)
.setInstanceCount(1)
.setTaskConfig(ownedImmediateTaskInfo);
Builder query = Query.roleScoped(ROLE);
Set<JobSummary> ownedImmediateJobSummaryOnly =
ImmutableSet.of(
new JobSummary()
.setJob(ownedImmediateJob)
.setStats(new JobStats().setActiveTaskCount(1)));
expect(cronPredictor.predictNextRun(CrontabEntry.parse(CRON_SCHEDULE)))
.andReturn(Optional.of(new Date(nextCronRunMs)))
.anyTimes();
storageUtil.expectTaskFetch(query);
expect(storageUtil.jobStore.fetchJobs())
.andReturn(IJobConfiguration.setFromBuilders(ownedCronJobOnly));
storageUtil.expectTaskFetch(query);
expect(storageUtil.jobStore.fetchJobs())
.andReturn(IJobConfiguration.setFromBuilders(bothCronJobs));
storageUtil.expectTaskFetch(query, ownedImmediateTask);
expect(storageUtil.jobStore.fetchJobs())
.andReturn(IJobConfiguration.setFromBuilders(unownedCronJobOnly));
storageUtil.expectTaskFetch(query);
expect(storageUtil.jobStore.fetchJobs()).andReturn(ImmutableSet.of());
// Handle the case where a cron job has a running task (same JobKey present in both stores).
storageUtil.expectTaskFetch(query, ownedCronJobScheduledTask);
expect(storageUtil.jobStore.fetchJobs())
.andReturn(IJobConfiguration.setFromBuilders(ImmutableSet.of(ownedCronJob)));
control.replay();
assertEquals(jobSummaryResponse(ownedCronJobSummaryOnly), thrift.getJobSummary(ROLE));
assertEquals(jobSummaryResponse(ownedCronJobSummaryOnly), thrift.getJobSummary(ROLE));
Response jobSummaryResponse = thrift.getJobSummary(ROLE);
assertEquals(
jobSummaryResponse(ownedImmediateJobSummaryOnly),
IResponse.build(jobSummaryResponse).newBuilder());
assertEquals(jobSummaryResponse(ImmutableSet.of()), thrift.getJobSummary(ROLE));
assertEquals(
jobSummaryResponse(ownedCronJobSummaryWithRunningTask), thrift.getJobSummary(ROLE));
}
/**
* Check validity of and populates defaults in a task configuration. This will return a deep copy
* of the provided task configuration with default configuration values applied, and configuration
* map values sanitized and applied to their respective struct fields.
*
* @param config Task config to validate and populate.
* @return A reference to the modified {@code config} (for chaining).
* @throws TaskDescriptionException If the task is invalid.
*/
public ITaskConfig validateAndPopulate(ITaskConfig config) throws TaskDescriptionException {
TaskConfig builder = config.newBuilder();
if (!builder.isSetRequestedPorts()) {
builder.setRequestedPorts(ImmutableSet.of());
}
if (config.isSetTier() && !UserProvidedStrings.isGoodIdentifier(config.getTier())) {
throw new TaskDescriptionException("Tier contains illegal characters: " + config.getTier());
}
try {
tierManager.getTier(config);
} catch (IllegalArgumentException e) {
throw new TaskDescriptionException(e.getMessage(), e);
}
if (!JobKeys.isValid(config.getJob())) {
// Job key is set but invalid
throw new TaskDescriptionException("Job key " + config.getJob() + " is invalid.");
}
// A task must either have an executor configuration or specify a Docker container.
if (!builder.isSetExecutorConfig()
&& !(builder.isSetContainer() && builder.getContainer().isSetDocker())) {
throw new TaskDescriptionException(NO_EXECUTOR_OR_CONTAINER);
}
// Docker containers don't require executors, validate the rest
if (builder.isSetExecutorConfig()) {
if (!builder.getExecutorConfig().isSetName()) {
throw new TaskDescriptionException(INVALID_EXECUTOR_CONFIG);
}
executorSettings
.getExecutorConfig(builder.getExecutorConfig().getName())
.orElseThrow(
() ->
new TaskDescriptionException(
"Configuration for executor '"
+ builder.getExecutorConfig().getName()
+ "' doesn't exist."));
}
// Maximize the usefulness of any thrown error message by checking required fields first.
for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) {
validator.validate(builder);
}
IConstraint constraint = getDedicatedConstraint(config);
if (constraint != null) {
if (!isValueConstraint(constraint.getConstraint())) {
throw new TaskDescriptionException("A dedicated constraint must be of value type.");
}
IValueConstraint valueConstraint = constraint.getConstraint().getValue();
if (valueConstraint.getValues().size() != 1) {
throw new TaskDescriptionException("A dedicated constraint must have exactly one value");
}
String dedicatedRole = getRole(valueConstraint);
if (!("*".equals(dedicatedRole) || config.getJob().getRole().equals(dedicatedRole))) {
throw new TaskDescriptionException(
"Only " + dedicatedRole + " may use hosts dedicated for that role.");
}
}
Optional<Container._Fields> containerType;
if (config.isSetContainer()) {
IContainer containerConfig = config.getContainer();
containerType = Optional.of(containerConfig.getSetField());
if (containerConfig.isSetDocker()) {
if (!containerConfig.getDocker().isSetImage()) {
throw new TaskDescriptionException("A container must specify an image.");
}
if (containerConfig.getDocker().getParameters().isEmpty()) {
for (Map.Entry<String, String> e : settings.defaultDockerParameters.entries()) {
builder
.getContainer()
.getDocker()
.addToParameters(new DockerParameter(e.getKey(), e.getValue()));
}
} else {
if (!settings.allowDockerParameters) {
throw new TaskDescriptionException(NO_DOCKER_PARAMETERS);
}
}
if (settings.requireDockerUseExecutor && !config.isSetExecutorConfig()) {
throw new TaskDescriptionException(EXECUTOR_REQUIRED_WITH_DOCKER);
}
}
} else {
// Default to mesos container type if unset.
containerType = Optional.of(Container._Fields.MESOS);
}
if (!containerType.isPresent()) {
throw new TaskDescriptionException("A job must have a container type.");
}
if (!settings.allowedContainerTypes.contains(containerType.get())) {
throw new TaskDescriptionException(
"This scheduler is not configured to allow the container type "
+ containerType.get().toString());
}
thriftBackfill.backfillTask(builder);
String types =
config
.getResources()
.stream()
.collect(Collectors.groupingBy(e -> ResourceType.fromResource(e)))
.entrySet()
.stream()
.filter(e -> !e.getKey().isMultipleAllowed() && e.getValue().size() > 1)
.map(r -> r.getKey().getAuroraName())
.sorted()
.collect(Collectors.joining(", "));
if (!Strings.isNullOrEmpty(types)) {
throw new TaskDescriptionException("Multiple resource values are not supported for " + types);
}
if (!settings.allowGpuResource
&& config
.getResources()
.stream()
.filter(r -> ResourceType.fromResource(r).equals(GPUS))
.findAny()
.isPresent()) {
throw new TaskDescriptionException("GPU resource support is disabled in this cluster.");
}
if (!settings.enableMesosFetcher && !config.getMesosFetcherUris().isEmpty()) {
throw new TaskDescriptionException(MESOS_FETCHER_DISABLED);
}
if (config.getContainer().isSetMesos()) {
IMesosContainer container = config.getContainer().getMesos();
if (!settings.allowContainerVolumes && !container.getVolumes().isEmpty()) {
throw new TaskDescriptionException(NO_CONTAINER_VOLUMES);
}
}
maybeFillLinks(builder);
return ITaskConfig.build(builder);
}
/**
* Check validity of and populates defaults in a task configuration. This will return a deep copy
* of the provided task configuration with default configuration values applied, and configuration
* map values sanitized and applied to their respective struct fields.
*
* @param config Task config to validate and populate.
* @return A reference to the modified {@code config} (for chaining).
* @throws TaskDescriptionException If the task is invalid.
*/
public static ITaskConfig validateAndPopulate(ITaskConfig config)
throws TaskDescriptionException {
TaskConfig builder = config.newBuilder();
if (!builder.isSetRequestedPorts()) {
builder.setRequestedPorts(ImmutableSet.of());
}
maybeFillLinks(builder);
if (!isGoodIdentifier(config.getJobName())) {
throw new TaskDescriptionException(
"Job name contains illegal characters: " + config.getJobName());
}
if (!isGoodIdentifier(config.getEnvironment())) {
throw new TaskDescriptionException(
"Environment contains illegal characters: " + config.getEnvironment());
}
if (config.isSetTier() && !isGoodIdentifier(config.getTier())) {
throw new TaskDescriptionException("Tier contains illegal characters: " + config.getTier());
}
if (config.isSetJob()) {
if (!JobKeys.isValid(config.getJob())) {
// Job key is set but invalid
throw new TaskDescriptionException("Job key " + config.getJob() + " is invalid.");
}
if (!config.getJob().getRole().equals(config.getOwner().getRole())) {
// Both owner and job key are set but don't match
throw new TaskDescriptionException("Role must match job owner.");
}
} else {
// TODO(maxim): Make sure both key and owner are populated to support older clients.
// Remove in 0.7.0. (AURORA-749).
// Job key is not set -> populate from owner, environment and name
assertOwnerValidity(config.getOwner());
builder.setJob(
JobKeys.from(config.getOwner().getRole(), config.getEnvironment(), config.getJobName())
.newBuilder());
}
if (!builder.isSetExecutorConfig()) {
throw new TaskDescriptionException("Configuration may not be null");
}
// Maximize the usefulness of any thrown error message by checking required fields first.
for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) {
validator.validate(builder);
}
IConstraint constraint = getDedicatedConstraint(config);
if (constraint != null) {
if (!isValueConstraint(constraint.getConstraint())) {
throw new TaskDescriptionException("A dedicated constraint must be of value type.");
}
IValueConstraint valueConstraint = constraint.getConstraint().getValue();
if (valueConstraint.getValues().size() != 1) {
throw new TaskDescriptionException("A dedicated constraint must have exactly one value");
}
String dedicatedRole = getRole(valueConstraint);
if (!config.getOwner().getRole().equals(dedicatedRole)) {
throw new TaskDescriptionException(
"Only " + dedicatedRole + " may use hosts dedicated for that role.");
}
}
Optional<Container._Fields> containerType;
if (config.isSetContainer()) {
IContainer containerConfig = config.getContainer();
containerType = Optional.of(containerConfig.getSetField());
if (containerConfig.isSetDocker()) {
if (!containerConfig.getDocker().isSetImage()) {
throw new TaskDescriptionException("A container must specify an image");
}
if (containerConfig.getDocker().isSetParameters()
&& !containerConfig.getDocker().getParameters().isEmpty()
&& !ENABLE_DOCKER_PARAMETERS.get()) {
throw new TaskDescriptionException("Docker parameters not allowed.");
}
}
} else {
// Default to mesos container type if unset.
containerType = Optional.of(Container._Fields.MESOS);
}
if (!containerType.isPresent()) {
throw new TaskDescriptionException("A job must have a container type.");
}
if (!ALLOWED_CONTAINER_TYPES.get().contains(containerType.get())) {
throw new TaskDescriptionException(
"The container type " + containerType.get().toString() + " is not allowed");
}
return ITaskConfig.build(builder);
}