Ejemplo n.º 1
0
 public void validate(TaskConfig task) throws TaskDescriptionException {
   if (!task.isSet(field)) {
     throw new TaskDescriptionException("Field " + field.getFieldName() + " is required.");
   }
   @SuppressWarnings("unchecked")
   T value = (T) task.getFieldValue(field);
   validator.validate(value);
 }
Ejemplo n.º 2
0
 private static void maybeFillLinks(TaskConfig task) {
   if (task.getTaskLinksSize() == 0) {
     ImmutableMap.Builder<String, String> links = ImmutableMap.builder();
     if (task.getRequestedPorts().contains("health")) {
       links.put("health", "http://%host%:%port:health%");
     }
     if (task.getRequestedPorts().contains("http")) {
       links.put("http", "http://%host%:%port:http%");
     }
     task.setTaskLinks(links.build());
   }
 }
Ejemplo n.º 3
0
 private static void maybeFillLinks(TaskConfig task) {
   if (task.getTaskLinksSize() == 0) {
     ImmutableMap.Builder<String, String> links = ImmutableMap.builder();
     for (IResource resource : ResourceManager.getTaskResources(ITaskConfig.build(task), PORTS)) {
       if (resource.getNamedPort().equals("health")) {
         links.put("health", "http://%host%:%port:health%");
       } else if (resource.getNamedPort().equals("http")) {
         links.put("http", "http://%host%:%port:http%");
       }
     }
     task.setTaskLinks(links.build());
   }
 }
  @Test
  public void testGetRoleSummary() throws Exception {
    final String BAZ_ROLE = "baz_role";
    final Identity BAZ_ROLE_IDENTITY = new Identity(BAZ_ROLE, USER);

    JobConfiguration cronJobOne =
        makeJob()
            .setCronSchedule("1 * * * *")
            .setKey(JOB_KEY.newBuilder())
            .setTaskConfig(nonProductionTask());
    JobConfiguration cronJobTwo =
        makeJob()
            .setCronSchedule("2 * * * *")
            .setKey(JOB_KEY.newBuilder().setName("cronJob2"))
            .setTaskConfig(nonProductionTask());

    JobConfiguration cronJobThree =
        makeJob()
            .setCronSchedule("3 * * * *")
            .setKey(JOB_KEY.newBuilder().setRole(BAZ_ROLE))
            .setTaskConfig(nonProductionTask())
            .setOwner(BAZ_ROLE_IDENTITY);

    Set<JobConfiguration> crons = ImmutableSet.of(cronJobOne, cronJobTwo, cronJobThree);

    TaskConfig immediateTaskConfig =
        defaultTask(false)
            .setJob(JOB_KEY.newBuilder().setName("immediate"))
            .setJobName("immediate")
            .setOwner(ROLE_IDENTITY);
    IScheduledTask task1 =
        IScheduledTask.build(
            new ScheduledTask().setAssignedTask(new AssignedTask().setTask(immediateTaskConfig)));
    IScheduledTask task2 =
        IScheduledTask.build(
            new ScheduledTask()
                .setAssignedTask(new AssignedTask().setTask(immediateTaskConfig.setNumCpus(2))));

    TaskConfig immediateTaskConfigTwo =
        defaultTask(false)
            .setJob(
                JOB_KEY.newBuilder().setRole(BAZ_ROLE_IDENTITY.getRole()).setName("immediateTwo"))
            .setJobName("immediateTwo")
            .setOwner(BAZ_ROLE_IDENTITY);
    IScheduledTask task3 =
        IScheduledTask.build(
            new ScheduledTask()
                .setAssignedTask(new AssignedTask().setTask(immediateTaskConfigTwo)));

    TaskConfig immediateTaskConfigThree =
        defaultTask(false)
            .setJob(
                JOB_KEY.newBuilder().setRole(BAZ_ROLE_IDENTITY.getRole()).setName("immediateThree"))
            .setJobName("immediateThree")
            .setOwner(BAZ_ROLE_IDENTITY);
    IScheduledTask task4 =
        IScheduledTask.build(
            new ScheduledTask()
                .setAssignedTask(new AssignedTask().setTask(immediateTaskConfigThree)));

    expect(storageUtil.taskStore.getJobKeys())
        .andReturn(
            FluentIterable.from(ImmutableSet.of(task1, task2, task3, task4))
                .transform(Tasks::getJob)
                .toSet());
    expect(storageUtil.jobStore.fetchJobs()).andReturn(IJobConfiguration.setFromBuilders(crons));

    RoleSummaryResult expectedResult = new RoleSummaryResult();
    expectedResult.addToSummaries(
        new RoleSummary().setRole(ROLE).setCronJobCount(2).setJobCount(1));
    expectedResult.addToSummaries(
        new RoleSummary().setRole(BAZ_ROLE).setCronJobCount(1).setJobCount(2));

    control.replay();

    Response response = assertOkResponse(thrift.getRoleSummary());
    assertEquals(expectedResult, response.getResult().getRoleSummaryResult());
  }
  @Test
  public void testGetJobs() throws Exception {
    TaskConfig ownedCronJobTask =
        nonProductionTask()
            .setJobName(JOB_KEY.getName())
            .setOwner(ROLE_IDENTITY)
            .setEnvironment(JOB_KEY.getEnvironment());
    JobConfiguration ownedCronJob =
        makeJob().setCronSchedule(CRON_SCHEDULE).setTaskConfig(ownedCronJobTask);
    IScheduledTask ownedCronJobScheduledTask =
        IScheduledTask.build(
            new ScheduledTask()
                .setAssignedTask(new AssignedTask().setTask(ownedCronJobTask))
                .setStatus(ScheduleStatus.ASSIGNED));
    Identity otherOwner = new Identity("other", "other");
    JobConfiguration unownedCronJob =
        makeJob()
            .setOwner(otherOwner)
            .setCronSchedule(CRON_SCHEDULE)
            .setKey(JOB_KEY.newBuilder().setRole("other"))
            .setTaskConfig(ownedCronJobTask.deepCopy().setOwner(otherOwner));
    TaskConfig ownedImmediateTaskInfo =
        defaultTask(false)
            .setJob(JOB_KEY.newBuilder().setName("immediate"))
            .setJobName("immediate")
            .setOwner(ROLE_IDENTITY);
    Set<JobConfiguration> ownedCronJobOnly = ImmutableSet.of(ownedCronJob);
    Set<JobConfiguration> unownedCronJobOnly = ImmutableSet.of(unownedCronJob);
    Set<JobConfiguration> bothCronJobs = ImmutableSet.of(ownedCronJob, unownedCronJob);
    IScheduledTask ownedImmediateTask =
        IScheduledTask.build(
            new ScheduledTask()
                .setAssignedTask(new AssignedTask().setTask(ownedImmediateTaskInfo))
                .setStatus(ScheduleStatus.ASSIGNED));
    JobConfiguration ownedImmediateJob =
        new JobConfiguration()
            .setKey(JOB_KEY.newBuilder().setName("immediate"))
            .setOwner(ROLE_IDENTITY)
            .setInstanceCount(1)
            .setTaskConfig(ownedImmediateTaskInfo);
    Query.Builder query = Query.roleScoped(ROLE).active();

    storageUtil.expectTaskFetch(query);
    expect(storageUtil.jobStore.fetchJobs())
        .andReturn(IJobConfiguration.setFromBuilders(ownedCronJobOnly));

    storageUtil.expectTaskFetch(query);
    expect(storageUtil.jobStore.fetchJobs())
        .andReturn(IJobConfiguration.setFromBuilders(bothCronJobs));

    storageUtil.expectTaskFetch(query, ownedImmediateTask);
    expect(storageUtil.jobStore.fetchJobs())
        .andReturn(IJobConfiguration.setFromBuilders(unownedCronJobOnly));

    expect(storageUtil.jobStore.fetchJobs()).andReturn(ImmutableSet.of());
    storageUtil.expectTaskFetch(query);

    // Handle the case where a cron job has a running task (same JobKey present in both stores).
    storageUtil.expectTaskFetch(query, ownedCronJobScheduledTask);
    expect(storageUtil.jobStore.fetchJobs())
        .andReturn(IJobConfiguration.setFromBuilders(ImmutableSet.of(ownedCronJob)));

    control.replay();

    assertJobsEqual(
        ownedCronJob,
        Iterables.getOnlyElement(thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs()));

    assertJobsEqual(
        ownedCronJob,
        Iterables.getOnlyElement(thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs()));

    Set<JobConfiguration> queryResult3 =
        thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs();
    assertJobsEqual(ownedImmediateJob, Iterables.getOnlyElement(queryResult3));
    assertEquals(
        ITaskConfig.build(ownedImmediateTaskInfo),
        ITaskConfig.build(Iterables.getOnlyElement(queryResult3).getTaskConfig()));

    assertTrue(thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs().isEmpty());

    assertJobsEqual(
        ownedCronJob,
        Iterables.getOnlyElement(thrift.getJobs(ROLE).getResult().getGetJobsResult().getConfigs()));
  }
  @Test
  public void testGetJobSummary() throws Exception {
    long nextCronRunMs = 100;
    TaskConfig ownedCronJobTask =
        nonProductionTask()
            .setJob(JOB_KEY.newBuilder())
            .setJobName(JOB_KEY.getName())
            .setOwner(ROLE_IDENTITY)
            .setEnvironment(JOB_KEY.getEnvironment());
    JobConfiguration ownedCronJob =
        makeJob().setCronSchedule(CRON_SCHEDULE).setTaskConfig(ownedCronJobTask);
    IScheduledTask ownedCronJobScheduledTask =
        IScheduledTask.build(
            new ScheduledTask()
                .setAssignedTask(new AssignedTask().setTask(ownedCronJobTask))
                .setStatus(ScheduleStatus.ASSIGNED));
    Identity otherOwner = new Identity("other", "other");
    JobConfiguration unownedCronJob =
        makeJob()
            .setOwner(otherOwner)
            .setCronSchedule(CRON_SCHEDULE)
            .setKey(JOB_KEY.newBuilder().setRole("other"))
            .setTaskConfig(ownedCronJobTask.deepCopy().setOwner(otherOwner));
    TaskConfig ownedImmediateTaskInfo =
        defaultTask(false)
            .setJob(JOB_KEY.newBuilder().setName("immediate"))
            .setJobName("immediate")
            .setOwner(ROLE_IDENTITY);
    Set<JobConfiguration> ownedCronJobOnly = ImmutableSet.of(ownedCronJob);
    Set<JobSummary> ownedCronJobSummaryOnly =
        ImmutableSet.of(
            new JobSummary()
                .setJob(ownedCronJob)
                .setStats(new JobStats())
                .setNextCronRunMs(nextCronRunMs));
    Set<JobSummary> ownedCronJobSummaryWithRunningTask =
        ImmutableSet.of(
            new JobSummary()
                .setJob(ownedCronJob)
                .setStats(new JobStats().setActiveTaskCount(1))
                .setNextCronRunMs(nextCronRunMs));
    Set<JobConfiguration> unownedCronJobOnly = ImmutableSet.of(unownedCronJob);
    Set<JobConfiguration> bothCronJobs = ImmutableSet.of(ownedCronJob, unownedCronJob);

    IScheduledTask ownedImmediateTask =
        IScheduledTask.build(
            new ScheduledTask()
                .setAssignedTask(new AssignedTask().setTask(ownedImmediateTaskInfo))
                .setStatus(ScheduleStatus.ASSIGNED));
    JobConfiguration ownedImmediateJob =
        new JobConfiguration()
            .setKey(JOB_KEY.newBuilder().setName("immediate"))
            .setOwner(ROLE_IDENTITY)
            .setInstanceCount(1)
            .setTaskConfig(ownedImmediateTaskInfo);
    Builder query = Query.roleScoped(ROLE);

    Set<JobSummary> ownedImmediateJobSummaryOnly =
        ImmutableSet.of(
            new JobSummary()
                .setJob(ownedImmediateJob)
                .setStats(new JobStats().setActiveTaskCount(1)));

    expect(cronPredictor.predictNextRun(CrontabEntry.parse(CRON_SCHEDULE)))
        .andReturn(Optional.of(new Date(nextCronRunMs)))
        .anyTimes();

    storageUtil.expectTaskFetch(query);
    expect(storageUtil.jobStore.fetchJobs())
        .andReturn(IJobConfiguration.setFromBuilders(ownedCronJobOnly));

    storageUtil.expectTaskFetch(query);
    expect(storageUtil.jobStore.fetchJobs())
        .andReturn(IJobConfiguration.setFromBuilders(bothCronJobs));

    storageUtil.expectTaskFetch(query, ownedImmediateTask);
    expect(storageUtil.jobStore.fetchJobs())
        .andReturn(IJobConfiguration.setFromBuilders(unownedCronJobOnly));

    storageUtil.expectTaskFetch(query);
    expect(storageUtil.jobStore.fetchJobs()).andReturn(ImmutableSet.of());

    // Handle the case where a cron job has a running task (same JobKey present in both stores).
    storageUtil.expectTaskFetch(query, ownedCronJobScheduledTask);
    expect(storageUtil.jobStore.fetchJobs())
        .andReturn(IJobConfiguration.setFromBuilders(ImmutableSet.of(ownedCronJob)));

    control.replay();

    assertEquals(jobSummaryResponse(ownedCronJobSummaryOnly), thrift.getJobSummary(ROLE));

    assertEquals(jobSummaryResponse(ownedCronJobSummaryOnly), thrift.getJobSummary(ROLE));

    Response jobSummaryResponse = thrift.getJobSummary(ROLE);
    assertEquals(
        jobSummaryResponse(ownedImmediateJobSummaryOnly),
        IResponse.build(jobSummaryResponse).newBuilder());

    assertEquals(jobSummaryResponse(ImmutableSet.of()), thrift.getJobSummary(ROLE));

    assertEquals(
        jobSummaryResponse(ownedCronJobSummaryWithRunningTask), thrift.getJobSummary(ROLE));
  }
Ejemplo n.º 7
0
  /**
   * Check validity of and populates defaults in a task configuration. This will return a deep copy
   * of the provided task configuration with default configuration values applied, and configuration
   * map values sanitized and applied to their respective struct fields.
   *
   * @param config Task config to validate and populate.
   * @return A reference to the modified {@code config} (for chaining).
   * @throws TaskDescriptionException If the task is invalid.
   */
  public ITaskConfig validateAndPopulate(ITaskConfig config) throws TaskDescriptionException {
    TaskConfig builder = config.newBuilder();

    if (!builder.isSetRequestedPorts()) {
      builder.setRequestedPorts(ImmutableSet.of());
    }

    if (config.isSetTier() && !UserProvidedStrings.isGoodIdentifier(config.getTier())) {
      throw new TaskDescriptionException("Tier contains illegal characters: " + config.getTier());
    }

    try {
      tierManager.getTier(config);
    } catch (IllegalArgumentException e) {
      throw new TaskDescriptionException(e.getMessage(), e);
    }

    if (!JobKeys.isValid(config.getJob())) {
      // Job key is set but invalid
      throw new TaskDescriptionException("Job key " + config.getJob() + " is invalid.");
    }

    // A task must either have an executor configuration or specify a Docker container.
    if (!builder.isSetExecutorConfig()
        && !(builder.isSetContainer() && builder.getContainer().isSetDocker())) {

      throw new TaskDescriptionException(NO_EXECUTOR_OR_CONTAINER);
    }

    // Docker containers don't require executors, validate the rest
    if (builder.isSetExecutorConfig()) {

      if (!builder.getExecutorConfig().isSetName()) {
        throw new TaskDescriptionException(INVALID_EXECUTOR_CONFIG);
      }

      executorSettings
          .getExecutorConfig(builder.getExecutorConfig().getName())
          .orElseThrow(
              () ->
                  new TaskDescriptionException(
                      "Configuration for executor '"
                          + builder.getExecutorConfig().getName()
                          + "' doesn't exist."));
    }

    // Maximize the usefulness of any thrown error message by checking required fields first.
    for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) {
      validator.validate(builder);
    }

    IConstraint constraint = getDedicatedConstraint(config);
    if (constraint != null) {
      if (!isValueConstraint(constraint.getConstraint())) {
        throw new TaskDescriptionException("A dedicated constraint must be of value type.");
      }

      IValueConstraint valueConstraint = constraint.getConstraint().getValue();

      if (valueConstraint.getValues().size() != 1) {
        throw new TaskDescriptionException("A dedicated constraint must have exactly one value");
      }

      String dedicatedRole = getRole(valueConstraint);
      if (!("*".equals(dedicatedRole) || config.getJob().getRole().equals(dedicatedRole))) {
        throw new TaskDescriptionException(
            "Only " + dedicatedRole + " may use hosts dedicated for that role.");
      }
    }

    Optional<Container._Fields> containerType;
    if (config.isSetContainer()) {
      IContainer containerConfig = config.getContainer();
      containerType = Optional.of(containerConfig.getSetField());
      if (containerConfig.isSetDocker()) {
        if (!containerConfig.getDocker().isSetImage()) {
          throw new TaskDescriptionException("A container must specify an image.");
        }
        if (containerConfig.getDocker().getParameters().isEmpty()) {
          for (Map.Entry<String, String> e : settings.defaultDockerParameters.entries()) {
            builder
                .getContainer()
                .getDocker()
                .addToParameters(new DockerParameter(e.getKey(), e.getValue()));
          }
        } else {
          if (!settings.allowDockerParameters) {
            throw new TaskDescriptionException(NO_DOCKER_PARAMETERS);
          }
        }

        if (settings.requireDockerUseExecutor && !config.isSetExecutorConfig()) {
          throw new TaskDescriptionException(EXECUTOR_REQUIRED_WITH_DOCKER);
        }
      }
    } else {
      // Default to mesos container type if unset.
      containerType = Optional.of(Container._Fields.MESOS);
    }

    if (!containerType.isPresent()) {
      throw new TaskDescriptionException("A job must have a container type.");
    }
    if (!settings.allowedContainerTypes.contains(containerType.get())) {
      throw new TaskDescriptionException(
          "This scheduler is not configured to allow the container type "
              + containerType.get().toString());
    }

    thriftBackfill.backfillTask(builder);

    String types =
        config
            .getResources()
            .stream()
            .collect(Collectors.groupingBy(e -> ResourceType.fromResource(e)))
            .entrySet()
            .stream()
            .filter(e -> !e.getKey().isMultipleAllowed() && e.getValue().size() > 1)
            .map(r -> r.getKey().getAuroraName())
            .sorted()
            .collect(Collectors.joining(", "));

    if (!Strings.isNullOrEmpty(types)) {
      throw new TaskDescriptionException("Multiple resource values are not supported for " + types);
    }

    if (!settings.allowGpuResource
        && config
            .getResources()
            .stream()
            .filter(r -> ResourceType.fromResource(r).equals(GPUS))
            .findAny()
            .isPresent()) {

      throw new TaskDescriptionException("GPU resource support is disabled in this cluster.");
    }

    if (!settings.enableMesosFetcher && !config.getMesosFetcherUris().isEmpty()) {
      throw new TaskDescriptionException(MESOS_FETCHER_DISABLED);
    }

    if (config.getContainer().isSetMesos()) {
      IMesosContainer container = config.getContainer().getMesos();
      if (!settings.allowContainerVolumes && !container.getVolumes().isEmpty()) {
        throw new TaskDescriptionException(NO_CONTAINER_VOLUMES);
      }
    }

    maybeFillLinks(builder);

    return ITaskConfig.build(builder);
  }
Ejemplo n.º 8
0
  /**
   * Check validity of and populates defaults in a task configuration. This will return a deep copy
   * of the provided task configuration with default configuration values applied, and configuration
   * map values sanitized and applied to their respective struct fields.
   *
   * @param config Task config to validate and populate.
   * @return A reference to the modified {@code config} (for chaining).
   * @throws TaskDescriptionException If the task is invalid.
   */
  public static ITaskConfig validateAndPopulate(ITaskConfig config)
      throws TaskDescriptionException {

    TaskConfig builder = config.newBuilder();

    if (!builder.isSetRequestedPorts()) {
      builder.setRequestedPorts(ImmutableSet.of());
    }

    maybeFillLinks(builder);

    if (!isGoodIdentifier(config.getJobName())) {
      throw new TaskDescriptionException(
          "Job name contains illegal characters: " + config.getJobName());
    }

    if (!isGoodIdentifier(config.getEnvironment())) {
      throw new TaskDescriptionException(
          "Environment contains illegal characters: " + config.getEnvironment());
    }

    if (config.isSetTier() && !isGoodIdentifier(config.getTier())) {
      throw new TaskDescriptionException("Tier contains illegal characters: " + config.getTier());
    }

    if (config.isSetJob()) {
      if (!JobKeys.isValid(config.getJob())) {
        // Job key is set but invalid
        throw new TaskDescriptionException("Job key " + config.getJob() + " is invalid.");
      }

      if (!config.getJob().getRole().equals(config.getOwner().getRole())) {
        // Both owner and job key are set but don't match
        throw new TaskDescriptionException("Role must match job owner.");
      }
    } else {
      // TODO(maxim): Make sure both key and owner are populated to support older clients.
      // Remove in 0.7.0. (AURORA-749).
      // Job key is not set -> populate from owner, environment and name
      assertOwnerValidity(config.getOwner());
      builder.setJob(
          JobKeys.from(config.getOwner().getRole(), config.getEnvironment(), config.getJobName())
              .newBuilder());
    }

    if (!builder.isSetExecutorConfig()) {
      throw new TaskDescriptionException("Configuration may not be null");
    }

    // Maximize the usefulness of any thrown error message by checking required fields first.
    for (RequiredFieldValidator<?> validator : REQUIRED_FIELDS_VALIDATORS) {
      validator.validate(builder);
    }

    IConstraint constraint = getDedicatedConstraint(config);
    if (constraint != null) {
      if (!isValueConstraint(constraint.getConstraint())) {
        throw new TaskDescriptionException("A dedicated constraint must be of value type.");
      }

      IValueConstraint valueConstraint = constraint.getConstraint().getValue();

      if (valueConstraint.getValues().size() != 1) {
        throw new TaskDescriptionException("A dedicated constraint must have exactly one value");
      }

      String dedicatedRole = getRole(valueConstraint);
      if (!config.getOwner().getRole().equals(dedicatedRole)) {
        throw new TaskDescriptionException(
            "Only " + dedicatedRole + " may use hosts dedicated for that role.");
      }
    }

    Optional<Container._Fields> containerType;
    if (config.isSetContainer()) {
      IContainer containerConfig = config.getContainer();
      containerType = Optional.of(containerConfig.getSetField());
      if (containerConfig.isSetDocker()) {
        if (!containerConfig.getDocker().isSetImage()) {
          throw new TaskDescriptionException("A container must specify an image");
        }
        if (containerConfig.getDocker().isSetParameters()
            && !containerConfig.getDocker().getParameters().isEmpty()
            && !ENABLE_DOCKER_PARAMETERS.get()) {
          throw new TaskDescriptionException("Docker parameters not allowed.");
        }
      }
    } else {
      // Default to mesos container type if unset.
      containerType = Optional.of(Container._Fields.MESOS);
    }
    if (!containerType.isPresent()) {
      throw new TaskDescriptionException("A job must have a container type.");
    }
    if (!ALLOWED_CONTAINER_TYPES.get().contains(containerType.get())) {
      throw new TaskDescriptionException(
          "The container type " + containerType.get().toString() + " is not allowed");
    }

    return ITaskConfig.build(builder);
  }