Ejemplo n.º 1
0
 /** Returns a {@link Map} of {@link JobId} to {@link Job} objects for all of the jobs known. */
 @Override
 public Map<JobId, Job> getJobs() {
   log.debug("getting jobs");
   final String folder = Paths.configJobs();
   final ZooKeeperClient client = provider.get("getJobs");
   try {
     final List<String> ids;
     try {
       ids = client.getChildren(folder);
     } catch (NoNodeException e) {
       return Maps.newHashMap();
     }
     final Map<JobId, Job> descriptors = Maps.newHashMap();
     for (final String id : ids) {
       final JobId jobId = JobId.fromString(id);
       final String path = Paths.configJob(jobId);
       final byte[] data = client.getData(path);
       final Job descriptor = parse(data, Job.class);
       descriptors.put(descriptor.getId(), descriptor);
     }
     return descriptors;
   } catch (KeeperException | IOException e) {
     throw new HeliosRuntimeException("getting jobs failed", e);
   }
 }
Ejemplo n.º 2
0
  protected void assertJobEquals(final Job expected, final Job actual) {
    final Builder expectedBuilder = expected.toBuilder();

    // hack to make sure that any environment variables that were folded into the created job
    // because of environment variables set at runtime on the test-running-agent are removed
    // from the actual when we assert the equality below
    final Builder actualBuilder = actual.toBuilder();
    final Map<String, String> metadata = Maps.newHashMap(actual.getMetadata());
    for (Map.Entry<String, String> entry : JobCreateCommand.DEFAULT_METADATA_ENVVARS.entrySet()) {
      final String envVar = entry.getKey();
      final String metadataKey = entry.getValue();
      final String envValue = System.getenv(envVar);
      if (envValue != null
          && actual.getMetadata().containsKey(metadataKey)
          && actual.getMetadata().get(metadataKey).equals(envValue)) {
        metadata.remove(metadataKey);
      }
    }
    actualBuilder.setMetadata(metadata);

    // Remove created timestamp set by master
    actualBuilder.setCreated(null);

    // copy the hash
    expectedBuilder.setHash(actualBuilder.build().getId().getHash());

    assertEquals(expectedBuilder.build(), actualBuilder.build());
  }
  @Override
  public void decorateContainerConfig(
      Job job, ImageInfo imageInfo, ContainerConfig.Builder containerConfig) {
    final ContainerConfig imageConfig = imageInfo.config();

    // Inject syslog-redirector in the entrypoint to capture std out/err
    final String syslogRedirectorPath =
        Optional.fromNullable(job.getEnv().get("SYSLOG_REDIRECTOR"))
            .or("/helios/syslog-redirector");

    final List<String> entrypoint =
        Lists.newArrayList(
            syslogRedirectorPath, "-h", syslogHostPort, "-n", job.getId().toString(), "--");
    if (imageConfig.entrypoint() != null) {
      entrypoint.addAll(imageConfig.entrypoint());
    }
    containerConfig.entrypoint(entrypoint);

    // If there's no explicit container cmd specified, copy over the one from the image.
    // Only setting the entrypoint causes dockerd to not use the image cmd.
    if ((containerConfig.cmd() == null || containerConfig.cmd().isEmpty())
        && imageConfig.cmd() != null) {
      containerConfig.cmd(imageConfig.cmd());
    }

    final Set<String> volumes = Sets.newHashSet();
    if (containerConfig.volumes() != null) {
      volumes.addAll(containerConfig.volumes());
    }
    volumes.add("/helios");
    containerConfig.volumes(volumes);
  }
Ejemplo n.º 4
0
 protected JobId createJob(
     final String name,
     final String version,
     final String image,
     final String hostname,
     final List<String> command,
     final Map<String, String> env,
     final Map<String, PortMapping> ports,
     final Map<ServiceEndpoint, ServicePorts> registration,
     final Integer gracePeriod,
     final Map<String, String> volumes,
     final Date expires)
     throws Exception {
   return createJob(
       Job.newBuilder()
           .setName(name)
           .setVersion(version)
           .setImage(image)
           .setHostname(hostname)
           .setCommand(command)
           .setEnv(env)
           .setPorts(ports)
           .setRegistration(registration)
           .setGracePeriod(gracePeriod)
           .setVolumes(volumes)
           .setExpires(expires)
           .build());
 }
Ejemplo n.º 5
0
  @Before
  public void setup() throws Exception {
    startDefaultMaster();

    client = defaultClient();
    startDefaultAgent(testHost());

    job =
        Job.newBuilder()
            .setName(testJobName)
            .setVersion(testJobVersion)
            .setImage(BUSYBOX)
            .addVolume("/volume")
            .addVolume("/hostname", "/etc/hostname")
            .setCommand(
                asList(
                    "sh",
                    "-c",
                    "echo foo > /volume/bar; "
                        + "nc -p 4711 -le dd if=/volume/bar;"
                        + "nc -p 4712 -lle dd if=/hostname"))
            .addPort("bar", PortMapping.of(4711))
            .addPort("hostname", PortMapping.of(4712))
            .setCreatingUser(TEST_USER)
            .build();
  }
Ejemplo n.º 6
0
 /** Adds a job into the configuration. */
 @Override
 public void addJob(final Job job) throws JobExistsException {
   log.info("adding job: {}", job);
   final JobId id = job.getId();
   final UUID operationId = UUID.randomUUID();
   final String creationPath = Paths.configJobCreation(id, operationId);
   final ZooKeeperClient client = provider.get("addJob");
   try {
     try {
       client.ensurePath(Paths.historyJob(id));
       client.transaction(
           create(Paths.configJob(id), job),
           create(Paths.configJobRefShort(id), id),
           create(Paths.configJobHosts(id)),
           create(creationPath));
     } catch (final NodeExistsException e) {
       if (client.exists(creationPath) != null) {
         // The job was created, we're done here
         return;
       }
       throw new JobExistsException(id.toString());
     }
   } catch (final KeeperException e) {
     throw new HeliosRuntimeException("adding job " + job + " failed", e);
   }
 }
Ejemplo n.º 7
0
 private List<Integer> staticPorts(final Job job) {
   final List<Integer> staticPorts = Lists.newArrayList();
   for (final PortMapping portMapping : job.getPorts().values()) {
     if (portMapping.getExternalPort() != null) {
       staticPorts.add(portMapping.getExternalPort());
     }
   }
   return staticPorts;
 }
Ejemplo n.º 8
0
  protected String createJobRawOutput(final Job job) throws Exception {
    final String name = job.getId().getName();
    checkArgument(name.contains(testTag), "Job name must contain testTag to enable cleanup");

    final String serializedConfig = Json.asNormalizedString(job);
    final File configFile = temporaryFolder.newFile();
    Files.write(serializedConfig, configFile, Charsets.UTF_8);

    final List<String> args = ImmutableList.of("-q", "-f", configFile.getAbsolutePath());
    return cli("create", args);
  }
Ejemplo n.º 9
0
  @Override
  public boolean equals(final Object o) {
    if (this == o) {
      return true;
    }
    if (o == null || getClass() != o.getClass()) {
      return false;
    }

    final Task task = (Task) o;

    if (goal != task.goal) {
      return false;
    }
    if (job != null ? !job.equals(task.job) : task.job != null) {
      return false;
    }

    return true;
  }
  @Test
  public void test() throws Exception {
    startDefaultMaster();
    startDefaultAgent(testHost());

    final HeliosClient client = defaultClient();

    // Create a job using an image exposing port 11211 but without mapping it
    final Job job1 =
        Job.newBuilder()
            .setName(testTag + "memcached")
            .setVersion("v1")
            .setImage("rohan/memcached-mini")
            .build();
    final JobId jobId1 = job1.getId();
    client.createJob(job1).get();

    // Create a job using an image exposing port 11211 and map it to a specific external port
    final Job job2 =
        Job.newBuilder()
            .setName(testTag + "memcached")
            .setVersion("v2")
            .setImage("rohan/memcached-mini")
            .setPorts(ImmutableMap.of("tcp", PortMapping.of(11211, externalPort)))
            .build();
    final JobId jobId2 = job2.getId();
    client.createJob(job2).get();

    // Wait for agent to come up
    awaitHostRegistered(client, testHost(), LONG_WAIT_MINUTES, MINUTES);
    awaitHostStatus(client, testHost(), UP, LONG_WAIT_MINUTES, MINUTES);

    // Deploy the jobs on the agent
    client.deploy(Deployment.of(jobId1, START), testHost()).get();
    client.deploy(Deployment.of(jobId2, START), testHost()).get();

    // Wait for the jobs to run
    awaitJobState(client, testHost(), jobId1, RUNNING, LONG_WAIT_MINUTES, MINUTES);
    awaitJobState(client, testHost(), jobId2, RUNNING, LONG_WAIT_MINUTES, MINUTES);
  }
Ejemplo n.º 11
0
 @Test
 public void testClient() throws Exception {
   final CreateJobResponse created = client.createJob(job).get();
   assertEquals(CreateJobResponse.Status.OK, created.getStatus());
   assertVolumes(job.getId());
 }
Ejemplo n.º 12
0
 @Override
 public int hashCode() {
   int result = goal != null ? goal.hashCode() : 0;
   result = 31 * result + (job != null ? job.hashCode() : 0);
   return result;
 }
Ejemplo n.º 13
0
 @Override
 public int compareTo(@NotNull final Job o) {
   return id.compareTo(o.getId());
 }