/** Adds a job into the configuration. */
@Override
public void addJob(final Job job) throws JobExistsException {
log.info("adding job: {}", job);
final JobId id = job.getId();
final UUID operationId = UUID.randomUUID();
final String creationPath = Paths.configJobCreation(id, operationId);
final ZooKeeperClient client = provider.get("addJob");
try {
try {
client.ensurePath(Paths.historyJob(id));
client.transaction(
create(Paths.configJob(id), job),
create(Paths.configJobRefShort(id), id),
create(Paths.configJobHosts(id)),
create(creationPath));
} catch (final NodeExistsException e) {
if (client.exists(creationPath) != null) {
// The job was created, we're done here
return;
}
throw new JobExistsException(id.toString());
}
} catch (final KeeperException e) {
throw new HeliosRuntimeException("adding job " + job + " failed", e);
}
}
/** Deletes a job from ZooKeeper. Ensures that job is not currently running anywhere. */
@Override
public Job removeJob(final JobId id) throws JobDoesNotExistException, JobStillDeployedException {
log.info("removing job: id={}", id);
final ZooKeeperClient client = provider.get("removeJob");
final Job job = getJob(client, id);
if (job == null) {
throw new JobDoesNotExistException(id);
}
// TODO (dano): handle retry failures
try {
final ImmutableList.Builder<ZooKeeperOperation> operations = ImmutableList.builder();
final UUID jobCreationOperationId = getJobCreation(client, id);
if (jobCreationOperationId != null) {
operations.add(delete(Paths.configJobCreation(id, jobCreationOperationId)));
}
operations.add(
delete(Paths.configJobHosts(id)),
delete(Paths.configJobRefShort(id)),
delete(Paths.configJob(id)));
client.transaction(operations.build());
} catch (final NoNodeException e) {
throw new JobDoesNotExistException(id);
} catch (final NotEmptyException e) {
throw new JobStillDeployedException(id, listJobHosts(client, id));
} catch (final KeeperException e) {
throw new HeliosRuntimeException("removing job " + id + " failed", e);
}
return job;
}
/** Undeploys the job specified by {@code jobId} on {@code host}. */
@Override
public Deployment undeployJob(final String host, final JobId jobId)
throws HostNotFoundException, JobNotDeployedException {
log.info("undeploying {}: {}", jobId, host);
final ZooKeeperClient client = provider.get("undeployJob");
assertHostExists(client, host);
final Deployment deployment = getDeployment(host, jobId);
if (deployment == null) {
throw new JobNotDeployedException(host, jobId);
}
// TODO (dano): Is this safe? can the ports of an undeployed job collide with a new deployment?
// TODO (drewc): If it's still in UNDEPLOY, that means the agent hasn't gotten to it
// yet, which means it probably won't see the new job yet either. However, it may spin up
// a new supervisor for the new job before the old one is done being torn down. So it can
// race and lose. With a little change to the Agent where we manage the supervisors, with
// some coordination, we could remove the race, such that this race goes away. Specifically,
// we're creating new Supervisors before updating existing ones. If we swap that, part of
// the problem goes away, but we'd need some coordination between the Supervisor and the
// agent such that the Agent could wait until the Supervisor had handled the Goal change.
// Additionally, since ZK guarantees we'll see the writes in the proper order, we wouldn't
// need to deal with seeing the new job before the UNDEPLOY.
final Job job = getJob(client, jobId);
final String path = Paths.configHostJob(host, jobId);
final Task task = new Task(job, UNDEPLOY);
final List<ZooKeeperOperation> operations =
Lists.newArrayList(set(path, task.toJsonBytes()), delete(Paths.configJobHost(jobId, host)));
final List<Integer> staticPorts = staticPorts(job);
for (int port : staticPorts) {
operations.add(delete(Paths.configHostPort(host, port)));
}
try {
client.transaction(operations);
} catch (NoNodeException e) {
if (e.getPath().equals(path)) {
// NoNodeException on updating the deployment node may happen due to retry failures.
// If the deployment isn't there anymore, we're done.
return deployment;
} else {
// The relation node deletes should not fail unless there is a programming error.
throw new HeliosRuntimeException("Removing deployment failed", e);
}
} catch (KeeperException e) {
throw new HeliosRuntimeException("Removing deployment failed", e);
}
return deployment;
}
// TODO(drewc): this kinda screams "long method"
private void deployJobRetry(
final ZooKeeperClient client, final String host, final Deployment deployment, int count)
throws JobDoesNotExistException, JobAlreadyDeployedException, HostNotFoundException,
JobPortAllocationConflictException {
if (count == 3) {
throw new HeliosRuntimeException(
"3 failures (possibly concurrent modifications) while " + "deploying. Giving up.");
}
log.info("deploying {}: {} (retry={})", deployment, host, count);
final JobId id = deployment.getJobId();
final Job job = getJob(id);
if (job == null) {
throw new JobDoesNotExistException(id);
}
final UUID operationId = UUID.randomUUID();
final String jobPath = Paths.configJob(id);
final String taskPath = Paths.configHostJob(host, id);
final String taskCreationPath = Paths.configHostJobCreation(host, id, operationId);
final List<Integer> staticPorts = staticPorts(job);
final Map<String, byte[]> portNodes = Maps.newHashMap();
final byte[] idJson = id.toJsonBytes();
for (final int port : staticPorts) {
final String path = Paths.configHostPort(host, port);
portNodes.put(path, idJson);
}
final Task task = new Task(job, deployment.getGoal());
final List<ZooKeeperOperation> operations =
Lists.newArrayList(
check(jobPath), create(portNodes), create(Paths.configJobHost(id, host)));
// Attempt to read a task here. If it's goal is UNDEPLOY, it's as good as not existing
try {
final Node existing = client.getNode(taskPath);
byte[] bytes = existing.getBytes();
Task readTask = Json.read(bytes, Task.class);
if (readTask.getGoal() != Goal.UNDEPLOY) {
throw new JobAlreadyDeployedException(host, id);
}
operations.add(check(taskPath, existing.getStat().getVersion()));
operations.add(set(taskPath, task));
} catch (NoNodeException e) {
operations.add(create(taskPath, task));
operations.add(create(taskCreationPath));
} catch (IOException | KeeperException e) {
throw new HeliosRuntimeException("reading existing task description failed", e);
}
// TODO (dano): Failure handling is racy wrt agent and job modifications.
try {
client.transaction(operations);
log.info("deployed {}: {} (retry={})", deployment, host, count);
} catch (NoNodeException e) {
// Either the job, the host or the task went away
assertJobExists(client, id);
assertHostExists(client, host);
// If the job and host still exists, we likely tried to redeploy a job that had an UNDEPLOY
// goal and lost the race with the agent removing the task before we could set it. Retry.
deployJobRetry(client, host, deployment, count + 1);
} catch (NodeExistsException e) {
// Check for conflict due to transaction retry
try {
if (client.exists(taskCreationPath) != null) {
// Our creation operation node existed, we're done here
return;
}
} catch (KeeperException ex) {
throw new HeliosRuntimeException("checking job deployment failed", ex);
}
try {
// Check if the job was already deployed
if (client.stat(taskPath) != null) {
throw new JobAlreadyDeployedException(host, id);
}
} catch (KeeperException ex) {
throw new HeliosRuntimeException("checking job deployment failed", e);
}
// Check for static port collisions
for (final int port : staticPorts) {
final String path = Paths.configHostPort(host, port);
try {
if (client.stat(path) == null) {
continue;
}
final byte[] b = client.getData(path);
final JobId existingJobId = parse(b, JobId.class);
throw new JobPortAllocationConflictException(id, existingJobId, host, port);
} catch (KeeperException | IOException ex) {
throw new HeliosRuntimeException("checking port allocations failed", e);
}
}
// Catch all for logic and ephemeral issues
throw new HeliosRuntimeException("deploying job failed", e);
} catch (KeeperException e) {
throw new HeliosRuntimeException("deploying job failed", e);
}
}
/**
* Undoes the effect of {@link ZooKeeperMasterModel#registerHost(String, String)}. Cleans up any
* leftover host-related things.
*/
@Override
public void deregisterHost(final String host)
throws HostNotFoundException, HostStillInUseException {
log.info("deregistering host: {}", host);
final ZooKeeperClient client = provider.get("deregisterHost");
// TODO (dano): handle retry failures
try {
final List<ZooKeeperOperation> operations = Lists.newArrayList();
// Remove all jobs deployed to this host
final List<JobId> jobs = listHostJobs(client, host);
if (jobs == null) {
if (client.exists(Paths.configHost(host)) == null) {
throw new HostNotFoundException("host [" + host + "] does not exist");
}
}
for (JobId job : jobs) {
final String hostJobPath = Paths.configHostJob(host, job);
final List<String> nodes = client.listRecursive(hostJobPath);
for (final String node : reverse(nodes)) {
operations.add(delete(node));
}
if (client.exists(Paths.configJobHost(job, host)) != null) {
operations.add(delete(Paths.configJobHost(job, host)));
}
// Clean out the history for each job
try {
final List<String> history = client.listRecursive(Paths.historyJobHost(job, host));
for (String s : reverse(history)) {
operations.add(delete(s));
}
} catch (NoNodeException ignore) {
}
}
operations.add(delete(Paths.configHostJobs(host)));
// Remove the host status
try {
final List<String> nodes = client.listRecursive(Paths.statusHost(host));
for (final String node : reverse(nodes)) {
operations.add(delete(node));
}
} catch (NoNodeException ignore) {
}
// Remove port allocations
try {
final List<String> ports = client.getChildren(Paths.configHostPorts(host));
for (final String port : ports) {
operations.add(delete(Paths.configHostPort(host, Integer.valueOf(port))));
}
operations.add(delete(Paths.configHostPorts(host)));
} catch (NoNodeException ignore) {
}
// Remove host id
String idPath = Paths.configHostId(host);
if (client.exists(idPath) != null) {
operations.add(delete(idPath));
}
// Remove host config root
operations.add(delete(Paths.configHost(host)));
client.transaction(operations);
} catch (NotEmptyException e) {
final HostStatus hostStatus = getHostStatus(host);
final List<JobId> jobs =
hostStatus != null
? ImmutableList.copyOf(hostStatus.getJobs().keySet())
: Collections.<JobId>emptyList();
throw new HostStillInUseException(host, jobs);
} catch (NoNodeException e) {
throw new HostNotFoundException(host);
} catch (KeeperException e) {
throw new HeliosRuntimeException(e);
}
}