private Map<JobId, Deployment> getTasks(final ZooKeeperClient client, final String host) {
final Map<JobId, Deployment> jobs = Maps.newHashMap();
try {
final String folder = Paths.configHostJobs(host);
final List<String> jobIds;
try {
jobIds = client.getChildren(folder);
} catch (KeeperException.NoNodeException e) {
return null;
}
for (final String jobIdString : jobIds) {
final JobId jobId = JobId.fromString(jobIdString);
final String containerPath = Paths.configHostJob(host, jobId);
try {
final byte[] data = client.getData(containerPath);
final Task task = parse(data, Task.class);
jobs.put(jobId, Deployment.of(jobId, task.getGoal()));
} catch (KeeperException.NoNodeException ignored) {
log.debug("deployment config node disappeared: {}", jobIdString);
}
}
} catch (KeeperException | IOException e) {
throw new HeliosRuntimeException("getting deployment config failed", e);
}
return jobs;
}
/** Returns a {@link Map} of {@link JobId} to {@link Job} objects for all of the jobs known. */
@Override
public Map<JobId, Job> getJobs() {
log.debug("getting jobs");
final String folder = Paths.configJobs();
final ZooKeeperClient client = provider.get("getJobs");
try {
final List<String> ids;
try {
ids = client.getChildren(folder);
} catch (NoNodeException e) {
return Maps.newHashMap();
}
final Map<JobId, Job> descriptors = Maps.newHashMap();
for (final String id : ids) {
final JobId jobId = JobId.fromString(id);
final String path = Paths.configJob(jobId);
final byte[] data = client.getData(path);
final Job descriptor = parse(data, Job.class);
descriptors.put(descriptor.getId(), descriptor);
}
return descriptors;
} catch (KeeperException | IOException e) {
throw new HeliosRuntimeException("getting jobs failed", e);
}
}
private UUID getJobCreation(final ZooKeeperClient client, final JobId id) throws KeeperException {
final String parent = Paths.configHostJobCreationParent(id);
final List<String> children = client.getChildren(parent);
for (final String child : children) {
if (Paths.isConfigJobCreation(id, parent, child)) {
return Paths.configJobCreationId(id, parent, child);
}
}
return null;
}
/** Given a jobId, returns the N most recent events in it's history in the cluster. */
@Override
public List<TaskStatusEvent> getJobHistory(final JobId jobId) throws JobDoesNotExistException {
final Job descriptor = getJob(jobId);
if (descriptor == null) {
throw new JobDoesNotExistException(jobId);
}
final ZooKeeperClient client = provider.get("getJobHistory");
final List<String> hosts;
try {
hosts = client.getChildren(Paths.historyJobHosts(jobId));
} catch (NoNodeException e) {
return emptyList();
} catch (KeeperException e) {
throw Throwables.propagate(e);
}
final List<TaskStatusEvent> jsEvents = Lists.newArrayList();
for (String host : hosts) {
final List<String> events;
try {
events = client.getChildren(Paths.historyJobHostEvents(jobId, host));
} catch (KeeperException e) {
throw Throwables.propagate(e);
}
for (String event : events) {
try {
byte[] data =
client.getData(Paths.historyJobHostEventsTimestamp(jobId, host, Long.valueOf(event)));
final TaskStatus status = Json.read(data, TaskStatus.class);
jsEvents.add(new TaskStatusEvent(status, Long.valueOf(event), host));
} catch (NoNodeException e) { // ignore, it went away before we read it
} catch (KeeperException | IOException e) {
throw Throwables.propagate(e);
}
}
}
return Ordering.from(EVENT_COMPARATOR).sortedCopy(jsEvents);
}
private List<String> listJobHosts(final ZooKeeperClient client, final JobId jobId)
throws JobDoesNotExistException {
final List<String> hosts;
try {
hosts = client.getChildren(Paths.configJobHosts(jobId));
} catch (NoNodeException e) {
throw new JobDoesNotExistException(jobId);
} catch (KeeperException e) {
throw new HeliosRuntimeException("failed to list hosts for job: " + jobId, e);
}
return hosts;
}
private List<JobId> listHostJobs(final ZooKeeperClient client, final String host) {
final List<String> jobIdStrings;
final String folder = Paths.statusHostJobs(host);
try {
jobIdStrings = client.getChildren(folder);
} catch (KeeperException.NoNodeException e) {
return null;
} catch (KeeperException e) {
throw new HeliosRuntimeException("List tasks for host failed: " + host, e);
}
final ImmutableList.Builder<JobId> jobIds = ImmutableList.builder();
for (String jobIdString : jobIdStrings) {
jobIds.add(JobId.fromString(jobIdString));
}
return jobIds.build();
}
/** Returns a list of the host names of the currently running masters. */
@Override
public List<String> getRunningMasters() {
final ZooKeeperClient client = provider.get("getRunningMasters");
try {
final List<String> masters = client.getChildren(Paths.statusMaster());
final ImmutableList.Builder<String> upMasters = ImmutableList.builder();
for (final String master : masters) {
if (client.exists(Paths.statusMasterUp(master)) != null) {
upMasters.add(master);
}
}
return upMasters.build();
} catch (KeeperException e) {
throw new HeliosRuntimeException("listing masters failed", e);
}
}
/**
* Undoes the effect of {@link ZooKeeperMasterModel#registerHost(String, String)}. Cleans up any
* leftover host-related things.
*/
@Override
public void deregisterHost(final String host)
throws HostNotFoundException, HostStillInUseException {
log.info("deregistering host: {}", host);
final ZooKeeperClient client = provider.get("deregisterHost");
// TODO (dano): handle retry failures
try {
final List<ZooKeeperOperation> operations = Lists.newArrayList();
// Remove all jobs deployed to this host
final List<JobId> jobs = listHostJobs(client, host);
if (jobs == null) {
if (client.exists(Paths.configHost(host)) == null) {
throw new HostNotFoundException("host [" + host + "] does not exist");
}
}
for (JobId job : jobs) {
final String hostJobPath = Paths.configHostJob(host, job);
final List<String> nodes = client.listRecursive(hostJobPath);
for (final String node : reverse(nodes)) {
operations.add(delete(node));
}
if (client.exists(Paths.configJobHost(job, host)) != null) {
operations.add(delete(Paths.configJobHost(job, host)));
}
// Clean out the history for each job
try {
final List<String> history = client.listRecursive(Paths.historyJobHost(job, host));
for (String s : reverse(history)) {
operations.add(delete(s));
}
} catch (NoNodeException ignore) {
}
}
operations.add(delete(Paths.configHostJobs(host)));
// Remove the host status
try {
final List<String> nodes = client.listRecursive(Paths.statusHost(host));
for (final String node : reverse(nodes)) {
operations.add(delete(node));
}
} catch (NoNodeException ignore) {
}
// Remove port allocations
try {
final List<String> ports = client.getChildren(Paths.configHostPorts(host));
for (final String port : ports) {
operations.add(delete(Paths.configHostPort(host, Integer.valueOf(port))));
}
operations.add(delete(Paths.configHostPorts(host)));
} catch (NoNodeException ignore) {
}
// Remove host id
String idPath = Paths.configHostId(host);
if (client.exists(idPath) != null) {
operations.add(delete(idPath));
}
// Remove host config root
operations.add(delete(Paths.configHost(host)));
client.transaction(operations);
} catch (NotEmptyException e) {
final HostStatus hostStatus = getHostStatus(host);
final List<JobId> jobs =
hostStatus != null
? ImmutableList.copyOf(hostStatus.getJobs().keySet())
: Collections.<JobId>emptyList();
throw new HostStillInUseException(host, jobs);
} catch (NoNodeException e) {
throw new HostNotFoundException(host);
} catch (KeeperException e) {
throw new HeliosRuntimeException(e);
}
}
