private ListenableFuture<Response> request(
final URI uri, final String method, final Object entity) {
final Map<String, List<String>> headers = Maps.newHashMap();
final byte[] entityBytes;
headers.put(
VersionCompatibility.HELIOS_VERSION_HEADER, Collections.singletonList(Version.POM_VERSION));
if (entity != null) {
headers.put("Content-Type", singletonList("application/json"));
headers.put("Charset", singletonList("utf-8"));
entityBytes = Json.asBytesUnchecked(entity);
} else {
entityBytes = new byte[] {};
}
final ListenableFuture<Response> f = dispatcher.request(uri, method, entityBytes, headers);
return transform(
f,
new Function<Response, Response>() {
@Override
public Response apply(final Response response) {
checkProtocolVersionStatus(response);
return response;
}
});
}
@Test
public void testDeploymentGroupInspectCommandJson() throws Exception {
when(options.getString("name")).thenReturn(NAME);
final int ret = command.run(options, client, out, true, null);
assertEquals(0, ret);
final DeploymentGroup output = Json.read(baos.toString(), DeploymentGroup.class);
assertEquals(DEPLOYMENT_GROUP, output);
}
protected void undeployJob(final JobId jobId, final String host) throws Exception {
final String undeployOutput = cli("undeploy", jobId.toString(), host);
assertThat(undeployOutput, containsString(host + ": done"));
final String output = cli("status", "--host", host, "--json");
final Map<JobId, JobStatus> statuses =
Json.readUnchecked(output, new TypeReference<Map<JobId, JobStatus>>() {});
final JobStatus status = statuses.get(jobId);
assertTrue(status == null || status.getDeployments().get(host) == null);
}
protected String createJobRawOutput(final Job job) throws Exception {
final String name = job.getId().getName();
checkArgument(name.contains(testTag), "Job name must contain testTag to enable cleanup");
final String serializedConfig = Json.asNormalizedString(job);
final File configFile = temporaryFolder.newFile();
Files.write(serializedConfig, configFile, Charsets.UTF_8);
final List<String> args = ImmutableList.of("-q", "-f", configFile.getAbsolutePath());
return cli("create", args);
}
@Test
public void testDeploymentGroupInspectCommandNotFoundJson() throws Exception {
when(options.getString("name")).thenReturn(NON_EXISTENT_NAME);
final int ret = command.run(options, client, out, true, null);
assertEquals(1, ret);
final Map<String, Object> output =
Json.read(baos.toString(), new TypeReference<Map<String, Object>>() {});
assertEquals("DEPLOYMENT_GROUP_NOT_FOUND", output.get("status"));
}
private <T> T tryGetEntity(
final ZooKeeperClient client, String path, TypeReference<T> type, String name) {
try {
final byte[] data = client.getData(path);
return Json.read(data, type);
} catch (NoNodeException e) {
return null;
} catch (KeeperException | IOException e) {
throw new HeliosRuntimeException("reading " + name + " info failed", e);
}
}
private Job getJob(final ZooKeeperClient client, final JobId id) {
final String path = Paths.configJob(id);
try {
final byte[] data = client.getData(path);
return Json.read(data, Job.class);
} catch (NoNodeException e) {
// Return null to indicate that the job does not exist
return null;
} catch (KeeperException | IOException e) {
throw new HeliosRuntimeException("getting job " + id + " failed", e);
}
}
protected void deployJob(final JobId jobId, final String host, final String token)
throws Exception {
final List<String> deployArgs = Lists.newArrayList(jobId.toString(), host);
if (token != null) {
deployArgs.addAll(ImmutableList.of("--token", token));
}
final String deployOutput = cli("deploy", deployArgs);
assertThat(deployOutput, containsString(host + ": done"));
final String output = cli("status", "--host", host, "--json");
final Map<JobId, JobStatus> statuses =
Json.readUnchecked(output, new TypeReference<Map<JobId, JobStatus>>() {});
assertTrue(statuses.keySet().contains(jobId));
}
/** Given a jobId, returns the N most recent events in it's history in the cluster. */
@Override
public List<TaskStatusEvent> getJobHistory(final JobId jobId) throws JobDoesNotExistException {
final Job descriptor = getJob(jobId);
if (descriptor == null) {
throw new JobDoesNotExistException(jobId);
}
final ZooKeeperClient client = provider.get("getJobHistory");
final List<String> hosts;
try {
hosts = client.getChildren(Paths.historyJobHosts(jobId));
} catch (NoNodeException e) {
return emptyList();
} catch (KeeperException e) {
throw Throwables.propagate(e);
}
final List<TaskStatusEvent> jsEvents = Lists.newArrayList();
for (String host : hosts) {
final List<String> events;
try {
events = client.getChildren(Paths.historyJobHostEvents(jobId, host));
} catch (KeeperException e) {
throw Throwables.propagate(e);
}
for (String event : events) {
try {
byte[] data =
client.getData(Paths.historyJobHostEventsTimestamp(jobId, host, Long.valueOf(event)));
final TaskStatus status = Json.read(data, TaskStatus.class);
jsEvents.add(new TaskStatusEvent(status, Long.valueOf(event), host));
} catch (NoNodeException e) { // ignore, it went away before we read it
} catch (KeeperException | IOException e) {
throw Throwables.propagate(e);
}
}
}
return Ordering.from(EVENT_COMPARATOR).sortedCopy(jsEvents);
}
@Override
public ListenableFuture<T> apply(@NotNull final Response reply) throws HeliosException {
if (reply.status() == HTTP_NOT_FOUND && !decodeableStatusCodes.contains(HTTP_NOT_FOUND)) {
return immediateFuture(null);
}
if (!decodeableStatusCodes.contains(reply.status())) {
throw new HeliosException("request failed: " + reply);
}
if (reply.payload().length == 0) {
throw new HeliosException("bad reply: " + reply);
}
final T result;
try {
result = Json.read(reply.payload(), javaType);
} catch (IOException e) {
throw new HeliosException("bad reply: " + reply, e);
}
return immediateFuture(result);
}
public Job build() {
final String configHash;
try {
configHash = hex(Json.sha1digest(p));
} catch (IOException e) {
throw propagate(e);
}
final String hash;
if (!Strings.isNullOrEmpty(this.hash)) {
hash = this.hash;
} else {
if (p.name != null && p.version != null) {
final String input = String.format("%s:%s:%s", p.name, p.version, configHash);
hash = hex(sha1digest(input.getBytes(UTF_8)));
} else {
hash = null;
}
}
final JobId id = new JobId(p.name, p.version, hash);
return new Job(id, p);
}
protected <T> T cliJson(final Class<T> klass, final String command, final List<String> args)
throws Exception {
final List<String> args0 = newArrayList("--json");
args0.addAll(args);
return Json.read(cli(command, args0), klass);
}
@Override
protected int runWithJobId(
final Namespace options,
final HeliosClient client,
final PrintStream out,
final boolean json,
final JobId jobId,
final BufferedReader stdin)
throws ExecutionException, InterruptedException, IOException {
final String name = options.getString(nameArg.getDest());
final long timeout = options.getLong(timeoutArg.getDest());
final int parallelism = options.getInt(parallelismArg.getDest());
final boolean async = options.getBoolean(asyncArg.getDest());
final long rolloutTimeout = options.getLong(rolloutTimeoutArg.getDest());
final boolean migrate = options.getBoolean(migrateArg.getDest());
final boolean overlap = options.getBoolean(overlapArg.getDest());
final String token = options.getString(tokenArg.getDest());
checkArgument(timeout > 0, "Timeout must be greater than 0");
checkArgument(parallelism > 0, "Parallelism must be greater than 0");
checkArgument(rolloutTimeout > 0, "Rollout timeout must be greater than 0");
final long startTime = timeSupplier.get();
final RolloutOptions rolloutOptions =
RolloutOptions.newBuilder()
.setTimeout(timeout)
.setParallelism(parallelism)
.setMigrate(migrate)
.setOverlap(overlap)
.setToken(token)
.build();
final RollingUpdateResponse response = client.rollingUpdate(name, jobId, rolloutOptions).get();
if (response.getStatus() != RollingUpdateResponse.Status.OK) {
if (!json) {
out.println("Failed: " + response);
} else {
out.println(response.toJsonString());
}
return 1;
}
if (!json) {
out.println(
format(
"Rolling update%s started: %s -> %s "
+ "(parallelism=%d, timeout=%d, overlap=%b, token=%s)%s",
async ? " (async)" : "",
name,
jobId.toShortString(),
parallelism,
timeout,
overlap,
token,
async ? "" : "\n"));
}
final Map<String, Object> jsonOutput = Maps.newHashMap();
jsonOutput.put("parallelism", parallelism);
jsonOutput.put("timeout", timeout);
jsonOutput.put("overlap", overlap);
jsonOutput.put("token", token);
if (async) {
if (json) {
jsonOutput.put("status", response.getStatus());
out.println(Json.asStringUnchecked(jsonOutput));
}
return 0;
}
String error = "";
boolean failed = false;
boolean timedOut = false;
final Set<String> reported = Sets.newHashSet();
while (true) {
final DeploymentGroupStatusResponse status = client.deploymentGroupStatus(name).get();
if (status == null) {
failed = true;
error = "Failed to fetch deployment-group status";
break;
}
if (!jobId.equals(status.getDeploymentGroup().getJobId())) {
// Another rolling-update was started, overriding this one -- exit
failed = true;
error = "Deployment-group job id changed during rolling-update";
break;
}
if (!json) {
for (DeploymentGroupStatusResponse.HostStatus hostStatus : status.getHostStatuses()) {
final JobId hostJobId = hostStatus.getJobId();
final String host = hostStatus.getHost();
final TaskStatus.State state = hostStatus.getState();
final boolean done =
hostJobId != null && hostJobId.equals(jobId) && state == TaskStatus.State.RUNNING;
if (done && reported.add(host)) {
out.println(
format(
"%s -> %s (%d/%d)",
host, state, reported.size(), status.getHostStatuses().size()));
}
}
}
if (status.getStatus() != DeploymentGroupStatusResponse.Status.ROLLING_OUT) {
if (status.getStatus() == DeploymentGroupStatusResponse.Status.FAILED) {
failed = true;
error = status.getError();
}
break;
}
if (timeSupplier.get() - startTime > TimeUnit.MINUTES.toMillis(rolloutTimeout)) {
// Rollout timed out
timedOut = true;
break;
}
sleepFunction.sleep(POLL_INTERVAL_MILLIS);
}
final double duration = (timeSupplier.get() - startTime) / 1000.0;
if (json) {
if (failed) {
jsonOutput.put("status", "FAILED");
jsonOutput.put("error", error);
} else if (timedOut) {
jsonOutput.put("status", "TIMEOUT");
} else {
jsonOutput.put("status", "DONE");
}
jsonOutput.put("duration", duration);
out.println(Json.asStringUnchecked(jsonOutput));
} else {
out.println();
if (failed) {
out.println(format("Failed: %s", error));
} else if (timedOut) {
out.println("Timed out! (rolling-update still in progress)");
} else {
out.println("Done.");
}
out.println(format("Duration: %.2f s", duration));
}
return (failed || timedOut) ? 1 : 0;
}
@Test
public void testNotInOperatorSerialization() throws Exception {
final HostSelector orig = HostSelector.parse("a notin (foo,bar)");
final HostSelector parsed = Json.read(Json.asString(orig), HostSelector.class);
assertEquals(orig, parsed);
}
// TODO(drewc): this kinda screams "long method"
private void deployJobRetry(
final ZooKeeperClient client, final String host, final Deployment deployment, int count)
throws JobDoesNotExistException, JobAlreadyDeployedException, HostNotFoundException,
JobPortAllocationConflictException {
if (count == 3) {
throw new HeliosRuntimeException(
"3 failures (possibly concurrent modifications) while " + "deploying. Giving up.");
}
log.info("deploying {}: {} (retry={})", deployment, host, count);
final JobId id = deployment.getJobId();
final Job job = getJob(id);
if (job == null) {
throw new JobDoesNotExistException(id);
}
final UUID operationId = UUID.randomUUID();
final String jobPath = Paths.configJob(id);
final String taskPath = Paths.configHostJob(host, id);
final String taskCreationPath = Paths.configHostJobCreation(host, id, operationId);
final List<Integer> staticPorts = staticPorts(job);
final Map<String, byte[]> portNodes = Maps.newHashMap();
final byte[] idJson = id.toJsonBytes();
for (final int port : staticPorts) {
final String path = Paths.configHostPort(host, port);
portNodes.put(path, idJson);
}
final Task task = new Task(job, deployment.getGoal());
final List<ZooKeeperOperation> operations =
Lists.newArrayList(
check(jobPath), create(portNodes), create(Paths.configJobHost(id, host)));
// Attempt to read a task here. If it's goal is UNDEPLOY, it's as good as not existing
try {
final Node existing = client.getNode(taskPath);
byte[] bytes = existing.getBytes();
Task readTask = Json.read(bytes, Task.class);
if (readTask.getGoal() != Goal.UNDEPLOY) {
throw new JobAlreadyDeployedException(host, id);
}
operations.add(check(taskPath, existing.getStat().getVersion()));
operations.add(set(taskPath, task));
} catch (NoNodeException e) {
operations.add(create(taskPath, task));
operations.add(create(taskCreationPath));
} catch (IOException | KeeperException e) {
throw new HeliosRuntimeException("reading existing task description failed", e);
}
// TODO (dano): Failure handling is racy wrt agent and job modifications.
try {
client.transaction(operations);
log.info("deployed {}: {} (retry={})", deployment, host, count);
} catch (NoNodeException e) {
// Either the job, the host or the task went away
assertJobExists(client, id);
assertHostExists(client, host);
// If the job and host still exists, we likely tried to redeploy a job that had an UNDEPLOY
// goal and lost the race with the agent removing the task before we could set it. Retry.
deployJobRetry(client, host, deployment, count + 1);
} catch (NodeExistsException e) {
// Check for conflict due to transaction retry
try {
if (client.exists(taskCreationPath) != null) {
// Our creation operation node existed, we're done here
return;
}
} catch (KeeperException ex) {
throw new HeliosRuntimeException("checking job deployment failed", ex);
}
try {
// Check if the job was already deployed
if (client.stat(taskPath) != null) {
throw new JobAlreadyDeployedException(host, id);
}
} catch (KeeperException ex) {
throw new HeliosRuntimeException("checking job deployment failed", e);
}
// Check for static port collisions
for (final int port : staticPorts) {
final String path = Paths.configHostPort(host, port);
try {
if (client.stat(path) == null) {
continue;
}
final byte[] b = client.getData(path);
final JobId existingJobId = parse(b, JobId.class);
throw new JobPortAllocationConflictException(id, existingJobId, host, port);
} catch (KeeperException | IOException ex) {
throw new HeliosRuntimeException("checking port allocations failed", e);
}
}
// Catch all for logic and ephemeral issues
throw new HeliosRuntimeException("deploying job failed", e);
} catch (KeeperException e) {
throw new HeliosRuntimeException("deploying job failed", e);
}
}
public String toJsonString() {
return Json.asStringUnchecked(this);
}
public static <T> ConvertResponseToPojo<T> create(
final Class<T> clazz, final Set<Integer> decodeableStatusCodes) {
return new ConvertResponseToPojo<>(Json.type(clazz), decodeableStatusCodes);
}
private <T> ListenableFuture<T> get(final URI uri, final Class<T> clazz) {
return get(uri, Json.type(clazz));
}
private <T> ListenableFuture<T> get(final URI uri, final TypeReference<T> typeReference) {
return get(uri, Json.type(typeReference));
}
