protected void waitForMasterToBeFullyUp() throws Exception {
log.debug("waitForMasterToBeFullyUp: beginning wait loop");
Polling.await(
WAIT_TIMEOUT_SECONDS,
SECONDS,
new Callable<Object>() {
@Override
public Object call() {
try {
// While MasterService will start listening for http requests on the main and admin
// ports
// as soon as it is started (without waiting for ZK to be available), the Healthcheck
// registered for Zookeeper connectivity will cause the HealthcheckServlet to not
// return
// 200 OK until ZK is connected to (and even better, until *everything* is healthy).
final HttpGet request = new HttpGet(masterAdminEndpoint + "/healthcheck");
try (CloseableHttpResponse response = httpClient.execute(request)) {
final int status = response.getStatusLine().getStatusCode();
log.debug("waitForMasterToBeFullyUp: healthcheck endpoint returned {}", status);
return status == HttpStatus.SC_OK;
}
} catch (Exception e) {
return null;
}
}
});
}
protected TaskStatus awaitJobThrottle(
final HeliosClient client,
final String host,
final JobId jobId,
final ThrottleState throttled,
final int timeout,
final TimeUnit timeunit)
throws Exception {
return Polling.await(
timeout,
timeunit,
new Callable<TaskStatus>() {
@Override
public TaskStatus call() throws Exception {
final HostStatus hostStatus = getOrNull(client.hostStatus(host));
if (hostStatus == null) {
return null;
}
final TaskStatus taskStatus = hostStatus.getStatuses().get(jobId);
return (taskStatus != null && taskStatus.getThrottled() == throttled)
? taskStatus
: null;
}
});
}
protected DeploymentGroupStatus awaitDeploymentGroupStatus(
final HeliosClient client, final String name, final DeploymentGroupStatus.State state)
throws Exception {
return Polling.await(
LONG_WAIT_SECONDS,
SECONDS,
new Callable<DeploymentGroupStatus>() {
@Override
public DeploymentGroupStatus call() throws Exception {
final DeploymentGroupStatusResponse response =
getOrNull(client.deploymentGroupStatus(name));
if (response != null) {
final DeploymentGroupStatus status = response.getDeploymentGroupStatus();
if (status.getState().equals(state)) {
return status;
} else if (status.getState().equals(DeploymentGroupStatus.State.FAILED)) {
assertEquals(state, status.getState());
}
}
return null;
}
});
}
protected TaskStatus awaitTaskState(
final JobId jobId, final String host, final TaskStatus.State state) throws Exception {
return Polling.await(
LONG_WAIT_SECONDS,
SECONDS,
new Callable<TaskStatus>() {
@Override
public TaskStatus call() throws Exception {
final String output = cli("status", "--json", "--job", jobId.toString());
final Map<JobId, JobStatus> statusMap;
try {
statusMap = Json.read(output, new TypeReference<Map<JobId, JobStatus>>() {});
} catch (IOException e) {
return null;
}
final JobStatus status = statusMap.get(jobId);
if (status == null) {
return null;
}
final TaskStatus taskStatus = status.getTaskStatuses().get(host);
if (taskStatus == null) {
return null;
}
if (taskStatus.getState() != state) {
return null;
}
return taskStatus;
}
});
}
private byte[] recv(final int port, final int n) throws Exception {
checkArgument(n > 0, "n must be > 0");
return Polling.await(
LONG_WAIT_SECONDS,
SECONDS,
new Callable<byte[]>() {
@Override
public byte[] call() {
try (final Socket s = new Socket(DOCKER_HOST.address(), port)) {
final byte[] bytes = new byte[n];
final InputStream is = s.getInputStream();
final int first = is.read();
// Check if the uml kernel slirp driver did an accept->close on us,
// i.e. the actual listener is not up yet
if (first == -1) {
return null;
}
bytes[0] = (byte) first;
for (int i = 1; i < n; i++) {
bytes[i] = (byte) is.read();
}
return bytes;
} catch (IOException e) {
return null;
}
}
});
}
protected void awaitHostRegistered(
final HeliosClient client, final String host, final int timeout, final TimeUnit timeUnit)
throws Exception {
Polling.await(
timeout,
timeUnit,
new Callable<HostStatus>() {
@Override
public HostStatus call() throws Exception {
return getOrNull(client.hostStatus(host));
}
});
}
protected void awaitHostRegistered(final String name, final long timeout, final TimeUnit timeUnit)
throws Exception {
Polling.await(
timeout,
timeUnit,
new Callable<Object>() {
@Override
public Object call() throws Exception {
final String output = cli("hosts", "-q");
return output.contains(name) ? true : null;
}
});
}
private void awaitHealthy(
final ZooKeeperHealthChecker hc, final int duration, final TimeUnit timeUnit)
throws Exception {
Polling.await(
duration,
timeUnit,
new Callable<Object>() {
@Override
public Object call() throws Exception {
return hc.check().isHealthy() ? true : null;
}
});
}
private void awaitState(final String state, final int duration, final TimeUnit timeUnit)
throws Exception {
Polling.await(
duration,
timeUnit,
new Callable<Object>() {
@Override
public Object call() throws Exception {
final List<Event> events = riemannClient.getEvents();
if (events.isEmpty()) {
return null;
}
final Event event = events.get(0);
return state.equals(event.getState()) ? event : null;
}
});
}
protected void awaitTaskGone(
final HeliosClient client,
final String host,
final JobId jobId,
final long timeout,
final TimeUnit timeunit)
throws Exception {
Polling.await(
timeout,
timeunit,
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
final HostStatus hostStatus = getOrNull(client.hostStatus(host));
final TaskStatus taskStatus = hostStatus.getStatuses().get(jobId);
final Deployment deployment = hostStatus.getJobs().get(jobId);
return taskStatus == null && deployment == null ? true : null;
}
});
}
protected HostStatus awaitHostStatus(
final HeliosClient client,
final String host,
final HostStatus.Status status,
final int timeout,
final TimeUnit timeUnit)
throws Exception {
return Polling.await(
timeout,
timeUnit,
new Callable<HostStatus>() {
@Override
public HostStatus call() throws Exception {
final HostStatus hostStatus = getOrNull(client.hostStatus(host));
if (hostStatus == null) {
return null;
}
return (hostStatus.getStatus() == status) ? hostStatus : null;
}
});
}
protected static void removeContainer(final DockerClient dockerClient, final String containerId)
throws Exception {
// Work around docker sometimes failing to remove a container directly after killing it
Polling.await(
1,
MINUTES,
new Callable<Object>() {
@Override
public Object call() throws Exception {
try {
dockerClient.killContainer(containerId);
} catch (DockerRequestException e) {
if (e.message().contains("is not running")) {
// Container already isn't running. So we continue.
} else {
throw e;
}
}
try {
dockerClient.removeContainer(containerId);
return true;
} catch (ContainerNotFoundException e) {
// We're done here
return true;
} catch (DockerException e) {
if ((e instanceof DockerRequestException)
&& ((DockerRequestException) e)
.message()
.contains("Driver btrfs failed to remove root filesystem")) {
// Workaround btrfs issue where removing containers throws an exception,
// but succeeds anyway.
return true;
} else {
return null;
}
}
}
});
}
@Before
public void dockerSetup() throws Exception {
final String portRange = System.getenv("DOCKER_PORT_RANGE");
final AllocatedPort allocatedPort;
final int probePort;
if (portRange != null) {
final String[] parts = portRange.split(":", 2);
dockerPortRange = Range.closedOpen(Integer.valueOf(parts[0]), Integer.valueOf(parts[1]));
allocatedPort =
Polling.await(
LONG_WAIT_SECONDS,
SECONDS,
new Callable<AllocatedPort>() {
@Override
public AllocatedPort call() throws Exception {
final int port =
ThreadLocalRandom.current()
.nextInt(
dockerPortRange.lowerEndpoint(), dockerPortRange.upperEndpoint());
return temporaryPorts.tryAcquire("docker-probe", port);
}
});
probePort = allocatedPort.port();
} else {
dockerPortRange = temporaryPorts.localPortRange("docker", 10);
probePort = dockerPortRange().lowerEndpoint();
allocatedPort = null;
}
try {
assertDockerReachable(probePort);
} finally {
if (allocatedPort != null) {
allocatedPort.release();
}
}
}
protected HostStatus awaitHostStatus(
final String name, final HostStatus.Status status, final int timeout, final TimeUnit timeUnit)
throws Exception {
return Polling.await(
timeout,
timeUnit,
new Callable<HostStatus>() {
@Override
public HostStatus call() throws Exception {
final String output = cli("hosts", name, "--json");
final Map<String, HostStatus> statuses;
try {
statuses = Json.read(output, new TypeReference<Map<String, HostStatus>>() {});
} catch (IOException e) {
return null;
}
final HostStatus hostStatus = statuses.get(name);
if (hostStatus == null) {
return null;
}
return (hostStatus.getStatus() == status) ? hostStatus : null;
}
});
}
private void assertDockerReachable(final int probePort) throws Exception {
try (final DockerClient docker = getNewDockerClient()) {
// Pull our base images
try {
docker.inspectImage(BUSYBOX);
} catch (ImageNotFoundException e) {
docker.pull(BUSYBOX);
}
try {
docker.inspectImage(ALPINE);
} catch (ImageNotFoundException e) {
docker.pull(ALPINE);
}
// Start a container with an exposed port
final HostConfig hostConfig =
HostConfig.builder()
.portBindings(
ImmutableMap.of("4711/tcp", singletonList(PortBinding.of("0.0.0.0", probePort))))
.build();
final ContainerConfig config =
ContainerConfig.builder()
.image(BUSYBOX)
.cmd("nc", "-p", "4711", "-lle", "cat")
.exposedPorts(ImmutableSet.of("4711/tcp"))
.hostConfig(hostConfig)
.build();
final ContainerCreation creation = docker.createContainer(config, testTag + "-probe");
final String containerId = creation.id();
docker.startContainer(containerId);
// Wait for container to come up
Polling.await(
5,
SECONDS,
new Callable<Object>() {
@Override
public Object call() throws Exception {
final ContainerInfo info = docker.inspectContainer(containerId);
return info.state().running() ? true : null;
}
});
log.info("Verifying that docker containers are reachable");
try {
Polling.awaitUnchecked(
5,
SECONDS,
new Callable<Object>() {
@Override
public Object call() throws Exception {
log.info("Probing: {}:{}", DOCKER_HOST.address(), probePort);
try (final Socket ignored = new Socket(DOCKER_HOST.address(), probePort)) {
return true;
} catch (IOException e) {
return false;
}
}
});
} catch (TimeoutException e) {
fail(
"Please ensure that DOCKER_HOST is set to an address that where containers can "
+ "be reached. If docker is running in a local VM, DOCKER_HOST must be set to the "
+ "address of that VM. If docker can only be reached on a limited port range, "
+ "set the environment variable DOCKER_PORT_RANGE=start:end");
}
docker.killContainer(containerId);
}
}
