protected void waitForMasterToBeFullyUp() throws Exception {
    log.debug("waitForMasterToBeFullyUp: beginning wait loop");
    Polling.await(
        WAIT_TIMEOUT_SECONDS,
        SECONDS,
        new Callable<Object>() {
          @Override
          public Object call() {
            try {
              // While MasterService will start listening for http requests on the main and admin
              // ports
              // as soon as it is started (without waiting for ZK to be available), the Healthcheck
              // registered for Zookeeper connectivity will cause the HealthcheckServlet to not
              // return
              // 200 OK until ZK is connected to (and even better, until *everything* is healthy).
              final HttpGet request = new HttpGet(masterAdminEndpoint + "/healthcheck");

              try (CloseableHttpResponse response = httpClient.execute(request)) {
                final int status = response.getStatusLine().getStatusCode();
                log.debug("waitForMasterToBeFullyUp: healthcheck endpoint returned {}", status);
                return status == HttpStatus.SC_OK;
              }
            } catch (Exception e) {
              return null;
            }
          }
        });
  }
 protected TaskStatus awaitJobThrottle(
     final HeliosClient client,
     final String host,
     final JobId jobId,
     final ThrottleState throttled,
     final int timeout,
     final TimeUnit timeunit)
     throws Exception {
   return Polling.await(
       timeout,
       timeunit,
       new Callable<TaskStatus>() {
         @Override
         public TaskStatus call() throws Exception {
           final HostStatus hostStatus = getOrNull(client.hostStatus(host));
           if (hostStatus == null) {
             return null;
           }
           final TaskStatus taskStatus = hostStatus.getStatuses().get(jobId);
           return (taskStatus != null && taskStatus.getThrottled() == throttled)
               ? taskStatus
               : null;
         }
       });
 }
  protected DeploymentGroupStatus awaitDeploymentGroupStatus(
      final HeliosClient client, final String name, final DeploymentGroupStatus.State state)
      throws Exception {
    return Polling.await(
        LONG_WAIT_SECONDS,
        SECONDS,
        new Callable<DeploymentGroupStatus>() {
          @Override
          public DeploymentGroupStatus call() throws Exception {
            final DeploymentGroupStatusResponse response =
                getOrNull(client.deploymentGroupStatus(name));

            if (response != null) {
              final DeploymentGroupStatus status = response.getDeploymentGroupStatus();
              if (status.getState().equals(state)) {
                return status;
              } else if (status.getState().equals(DeploymentGroupStatus.State.FAILED)) {
                assertEquals(state, status.getState());
              }
            }

            return null;
          }
        });
  }
 protected TaskStatus awaitTaskState(
     final JobId jobId, final String host, final TaskStatus.State state) throws Exception {
   return Polling.await(
       LONG_WAIT_SECONDS,
       SECONDS,
       new Callable<TaskStatus>() {
         @Override
         public TaskStatus call() throws Exception {
           final String output = cli("status", "--json", "--job", jobId.toString());
           final Map<JobId, JobStatus> statusMap;
           try {
             statusMap = Json.read(output, new TypeReference<Map<JobId, JobStatus>>() {});
           } catch (IOException e) {
             return null;
           }
           final JobStatus status = statusMap.get(jobId);
           if (status == null) {
             return null;
           }
           final TaskStatus taskStatus = status.getTaskStatuses().get(host);
           if (taskStatus == null) {
             return null;
           }
           if (taskStatus.getState() != state) {
             return null;
           }
           return taskStatus;
         }
       });
 }
Example #5
0
 private byte[] recv(final int port, final int n) throws Exception {
   checkArgument(n > 0, "n must be > 0");
   return Polling.await(
       LONG_WAIT_SECONDS,
       SECONDS,
       new Callable<byte[]>() {
         @Override
         public byte[] call() {
           try (final Socket s = new Socket(DOCKER_HOST.address(), port)) {
             final byte[] bytes = new byte[n];
             final InputStream is = s.getInputStream();
             final int first = is.read();
             // Check if the uml kernel slirp driver did an accept->close on us,
             // i.e. the actual listener is not up yet
             if (first == -1) {
               return null;
             }
             bytes[0] = (byte) first;
             for (int i = 1; i < n; i++) {
               bytes[i] = (byte) is.read();
             }
             return bytes;
           } catch (IOException e) {
             return null;
           }
         }
       });
 }
 protected void awaitHostRegistered(
     final HeliosClient client, final String host, final int timeout, final TimeUnit timeUnit)
     throws Exception {
   Polling.await(
       timeout,
       timeUnit,
       new Callable<HostStatus>() {
         @Override
         public HostStatus call() throws Exception {
           return getOrNull(client.hostStatus(host));
         }
       });
 }
 protected void awaitHostRegistered(final String name, final long timeout, final TimeUnit timeUnit)
     throws Exception {
   Polling.await(
       timeout,
       timeUnit,
       new Callable<Object>() {
         @Override
         public Object call() throws Exception {
           final String output = cli("hosts", "-q");
           return output.contains(name) ? true : null;
         }
       });
 }
 private void awaitHealthy(
     final ZooKeeperHealthChecker hc, final int duration, final TimeUnit timeUnit)
     throws Exception {
   Polling.await(
       duration,
       timeUnit,
       new Callable<Object>() {
         @Override
         public Object call() throws Exception {
           return hc.check().isHealthy() ? true : null;
         }
       });
 }
 private void awaitState(final String state, final int duration, final TimeUnit timeUnit)
     throws Exception {
   Polling.await(
       duration,
       timeUnit,
       new Callable<Object>() {
         @Override
         public Object call() throws Exception {
           final List<Event> events = riemannClient.getEvents();
           if (events.isEmpty()) {
             return null;
           }
           final Event event = events.get(0);
           return state.equals(event.getState()) ? event : null;
         }
       });
 }
 protected void awaitTaskGone(
     final HeliosClient client,
     final String host,
     final JobId jobId,
     final long timeout,
     final TimeUnit timeunit)
     throws Exception {
   Polling.await(
       timeout,
       timeunit,
       new Callable<Boolean>() {
         @Override
         public Boolean call() throws Exception {
           final HostStatus hostStatus = getOrNull(client.hostStatus(host));
           final TaskStatus taskStatus = hostStatus.getStatuses().get(jobId);
           final Deployment deployment = hostStatus.getJobs().get(jobId);
           return taskStatus == null && deployment == null ? true : null;
         }
       });
 }
 protected HostStatus awaitHostStatus(
     final HeliosClient client,
     final String host,
     final HostStatus.Status status,
     final int timeout,
     final TimeUnit timeUnit)
     throws Exception {
   return Polling.await(
       timeout,
       timeUnit,
       new Callable<HostStatus>() {
         @Override
         public HostStatus call() throws Exception {
           final HostStatus hostStatus = getOrNull(client.hostStatus(host));
           if (hostStatus == null) {
             return null;
           }
           return (hostStatus.getStatus() == status) ? hostStatus : null;
         }
       });
 }
  protected static void removeContainer(final DockerClient dockerClient, final String containerId)
      throws Exception {
    // Work around docker sometimes failing to remove a container directly after killing it
    Polling.await(
        1,
        MINUTES,
        new Callable<Object>() {
          @Override
          public Object call() throws Exception {
            try {
              dockerClient.killContainer(containerId);
            } catch (DockerRequestException e) {
              if (e.message().contains("is not running")) {
                // Container already isn't running. So we continue.
              } else {
                throw e;
              }
            }

            try {
              dockerClient.removeContainer(containerId);
              return true;
            } catch (ContainerNotFoundException e) {
              // We're done here
              return true;
            } catch (DockerException e) {
              if ((e instanceof DockerRequestException)
                  && ((DockerRequestException) e)
                      .message()
                      .contains("Driver btrfs failed to remove root filesystem")) {
                // Workaround btrfs issue where removing containers throws an exception,
                // but succeeds anyway.
                return true;
              } else {
                return null;
              }
            }
          }
        });
  }
  @Before
  public void dockerSetup() throws Exception {
    final String portRange = System.getenv("DOCKER_PORT_RANGE");

    final AllocatedPort allocatedPort;
    final int probePort;
    if (portRange != null) {
      final String[] parts = portRange.split(":", 2);
      dockerPortRange = Range.closedOpen(Integer.valueOf(parts[0]), Integer.valueOf(parts[1]));
      allocatedPort =
          Polling.await(
              LONG_WAIT_SECONDS,
              SECONDS,
              new Callable<AllocatedPort>() {
                @Override
                public AllocatedPort call() throws Exception {
                  final int port =
                      ThreadLocalRandom.current()
                          .nextInt(
                              dockerPortRange.lowerEndpoint(), dockerPortRange.upperEndpoint());
                  return temporaryPorts.tryAcquire("docker-probe", port);
                }
              });
      probePort = allocatedPort.port();
    } else {
      dockerPortRange = temporaryPorts.localPortRange("docker", 10);
      probePort = dockerPortRange().lowerEndpoint();
      allocatedPort = null;
    }

    try {
      assertDockerReachable(probePort);
    } finally {
      if (allocatedPort != null) {
        allocatedPort.release();
      }
    }
  }
 protected HostStatus awaitHostStatus(
     final String name, final HostStatus.Status status, final int timeout, final TimeUnit timeUnit)
     throws Exception {
   return Polling.await(
       timeout,
       timeUnit,
       new Callable<HostStatus>() {
         @Override
         public HostStatus call() throws Exception {
           final String output = cli("hosts", name, "--json");
           final Map<String, HostStatus> statuses;
           try {
             statuses = Json.read(output, new TypeReference<Map<String, HostStatus>>() {});
           } catch (IOException e) {
             return null;
           }
           final HostStatus hostStatus = statuses.get(name);
           if (hostStatus == null) {
             return null;
           }
           return (hostStatus.getStatus() == status) ? hostStatus : null;
         }
       });
 }
  private void assertDockerReachable(final int probePort) throws Exception {
    try (final DockerClient docker = getNewDockerClient()) {
      // Pull our base images
      try {
        docker.inspectImage(BUSYBOX);
      } catch (ImageNotFoundException e) {
        docker.pull(BUSYBOX);
      }

      try {
        docker.inspectImage(ALPINE);
      } catch (ImageNotFoundException e) {
        docker.pull(ALPINE);
      }

      // Start a container with an exposed port
      final HostConfig hostConfig =
          HostConfig.builder()
              .portBindings(
                  ImmutableMap.of("4711/tcp", singletonList(PortBinding.of("0.0.0.0", probePort))))
              .build();
      final ContainerConfig config =
          ContainerConfig.builder()
              .image(BUSYBOX)
              .cmd("nc", "-p", "4711", "-lle", "cat")
              .exposedPorts(ImmutableSet.of("4711/tcp"))
              .hostConfig(hostConfig)
              .build();
      final ContainerCreation creation = docker.createContainer(config, testTag + "-probe");
      final String containerId = creation.id();
      docker.startContainer(containerId);

      // Wait for container to come up
      Polling.await(
          5,
          SECONDS,
          new Callable<Object>() {
            @Override
            public Object call() throws Exception {
              final ContainerInfo info = docker.inspectContainer(containerId);
              return info.state().running() ? true : null;
            }
          });

      log.info("Verifying that docker containers are reachable");
      try {
        Polling.awaitUnchecked(
            5,
            SECONDS,
            new Callable<Object>() {
              @Override
              public Object call() throws Exception {
                log.info("Probing: {}:{}", DOCKER_HOST.address(), probePort);
                try (final Socket ignored = new Socket(DOCKER_HOST.address(), probePort)) {
                  return true;
                } catch (IOException e) {
                  return false;
                }
              }
            });
      } catch (TimeoutException e) {
        fail(
            "Please ensure that DOCKER_HOST is set to an address that where containers can "
                + "be reached. If docker is running in a local VM, DOCKER_HOST must be set to the "
                + "address of that VM. If docker can only be reached on a limited port range, "
                + "set the environment variable DOCKER_PORT_RANGE=start:end");
      }

      docker.killContainer(containerId);
    }
  }