@Test
  @OperateOnDeployment(DEPLOYMENT_1)
  public void testSerialized(@ArquillianResource(SimpleServlet.class) URL baseURL)
      throws IOException, URISyntaxException {

    // returns the URL of the deployment (http://127.0.0.1:8180/distributable)
    URI uri = SimpleServlet.createURI(baseURL);

    try (CloseableHttpClient client = TestHttpClientUtils.promiscuousCookieHttpClient()) {
      HttpResponse response = client.execute(new HttpGet(uri));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(1, Integer.parseInt(response.getFirstHeader("value").getValue()));
        Assert.assertFalse(Boolean.valueOf(response.getFirstHeader("serialized").getValue()));
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      response = client.execute(new HttpGet(uri));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(2, Integer.parseInt(response.getFirstHeader("value").getValue()));
        // This won't be true unless we have somewhere to which to replicate
        Assert.assertTrue(Boolean.valueOf(response.getFirstHeader("serialized").getValue()));
      } finally {
        HttpClientUtils.closeQuietly(response);
      }
    }
  }
  @Test
  public void testSingletonService(@ArquillianResource(MyServiceServlet.class) URL baseURL)
      throws IOException, URISyntaxException {

    // URLs look like "http://IP:PORT/singleton/service"
    URI defaultURI = MyServiceServlet.createURI(baseURL, MyServiceActivator.DEFAULT_SERVICE_NAME);
    URI quorumURI = MyServiceServlet.createURI(baseURL, MyServiceActivator.QUORUM_SERVICE_NAME);

    try (CloseableHttpClient client = TestHttpClientUtils.promiscuousCookieHttpClient()) {
      HttpResponse response = client.execute(new HttpGet(defaultURI));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(NODE_1, response.getFirstHeader("node").getValue());
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Service should be started regardless of whether a quorum was required.
      response = client.execute(new HttpGet(quorumURI));
      try {
        assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(NODE_1, response.getFirstHeader("node").getValue());
      } finally {
        HttpClientUtils.closeQuietly(response);
      }
    }
  }
  @Test
  @OperateOnDeployment(DEPLOYMENT_2) // For change, operate on the 2nd deployment first
  public void testSessionReplication(
      @ArquillianResource(SimpleServlet.class) @OperateOnDeployment(DEPLOYMENT_1) URL baseURL1,
      @ArquillianResource(SimpleServlet.class) @OperateOnDeployment(DEPLOYMENT_2) URL baseURL2)
      throws IOException, URISyntaxException {

    URI url1 = SimpleServlet.createURI(baseURL1);
    URI url2 = SimpleServlet.createURI(baseURL2);

    try (CloseableHttpClient client = TestHttpClientUtils.promiscuousCookieHttpClient()) {
      HttpResponse response = client.execute(new HttpGet(url1));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(1, Integer.parseInt(response.getFirstHeader("value").getValue()));
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Lets do this twice to have more debug info if failover is slow.
      response = client.execute(new HttpGet(url1));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(2, Integer.parseInt(response.getFirstHeader("value").getValue()));
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Lets wait for the session to replicate
      waitForReplication(GRACE_TIME_TO_REPLICATE);

      // Now check on the 2nd server
      response = client.execute(new HttpGet(url2));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(3, Integer.parseInt(response.getFirstHeader("value").getValue()));
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Lets do one more check.
      response = client.execute(new HttpGet(url2));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(4, Integer.parseInt(response.getFirstHeader("value").getValue()));
      } finally {
        HttpClientUtils.closeQuietly(response);
      }
    }
  }
  private void abstractGracefulServe(URL baseURL, boolean undeployOnly)
      throws URISyntaxException, IOException, InterruptedException {

    try (CloseableHttpClient client = TestHttpClientUtils.promiscuousCookieHttpClient()) {
      URI uri = SimpleServlet.createURI(baseURL);

      // Make sure a normal request will succeed
      HttpResponse response = client.execute(new HttpGet(uri));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Send a long request - in parallel
      URI longRunningURI = SimpleServlet.createURI(baseURL, REQUEST_DURATION);
      ExecutorService executor = Executors.newSingleThreadExecutor();
      Future<HttpResponse> future = executor.submit(new RequestTask(client, longRunningURI));

      // Make sure long request has started
      Thread.sleep(1000);

      if (undeployOnly) {
        // Undeploy the app only.
        undeploy(DEPLOYMENT_1);
      } else {
        // Shutdown server.
        stop(CONTAINER_1);
      }

      // Get result of long request
      // This request should succeed since it initiated before server shutdown
      try {
        response = future.get();
        try {
          Assert.assertEquals(
              "Request should succeed since it initiated before undeply or shutdown.",
              HttpServletResponse.SC_OK,
              response.getStatusLine().getStatusCode());
        } finally {
          HttpClientUtils.closeQuietly(response);
        }
      } catch (ExecutionException e) {
        e.printStackTrace(System.err);
        Assert.fail(e.getCause().getMessage());
      }

      if (undeployOnly) {
        // If we are only undeploying, then subsequent requests should return 404.
        response = client.execute(new HttpGet(uri));
        try {
          Assert.assertEquals(
              "If we are only undeploying, then subsequent requests should return 404.",
              HttpServletResponse.SC_NOT_FOUND,
              response.getStatusLine().getStatusCode());
        } finally {
          HttpClientUtils.closeQuietly(response);
        }
      }
    }
  }
  private void testFailover(Lifecycle lifecycle, URL baseURL1, URL baseURL2) throws Exception {
    HttpClient client = TestHttpClientUtils.promiscuousCookieHttpClient();

    URI uri1 = SimpleServlet.createURI(baseURL1);
    URI uri2 = SimpleServlet.createURI(baseURL2);

    this.establishTopology(baseURL1, NODES);

    try {
      HttpResponse response = client.execute(new HttpGet(uri1));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            1, Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        Assert.assertEquals(NODE_1, entry.getValue());
        Assert.assertEquals(
            entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Let's do this twice to have more debug info if failover is slow.
      response = client.execute(new HttpGet(uri1));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            2, Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        if (entry != null) {
          Assert.assertEquals(NODE_1, entry.getValue());
          Assert.assertEquals(
              entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
        }
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Gracefully undeploy from/shutdown the 1st container.
      lifecycle.stop(NODE_1);

      this.establishTopology(baseURL2, NODE_2);

      // Now check on the 2nd server

      response = client.execute(new HttpGet(uri2));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            "Session failed to replicate after container 1 was shutdown.",
            3,
            Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        Assert.assertEquals(NODE_2, entry.getValue());
        Assert.assertEquals(
            entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Let's do one more check.
      response = client.execute(new HttpGet(uri2));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            4, Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        if (entry != null) {
          Assert.assertEquals(NODE_2, entry.getValue());
          Assert.assertEquals(
              entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
        }
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      lifecycle.start(NODE_1);

      this.establishTopology(baseURL2, NODES);

      response = client.execute(new HttpGet(uri2));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            "Session failed to replicate after container 1 was brough up.",
            5,
            Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        if (entry != null) {
          Assert.assertEquals(NODE_1, entry.getValue());
          Assert.assertEquals(
              entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
        }
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // The previous and next requests intentially hit the non-owning node
      this.nonOwnerTask.run();

      // Let's do this twice to have more debug info if failover is slow.
      response = client.execute(new HttpGet(uri2));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            6, Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        if (entry != null) {
          Assert.assertEquals(NODE_1, entry.getValue());
          Assert.assertEquals(
              entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
        }
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Until graceful undeploy is supported, we need to wait for replication to complete before
      // undeploy (WFLY-6769).
      if (lifecycle instanceof RedeployLifecycle) {
        Thread.sleep(GRACE_TIME_TO_REPLICATE);
      }

      // Gracefully undeploy from/shutdown the 1st container.
      lifecycle.stop(NODE_2);

      this.establishTopology(baseURL1, NODE_1);

      // Now check on the 2nd server

      response = client.execute(new HttpGet(uri1));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            "Session failed to replicate after container 1 was shutdown.",
            7,
            Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        if (entry != null) {
          Assert.assertEquals(NODE_1, entry.getValue());
          Assert.assertEquals(
              entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
        }
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Let's do one more check.
      response = client.execute(new HttpGet(uri1));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            8, Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        if (entry != null) {
          Assert.assertEquals(NODE_1, entry.getValue());
          Assert.assertEquals(
              entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
        }
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      lifecycle.start(NODE_2);

      this.establishTopology(baseURL1, NODES);

      response = client.execute(new HttpGet(uri1));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            "Session failed to replicate after container 1 was brought up.",
            9,
            Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        if (entry != null) {
          Assert.assertEquals(NODE_2, entry.getValue());
          Assert.assertEquals(
              entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
        }
      } finally {
        HttpClientUtils.closeQuietly(response);
      }

      // Let's do this twice to have more debug info if failover is slow.
      response = client.execute(new HttpGet(uri1));
      try {
        Assert.assertEquals(HttpServletResponse.SC_OK, response.getStatusLine().getStatusCode());
        Assert.assertEquals(
            10, Integer.parseInt(response.getFirstHeader(SimpleServlet.VALUE_HEADER).getValue()));
        Map.Entry<String, String> entry = parseSessionRoute(response);
        if (entry != null) {
          Assert.assertEquals(NODE_2, entry.getValue());
          Assert.assertEquals(
              entry.getKey(), response.getFirstHeader(SimpleServlet.SESSION_ID_HEADER).getValue());
        }
      } finally {
        HttpClientUtils.closeQuietly(response);
      }
    } finally {
      HttpClientUtils.closeQuietly(client);
    }
  }