@Test(groups = {"smoke", "system"})
 public void testRabbitMQNodesUpAndRunning() throws Exception {
   for (String host : m_hosts) {
     assertTrue(
         "RabbitMQ not running on host " + host + " according to ender",
         enderStatusCheck(host, true, "rabbitmq"));
     SSH remoteSSH = getSSH(host);
     checkClusterStatus(remoteSSH, host);
     checkHApolicy(remoteSSH, host);
     remoteSSH.close();
   }
 }
  private void checkHApolicy(SSH remoteSSH, String host) throws Exception {
    boolean checkThree = false;

    // make sure HA policy is applied
    //            Listing policies ...
    //            /	ha-roundrobin	roundrobin:.*	{"ha-mode":"all"}	0
    //            ...done.
    assertEquals(
        "didn't run ha policy command successfully on host " + host,
        0,
        remoteSSH.execCommand("/opt/rabbitmq/current/sbin/rabbitmqctl list_policies", 15000));
    assertEquals("There is stderr on host " + host + "!", 0, remoteSSH.readStderr().length);
    String[] policiesOutput = remoteSSH.readStdout();
    for (String line : policiesOutput) {
      if (line.contains("ha-roundrobin")) {
        assertTrue(
            "should have HA policy output on host " + host,
            line.contains("roundrobin:.*") && line.contains("ha-mode\":\"all\""));
        checkThree = true;
      }
    }
    assertTrue("HA policy not found on host " + host + "!", checkThree);
  }
  private void checkClusterStatus(SSH remoteSSH, String host) throws Exception {
    // make sure cluster is the same for each node
    //            [{nodes,[{disc,['rabbit@aggregator-1','rabbit@cassandra-1']}]},
    //            {running_nodes,['rabbit@aggregator-1','rabbit@cassandra-1']},
    //            {partitions,[]}]
    //            ...done.

    int result = remoteSSH.execCommand("/opt/rabbitmq/current/sbin/rabbitmqctl cluster_status");

    // sometimes fails the first time... try again and if it fails again then fail out of the test
    if (result != 0) {
      String errMsg =
          "cluster_status failed the first time: stdout = "
              + remoteSSH.stdoutAsString()
              + "\nstderr = "
              + remoteSSH.readStderr()
              + "\nretrying...";
      // System.out.println(errMsg);
      LOGGER.warn(errMsg);
      assertEquals(
          "didn't run cluster status command successfully 2nd time on host " + host,
          0,
          remoteSSH.execCommand("/opt/rabbitmq/current/sbin/rabbitmqctl cluster_status", 15000));
    }

    assertEquals("There is stderr on host " + host + "!", 0, remoteSSH.readStderr().length);
    String[] clusterOutput = remoteSSH.readStdout();
    List<String> externalHosts = new LinkedList<String>();
    externalHosts.addAll(m_hosts);
    List<String> runningHosts = new LinkedList<String>();
    runningHosts.addAll(externalHosts);
    for (String line : clusterOutput) {
      if (line.contains("[{nodes,[{disc,[")) {
        String[] split = getRabbitHosts(line);
        for (String s : split) {
          String exHost = getSystemOfRecord().getExternalAddress(s);
          externalHosts.remove(exHost);
        }
      } else if (line.contains("running_nodes")) {
        String[] split = getRabbitHosts(line);
        for (String s : split) {
          String exHost = getSystemOfRecord().getExternalAddress(s);
          runningHosts.remove(exHost);
        }
      }
    }
    assertTrue("Did not find both hosts listed in nodes line.", externalHosts.size() == 0);
    assertTrue("Did not find both hosts listed in running line.", runningHosts.size() == 0);
  }