protected void ensurePoolIntegrity(
      Connection conn, String masterIp, String username, Queue<String> password, int wait) {
    try {
      // try recoverSlave first
      Set<Host> rcSlaves = Pool.recoverSlaves(conn);
      // wait 10 second
      forceSleep(10);
      for (Host slave : rcSlaves) {
        for (int i = 0; i < 30; i++) {
          Connection slaveConn = null;
          try {

            String slaveIp = slave.getAddress(conn);
            s_logger.debug("Logging on as the slave to " + slaveIp);
            slaveConn = new Connection(getURL(slaveIp), 10);
            slaveLocalLoginWithPassword(slaveConn, username, password);
            Pool.Record pr = getPoolRecord(slaveConn);
            String mIp = pr.master.getAddress(slaveConn);
            if (mIp.trim().equals(masterIp.trim())) {
              break;
            }
          } catch (Exception e) {
          } finally {
            localLogout(slaveConn);
            slaveConn = null;
          }
          // wait 2 second
          forceSleep(2);
        }
      }
      // then try emergency reset master
      Set<Host> slaves = Host.getAll(conn);
      for (Host slave : slaves) {
        String slaveIp = slave.getAddress(conn);
        Connection slaveConn = null;
        try {
          s_logger.debug("Logging on as the slave to " + slaveIp);

          slaveConn = new Connection(getURL(slaveIp), 10);
          slaveLocalLoginWithPassword(slaveConn, username, password);
          Pool.Record slavePoolr = getPoolRecord(slaveConn);
          String ip = slavePoolr.master.getAddress(slaveConn);
          if (!masterIp.trim().equals(ip.trim())) {
            PoolEmergencyResetMaster(slaveIp, masterIp, username, password);
          }
        } catch (Exception e) {
          s_logger.debug("Unable to login to slave " + slaveIp + " error " + e.getMessage());
        } finally {
          localLogout(slaveConn);
          slaveConn = null;
        }
      }
    } catch (Exception e) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Catch " + e.getClass().getName() + " due to " + e.toString());
      }
    }
  }
  private void PoolEmergencyResetMaster(
      String slaveIp, String masterIp, String username, Queue<String> password) {
    if (!s_managePool) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Don't manage pool on error so sleeping for " + s_sleepOnError);
        try {
          Thread.sleep(s_sleepOnError);
        } catch (InterruptedException ie) {
        }
      }
      return;
    }

    Connection slaveConn = null;
    try {
      s_logger.debug("Trying to reset master of slave " + slaveIp + " to " + masterIp);
      slaveConn = new Connection(getURL(slaveIp), 10);
      slaveLocalLoginWithPassword(slaveConn, username, password);
      Pool.emergencyResetMaster(slaveConn, masterIp);
      forceSleep(10);
      for (int i = 0; i < 30; i++) {
        try {
          slaveLocalLoginWithPassword(slaveConn, username, password);
          Pool.Record pr = getPoolRecord(slaveConn);
          String mIp = pr.master.getAddress(slaveConn);
          if (mIp.trim().equals(masterIp.trim())) {
            s_logger.debug("Succeeded to reset master of slave " + slaveIp + " to " + masterIp);
            return;
          }
        } catch (Exception e) {
        } finally {
          localLogout(slaveConn);
          slaveConn = null;
        }
        // wait 2 second
        forceSleep(2);
      }
      throw new CloudRuntimeException(
          "Unable to reset master of slave " + slaveIp + " to " + masterIp + "after 30 retry");
    } catch (Exception e) {
      throw new CloudRuntimeException(
          "Unable to reset master of slave "
              + slaveIp
              + " to "
              + masterIp
              + " due to "
              + e.toString());
    } finally {
      localLogout(slaveConn);
      slaveConn = null;
    }
  }
  void PoolEmergencyTransitionToMaster(String slaveIp, String username, Queue<String> password) {
    if (!s_managePool) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Don't manage pool on error so sleeping for " + s_sleepOnError);
        try {
          Thread.sleep(s_sleepOnError);
        } catch (InterruptedException ie) {
        }
      }
      return;
    }

    Connection slaveConn = null;
    Connection c = null;
    try {
      s_logger.debug("Trying to transition master to " + slaveIp);
      slaveConn = new Connection(getURL(slaveIp), 10);
      slaveLocalLoginWithPassword(slaveConn, username, password);
      Pool.emergencyTransitionToMaster(slaveConn);
      // restart xapi in 10 sec
      forceSleep(10);
      // check if the master of this host is set correctly.
      c = new Connection(getURL(slaveIp), 10);
      for (int i = 0; i < 30; i++) {
        try {
          loginWithPassword(c, username, password, APIVersion.latest().toString());
          s_logger.debug("Succeeded to transition master to " + slaveIp);
          return;
        } catch (Types.HostIsSlave e) {
          s_logger.debug("HostIsSlave: Still waiting for the conversion to the master " + slaveIp);
        } catch (Exception e) {
          s_logger.debug("Exception: Still waiting for the conversion to the master");
        }
        forceSleep(2);
      }
      throw new RuntimeException("EmergencyTransitionToMaster failed after retry 30 times");
    } catch (Exception e) {
      throw new RuntimeException("EmergencyTransitionToMaster failed due to " + e.getMessage());
    } finally {
      localLogout(slaveConn);
      slaveConn = null;
      if (c != null) {
        try {
          Session.logout(c);
          c.dispose();
        } catch (Exception e) {
        }
      }
    }
  }
  protected void join(Connection conn, String masterIp, String username, Queue<String> password)
      throws BadServerResponse, XenAPIException, XmlRpcException,
          Types.JoiningHostCannotContainSharedSrs {

    boolean logged_in = false;
    Exception ex = null;
    while (!logged_in) {
      try {
        Pool.join(conn, masterIp, username, password.peek());
        logged_in = true;
      } catch (BadServerResponse e) {
        logged_in = false;
        ex = e;
      } catch (XenAPIException e) {
        logged_in = false;
        ex = e;
      } catch (XmlRpcException e) {
        logged_in = false;
        ex = e;
      }
      if (logged_in && conn != null) {
        break;
      } else {
        if (password.size() > 1) {
          password.remove();
          continue;
        } else {
          // the last password did not work leave it and flag error
          if (ex instanceof BadServerResponse) {
            throw (BadServerResponse) ex;
          } else if (ex instanceof XmlRpcException) {
            throw (XmlRpcException) ex;
          } else if (ex instanceof Types.SessionAuthenticationFailed) {
            throw (Types.SessionAuthenticationFailed) ex;
          } else if (ex instanceof XenAPIException) {
            throw (XenAPIException) ex;
          }
          break;
        }
      }
    }
  }
  public static Pool.Record getPoolRecord(Connection conn) throws XmlRpcException, XenAPIException {
    Map<Pool, Pool.Record> pools = Pool.getAllRecords(conn);
    assert pools.size() == 1 : "Pool size is not one....hmmm....wth? " + pools.size();

    return pools.values().iterator().next();
  }
  public void switchMaster(
      String slaveIp,
      String poolUuid,
      Connection conn,
      Host host,
      String username,
      Queue<String> password,
      int wait)
      throws XmlRpcException, XenAPIException {
    synchronized (poolUuid.intern()) {
      String masterIp = host.getAddress(conn);
      s_logger.debug("Designating the new master to " + masterIp);
      Pool.designateNewMaster(conn, host);
      Connection slaveConn = null;
      Connection masterConn = null;
      int retry = 30;
      for (int i = 0; i < retry; i++) {
        forceSleep(5);
        try {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug("Logging on as the slave to " + slaveIp);
          }
          slaveConn = null;
          masterConn = null;
          Session slaveSession = null;

          slaveConn = new Connection(getURL(slaveIp), 10);
          slaveSession = slaveLocalLoginWithPassword(slaveConn, username, password);

          if (s_logger.isDebugEnabled()) {
            s_logger.debug("Slave logon successful. session= " + slaveSession);
          }

          Pool.Record pr = getPoolRecord(slaveConn);
          Host master = pr.master;
          String ma = master.getAddress(slaveConn);
          if (!ma.trim().equals(masterIp.trim())) {
            continue;
          }
          s_logger.debug("Logging on as the master to " + masterIp);
          masterConn = new Connection(getURL(masterIp), 10);
          loginWithPassword(masterConn, username, password, APIVersion.latest().toString());
          removeConnect(poolUuid);
          ensurePoolIntegrity(masterConn, masterIp, username, password, wait);
          return;
        } catch (Types.HostIsSlave e) {
          s_logger.debug("HostIsSlaveException: Still waiting for the conversion to the master");
        } catch (XmlRpcException e) {
          s_logger.debug(
              "XmlRpcException: Still waiting for the conversion to the master " + e.getMessage());
        } catch (Exception e) {
          s_logger.debug(
              "Exception: Still waiting for the conversion to the master" + e.getMessage());
        } finally {
          if (masterConn != null) {
            try {
              Session.logout(masterConn);
            } catch (Exception e) {
              s_logger.debug("Unable to log out of session: " + e.getMessage());
            }
            masterConn.dispose();
            masterConn = null;
          }
          localLogout(slaveConn);
          slaveConn = null;
        }
      }
      throw new CloudRuntimeException(
          "Unable to logon to the new master after " + retry + " retries");
    }
  }