public String getMasterIp(String ip, String username, Queue<String> password)
      throws XenAPIException {
    Connection slaveConn = null;
    try {
      slaveConn = new Connection(getURL(ip), 10);
      slaveLocalLoginWithPassword(slaveConn, username, password);

      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Slave logon to " + ip);
      }
      String masterIp = null;
      Pool.Record pr = getPoolRecord(slaveConn);
      Host master = pr.master;
      masterIp = master.getAddress(slaveConn);
      return masterIp;
    } catch (Types.SessionAuthenticationFailed e) {
      s_logger.debug("Failed to slave local login to " + ip + " due to " + e.toString());
      throw e;
    } catch (Exception e) {
      s_logger.debug("Failed to slave local login to " + ip + " due to " + e.toString());
    } finally {
      localLogout(slaveConn);
      slaveConn = null;
    }
    throw new RuntimeException("can not get master ip");
  }
  protected void ensurePoolIntegrity(
      Connection conn, String masterIp, String username, Queue<String> password, int wait) {
    try {
      // try recoverSlave first
      Set<Host> rcSlaves = Pool.recoverSlaves(conn);
      // wait 10 second
      forceSleep(10);
      for (Host slave : rcSlaves) {
        for (int i = 0; i < 30; i++) {
          Connection slaveConn = null;
          try {

            String slaveIp = slave.getAddress(conn);
            s_logger.debug("Logging on as the slave to " + slaveIp);
            slaveConn = new Connection(getURL(slaveIp), 10);
            slaveLocalLoginWithPassword(slaveConn, username, password);
            Pool.Record pr = getPoolRecord(slaveConn);
            String mIp = pr.master.getAddress(slaveConn);
            if (mIp.trim().equals(masterIp.trim())) {
              break;
            }
          } catch (Exception e) {
          } finally {
            localLogout(slaveConn);
            slaveConn = null;
          }
          // wait 2 second
          forceSleep(2);
        }
      }
      // then try emergency reset master
      Set<Host> slaves = Host.getAll(conn);
      for (Host slave : slaves) {
        String slaveIp = slave.getAddress(conn);
        Connection slaveConn = null;
        try {
          s_logger.debug("Logging on as the slave to " + slaveIp);

          slaveConn = new Connection(getURL(slaveIp), 10);
          slaveLocalLoginWithPassword(slaveConn, username, password);
          Pool.Record slavePoolr = getPoolRecord(slaveConn);
          String ip = slavePoolr.master.getAddress(slaveConn);
          if (!masterIp.trim().equals(ip.trim())) {
            PoolEmergencyResetMaster(slaveIp, masterIp, username, password);
          }
        } catch (Exception e) {
          s_logger.debug("Unable to login to slave " + slaveIp + " error " + e.getMessage());
        } finally {
          localLogout(slaveConn);
          slaveConn = null;
        }
      }
    } catch (Exception e) {
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Catch " + e.getClass().getName() + " due to " + e.toString());
      }
    }
  }
  @Override
  public Answer execute(final ScaleVmCommand command, final CitrixResourceBase citrixResourceBase) {
    final VirtualMachineTO vmSpec = command.getVirtualMachine();
    final String vmName = vmSpec.getName();
    try {
      final Connection conn = citrixResourceBase.getConnection();
      final Set<VM> vms = VM.getByNameLabel(conn, vmName);
      final Host host = Host.getByUuid(conn, citrixResourceBase.getHost().getUuid());

      // If DMC is not enable then don't execute this command.
      if (!citrixResourceBase.isDmcEnabled(conn, host)) {
        throw new CloudRuntimeException(
            "Unable to scale the vm: "
                + vmName
                + " as DMC - Dynamic memory control is not enabled for the XenServer:"
                + citrixResourceBase.getHost().getUuid()
                + " ,check your license and hypervisor version.");
      }

      if (vms == null || vms.size() == 0) {
        s_logger.info(
            "No running VM "
                + vmName
                + " exists on XenServer"
                + citrixResourceBase.getHost().getUuid());
        return new ScaleVmAnswer(command, false, "VM does not exist");
      }

      // stop vm which is running on this host or is in halted state
      final Iterator<VM> iter = vms.iterator();
      while (iter.hasNext()) {
        final VM vm = iter.next();
        final VM.Record vmr = vm.getRecord(conn);

        if (vmr.powerState == VmPowerState.HALTED
            || vmr.powerState == VmPowerState.RUNNING
                && !citrixResourceBase.isRefNull(vmr.residentOn)
                && !vmr.residentOn.getUuid(conn).equals(citrixResourceBase.getHost().getUuid())) {
          iter.remove();
        }
      }

      for (final VM vm : vms) {
        vm.getRecord(conn);
        try {
          citrixResourceBase.scaleVM(conn, vm, vmSpec, host);
        } catch (final Exception e) {
          final String msg =
              "Catch exception "
                  + e.getClass().getName()
                  + " when scaling VM:"
                  + vmName
                  + " due to "
                  + e.toString();
          s_logger.debug(msg);
          return new ScaleVmAnswer(command, false, msg);
        }
      }
      final String msg = "scaling VM " + vmName + " is successful on host " + host;
      s_logger.debug(msg);
      return new ScaleVmAnswer(command, true, msg);

    } catch (final XenAPIException e) {
      final String msg = "Upgrade Vm " + vmName + " fail due to " + e.toString();
      s_logger.warn(msg, e);
      return new ScaleVmAnswer(command, false, msg);
    } catch (final XmlRpcException e) {
      final String msg = "Upgrade Vm " + vmName + " fail due to " + e.getMessage();
      s_logger.warn(msg, e);
      return new ScaleVmAnswer(command, false, msg);
    } catch (final Exception e) {
      final String msg = "Unable to upgrade " + vmName + " due to " + e.getMessage();
      s_logger.warn(msg, e);
      return new ScaleVmAnswer(command, false, msg);
    }
  }
  public Connection connect(
      String hostUuid,
      String poolUuid,
      String ipAddress,
      String username,
      Queue<String> password,
      int wait) {
    XenServerConnection mConn = null;
    Connection sConn = null;
    String masterIp = null;
    if (hostUuid == null
        || poolUuid == null
        || ipAddress == null
        || username == null
        || password == null) {
      String msg =
          "Connect some parameter are null hostUuid:"
              + hostUuid
              + " ,poolUuid:"
              + poolUuid
              + " ,ipAddress:"
              + ipAddress;
      s_logger.debug(msg);
      throw new CloudRuntimeException(msg);
    }
    Host host = null;
    synchronized (poolUuid.intern()) {
      // Let's see if it is an existing connection.
      mConn = getConnect(poolUuid);
      if (mConn != null) {
        try {
          host = Host.getByUuid(mConn, hostUuid);
        } catch (Types.SessionInvalid e) {
          s_logger.debug(
              "Session thgrough ip "
                  + mConn.getIp()
                  + " is invalid for pool("
                  + poolUuid
                  + ") due to "
                  + e.toString());
          try {
            loginWithPassword(
                mConn, mConn.getUsername(), mConn.getPassword(), APIVersion.latest().toString());
          } catch (Exception e1) {
            if (s_logger.isDebugEnabled()) {
              s_logger.debug(
                  "connect through IP("
                      + mConn.getIp()
                      + " for pool("
                      + poolUuid
                      + ") is broken due to "
                      + e.toString());
            }
            removeConnect(poolUuid);
            mConn = null;
          }
        } catch (UuidInvalid e) {
          String msg =
              "Host("
                  + hostUuid
                  + ") doesn't belong to pool("
                  + poolUuid
                  + "), please execute 'xe pool-join master-address="
                  + mConn.getIp()
                  + " master-username="******"connect through IP("
                    + mConn.getIp()
                    + " for pool("
                    + poolUuid
                    + ") is broken due to "
                    + e.toString());
          }
          removeConnect(poolUuid);
          mConn = null;
        }
      }

      if (mConn == null) {
        try {
          try {
            if (s_logger.isDebugEnabled()) {
              s_logger.debug("Logging on as the slave to " + ipAddress);
            }
            sConn = new Connection(getURL(ipAddress), 5);
            slaveLocalLoginWithPassword(sConn, username, password);
          } catch (Exception e) {
            String msg =
                "Unable to create slave connection to host("
                    + hostUuid
                    + ") due to "
                    + e.toString();
            if (s_logger.isDebugEnabled()) {
              s_logger.debug(msg);
            }
            throw new CloudRuntimeException(msg, e);
          }
          Pool.Record pr = null;
          try {
            pr = getPoolRecord(sConn);
          } catch (Exception e) {
            PoolEmergencyTransitionToMaster(ipAddress, username, password);
            mConn =
                new XenServerConnection(
                    getURL(ipAddress), ipAddress, username, password, _retries, _interval, wait);
            try {
              loginWithPassword(mConn, username, password, APIVersion.latest().toString());
              pr = getPoolRecord(mConn);
            } catch (Exception e1) {
              String msg =
                  "Unable to create master connection to host("
                      + hostUuid
                      + ") after transition it to master, due to "
                      + e1.toString();
              if (s_logger.isDebugEnabled()) {
                s_logger.debug(msg);
              }
              throw new CloudRuntimeException(msg, e1);
            }
            if (!pr.uuid.equals(poolUuid)) {
              String msg =
                  "host("
                      + hostUuid
                      + ") should be in pool("
                      + poolUuid
                      + "), but it is actually in pool("
                      + pr.uuid
                      + ")";
              if (s_logger.isDebugEnabled()) {
                s_logger.debug(msg);
              }
              throw new CloudRuntimeException(msg);
            } else {
              if (s_managePool) {
                ensurePoolIntegrity(mConn, ipAddress, username, password, wait);
              }
              addConnect(poolUuid, mConn);
              return mConn;
            }
          }
          if (!pr.uuid.equals(poolUuid)) {
            String msg =
                "host("
                    + hostUuid
                    + ") should be in pool("
                    + poolUuid
                    + "), but it is actually in pool("
                    + pr.uuid
                    + ")";
            if (s_logger.isDebugEnabled()) {
              s_logger.debug(msg);
            }
            throw new CloudRuntimeException(msg);
          }
          try {
            masterIp = pr.master.getAddress(sConn);
            mConn =
                new XenServerConnection(
                    getURL(masterIp), masterIp, username, password, _retries, _interval, wait);
            loginWithPassword(mConn, username, password, APIVersion.latest().toString());
            addConnect(poolUuid, mConn);
            return mConn;
          } catch (Exception e) {
            String msg = "Unable to logon in " + masterIp + " as master in pool(" + poolUuid + ")";
            if (s_logger.isDebugEnabled()) {
              s_logger.debug(msg);
            }
            throw new CloudRuntimeException(msg);
          }
        } finally {
          localLogout(sConn);
          sConn = null;
        }
      }
    }

    if (mConn != null) {
      if (s_managePool) {
        try {
          Map<String, String> args = new HashMap<String, String>();
          host.callPlugin(mConn, "echo", "main", args);
        } catch (Types.SessionInvalid e) {
          if (s_logger.isDebugEnabled()) {
            String msg =
                "Catch Exception: "
                    + e.getClass().getName()
                    + " Can't connect host "
                    + ipAddress
                    + " due to "
                    + e.toString();
            s_logger.debug(msg);
          }
          PoolEmergencyResetMaster(
              ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword());
        } catch (Types.CannotContactHost e) {
          if (s_logger.isDebugEnabled()) {
            String msg =
                "Catch Exception: "
                    + e.getClass().getName()
                    + " Can't connect host "
                    + ipAddress
                    + " due to "
                    + e.toString();
            s_logger.debug(msg);
          }
          PoolEmergencyResetMaster(
              ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword());
        } catch (Types.HostOffline e) {
          if (s_logger.isDebugEnabled()) {
            String msg =
                "Catch Exception: "
                    + e.getClass().getName()
                    + " Host is offline "
                    + ipAddress
                    + " due to "
                    + e.toString();
            s_logger.debug(msg);
          }
          PoolEmergencyResetMaster(
              ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword());
        } catch (Types.HostNotLive e) {
          String msg =
              "Catch Exception: "
                  + e.getClass().getName()
                  + " Host Not Live "
                  + ipAddress
                  + " due to "
                  + e.toString();
          if (s_logger.isDebugEnabled()) {
            s_logger.debug(msg);
          }
          PoolEmergencyResetMaster(
              ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword());
        } catch (Exception e) {
          String msg = "Echo test failed on host " + hostUuid + " IP " + ipAddress;
          s_logger.warn(msg, e);
          throw new CloudRuntimeException(msg, e);
        }
      }
    }
    return mConn;
  }
  public void switchMaster(
      String slaveIp,
      String poolUuid,
      Connection conn,
      Host host,
      String username,
      Queue<String> password,
      int wait)
      throws XmlRpcException, XenAPIException {
    synchronized (poolUuid.intern()) {
      String masterIp = host.getAddress(conn);
      s_logger.debug("Designating the new master to " + masterIp);
      Pool.designateNewMaster(conn, host);
      Connection slaveConn = null;
      Connection masterConn = null;
      int retry = 30;
      for (int i = 0; i < retry; i++) {
        forceSleep(5);
        try {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug("Logging on as the slave to " + slaveIp);
          }
          slaveConn = null;
          masterConn = null;
          Session slaveSession = null;

          slaveConn = new Connection(getURL(slaveIp), 10);
          slaveSession = slaveLocalLoginWithPassword(slaveConn, username, password);

          if (s_logger.isDebugEnabled()) {
            s_logger.debug("Slave logon successful. session= " + slaveSession);
          }

          Pool.Record pr = getPoolRecord(slaveConn);
          Host master = pr.master;
          String ma = master.getAddress(slaveConn);
          if (!ma.trim().equals(masterIp.trim())) {
            continue;
          }
          s_logger.debug("Logging on as the master to " + masterIp);
          masterConn = new Connection(getURL(masterIp), 10);
          loginWithPassword(masterConn, username, password, APIVersion.latest().toString());
          removeConnect(poolUuid);
          ensurePoolIntegrity(masterConn, masterIp, username, password, wait);
          return;
        } catch (Types.HostIsSlave e) {
          s_logger.debug("HostIsSlaveException: Still waiting for the conversion to the master");
        } catch (XmlRpcException e) {
          s_logger.debug(
              "XmlRpcException: Still waiting for the conversion to the master " + e.getMessage());
        } catch (Exception e) {
          s_logger.debug(
              "Exception: Still waiting for the conversion to the master" + e.getMessage());
        } finally {
          if (masterConn != null) {
            try {
              Session.logout(masterConn);
            } catch (Exception e) {
              s_logger.debug("Unable to log out of session: " + e.getMessage());
            }
            masterConn.dispose();
            masterConn = null;
          }
          localLogout(slaveConn);
          slaveConn = null;
        }
      }
      throw new CloudRuntimeException(
          "Unable to logon to the new master after " + retry + " retries");
    }
  }
  public boolean joinPool(
      Connection conn, String hostIp, String masterIp, String username, Queue<String> password) {
    try {
      join(conn, masterIp, username, password);
      if (s_logger.isDebugEnabled()) {
        s_logger.debug("Host(" + hostIp + ") Join the pool at " + masterIp);
      }
      try {
        // slave will restart xapi in 10 sec
        Thread.sleep(10000);
      } catch (InterruptedException e) {
      }
      for (int i = 0; i < 15; i++) {
        Connection slaveConn = null;
        Session slaveSession = null;
        try {
          if (s_logger.isDebugEnabled()) {
            s_logger.debug("Logging on as the slave to " + hostIp);
          }
          slaveConn = new Connection(getURL(hostIp), 10);
          slaveSession = slaveLocalLoginWithPassword(slaveConn, username, password);
          if (s_logger.isDebugEnabled()) {
            s_logger.debug("Slave logon successful. session= " + slaveSession);
          }
          Pool.Record pr = getPoolRecord(slaveConn);
          Host master = pr.master;
          String ma = master.getAddress(slaveConn);
          if (ma.trim().equals(masterIp.trim())) {
            if (s_logger.isDebugEnabled()) {
              s_logger.debug("Host(" + hostIp + ") Joined the pool at " + masterIp);
            }
            return true;
          }
        } catch (Exception e) {
        } finally {
          if (slaveSession != null) {
            try {
              Session.logout(slaveConn);
            } catch (Exception e) {
            }
            slaveConn.dispose();
          }
        }
        try {
          Thread.sleep(3000);
        } catch (InterruptedException e) {
        }
      }

    } catch (Exception e) {
      String msg =
          "Catch "
              + e.getClass().getName()
              + " Unable to allow host "
              + hostIp
              + " to join pool "
              + masterIp
              + " due to "
              + e.toString();
      s_logger.warn(msg, e);
    }
    if (s_logger.isDebugEnabled()) {
      s_logger.debug("Host(" + hostIp + ") unable to Join the pool at " + masterIp);
    }
    return false;
  }