public String getMasterIp(String ip, String username, Queue<String> password)
throws XenAPIException {
Connection slaveConn = null;
try {
slaveConn = new Connection(getURL(ip), 10);
slaveLocalLoginWithPassword(slaveConn, username, password);
if (s_logger.isDebugEnabled()) {
s_logger.debug("Slave logon to " + ip);
}
String masterIp = null;
Pool.Record pr = getPoolRecord(slaveConn);
Host master = pr.master;
masterIp = master.getAddress(slaveConn);
return masterIp;
} catch (Types.SessionAuthenticationFailed e) {
s_logger.debug("Failed to slave local login to " + ip + " due to " + e.toString());
throw e;
} catch (Exception e) {
s_logger.debug("Failed to slave local login to " + ip + " due to " + e.toString());
} finally {
localLogout(slaveConn);
slaveConn = null;
}
throw new RuntimeException("can not get master ip");
}
protected void ensurePoolIntegrity(
Connection conn, String masterIp, String username, Queue<String> password, int wait) {
try {
// try recoverSlave first
Set<Host> rcSlaves = Pool.recoverSlaves(conn);
// wait 10 second
forceSleep(10);
for (Host slave : rcSlaves) {
for (int i = 0; i < 30; i++) {
Connection slaveConn = null;
try {
String slaveIp = slave.getAddress(conn);
s_logger.debug("Logging on as the slave to " + slaveIp);
slaveConn = new Connection(getURL(slaveIp), 10);
slaveLocalLoginWithPassword(slaveConn, username, password);
Pool.Record pr = getPoolRecord(slaveConn);
String mIp = pr.master.getAddress(slaveConn);
if (mIp.trim().equals(masterIp.trim())) {
break;
}
} catch (Exception e) {
} finally {
localLogout(slaveConn);
slaveConn = null;
}
// wait 2 second
forceSleep(2);
}
}
// then try emergency reset master
Set<Host> slaves = Host.getAll(conn);
for (Host slave : slaves) {
String slaveIp = slave.getAddress(conn);
Connection slaveConn = null;
try {
s_logger.debug("Logging on as the slave to " + slaveIp);
slaveConn = new Connection(getURL(slaveIp), 10);
slaveLocalLoginWithPassword(slaveConn, username, password);
Pool.Record slavePoolr = getPoolRecord(slaveConn);
String ip = slavePoolr.master.getAddress(slaveConn);
if (!masterIp.trim().equals(ip.trim())) {
PoolEmergencyResetMaster(slaveIp, masterIp, username, password);
}
} catch (Exception e) {
s_logger.debug("Unable to login to slave " + slaveIp + " error " + e.getMessage());
} finally {
localLogout(slaveConn);
slaveConn = null;
}
}
} catch (Exception e) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Catch " + e.getClass().getName() + " due to " + e.toString());
}
}
}
@Override
public Answer execute(final ScaleVmCommand command, final CitrixResourceBase citrixResourceBase) {
final VirtualMachineTO vmSpec = command.getVirtualMachine();
final String vmName = vmSpec.getName();
try {
final Connection conn = citrixResourceBase.getConnection();
final Set<VM> vms = VM.getByNameLabel(conn, vmName);
final Host host = Host.getByUuid(conn, citrixResourceBase.getHost().getUuid());
// If DMC is not enable then don't execute this command.
if (!citrixResourceBase.isDmcEnabled(conn, host)) {
throw new CloudRuntimeException(
"Unable to scale the vm: "
+ vmName
+ " as DMC - Dynamic memory control is not enabled for the XenServer:"
+ citrixResourceBase.getHost().getUuid()
+ " ,check your license and hypervisor version.");
}
if (vms == null || vms.size() == 0) {
s_logger.info(
"No running VM "
+ vmName
+ " exists on XenServer"
+ citrixResourceBase.getHost().getUuid());
return new ScaleVmAnswer(command, false, "VM does not exist");
}
// stop vm which is running on this host or is in halted state
final Iterator<VM> iter = vms.iterator();
while (iter.hasNext()) {
final VM vm = iter.next();
final VM.Record vmr = vm.getRecord(conn);
if (vmr.powerState == VmPowerState.HALTED
|| vmr.powerState == VmPowerState.RUNNING
&& !citrixResourceBase.isRefNull(vmr.residentOn)
&& !vmr.residentOn.getUuid(conn).equals(citrixResourceBase.getHost().getUuid())) {
iter.remove();
}
}
for (final VM vm : vms) {
vm.getRecord(conn);
try {
citrixResourceBase.scaleVM(conn, vm, vmSpec, host);
} catch (final Exception e) {
final String msg =
"Catch exception "
+ e.getClass().getName()
+ " when scaling VM:"
+ vmName
+ " due to "
+ e.toString();
s_logger.debug(msg);
return new ScaleVmAnswer(command, false, msg);
}
}
final String msg = "scaling VM " + vmName + " is successful on host " + host;
s_logger.debug(msg);
return new ScaleVmAnswer(command, true, msg);
} catch (final XenAPIException e) {
final String msg = "Upgrade Vm " + vmName + " fail due to " + e.toString();
s_logger.warn(msg, e);
return new ScaleVmAnswer(command, false, msg);
} catch (final XmlRpcException e) {
final String msg = "Upgrade Vm " + vmName + " fail due to " + e.getMessage();
s_logger.warn(msg, e);
return new ScaleVmAnswer(command, false, msg);
} catch (final Exception e) {
final String msg = "Unable to upgrade " + vmName + " due to " + e.getMessage();
s_logger.warn(msg, e);
return new ScaleVmAnswer(command, false, msg);
}
}
public Connection connect(
String hostUuid,
String poolUuid,
String ipAddress,
String username,
Queue<String> password,
int wait) {
XenServerConnection mConn = null;
Connection sConn = null;
String masterIp = null;
if (hostUuid == null
|| poolUuid == null
|| ipAddress == null
|| username == null
|| password == null) {
String msg =
"Connect some parameter are null hostUuid:"
+ hostUuid
+ " ,poolUuid:"
+ poolUuid
+ " ,ipAddress:"
+ ipAddress;
s_logger.debug(msg);
throw new CloudRuntimeException(msg);
}
Host host = null;
synchronized (poolUuid.intern()) {
// Let's see if it is an existing connection.
mConn = getConnect(poolUuid);
if (mConn != null) {
try {
host = Host.getByUuid(mConn, hostUuid);
} catch (Types.SessionInvalid e) {
s_logger.debug(
"Session thgrough ip "
+ mConn.getIp()
+ " is invalid for pool("
+ poolUuid
+ ") due to "
+ e.toString());
try {
loginWithPassword(
mConn, mConn.getUsername(), mConn.getPassword(), APIVersion.latest().toString());
} catch (Exception e1) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(
"connect through IP("
+ mConn.getIp()
+ " for pool("
+ poolUuid
+ ") is broken due to "
+ e.toString());
}
removeConnect(poolUuid);
mConn = null;
}
} catch (UuidInvalid e) {
String msg =
"Host("
+ hostUuid
+ ") doesn't belong to pool("
+ poolUuid
+ "), please execute 'xe pool-join master-address="
+ mConn.getIp()
+ " master-username="******"connect through IP("
+ mConn.getIp()
+ " for pool("
+ poolUuid
+ ") is broken due to "
+ e.toString());
}
removeConnect(poolUuid);
mConn = null;
}
}
if (mConn == null) {
try {
try {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Logging on as the slave to " + ipAddress);
}
sConn = new Connection(getURL(ipAddress), 5);
slaveLocalLoginWithPassword(sConn, username, password);
} catch (Exception e) {
String msg =
"Unable to create slave connection to host("
+ hostUuid
+ ") due to "
+ e.toString();
if (s_logger.isDebugEnabled()) {
s_logger.debug(msg);
}
throw new CloudRuntimeException(msg, e);
}
Pool.Record pr = null;
try {
pr = getPoolRecord(sConn);
} catch (Exception e) {
PoolEmergencyTransitionToMaster(ipAddress, username, password);
mConn =
new XenServerConnection(
getURL(ipAddress), ipAddress, username, password, _retries, _interval, wait);
try {
loginWithPassword(mConn, username, password, APIVersion.latest().toString());
pr = getPoolRecord(mConn);
} catch (Exception e1) {
String msg =
"Unable to create master connection to host("
+ hostUuid
+ ") after transition it to master, due to "
+ e1.toString();
if (s_logger.isDebugEnabled()) {
s_logger.debug(msg);
}
throw new CloudRuntimeException(msg, e1);
}
if (!pr.uuid.equals(poolUuid)) {
String msg =
"host("
+ hostUuid
+ ") should be in pool("
+ poolUuid
+ "), but it is actually in pool("
+ pr.uuid
+ ")";
if (s_logger.isDebugEnabled()) {
s_logger.debug(msg);
}
throw new CloudRuntimeException(msg);
} else {
if (s_managePool) {
ensurePoolIntegrity(mConn, ipAddress, username, password, wait);
}
addConnect(poolUuid, mConn);
return mConn;
}
}
if (!pr.uuid.equals(poolUuid)) {
String msg =
"host("
+ hostUuid
+ ") should be in pool("
+ poolUuid
+ "), but it is actually in pool("
+ pr.uuid
+ ")";
if (s_logger.isDebugEnabled()) {
s_logger.debug(msg);
}
throw new CloudRuntimeException(msg);
}
try {
masterIp = pr.master.getAddress(sConn);
mConn =
new XenServerConnection(
getURL(masterIp), masterIp, username, password, _retries, _interval, wait);
loginWithPassword(mConn, username, password, APIVersion.latest().toString());
addConnect(poolUuid, mConn);
return mConn;
} catch (Exception e) {
String msg = "Unable to logon in " + masterIp + " as master in pool(" + poolUuid + ")";
if (s_logger.isDebugEnabled()) {
s_logger.debug(msg);
}
throw new CloudRuntimeException(msg);
}
} finally {
localLogout(sConn);
sConn = null;
}
}
}
if (mConn != null) {
if (s_managePool) {
try {
Map<String, String> args = new HashMap<String, String>();
host.callPlugin(mConn, "echo", "main", args);
} catch (Types.SessionInvalid e) {
if (s_logger.isDebugEnabled()) {
String msg =
"Catch Exception: "
+ e.getClass().getName()
+ " Can't connect host "
+ ipAddress
+ " due to "
+ e.toString();
s_logger.debug(msg);
}
PoolEmergencyResetMaster(
ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword());
} catch (Types.CannotContactHost e) {
if (s_logger.isDebugEnabled()) {
String msg =
"Catch Exception: "
+ e.getClass().getName()
+ " Can't connect host "
+ ipAddress
+ " due to "
+ e.toString();
s_logger.debug(msg);
}
PoolEmergencyResetMaster(
ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword());
} catch (Types.HostOffline e) {
if (s_logger.isDebugEnabled()) {
String msg =
"Catch Exception: "
+ e.getClass().getName()
+ " Host is offline "
+ ipAddress
+ " due to "
+ e.toString();
s_logger.debug(msg);
}
PoolEmergencyResetMaster(
ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword());
} catch (Types.HostNotLive e) {
String msg =
"Catch Exception: "
+ e.getClass().getName()
+ " Host Not Live "
+ ipAddress
+ " due to "
+ e.toString();
if (s_logger.isDebugEnabled()) {
s_logger.debug(msg);
}
PoolEmergencyResetMaster(
ipAddress, mConn.getIp(), mConn.getUsername(), mConn.getPassword());
} catch (Exception e) {
String msg = "Echo test failed on host " + hostUuid + " IP " + ipAddress;
s_logger.warn(msg, e);
throw new CloudRuntimeException(msg, e);
}
}
}
return mConn;
}
public void switchMaster(
String slaveIp,
String poolUuid,
Connection conn,
Host host,
String username,
Queue<String> password,
int wait)
throws XmlRpcException, XenAPIException {
synchronized (poolUuid.intern()) {
String masterIp = host.getAddress(conn);
s_logger.debug("Designating the new master to " + masterIp);
Pool.designateNewMaster(conn, host);
Connection slaveConn = null;
Connection masterConn = null;
int retry = 30;
for (int i = 0; i < retry; i++) {
forceSleep(5);
try {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Logging on as the slave to " + slaveIp);
}
slaveConn = null;
masterConn = null;
Session slaveSession = null;
slaveConn = new Connection(getURL(slaveIp), 10);
slaveSession = slaveLocalLoginWithPassword(slaveConn, username, password);
if (s_logger.isDebugEnabled()) {
s_logger.debug("Slave logon successful. session= " + slaveSession);
}
Pool.Record pr = getPoolRecord(slaveConn);
Host master = pr.master;
String ma = master.getAddress(slaveConn);
if (!ma.trim().equals(masterIp.trim())) {
continue;
}
s_logger.debug("Logging on as the master to " + masterIp);
masterConn = new Connection(getURL(masterIp), 10);
loginWithPassword(masterConn, username, password, APIVersion.latest().toString());
removeConnect(poolUuid);
ensurePoolIntegrity(masterConn, masterIp, username, password, wait);
return;
} catch (Types.HostIsSlave e) {
s_logger.debug("HostIsSlaveException: Still waiting for the conversion to the master");
} catch (XmlRpcException e) {
s_logger.debug(
"XmlRpcException: Still waiting for the conversion to the master " + e.getMessage());
} catch (Exception e) {
s_logger.debug(
"Exception: Still waiting for the conversion to the master" + e.getMessage());
} finally {
if (masterConn != null) {
try {
Session.logout(masterConn);
} catch (Exception e) {
s_logger.debug("Unable to log out of session: " + e.getMessage());
}
masterConn.dispose();
masterConn = null;
}
localLogout(slaveConn);
slaveConn = null;
}
}
throw new CloudRuntimeException(
"Unable to logon to the new master after " + retry + " retries");
}
}
public boolean joinPool(
Connection conn, String hostIp, String masterIp, String username, Queue<String> password) {
try {
join(conn, masterIp, username, password);
if (s_logger.isDebugEnabled()) {
s_logger.debug("Host(" + hostIp + ") Join the pool at " + masterIp);
}
try {
// slave will restart xapi in 10 sec
Thread.sleep(10000);
} catch (InterruptedException e) {
}
for (int i = 0; i < 15; i++) {
Connection slaveConn = null;
Session slaveSession = null;
try {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Logging on as the slave to " + hostIp);
}
slaveConn = new Connection(getURL(hostIp), 10);
slaveSession = slaveLocalLoginWithPassword(slaveConn, username, password);
if (s_logger.isDebugEnabled()) {
s_logger.debug("Slave logon successful. session= " + slaveSession);
}
Pool.Record pr = getPoolRecord(slaveConn);
Host master = pr.master;
String ma = master.getAddress(slaveConn);
if (ma.trim().equals(masterIp.trim())) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Host(" + hostIp + ") Joined the pool at " + masterIp);
}
return true;
}
} catch (Exception e) {
} finally {
if (slaveSession != null) {
try {
Session.logout(slaveConn);
} catch (Exception e) {
}
slaveConn.dispose();
}
}
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
}
}
} catch (Exception e) {
String msg =
"Catch "
+ e.getClass().getName()
+ " Unable to allow host "
+ hostIp
+ " to join pool "
+ masterIp
+ " due to "
+ e.toString();
s_logger.warn(msg, e);
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("Host(" + hostIp + ") unable to Join the pool at " + masterIp);
}
return false;
}