protected void ensurePoolIntegrity(
Connection conn, String masterIp, String username, Queue<String> password, int wait) {
try {
// try recoverSlave first
Set<Host> rcSlaves = Pool.recoverSlaves(conn);
// wait 10 second
forceSleep(10);
for (Host slave : rcSlaves) {
for (int i = 0; i < 30; i++) {
Connection slaveConn = null;
try {
String slaveIp = slave.getAddress(conn);
s_logger.debug("Logging on as the slave to " + slaveIp);
slaveConn = new Connection(getURL(slaveIp), 10);
slaveLocalLoginWithPassword(slaveConn, username, password);
Pool.Record pr = getPoolRecord(slaveConn);
String mIp = pr.master.getAddress(slaveConn);
if (mIp.trim().equals(masterIp.trim())) {
break;
}
} catch (Exception e) {
} finally {
localLogout(slaveConn);
slaveConn = null;
}
// wait 2 second
forceSleep(2);
}
}
// then try emergency reset master
Set<Host> slaves = Host.getAll(conn);
for (Host slave : slaves) {
String slaveIp = slave.getAddress(conn);
Connection slaveConn = null;
try {
s_logger.debug("Logging on as the slave to " + slaveIp);
slaveConn = new Connection(getURL(slaveIp), 10);
slaveLocalLoginWithPassword(slaveConn, username, password);
Pool.Record slavePoolr = getPoolRecord(slaveConn);
String ip = slavePoolr.master.getAddress(slaveConn);
if (!masterIp.trim().equals(ip.trim())) {
PoolEmergencyResetMaster(slaveIp, masterIp, username, password);
}
} catch (Exception e) {
s_logger.debug("Unable to login to slave " + slaveIp + " error " + e.getMessage());
} finally {
localLogout(slaveConn);
slaveConn = null;
}
}
} catch (Exception e) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Catch " + e.getClass().getName() + " due to " + e.toString());
}
}
}
private void PoolEmergencyResetMaster(
String slaveIp, String masterIp, String username, Queue<String> password) {
if (!s_managePool) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Don't manage pool on error so sleeping for " + s_sleepOnError);
try {
Thread.sleep(s_sleepOnError);
} catch (InterruptedException ie) {
}
}
return;
}
Connection slaveConn = null;
try {
s_logger.debug("Trying to reset master of slave " + slaveIp + " to " + masterIp);
slaveConn = new Connection(getURL(slaveIp), 10);
slaveLocalLoginWithPassword(slaveConn, username, password);
Pool.emergencyResetMaster(slaveConn, masterIp);
forceSleep(10);
for (int i = 0; i < 30; i++) {
try {
slaveLocalLoginWithPassword(slaveConn, username, password);
Pool.Record pr = getPoolRecord(slaveConn);
String mIp = pr.master.getAddress(slaveConn);
if (mIp.trim().equals(masterIp.trim())) {
s_logger.debug("Succeeded to reset master of slave " + slaveIp + " to " + masterIp);
return;
}
} catch (Exception e) {
} finally {
localLogout(slaveConn);
slaveConn = null;
}
// wait 2 second
forceSleep(2);
}
throw new CloudRuntimeException(
"Unable to reset master of slave " + slaveIp + " to " + masterIp + "after 30 retry");
} catch (Exception e) {
throw new CloudRuntimeException(
"Unable to reset master of slave "
+ slaveIp
+ " to "
+ masterIp
+ " due to "
+ e.toString());
} finally {
localLogout(slaveConn);
slaveConn = null;
}
}
void PoolEmergencyTransitionToMaster(String slaveIp, String username, Queue<String> password) {
if (!s_managePool) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Don't manage pool on error so sleeping for " + s_sleepOnError);
try {
Thread.sleep(s_sleepOnError);
} catch (InterruptedException ie) {
}
}
return;
}
Connection slaveConn = null;
Connection c = null;
try {
s_logger.debug("Trying to transition master to " + slaveIp);
slaveConn = new Connection(getURL(slaveIp), 10);
slaveLocalLoginWithPassword(slaveConn, username, password);
Pool.emergencyTransitionToMaster(slaveConn);
// restart xapi in 10 sec
forceSleep(10);
// check if the master of this host is set correctly.
c = new Connection(getURL(slaveIp), 10);
for (int i = 0; i < 30; i++) {
try {
loginWithPassword(c, username, password, APIVersion.latest().toString());
s_logger.debug("Succeeded to transition master to " + slaveIp);
return;
} catch (Types.HostIsSlave e) {
s_logger.debug("HostIsSlave: Still waiting for the conversion to the master " + slaveIp);
} catch (Exception e) {
s_logger.debug("Exception: Still waiting for the conversion to the master");
}
forceSleep(2);
}
throw new RuntimeException("EmergencyTransitionToMaster failed after retry 30 times");
} catch (Exception e) {
throw new RuntimeException("EmergencyTransitionToMaster failed due to " + e.getMessage());
} finally {
localLogout(slaveConn);
slaveConn = null;
if (c != null) {
try {
Session.logout(c);
c.dispose();
} catch (Exception e) {
}
}
}
}
protected void join(Connection conn, String masterIp, String username, Queue<String> password)
throws BadServerResponse, XenAPIException, XmlRpcException,
Types.JoiningHostCannotContainSharedSrs {
boolean logged_in = false;
Exception ex = null;
while (!logged_in) {
try {
Pool.join(conn, masterIp, username, password.peek());
logged_in = true;
} catch (BadServerResponse e) {
logged_in = false;
ex = e;
} catch (XenAPIException e) {
logged_in = false;
ex = e;
} catch (XmlRpcException e) {
logged_in = false;
ex = e;
}
if (logged_in && conn != null) {
break;
} else {
if (password.size() > 1) {
password.remove();
continue;
} else {
// the last password did not work leave it and flag error
if (ex instanceof BadServerResponse) {
throw (BadServerResponse) ex;
} else if (ex instanceof XmlRpcException) {
throw (XmlRpcException) ex;
} else if (ex instanceof Types.SessionAuthenticationFailed) {
throw (Types.SessionAuthenticationFailed) ex;
} else if (ex instanceof XenAPIException) {
throw (XenAPIException) ex;
}
break;
}
}
}
}
public static Pool.Record getPoolRecord(Connection conn) throws XmlRpcException, XenAPIException {
Map<Pool, Pool.Record> pools = Pool.getAllRecords(conn);
assert pools.size() == 1 : "Pool size is not one....hmmm....wth? " + pools.size();
return pools.values().iterator().next();
}
public void switchMaster(
String slaveIp,
String poolUuid,
Connection conn,
Host host,
String username,
Queue<String> password,
int wait)
throws XmlRpcException, XenAPIException {
synchronized (poolUuid.intern()) {
String masterIp = host.getAddress(conn);
s_logger.debug("Designating the new master to " + masterIp);
Pool.designateNewMaster(conn, host);
Connection slaveConn = null;
Connection masterConn = null;
int retry = 30;
for (int i = 0; i < retry; i++) {
forceSleep(5);
try {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Logging on as the slave to " + slaveIp);
}
slaveConn = null;
masterConn = null;
Session slaveSession = null;
slaveConn = new Connection(getURL(slaveIp), 10);
slaveSession = slaveLocalLoginWithPassword(slaveConn, username, password);
if (s_logger.isDebugEnabled()) {
s_logger.debug("Slave logon successful. session= " + slaveSession);
}
Pool.Record pr = getPoolRecord(slaveConn);
Host master = pr.master;
String ma = master.getAddress(slaveConn);
if (!ma.trim().equals(masterIp.trim())) {
continue;
}
s_logger.debug("Logging on as the master to " + masterIp);
masterConn = new Connection(getURL(masterIp), 10);
loginWithPassword(masterConn, username, password, APIVersion.latest().toString());
removeConnect(poolUuid);
ensurePoolIntegrity(masterConn, masterIp, username, password, wait);
return;
} catch (Types.HostIsSlave e) {
s_logger.debug("HostIsSlaveException: Still waiting for the conversion to the master");
} catch (XmlRpcException e) {
s_logger.debug(
"XmlRpcException: Still waiting for the conversion to the master " + e.getMessage());
} catch (Exception e) {
s_logger.debug(
"Exception: Still waiting for the conversion to the master" + e.getMessage());
} finally {
if (masterConn != null) {
try {
Session.logout(masterConn);
} catch (Exception e) {
s_logger.debug("Unable to log out of session: " + e.getMessage());
}
masterConn.dispose();
masterConn = null;
}
localLogout(slaveConn);
slaveConn = null;
}
}
throw new CloudRuntimeException(
"Unable to logon to the new master after " + retry + " retries");
}
}