private void reconnectVdc(String vdcId) throws Exception {
URI id = new URI(vdcId);
VirtualDataCenter vdc = dbClient.queryObject(VirtualDataCenter.class, id);
vdc.setConnectionStatus(VirtualDataCenter.ConnectionStatus.CONNECTED);
dbClient.updateAndReindexObject(vdc);
dbClient.removeVdcNodesFromBlacklist(vdc);
}
private VirtualDataCenter getLocalVdc() {
List<URI> ids = dbClient.queryByType(VirtualDataCenter.class, true);
for (URI id : ids) {
VirtualDataCenter vdc = dbClient.queryObject(VirtualDataCenter.class, id);
if (vdc.getLocal() == true) {
return vdc;
}
}
throw new RuntimeException("Failed to find local vdc");
}
private void updateBlackListForReconnectedVdc() {
List<URI> vdcIds = dbClient.queryByType(VirtualDataCenter.class, true);
dbClient.clearBlackList();
log.info("After clear, get current black list {}", dbClient.getBlacklist());
for (URI vdcId : vdcIds) {
VirtualDataCenter vdc = dbClient.queryObject(VirtualDataCenter.class, vdcId);
if (vdc.getConnectionStatus() == VirtualDataCenter.ConnectionStatus.DISCONNECTED) {
log.info("Add vdc {} with status {} to blacklist", vdc.getId(), vdc.getConnectionStatus());
dbClient.addVdcNodesToBlacklist(vdc);
}
}
}
private void updateVdcStatusInDB(List<VdcConfig> vdcConfigs) {
List<URI> vdcIdIter = new ArrayList<>();
List<URI> vdcIds = dbClient.queryByType(VirtualDataCenter.class, true);
for (URI id : vdcIds) {
vdcIdIter.add(id);
}
for (VdcConfig vdcConfig : vdcConfigs) {
log.info("current config's id is {}", vdcConfig.getId());
if (vdcIdIter.contains(vdcConfig.getId())) {
VirtualDataCenter vdc = dbClient.queryObject(VirtualDataCenter.class, vdcConfig.getId());
vdc.setConnectionStatus(
VirtualDataCenter.ConnectionStatus.valueOf(vdcConfig.getConnectionStatus()));
dbClient.updateAndReindexObject(vdc);
}
}
}
private boolean isDisconnectedEachOther(List<String> blackList, List<String> whiteList) {
VirtualDataCenter myVdc = getLocalVdc();
Collection<String> addresses = myVdc.queryHostIPAddressesMap().values();
log.info("local vdc IP addresses:{}", addresses);
boolean found = false;
for (String addr : addresses) {
if (blackList.contains(addr)) {
log.info("The addr {} is in the blackList {}", addr, blackList);
found = true;
break;
}
}
if (found == false) {
return false; // not disconnected each other
}
Collection<List<String>> localBlackLists = dbClient.getBlacklist().values();
log.info("The localBackLists={}", localBlackLists);
List<String> localBlackList = new ArrayList();
for (List<String> list : localBlackLists) {
localBlackList = list;
}
return localBlackList.containsAll(whiteList);
}
private <T extends DataObject> boolean hasDataInCF(Class<T> clazz) {
if (excludeClasses.contains(clazz)) {
return false; // ignore the data in those CFs
}
// true: query only active object ids, for below reason:
// add VDC should succeed just when remove the data in vdc2.
List<URI> ids = dbClient.queryByType(clazz, true, null, 2);
if (clazz.equals(TenantOrg.class) || clazz.equals(ObjectStore.class)) {
if (ids.size() > 1) {
// at least one non-root tenant exist
return true;
}
return false;
}
if (!ids.isEmpty()) {
log.info("The class {} has data e.g. id={}", clazz.getSimpleName(), ids.get(0));
return true;
}
return false;
}
@POST
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Path("/resetblacklist")
public Response resetBlackListForVdc(@QueryParam("vdc_short_id") String vdcShortId) {
try {
log.info("Reset blacklist for {}", vdcShortId);
URI vdcId = VdcUtil.getVdcUrn(vdcShortId);
VirtualDataCenter vdc = dbClient.queryObject(VirtualDataCenter.class, vdcId);
dbClient.removeVdcNodesFromBlacklist(vdc);
return Response.ok().build();
} catch (InternalException ex) {
throw ex;
} catch (Exception ex) {
log.error("Reset blacklist vdc error", ex);
throw GeoException.fatals.reconnectVdcIncompatible();
}
}
@PUT
@Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
public Response syncVdcConfig(VdcConfigSyncParam param) {
log.info("Acquired gobal lock, vdc {}...", param.getVirtualDataCenters().size());
VdcConfig.ConfigChangeType type =
VdcConfig.ConfigChangeType.valueOf(param.getConfigChangeType());
log.info("Current config change type is {}", type);
switch (type) {
case DISCONNECT_VDC:
String disconntecedvdcId = param.getAssignedVdcId();
try {
disconnectVdc(disconntecedvdcId);
} catch (Exception e) {
throw GeoException.fatals.disconnectRemoteSyncFailed(disconntecedvdcId, e.getMessage());
}
break;
case RECONNECT_VDC:
String reconnectVdcId = param.getAssignedVdcId();
try {
VirtualDataCenter localVdc = VdcUtil.getLocalVdc();
// If the local vdc is the one need to be reconnected back, trigger a node repair for
// geodb
if (localVdc.getId().toString().equals(reconnectVdcId)) {
log.info(
"Perform sync vdc config operation and node repair for the reconnected vdc {}",
reconnectVdcId);
VirtualDataCenter reconnVdc =
dbClient.queryObject(VirtualDataCenter.class, new URI(reconnectVdcId));
// Update operated local db, make sure all the vdcs in operated vdc'd db have the same
// status with others.
log.info("Reconnect ops update local db for operatedVdc");
updateVdcStatusInDB(param.getVirtualDataCenters());
log.info("Reconnect ops update local db done");
// Clean blacklist for reconnected vdc
log.info("Reconnect ops to clean blacklist for reconnected vdc.");
updateBlackListForReconnectedVdc();
log.info("Reconnect ops: new blacklist is {}", dbClient.getBlacklist());
helper.syncVdcConfig(param.getVirtualDataCenters(), null);
log.info("Current strategy options is {}", dbClient.getGeoStrategyOptions());
log.info("Current schema version for Geo is {}", dbClient.getGeoSchemaVersions());
geoBackgroundTasks.startGeodbNodeRepair();
} else {
reconnectVdc(reconnectVdcId);
}
} catch (Exception e) {
throw GeoException.fatals.reconnectRemoteSyncFailed(reconnectVdcId, e.getMessage());
}
break;
default:
Iterator<URI> srcVdcIdIter = dbClient.queryByType(VirtualDataCenter.class, true).iterator();
String assignedVdcId = param.getAssignedVdcId();
String geoEncryptionKey = param.getGeoEncryptionKey();
// for add-vdc
if (assignedVdcId != null && geoEncryptionKey != null) {
log.info("This vdc will be added to a geo system.");
if (!srcVdcIdIter.hasNext()) {
throw GeoException.fatals.connectVDCLocalMultipleVDC(assignedVdcId);
}
URI srcVdcId = srcVdcIdIter.next();
// Delete the local vdc record
VirtualDataCenter existingVdc = dbClient.queryObject(VirtualDataCenter.class, srcVdcId);
dbClient.markForDeletion(existingVdc);
log.info(
"The existing vdc {} has been removed. The current vdc id will be {}.",
srcVdcId,
assignedVdcId);
helper.setGeoEncryptionKey(geoEncryptionKey);
log.info("geo encryption key has been updated");
helper.resetStaleLocalObjects();
dbClient.stopClusterGossiping();
} else if (assignedVdcId == null && geoEncryptionKey == null) {
log.info("Sync'ing new vdc info to existing geo system.");
} else {
throw GeoException.fatals.remoteVDCGeoEncryptionMissing();
}
helper.syncVdcConfig(param.getVirtualDataCenters(), assignedVdcId);
if (isRemoveOp(param)) {
log.info("Disable grossip to avoid schema version disagreement errors");
dbClient.stopClusterGossiping();
}
break;
}
return Response.ok().build();
}
/**
* Do more precheck For disconnecting a vdc, check if there is a VDC that is under disconnecting
* If yes, return the VDC under disconnecting, otherwise set the VDC (given by parameter) status
* to DISCONNECTING
*
* @param checkParam
* @return VdcPreCheckResponse
*/
@POST
@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
@Path("/precheck2")
public VdcPreCheckResponse2 precheckVdcConfig(VdcPreCheckParam2 checkParam) {
log.info("Start vdc config precheck2 for {} ...", checkParam.getConfigChangeType());
if (service.getId().endsWith("standalone")) {
throw GeoException.fatals.remoteVDCWrongStandaloneInstall();
}
VdcConfig.ConfigChangeType type = checkParam.getConfigChangeType();
VirtualDataCenter vdc = null;
VdcPreCheckResponse2 resp2 = new VdcPreCheckResponse2();
resp2.setCompatible(true);
// BZ
// TODO Need to use a different method to update info on lock on a remote system.
// Need to use a different field (not connection status of VDC object) as a locking mechanism)
boolean precheckFailed = checkParam.isPrecheckFailed();
switch (type) {
case DISCONNECT_VDC:
log.info("Precheck2 for disconnect ops");
vdc = helper.getDisconnectingVdc();
if (checkParam.getIsAllNotReachable()) {
URI targetVdcId = checkParam.getVdcIds().get(0);
log.info("Precheck2 to check the disconnect vdc {} is reachable", targetVdcId);
VirtualDataCenter targetVdc = dbClient.queryObject(VirtualDataCenter.class, targetVdcId);
resp2.setIsAllNodesNotReachable(
!helper.areNodesReachable(
getLocalVdc().getShortId(),
targetVdc.getHostIPv4AddressesMap(),
targetVdc.getHostIPv6AddressesMap(),
checkParam.getIsAllNotReachable()));
break;
}
if (precheckFailed) {
log.info("Precheck2 to update reconnect precheck fail status");
String vdcState = checkParam.getDefaultVdcState();
if (StringUtils.isNotEmpty(vdcState)) {
vdc.setConnectionStatus(VirtualDataCenter.ConnectionStatus.valueOf(vdcState));
dbClient.updateAndReindexObject(vdc);
}
break;
}
if (vdc == null) {
// no DISCONNECTING_VDC
log.info("Precheck2: there is no disconnecting vdc");
URI srcVdcId = checkParam.getVdcIds().get(1);
VirtualDataCenter srcVdc = dbClient.queryObject(VirtualDataCenter.class, srcVdcId);
if (srcVdc.getConnectionStatus() == VirtualDataCenter.ConnectionStatus.DISCONNECTED) {
resp2.setCompatible(false);
break;
}
// BZ
// TODO need to use a different field to set locks on concurrent VDC operation
URI id = checkParam.getVdcIds().get(0);
vdc = dbClient.queryObject(VirtualDataCenter.class, id);
vdc.setConnectionStatus(VirtualDataCenter.ConnectionStatus.DISCONNECTING);
dbClient.updateAndReindexObject(vdc);
} else {
resp2 = toVirtualDataCenterResponse2(vdc, true, null);
}
break;
case RECONNECT_VDC:
log.info("Precheck2 for reconnect ops checkParam={}", checkParam);
List<String> blackList = checkParam.getBlackList();
List<String> whiteList = checkParam.getWhiteList();
log.info("Precheck2 to check if two vdc disconnect each other");
resp2.setCompatible(true);
if (isDisconnectedEachOther(blackList, whiteList)) {
log.info("Precheck2: two vdc have disconnected each other");
resp2.setCompatible(false);
break;
}
if (precheckFailed) {
log.info("Precheck2 to update reconnect precheck fail status");
URI targetVdcId = checkParam.getVdcIds().get(0);
log.info("Precheck2 to check the disconnect vdc {} is reachable", targetVdcId);
VirtualDataCenter targetVdc = dbClient.queryObject(VirtualDataCenter.class, targetVdcId);
String vdcState = checkParam.getDefaultVdcState();
if (StringUtils.isNotEmpty(vdcState)) {
targetVdc.setConnectionStatus(VirtualDataCenter.ConnectionStatus.valueOf(vdcState));
dbClient.updateAndReindexObject(targetVdc);
}
break;
}
break;
}
log.info("Precheck2 done, resp is {}", resp2.toString());
return resp2;
}