@Override
public long updateEnvClouds(long envId, List<CmsCIRelation> cloudRels, String userId) {
// for now we will handle just new clouds
List<CmsCIRelation> existingCloudRels =
cmProcessor.getFromCIRelationsNaked(envId, BASE_CONSUMES, ACCOUNT_CLOUD);
Set<Long> existingCloudIds = new HashSet<Long>();
for (CmsCIRelation rel : existingCloudRels) {
existingCloudIds.add(rel.getToCiId());
}
boolean needUpdate = false;
for (CmsCIRelation requestRel : cloudRels) {
if (!existingCloudIds.contains(requestRel.getToCiId())) {
// this is new cloud lets add env->cloud rel
cmProcessor.createRelation(requestRel);
needUpdate = true;
} else {
cmProcessor.updateRelation(requestRel);
existingCloudIds.remove(requestRel.getToCiId());
}
}
if (!existingCloudIds.isEmpty()) {
// looks like we need to delete some clouds
// first lets see if we have any open releases
processCloudDeletions(envId, existingCloudIds);
}
if (needUpdate) {
CmsCI env = getEnv(envId);
String nsPath = env.getNsPath() + "/" + env.getCiName() + "/manifest";
List<CmsRfcRelation> compOfRels =
cmRfcMrgProcessor.getFromCIRelations(
envId, MANIFEST_COMPOSED_OF, MANIFEST_PLATFORM, "dj");
for (CmsRfcRelation compOfRel : compOfRels) {
CmsRfcCI platform = compOfRel.getToRfcCi();
String platNs = platform.getNsPath();
manifestRfcProcessor.processClouds(env, platform, platNs, nsPath, userId, null, null, null);
Set<String> missingSrvs = cloudUtil.getMissingServices(platform.getCiId());
if (missingSrvs.size() > 0) {
logger.info(
">>>>> Not all services available for platform: "
+ platform.getCiName()
+ ", the missing services: "
+ missingSrvs.toString());
manifestRfcProcessor.disablePlatform(platform.getCiId(), userId);
}
logger.info("Done working on platform " + platform.getCiName());
}
return populateParentRelease(env, nsPath);
} else {
return 0;
}
}
public void restore(boolean writeMode, Long manifestIdDefault) {
List<CmsCIRelation> watchedBys;
long counter = 0;
long restoredMonitors = 0;
if (manifestIdDefault == null) {
watchedBys =
cmProcessor.getCIRelationsNsLikeNakedNoAttrs("/", "manifest.WatchedBy", null, null, null);
} else {
watchedBys = cmManager.getFromCIRelations(manifestIdDefault, "manifest.WatchedBy", null);
}
long startTime = System.currentTimeMillis();
Map<Long, List<CmsCIRelation>> monitors =
watchedBys.stream().collect(Collectors.groupingBy(CmsCIRelation::getFromCiId));
for (Long manifestId : monitors.keySet()) {
counter++;
List<CmsCIRelation> realizedAsRels =
cmManager.getFromCIRelations(manifestId, "base.RealizedAs", null);
if (realizedAsRels.isEmpty()) continue;
Set<Long> cmsBomCiIds =
realizedAsRels.stream().map(CmsCIRelation::getToCiId).collect(Collectors.toSet());
restoreRealizedAsMapping(manifestId, cmsBomCiIds, writeMode);
boolean isFirstBom = true;
for (CmsCIRelation relation : realizedAsRels) {
restoreManifestMapping(manifestId, relation.getToCiId(), writeMode);
if (isFirstBom) {
for (CmsCIRelation rel : monitors.get(manifestId)) {
if (rel.getToCi() == null) {
rel.setToCi(cmProcessor.getCiById(rel.getToCiId()));
}
if (restoreMonitor(manifestId, relation.getToCiId(), rel.getToCi(), writeMode)) {
restoredMonitors++;
}
}
}
isFirstBom = false;
}
if (counter % 100 == 0) {
logger.info(
"Time to process "
+ counter
+ " - "
+ (System.currentTimeMillis() - startTime)
+ " ms!");
}
}
logger.info(">>>>>>>>>>> Monitor restored " + restoredMonitors + ";");
logger.info(
">>>>>>>>>>> Monitor restoration is done!!!"
+ counter
+ " - "
+ (System.currentTimeMillis() - startTime)
+ " ms!");
}
private long checkPlatformPackCompliance(
List<CmsCIRelation> designPlatRels, CmsCI env, String nsPath, String userId) {
List<CmsCIRelation> manifestPlatRels =
cmProcessor.getFromCIRelations(
env.getCiId(), MANIFEST_COMPOSED_OF, null, MANIFEST_PLATFORM);
Map<String, String> manifestPlatPacks = new HashMap<String, String>(manifestPlatRels.size());
for (CmsCIRelation manifestRel : manifestPlatRels) {
CmsCI plat = manifestRel.getToCi();
String key = plat.getCiName() + ":" + plat.getAttribute("major_version").getDjValue();
String value =
plat.getAttribute("source").getDjValue()
+ ":"
+ plat.getAttribute("pack").getDjValue()
+ ":"
+ plat.getAttribute("version").getDjValue();
manifestPlatPacks.put(key, value);
}
long newReleaseId = 0;
for (CmsCIRelation designRel : designPlatRels) {
CmsCI dPlat = designRel.getToCi();
String key = dPlat.getCiName() + ":" + dPlat.getAttribute("major_version").getDjValue();
String value =
dPlat.getAttribute("source").getDjValue()
+ ":"
+ dPlat.getAttribute("pack").getDjValue()
+ ":"
+ dPlat.getAttribute("version").getDjValue();
if (manifestPlatPacks.containsKey(key) && !value.equals(manifestPlatPacks.get(key))) {
String platNsPath =
nsPath
+ "/"
+ dPlat.getCiName()
+ "/"
+ dPlat.getAttribute("major_version").getDfValue();
List<CmsRfcCI> mPlats =
cmRfcMrgProcessor.getDfDjCi(platNsPath, MANIFEST_PLATFORM, dPlat.getCiName(), "dj");
if (mPlats.size() > 0) {
newReleaseId = manifestRfcProcessor.deleteManifestPlatform(mPlats.get(0), userId);
}
}
}
return newReleaseId;
}
private void processCloudDeletions(long envId, Set<Long> cloudsToRemove) {
// looks like we need to delete some clouds
// first lets see if we have any open releases
CmsCI env = getEnv(envId);
String manifestNsPath = env.getNsPath() + "/" + env.getCiName() + "/manifest";
String bomNsPath = env.getNsPath() + "/" + env.getCiName() + "/bom";
List<CmsRelease> manifestOpenReleases =
rfcProcessor.getReleaseBy3(manifestNsPath, null, RELEASE_STATE_OPEN);
if (manifestOpenReleases.size() > 0) {
// throw exception on open release
throw new TransistorException(
CmsError.TRANSISTOR_OPEN_MANIFEST_RELEASE,
"There is open release in this environment, you have to commit or discard before deleteing the clouds!");
}
List<CmsRelease> bomOpenReleases =
rfcProcessor.getReleaseBy3(bomNsPath, null, RELEASE_STATE_OPEN);
if (bomOpenReleases.size() > 0) {
// throw exception on open release
throw new TransistorException(
CmsError.TRANSISTOR_OPEN_BOM_RELEASE,
"There is open BOM release in this environment, you have to deploy or discard before deleteing the clouds!");
}
// if we still here lets check if there are any boms deployed to the clouds that user tries to
// remove
for (long cloidId : cloudsToRemove) {
long deployedToRelsCount =
cmProcessor.getCountToCIRelationsByNS(
cloidId, BASE_DEPLOYED_TO, null, null, bomNsPath, true);
if (deployedToRelsCount > 0) {
// throw exception that there are active boms
throw new TransistorException(
CmsError.TRANSISTOR_BOM_INSTANCES_EXIST,
"There are deployed instances in the cloud, please put the cloud in offline mode for every platform and deploy!");
}
}
// if everything cool, let remove Consumes relations from platforms and env
for (long cloidId : cloudsToRemove) {
List<CmsCIRelation> platformRels =
cmProcessor.getFromCIRelationsNaked(envId, MANIFEST_COMPOSED_OF, MANIFEST_PLATFORM);
for (CmsCIRelation platRel : platformRels) {
List<CmsCIRelation> platformCloudRels =
cmProcessor.getFromToCIRelations(platRel.getToCiId(), BASE_CONSUMES, cloidId);
for (CmsCIRelation platCloudRel : platformCloudRels) {
cmProcessor.deleteRelation(platCloudRel.getCiRelationId());
}
}
// now lets remove env consumes rel
List<CmsCIRelation> envCloudRels =
cmProcessor.getFromToCIRelations(envId, BASE_CONSUMES, cloidId);
for (CmsCIRelation cloudRel : envCloudRels) {
cmProcessor.deleteRelation(cloudRel.getCiRelationId());
}
}
}
private Map<String, TreeSet<String>> getMissingCloudServices(
long manifestPlatCiId, Set<String> requiredServices) {
Map<String, TreeSet<String>> missingCloud2Services = new TreeMap<>();
// get clouds
List<CmsRfcRelation> cloudRelations =
cmRfcMrgProcessor.getFromCIRelations(
manifestPlatCiId, BASE_CONSUMES, ACCOUNT_CLOUD_CLASS, "dj");
// get services for all clouds
cloudRelations.forEach(
rel -> {
Set<String> cloudServices =
cmProcessor
.getFromCIRelationsNaked(rel.getToCiId(), BASE_PROVIDES, null)
.stream()
.filter(this::isService)
.map(cmsCIRelation -> cmsCIRelation.getAttribute("service").getDjValue())
.collect(Collectors.toSet());
String cloud = rel.getToRfcCi().getCiName();
// check if service is configured
requiredServices
.stream()
.filter(s -> !cloudServices.contains(s))
.forEach(
s -> missingCloud2Services.computeIfAbsent(cloud, k -> new TreeSet<>()).add(s));
logger.debug(
"cloud: "
+ cloud
+ " required services:: "
+ requiredServices.toString()
+ " missingServices "
+ missingCloud2Services.keySet());
});
return missingCloud2Services;
}
@Override
public long generateEnvManifest(long envId, String userId, Map<String, String> platModes) {
long t1 = System.currentTimeMillis();
String oldThreadName = Thread.currentThread().getName();
Thread.currentThread().setName(getProcessingThreadName(oldThreadName, envId));
List<CmsCIRelation> assemblyRels =
cmProcessor.getToCIRelations(envId, BASE_REALIZED_IN, null, ACCOUNT_ASSEMBLY);
CmsCI assembly = null;
if (assemblyRels.size() > 0) {
assembly = assemblyRels.get(0).getFromCi();
} else {
String error = "Can not get assembly for envId = " + envId;
logger.error(error);
throw new TransistorException(CmsError.TRANSISTOR_CANNOT_GET_ASSEMBLY, error);
}
CmsCI env = getEnv(envId);
String nsPath = env.getNsPath() + "/" + env.getCiName() + "/manifest";
if (hasOpenManifestRelease(nsPath)) {
String message =
"This environment has an open release. It needs to be discarded or committed before the design pull: "
+ env.getNsPath()
+ "/"
+ env.getCiName();
logger.info(message);
throw new TransistorException(CmsError.TRANSISTOR_OPEN_MANIFEST_RELEASE, message);
}
Long nsId = trUtil.verifyAndCreateNS(nsPath);
logger.info("Created nsId " + nsId);
// Long releaseId = createManifestRelease(nsPath,userId);
List<CmsCIRelation> designPlatRels =
cmProcessor.getFromCIRelations(assembly.getCiId(), null, "ComposedOf", CATALOG_PLATFORM);
// we need to reset all pending deletions cis just in case there was one added back
cmProcessor.resetDeletionsByNs(nsPath);
// check for edge case scenario when there is new design platform with the same name as old one
// but different pack
long releaseId = checkPlatformPackCompliance(designPlatRels, env, nsPath, userId);
if (releaseId > 0) {
// stop any processing and return new release id
return releaseId;
}
final CountDownLatch latch = new CountDownLatch(designPlatRels.size());
List<Future<DesignCIManifestRfcTouple>> submittedFutureTasks =
new ArrayList<Future<DesignCIManifestRfcTouple>>();
Map<Long, CmsRfcCI> design2manifestPlatMap = new HashMap<Long, CmsRfcCI>();
for (CmsCIRelation platRelation : designPlatRels) {
String availMode = null;
if (platModes != null) {
availMode = platModes.get(String.valueOf(platRelation.getToCiId()));
if (availMode != null && availMode.length() == 0) {
availMode = "default";
}
}
Future<DesignCIManifestRfcTouple> future =
executorService.submit(
new ManifestRfcProcessorTask(env, nsPath, userId, availMode, latch, platRelation));
submittedFutureTasks.add(future);
}
boolean allPlatsProcessed = false;
try {
// latch.await(); //wait till all platform processing threads return
allPlatsProcessed =
latch.await(
timeoutInMilliSeconds,
TimeUnit
.MILLISECONDS); // wait for all platform processing threads to finish with timeout
// of 10 mins
if (!allPlatsProcessed) {
logger.error(
"All platforms not processed within timeout duration of " + timeoutInMilliSeconds);
throw new TransistorException(
CmsError.TRANSISTOR_OPEN_MANIFEST_RELEASE,
"Failed to pull latest design for all platform within timeout duration of "
+ timeoutInMilliSeconds
+ " millis");
}
} catch (InterruptedException ie) {
for (Future<DesignCIManifestRfcTouple> job : submittedFutureTasks) {
job.cancel(true);
}
throw new TransistorException(
CmsError.TRANSISTOR_OPEN_MANIFEST_RELEASE, "Design pull process interrupted. ");
}
for (Future<DesignCIManifestRfcTouple> task : submittedFutureTasks) {
DesignCIManifestRfcTouple touple;
try {
touple = task.get();
processPlatformRfcs(touple.manifestPlatformRfcs, userId);
CmsRfcCI manifestPlatformRfc = touple.manifestPlatformRfcs.getManifestPlatformRfc();
Set<String> missingSrvs = cloudUtil.getMissingServices(manifestPlatformRfc.getCiId());
if (missingSrvs.size() > 0) {
logger.info(
">>>>> Not all services available for platform: "
+ manifestPlatformRfc.getCiName()
+ ", the missing services: "
+ missingSrvs.toString());
disablePlatform(manifestPlatformRfc.getCiId(), userId);
}
logger.info("New release id = " + manifestPlatformRfc.getReleaseId());
logger.info("Done working on platform " + manifestPlatformRfc.getNsPath());
design2manifestPlatMap.put(touple.designPlatCI, manifestPlatformRfc);
} catch (Exception e) {
logger.error("Error in pulling latest design for all platforms ", e);
throw new TransistorException(
CmsError.TRANSISTOR_OPEN_MANIFEST_RELEASE,
"Error in pulling latest design for all platforms ");
}
}
// now we need to process linkedTo relations
manifestRfcProcessor.processLinkedTo(design2manifestPlatMap, nsPath, userId);
// now lets delete old existing plats that do not exists in new manifest
manifestRfcProcessor.processDeletedPlatforms(
design2manifestPlatMap.values(), env, nsPath, userId);
// process global variables from design
manifestRfcProcessor.processGlobalVars(assembly.getCiId(), env, nsPath, userId);
long t2 = System.currentTimeMillis();
long envReleaseId = populateParentRelease(env, nsPath);
logger.info(
"Pull design for "
+ nsPath
+ " completed in "
+ (t2 - t1)
+ " millis (releaseId "
+ envReleaseId
+ ")");
return envReleaseId;
}
private CmsCI getEnv(long envId) {
return cmProcessor.getCiById(envId);
}
@Override
public long generateEnvManifest(long envId, String userId, Map<String, String> platModes) {
long t1 = System.currentTimeMillis();
String oldThreadName = Thread.currentThread().getName();
Thread.currentThread().setName(getProcessingThreadName(oldThreadName, envId));
List<CmsCIRelation> assemblyRels =
cmProcessor.getToCIRelations(envId, BASE_REALIZED_IN, null, ACCOUNT_ASSEMBLY);
CmsCI assembly = getAssembly(envId, assemblyRels);
CmsCI env = getEnv(envId);
String nsPath = getManifestNsPath(env);
// check for openRelease
check4OpenRelease(env, nsPath);
Long nsId = trUtil.verifyAndCreateNS(nsPath);
logger.info("Created nsId " + nsId);
List<CmsCIRelation> designPlatRels =
cmProcessor.getFromCIRelations(assembly.getCiId(), null, "ComposedOf", CATALOG_PLATFORM);
// we need to reset all pending deletions cis just in case there was one added back
cmProcessor.resetDeletionsByNs(nsPath);
// check for edge case scenario when there is new design platform with the same name as old one
// but different pack
long releaseId = checkPlatformPackCompliance(designPlatRels, env, nsPath, userId);
if (releaseId > 0) {
// stop any processing and return new release id
return releaseId;
}
final CountDownLatch latch = new CountDownLatch(designPlatRels.size());
List<Future<DesignCIManifestRfcTouple>> submittedFutureTasks = new ArrayList<>();
for (CmsCIRelation platRelation : designPlatRels) {
String availMode = getPlatformAvailabiltyMode(platModes, platRelation);
Future<DesignCIManifestRfcTouple> future =
executorService.submit(
new ManifestRfcProcessorTask(env, nsPath, userId, availMode, latch, platRelation));
submittedFutureTasks.add(future);
}
boolean allPlatsProcessed;
try {
allPlatsProcessed =
latch.await(
timeoutInMilliSeconds,
TimeUnit
.MILLISECONDS); // wait for all platform processing threads to finish with timeout
// of 10 mins
if (!allPlatsProcessed) {
logger.error(
"All platforms not processed within timeout duration of " + timeoutInMilliSeconds);
throw new TransistorException(
CmsError.TRANSISTOR_OPEN_MANIFEST_RELEASE,
"Failed to pull latest design for all platform within timeout duration of "
+ timeoutInMilliSeconds
+ " millis");
}
} catch (InterruptedException ie) {
for (Future<DesignCIManifestRfcTouple> job : submittedFutureTasks) {
job.cancel(true);
}
throw new TransistorException(
CmsError.TRANSISTOR_OPEN_MANIFEST_RELEASE, "Design pull process interrupted. ");
}
Map<Long, CmsRfcCI> design2manifestPlatMap = new HashMap<>();
for (Future<DesignCIManifestRfcTouple> task : submittedFutureTasks) {
DesignCIManifestRfcTouple touple;
try {
touple = task.get();
processPlatformRfcs(touple.manifestPlatformRfcs, userId);
CmsRfcCI manifestPlatformRfc = touple.manifestPlatformRfcs.getManifestPlatformRfc();
logger.info(
"Finished working on ="
+ manifestPlatformRfc.getNsPath()
+ " release id = "
+ manifestPlatformRfc.getReleaseId());
design2manifestPlatMap.put(touple.designPlatCI, manifestPlatformRfc);
} catch (Exception e) {
logger.error("Error in pulling latest design for all platforms ", e);
throw new TransistorException(
CmsError.TRANSISTOR_OPEN_MANIFEST_RELEASE,
"Error in pulling latest design for all platforms ");
}
}
check4MissingServices(design2manifestPlatMap);
// now we need to process linkedTo relations
manifestRfcProcessor.processLinkedTo(design2manifestPlatMap, nsPath, userId);
// now lets delete old existing plats that do not exists in new manifest
manifestRfcProcessor.processDeletedPlatforms(
design2manifestPlatMap.values(), env, nsPath, userId);
// process global variables from design
manifestRfcProcessor.processGlobalVars(assembly.getCiId(), env, nsPath, userId);
long t2 = System.currentTimeMillis();
long envReleaseId = populateParentRelease(env, nsPath);
logger.info(
"Pull design for "
+ nsPath
+ " completed in "
+ (t2 - t1)
+ " millis (releaseId "
+ envReleaseId
+ ")");
return envReleaseId;
}