@Override
public List<Integer> getServiceInstanceUsedOrderIds(Service service, String launchConfigName) {
Environment env =
objectManager.findOne(Environment.class, ENVIRONMENT.ID, service.getEnvironmentId());
// get all existing instances to check if the name is in use by the instance of the same service
List<Integer> usedIds = new ArrayList<>();
// list all the instances
List<? extends ServiceExposeMap> instanceServiceMaps =
exposeMapDao.getNonRemovedServiceInstanceMap(service.getId());
for (ServiceExposeMap instanceServiceMap : instanceServiceMaps) {
Instance instance =
objectManager.loadResource(Instance.class, instanceServiceMap.getInstanceId());
if (ServiceDiscoveryUtil.isServiceGeneratedName(env, service, instance)) {
String configName =
launchConfigName == null
|| launchConfigName.equals(ServiceDiscoveryConstants.PRIMARY_LAUNCH_CONFIG_NAME)
? ""
: launchConfigName + "_";
String id =
instance
.getName()
.replace(
String.format("%s_%s_%s", env.getName(), service.getName(), configName), "");
if (id.matches("\\d+")) {
usedIds.add(Integer.valueOf(id));
}
}
}
return usedIds;
}
public Condition getInstanceHostConstraint(Instance instance) {
if (StringUtils.isEmpty(instance.getDeploymentUnitUuid())) {
return INSTANCE_HOST_MAP.INSTANCE_ID.eq(instance.getId());
} else {
return INSTANCE.DEPLOYMENT_UNIT_UUID.eq(instance.getDeploymentUnitUuid());
}
}
protected Map<String, Object> lookupCacheInstanceData(long instanceId) {
Instance instance = objectManager.loadResource(Instance.class, instanceId);
if (instance == null) {
return Collections.emptyMap();
}
Map<String, Object> newData = new HashMap<>();
newData.put(DataUtils.FIELDS, instance.getData().get(DataUtils.FIELDS));
return newData;
}
protected void addInstanceToDeploymentUnits(
Map<String, List<Instance>> deploymentUnitInstancesToUpgrade, Instance instance) {
List<Instance> toRemove =
deploymentUnitInstancesToUpgrade.get(instance.getDeploymentUnitUuid());
if (toRemove == null) {
toRemove = new ArrayList<Instance>();
}
toRemove.add(instance);
deploymentUnitInstancesToUpgrade.put(instance.getDeploymentUnitUuid(), toRemove);
}
@Override
public boolean isUnhealthy() {
if (instance != null) {
return instance.getHealthState() != null
&& (instance
.getHealthState()
.equalsIgnoreCase(HealthcheckConstants.HEALTH_STATE_UNHEALTHY)
|| instance
.getHealthState()
.equalsIgnoreCase(HealthcheckConstants.HEALTH_STATE_UPDATING_UNHEALTHY));
}
return false;
}
@Override
public void stop() {
if (instance != null && instance.getState().equals(InstanceConstants.STATE_RUNNING)) {
context.objectProcessManager.scheduleProcessInstanceAsync(
InstanceConstants.PROCESS_STOP, instance, null);
}
}
/**
* Supported macros ${service_name} ${stack_name} LEGACY: ${project_name}
*
* @param valueStr
* @param instance
* @return
*/
private String evaluateMacros(String valueStr, Instance instance) {
if (valueStr.indexOf(SERVICE_NAME_MACRO) != -1
|| valueStr.indexOf(STACK_NAME_MACRO) != -1
|| valueStr.indexOf(PROJECT_NAME_MACRO) != -1) {
List<Label> labels = labelsDao.getLabelsForInstance(instance.getId());
String serviceLaunchConfigName = "";
String stackName = "";
for (Label label : labels) {
if (LABEL_STACK_NAME.equals(label.getKey())) {
stackName = label.getValue();
} else if (LABEL_STACK_SERVICE_NAME.equals(label.getKey())) {
if (label.getValue() != null) {
int i = label.getValue().indexOf('/');
if (i != -1) {
serviceLaunchConfigName = label.getValue().substring(i + 1);
}
}
}
}
if (!StringUtils.isBlank(stackName)) {
valueStr = valueStr.replace(STACK_NAME_MACRO, stackName);
// LEGACY: ${project_name} rename ${stack_name}
valueStr = valueStr.replace(PROJECT_NAME_MACRO, stackName);
}
if (!StringUtils.isBlank(serviceLaunchConfigName)) {
valueStr = valueStr.replace(SERVICE_NAME_MACRO, serviceLaunchConfigName);
}
}
return valueStr;
}
protected void removeLoadBalancerInstance(
LoadBalancer loadBalancer, LoadBalancerHostMap hostMap) {
Instance lbInstance = lbInstanceManager.getLoadBalancerInstance(hostMap);
if (lbInstance != null
&& !(lbInstance.getState().equalsIgnoreCase(CommonStatesConstants.REMOVED)
|| lbInstance.getState().equals(CommonStatesConstants.REMOVING))) {
// try to remove first
try {
objectProcessManager.scheduleStandardProcess(StandardProcess.REMOVE, lbInstance, null);
} catch (ProcessCancelException e) {
objectProcessManager.scheduleProcessInstance(
InstanceConstants.PROCESS_STOP,
lbInstance,
CollectionUtils.asMap(InstanceConstants.REMOVE_OPTION, true));
}
}
}
protected void modifyDisk(long hostId, Instance instance, boolean add) {
CacheManager cm = CacheManager.getCacheManagerInstance(this.objectManager);
HostInfo hostInfo = cm.getHostInfo(hostId, false);
if (hostInfo == null) {
// we never tried to schedule disks at all in the past
return;
}
InstanceInfo instanceInfo = hostInfo.getInstanceInfo(instance.getId());
if (instanceInfo == null) {
// we never tried to schedule disks at all during constraint scheduling
return;
}
if (add) {
Map<Pair<String, Long>, DiskInfo> volumeToDiskMapping =
AllocatorUtils.allocateDiskForVolumes(hostId, instance, this.objectManager);
if (volumeToDiskMapping == null) {
return;
}
for (Entry<Pair<String, Long>, DiskInfo> mapping : volumeToDiskMapping.entrySet()) {
Pair<String, Long> vol = mapping.getKey();
DiskInfo disk = mapping.getValue();
Long allocated = disk.getAllocatedSize();
disk.addAllocatedSize(vol.getRight());
log.info(
"allocated disk space on disk [{}] with total = {}, {} {} {} = {} as used",
disk.getDiskDevicePath(),
disk.getCapacity(),
allocated,
"+",
vol.getRight(),
allocated + vol.getRight());
// record to cache for deletion purpose
instanceInfo.addReservedSize(disk.getDiskDevicePath(), vol.getRight());
}
} else {
for (Entry<String, Long> diskAllocated : instanceInfo.getAllocatedDisks()) {
String diskDevicePath = diskAllocated.getKey();
Long reserveSize = diskAllocated.getValue();
DiskInfo diskInfo = hostInfo.getDiskInfo(diskDevicePath);
diskInfo.freeAllocatedSize(reserveSize);
log.info(
"freed disk space on disk [{}] with total = {}, {} {} {} = {} as used",
diskInfo.getDiskDevicePath(),
diskInfo.getCapacity(),
diskInfo.getAllocatedSize(),
"-",
reserveSize,
diskInfo.getAllocatedSize() - reserveSize);
// release the reserved disk for this instance
instanceInfo.releaseDisk(diskDevicePath);
}
}
}
@Override
public List<? extends Service> findServicesFor(Instance instance) {
return create()
.select(SERVICE.fields())
.from(SERVICE)
.join(SERVICE_EXPOSE_MAP)
.on(SERVICE_EXPOSE_MAP.SERVICE_ID.eq(SERVICE.ID))
.where(SERVICE_EXPOSE_MAP.INSTANCE_ID.eq(instance.getId()))
.fetchInto(ServiceRecord.class);
}
protected List<? extends Instance> getServiceInstancesToRestart(Service service) {
// get all instances of the service
List<? extends Instance> instances = exposeMapDao.listServiceManagedInstances(service.getId());
List<Instance> toRestart = new ArrayList<>();
ServiceRestart svcRestart =
DataAccessor.field(
service, ServiceDiscoveryConstants.FIELD_RESTART, jsonMapper, ServiceRestart.class);
RollingRestartStrategy strategy = svcRestart.getRollingRestartStrategy();
Map<Long, Long> instanceToStartCount = strategy.getInstanceToStartCount();
// compare its start_count with one set on the service restart field
for (Instance instance : instances) {
if (instanceToStartCount.containsKey(instance.getId())) {
Long previousStartCount = instanceToStartCount.get(instance.getId());
if (previousStartCount == instance.getStartCount()) {
toRestart.add(instance);
}
}
}
return toRestart;
}
protected void processPorts(Instance instance) {
Set<String> portSpecs = new HashSet<>();
for (Port port : objectManager.children(instance, Port.class)) {
if (port.getRemoved() != null) {
continue;
}
portSpecs.add(new PortSpec(port).toSpec());
}
objectManager.setFields(instance, InstanceConstants.FIELD_PORTS, new ArrayList<>(portSpecs));
instanceDao.clearCacheInstanceData(instance.getId());
}
protected void stopInstances(Map<String, List<Instance>> deploymentUnitInstancesToStop) {
List<Instance> toStop = new ArrayList<>();
for (String key : deploymentUnitInstancesToStop.keySet()) {
toStop.addAll(deploymentUnitInstancesToStop.get(key));
}
for (Instance instance : toStop) {
if (!instance.getState().equalsIgnoreCase(InstanceConstants.STATE_STOPPED)) {
objectProcessMgr.scheduleProcessInstanceAsync(
InstanceConstants.PROCESS_STOP, instance, null);
}
}
for (Instance instance : toStop) {
resourceMntr.waitFor(
instance,
new ResourcePredicate<Instance>() {
@Override
public boolean evaluate(Instance obj) {
return InstanceConstants.STATE_STOPPED.equals(obj.getState());
}
});
}
}
@Override
protected Object getAgentResource(
ProcessState state, ProcessInstance process, Object dataResource) {
Instance instance = getInstance(state);
if (!isPod(instance)) {
return false;
}
Long accountId = instance.getAccountId();
List<Long> agentIds =
agentInstanceDao.getAgentProvider(
SystemLabels.LABEL_AGENT_SERVICE_LABELS_PROVIDER, accountId);
Long agentId = agentIds.size() == 0 ? null : agentIds.get(0);
if ((instance instanceof Instance)
&& (agentIds.contains(instance.getAgentId()) || instance.getSystem())) {
return null;
}
if (agentId == null) {
throw new ExecutionException("Failed to find labels provider", instance);
}
return agentId;
}
public void cleanupUpgradedInstances(Service service) {
List<? extends ServiceExposeMap> maps = exposeMapDao.getInstancesSetForUpgrade(service.getId());
List<Instance> waitList = new ArrayList<>();
for (ServiceExposeMap map : maps) {
Instance instance = objectManager.loadResource(Instance.class, map.getInstanceId());
if (instance == null
|| instance.getState().equals(CommonStatesConstants.REMOVED)
|| instance.getState().equals(CommonStatesConstants.REMOVING)) {
continue;
}
try {
objectProcessMgr.scheduleProcessInstanceAsync(
InstanceConstants.PROCESS_REMOVE, instance, null);
} catch (ProcessCancelException ex) {
// in case instance was manually restarted
objectProcessMgr.scheduleProcessInstanceAsync(
InstanceConstants.PROCESS_STOP,
instance,
ProcessUtils.chainInData(
new HashMap<String, Object>(),
InstanceConstants.PROCESS_STOP,
InstanceConstants.PROCESS_REMOVE));
}
}
for (Instance instance : waitList) {
resourceMntr.waitFor(
instance,
new ResourcePredicate<Instance>() {
@Override
public boolean evaluate(Instance obj) {
return CommonStatesConstants.REMOVED.equals(obj.getState());
}
});
}
}
@Override
protected void postProcessEvent(
EventVO<?> event,
Event reply,
ProcessState state,
ProcessInstance process,
Object eventResource,
Object dataResource,
Object agentResource) {
Map<String, String> labels =
CollectionUtils.toMap(
CollectionUtils.getNestedValue(
reply.getData(), "instance", "+data", "+fields", "+labels"));
if (labels.size() == 0) {
labels =
CollectionUtils.toMap(
CollectionUtils.getNestedValue(
reply.getData(), "instanceHostMap", "instance", "+data", "+fields", "+labels"));
} else {
CollectionUtils.setNestedValue(
CollectionUtils.toMap(reply.getData()),
labels,
"instanceHostMap",
"instance",
"+data",
"+fields",
"+labels");
}
Instance instance = getInstance(state);
for (Map.Entry<String, String> label : labels.entrySet()) {
labelsService.createContainerLabel(
instance.getAccountId(), instance.getId(), label.getKey(), label.getValue());
}
}
@Override
public boolean recordCandidate(AllocationAttempt attempt, AllocationCandidate candidate) {
Long newHost = candidate.getHost();
if (newHost != null) {
for (Instance instance : attempt.getInstances()) {
log.info("Associating instance [{}] to host [{}]", instance.getId(), newHost);
objectManager.create(
InstanceHostMap.class,
INSTANCE_HOST_MAP.HOST_ID,
newHost,
INSTANCE_HOST_MAP.INSTANCE_ID,
instance.getId());
modifyDisk(newHost, instance, true);
List<Volume> vols = InstanceHelpers.extractVolumesFromMounts(instance, objectManager);
for (Volume v : vols) {
if (VolumeConstants.ACCESS_MODE_SINGLE_HOST_RW.equals(v.getAccessMode())) {
objectManager.setFields(v, VOLUME.HOST_ID, newHost);
}
}
}
}
Map<Long, Set<Long>> existingPools = attempt.getPoolIds();
Map<Long, Set<Long>> newPools = candidate.getPools();
if (!existingPools.keySet().equals(newPools.keySet())) {
throw new IllegalStateException(
String.format(
"Volumes don't match. currently %s, new %s",
existingPools.keySet(), newPools.keySet()));
}
for (Map.Entry<Long, Set<Long>> entry : newPools.entrySet()) {
long volumeId = entry.getKey();
Set<Long> existingPoolsForVol = existingPools.get(entry.getKey());
Set<Long> newPoolsForVol = entry.getValue();
if (existingPoolsForVol == null || existingPoolsForVol.size() == 0) {
for (long poolId : newPoolsForVol) {
log.info("Associating volume [{}] to storage pool [{}]", volumeId, poolId);
objectManager.create(
VolumeStoragePoolMap.class,
VOLUME_STORAGE_POOL_MAP.VOLUME_ID,
volumeId,
VOLUME_STORAGE_POOL_MAP.STORAGE_POOL_ID,
poolId);
}
} else if (!existingPoolsForVol.equals(newPoolsForVol)) {
throw new IllegalStateException(
String.format(
"Can not move volume %s, currently: %s, new: %s",
volumeId, existingPools, newPools));
}
}
for (Nic nic : attempt.getNics()) {
Long subnetId = candidate.getSubnetIds().get(nic.getId());
if (subnetId == null
|| (nic.getSubnetId() != null && subnetId.longValue() == nic.getSubnetId())) {
continue;
}
log.info("Associating nic [{}] to subnet [{}]", nic.getId(), subnetId);
int i =
create().update(NIC).set(NIC.SUBNET_ID, subnetId).where(NIC.ID.eq(nic.getId())).execute();
if (i != 1) {
throw new IllegalStateException(
"Expected to update nic id ["
+ nic.getId()
+ "] with subnet ["
+ subnetId
+ "] but update ["
+ i
+ "] rows");
}
}
return true;
}
@Override
public boolean isHealthCheckInitializing() {
return instance != null
&& HealthcheckConstants.HEALTH_STATE_INITIALIZING.equals(instance.getHealthState());
}
@Override
public List<DnsEntryData> getServiceDnsData(
final Instance instance, final boolean isVIPProvider) {
MultiRecordMapper<ServiceDnsEntryData> mapper =
new MultiRecordMapper<ServiceDnsEntryData>() {
@Override
protected ServiceDnsEntryData map(List<Object> input) {
Service clientService = (Service) input.get(0);
Service targetService = (Service) input.get(1);
ServiceConsumeMap consumeMap = (ServiceConsumeMap) input.get(2);
ServiceDnsEntryData data =
new ServiceDnsEntryData(clientService, targetService, consumeMap);
return data;
}
};
ServiceTable clientService = mapper.add(SERVICE);
ServiceTable targetService = mapper.add(SERVICE);
ServiceConsumeMapTable serviceConsumeMap = mapper.add(SERVICE_CONSUME_MAP);
// there are 2 conditions below linked with OR clause
// first condition - means to return all non-dns clientService + target service map within the
// same stack
// that are not linked
// second condition - return only clientService + targetService with explicit links
Condition condition =
(clientService
.KIND
.ne(ServiceDiscoveryConstants.KIND.DNSSERVICE.name())
.and(targetService.ENVIRONMENT_ID.eq(clientService.ENVIRONMENT_ID))
.and(serviceConsumeMap.ID.isNull()))
.or(
serviceConsumeMap
.ID
.isNotNull()
.and(serviceConsumeMap.REMOVED.isNull())
.and(
serviceConsumeMap.STATE.in(
CommonStatesConstants.ACTIVATING, CommonStatesConstants.ACTIVE)));
List<ServiceDnsEntryData> serviceDnsEntries =
create()
.select(mapper.fields())
.from(clientService)
.join(targetService)
.on(targetService.ACCOUNT_ID.eq(clientService.ACCOUNT_ID))
.leftOuterJoin(serviceConsumeMap)
.on(
serviceConsumeMap
.SERVICE_ID
.eq(clientService.ID)
.and(serviceConsumeMap.CONSUMED_SERVICE_ID.eq(targetService.ID))
.and(serviceConsumeMap.REMOVED.isNull()))
.where(targetService.REMOVED.isNull())
.and(clientService.REMOVED.isNull())
.and(condition)
.fetch()
.map(mapper);
Nic nic = ntwkDao.getPrimaryNic(instance.getId());
long vnetId = nic.getVnetId();
return convertToDnsEntryData(isVIPProvider, serviceDnsEntries, instance.getAccountId(), vnetId);
}
@Override
public List<DnsEntryData> getInstanceLinksDnsData(final Instance instance) {
MultiRecordMapper<DnsEntryData> mapper =
new MultiRecordMapper<DnsEntryData>() {
@Override
protected DnsEntryData map(List<Object> input) {
DnsEntryData data = new DnsEntryData();
Map<String, Map<String, String>> resolve = new HashMap<>();
Map<String, String> ips = new HashMap<>();
String targetInstanceName =
input.get(4) == null ? null : ((Instance) input.get(4)).getName();
ips.put(((IpAddress) input.get(1)).getAddress(), targetInstanceName);
// add all instance links
resolve.put(((InstanceLink) input.get(0)).getLinkName(), ips);
data.setSourceIpAddress(((IpAddress) input.get(2)).getAddress());
data.setResolveServicesAndContainers(resolve);
data.setInstance((Instance) input.get(3));
return data;
}
};
InstanceLinkTable instanceLink = mapper.add(INSTANCE_LINK);
IpAddressTable targetIpAddress = mapper.add(IP_ADDRESS);
IpAddressTable clientIpAddress = mapper.add(IP_ADDRESS);
InstanceTable clientInstance = mapper.add(INSTANCE);
InstanceTable targetInstance = mapper.add(INSTANCE);
NicTable clientNic = NIC.as("client_nic");
NicTable targetNic = NIC.as("target_nic");
IpAddressNicMapTable clientNicIpTable = IP_ADDRESS_NIC_MAP.as("client_nic_ip");
IpAddressNicMapTable targetNicIpTable = IP_ADDRESS_NIC_MAP.as("target_nic_ip");
return create()
.select(mapper.fields())
.from(NIC)
.join(clientNic)
.on(NIC.VNET_ID.eq(clientNic.VNET_ID))
.join(instanceLink)
.on(instanceLink.INSTANCE_ID.eq(clientNic.INSTANCE_ID))
.join(targetNic)
.on(targetNic.INSTANCE_ID.eq(instanceLink.TARGET_INSTANCE_ID))
.join(targetInstance)
.on(targetNic.INSTANCE_ID.eq(targetInstance.ID))
.join(targetNicIpTable)
.on(targetNicIpTable.NIC_ID.eq(targetNic.ID))
.join(targetIpAddress)
.on(targetNicIpTable.IP_ADDRESS_ID.eq(targetIpAddress.ID))
.join(clientNicIpTable)
.on(clientNicIpTable.NIC_ID.eq(clientNic.ID))
.join(clientIpAddress)
.on(clientNicIpTable.IP_ADDRESS_ID.eq(clientIpAddress.ID))
.join(clientInstance)
.on(clientNic.INSTANCE_ID.eq(clientInstance.ID))
.where(
NIC.INSTANCE_ID
.eq(instance.getId())
.and(NIC.VNET_ID.isNotNull())
.and(NIC.REMOVED.isNull())
.and(targetIpAddress.ROLE.eq(IpAddressConstants.ROLE_PRIMARY))
.and(targetIpAddress.REMOVED.isNull())
.and(clientIpAddress.ROLE.eq(IpAddressConstants.ROLE_PRIMARY))
.and(clientIpAddress.REMOVED.isNull())
.and(targetNicIpTable.REMOVED.isNull())
.and(clientNic.REMOVED.isNull())
.and(targetNic.REMOVED.isNull())
.and(instanceLink.REMOVED.isNull())
.and(instanceLink.SERVICE_CONSUME_MAP_ID.isNull())
.and(
targetInstance.STATE.in(
InstanceConstants.STATE_RUNNING, InstanceConstants.STATE_STARTING))
.and(
targetInstance
.HEALTH_STATE
.isNull()
.or(
targetInstance.HEALTH_STATE.eq(
HealthcheckConstants.HEALTH_STATE_HEALTHY))))
.fetch()
.map(mapper);
}
@Override
public HandlerResult handle(ProcessState state, ProcessInstance process) {
Account account = (Account) state.getResource();
for (Certificate cert : getObjectManager().children(account, Certificate.class)) {
if (cert.getRemoved() != null) {
continue;
}
deactivateThenRemove(cert, state.getData());
}
for (Credential cred : getObjectManager().children(account, Credential.class)) {
if (cred.getRemoved() != null) {
continue;
}
deactivateThenRemove(cred, state.getData());
}
for (Host host : getObjectManager().children(account, Host.class)) {
try {
deactivateThenRemove(host, state.getData());
} catch (ProcessCancelException e) {
// ignore
}
purge(host, null);
}
for (PhysicalHost host : getObjectManager().children(account, PhysicalHost.class)) {
try {
getObjectProcessManager().executeStandardProcess(StandardProcess.REMOVE, host, null);
} catch (ProcessCancelException e) {
// ignore
}
}
for (Agent agent : getObjectManager().children(account, Agent.class)) {
if (agent.getRemoved() != null) {
continue;
}
deactivateThenRemove(agent, state.getData());
}
for (Environment env : getObjectManager().children(account, Environment.class)) {
if (env.getRemoved() != null) {
continue;
}
objectProcessManager.scheduleStandardProcessAsync(StandardProcess.REMOVE, env, null);
}
for (Instance instance : instanceDao.listNonRemovedInstances(account, false)) {
deleteAgentAccount(instance.getAgentId(), state.getData());
try {
objectProcessManager.scheduleStandardProcess(StandardProcess.REMOVE, instance, null);
} catch (ProcessCancelException e) {
objectProcessManager.scheduleProcessInstance(
InstanceConstants.PROCESS_STOP,
instance,
CollectionUtils.asMap(InstanceConstants.REMOVE_OPTION, true));
}
}
accountDao.deleteProjectMemberEntries(account);
return null;
}