/**
* Remove kubernetes services if available for application cluster.
*
* @param applicationId
* @param clusterId
*/
public static void removeKubernetesServices(String applicationId, String clusterId) {
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
if (clusterContext != null) {
String kubernetesClusterId = clusterContext.getKubernetesClusterId();
if (org.apache.commons.lang3.StringUtils.isNotBlank(kubernetesClusterId)) {
KubernetesClusterContext kubernetesClusterContext =
CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId);
if (kubernetesClusterContext != null) {
KubernetesApiClient kubernetesApiClient = kubernetesClusterContext.getKubApi();
for (KubernetesService kubernetesService : clusterContext.getKubernetesServices()) {
log.info(
String.format(
"Deleting kubernetes service: [application-id] %s " + "[service-id] %s",
applicationId, kubernetesService.getId()));
try {
kubernetesApiClient.deleteService(kubernetesService.getId());
} catch (KubernetesClientException e) {
log.error(
String.format(
"Could not delete kubernetes service: [application-id] %s "
+ "[service-id] %s",
applicationId, kubernetesService.getId()));
}
}
}
}
}
}
/**
* Terminate all the containers belong to a cluster by cluster id.
*
* @param clusterId
* @return
* @throws InvalidClusterException
*/
public MemberContext[] terminateContainers(String clusterId) throws InvalidClusterException {
Lock lock = null;
try {
lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
handleNullObject(
clusterContext,
"Could not terminate containers, cluster not found: [cluster-id] " + clusterId);
String kubernetesClusterId = clusterContext.getKubernetesClusterId();
handleNullObject(
kubernetesClusterId,
"Could not terminate containers, kubernetes cluster id not found: "
+ "[cluster-id] "
+ clusterId);
KubernetesClusterContext kubClusterContext =
CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId);
handleNullObject(
kubClusterContext,
"Could not terminate containers, kubernetes cluster not found: "
+ "[kubernetes-cluster-id] "
+ kubernetesClusterId);
KubernetesApiClient kubApi = kubClusterContext.getKubApi();
// Remove kubernetes services
List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
if (kubernetesServices != null) {
for (KubernetesService kubernetesService : kubernetesServices) {
try {
kubApi.deleteService(kubernetesService.getId());
int allocatedPort = kubernetesService.getPort();
kubClusterContext.deallocatePort(allocatedPort);
} catch (KubernetesClientException e) {
log.error("Could not remove kubernetes service: [cluster-id] " + clusterId, e);
}
}
}
List<MemberContext> memberContextsRemoved = new ArrayList<MemberContext>();
List<MemberContext> memberContexts =
CloudControllerContext.getInstance().getMemberContextsOfClusterId(clusterId);
if (memberContexts != null) {
for (MemberContext memberContext : memberContexts) {
try {
MemberContext memberContextRemoved = terminateContainer(memberContext.getMemberId());
memberContextsRemoved.add(memberContextRemoved);
} catch (MemberTerminationFailedException e) {
String message =
"Could not terminate container: [member-id] " + memberContext.getMemberId();
log.error(message);
}
}
}
// Persist changes
CloudControllerContext.getInstance().persist();
return memberContextsRemoved.toArray(new MemberContext[memberContextsRemoved.size()]);
} finally {
if (lock != null) {
CloudControllerContext.getInstance().releaseWriteLock(lock);
}
}
}
/**
* Creates and returns proxy services for the cluster.
*
* @param kubernetesApi
* @param clusterContext
* @param kubernetesCluster
* @param kubernetesClusterContext
* @throws KubernetesClientException
*/
private void createKubernetesServices(
KubernetesApiClient kubernetesApi,
ClusterContext clusterContext,
KubernetesCluster kubernetesCluster,
KubernetesClusterContext kubernetesClusterContext)
throws KubernetesClientException {
String clusterId = clusterContext.getClusterId();
String cartridgeType = clusterContext.getCartridgeUuid();
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String message =
"Could not create kubernetes services, cartridge not found: [cartridge] " + cartridgeType;
log.error(message);
throw new RuntimeException(message);
}
String sessionAffinity = null;
Property sessionAffinityProperty =
cartridge.getProperties().getProperty(KUBERNETES_SERVICE_SESSION_AFFINITY);
if (sessionAffinityProperty != null) {
sessionAffinity = sessionAffinityProperty.getValue();
}
List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
if (kubernetesServices == null) {
kubernetesServices = new ArrayList<KubernetesService>();
}
// Prepare minion public IP addresses
List<String> minionPrivateIPList = new ArrayList<String>();
List<String> minionPublicIPList = new ArrayList<String>();
KubernetesHost[] kubernetesHosts = kubernetesCluster.getKubernetesHosts();
if ((kubernetesHosts == null)
|| (kubernetesHosts.length == 0)
|| (kubernetesHosts[0] == null)) {
throw new RuntimeException(
"Hosts not found in kubernetes cluster: [cluster] " + kubernetesCluster.getClusterUuid());
}
for (KubernetesHost host : kubernetesHosts) {
if (host != null) {
minionPrivateIPList.add(host.getPrivateIPAddress());
minionPublicIPList.add(host.getPublicIPAddress());
}
}
if (log.isDebugEnabled()) {
log.debug(String.format("Minion private IPs: %s", minionPrivateIPList));
}
Collection<ClusterPortMapping> clusterPortMappings =
CloudControllerContext.getInstance()
.getClusterPortMappings(clusterContext.getApplicationUuid(), clusterId);
if (clusterPortMappings != null) {
for (ClusterPortMapping clusterPortMapping : clusterPortMappings) {
// Skip if already created
int containerPort = clusterPortMapping.getPort();
if (kubernetesServiceExist(kubernetesServices, containerPort)) {
continue;
}
// Find next service sequence no
long serviceSeqNo = kubernetesClusterContext.getServiceSeqNo().incrementAndGet();
String serviceId =
KubernetesIaasUtil.fixSpecialCharacters("service" + "-" + (serviceSeqNo));
String serviceLabel = DigestUtils.md5Hex(clusterId);
if (log.isInfoEnabled()) {
log.info(
String.format(
"Creating kubernetes service: [cluster] %s [service] %s [service-label] %s "
+ "[protocol] %s [service-port] %d [container-port] %s",
clusterId,
serviceId,
serviceLabel,
clusterPortMapping.getProtocol(),
clusterPortMapping.getKubernetesServicePort(),
containerPort));
}
// Create kubernetes service for port mapping
int servicePort = clusterPortMapping.getKubernetesServicePort();
String serviceType = clusterPortMapping.getKubernetesServiceType();
String containerPortName =
KubernetesIaasUtil.preparePortNameFromPortMapping(clusterPortMapping);
try {
// If kubernetes service is already created, skip creating a new one
if (kubernetesApi.getService(serviceId) == null) {
// Services need to use minions private IP addresses for creating iptable rules
kubernetesApi.createService(
serviceId,
serviceLabel,
servicePort,
serviceType,
containerPortName,
containerPort,
sessionAffinity);
} else {
if (log.isDebugEnabled()) {
log.debug(
String.format(
"Kubernetes service is already created: [cluster] %s [service] %s "
+ "[protocol] %s [service-port] %d [container-port] %d",
clusterId,
serviceId,
clusterPortMapping.getProtocol(),
servicePort,
containerPort));
}
}
} finally {
// Persist kubernetes service sequence no
CloudControllerContext.getInstance().persist();
}
try {
Thread.sleep(1000);
} catch (InterruptedException ignore) {
}
Service service = kubernetesApi.getService(serviceId);
KubernetesService kubernetesService = new KubernetesService();
kubernetesService.setId(service.getMetadata().getName());
kubernetesService.setPortalIP(service.getSpec().getClusterIP());
// Expose minions public IP addresses as they need to be accessed by external networks
String[] minionPublicIPArray =
minionPublicIPList.toArray(new String[minionPublicIPList.size()]);
kubernetesService.setPublicIPs(minionPublicIPArray);
kubernetesService.setProtocol(clusterPortMapping.getProtocol());
kubernetesService.setPortName(clusterPortMapping.getName());
String kubernetesServiceType = service.getSpec().getType();
kubernetesService.setServiceType(kubernetesServiceType);
if (kubernetesServiceType.equals(KubernetesConstants.NODE_PORT)) {
kubernetesService.setPort(service.getSpec().getPorts().get(0).getNodePort());
} else {
kubernetesService.setPort(service.getSpec().getPorts().get(0).getPort());
}
kubernetesService.setContainerPort(containerPort);
kubernetesServices.add(kubernetesService);
if (log.isInfoEnabled()) {
log.info(
String.format(
"Kubernetes service successfully created: [cluster] %s [service] %s "
+ "[protocol] %s [node-port] %d [container-port] %s",
clusterId,
serviceId,
clusterPortMapping.getProtocol(),
servicePort,
containerPort));
}
}
}
// Add kubernetes services to cluster context and persist
clusterContext.setKubernetesServices(kubernetesServices);
CloudControllerContext.getInstance().persist();
}
public static void handleClusterActivatedEvent(
ClusterStatusClusterActivatedEvent clusterStatusClusterActivatedEvent) {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(clusterStatusClusterActivatedEvent.getServiceName());
// update the status of the cluster
if (service == null) {
log.warn(
String.format(
"Service %s does not exist", clusterStatusClusterActivatedEvent.getServiceName()));
return;
}
Cluster cluster = service.getCluster(clusterStatusClusterActivatedEvent.getClusterId());
if (cluster == null) {
log.warn(
String.format(
"Cluster %s does not exist", clusterStatusClusterActivatedEvent.getClusterId()));
return;
}
String clusterId = cluster.getClusterId();
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
if (clusterContext == null) {
log.warn("Cluster context not found: [cluster-id] " + clusterId);
return;
}
ClusterInstanceActivatedEvent clusterInstanceActivatedEvent =
new ClusterInstanceActivatedEvent(
clusterStatusClusterActivatedEvent.getAppId(),
clusterStatusClusterActivatedEvent.getServiceName(),
clusterStatusClusterActivatedEvent.getClusterId(),
clusterStatusClusterActivatedEvent.getInstanceId());
try {
TopologyManager.acquireWriteLock();
List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
cluster.setKubernetesServices(kubernetesServices);
if (kubernetesServices != null) {
// Set kubernetes services
cluster.setKubernetesServices(kubernetesServices);
try {
// Generate access URLs for kubernetes services
for (KubernetesService kubernetesService : kubernetesServices) {
if (kubernetesService.getServiceType().equals(KubernetesConstants.NODE_PORT)) {
// Public IP = Kubernetes minion public IP
String[] publicIPs = kubernetesService.getPublicIPs();
if ((publicIPs != null) && (publicIPs.length > 0)) {
for (String publicIP : publicIPs) {
// There can be a String array with null values
if (publicIP != null) {
// Using type URI since only http, https, ftp, file, jar protocols are supported
// in URL
URI accessURL =
new URI(
kubernetesService.getProtocol(),
null,
publicIP,
kubernetesService.getPort(),
null,
null,
null);
cluster.addAccessUrl(accessURL.toString());
clusterInstanceActivatedEvent.addAccessUrl(accessURL.toString());
} else {
log.error(
String.format(
"Could not create access URL for [Kubernetes-service] %s , "
+ "since Public IP is not available",
kubernetesService.getId()));
}
}
}
}
}
} catch (URISyntaxException e) {
log.error("Could not create access URLs for Kubernetes services", e);
}
}
ClusterInstance context =
cluster.getInstanceContexts(clusterStatusClusterActivatedEvent.getInstanceId());
if (context == null) {
log.warn(
"Cluster instance context is not found for [cluster] "
+ clusterStatusClusterActivatedEvent.getClusterId()
+ " [instance-id] "
+ clusterStatusClusterActivatedEvent.getInstanceId());
return;
}
ClusterStatus status = ClusterStatus.Active;
if (context.isStateTransitionValid(status)) {
context.setStatus(status);
log.info("Cluster activated adding status started for " + cluster.getClusterId());
TopologyManager.updateTopology(topology);
// publish event
TopologyEventPublisher.sendClusterActivatedEvent(clusterInstanceActivatedEvent);
} else {
log.error(
String.format(
"Cluster state transition is not valid: [cluster-id] %s "
+ " [instance-id] %s [current-status] %s [status-requested] %s",
clusterStatusClusterActivatedEvent.getClusterId(),
clusterStatusClusterActivatedEvent.getInstanceId(),
context.getStatus(),
status));
return;
}
} finally {
TopologyManager.releaseWriteLock();
}
}
public static void handleMemberActivated(InstanceActivatedEvent instanceActivatedEvent) {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(instanceActivatedEvent.getServiceName());
if (service == null) {
log.warn(String.format("Service %s does not exist", instanceActivatedEvent.getServiceName()));
return;
}
Cluster cluster = service.getCluster(instanceActivatedEvent.getClusterId());
if (cluster == null) {
log.warn(String.format("Cluster %s does not exist", instanceActivatedEvent.getClusterId()));
return;
}
Member member = cluster.getMember(instanceActivatedEvent.getMemberId());
if (member == null) {
log.warn(String.format("Member %s does not exist", instanceActivatedEvent.getMemberId()));
return;
}
MemberActivatedEvent memberActivatedEvent =
new MemberActivatedEvent(
instanceActivatedEvent.getServiceName(),
instanceActivatedEvent.getClusterId(),
instanceActivatedEvent.getClusterInstanceId(),
instanceActivatedEvent.getMemberId(),
instanceActivatedEvent.getNetworkPartitionId(),
instanceActivatedEvent.getPartitionId());
// grouping - set grouid
// TODO
memberActivatedEvent.setApplicationId(null);
try {
TopologyManager.acquireWriteLock();
// try update lifecycle state
if (!member.isStateTransitionValid(MemberStatus.Active)) {
log.error(
"Invalid state transition from ["
+ member.getStatus()
+ "] to ["
+ MemberStatus.Active
+ "]");
return;
} else {
member.setStatus(MemberStatus.Active);
// Set member ports
try {
Cartridge cartridge =
CloudControllerContext.getInstance().getCartridge(service.getServiceName());
if (cartridge == null) {
throw new RuntimeException(
String.format(
"Cartridge not found: [cartridge-type] %s", service.getServiceName()));
}
Port port;
int portValue;
List<PortMapping> portMappings = Arrays.asList(cartridge.getPortMappings());
String clusterId = cluster.getClusterId();
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
for (PortMapping portMapping : portMappings) {
if (kubernetesServices != null) {
portValue = findKubernetesServicePort(clusterId, kubernetesServices, portMapping);
} else {
portValue = portMapping.getPort();
}
port = new Port(portMapping.getProtocol(), portValue, portMapping.getProxyPort());
member.addPort(port);
memberActivatedEvent.addPort(port);
}
} catch (Exception e) {
String message =
String.format(
"Could not add member ports: [service-name] %s [member-id] %s",
memberActivatedEvent.getServiceName(), memberActivatedEvent.getMemberId());
log.error(message, e);
}
// Set member ip addresses
memberActivatedEvent.setDefaultPrivateIP(member.getDefaultPrivateIP());
memberActivatedEvent.setMemberPrivateIPs(member.getMemberPrivateIPs());
memberActivatedEvent.setDefaultPublicIP(member.getDefaultPublicIP());
memberActivatedEvent.setMemberPublicIPs(member.getMemberPublicIPs());
TopologyManager.updateTopology(topology);
// Publish member activated event
TopologyEventPublisher.sendMemberActivatedEvent(memberActivatedEvent);
// Publish statistics data
BAMUsageDataPublisher.publish(
memberActivatedEvent.getMemberId(),
memberActivatedEvent.getPartitionId(),
memberActivatedEvent.getNetworkPartitionId(),
memberActivatedEvent.getClusterId(),
memberActivatedEvent.getServiceName(),
MemberStatus.Active.toString(),
null);
}
} finally {
TopologyManager.releaseWriteLock();
}
}