/**
* Creates and returns proxy services for the cluster.
*
* @param kubernetesApi
* @param clusterContext
* @param kubernetesCluster
* @param kubernetesClusterContext
* @throws KubernetesClientException
*/
private void createKubernetesServices(
KubernetesApiClient kubernetesApi,
ClusterContext clusterContext,
KubernetesCluster kubernetesCluster,
KubernetesClusterContext kubernetesClusterContext)
throws KubernetesClientException {
String clusterId = clusterContext.getClusterId();
String cartridgeType = clusterContext.getCartridgeUuid();
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String message =
"Could not create kubernetes services, cartridge not found: [cartridge] " + cartridgeType;
log.error(message);
throw new RuntimeException(message);
}
String sessionAffinity = null;
Property sessionAffinityProperty =
cartridge.getProperties().getProperty(KUBERNETES_SERVICE_SESSION_AFFINITY);
if (sessionAffinityProperty != null) {
sessionAffinity = sessionAffinityProperty.getValue();
}
List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
if (kubernetesServices == null) {
kubernetesServices = new ArrayList<KubernetesService>();
}
// Prepare minion public IP addresses
List<String> minionPrivateIPList = new ArrayList<String>();
List<String> minionPublicIPList = new ArrayList<String>();
KubernetesHost[] kubernetesHosts = kubernetesCluster.getKubernetesHosts();
if ((kubernetesHosts == null)
|| (kubernetesHosts.length == 0)
|| (kubernetesHosts[0] == null)) {
throw new RuntimeException(
"Hosts not found in kubernetes cluster: [cluster] " + kubernetesCluster.getClusterUuid());
}
for (KubernetesHost host : kubernetesHosts) {
if (host != null) {
minionPrivateIPList.add(host.getPrivateIPAddress());
minionPublicIPList.add(host.getPublicIPAddress());
}
}
if (log.isDebugEnabled()) {
log.debug(String.format("Minion private IPs: %s", minionPrivateIPList));
}
Collection<ClusterPortMapping> clusterPortMappings =
CloudControllerContext.getInstance()
.getClusterPortMappings(clusterContext.getApplicationUuid(), clusterId);
if (clusterPortMappings != null) {
for (ClusterPortMapping clusterPortMapping : clusterPortMappings) {
// Skip if already created
int containerPort = clusterPortMapping.getPort();
if (kubernetesServiceExist(kubernetesServices, containerPort)) {
continue;
}
// Find next service sequence no
long serviceSeqNo = kubernetesClusterContext.getServiceSeqNo().incrementAndGet();
String serviceId =
KubernetesIaasUtil.fixSpecialCharacters("service" + "-" + (serviceSeqNo));
String serviceLabel = DigestUtils.md5Hex(clusterId);
if (log.isInfoEnabled()) {
log.info(
String.format(
"Creating kubernetes service: [cluster] %s [service] %s [service-label] %s "
+ "[protocol] %s [service-port] %d [container-port] %s",
clusterId,
serviceId,
serviceLabel,
clusterPortMapping.getProtocol(),
clusterPortMapping.getKubernetesServicePort(),
containerPort));
}
// Create kubernetes service for port mapping
int servicePort = clusterPortMapping.getKubernetesServicePort();
String serviceType = clusterPortMapping.getKubernetesServiceType();
String containerPortName =
KubernetesIaasUtil.preparePortNameFromPortMapping(clusterPortMapping);
try {
// If kubernetes service is already created, skip creating a new one
if (kubernetesApi.getService(serviceId) == null) {
// Services need to use minions private IP addresses for creating iptable rules
kubernetesApi.createService(
serviceId,
serviceLabel,
servicePort,
serviceType,
containerPortName,
containerPort,
sessionAffinity);
} else {
if (log.isDebugEnabled()) {
log.debug(
String.format(
"Kubernetes service is already created: [cluster] %s [service] %s "
+ "[protocol] %s [service-port] %d [container-port] %d",
clusterId,
serviceId,
clusterPortMapping.getProtocol(),
servicePort,
containerPort));
}
}
} finally {
// Persist kubernetes service sequence no
CloudControllerContext.getInstance().persist();
}
try {
Thread.sleep(1000);
} catch (InterruptedException ignore) {
}
Service service = kubernetesApi.getService(serviceId);
KubernetesService kubernetesService = new KubernetesService();
kubernetesService.setId(service.getMetadata().getName());
kubernetesService.setPortalIP(service.getSpec().getClusterIP());
// Expose minions public IP addresses as they need to be accessed by external networks
String[] minionPublicIPArray =
minionPublicIPList.toArray(new String[minionPublicIPList.size()]);
kubernetesService.setPublicIPs(minionPublicIPArray);
kubernetesService.setProtocol(clusterPortMapping.getProtocol());
kubernetesService.setPortName(clusterPortMapping.getName());
String kubernetesServiceType = service.getSpec().getType();
kubernetesService.setServiceType(kubernetesServiceType);
if (kubernetesServiceType.equals(KubernetesConstants.NODE_PORT)) {
kubernetesService.setPort(service.getSpec().getPorts().get(0).getNodePort());
} else {
kubernetesService.setPort(service.getSpec().getPorts().get(0).getPort());
}
kubernetesService.setContainerPort(containerPort);
kubernetesServices.add(kubernetesService);
if (log.isInfoEnabled()) {
log.info(
String.format(
"Kubernetes service successfully created: [cluster] %s [service] %s "
+ "[protocol] %s [node-port] %d [container-port] %s",
clusterId,
serviceId,
clusterPortMapping.getProtocol(),
servicePort,
containerPort));
}
}
}
// Add kubernetes services to cluster context and persist
clusterContext.setKubernetesServices(kubernetesServices);
CloudControllerContext.getInstance().persist();
}
/**
* Starts a container via kubernetes for the given member context.
*
* @param memberContext
* @return
* @throws CartridgeNotFoundException
*/
public MemberContext startContainer(MemberContext memberContext)
throws CartridgeNotFoundException {
Lock lock = null;
try {
lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();
handleNullObject(memberContext, "member context is null");
log.info(
String.format(
"Starting container: [application] %s [cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId()));
// Validate cluster id
String clusterId = memberContext.getClusterId();
String memberId = memberContext.getMemberId();
handleNullObject(clusterId, "cluster id is null in member context");
// Validate cluster context
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
handleNullObject(
clusterContext,
String.format(
"Cluster context not found: [application] %s [cartridge] %s " + "[cluster] %s",
memberContext.getApplicationId(), memberContext.getCartridgeType(), clusterId));
// Validate partition
Partition partition = memberContext.getPartition();
handleNullObject(
partition,
String.format(
"partition not found in member context: [application] %s "
+ "[cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId()));
// Validate cartridge
String cartridgeType = clusterContext.getCartridgeUuid();
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String msg =
String.format(
"Cartridge not found: [application] %s [cartridge] %s",
memberContext.getApplicationId(), memberContext.getCartridgeType());
log.error(msg);
throw new CartridgeNotFoundException(msg);
}
String kubernetesClusterId = partition.getKubernetesClusterId();
clusterContext.setKubernetesClusterId(kubernetesClusterId);
KubernetesCluster kubernetesCluster =
CloudControllerContext.getInstance().getKubernetesCluster(kubernetesClusterId);
handleNullObject(
kubernetesCluster,
"kubernetes cluster not found: "
+ "[kubernetes-cluster] "
+ kubernetesClusterId
+ " [cluster] "
+ clusterId
+ " [member] "
+ memberId);
// Prepare kubernetes context
String kubernetesMasterIp = kubernetesCluster.getKubernetesMaster().getPrivateIPAddress();
PortRange kubernetesPortRange = kubernetesCluster.getPortRange();
String kubernetesMasterPort =
CloudControllerUtil.getProperty(
kubernetesCluster.getKubernetesMaster().getProperties(),
StratosConstants.KUBERNETES_MASTER_PORT,
StratosConstants.KUBERNETES_MASTER_DEFAULT_PORT);
// Add kubernetes cluster payload parameters to payload
if ((kubernetesCluster.getProperties() != null)
&& (kubernetesCluster.getProperties().getProperties() != null)) {
for (Property property : kubernetesCluster.getProperties().getProperties()) {
if (property != null) {
if (property.getName().startsWith(PAYLOAD_PARAMETER_PREFIX)) {
String name = property.getName().replace(PAYLOAD_PARAMETER_PREFIX, "");
payload.add(new NameValuePair(name, property.getValue()));
}
}
}
}
KubernetesClusterContext kubernetesClusterContext =
getKubernetesClusterContext(
kubernetesClusterId,
kubernetesMasterIp,
kubernetesMasterPort,
kubernetesPortRange.getUpper(),
kubernetesPortRange.getLower());
// Generate kubernetes service ports and update port mappings in cartridge
generateKubernetesServicePorts(
clusterContext.getApplicationUuid(),
clusterContext.getClusterId(),
kubernetesClusterContext,
cartridge);
// Create kubernetes services for port mappings
KubernetesApiClient kubernetesApi = kubernetesClusterContext.getKubApi();
createKubernetesServices(
kubernetesApi, clusterContext, kubernetesCluster, kubernetesClusterContext);
// Create pod
createPod(clusterContext, memberContext, kubernetesApi, kubernetesClusterContext);
// Wait for pod status to be changed to running
Pod pod = waitForPodToBeActivated(memberContext, kubernetesApi);
// Update member context
updateMemberContext(memberContext, pod, kubernetesCluster);
log.info(
String.format(
"Container started successfully: [application] %s [cartridge] %s [member] %s "
+ "[pod] %s [cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
memberContext.getKubernetesPodId(),
memberContext.getInstanceMetadata().getCpu(),
memberContext.getInstanceMetadata().getRam()));
return memberContext;
} catch (Exception e) {
String msg =
String.format(
"Could not start container: [application] %s [cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId());
log.error(msg, e);
throw new RuntimeException(msg, e);
} finally {
if (lock != null) {
CloudControllerContext.getInstance().releaseWriteLock(lock);
}
}
}
/**
* Create new pod and pass environment variables.
*
* @param memberContext
* @param kubernetesApi
* @param kubernetesClusterContext
* @throws KubernetesClientException
*/
private void createPod(
ClusterContext clusterContext,
MemberContext memberContext,
KubernetesApiClient kubernetesApi,
KubernetesClusterContext kubernetesClusterContext)
throws KubernetesClientException {
String applicationId = memberContext.getApplicationId();
String cartridgeType = memberContext.getCartridgeType();
String clusterId = memberContext.getClusterId();
String memberId = memberContext.getMemberId();
if (log.isInfoEnabled()) {
log.info(
String.format(
"Creating kubernetes pod: [application] %s [cartridge] %s [member] %s",
applicationId, cartridgeType, memberId));
}
Partition partition = memberContext.getPartition();
if (partition == null) {
String message =
String.format(
"Partition not found in member context: [application] %s [cartridge] %s "
+ "[member] %s ",
applicationId, cartridgeType, memberId);
log.error(message);
throw new RuntimeException(message);
}
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String message = "Could not find cartridge: [cartridge] " + cartridgeType;
log.error(message);
throw new RuntimeException(message);
}
// Set default values to zero to avoid cpu and memory restrictions
int cpu = Integer.getInteger(KUBERNETES_CONTAINER_CPU_DEFAULT, 0);
int memory = Integer.getInteger(KUBERNETES_CONTAINER_MEMORY_DEFAULT, 0);
Property cpuProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_CPU);
if (cpuProperty != null) {
cpu = Integer.parseInt(cpuProperty.getValue());
}
Property memoryProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_MEMORY);
if (memoryProperty != null) {
memory = Integer.parseInt(memoryProperty.getValue());
}
IaasProvider iaasProvider =
CloudControllerContext.getInstance()
.getIaasProviderOfPartition(cartridge.getUuid(), partition.getUuid());
if (iaasProvider == null) {
String message = "Could not find iaas provider: [partition] " + partition.getUuid();
log.error(message);
throw new RuntimeException(message);
}
// Add dynamic payload to the member context
memberContext.setDynamicPayload(payload.toArray(new NameValuePair[payload.size()]));
// Create pod
long podSeqNo = kubernetesClusterContext.getPodSeqNo().incrementAndGet();
String podId = "pod" + "-" + podSeqNo;
String podLabel = DigestUtils.md5Hex(clusterId);
String dockerImage = iaasProvider.getImage();
List<EnvVar> environmentVariables =
KubernetesIaasUtil.prepareEnvironmentVariables(clusterContext, memberContext);
List<ContainerPort> ports =
KubernetesIaasUtil.convertPortMappings(Arrays.asList(cartridge.getPortMappings()));
log.info(
String.format(
"Starting pod: [application] %s [cartridge] %s [member] %s "
+ "[cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
cpu,
memory));
kubernetesApi.createPod(podId, podLabel, dockerImage, cpu, memory, ports, environmentVariables);
log.info(
String.format(
"Pod started successfully: [application] %s [cartridge] %s [member] %s "
+ "[pod] %s [cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
podId,
cpu,
memory));
// Add pod id to member context
memberContext.setKubernetesPodId(podId);
memberContext.setKubernetesPodLabel(podLabel);
// Create instance metadata
InstanceMetadata instanceMetadata = new InstanceMetadata();
instanceMetadata.setImageId(dockerImage);
instanceMetadata.setCpu(cpu);
instanceMetadata.setRam(memory);
memberContext.setInstanceMetadata(instanceMetadata);
// Persist cloud controller context
CloudControllerContext.getInstance().persist();
}
public static void handleServiceCreated(List<Cartridge> cartridgeList) {
Service service;
Topology topology = TopologyManager.getTopology();
if (cartridgeList == null) {
log.warn(String.format("Cartridge list is empty"));
return;
}
try {
TopologyManager.acquireWriteLock();
for (Cartridge cartridge : cartridgeList) {
if (!topology.serviceExists(cartridge.getType())) {
ServiceType serviceType =
cartridge.isMultiTenant() ? ServiceType.MultiTenant : ServiceType.SingleTenant;
service = new Service(cartridge.getType(), serviceType);
Properties properties = new Properties();
try {
Property[] propertyArray = null;
if (cartridge.getProperties() != null) {
if (cartridge.getProperties().getProperties() != null) {
propertyArray = cartridge.getProperties().getProperties();
}
}
List<Property> propertyList = new ArrayList<Property>();
if (propertyArray != null) {
propertyList = Arrays.asList(propertyArray);
if (propertyList != null) {
for (Property property : propertyList) {
properties.setProperty(property.getName(), property.getValue());
}
}
}
} catch (Exception e) {
log.error(e);
}
service.setProperties(properties);
if (cartridge.getPortMappings() != null) {
List<PortMapping> portMappings = Arrays.asList(cartridge.getPortMappings());
Port port;
// adding ports to the event
for (PortMapping portMapping : portMappings) {
port =
new Port(
portMapping.getProtocol(), portMapping.getPort(), portMapping.getProxyPort());
service.addPort(port);
}
}
topology.addService(service);
TopologyManager.updateTopology(topology);
}
}
} finally {
TopologyManager.releaseWriteLock();
}
TopologyEventPublisher.sendServiceCreateEvent(cartridgeList);
}