private void updateMemberContext(
MemberContext memberContext, Pod pod, KubernetesCluster kubernetesCluster) {
String memberPrivateIPAddress = pod.getStatus().getPodIP();
String podHostIPAddress = pod.getStatus().getHostIP();
String memberPublicIPAddress = podHostIPAddress;
String kubernetesHostPublicIP =
findKubernetesHostPublicIPAddress(kubernetesCluster, podHostIPAddress);
if (StringUtils.isNotBlank(kubernetesHostPublicIP)) {
memberPublicIPAddress = kubernetesHostPublicIP;
if (log.isInfoEnabled()) {
log.info(
String.format(
"Member public IP address set to kubernetes host public IP address:"
+ "[pod-host-ip] %s [kubernetes-host-public-ip] %s",
podHostIPAddress, kubernetesHostPublicIP));
}
}
memberContext.setInstanceId(pod.getMetadata().getName());
memberContext.setDefaultPrivateIP(memberPrivateIPAddress);
memberContext.setPrivateIPs(new String[] {memberPrivateIPAddress});
memberContext.setDefaultPublicIP(memberPublicIPAddress);
memberContext.setPublicIPs(new String[] {memberPublicIPAddress});
memberContext.setInitTime(memberContext.getInitTime());
memberContext.setProperties(memberContext.getProperties());
}
/**
* Add member object to the topology and publish member created event
*
* @param memberContext
*/
public static void handleMemberCreatedEvent(MemberContext memberContext) {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(memberContext.getCartridgeType());
String clusterId = memberContext.getClusterId();
Cluster cluster = service.getCluster(clusterId);
String memberId = memberContext.getMemberId();
String clusterInstanceId = memberContext.getClusterInstanceId();
String networkPartitionId = memberContext.getNetworkPartitionId();
String partitionId = memberContext.getPartition().getId();
String lbClusterId = memberContext.getLbClusterId();
long initTime = memberContext.getInitTime();
if (cluster.memberExists(memberId)) {
log.warn(String.format("Member %s already exists", memberId));
return;
}
try {
TopologyManager.acquireWriteLock();
Member member =
new Member(
service.getServiceName(),
clusterId,
memberId,
clusterInstanceId,
networkPartitionId,
partitionId,
memberContext.getLoadBalancingIPType(),
initTime);
member.setStatus(MemberStatus.Created);
member.setLbClusterId(lbClusterId);
member.setProperties(CloudControllerUtil.toJavaUtilProperties(memberContext.getProperties()));
cluster.addMember(member);
TopologyManager.updateTopology(topology);
} finally {
TopologyManager.releaseWriteLock();
}
TopologyEventPublisher.sendMemberCreatedEvent(memberContext);
}
/**
* Terminate a container by member id
*
* @param memberId
* @return
* @throws MemberTerminationFailedException
*/
public MemberContext terminateContainer(String memberId) throws MemberTerminationFailedException {
Lock lock = null;
try {
lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();
handleNullObject(memberId, "Could not terminate container, member id is null");
MemberContext memberContext =
CloudControllerContext.getInstance().getMemberContextOfMemberId(memberId);
handleNullObject(
memberContext,
"Could not terminate container, member context not found: [member-id] " + memberId);
String clusterId = memberContext.getClusterId();
handleNullObject(
clusterId, "Could not terminate container, cluster id is null: [member-id] " + memberId);
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
handleNullObject(
clusterContext,
String.format(
"Could not terminate container, cluster context not found: "
+ "[cluster-id] %s [member-id] %s",
clusterId, memberId));
String kubernetesClusterId = clusterContext.getKubernetesClusterId();
handleNullObject(
kubernetesClusterId,
String.format(
"Could not terminate container, kubernetes cluster "
+ "context id is null: [cluster-id] %s [member-id] %s",
clusterId, memberId));
KubernetesClusterContext kubernetesClusterContext =
CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId);
handleNullObject(
kubernetesClusterContext,
String.format(
"Could not terminate container, kubernetes cluster "
+ "context not found: [cluster-id] %s [member-id] %s",
clusterId, memberId));
KubernetesApiClient kubApi = kubernetesClusterContext.getKubApi();
try {
log.info(
String.format(
"Removing kubernetes pod: [application] %s [cartridge] %s [member] %s [pod] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
memberContext.getKubernetesPodId()));
// Remove pod
kubApi.deletePod(memberContext.getKubernetesPodId());
// Persist changes
CloudControllerContext.getInstance().persist();
log.info(
String.format(
"Kubernetes pod removed successfully: [application] %s [cartridge] %s "
+ "[member] %s [pod] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
memberContext.getKubernetesPodId()));
} catch (KubernetesClientException ignore) {
// we can't do nothing here
log.warn(
String.format("Could not delete pod: [pod-id] %s", memberContext.getKubernetesPodId()));
}
return memberContext;
} finally {
if (lock != null) {
CloudControllerContext.getInstance().releaseWriteLock(lock);
}
}
}
/**
* Terminate all the containers belong to a cluster by cluster id.
*
* @param clusterId
* @return
* @throws InvalidClusterException
*/
public MemberContext[] terminateContainers(String clusterId) throws InvalidClusterException {
Lock lock = null;
try {
lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
handleNullObject(
clusterContext,
"Could not terminate containers, cluster not found: [cluster-id] " + clusterId);
String kubernetesClusterId = clusterContext.getKubernetesClusterId();
handleNullObject(
kubernetesClusterId,
"Could not terminate containers, kubernetes cluster id not found: "
+ "[cluster-id] "
+ clusterId);
KubernetesClusterContext kubClusterContext =
CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId);
handleNullObject(
kubClusterContext,
"Could not terminate containers, kubernetes cluster not found: "
+ "[kubernetes-cluster-id] "
+ kubernetesClusterId);
KubernetesApiClient kubApi = kubClusterContext.getKubApi();
// Remove kubernetes services
List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
if (kubernetesServices != null) {
for (KubernetesService kubernetesService : kubernetesServices) {
try {
kubApi.deleteService(kubernetesService.getId());
int allocatedPort = kubernetesService.getPort();
kubClusterContext.deallocatePort(allocatedPort);
} catch (KubernetesClientException e) {
log.error("Could not remove kubernetes service: [cluster-id] " + clusterId, e);
}
}
}
List<MemberContext> memberContextsRemoved = new ArrayList<MemberContext>();
List<MemberContext> memberContexts =
CloudControllerContext.getInstance().getMemberContextsOfClusterId(clusterId);
if (memberContexts != null) {
for (MemberContext memberContext : memberContexts) {
try {
MemberContext memberContextRemoved = terminateContainer(memberContext.getMemberId());
memberContextsRemoved.add(memberContextRemoved);
} catch (MemberTerminationFailedException e) {
String message =
"Could not terminate container: [member-id] " + memberContext.getMemberId();
log.error(message);
}
}
}
// Persist changes
CloudControllerContext.getInstance().persist();
return memberContextsRemoved.toArray(new MemberContext[memberContextsRemoved.size()]);
} finally {
if (lock != null) {
CloudControllerContext.getInstance().releaseWriteLock(lock);
}
}
}
/**
* Create new pod and pass environment variables.
*
* @param memberContext
* @param kubernetesApi
* @param kubernetesClusterContext
* @throws KubernetesClientException
*/
private void createPod(
ClusterContext clusterContext,
MemberContext memberContext,
KubernetesApiClient kubernetesApi,
KubernetesClusterContext kubernetesClusterContext)
throws KubernetesClientException {
String applicationId = memberContext.getApplicationId();
String cartridgeType = memberContext.getCartridgeType();
String clusterId = memberContext.getClusterId();
String memberId = memberContext.getMemberId();
if (log.isInfoEnabled()) {
log.info(
String.format(
"Creating kubernetes pod: [application] %s [cartridge] %s [member] %s",
applicationId, cartridgeType, memberId));
}
Partition partition = memberContext.getPartition();
if (partition == null) {
String message =
String.format(
"Partition not found in member context: [application] %s [cartridge] %s "
+ "[member] %s ",
applicationId, cartridgeType, memberId);
log.error(message);
throw new RuntimeException(message);
}
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String message = "Could not find cartridge: [cartridge] " + cartridgeType;
log.error(message);
throw new RuntimeException(message);
}
// Set default values to zero to avoid cpu and memory restrictions
int cpu = Integer.getInteger(KUBERNETES_CONTAINER_CPU_DEFAULT, 0);
int memory = Integer.getInteger(KUBERNETES_CONTAINER_MEMORY_DEFAULT, 0);
Property cpuProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_CPU);
if (cpuProperty != null) {
cpu = Integer.parseInt(cpuProperty.getValue());
}
Property memoryProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_MEMORY);
if (memoryProperty != null) {
memory = Integer.parseInt(memoryProperty.getValue());
}
IaasProvider iaasProvider =
CloudControllerContext.getInstance()
.getIaasProviderOfPartition(cartridge.getUuid(), partition.getUuid());
if (iaasProvider == null) {
String message = "Could not find iaas provider: [partition] " + partition.getUuid();
log.error(message);
throw new RuntimeException(message);
}
// Add dynamic payload to the member context
memberContext.setDynamicPayload(payload.toArray(new NameValuePair[payload.size()]));
// Create pod
long podSeqNo = kubernetesClusterContext.getPodSeqNo().incrementAndGet();
String podId = "pod" + "-" + podSeqNo;
String podLabel = DigestUtils.md5Hex(clusterId);
String dockerImage = iaasProvider.getImage();
List<EnvVar> environmentVariables =
KubernetesIaasUtil.prepareEnvironmentVariables(clusterContext, memberContext);
List<ContainerPort> ports =
KubernetesIaasUtil.convertPortMappings(Arrays.asList(cartridge.getPortMappings()));
log.info(
String.format(
"Starting pod: [application] %s [cartridge] %s [member] %s "
+ "[cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
cpu,
memory));
kubernetesApi.createPod(podId, podLabel, dockerImage, cpu, memory, ports, environmentVariables);
log.info(
String.format(
"Pod started successfully: [application] %s [cartridge] %s [member] %s "
+ "[pod] %s [cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
podId,
cpu,
memory));
// Add pod id to member context
memberContext.setKubernetesPodId(podId);
memberContext.setKubernetesPodLabel(podLabel);
// Create instance metadata
InstanceMetadata instanceMetadata = new InstanceMetadata();
instanceMetadata.setImageId(dockerImage);
instanceMetadata.setCpu(cpu);
instanceMetadata.setRam(memory);
memberContext.setInstanceMetadata(instanceMetadata);
// Persist cloud controller context
CloudControllerContext.getInstance().persist();
}
private Pod waitForPodToBeActivated(
MemberContext memberContext, KubernetesApiClient kubernetesApi)
throws KubernetesClientException, InterruptedException {
Pod pod;
boolean podCreated = false;
boolean podRunning = false;
long startTime = System.currentTimeMillis();
while (!podRunning) {
pod = kubernetesApi.getPod(memberContext.getKubernetesPodId());
if (pod != null) {
podCreated = true;
if (pod.getStatus().getPhase().equals(KubernetesConstants.POD_STATUS_RUNNING)) {
log.info(
String.format(
"Pod status changed to running: [application] %s [cartridge] %s [member] %s "
+ "[pod] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
pod.getMetadata().getName()));
return pod;
} else {
log.info(
String.format(
"Waiting pod status to be changed to running: [application] %s "
+ "[cartridge] %s [member] %s [pod] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
pod.getMetadata().getName()));
}
} else {
log.info(
String.format(
"Waiting for pod to be created: [application] %s "
+ "[cartridge] %s [member] %s [pod] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
memberContext.getKubernetesPodId()));
}
if ((System.currentTimeMillis() - startTime) > podActivationTimeout) {
break;
}
Thread.sleep(5000);
}
String message;
if (podCreated) {
// Pod created but status did not change to running
message =
String.format(
"Pod status did not change to running within %d sec: "
+ "[application] %s [cartridge] %s [member] %s [pod] %s",
(podActivationTimeout.intValue() / 1000),
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
memberContext.getKubernetesPodId());
log.error(message);
} else {
// Pod did not create
message =
String.format(
"Pod did not create within %d sec: "
+ "[application] %s [cartridge] %s [member] %s [pod] %s",
(podActivationTimeout.intValue() / 1000),
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
memberContext.getKubernetesPodId());
log.error(message);
}
throw new RuntimeException(message);
}
/**
* Starts a container via kubernetes for the given member context.
*
* @param memberContext
* @return
* @throws CartridgeNotFoundException
*/
public MemberContext startContainer(MemberContext memberContext)
throws CartridgeNotFoundException {
Lock lock = null;
try {
lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();
handleNullObject(memberContext, "member context is null");
log.info(
String.format(
"Starting container: [application] %s [cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId()));
// Validate cluster id
String clusterId = memberContext.getClusterId();
String memberId = memberContext.getMemberId();
handleNullObject(clusterId, "cluster id is null in member context");
// Validate cluster context
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
handleNullObject(
clusterContext,
String.format(
"Cluster context not found: [application] %s [cartridge] %s " + "[cluster] %s",
memberContext.getApplicationId(), memberContext.getCartridgeType(), clusterId));
// Validate partition
Partition partition = memberContext.getPartition();
handleNullObject(
partition,
String.format(
"partition not found in member context: [application] %s "
+ "[cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId()));
// Validate cartridge
String cartridgeType = clusterContext.getCartridgeUuid();
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String msg =
String.format(
"Cartridge not found: [application] %s [cartridge] %s",
memberContext.getApplicationId(), memberContext.getCartridgeType());
log.error(msg);
throw new CartridgeNotFoundException(msg);
}
String kubernetesClusterId = partition.getKubernetesClusterId();
clusterContext.setKubernetesClusterId(kubernetesClusterId);
KubernetesCluster kubernetesCluster =
CloudControllerContext.getInstance().getKubernetesCluster(kubernetesClusterId);
handleNullObject(
kubernetesCluster,
"kubernetes cluster not found: "
+ "[kubernetes-cluster] "
+ kubernetesClusterId
+ " [cluster] "
+ clusterId
+ " [member] "
+ memberId);
// Prepare kubernetes context
String kubernetesMasterIp = kubernetesCluster.getKubernetesMaster().getPrivateIPAddress();
PortRange kubernetesPortRange = kubernetesCluster.getPortRange();
String kubernetesMasterPort =
CloudControllerUtil.getProperty(
kubernetesCluster.getKubernetesMaster().getProperties(),
StratosConstants.KUBERNETES_MASTER_PORT,
StratosConstants.KUBERNETES_MASTER_DEFAULT_PORT);
// Add kubernetes cluster payload parameters to payload
if ((kubernetesCluster.getProperties() != null)
&& (kubernetesCluster.getProperties().getProperties() != null)) {
for (Property property : kubernetesCluster.getProperties().getProperties()) {
if (property != null) {
if (property.getName().startsWith(PAYLOAD_PARAMETER_PREFIX)) {
String name = property.getName().replace(PAYLOAD_PARAMETER_PREFIX, "");
payload.add(new NameValuePair(name, property.getValue()));
}
}
}
}
KubernetesClusterContext kubernetesClusterContext =
getKubernetesClusterContext(
kubernetesClusterId,
kubernetesMasterIp,
kubernetesMasterPort,
kubernetesPortRange.getUpper(),
kubernetesPortRange.getLower());
// Generate kubernetes service ports and update port mappings in cartridge
generateKubernetesServicePorts(
clusterContext.getApplicationUuid(),
clusterContext.getClusterId(),
kubernetesClusterContext,
cartridge);
// Create kubernetes services for port mappings
KubernetesApiClient kubernetesApi = kubernetesClusterContext.getKubApi();
createKubernetesServices(
kubernetesApi, clusterContext, kubernetesCluster, kubernetesClusterContext);
// Create pod
createPod(clusterContext, memberContext, kubernetesApi, kubernetesClusterContext);
// Wait for pod status to be changed to running
Pod pod = waitForPodToBeActivated(memberContext, kubernetesApi);
// Update member context
updateMemberContext(memberContext, pod, kubernetesCluster);
log.info(
String.format(
"Container started successfully: [application] %s [cartridge] %s [member] %s "
+ "[pod] %s [cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
memberContext.getKubernetesPodId(),
memberContext.getInstanceMetadata().getCpu(),
memberContext.getInstanceMetadata().getRam()));
return memberContext;
} catch (Exception e) {
String msg =
String.format(
"Could not start container: [application] %s [cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId());
log.error(msg, e);
throw new RuntimeException(msg, e);
} finally {
if (lock != null) {
CloudControllerContext.getInstance().releaseWriteLock(lock);
}
}
}
@Override
public void terminateInstance(MemberContext memberContext)
throws InvalidCartridgeTypeException, InvalidMemberException,
MemberTerminationFailedException {
terminateContainer(memberContext.getMemberId());
}
/**
* Update member status to initialized and publish member initialized event
*
* @param memberContext
*/
public static void handleMemberInitializedEvent(MemberContext memberContext) {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(memberContext.getCartridgeType());
if (service == null) {
log.warn(String.format("Service %s does not exist", memberContext.getCartridgeType()));
return;
}
if (!service.clusterExists(memberContext.getClusterId())) {
log.warn(
String.format(
"Cluster %s does not exist in service %s",
memberContext.getClusterId(), memberContext.getCartridgeType()));
return;
}
Member member =
service.getCluster(memberContext.getClusterId()).getMember(memberContext.getMemberId());
if (member == null) {
log.warn(String.format("Member %s does not exist", memberContext.getMemberId()));
return;
}
try {
TopologyManager.acquireWriteLock();
// Set ip addresses
member.setDefaultPrivateIP(memberContext.getDefaultPrivateIP());
if (memberContext.getPrivateIPs() != null) {
member.setMemberPrivateIPs(Arrays.asList(memberContext.getPrivateIPs()));
}
member.setDefaultPublicIP(memberContext.getDefaultPublicIP());
if (memberContext.getPublicIPs() != null) {
member.setMemberPublicIPs(Arrays.asList(memberContext.getPublicIPs()));
}
// try update lifecycle state
if (!member.isStateTransitionValid(MemberStatus.Initialized)) {
log.error(
"Invalid state transition from "
+ member.getStatus()
+ " to "
+ MemberStatus.Initialized);
return;
} else {
member.setStatus(MemberStatus.Initialized);
log.info("Member status updated to initialized");
TopologyManager.updateTopology(topology);
TopologyEventPublisher.sendMemberInitializedEvent(memberContext);
// publishing data
BAMUsageDataPublisher.publish(
memberContext.getMemberId(),
memberContext.getPartition().getId(),
memberContext.getNetworkPartitionId(),
memberContext.getClusterId(),
memberContext.getCartridgeType(),
MemberStatus.Initialized.toString(),
null);
}
} finally {
TopologyManager.releaseWriteLock();
}
}