private Pod waitForPodToBeActivated( MemberContext memberContext, KubernetesApiClient kubernetesApi) throws KubernetesClientException, InterruptedException { Pod pod; boolean podCreated = false; boolean podRunning = false; long startTime = System.currentTimeMillis(); while (!podRunning) { pod = kubernetesApi.getPod(memberContext.getKubernetesPodId()); if (pod != null) { podCreated = true; if (pod.getStatus().getPhase().equals(KubernetesConstants.POD_STATUS_RUNNING)) { log.info( String.format( "Pod status changed to running: [application] %s [cartridge] %s [member] %s " + "[pod] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), pod.getMetadata().getName())); return pod; } else { log.info( String.format( "Waiting pod status to be changed to running: [application] %s " + "[cartridge] %s [member] %s [pod] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), pod.getMetadata().getName())); } } else { log.info( String.format( "Waiting for pod to be created: [application] %s " + "[cartridge] %s [member] %s [pod] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId())); } if ((System.currentTimeMillis() - startTime) > podActivationTimeout) { break; } Thread.sleep(5000); } String message; if (podCreated) { // Pod created but status did not change to running message = String.format( "Pod status did not change to running within %d sec: " + "[application] %s [cartridge] %s [member] %s [pod] %s", (podActivationTimeout.intValue() / 1000), memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId()); log.error(message); } else { // Pod did not create message = String.format( "Pod did not create within %d sec: " + "[application] %s [cartridge] %s [member] %s [pod] %s", (podActivationTimeout.intValue() / 1000), memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId()); log.error(message); } throw new RuntimeException(message); }
/** * Terminate a container by member id * * @param memberId * @return * @throws MemberTerminationFailedException */ public MemberContext terminateContainer(String memberId) throws MemberTerminationFailedException { Lock lock = null; try { lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock(); handleNullObject(memberId, "Could not terminate container, member id is null"); MemberContext memberContext = CloudControllerContext.getInstance().getMemberContextOfMemberId(memberId); handleNullObject( memberContext, "Could not terminate container, member context not found: [member-id] " + memberId); String clusterId = memberContext.getClusterId(); handleNullObject( clusterId, "Could not terminate container, cluster id is null: [member-id] " + memberId); ClusterContext clusterContext = CloudControllerContext.getInstance().getClusterContext(clusterId); handleNullObject( clusterContext, String.format( "Could not terminate container, cluster context not found: " + "[cluster-id] %s [member-id] %s", clusterId, memberId)); String kubernetesClusterId = clusterContext.getKubernetesClusterId(); handleNullObject( kubernetesClusterId, String.format( "Could not terminate container, kubernetes cluster " + "context id is null: [cluster-id] %s [member-id] %s", clusterId, memberId)); KubernetesClusterContext kubernetesClusterContext = CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId); handleNullObject( kubernetesClusterContext, String.format( "Could not terminate container, kubernetes cluster " + "context not found: [cluster-id] %s [member-id] %s", clusterId, memberId)); KubernetesApiClient kubApi = kubernetesClusterContext.getKubApi(); try { log.info( String.format( "Removing kubernetes pod: [application] %s [cartridge] %s [member] %s [pod] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId())); // Remove pod kubApi.deletePod(memberContext.getKubernetesPodId()); // Persist changes CloudControllerContext.getInstance().persist(); log.info( String.format( "Kubernetes pod removed successfully: [application] %s [cartridge] %s " + "[member] %s [pod] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId())); } catch (KubernetesClientException ignore) { // we can't do nothing here log.warn( String.format("Could not delete pod: [pod-id] %s", memberContext.getKubernetesPodId())); } return memberContext; } finally { if (lock != null) { CloudControllerContext.getInstance().releaseWriteLock(lock); } } }
/** * Starts a container via kubernetes for the given member context. * * @param memberContext * @return * @throws CartridgeNotFoundException */ public MemberContext startContainer(MemberContext memberContext) throws CartridgeNotFoundException { Lock lock = null; try { lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock(); handleNullObject(memberContext, "member context is null"); log.info( String.format( "Starting container: [application] %s [cartridge] %s [member] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId())); // Validate cluster id String clusterId = memberContext.getClusterId(); String memberId = memberContext.getMemberId(); handleNullObject(clusterId, "cluster id is null in member context"); // Validate cluster context ClusterContext clusterContext = CloudControllerContext.getInstance().getClusterContext(clusterId); handleNullObject( clusterContext, String.format( "Cluster context not found: [application] %s [cartridge] %s " + "[cluster] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), clusterId)); // Validate partition Partition partition = memberContext.getPartition(); handleNullObject( partition, String.format( "partition not found in member context: [application] %s " + "[cartridge] %s [member] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId())); // Validate cartridge String cartridgeType = clusterContext.getCartridgeUuid(); Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType); if (cartridge == null) { String msg = String.format( "Cartridge not found: [application] %s [cartridge] %s", memberContext.getApplicationId(), memberContext.getCartridgeType()); log.error(msg); throw new CartridgeNotFoundException(msg); } String kubernetesClusterId = partition.getKubernetesClusterId(); clusterContext.setKubernetesClusterId(kubernetesClusterId); KubernetesCluster kubernetesCluster = CloudControllerContext.getInstance().getKubernetesCluster(kubernetesClusterId); handleNullObject( kubernetesCluster, "kubernetes cluster not found: " + "[kubernetes-cluster] " + kubernetesClusterId + " [cluster] " + clusterId + " [member] " + memberId); // Prepare kubernetes context String kubernetesMasterIp = kubernetesCluster.getKubernetesMaster().getPrivateIPAddress(); PortRange kubernetesPortRange = kubernetesCluster.getPortRange(); String kubernetesMasterPort = CloudControllerUtil.getProperty( kubernetesCluster.getKubernetesMaster().getProperties(), StratosConstants.KUBERNETES_MASTER_PORT, StratosConstants.KUBERNETES_MASTER_DEFAULT_PORT); // Add kubernetes cluster payload parameters to payload if ((kubernetesCluster.getProperties() != null) && (kubernetesCluster.getProperties().getProperties() != null)) { for (Property property : kubernetesCluster.getProperties().getProperties()) { if (property != null) { if (property.getName().startsWith(PAYLOAD_PARAMETER_PREFIX)) { String name = property.getName().replace(PAYLOAD_PARAMETER_PREFIX, ""); payload.add(new NameValuePair(name, property.getValue())); } } } } KubernetesClusterContext kubernetesClusterContext = getKubernetesClusterContext( kubernetesClusterId, kubernetesMasterIp, kubernetesMasterPort, kubernetesPortRange.getUpper(), kubernetesPortRange.getLower()); // Generate kubernetes service ports and update port mappings in cartridge generateKubernetesServicePorts( clusterContext.getApplicationUuid(), clusterContext.getClusterId(), kubernetesClusterContext, cartridge); // Create kubernetes services for port mappings KubernetesApiClient kubernetesApi = kubernetesClusterContext.getKubApi(); createKubernetesServices( kubernetesApi, clusterContext, kubernetesCluster, kubernetesClusterContext); // Create pod createPod(clusterContext, memberContext, kubernetesApi, kubernetesClusterContext); // Wait for pod status to be changed to running Pod pod = waitForPodToBeActivated(memberContext, kubernetesApi); // Update member context updateMemberContext(memberContext, pod, kubernetesCluster); log.info( String.format( "Container started successfully: [application] %s [cartridge] %s [member] %s " + "[pod] %s [cpu] %d [memory] %d MB", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId(), memberContext.getInstanceMetadata().getCpu(), memberContext.getInstanceMetadata().getRam())); return memberContext; } catch (Exception e) { String msg = String.format( "Could not start container: [application] %s [cartridge] %s [member] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId()); log.error(msg, e); throw new RuntimeException(msg, e); } finally { if (lock != null) { CloudControllerContext.getInstance().releaseWriteLock(lock); } } }