/** * Creates and returns proxy services for the cluster. * * @param kubernetesApi * @param clusterContext * @param kubernetesCluster * @param kubernetesClusterContext * @throws KubernetesClientException */ private void createKubernetesServices( KubernetesApiClient kubernetesApi, ClusterContext clusterContext, KubernetesCluster kubernetesCluster, KubernetesClusterContext kubernetesClusterContext) throws KubernetesClientException { String clusterId = clusterContext.getClusterId(); String cartridgeType = clusterContext.getCartridgeUuid(); Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType); if (cartridge == null) { String message = "Could not create kubernetes services, cartridge not found: [cartridge] " + cartridgeType; log.error(message); throw new RuntimeException(message); } String sessionAffinity = null; Property sessionAffinityProperty = cartridge.getProperties().getProperty(KUBERNETES_SERVICE_SESSION_AFFINITY); if (sessionAffinityProperty != null) { sessionAffinity = sessionAffinityProperty.getValue(); } List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices(); if (kubernetesServices == null) { kubernetesServices = new ArrayList<KubernetesService>(); } // Prepare minion public IP addresses List<String> minionPrivateIPList = new ArrayList<String>(); List<String> minionPublicIPList = new ArrayList<String>(); KubernetesHost[] kubernetesHosts = kubernetesCluster.getKubernetesHosts(); if ((kubernetesHosts == null) || (kubernetesHosts.length == 0) || (kubernetesHosts[0] == null)) { throw new RuntimeException( "Hosts not found in kubernetes cluster: [cluster] " + kubernetesCluster.getClusterUuid()); } for (KubernetesHost host : kubernetesHosts) { if (host != null) { minionPrivateIPList.add(host.getPrivateIPAddress()); minionPublicIPList.add(host.getPublicIPAddress()); } } if (log.isDebugEnabled()) { log.debug(String.format("Minion private IPs: %s", minionPrivateIPList)); } Collection<ClusterPortMapping> clusterPortMappings = CloudControllerContext.getInstance() .getClusterPortMappings(clusterContext.getApplicationUuid(), clusterId); if (clusterPortMappings != null) { for (ClusterPortMapping clusterPortMapping : clusterPortMappings) { // Skip if already created int containerPort = clusterPortMapping.getPort(); if (kubernetesServiceExist(kubernetesServices, containerPort)) { continue; } // Find next service sequence no long serviceSeqNo = kubernetesClusterContext.getServiceSeqNo().incrementAndGet(); String serviceId = KubernetesIaasUtil.fixSpecialCharacters("service" + "-" + (serviceSeqNo)); String serviceLabel = DigestUtils.md5Hex(clusterId); if (log.isInfoEnabled()) { log.info( String.format( "Creating kubernetes service: [cluster] %s [service] %s [service-label] %s " + "[protocol] %s [service-port] %d [container-port] %s", clusterId, serviceId, serviceLabel, clusterPortMapping.getProtocol(), clusterPortMapping.getKubernetesServicePort(), containerPort)); } // Create kubernetes service for port mapping int servicePort = clusterPortMapping.getKubernetesServicePort(); String serviceType = clusterPortMapping.getKubernetesServiceType(); String containerPortName = KubernetesIaasUtil.preparePortNameFromPortMapping(clusterPortMapping); try { // If kubernetes service is already created, skip creating a new one if (kubernetesApi.getService(serviceId) == null) { // Services need to use minions private IP addresses for creating iptable rules kubernetesApi.createService( serviceId, serviceLabel, servicePort, serviceType, containerPortName, containerPort, sessionAffinity); } else { if (log.isDebugEnabled()) { log.debug( String.format( "Kubernetes service is already created: [cluster] %s [service] %s " + "[protocol] %s [service-port] %d [container-port] %d", clusterId, serviceId, clusterPortMapping.getProtocol(), servicePort, containerPort)); } } } finally { // Persist kubernetes service sequence no CloudControllerContext.getInstance().persist(); } try { Thread.sleep(1000); } catch (InterruptedException ignore) { } Service service = kubernetesApi.getService(serviceId); KubernetesService kubernetesService = new KubernetesService(); kubernetesService.setId(service.getMetadata().getName()); kubernetesService.setPortalIP(service.getSpec().getClusterIP()); // Expose minions public IP addresses as they need to be accessed by external networks String[] minionPublicIPArray = minionPublicIPList.toArray(new String[minionPublicIPList.size()]); kubernetesService.setPublicIPs(minionPublicIPArray); kubernetesService.setProtocol(clusterPortMapping.getProtocol()); kubernetesService.setPortName(clusterPortMapping.getName()); String kubernetesServiceType = service.getSpec().getType(); kubernetesService.setServiceType(kubernetesServiceType); if (kubernetesServiceType.equals(KubernetesConstants.NODE_PORT)) { kubernetesService.setPort(service.getSpec().getPorts().get(0).getNodePort()); } else { kubernetesService.setPort(service.getSpec().getPorts().get(0).getPort()); } kubernetesService.setContainerPort(containerPort); kubernetesServices.add(kubernetesService); if (log.isInfoEnabled()) { log.info( String.format( "Kubernetes service successfully created: [cluster] %s [service] %s " + "[protocol] %s [node-port] %d [container-port] %s", clusterId, serviceId, clusterPortMapping.getProtocol(), servicePort, containerPort)); } } } // Add kubernetes services to cluster context and persist clusterContext.setKubernetesServices(kubernetesServices); CloudControllerContext.getInstance().persist(); }
private Pod waitForPodToBeActivated( MemberContext memberContext, KubernetesApiClient kubernetesApi) throws KubernetesClientException, InterruptedException { Pod pod; boolean podCreated = false; boolean podRunning = false; long startTime = System.currentTimeMillis(); while (!podRunning) { pod = kubernetesApi.getPod(memberContext.getKubernetesPodId()); if (pod != null) { podCreated = true; if (pod.getStatus().getPhase().equals(KubernetesConstants.POD_STATUS_RUNNING)) { log.info( String.format( "Pod status changed to running: [application] %s [cartridge] %s [member] %s " + "[pod] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), pod.getMetadata().getName())); return pod; } else { log.info( String.format( "Waiting pod status to be changed to running: [application] %s " + "[cartridge] %s [member] %s [pod] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), pod.getMetadata().getName())); } } else { log.info( String.format( "Waiting for pod to be created: [application] %s " + "[cartridge] %s [member] %s [pod] %s", memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId())); } if ((System.currentTimeMillis() - startTime) > podActivationTimeout) { break; } Thread.sleep(5000); } String message; if (podCreated) { // Pod created but status did not change to running message = String.format( "Pod status did not change to running within %d sec: " + "[application] %s [cartridge] %s [member] %s [pod] %s", (podActivationTimeout.intValue() / 1000), memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId()); log.error(message); } else { // Pod did not create message = String.format( "Pod did not create within %d sec: " + "[application] %s [cartridge] %s [member] %s [pod] %s", (podActivationTimeout.intValue() / 1000), memberContext.getApplicationId(), memberContext.getCartridgeType(), memberContext.getMemberId(), memberContext.getKubernetesPodId()); log.error(message); } throw new RuntimeException(message); }