Beispiel #1
0
  /**
   * Generate kubernetes service ports for cluster.
   *
   * @param kubernetesClusterContext
   * @param clusterId
   * @param cartridge
   */
  private void generateKubernetesServicePorts(
      String applicationId,
      String clusterId,
      KubernetesClusterContext kubernetesClusterContext,
      Cartridge cartridge) {

    synchronized (KubernetesIaas.class) {
      if (cartridge != null) {

        StringBuilder portMappingStrBuilder = new StringBuilder();
        for (PortMapping portMapping : Arrays.asList(cartridge.getPortMappings())) {

          Collection<ClusterPortMapping> clusterPortMappings =
              CloudControllerContext.getInstance().getClusterPortMappings(applicationId, clusterId);
          if (clusterPortMappings == null) {
            throw new CloudControllerException(
                String.format(
                    "Cluster port mappings not found: " + "[application-id] %s [cluster-id] %s",
                    applicationId, clusterId));
          }

          ClusterPortMapping clusterPortMapping =
              findClusterPortMapping(clusterPortMappings, portMapping);
          if (clusterPortMappings == null) {
            throw new CloudControllerException(
                String.format(
                    "Cluster port mapping not found: "
                        + "[application-id] %s [cluster-id] %s [transport] %s",
                    applicationId, clusterId, portMapping.getName()));
          }

          if (clusterPortMapping.getKubernetesServiceType() == null) {
            throw new CloudControllerException(
                String.format(
                    "Kubernetes service type not "
                        + "found [application-id] %s [cluster-id] %s [cartridge] %s",
                    applicationId, clusterId, cartridge));
          }

          String serviceType = portMapping.getKubernetesPortType();
          clusterPortMapping.setKubernetesServiceType(serviceType);

          // If kubernetes service port is already set, skip setting a new one
          if (clusterPortMapping.getKubernetesServicePort() == 0) {
            if (serviceType.equals(KubernetesConstants.NODE_PORT)) {
              int nextServicePort = kubernetesClusterContext.getNextServicePort();
              if (nextServicePort == -1) {
                throw new RuntimeException(
                    String.format(
                        "Could not generate service port: [cluster-id] %s " + "[port] %d",
                        clusterId, portMapping.getPort()));
              }
              clusterPortMapping.setKubernetesServicePort(nextServicePort);
            } else {
              clusterPortMapping.setKubernetesServicePort(portMapping.getPort());
            }
          } else {
            if (log.isDebugEnabled()) {
              log.debug(
                  String.format(
                      "Kubernetes service port is already set: [application-id] %s "
                          + "[cluster-id] %s [port] %d [service-port] %d",
                      applicationId,
                      clusterId,
                      clusterPortMapping.getPort(),
                      clusterPortMapping.getKubernetesServicePort()));
            }
          }

          // Add port mappings to payload
          if (portMappingStrBuilder.toString().length() > 0) {
            portMappingStrBuilder.append(";");
          }
          portMappingStrBuilder.append(
              String.format(
                  "NAME:%s|PROTOCOL:%s|PORT:%d|PROXY_PORT:%d",
                  clusterPortMapping.getName(),
                  clusterPortMapping.getProtocol(),
                  clusterPortMapping.getKubernetesServicePort(),
                  clusterPortMapping.getProxyPort()));

          if (log.isInfoEnabled()) {
            log.info(
                String.format(
                    "Kubernetes service port generated: [application-id] %s "
                        + "[cluster-id] %s [port] %d [service-port] %d",
                    applicationId,
                    clusterId,
                    clusterPortMapping.getPort(),
                    clusterPortMapping.getKubernetesServicePort()));
          }
        }

        NameValuePair nameValuePair =
            new NameValuePair(PORT_MAPPINGS, portMappingStrBuilder.toString());
        payload.add(nameValuePair);

        // Persist service ports added to cluster port mappings
        CloudControllerContext.getInstance().persist();
      }
    }
  }
Beispiel #2
0
  /**
   * Creates and returns proxy services for the cluster.
   *
   * @param kubernetesApi
   * @param clusterContext
   * @param kubernetesCluster
   * @param kubernetesClusterContext
   * @throws KubernetesClientException
   */
  private void createKubernetesServices(
      KubernetesApiClient kubernetesApi,
      ClusterContext clusterContext,
      KubernetesCluster kubernetesCluster,
      KubernetesClusterContext kubernetesClusterContext)
      throws KubernetesClientException {

    String clusterId = clusterContext.getClusterId();
    String cartridgeType = clusterContext.getCartridgeUuid();

    Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
    if (cartridge == null) {
      String message =
          "Could not create kubernetes services, cartridge not found: [cartridge] " + cartridgeType;
      log.error(message);
      throw new RuntimeException(message);
    }

    String sessionAffinity = null;
    Property sessionAffinityProperty =
        cartridge.getProperties().getProperty(KUBERNETES_SERVICE_SESSION_AFFINITY);
    if (sessionAffinityProperty != null) {
      sessionAffinity = sessionAffinityProperty.getValue();
    }

    List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
    if (kubernetesServices == null) {
      kubernetesServices = new ArrayList<KubernetesService>();
    }

    // Prepare minion public IP addresses
    List<String> minionPrivateIPList = new ArrayList<String>();
    List<String> minionPublicIPList = new ArrayList<String>();
    KubernetesHost[] kubernetesHosts = kubernetesCluster.getKubernetesHosts();
    if ((kubernetesHosts == null)
        || (kubernetesHosts.length == 0)
        || (kubernetesHosts[0] == null)) {
      throw new RuntimeException(
          "Hosts not found in kubernetes cluster: [cluster] " + kubernetesCluster.getClusterUuid());
    }
    for (KubernetesHost host : kubernetesHosts) {
      if (host != null) {
        minionPrivateIPList.add(host.getPrivateIPAddress());
        minionPublicIPList.add(host.getPublicIPAddress());
      }
    }
    if (log.isDebugEnabled()) {
      log.debug(String.format("Minion private IPs: %s", minionPrivateIPList));
    }

    Collection<ClusterPortMapping> clusterPortMappings =
        CloudControllerContext.getInstance()
            .getClusterPortMappings(clusterContext.getApplicationUuid(), clusterId);

    if (clusterPortMappings != null) {
      for (ClusterPortMapping clusterPortMapping : clusterPortMappings) {

        // Skip if already created
        int containerPort = clusterPortMapping.getPort();
        if (kubernetesServiceExist(kubernetesServices, containerPort)) {
          continue;
        }

        // Find next service sequence no
        long serviceSeqNo = kubernetesClusterContext.getServiceSeqNo().incrementAndGet();
        String serviceId =
            KubernetesIaasUtil.fixSpecialCharacters("service" + "-" + (serviceSeqNo));
        String serviceLabel = DigestUtils.md5Hex(clusterId);

        if (log.isInfoEnabled()) {
          log.info(
              String.format(
                  "Creating kubernetes service: [cluster] %s [service] %s [service-label] %s "
                      + "[protocol] %s [service-port] %d [container-port] %s",
                  clusterId,
                  serviceId,
                  serviceLabel,
                  clusterPortMapping.getProtocol(),
                  clusterPortMapping.getKubernetesServicePort(),
                  containerPort));
        }

        // Create kubernetes service for port mapping
        int servicePort = clusterPortMapping.getKubernetesServicePort();
        String serviceType = clusterPortMapping.getKubernetesServiceType();
        String containerPortName =
            KubernetesIaasUtil.preparePortNameFromPortMapping(clusterPortMapping);

        try {
          // If kubernetes service is already created, skip creating a new one
          if (kubernetesApi.getService(serviceId) == null) {
            // Services need to use minions private IP addresses for creating iptable rules
            kubernetesApi.createService(
                serviceId,
                serviceLabel,
                servicePort,
                serviceType,
                containerPortName,
                containerPort,
                sessionAffinity);
          } else {
            if (log.isDebugEnabled()) {
              log.debug(
                  String.format(
                      "Kubernetes service is already created: [cluster] %s [service] %s "
                          + "[protocol] %s [service-port] %d [container-port] %d",
                      clusterId,
                      serviceId,
                      clusterPortMapping.getProtocol(),
                      servicePort,
                      containerPort));
            }
          }
        } finally {
          // Persist kubernetes service sequence no
          CloudControllerContext.getInstance().persist();
        }

        try {
          Thread.sleep(1000);
        } catch (InterruptedException ignore) {
        }

        Service service = kubernetesApi.getService(serviceId);

        KubernetesService kubernetesService = new KubernetesService();
        kubernetesService.setId(service.getMetadata().getName());
        kubernetesService.setPortalIP(service.getSpec().getClusterIP());
        // Expose minions public IP addresses as they need to be accessed by external networks
        String[] minionPublicIPArray =
            minionPublicIPList.toArray(new String[minionPublicIPList.size()]);
        kubernetesService.setPublicIPs(minionPublicIPArray);
        kubernetesService.setProtocol(clusterPortMapping.getProtocol());
        kubernetesService.setPortName(clusterPortMapping.getName());

        String kubernetesServiceType = service.getSpec().getType();
        kubernetesService.setServiceType(kubernetesServiceType);

        if (kubernetesServiceType.equals(KubernetesConstants.NODE_PORT)) {
          kubernetesService.setPort(service.getSpec().getPorts().get(0).getNodePort());
        } else {
          kubernetesService.setPort(service.getSpec().getPorts().get(0).getPort());
        }

        kubernetesService.setContainerPort(containerPort);
        kubernetesServices.add(kubernetesService);

        if (log.isInfoEnabled()) {
          log.info(
              String.format(
                  "Kubernetes service successfully created: [cluster] %s [service] %s "
                      + "[protocol] %s [node-port] %d [container-port] %s",
                  clusterId,
                  serviceId,
                  clusterPortMapping.getProtocol(),
                  servicePort,
                  containerPort));
        }
      }
    }

    // Add kubernetes services to cluster context and persist
    clusterContext.setKubernetesServices(kubernetesServices);
    CloudControllerContext.getInstance().persist();
  }