Пример #1
0
  /**
   * Creates and returns proxy services for the cluster.
   *
   * @param kubernetesApi
   * @param clusterContext
   * @param kubernetesCluster
   * @param kubernetesClusterContext
   * @throws KubernetesClientException
   */
  private void createKubernetesServices(
      KubernetesApiClient kubernetesApi,
      ClusterContext clusterContext,
      KubernetesCluster kubernetesCluster,
      KubernetesClusterContext kubernetesClusterContext)
      throws KubernetesClientException {

    String clusterId = clusterContext.getClusterId();
    String cartridgeType = clusterContext.getCartridgeUuid();

    Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
    if (cartridge == null) {
      String message =
          "Could not create kubernetes services, cartridge not found: [cartridge] " + cartridgeType;
      log.error(message);
      throw new RuntimeException(message);
    }

    String sessionAffinity = null;
    Property sessionAffinityProperty =
        cartridge.getProperties().getProperty(KUBERNETES_SERVICE_SESSION_AFFINITY);
    if (sessionAffinityProperty != null) {
      sessionAffinity = sessionAffinityProperty.getValue();
    }

    List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
    if (kubernetesServices == null) {
      kubernetesServices = new ArrayList<KubernetesService>();
    }

    // Prepare minion public IP addresses
    List<String> minionPrivateIPList = new ArrayList<String>();
    List<String> minionPublicIPList = new ArrayList<String>();
    KubernetesHost[] kubernetesHosts = kubernetesCluster.getKubernetesHosts();
    if ((kubernetesHosts == null)
        || (kubernetesHosts.length == 0)
        || (kubernetesHosts[0] == null)) {
      throw new RuntimeException(
          "Hosts not found in kubernetes cluster: [cluster] " + kubernetesCluster.getClusterUuid());
    }
    for (KubernetesHost host : kubernetesHosts) {
      if (host != null) {
        minionPrivateIPList.add(host.getPrivateIPAddress());
        minionPublicIPList.add(host.getPublicIPAddress());
      }
    }
    if (log.isDebugEnabled()) {
      log.debug(String.format("Minion private IPs: %s", minionPrivateIPList));
    }

    Collection<ClusterPortMapping> clusterPortMappings =
        CloudControllerContext.getInstance()
            .getClusterPortMappings(clusterContext.getApplicationUuid(), clusterId);

    if (clusterPortMappings != null) {
      for (ClusterPortMapping clusterPortMapping : clusterPortMappings) {

        // Skip if already created
        int containerPort = clusterPortMapping.getPort();
        if (kubernetesServiceExist(kubernetesServices, containerPort)) {
          continue;
        }

        // Find next service sequence no
        long serviceSeqNo = kubernetesClusterContext.getServiceSeqNo().incrementAndGet();
        String serviceId =
            KubernetesIaasUtil.fixSpecialCharacters("service" + "-" + (serviceSeqNo));
        String serviceLabel = DigestUtils.md5Hex(clusterId);

        if (log.isInfoEnabled()) {
          log.info(
              String.format(
                  "Creating kubernetes service: [cluster] %s [service] %s [service-label] %s "
                      + "[protocol] %s [service-port] %d [container-port] %s",
                  clusterId,
                  serviceId,
                  serviceLabel,
                  clusterPortMapping.getProtocol(),
                  clusterPortMapping.getKubernetesServicePort(),
                  containerPort));
        }

        // Create kubernetes service for port mapping
        int servicePort = clusterPortMapping.getKubernetesServicePort();
        String serviceType = clusterPortMapping.getKubernetesServiceType();
        String containerPortName =
            KubernetesIaasUtil.preparePortNameFromPortMapping(clusterPortMapping);

        try {
          // If kubernetes service is already created, skip creating a new one
          if (kubernetesApi.getService(serviceId) == null) {
            // Services need to use minions private IP addresses for creating iptable rules
            kubernetesApi.createService(
                serviceId,
                serviceLabel,
                servicePort,
                serviceType,
                containerPortName,
                containerPort,
                sessionAffinity);
          } else {
            if (log.isDebugEnabled()) {
              log.debug(
                  String.format(
                      "Kubernetes service is already created: [cluster] %s [service] %s "
                          + "[protocol] %s [service-port] %d [container-port] %d",
                      clusterId,
                      serviceId,
                      clusterPortMapping.getProtocol(),
                      servicePort,
                      containerPort));
            }
          }
        } finally {
          // Persist kubernetes service sequence no
          CloudControllerContext.getInstance().persist();
        }

        try {
          Thread.sleep(1000);
        } catch (InterruptedException ignore) {
        }

        Service service = kubernetesApi.getService(serviceId);

        KubernetesService kubernetesService = new KubernetesService();
        kubernetesService.setId(service.getMetadata().getName());
        kubernetesService.setPortalIP(service.getSpec().getClusterIP());
        // Expose minions public IP addresses as they need to be accessed by external networks
        String[] minionPublicIPArray =
            minionPublicIPList.toArray(new String[minionPublicIPList.size()]);
        kubernetesService.setPublicIPs(minionPublicIPArray);
        kubernetesService.setProtocol(clusterPortMapping.getProtocol());
        kubernetesService.setPortName(clusterPortMapping.getName());

        String kubernetesServiceType = service.getSpec().getType();
        kubernetesService.setServiceType(kubernetesServiceType);

        if (kubernetesServiceType.equals(KubernetesConstants.NODE_PORT)) {
          kubernetesService.setPort(service.getSpec().getPorts().get(0).getNodePort());
        } else {
          kubernetesService.setPort(service.getSpec().getPorts().get(0).getPort());
        }

        kubernetesService.setContainerPort(containerPort);
        kubernetesServices.add(kubernetesService);

        if (log.isInfoEnabled()) {
          log.info(
              String.format(
                  "Kubernetes service successfully created: [cluster] %s [service] %s "
                      + "[protocol] %s [node-port] %d [container-port] %s",
                  clusterId,
                  serviceId,
                  clusterPortMapping.getProtocol(),
                  servicePort,
                  containerPort));
        }
      }
    }

    // Add kubernetes services to cluster context and persist
    clusterContext.setKubernetesServices(kubernetesServices);
    CloudControllerContext.getInstance().persist();
  }
Пример #2
0
  /**
   * Starts a container via kubernetes for the given member context.
   *
   * @param memberContext
   * @return
   * @throws CartridgeNotFoundException
   */
  public MemberContext startContainer(MemberContext memberContext)
      throws CartridgeNotFoundException {
    Lock lock = null;
    try {
      lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();

      handleNullObject(memberContext, "member context is null");
      log.info(
          String.format(
              "Starting container: [application] %s [cartridge] %s [member] %s",
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId()));

      // Validate cluster id
      String clusterId = memberContext.getClusterId();
      String memberId = memberContext.getMemberId();
      handleNullObject(clusterId, "cluster id is null in member context");

      // Validate cluster context
      ClusterContext clusterContext =
          CloudControllerContext.getInstance().getClusterContext(clusterId);
      handleNullObject(
          clusterContext,
          String.format(
              "Cluster context not found: [application] %s [cartridge] %s " + "[cluster] %s",
              memberContext.getApplicationId(), memberContext.getCartridgeType(), clusterId));

      // Validate partition
      Partition partition = memberContext.getPartition();
      handleNullObject(
          partition,
          String.format(
              "partition not found in member context: [application] %s "
                  + "[cartridge] %s [member] %s",
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId()));

      // Validate cartridge
      String cartridgeType = clusterContext.getCartridgeUuid();
      Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
      if (cartridge == null) {
        String msg =
            String.format(
                "Cartridge not found: [application] %s [cartridge] %s",
                memberContext.getApplicationId(), memberContext.getCartridgeType());
        log.error(msg);
        throw new CartridgeNotFoundException(msg);
      }

      String kubernetesClusterId = partition.getKubernetesClusterId();
      clusterContext.setKubernetesClusterId(kubernetesClusterId);
      KubernetesCluster kubernetesCluster =
          CloudControllerContext.getInstance().getKubernetesCluster(kubernetesClusterId);
      handleNullObject(
          kubernetesCluster,
          "kubernetes cluster not found: "
              + "[kubernetes-cluster] "
              + kubernetesClusterId
              + " [cluster] "
              + clusterId
              + " [member] "
              + memberId);

      // Prepare kubernetes context
      String kubernetesMasterIp = kubernetesCluster.getKubernetesMaster().getPrivateIPAddress();
      PortRange kubernetesPortRange = kubernetesCluster.getPortRange();
      String kubernetesMasterPort =
          CloudControllerUtil.getProperty(
              kubernetesCluster.getKubernetesMaster().getProperties(),
              StratosConstants.KUBERNETES_MASTER_PORT,
              StratosConstants.KUBERNETES_MASTER_DEFAULT_PORT);

      // Add kubernetes cluster payload parameters to payload
      if ((kubernetesCluster.getProperties() != null)
          && (kubernetesCluster.getProperties().getProperties() != null)) {
        for (Property property : kubernetesCluster.getProperties().getProperties()) {
          if (property != null) {
            if (property.getName().startsWith(PAYLOAD_PARAMETER_PREFIX)) {
              String name = property.getName().replace(PAYLOAD_PARAMETER_PREFIX, "");
              payload.add(new NameValuePair(name, property.getValue()));
            }
          }
        }
      }

      KubernetesClusterContext kubernetesClusterContext =
          getKubernetesClusterContext(
              kubernetesClusterId,
              kubernetesMasterIp,
              kubernetesMasterPort,
              kubernetesPortRange.getUpper(),
              kubernetesPortRange.getLower());

      // Generate kubernetes service ports and update port mappings in cartridge
      generateKubernetesServicePorts(
          clusterContext.getApplicationUuid(),
          clusterContext.getClusterId(),
          kubernetesClusterContext,
          cartridge);

      // Create kubernetes services for port mappings
      KubernetesApiClient kubernetesApi = kubernetesClusterContext.getKubApi();
      createKubernetesServices(
          kubernetesApi, clusterContext, kubernetesCluster, kubernetesClusterContext);

      // Create pod
      createPod(clusterContext, memberContext, kubernetesApi, kubernetesClusterContext);

      // Wait for pod status to be changed to running
      Pod pod = waitForPodToBeActivated(memberContext, kubernetesApi);

      // Update member context
      updateMemberContext(memberContext, pod, kubernetesCluster);

      log.info(
          String.format(
              "Container started successfully: [application] %s [cartridge] %s [member] %s "
                  + "[pod] %s [cpu] %d [memory] %d MB",
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId(),
              memberContext.getKubernetesPodId(),
              memberContext.getInstanceMetadata().getCpu(),
              memberContext.getInstanceMetadata().getRam()));
      return memberContext;
    } catch (Exception e) {
      String msg =
          String.format(
              "Could not start container: [application] %s [cartridge] %s [member] %s",
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId());
      log.error(msg, e);
      throw new RuntimeException(msg, e);
    } finally {
      if (lock != null) {
        CloudControllerContext.getInstance().releaseWriteLock(lock);
      }
    }
  }
Пример #3
0
  /**
   * Create new pod and pass environment variables.
   *
   * @param memberContext
   * @param kubernetesApi
   * @param kubernetesClusterContext
   * @throws KubernetesClientException
   */
  private void createPod(
      ClusterContext clusterContext,
      MemberContext memberContext,
      KubernetesApiClient kubernetesApi,
      KubernetesClusterContext kubernetesClusterContext)
      throws KubernetesClientException {

    String applicationId = memberContext.getApplicationId();
    String cartridgeType = memberContext.getCartridgeType();
    String clusterId = memberContext.getClusterId();
    String memberId = memberContext.getMemberId();

    if (log.isInfoEnabled()) {
      log.info(
          String.format(
              "Creating kubernetes pod: [application] %s [cartridge] %s [member] %s",
              applicationId, cartridgeType, memberId));
    }

    Partition partition = memberContext.getPartition();
    if (partition == null) {
      String message =
          String.format(
              "Partition not found in member context: [application] %s [cartridge] %s "
                  + "[member] %s ",
              applicationId, cartridgeType, memberId);
      log.error(message);
      throw new RuntimeException(message);
    }

    Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
    if (cartridge == null) {
      String message = "Could not find cartridge: [cartridge] " + cartridgeType;
      log.error(message);
      throw new RuntimeException(message);
    }

    // Set default values to zero to avoid cpu and memory restrictions
    int cpu = Integer.getInteger(KUBERNETES_CONTAINER_CPU_DEFAULT, 0);
    int memory = Integer.getInteger(KUBERNETES_CONTAINER_MEMORY_DEFAULT, 0);
    Property cpuProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_CPU);
    if (cpuProperty != null) {
      cpu = Integer.parseInt(cpuProperty.getValue());
    }
    Property memoryProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_MEMORY);
    if (memoryProperty != null) {
      memory = Integer.parseInt(memoryProperty.getValue());
    }

    IaasProvider iaasProvider =
        CloudControllerContext.getInstance()
            .getIaasProviderOfPartition(cartridge.getUuid(), partition.getUuid());
    if (iaasProvider == null) {
      String message = "Could not find iaas provider: [partition] " + partition.getUuid();
      log.error(message);
      throw new RuntimeException(message);
    }

    // Add dynamic payload to the member context
    memberContext.setDynamicPayload(payload.toArray(new NameValuePair[payload.size()]));

    // Create pod
    long podSeqNo = kubernetesClusterContext.getPodSeqNo().incrementAndGet();
    String podId = "pod" + "-" + podSeqNo;
    String podLabel = DigestUtils.md5Hex(clusterId);
    String dockerImage = iaasProvider.getImage();
    List<EnvVar> environmentVariables =
        KubernetesIaasUtil.prepareEnvironmentVariables(clusterContext, memberContext);

    List<ContainerPort> ports =
        KubernetesIaasUtil.convertPortMappings(Arrays.asList(cartridge.getPortMappings()));

    log.info(
        String.format(
            "Starting pod: [application] %s [cartridge] %s [member] %s "
                + "[cpu] %d [memory] %d MB",
            memberContext.getApplicationId(),
            memberContext.getCartridgeType(),
            memberContext.getMemberId(),
            cpu,
            memory));

    kubernetesApi.createPod(podId, podLabel, dockerImage, cpu, memory, ports, environmentVariables);

    log.info(
        String.format(
            "Pod started successfully: [application] %s [cartridge] %s [member] %s "
                + "[pod] %s [cpu] %d [memory] %d MB",
            memberContext.getApplicationId(),
            memberContext.getCartridgeType(),
            memberContext.getMemberId(),
            podId,
            cpu,
            memory));

    // Add pod id to member context
    memberContext.setKubernetesPodId(podId);
    memberContext.setKubernetesPodLabel(podLabel);

    // Create instance metadata
    InstanceMetadata instanceMetadata = new InstanceMetadata();
    instanceMetadata.setImageId(dockerImage);
    instanceMetadata.setCpu(cpu);
    instanceMetadata.setRam(memory);
    memberContext.setInstanceMetadata(instanceMetadata);

    // Persist cloud controller context
    CloudControllerContext.getInstance().persist();
  }
Пример #4
0
  public static void handleServiceCreated(List<Cartridge> cartridgeList) {
    Service service;
    Topology topology = TopologyManager.getTopology();
    if (cartridgeList == null) {
      log.warn(String.format("Cartridge list is empty"));
      return;
    }

    try {

      TopologyManager.acquireWriteLock();
      for (Cartridge cartridge : cartridgeList) {
        if (!topology.serviceExists(cartridge.getType())) {
          ServiceType serviceType =
              cartridge.isMultiTenant() ? ServiceType.MultiTenant : ServiceType.SingleTenant;
          service = new Service(cartridge.getType(), serviceType);
          Properties properties = new Properties();

          try {
            Property[] propertyArray = null;

            if (cartridge.getProperties() != null) {
              if (cartridge.getProperties().getProperties() != null) {
                propertyArray = cartridge.getProperties().getProperties();
              }
            }

            List<Property> propertyList = new ArrayList<Property>();
            if (propertyArray != null) {
              propertyList = Arrays.asList(propertyArray);
              if (propertyList != null) {
                for (Property property : propertyList) {
                  properties.setProperty(property.getName(), property.getValue());
                }
              }
            }
          } catch (Exception e) {
            log.error(e);
          }

          service.setProperties(properties);
          if (cartridge.getPortMappings() != null) {
            List<PortMapping> portMappings = Arrays.asList(cartridge.getPortMappings());
            Port port;
            // adding ports to the event
            for (PortMapping portMapping : portMappings) {
              port =
                  new Port(
                      portMapping.getProtocol(), portMapping.getPort(), portMapping.getProxyPort());
              service.addPort(port);
            }
          }

          topology.addService(service);
          TopologyManager.updateTopology(topology);
        }
      }
    } finally {
      TopologyManager.releaseWriteLock();
    }
    TopologyEventPublisher.sendServiceCreateEvent(cartridgeList);
  }