private void updateMemberContext(
      MemberContext memberContext, Pod pod, KubernetesCluster kubernetesCluster) {

    String memberPrivateIPAddress = pod.getStatus().getPodIP();
    String podHostIPAddress = pod.getStatus().getHostIP();
    String memberPublicIPAddress = podHostIPAddress;
    String kubernetesHostPublicIP =
        findKubernetesHostPublicIPAddress(kubernetesCluster, podHostIPAddress);

    if (StringUtils.isNotBlank(kubernetesHostPublicIP)) {
      memberPublicIPAddress = kubernetesHostPublicIP;
      if (log.isInfoEnabled()) {
        log.info(
            String.format(
                "Member public IP address set to kubernetes host public IP address:"
                    + "[pod-host-ip] %s [kubernetes-host-public-ip] %s",
                podHostIPAddress, kubernetesHostPublicIP));
      }
    }

    memberContext.setInstanceId(pod.getMetadata().getName());
    memberContext.setDefaultPrivateIP(memberPrivateIPAddress);
    memberContext.setPrivateIPs(new String[] {memberPrivateIPAddress});
    memberContext.setDefaultPublicIP(memberPublicIPAddress);
    memberContext.setPublicIPs(new String[] {memberPublicIPAddress});
    memberContext.setInitTime(memberContext.getInitTime());
    memberContext.setProperties(memberContext.getProperties());
  }
Exemple #2
0
  /**
   * Add member object to the topology and publish member created event
   *
   * @param memberContext
   */
  public static void handleMemberCreatedEvent(MemberContext memberContext) {
    Topology topology = TopologyManager.getTopology();

    Service service = topology.getService(memberContext.getCartridgeType());
    String clusterId = memberContext.getClusterId();
    Cluster cluster = service.getCluster(clusterId);
    String memberId = memberContext.getMemberId();
    String clusterInstanceId = memberContext.getClusterInstanceId();
    String networkPartitionId = memberContext.getNetworkPartitionId();
    String partitionId = memberContext.getPartition().getId();
    String lbClusterId = memberContext.getLbClusterId();
    long initTime = memberContext.getInitTime();

    if (cluster.memberExists(memberId)) {
      log.warn(String.format("Member %s already exists", memberId));
      return;
    }

    try {
      TopologyManager.acquireWriteLock();
      Member member =
          new Member(
              service.getServiceName(),
              clusterId,
              memberId,
              clusterInstanceId,
              networkPartitionId,
              partitionId,
              memberContext.getLoadBalancingIPType(),
              initTime);
      member.setStatus(MemberStatus.Created);
      member.setLbClusterId(lbClusterId);
      member.setProperties(CloudControllerUtil.toJavaUtilProperties(memberContext.getProperties()));
      cluster.addMember(member);
      TopologyManager.updateTopology(topology);
    } finally {
      TopologyManager.releaseWriteLock();
    }

    TopologyEventPublisher.sendMemberCreatedEvent(memberContext);
  }
  /**
   * Terminate a container by member id
   *
   * @param memberId
   * @return
   * @throws MemberTerminationFailedException
   */
  public MemberContext terminateContainer(String memberId) throws MemberTerminationFailedException {
    Lock lock = null;
    try {
      lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();
      handleNullObject(memberId, "Could not terminate container, member id is null");

      MemberContext memberContext =
          CloudControllerContext.getInstance().getMemberContextOfMemberId(memberId);
      handleNullObject(
          memberContext,
          "Could not terminate container, member context not found: [member-id] " + memberId);

      String clusterId = memberContext.getClusterId();
      handleNullObject(
          clusterId, "Could not terminate container, cluster id is null: [member-id] " + memberId);

      ClusterContext clusterContext =
          CloudControllerContext.getInstance().getClusterContext(clusterId);
      handleNullObject(
          clusterContext,
          String.format(
              "Could not terminate container, cluster context not found: "
                  + "[cluster-id] %s [member-id] %s",
              clusterId, memberId));

      String kubernetesClusterId = clusterContext.getKubernetesClusterId();
      handleNullObject(
          kubernetesClusterId,
          String.format(
              "Could not terminate container, kubernetes cluster "
                  + "context id is null: [cluster-id] %s [member-id] %s",
              clusterId, memberId));

      KubernetesClusterContext kubernetesClusterContext =
          CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId);
      handleNullObject(
          kubernetesClusterContext,
          String.format(
              "Could not terminate container, kubernetes cluster "
                  + "context not found: [cluster-id] %s [member-id] %s",
              clusterId, memberId));
      KubernetesApiClient kubApi = kubernetesClusterContext.getKubApi();

      try {
        log.info(
            String.format(
                "Removing kubernetes pod: [application] %s [cartridge] %s [member] %s [pod] %s",
                memberContext.getApplicationId(),
                memberContext.getCartridgeType(),
                memberContext.getMemberId(),
                memberContext.getKubernetesPodId()));

        // Remove pod
        kubApi.deletePod(memberContext.getKubernetesPodId());
        // Persist changes
        CloudControllerContext.getInstance().persist();

        log.info(
            String.format(
                "Kubernetes pod removed successfully: [application] %s [cartridge] %s "
                    + "[member] %s [pod] %s",
                memberContext.getApplicationId(),
                memberContext.getCartridgeType(),
                memberContext.getMemberId(),
                memberContext.getKubernetesPodId()));
      } catch (KubernetesClientException ignore) {
        // we can't do nothing here
        log.warn(
            String.format("Could not delete pod: [pod-id] %s", memberContext.getKubernetesPodId()));
      }
      return memberContext;
    } finally {
      if (lock != null) {
        CloudControllerContext.getInstance().releaseWriteLock(lock);
      }
    }
  }
  /**
   * Terminate all the containers belong to a cluster by cluster id.
   *
   * @param clusterId
   * @return
   * @throws InvalidClusterException
   */
  public MemberContext[] terminateContainers(String clusterId) throws InvalidClusterException {
    Lock lock = null;
    try {
      lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();

      ClusterContext clusterContext =
          CloudControllerContext.getInstance().getClusterContext(clusterId);
      handleNullObject(
          clusterContext,
          "Could not terminate containers, cluster not found: [cluster-id] " + clusterId);

      String kubernetesClusterId = clusterContext.getKubernetesClusterId();
      handleNullObject(
          kubernetesClusterId,
          "Could not terminate containers, kubernetes cluster id not found: "
              + "[cluster-id] "
              + clusterId);

      KubernetesClusterContext kubClusterContext =
          CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId);
      handleNullObject(
          kubClusterContext,
          "Could not terminate containers, kubernetes cluster not found: "
              + "[kubernetes-cluster-id] "
              + kubernetesClusterId);

      KubernetesApiClient kubApi = kubClusterContext.getKubApi();

      // Remove kubernetes services
      List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
      if (kubernetesServices != null) {
        for (KubernetesService kubernetesService : kubernetesServices) {
          try {
            kubApi.deleteService(kubernetesService.getId());
            int allocatedPort = kubernetesService.getPort();
            kubClusterContext.deallocatePort(allocatedPort);
          } catch (KubernetesClientException e) {
            log.error("Could not remove kubernetes service: [cluster-id] " + clusterId, e);
          }
        }
      }

      List<MemberContext> memberContextsRemoved = new ArrayList<MemberContext>();
      List<MemberContext> memberContexts =
          CloudControllerContext.getInstance().getMemberContextsOfClusterId(clusterId);
      if (memberContexts != null) {
        for (MemberContext memberContext : memberContexts) {
          try {
            MemberContext memberContextRemoved = terminateContainer(memberContext.getMemberId());
            memberContextsRemoved.add(memberContextRemoved);
          } catch (MemberTerminationFailedException e) {
            String message =
                "Could not terminate container: [member-id] " + memberContext.getMemberId();
            log.error(message);
          }
        }
      }

      // Persist changes
      CloudControllerContext.getInstance().persist();
      return memberContextsRemoved.toArray(new MemberContext[memberContextsRemoved.size()]);
    } finally {
      if (lock != null) {
        CloudControllerContext.getInstance().releaseWriteLock(lock);
      }
    }
  }
  /**
   * Create new pod and pass environment variables.
   *
   * @param memberContext
   * @param kubernetesApi
   * @param kubernetesClusterContext
   * @throws KubernetesClientException
   */
  private void createPod(
      ClusterContext clusterContext,
      MemberContext memberContext,
      KubernetesApiClient kubernetesApi,
      KubernetesClusterContext kubernetesClusterContext)
      throws KubernetesClientException {

    String applicationId = memberContext.getApplicationId();
    String cartridgeType = memberContext.getCartridgeType();
    String clusterId = memberContext.getClusterId();
    String memberId = memberContext.getMemberId();

    if (log.isInfoEnabled()) {
      log.info(
          String.format(
              "Creating kubernetes pod: [application] %s [cartridge] %s [member] %s",
              applicationId, cartridgeType, memberId));
    }

    Partition partition = memberContext.getPartition();
    if (partition == null) {
      String message =
          String.format(
              "Partition not found in member context: [application] %s [cartridge] %s "
                  + "[member] %s ",
              applicationId, cartridgeType, memberId);
      log.error(message);
      throw new RuntimeException(message);
    }

    Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
    if (cartridge == null) {
      String message = "Could not find cartridge: [cartridge] " + cartridgeType;
      log.error(message);
      throw new RuntimeException(message);
    }

    // Set default values to zero to avoid cpu and memory restrictions
    int cpu = Integer.getInteger(KUBERNETES_CONTAINER_CPU_DEFAULT, 0);
    int memory = Integer.getInteger(KUBERNETES_CONTAINER_MEMORY_DEFAULT, 0);
    Property cpuProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_CPU);
    if (cpuProperty != null) {
      cpu = Integer.parseInt(cpuProperty.getValue());
    }
    Property memoryProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_MEMORY);
    if (memoryProperty != null) {
      memory = Integer.parseInt(memoryProperty.getValue());
    }

    IaasProvider iaasProvider =
        CloudControllerContext.getInstance()
            .getIaasProviderOfPartition(cartridge.getUuid(), partition.getUuid());
    if (iaasProvider == null) {
      String message = "Could not find iaas provider: [partition] " + partition.getUuid();
      log.error(message);
      throw new RuntimeException(message);
    }

    // Add dynamic payload to the member context
    memberContext.setDynamicPayload(payload.toArray(new NameValuePair[payload.size()]));

    // Create pod
    long podSeqNo = kubernetesClusterContext.getPodSeqNo().incrementAndGet();
    String podId = "pod" + "-" + podSeqNo;
    String podLabel = DigestUtils.md5Hex(clusterId);
    String dockerImage = iaasProvider.getImage();
    List<EnvVar> environmentVariables =
        KubernetesIaasUtil.prepareEnvironmentVariables(clusterContext, memberContext);

    List<ContainerPort> ports =
        KubernetesIaasUtil.convertPortMappings(Arrays.asList(cartridge.getPortMappings()));

    log.info(
        String.format(
            "Starting pod: [application] %s [cartridge] %s [member] %s "
                + "[cpu] %d [memory] %d MB",
            memberContext.getApplicationId(),
            memberContext.getCartridgeType(),
            memberContext.getMemberId(),
            cpu,
            memory));

    kubernetesApi.createPod(podId, podLabel, dockerImage, cpu, memory, ports, environmentVariables);

    log.info(
        String.format(
            "Pod started successfully: [application] %s [cartridge] %s [member] %s "
                + "[pod] %s [cpu] %d [memory] %d MB",
            memberContext.getApplicationId(),
            memberContext.getCartridgeType(),
            memberContext.getMemberId(),
            podId,
            cpu,
            memory));

    // Add pod id to member context
    memberContext.setKubernetesPodId(podId);
    memberContext.setKubernetesPodLabel(podLabel);

    // Create instance metadata
    InstanceMetadata instanceMetadata = new InstanceMetadata();
    instanceMetadata.setImageId(dockerImage);
    instanceMetadata.setCpu(cpu);
    instanceMetadata.setRam(memory);
    memberContext.setInstanceMetadata(instanceMetadata);

    // Persist cloud controller context
    CloudControllerContext.getInstance().persist();
  }
  private Pod waitForPodToBeActivated(
      MemberContext memberContext, KubernetesApiClient kubernetesApi)
      throws KubernetesClientException, InterruptedException {

    Pod pod;
    boolean podCreated = false;
    boolean podRunning = false;
    long startTime = System.currentTimeMillis();

    while (!podRunning) {
      pod = kubernetesApi.getPod(memberContext.getKubernetesPodId());
      if (pod != null) {
        podCreated = true;
        if (pod.getStatus().getPhase().equals(KubernetesConstants.POD_STATUS_RUNNING)) {
          log.info(
              String.format(
                  "Pod status changed to running: [application] %s [cartridge] %s [member] %s "
                      + "[pod] %s",
                  memberContext.getApplicationId(),
                  memberContext.getCartridgeType(),
                  memberContext.getMemberId(),
                  pod.getMetadata().getName()));
          return pod;
        } else {
          log.info(
              String.format(
                  "Waiting pod status to be changed to running: [application] %s "
                      + "[cartridge] %s [member] %s [pod] %s",
                  memberContext.getApplicationId(),
                  memberContext.getCartridgeType(),
                  memberContext.getMemberId(),
                  pod.getMetadata().getName()));
        }
      } else {
        log.info(
            String.format(
                "Waiting for pod to be created: [application] %s "
                    + "[cartridge] %s [member] %s [pod] %s",
                memberContext.getApplicationId(),
                memberContext.getCartridgeType(),
                memberContext.getMemberId(),
                memberContext.getKubernetesPodId()));
      }

      if ((System.currentTimeMillis() - startTime) > podActivationTimeout) {
        break;
      }
      Thread.sleep(5000);
    }

    String message;
    if (podCreated) {
      // Pod created but status did not change to running
      message =
          String.format(
              "Pod status did not change to running within %d sec: "
                  + "[application] %s [cartridge] %s [member] %s [pod] %s",
              (podActivationTimeout.intValue() / 1000),
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId(),
              memberContext.getKubernetesPodId());
      log.error(message);
    } else {
      // Pod did not create
      message =
          String.format(
              "Pod did not create within %d sec: "
                  + "[application] %s [cartridge] %s [member] %s [pod] %s",
              (podActivationTimeout.intValue() / 1000),
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId(),
              memberContext.getKubernetesPodId());
      log.error(message);
    }

    throw new RuntimeException(message);
  }
  /**
   * Starts a container via kubernetes for the given member context.
   *
   * @param memberContext
   * @return
   * @throws CartridgeNotFoundException
   */
  public MemberContext startContainer(MemberContext memberContext)
      throws CartridgeNotFoundException {
    Lock lock = null;
    try {
      lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();

      handleNullObject(memberContext, "member context is null");
      log.info(
          String.format(
              "Starting container: [application] %s [cartridge] %s [member] %s",
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId()));

      // Validate cluster id
      String clusterId = memberContext.getClusterId();
      String memberId = memberContext.getMemberId();
      handleNullObject(clusterId, "cluster id is null in member context");

      // Validate cluster context
      ClusterContext clusterContext =
          CloudControllerContext.getInstance().getClusterContext(clusterId);
      handleNullObject(
          clusterContext,
          String.format(
              "Cluster context not found: [application] %s [cartridge] %s " + "[cluster] %s",
              memberContext.getApplicationId(), memberContext.getCartridgeType(), clusterId));

      // Validate partition
      Partition partition = memberContext.getPartition();
      handleNullObject(
          partition,
          String.format(
              "partition not found in member context: [application] %s "
                  + "[cartridge] %s [member] %s",
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId()));

      // Validate cartridge
      String cartridgeType = clusterContext.getCartridgeUuid();
      Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
      if (cartridge == null) {
        String msg =
            String.format(
                "Cartridge not found: [application] %s [cartridge] %s",
                memberContext.getApplicationId(), memberContext.getCartridgeType());
        log.error(msg);
        throw new CartridgeNotFoundException(msg);
      }

      String kubernetesClusterId = partition.getKubernetesClusterId();
      clusterContext.setKubernetesClusterId(kubernetesClusterId);
      KubernetesCluster kubernetesCluster =
          CloudControllerContext.getInstance().getKubernetesCluster(kubernetesClusterId);
      handleNullObject(
          kubernetesCluster,
          "kubernetes cluster not found: "
              + "[kubernetes-cluster] "
              + kubernetesClusterId
              + " [cluster] "
              + clusterId
              + " [member] "
              + memberId);

      // Prepare kubernetes context
      String kubernetesMasterIp = kubernetesCluster.getKubernetesMaster().getPrivateIPAddress();
      PortRange kubernetesPortRange = kubernetesCluster.getPortRange();
      String kubernetesMasterPort =
          CloudControllerUtil.getProperty(
              kubernetesCluster.getKubernetesMaster().getProperties(),
              StratosConstants.KUBERNETES_MASTER_PORT,
              StratosConstants.KUBERNETES_MASTER_DEFAULT_PORT);

      // Add kubernetes cluster payload parameters to payload
      if ((kubernetesCluster.getProperties() != null)
          && (kubernetesCluster.getProperties().getProperties() != null)) {
        for (Property property : kubernetesCluster.getProperties().getProperties()) {
          if (property != null) {
            if (property.getName().startsWith(PAYLOAD_PARAMETER_PREFIX)) {
              String name = property.getName().replace(PAYLOAD_PARAMETER_PREFIX, "");
              payload.add(new NameValuePair(name, property.getValue()));
            }
          }
        }
      }

      KubernetesClusterContext kubernetesClusterContext =
          getKubernetesClusterContext(
              kubernetesClusterId,
              kubernetesMasterIp,
              kubernetesMasterPort,
              kubernetesPortRange.getUpper(),
              kubernetesPortRange.getLower());

      // Generate kubernetes service ports and update port mappings in cartridge
      generateKubernetesServicePorts(
          clusterContext.getApplicationUuid(),
          clusterContext.getClusterId(),
          kubernetesClusterContext,
          cartridge);

      // Create kubernetes services for port mappings
      KubernetesApiClient kubernetesApi = kubernetesClusterContext.getKubApi();
      createKubernetesServices(
          kubernetesApi, clusterContext, kubernetesCluster, kubernetesClusterContext);

      // Create pod
      createPod(clusterContext, memberContext, kubernetesApi, kubernetesClusterContext);

      // Wait for pod status to be changed to running
      Pod pod = waitForPodToBeActivated(memberContext, kubernetesApi);

      // Update member context
      updateMemberContext(memberContext, pod, kubernetesCluster);

      log.info(
          String.format(
              "Container started successfully: [application] %s [cartridge] %s [member] %s "
                  + "[pod] %s [cpu] %d [memory] %d MB",
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId(),
              memberContext.getKubernetesPodId(),
              memberContext.getInstanceMetadata().getCpu(),
              memberContext.getInstanceMetadata().getRam()));
      return memberContext;
    } catch (Exception e) {
      String msg =
          String.format(
              "Could not start container: [application] %s [cartridge] %s [member] %s",
              memberContext.getApplicationId(),
              memberContext.getCartridgeType(),
              memberContext.getMemberId());
      log.error(msg, e);
      throw new RuntimeException(msg, e);
    } finally {
      if (lock != null) {
        CloudControllerContext.getInstance().releaseWriteLock(lock);
      }
    }
  }
 @Override
 public void terminateInstance(MemberContext memberContext)
     throws InvalidCartridgeTypeException, InvalidMemberException,
         MemberTerminationFailedException {
   terminateContainer(memberContext.getMemberId());
 }
Exemple #9
0
  /**
   * Update member status to initialized and publish member initialized event
   *
   * @param memberContext
   */
  public static void handleMemberInitializedEvent(MemberContext memberContext) {
    Topology topology = TopologyManager.getTopology();
    Service service = topology.getService(memberContext.getCartridgeType());
    if (service == null) {
      log.warn(String.format("Service %s does not exist", memberContext.getCartridgeType()));
      return;
    }
    if (!service.clusterExists(memberContext.getClusterId())) {
      log.warn(
          String.format(
              "Cluster %s does not exist in service %s",
              memberContext.getClusterId(), memberContext.getCartridgeType()));
      return;
    }

    Member member =
        service.getCluster(memberContext.getClusterId()).getMember(memberContext.getMemberId());
    if (member == null) {
      log.warn(String.format("Member %s does not exist", memberContext.getMemberId()));
      return;
    }

    try {
      TopologyManager.acquireWriteLock();

      // Set ip addresses
      member.setDefaultPrivateIP(memberContext.getDefaultPrivateIP());
      if (memberContext.getPrivateIPs() != null) {
        member.setMemberPrivateIPs(Arrays.asList(memberContext.getPrivateIPs()));
      }
      member.setDefaultPublicIP(memberContext.getDefaultPublicIP());
      if (memberContext.getPublicIPs() != null) {
        member.setMemberPublicIPs(Arrays.asList(memberContext.getPublicIPs()));
      }

      // try update lifecycle state
      if (!member.isStateTransitionValid(MemberStatus.Initialized)) {
        log.error(
            "Invalid state transition from "
                + member.getStatus()
                + " to "
                + MemberStatus.Initialized);
        return;
      } else {
        member.setStatus(MemberStatus.Initialized);
        log.info("Member status updated to initialized");

        TopologyManager.updateTopology(topology);

        TopologyEventPublisher.sendMemberInitializedEvent(memberContext);
        // publishing data
        BAMUsageDataPublisher.publish(
            memberContext.getMemberId(),
            memberContext.getPartition().getId(),
            memberContext.getNetworkPartitionId(),
            memberContext.getClusterId(),
            memberContext.getCartridgeType(),
            MemberStatus.Initialized.toString(),
            null);
      }
    } finally {
      TopologyManager.releaseWriteLock();
    }
  }