Beispiel #1
0
  /**
   * Remove kubernetes services if available for application cluster.
   *
   * @param applicationId
   * @param clusterId
   */
  public static void removeKubernetesServices(String applicationId, String clusterId) {

    ClusterContext clusterContext =
        CloudControllerContext.getInstance().getClusterContext(clusterId);
    if (clusterContext != null) {
      String kubernetesClusterId = clusterContext.getKubernetesClusterId();
      if (org.apache.commons.lang3.StringUtils.isNotBlank(kubernetesClusterId)) {
        KubernetesClusterContext kubernetesClusterContext =
            CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId);
        if (kubernetesClusterContext != null) {
          KubernetesApiClient kubernetesApiClient = kubernetesClusterContext.getKubApi();
          for (KubernetesService kubernetesService : clusterContext.getKubernetesServices()) {
            log.info(
                String.format(
                    "Deleting kubernetes service: [application-id] %s " + "[service-id] %s",
                    applicationId, kubernetesService.getId()));
            try {
              kubernetesApiClient.deleteService(kubernetesService.getId());
            } catch (KubernetesClientException e) {
              log.error(
                  String.format(
                      "Could not delete kubernetes service: [application-id] %s "
                          + "[service-id] %s",
                      applicationId, kubernetesService.getId()));
            }
          }
        }
      }
    }
  }
Beispiel #2
0
  /**
   * Terminate all the containers belong to a cluster by cluster id.
   *
   * @param clusterId
   * @return
   * @throws InvalidClusterException
   */
  public MemberContext[] terminateContainers(String clusterId) throws InvalidClusterException {
    Lock lock = null;
    try {
      lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();

      ClusterContext clusterContext =
          CloudControllerContext.getInstance().getClusterContext(clusterId);
      handleNullObject(
          clusterContext,
          "Could not terminate containers, cluster not found: [cluster-id] " + clusterId);

      String kubernetesClusterId = clusterContext.getKubernetesClusterId();
      handleNullObject(
          kubernetesClusterId,
          "Could not terminate containers, kubernetes cluster id not found: "
              + "[cluster-id] "
              + clusterId);

      KubernetesClusterContext kubClusterContext =
          CloudControllerContext.getInstance().getKubernetesClusterContext(kubernetesClusterId);
      handleNullObject(
          kubClusterContext,
          "Could not terminate containers, kubernetes cluster not found: "
              + "[kubernetes-cluster-id] "
              + kubernetesClusterId);

      KubernetesApiClient kubApi = kubClusterContext.getKubApi();

      // Remove kubernetes services
      List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
      if (kubernetesServices != null) {
        for (KubernetesService kubernetesService : kubernetesServices) {
          try {
            kubApi.deleteService(kubernetesService.getId());
            int allocatedPort = kubernetesService.getPort();
            kubClusterContext.deallocatePort(allocatedPort);
          } catch (KubernetesClientException e) {
            log.error("Could not remove kubernetes service: [cluster-id] " + clusterId, e);
          }
        }
      }

      List<MemberContext> memberContextsRemoved = new ArrayList<MemberContext>();
      List<MemberContext> memberContexts =
          CloudControllerContext.getInstance().getMemberContextsOfClusterId(clusterId);
      if (memberContexts != null) {
        for (MemberContext memberContext : memberContexts) {
          try {
            MemberContext memberContextRemoved = terminateContainer(memberContext.getMemberId());
            memberContextsRemoved.add(memberContextRemoved);
          } catch (MemberTerminationFailedException e) {
            String message =
                "Could not terminate container: [member-id] " + memberContext.getMemberId();
            log.error(message);
          }
        }
      }

      // Persist changes
      CloudControllerContext.getInstance().persist();
      return memberContextsRemoved.toArray(new MemberContext[memberContextsRemoved.size()]);
    } finally {
      if (lock != null) {
        CloudControllerContext.getInstance().releaseWriteLock(lock);
      }
    }
  }
Beispiel #3
0
  /**
   * Creates and returns proxy services for the cluster.
   *
   * @param kubernetesApi
   * @param clusterContext
   * @param kubernetesCluster
   * @param kubernetesClusterContext
   * @throws KubernetesClientException
   */
  private void createKubernetesServices(
      KubernetesApiClient kubernetesApi,
      ClusterContext clusterContext,
      KubernetesCluster kubernetesCluster,
      KubernetesClusterContext kubernetesClusterContext)
      throws KubernetesClientException {

    String clusterId = clusterContext.getClusterId();
    String cartridgeType = clusterContext.getCartridgeUuid();

    Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
    if (cartridge == null) {
      String message =
          "Could not create kubernetes services, cartridge not found: [cartridge] " + cartridgeType;
      log.error(message);
      throw new RuntimeException(message);
    }

    String sessionAffinity = null;
    Property sessionAffinityProperty =
        cartridge.getProperties().getProperty(KUBERNETES_SERVICE_SESSION_AFFINITY);
    if (sessionAffinityProperty != null) {
      sessionAffinity = sessionAffinityProperty.getValue();
    }

    List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
    if (kubernetesServices == null) {
      kubernetesServices = new ArrayList<KubernetesService>();
    }

    // Prepare minion public IP addresses
    List<String> minionPrivateIPList = new ArrayList<String>();
    List<String> minionPublicIPList = new ArrayList<String>();
    KubernetesHost[] kubernetesHosts = kubernetesCluster.getKubernetesHosts();
    if ((kubernetesHosts == null)
        || (kubernetesHosts.length == 0)
        || (kubernetesHosts[0] == null)) {
      throw new RuntimeException(
          "Hosts not found in kubernetes cluster: [cluster] " + kubernetesCluster.getClusterUuid());
    }
    for (KubernetesHost host : kubernetesHosts) {
      if (host != null) {
        minionPrivateIPList.add(host.getPrivateIPAddress());
        minionPublicIPList.add(host.getPublicIPAddress());
      }
    }
    if (log.isDebugEnabled()) {
      log.debug(String.format("Minion private IPs: %s", minionPrivateIPList));
    }

    Collection<ClusterPortMapping> clusterPortMappings =
        CloudControllerContext.getInstance()
            .getClusterPortMappings(clusterContext.getApplicationUuid(), clusterId);

    if (clusterPortMappings != null) {
      for (ClusterPortMapping clusterPortMapping : clusterPortMappings) {

        // Skip if already created
        int containerPort = clusterPortMapping.getPort();
        if (kubernetesServiceExist(kubernetesServices, containerPort)) {
          continue;
        }

        // Find next service sequence no
        long serviceSeqNo = kubernetesClusterContext.getServiceSeqNo().incrementAndGet();
        String serviceId =
            KubernetesIaasUtil.fixSpecialCharacters("service" + "-" + (serviceSeqNo));
        String serviceLabel = DigestUtils.md5Hex(clusterId);

        if (log.isInfoEnabled()) {
          log.info(
              String.format(
                  "Creating kubernetes service: [cluster] %s [service] %s [service-label] %s "
                      + "[protocol] %s [service-port] %d [container-port] %s",
                  clusterId,
                  serviceId,
                  serviceLabel,
                  clusterPortMapping.getProtocol(),
                  clusterPortMapping.getKubernetesServicePort(),
                  containerPort));
        }

        // Create kubernetes service for port mapping
        int servicePort = clusterPortMapping.getKubernetesServicePort();
        String serviceType = clusterPortMapping.getKubernetesServiceType();
        String containerPortName =
            KubernetesIaasUtil.preparePortNameFromPortMapping(clusterPortMapping);

        try {
          // If kubernetes service is already created, skip creating a new one
          if (kubernetesApi.getService(serviceId) == null) {
            // Services need to use minions private IP addresses for creating iptable rules
            kubernetesApi.createService(
                serviceId,
                serviceLabel,
                servicePort,
                serviceType,
                containerPortName,
                containerPort,
                sessionAffinity);
          } else {
            if (log.isDebugEnabled()) {
              log.debug(
                  String.format(
                      "Kubernetes service is already created: [cluster] %s [service] %s "
                          + "[protocol] %s [service-port] %d [container-port] %d",
                      clusterId,
                      serviceId,
                      clusterPortMapping.getProtocol(),
                      servicePort,
                      containerPort));
            }
          }
        } finally {
          // Persist kubernetes service sequence no
          CloudControllerContext.getInstance().persist();
        }

        try {
          Thread.sleep(1000);
        } catch (InterruptedException ignore) {
        }

        Service service = kubernetesApi.getService(serviceId);

        KubernetesService kubernetesService = new KubernetesService();
        kubernetesService.setId(service.getMetadata().getName());
        kubernetesService.setPortalIP(service.getSpec().getClusterIP());
        // Expose minions public IP addresses as they need to be accessed by external networks
        String[] minionPublicIPArray =
            minionPublicIPList.toArray(new String[minionPublicIPList.size()]);
        kubernetesService.setPublicIPs(minionPublicIPArray);
        kubernetesService.setProtocol(clusterPortMapping.getProtocol());
        kubernetesService.setPortName(clusterPortMapping.getName());

        String kubernetesServiceType = service.getSpec().getType();
        kubernetesService.setServiceType(kubernetesServiceType);

        if (kubernetesServiceType.equals(KubernetesConstants.NODE_PORT)) {
          kubernetesService.setPort(service.getSpec().getPorts().get(0).getNodePort());
        } else {
          kubernetesService.setPort(service.getSpec().getPorts().get(0).getPort());
        }

        kubernetesService.setContainerPort(containerPort);
        kubernetesServices.add(kubernetesService);

        if (log.isInfoEnabled()) {
          log.info(
              String.format(
                  "Kubernetes service successfully created: [cluster] %s [service] %s "
                      + "[protocol] %s [node-port] %d [container-port] %s",
                  clusterId,
                  serviceId,
                  clusterPortMapping.getProtocol(),
                  servicePort,
                  containerPort));
        }
      }
    }

    // Add kubernetes services to cluster context and persist
    clusterContext.setKubernetesServices(kubernetesServices);
    CloudControllerContext.getInstance().persist();
  }
Beispiel #4
0
  public static void handleClusterActivatedEvent(
      ClusterStatusClusterActivatedEvent clusterStatusClusterActivatedEvent) {

    Topology topology = TopologyManager.getTopology();
    Service service = topology.getService(clusterStatusClusterActivatedEvent.getServiceName());
    // update the status of the cluster
    if (service == null) {
      log.warn(
          String.format(
              "Service %s does not exist", clusterStatusClusterActivatedEvent.getServiceName()));
      return;
    }

    Cluster cluster = service.getCluster(clusterStatusClusterActivatedEvent.getClusterId());
    if (cluster == null) {
      log.warn(
          String.format(
              "Cluster %s does not exist", clusterStatusClusterActivatedEvent.getClusterId()));
      return;
    }

    String clusterId = cluster.getClusterId();
    ClusterContext clusterContext =
        CloudControllerContext.getInstance().getClusterContext(clusterId);
    if (clusterContext == null) {
      log.warn("Cluster context not found: [cluster-id] " + clusterId);
      return;
    }

    ClusterInstanceActivatedEvent clusterInstanceActivatedEvent =
        new ClusterInstanceActivatedEvent(
            clusterStatusClusterActivatedEvent.getAppId(),
            clusterStatusClusterActivatedEvent.getServiceName(),
            clusterStatusClusterActivatedEvent.getClusterId(),
            clusterStatusClusterActivatedEvent.getInstanceId());
    try {
      TopologyManager.acquireWriteLock();
      List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
      cluster.setKubernetesServices(kubernetesServices);

      if (kubernetesServices != null) {
        // Set kubernetes services
        cluster.setKubernetesServices(kubernetesServices);
        try {
          // Generate access URLs for kubernetes services
          for (KubernetesService kubernetesService : kubernetesServices) {

            if (kubernetesService.getServiceType().equals(KubernetesConstants.NODE_PORT)) {
              // Public IP = Kubernetes minion public IP
              String[] publicIPs = kubernetesService.getPublicIPs();
              if ((publicIPs != null) && (publicIPs.length > 0)) {
                for (String publicIP : publicIPs) {
                  // There can be a String array with null values
                  if (publicIP != null) {
                    // Using type URI since only http, https, ftp, file, jar protocols are supported
                    // in URL
                    URI accessURL =
                        new URI(
                            kubernetesService.getProtocol(),
                            null,
                            publicIP,
                            kubernetesService.getPort(),
                            null,
                            null,
                            null);
                    cluster.addAccessUrl(accessURL.toString());
                    clusterInstanceActivatedEvent.addAccessUrl(accessURL.toString());
                  } else {
                    log.error(
                        String.format(
                            "Could not create access URL for [Kubernetes-service] %s , "
                                + "since Public IP is not available",
                            kubernetesService.getId()));
                  }
                }
              }
            }
          }
        } catch (URISyntaxException e) {
          log.error("Could not create access URLs for Kubernetes services", e);
        }
      }

      ClusterInstance context =
          cluster.getInstanceContexts(clusterStatusClusterActivatedEvent.getInstanceId());

      if (context == null) {
        log.warn(
            "Cluster instance context is not found for [cluster] "
                + clusterStatusClusterActivatedEvent.getClusterId()
                + " [instance-id] "
                + clusterStatusClusterActivatedEvent.getInstanceId());
        return;
      }
      ClusterStatus status = ClusterStatus.Active;
      if (context.isStateTransitionValid(status)) {
        context.setStatus(status);
        log.info("Cluster activated adding status started for " + cluster.getClusterId());
        TopologyManager.updateTopology(topology);
        // publish event
        TopologyEventPublisher.sendClusterActivatedEvent(clusterInstanceActivatedEvent);
      } else {
        log.error(
            String.format(
                "Cluster state transition is not valid: [cluster-id] %s "
                    + " [instance-id] %s [current-status] %s [status-requested] %s",
                clusterStatusClusterActivatedEvent.getClusterId(),
                clusterStatusClusterActivatedEvent.getInstanceId(),
                context.getStatus(),
                status));
        return;
      }
    } finally {
      TopologyManager.releaseWriteLock();
    }
  }
Beispiel #5
0
  public static void handleMemberActivated(InstanceActivatedEvent instanceActivatedEvent) {
    Topology topology = TopologyManager.getTopology();
    Service service = topology.getService(instanceActivatedEvent.getServiceName());
    if (service == null) {
      log.warn(String.format("Service %s does not exist", instanceActivatedEvent.getServiceName()));
      return;
    }

    Cluster cluster = service.getCluster(instanceActivatedEvent.getClusterId());
    if (cluster == null) {
      log.warn(String.format("Cluster %s does not exist", instanceActivatedEvent.getClusterId()));
      return;
    }

    Member member = cluster.getMember(instanceActivatedEvent.getMemberId());
    if (member == null) {
      log.warn(String.format("Member %s does not exist", instanceActivatedEvent.getMemberId()));
      return;
    }

    MemberActivatedEvent memberActivatedEvent =
        new MemberActivatedEvent(
            instanceActivatedEvent.getServiceName(),
            instanceActivatedEvent.getClusterId(),
            instanceActivatedEvent.getClusterInstanceId(),
            instanceActivatedEvent.getMemberId(),
            instanceActivatedEvent.getNetworkPartitionId(),
            instanceActivatedEvent.getPartitionId());

    // grouping - set grouid
    // TODO
    memberActivatedEvent.setApplicationId(null);
    try {
      TopologyManager.acquireWriteLock();
      // try update lifecycle state
      if (!member.isStateTransitionValid(MemberStatus.Active)) {
        log.error(
            "Invalid state transition from ["
                + member.getStatus()
                + "] to ["
                + MemberStatus.Active
                + "]");
        return;
      } else {
        member.setStatus(MemberStatus.Active);

        // Set member ports
        try {
          Cartridge cartridge =
              CloudControllerContext.getInstance().getCartridge(service.getServiceName());
          if (cartridge == null) {
            throw new RuntimeException(
                String.format(
                    "Cartridge not found: [cartridge-type] %s", service.getServiceName()));
          }

          Port port;
          int portValue;
          List<PortMapping> portMappings = Arrays.asList(cartridge.getPortMappings());
          String clusterId = cluster.getClusterId();
          ClusterContext clusterContext =
              CloudControllerContext.getInstance().getClusterContext(clusterId);
          List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();

          for (PortMapping portMapping : portMappings) {
            if (kubernetesServices != null) {
              portValue = findKubernetesServicePort(clusterId, kubernetesServices, portMapping);
            } else {
              portValue = portMapping.getPort();
            }
            port = new Port(portMapping.getProtocol(), portValue, portMapping.getProxyPort());
            member.addPort(port);
            memberActivatedEvent.addPort(port);
          }
        } catch (Exception e) {
          String message =
              String.format(
                  "Could not add member ports: [service-name] %s [member-id] %s",
                  memberActivatedEvent.getServiceName(), memberActivatedEvent.getMemberId());
          log.error(message, e);
        }

        // Set member ip addresses
        memberActivatedEvent.setDefaultPrivateIP(member.getDefaultPrivateIP());
        memberActivatedEvent.setMemberPrivateIPs(member.getMemberPrivateIPs());
        memberActivatedEvent.setDefaultPublicIP(member.getDefaultPublicIP());
        memberActivatedEvent.setMemberPublicIPs(member.getMemberPublicIPs());
        TopologyManager.updateTopology(topology);

        // Publish member activated event
        TopologyEventPublisher.sendMemberActivatedEvent(memberActivatedEvent);

        // Publish statistics data
        BAMUsageDataPublisher.publish(
            memberActivatedEvent.getMemberId(),
            memberActivatedEvent.getPartitionId(),
            memberActivatedEvent.getNetworkPartitionId(),
            memberActivatedEvent.getClusterId(),
            memberActivatedEvent.getServiceName(),
            MemberStatus.Active.toString(),
            null);
      }
    } finally {
      TopologyManager.releaseWriteLock();
    }
  }