public static void handleApplicationClustersCreated(String appId, List<Cluster> appClusters) {
TopologyManager.acquireWriteLock();
try {
Topology topology = TopologyManager.getTopology();
for (Cluster cluster : appClusters) {
Service service = topology.getService(cluster.getServiceName());
if (service == null) {
log.error(
"Service "
+ cluster.getServiceName()
+ " not found in Topology, unable to create Application cluster");
} else {
service.addCluster(cluster);
log.info("Application Cluster " + cluster.getClusterId() + " created in CC topology");
}
}
TopologyManager.updateTopology(topology);
} finally {
TopologyManager.releaseWriteLock();
}
log.debug("Creating cluster port mappings: [appication-id] " + appId);
for (Cluster cluster : appClusters) {
String cartridgeType = cluster.getServiceName();
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
throw new CloudControllerException(
"Cartridge not found: [cartridge-type] " + cartridgeType);
}
for (PortMapping portMapping : cartridge.getPortMappings()) {
ClusterPortMapping clusterPortMapping =
new ClusterPortMapping(
appId,
cluster.getClusterId(),
portMapping.getName(),
portMapping.getProtocol(),
portMapping.getPort(),
portMapping.getProxyPort());
if (portMapping.getKubernetesPortType() != null) {
clusterPortMapping.setKubernetesServiceType(portMapping.getKubernetesPortType());
}
CloudControllerContext.getInstance().addClusterPortMapping(clusterPortMapping);
log.debug("Cluster port mapping created: " + clusterPortMapping.toString());
}
}
// Persist cluster port mappings
CloudControllerContext.getInstance().persist();
// Send application clusters created event
TopologyEventPublisher.sendApplicationClustersCreated(appId, appClusters);
}
/**
* Generate kubernetes service ports for cluster.
*
* @param kubernetesClusterContext
* @param clusterId
* @param cartridge
*/
private void generateKubernetesServicePorts(
String applicationId,
String clusterId,
KubernetesClusterContext kubernetesClusterContext,
Cartridge cartridge) {
synchronized (KubernetesIaas.class) {
if (cartridge != null) {
StringBuilder portMappingStrBuilder = new StringBuilder();
for (PortMapping portMapping : Arrays.asList(cartridge.getPortMappings())) {
Collection<ClusterPortMapping> clusterPortMappings =
CloudControllerContext.getInstance().getClusterPortMappings(applicationId, clusterId);
if (clusterPortMappings == null) {
throw new CloudControllerException(
String.format(
"Cluster port mappings not found: " + "[application-id] %s [cluster-id] %s",
applicationId, clusterId));
}
ClusterPortMapping clusterPortMapping =
findClusterPortMapping(clusterPortMappings, portMapping);
if (clusterPortMappings == null) {
throw new CloudControllerException(
String.format(
"Cluster port mapping not found: "
+ "[application-id] %s [cluster-id] %s [transport] %s",
applicationId, clusterId, portMapping.getName()));
}
if (clusterPortMapping.getKubernetesServiceType() == null) {
throw new CloudControllerException(
String.format(
"Kubernetes service type not "
+ "found [application-id] %s [cluster-id] %s [cartridge] %s",
applicationId, clusterId, cartridge));
}
String serviceType = portMapping.getKubernetesPortType();
clusterPortMapping.setKubernetesServiceType(serviceType);
// If kubernetes service port is already set, skip setting a new one
if (clusterPortMapping.getKubernetesServicePort() == 0) {
if (serviceType.equals(KubernetesConstants.NODE_PORT)) {
int nextServicePort = kubernetesClusterContext.getNextServicePort();
if (nextServicePort == -1) {
throw new RuntimeException(
String.format(
"Could not generate service port: [cluster-id] %s " + "[port] %d",
clusterId, portMapping.getPort()));
}
clusterPortMapping.setKubernetesServicePort(nextServicePort);
} else {
clusterPortMapping.setKubernetesServicePort(portMapping.getPort());
}
} else {
if (log.isDebugEnabled()) {
log.debug(
String.format(
"Kubernetes service port is already set: [application-id] %s "
+ "[cluster-id] %s [port] %d [service-port] %d",
applicationId,
clusterId,
clusterPortMapping.getPort(),
clusterPortMapping.getKubernetesServicePort()));
}
}
// Add port mappings to payload
if (portMappingStrBuilder.toString().length() > 0) {
portMappingStrBuilder.append(";");
}
portMappingStrBuilder.append(
String.format(
"NAME:%s|PROTOCOL:%s|PORT:%d|PROXY_PORT:%d",
clusterPortMapping.getName(),
clusterPortMapping.getProtocol(),
clusterPortMapping.getKubernetesServicePort(),
clusterPortMapping.getProxyPort()));
if (log.isInfoEnabled()) {
log.info(
String.format(
"Kubernetes service port generated: [application-id] %s "
+ "[cluster-id] %s [port] %d [service-port] %d",
applicationId,
clusterId,
clusterPortMapping.getPort(),
clusterPortMapping.getKubernetesServicePort()));
}
}
NameValuePair nameValuePair =
new NameValuePair(PORT_MAPPINGS, portMappingStrBuilder.toString());
payload.add(nameValuePair);
// Persist service ports added to cluster port mappings
CloudControllerContext.getInstance().persist();
}
}
}
/**
* Create new pod and pass environment variables.
*
* @param memberContext
* @param kubernetesApi
* @param kubernetesClusterContext
* @throws KubernetesClientException
*/
private void createPod(
ClusterContext clusterContext,
MemberContext memberContext,
KubernetesApiClient kubernetesApi,
KubernetesClusterContext kubernetesClusterContext)
throws KubernetesClientException {
String applicationId = memberContext.getApplicationId();
String cartridgeType = memberContext.getCartridgeType();
String clusterId = memberContext.getClusterId();
String memberId = memberContext.getMemberId();
if (log.isInfoEnabled()) {
log.info(
String.format(
"Creating kubernetes pod: [application] %s [cartridge] %s [member] %s",
applicationId, cartridgeType, memberId));
}
Partition partition = memberContext.getPartition();
if (partition == null) {
String message =
String.format(
"Partition not found in member context: [application] %s [cartridge] %s "
+ "[member] %s ",
applicationId, cartridgeType, memberId);
log.error(message);
throw new RuntimeException(message);
}
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String message = "Could not find cartridge: [cartridge] " + cartridgeType;
log.error(message);
throw new RuntimeException(message);
}
// Set default values to zero to avoid cpu and memory restrictions
int cpu = Integer.getInteger(KUBERNETES_CONTAINER_CPU_DEFAULT, 0);
int memory = Integer.getInteger(KUBERNETES_CONTAINER_MEMORY_DEFAULT, 0);
Property cpuProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_CPU);
if (cpuProperty != null) {
cpu = Integer.parseInt(cpuProperty.getValue());
}
Property memoryProperty = cartridge.getProperties().getProperty(KUBERNETES_CONTAINER_MEMORY);
if (memoryProperty != null) {
memory = Integer.parseInt(memoryProperty.getValue());
}
IaasProvider iaasProvider =
CloudControllerContext.getInstance()
.getIaasProviderOfPartition(cartridge.getUuid(), partition.getUuid());
if (iaasProvider == null) {
String message = "Could not find iaas provider: [partition] " + partition.getUuid();
log.error(message);
throw new RuntimeException(message);
}
// Add dynamic payload to the member context
memberContext.setDynamicPayload(payload.toArray(new NameValuePair[payload.size()]));
// Create pod
long podSeqNo = kubernetesClusterContext.getPodSeqNo().incrementAndGet();
String podId = "pod" + "-" + podSeqNo;
String podLabel = DigestUtils.md5Hex(clusterId);
String dockerImage = iaasProvider.getImage();
List<EnvVar> environmentVariables =
KubernetesIaasUtil.prepareEnvironmentVariables(clusterContext, memberContext);
List<ContainerPort> ports =
KubernetesIaasUtil.convertPortMappings(Arrays.asList(cartridge.getPortMappings()));
log.info(
String.format(
"Starting pod: [application] %s [cartridge] %s [member] %s "
+ "[cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
cpu,
memory));
kubernetesApi.createPod(podId, podLabel, dockerImage, cpu, memory, ports, environmentVariables);
log.info(
String.format(
"Pod started successfully: [application] %s [cartridge] %s [member] %s "
+ "[pod] %s [cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
podId,
cpu,
memory));
// Add pod id to member context
memberContext.setKubernetesPodId(podId);
memberContext.setKubernetesPodLabel(podLabel);
// Create instance metadata
InstanceMetadata instanceMetadata = new InstanceMetadata();
instanceMetadata.setImageId(dockerImage);
instanceMetadata.setCpu(cpu);
instanceMetadata.setRam(memory);
memberContext.setInstanceMetadata(instanceMetadata);
// Persist cloud controller context
CloudControllerContext.getInstance().persist();
}
public static void handleMemberActivated(InstanceActivatedEvent instanceActivatedEvent) {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(instanceActivatedEvent.getServiceName());
if (service == null) {
log.warn(String.format("Service %s does not exist", instanceActivatedEvent.getServiceName()));
return;
}
Cluster cluster = service.getCluster(instanceActivatedEvent.getClusterId());
if (cluster == null) {
log.warn(String.format("Cluster %s does not exist", instanceActivatedEvent.getClusterId()));
return;
}
Member member = cluster.getMember(instanceActivatedEvent.getMemberId());
if (member == null) {
log.warn(String.format("Member %s does not exist", instanceActivatedEvent.getMemberId()));
return;
}
MemberActivatedEvent memberActivatedEvent =
new MemberActivatedEvent(
instanceActivatedEvent.getServiceName(),
instanceActivatedEvent.getClusterId(),
instanceActivatedEvent.getClusterInstanceId(),
instanceActivatedEvent.getMemberId(),
instanceActivatedEvent.getNetworkPartitionId(),
instanceActivatedEvent.getPartitionId());
// grouping - set grouid
// TODO
memberActivatedEvent.setApplicationId(null);
try {
TopologyManager.acquireWriteLock();
// try update lifecycle state
if (!member.isStateTransitionValid(MemberStatus.Active)) {
log.error(
"Invalid state transition from ["
+ member.getStatus()
+ "] to ["
+ MemberStatus.Active
+ "]");
return;
} else {
member.setStatus(MemberStatus.Active);
// Set member ports
try {
Cartridge cartridge =
CloudControllerContext.getInstance().getCartridge(service.getServiceName());
if (cartridge == null) {
throw new RuntimeException(
String.format(
"Cartridge not found: [cartridge-type] %s", service.getServiceName()));
}
Port port;
int portValue;
List<PortMapping> portMappings = Arrays.asList(cartridge.getPortMappings());
String clusterId = cluster.getClusterId();
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
List<KubernetesService> kubernetesServices = clusterContext.getKubernetesServices();
for (PortMapping portMapping : portMappings) {
if (kubernetesServices != null) {
portValue = findKubernetesServicePort(clusterId, kubernetesServices, portMapping);
} else {
portValue = portMapping.getPort();
}
port = new Port(portMapping.getProtocol(), portValue, portMapping.getProxyPort());
member.addPort(port);
memberActivatedEvent.addPort(port);
}
} catch (Exception e) {
String message =
String.format(
"Could not add member ports: [service-name] %s [member-id] %s",
memberActivatedEvent.getServiceName(), memberActivatedEvent.getMemberId());
log.error(message, e);
}
// Set member ip addresses
memberActivatedEvent.setDefaultPrivateIP(member.getDefaultPrivateIP());
memberActivatedEvent.setMemberPrivateIPs(member.getMemberPrivateIPs());
memberActivatedEvent.setDefaultPublicIP(member.getDefaultPublicIP());
memberActivatedEvent.setMemberPublicIPs(member.getMemberPublicIPs());
TopologyManager.updateTopology(topology);
// Publish member activated event
TopologyEventPublisher.sendMemberActivatedEvent(memberActivatedEvent);
// Publish statistics data
BAMUsageDataPublisher.publish(
memberActivatedEvent.getMemberId(),
memberActivatedEvent.getPartitionId(),
memberActivatedEvent.getNetworkPartitionId(),
memberActivatedEvent.getClusterId(),
memberActivatedEvent.getServiceName(),
MemberStatus.Active.toString(),
null);
}
} finally {
TopologyManager.releaseWriteLock();
}
}
public static void handleServiceCreated(List<Cartridge> cartridgeList) {
Service service;
Topology topology = TopologyManager.getTopology();
if (cartridgeList == null) {
log.warn(String.format("Cartridge list is empty"));
return;
}
try {
TopologyManager.acquireWriteLock();
for (Cartridge cartridge : cartridgeList) {
if (!topology.serviceExists(cartridge.getType())) {
ServiceType serviceType =
cartridge.isMultiTenant() ? ServiceType.MultiTenant : ServiceType.SingleTenant;
service = new Service(cartridge.getType(), serviceType);
Properties properties = new Properties();
try {
Property[] propertyArray = null;
if (cartridge.getProperties() != null) {
if (cartridge.getProperties().getProperties() != null) {
propertyArray = cartridge.getProperties().getProperties();
}
}
List<Property> propertyList = new ArrayList<Property>();
if (propertyArray != null) {
propertyList = Arrays.asList(propertyArray);
if (propertyList != null) {
for (Property property : propertyList) {
properties.setProperty(property.getName(), property.getValue());
}
}
}
} catch (Exception e) {
log.error(e);
}
service.setProperties(properties);
if (cartridge.getPortMappings() != null) {
List<PortMapping> portMappings = Arrays.asList(cartridge.getPortMappings());
Port port;
// adding ports to the event
for (PortMapping portMapping : portMappings) {
port =
new Port(
portMapping.getProtocol(), portMapping.getPort(), portMapping.getProxyPort());
service.addPort(port);
}
}
topology.addService(service);
TopologyManager.updateTopology(topology);
}
}
} finally {
TopologyManager.releaseWriteLock();
}
TopologyEventPublisher.sendServiceCreateEvent(cartridgeList);
}