private String readProperty(
String property, org.apache.stratos.common.Properties properties, String object) {
String propVal = CloudControllerUtil.getProperty(properties, property);
handleNullObject(
propVal,
"Property validation failed. Could not find property: '" + property + " in " + object);
return propVal;
}
private void loadPayload(final List<OMNode> nodes, final ServiceContext serviceCtxt) {
if (nodes == null || nodes.isEmpty()) {
return;
}
// read payload element
if (nodes.get(0).getType() == OMNode.ELEMENT_NODE) {
OMElement node = (OMElement) nodes.get(0);
if (node.getText() != null) {
byte[] payload = CloudControllerUtil.getBytesFromFile(node.getText());
serviceCtxt.setPayload(payload);
}
}
}
/**
* Add member object to the topology and publish member created event
*
* @param memberContext
*/
public static void handleMemberCreatedEvent(MemberContext memberContext) {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(memberContext.getCartridgeType());
String clusterId = memberContext.getClusterId();
Cluster cluster = service.getCluster(clusterId);
String memberId = memberContext.getMemberId();
String clusterInstanceId = memberContext.getClusterInstanceId();
String networkPartitionId = memberContext.getNetworkPartitionId();
String partitionId = memberContext.getPartition().getId();
String lbClusterId = memberContext.getLbClusterId();
long initTime = memberContext.getInitTime();
if (cluster.memberExists(memberId)) {
log.warn(String.format("Member %s already exists", memberId));
return;
}
try {
TopologyManager.acquireWriteLock();
Member member =
new Member(
service.getServiceName(),
clusterId,
memberId,
clusterInstanceId,
networkPartitionId,
partitionId,
memberContext.getLoadBalancingIPType(),
initTime);
member.setStatus(MemberStatus.Created);
member.setLbClusterId(lbClusterId);
member.setProperties(CloudControllerUtil.toJavaUtilProperties(memberContext.getProperties()));
cluster.addMember(member);
TopologyManager.updateTopology(topology);
} finally {
TopologyManager.releaseWriteLock();
}
TopologyEventPublisher.sendMemberCreatedEvent(memberContext);
}
@Override
public List<String> associateAddresses(NodeMetadata node) {
ComputeServiceContext context = iaasProvider.getComputeService().getContext();
String region = ComputeServiceBuilderUtil.extractRegion(iaasProvider);
if (StringUtils.isEmpty(region)) {
throw new RuntimeException(
"Could not find region in iaas provider: " + iaasProvider.getName());
}
NovaApi novaApi = context.unwrapApi(NovaApi.class);
FloatingIPApi floatingIPApi = novaApi.getFloatingIPExtensionForZone(region).get();
String ip = null;
// first try to find an unassigned IP.
FluentIterable<FloatingIP> floatingIPs = floatingIPApi.list();
ArrayList<FloatingIP> unassignedIps =
Lists.newArrayList(
Iterables.filter(
floatingIPs,
new Predicate<FloatingIP>() {
@Override
public boolean apply(FloatingIP floatingIP) {
return floatingIP.getInstanceId() == null;
}
}));
if (!unassignedIps.isEmpty()) {
// try to prevent multiple parallel launches from choosing the same ip.
Collections.shuffle(unassignedIps);
ip = Iterables.getLast(unassignedIps).getIp();
}
// if no unassigned IP is available, we'll try to allocate an IP.
if (StringUtils.isEmpty(ip)) {
String floatingIpPool =
iaasProvider.getProperty(CloudControllerConstants.DEFAULT_FLOATING_IP_POOL);
FloatingIP allocatedFloatingIP;
if (StringUtils.isEmpty(floatingIpPool)) {
allocatedFloatingIP = floatingIPApi.create();
} else {
log.debug(
String.format(
"Trying to allocate a floating IP address from IP pool %s", floatingIpPool));
allocatedFloatingIP = floatingIPApi.allocateFromPool(floatingIpPool);
}
if (allocatedFloatingIP == null) {
String msg =
String.format(
"Floating IP API did not return a floating IP address from IP pool %s",
floatingIpPool);
log.error(msg);
throw new CloudControllerException(msg);
}
ip = allocatedFloatingIP.getIp();
}
// wait till the fixed IP address gets assigned - this is needed before
// we associate a public IP
log.info(
String.format(
"Waiting for private IP addresses get allocated: [node-id] %s", node.getId()));
while (node.getPrivateAddresses() == null) {
CloudControllerUtil.sleep(1000);
}
log.info(String.format("Private IP addresses allocated: %s", node.getPrivateAddresses()));
if ((node.getPublicAddresses() != null) && (node.getPublicAddresses().iterator().hasNext())) {
log.info(
"Public IP address "
+ node.getPublicAddresses().iterator().next()
+ " is already allocated to the instance: [node-id] "
+ node.getId());
return null;
}
int retries = 0;
int retryCount = Integer.getInteger("stratos.public.ip.association.retry.count", 5);
while ((retries < retryCount) && (!associateIp(floatingIPApi, ip, node.getProviderId()))) {
// wait for 5s
CloudControllerUtil.sleep(5000);
retries++;
}
log.info(
String.format(
"Successfully associated an IP address: [node-id] %s [ip] %s", node.getId(), ip));
List<String> allocatedIPAddresses = new ArrayList<String>();
allocatedIPAddresses.add(ip);
return allocatedIPAddresses;
}
@Override
public String associatePredefinedAddress(NodeMetadata node, String ip) {
if (log.isDebugEnabled()) {
log.debug(
String.format(
"Trying to associate predefined IP address: [node-id] %s [ip] %s", node.getId(), ip));
}
ComputeServiceContext context = iaasProvider.getComputeService().getContext();
String region = ComputeServiceBuilderUtil.extractRegion(iaasProvider);
FloatingIPApi floatingIPApi =
context.unwrapApi(NovaApi.class).getFloatingIPExtensionForZone(region).get();
// get the list of all unassigned IP.
ArrayList<FloatingIP> unassignedFloatingIPs =
Lists.newArrayList(
Iterables.filter(
floatingIPApi.list(),
new Predicate<FloatingIP>() {
@Override
public boolean apply(FloatingIP floatingIP) {
return StringUtils.isEmpty(floatingIP.getFixedIp());
}
}));
boolean isAvailable = false;
for (FloatingIP floatingIP : unassignedFloatingIPs) {
if (log.isDebugEnabled()) {
log.debug(
"OpenstackNovaIaas:associatePredefinedAddress:iterating over available floatingip:"
+ floatingIP);
}
if (ip.equals(floatingIP.getIp())) {
if (log.isDebugEnabled()) {
log.debug(
String.format(
"OpenstackNovaIaas:associatePredefinedAddress:floating ip in use:%s /ip:%s",
floatingIP, ip));
}
isAvailable = true;
break;
}
}
if (isAvailable) {
// assign ip
if (log.isDebugEnabled()) {
log.debug("OpenstackNovaIaas:associatePredefinedAddress:assign floating ip:" + ip);
}
// exercise same code as in associateAddress()
// wait till the fixed IP address gets assigned - this is needed before
// we associate a public IP
while (node.getPrivateAddresses() == null) {
CloudControllerUtil.sleep(1000);
}
int retries = 0;
int retryCount = Integer.getInteger("stratos.public.ip.association.retry.count", 5);
while (retries < retryCount && !associateIp(floatingIPApi, ip, node.getProviderId())) {
// wait for 5s
CloudControllerUtil.sleep(5000);
retries++;
}
NodeMetadataBuilder.fromNodeMetadata(node).publicAddresses(ImmutableSet.of(ip)).build();
log.info(
String.format(
"Successfully associated predefined IP address: [node-id] %s [ip] %s ",
node.getId(), ip));
return ip;
} else {
log.warn(
String.format(
"Could not associate predefined IP address: [node-id] %s [ip] %s ",
node.getId(), ip));
return null;
}
}
@Override
public List<String> associateAddresses(NodeMetadata node) {
assertNotNull(node, "Node cannot be null");
if (null == neutronApi || null == portApi || null == floatingIPApi) {
buildNeutronApi();
}
// internal network uuid to floating networks map, as defined in cartridge definition
Map<String, List<FloatingNetwork>> networkUuidToFloatingNetworksMap =
getNetworkUuidToFloatingNetworksMap(iaasProvider.getNetworkInterfaces());
// private IP to floating networks map, as defined in cartridge definition
Map<String, List<FloatingNetwork>> fixedIPToFloatingNetworksMap =
getFixedIPToFloatingNetworksMap(iaasProvider.getNetworkInterfaces());
// list of IPs allocated to this node
List<String> associatedFloatingIPs = new ArrayList<String>();
// wait until node gets private IPs
while (node.getPrivateAddresses() == null) {
CloudControllerUtil.sleep(1000);
}
// loop through all the fixed IPs of this node
// and see whether we need to assign floating IP to each according to the cartridge deployment
for (String privateIPOfTheNode : node.getPrivateAddresses()) {
Port portOfTheFixedIP = getPortByFixedIP(privateIPOfTheNode);
if (null == portOfTheFixedIP) {
// we can't assign floating IP if port is null
// it can't happen, a fixed/private IP can't live without a port
// but doing a null check to be on the safe side
if (log.isDebugEnabled()) {
String msg = String.format("Port not found for fixed IP %s", privateIPOfTheNode);
log.debug(msg);
}
continue;
}
// get list of floating networks associated with each network interfaces (refer cartridge
// definition)
List<FloatingNetwork> floatingNetworks =
networkUuidToFloatingNetworksMap.get(portOfTheFixedIP.getNetworkId());
// if no floating networks is defined for a network interface, no need to assign any floating
// IPs, skip the current iteration
if (null == floatingNetworks || floatingNetworks.isEmpty()) {
// since no floating networks found in networkUuidToFloatingNetworksMap,
// we will search in fixedIPToFloatingNetworksMap
floatingNetworks = fixedIPToFloatingNetworksMap.get(privateIPOfTheNode);
if (null == floatingNetworks || floatingNetworks.isEmpty()) {
if (log.isDebugEnabled()) {
String msg =
String.format(
"No floating networks defined for the network interface %s",
portOfTheFixedIP.getNetworkId());
log.debug(msg);
}
}
continue;
}
// if floating networks defined for a network interface, assign one floating IP from each
// floating network
for (FloatingNetwork floatingNetwork : floatingNetworks) {
FloatingIP allocatedFloatingIP = null;
if (floatingNetwork.getNetworkUuid() != null
&& !floatingNetwork.getNetworkUuid().isEmpty()) {
allocatedFloatingIP =
assignFloatingIP(portOfTheFixedIP, floatingNetwork.getNetworkUuid());
} else if (floatingNetwork.getFloatingIP() != null
&& !floatingNetwork.getFloatingIP().isEmpty()) {
allocatedFloatingIP =
assignPredefinedFloatingIP(portOfTheFixedIP, floatingNetwork.getFloatingIP());
} else {
String msg =
String.format(
"Neither floating network uuid or floating IP defined for the floating network %s",
floatingNetwork.getName());
log.error(msg);
throw new CloudControllerException(msg);
}
String allocatedFloatingIPNullMsg =
String.format(
"Error occured while assigning floating IP. "
+ "Please check whether the floating network %s can be reached from the fixed IP range",
floatingNetwork.getNetworkUuid());
assertNotNull(allocatedFloatingIP, allocatedFloatingIPNullMsg);
String allocatedFloatingIPAddressNullOrEmptyMsg =
String.format(
"Error occured while assigning floating IP. "
+ "Please check whether the floating network %s can be reached from the fixed IP range",
floatingNetwork.getNetworkUuid());
assertNotNullAndNotEmpty(
allocatedFloatingIP.getFloatingIpAddress(), allocatedFloatingIPAddressNullOrEmptyMsg);
associatedFloatingIPs.add(allocatedFloatingIP.getFloatingIpAddress());
}
}
return associatedFloatingIPs;
}
/**
* Starts a container via kubernetes for the given member context.
*
* @param memberContext
* @return
* @throws CartridgeNotFoundException
*/
public MemberContext startContainer(MemberContext memberContext)
throws CartridgeNotFoundException {
Lock lock = null;
try {
lock = CloudControllerContext.getInstance().acquireMemberContextWriteLock();
handleNullObject(memberContext, "member context is null");
log.info(
String.format(
"Starting container: [application] %s [cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId()));
// Validate cluster id
String clusterId = memberContext.getClusterId();
String memberId = memberContext.getMemberId();
handleNullObject(clusterId, "cluster id is null in member context");
// Validate cluster context
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(clusterId);
handleNullObject(
clusterContext,
String.format(
"Cluster context not found: [application] %s [cartridge] %s " + "[cluster] %s",
memberContext.getApplicationId(), memberContext.getCartridgeType(), clusterId));
// Validate partition
Partition partition = memberContext.getPartition();
handleNullObject(
partition,
String.format(
"partition not found in member context: [application] %s "
+ "[cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId()));
// Validate cartridge
String cartridgeType = clusterContext.getCartridgeUuid();
Cartridge cartridge = CloudControllerContext.getInstance().getCartridge(cartridgeType);
if (cartridge == null) {
String msg =
String.format(
"Cartridge not found: [application] %s [cartridge] %s",
memberContext.getApplicationId(), memberContext.getCartridgeType());
log.error(msg);
throw new CartridgeNotFoundException(msg);
}
String kubernetesClusterId = partition.getKubernetesClusterId();
clusterContext.setKubernetesClusterId(kubernetesClusterId);
KubernetesCluster kubernetesCluster =
CloudControllerContext.getInstance().getKubernetesCluster(kubernetesClusterId);
handleNullObject(
kubernetesCluster,
"kubernetes cluster not found: "
+ "[kubernetes-cluster] "
+ kubernetesClusterId
+ " [cluster] "
+ clusterId
+ " [member] "
+ memberId);
// Prepare kubernetes context
String kubernetesMasterIp = kubernetesCluster.getKubernetesMaster().getPrivateIPAddress();
PortRange kubernetesPortRange = kubernetesCluster.getPortRange();
String kubernetesMasterPort =
CloudControllerUtil.getProperty(
kubernetesCluster.getKubernetesMaster().getProperties(),
StratosConstants.KUBERNETES_MASTER_PORT,
StratosConstants.KUBERNETES_MASTER_DEFAULT_PORT);
// Add kubernetes cluster payload parameters to payload
if ((kubernetesCluster.getProperties() != null)
&& (kubernetesCluster.getProperties().getProperties() != null)) {
for (Property property : kubernetesCluster.getProperties().getProperties()) {
if (property != null) {
if (property.getName().startsWith(PAYLOAD_PARAMETER_PREFIX)) {
String name = property.getName().replace(PAYLOAD_PARAMETER_PREFIX, "");
payload.add(new NameValuePair(name, property.getValue()));
}
}
}
}
KubernetesClusterContext kubernetesClusterContext =
getKubernetesClusterContext(
kubernetesClusterId,
kubernetesMasterIp,
kubernetesMasterPort,
kubernetesPortRange.getUpper(),
kubernetesPortRange.getLower());
// Generate kubernetes service ports and update port mappings in cartridge
generateKubernetesServicePorts(
clusterContext.getApplicationUuid(),
clusterContext.getClusterId(),
kubernetesClusterContext,
cartridge);
// Create kubernetes services for port mappings
KubernetesApiClient kubernetesApi = kubernetesClusterContext.getKubApi();
createKubernetesServices(
kubernetesApi, clusterContext, kubernetesCluster, kubernetesClusterContext);
// Create pod
createPod(clusterContext, memberContext, kubernetesApi, kubernetesClusterContext);
// Wait for pod status to be changed to running
Pod pod = waitForPodToBeActivated(memberContext, kubernetesApi);
// Update member context
updateMemberContext(memberContext, pod, kubernetesCluster);
log.info(
String.format(
"Container started successfully: [application] %s [cartridge] %s [member] %s "
+ "[pod] %s [cpu] %d [memory] %d MB",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId(),
memberContext.getKubernetesPodId(),
memberContext.getInstanceMetadata().getCpu(),
memberContext.getInstanceMetadata().getRam()));
return memberContext;
} catch (Exception e) {
String msg =
String.format(
"Could not start container: [application] %s [cartridge] %s [member] %s",
memberContext.getApplicationId(),
memberContext.getCartridgeType(),
memberContext.getMemberId());
log.error(msg, e);
throw new RuntimeException(msg, e);
} finally {
if (lock != null) {
CloudControllerContext.getInstance().releaseWriteLock(lock);
}
}
}
@Override
public synchronized List<String> associateAddresses(NodeMetadata node) {
IaasProvider iaasInfo = getIaasProvider();
ComputeServiceContext context = iaasInfo.getComputeService().getContext();
ElasticIPAddressApi elasticIPAddressApi =
context.unwrapApi(AWSEC2Api.class).getElasticIPAddressApi().get();
String region = ComputeServiceBuilderUtil.extractRegion(iaasInfo);
String ip = null;
// first try to find an unassigned IP.
ArrayList<PublicIpInstanceIdPair> unassignedIps =
Lists.newArrayList(
Iterables.filter(
elasticIPAddressApi.describeAddressesInRegion(region),
new Predicate<PublicIpInstanceIdPair>() {
@Override
public boolean apply(PublicIpInstanceIdPair arg0) {
return arg0.getInstanceId() == null;
}
}));
if (!unassignedIps.isEmpty()) {
// try to prevent multiple parallel launches from choosing the same
// ip.
Collections.shuffle(unassignedIps);
ip = Iterables.getLast(unassignedIps).getPublicIp();
}
// if no unassigned IP is available, we'll try to allocate an IP.
if (ip == null || ip.isEmpty()) {
try {
ip = elasticIPAddressApi.allocateAddressInRegion(region);
log.info("Allocated ip [" + ip + "]");
} catch (Exception e) {
String msg = "Failed to allocate an IP address. All IP addresses are in use.";
log.error(msg, e);
throw new CloudControllerException(msg, e);
}
}
String id = node.getProviderId();
// wait till the fixed IP address gets assigned - this is needed before
// we associate a
// public IP
while (node.getPrivateAddresses() == null) {
CloudControllerUtil.sleep(1000);
}
int retries = 0;
while (retries < 12 && !associatePublicIp(elasticIPAddressApi, region, ip, id)) {
// wait for 5s
CloudControllerUtil.sleep(5000);
retries++;
}
log.debug("Successfully associated an IP address " + ip + " for node with id: " + node.getId());
List<String> associatedIPs = new ArrayList<String>();
associatedIPs.add(ip);
return associatedIPs;
}