private void checkSecurityGroupCapacity(Reservation reservation, SecurityGroup securityGroup) {
LOG.debug(String.format("checking security group: %s", securityGroup));
int numberOfInstancesInSecurityGroup = securityGroup.getInstances().size();
if (numberOfInstancesInSecurityGroup == 0) {
LOG.debug(String.format("No instances in security group: %s", securityGroup));
return;
}
if (securityGroup.getNetmask() == null) {
LOG.debug(String.format("Unable to get the netmask for security group: %s", securityGroup));
return;
}
int availableNumberOfInstancesInSecurityGroup = getSecurityGroupCapacity(securityGroup);
LOG.debug(
String.format(
"SecurityGroup: %s - Number of Instances: %d - Capacity in SecurityGroup: %d",
securityGroup.getUrl(),
numberOfInstancesInSecurityGroup,
availableNumberOfInstancesInSecurityGroup));
if (numberOfInstancesInSecurityGroup + reservation.getMinCount()
> availableNumberOfInstancesInSecurityGroup
- NUMBER_OF_RESERVED_ADDRESSES_IN_SECURITY_GROUP)
throw new IllegalStateException(
"unable to run instances as no more capacity in security group");
}
public void refreshVirtualNetworks() {
LOG.debug(String.format("Refreshing virtual networks"));
Set<URI> vlanIds =
consumedUriResourceRegistry.getResourceIdsByScheme(
ResourceSchemes.VIRTUAL_NETWORK.toString());
/*
* Because the vlanId gets inserted into the consumedUriResourceRegistry a few continuations after a security
* group gets added, sometimes this fails and then the pibr never actually gets onto the network manager. So we
* do some reconciliation between the security group and vlan registries. For now, we aren't looking for entries
* in the vlan registry that aren't in the security group list...needs some thought on whether that is needed but
* certainly not until we see some issues pertaining to that!
*/
List<SecurityGroup> securityGroups = consumedDhtResourceRegistry.getByType(SecurityGroup.class);
for (SecurityGroup securityGroup : securityGroups) {
Long vlanId = securityGroup.getVlanId();
if (null == vlanId) continue;
URI vlanUri = URI.create(String.format(S_D, ResourceSchemes.VIRTUAL_NETWORK, vlanId));
if (!vlanIds.contains(vlanUri)) {
LOG.debug(
String.format(
"Vlan Id %d present in the security groups in consumedDhtResourceRegistry is not present in the list of vlans in consumedUriResourceRegistry",
securityGroup.getVlanId()));
consumedUriResourceRegistry.registerConsumer(
vlanUri, securityGroup.getSecurityGroupId(), new LoggingContinuation<Boolean>());
networkCommandRunner.addManagedNetwork(vlanId, privateInterface);
}
}
List<String> deviceList = deviceUtils.getDeviceList();
for (URI uri : vlanIds) {
long vlanId = Long.parseLong(uri.getSchemeSpecificPart());
LOG.debug(String.format("Checking to see if vlan %d exists", vlanId));
if (!deviceUtils.deviceExists(
networkCommandRunner.getVlanInterface(vlanId, privateInterface), deviceList)) {
networkCommandRunner.addManagedNetwork(vlanId, privateInterface);
}
}
Collection<String> allVlanDevicesForInterface =
deviceUtils.getAllVlanDevicesForInterface(privateInterface, deviceList);
for (String vlan : allVlanDevicesForInterface) {
if (vlan.contains(DOT)) {
long vlanId = Long.parseLong(vlan.split(String.format("\\%s", DOT))[1]);
if (!vlanIds.contains(
URI.create(String.format(S_D, ResourceSchemes.VIRTUAL_NETWORK, vlanId))))
addressDeleteQueue.add(
new VlanDeleteItem(vlanId, privateInterface, networkCommandRunner));
}
}
}
public synchronized List<String> generateFilterTable(
List<SecurityGroup> securityGroups, List<String> existingChains) {
LOG.debug(String.format("generateFilterTable(%s, %s)", securityGroups, existingChains));
lines = new ArrayList<String>();
// if (!existingChains.contains(PI_CHAIN))
// addChain(PI_CHAIN);
// addForwardAllToChain(PI_CHAIN);
for (SecurityGroup securityGroup : securityGroups) {
if (!validateSecurityGroup(securityGroup)) {
LOG.debug(String.format(SKIPPING_SECURITY_GROUP_S_AS_IT_IS_NOT_POPULATED, securityGroup));
continue;
}
LOG.debug(String.format("Processing Security group: %s for filter rules.", securityGroup));
String filterChainName =
getChainNameForSecurityGroup(FLTR_PREFIX, securityGroup.getSecurityGroupId());
String destinationNetwork =
String.format(S_SLASH_S, securityGroup.getNetworkAddress(), securityGroup.getSlashnet());
if (!existingChains.contains(filterChainName))
lines.add(ipTablesHelper.addChain(filterChainName));
lines.add(
ipTablesHelper.appendForwardChainToChain(PI_CHAIN, destinationNetwork, filterChainName));
// TODO: Handle case where rule name specificed instead of network
for (NetworkRule networkRule : securityGroup.getNetworkRules()) {
for (String sourceNetwork : networkRule.getSourceNetworks()) {
addFilterRule(
filterChainName,
sourceNetwork,
destinationNetwork,
networkRule.getNetworkProtocol(),
networkRule.getPortRangeMin(),
networkRule.getPortRangeMax());
}
}
}
logFlushRules("Generated filter iptables:\n%s", lines);
return lines;
}
@Override
public SecurityGroup update(SecurityGroup existingEntity, SecurityGroup requestedEntity) {
if (null == existingEntity) return null;
for (String instanceId : reservationInstanceIds) {
existingEntity.getInstances().put(instanceId, new InstanceAddress());
}
return existingEntity;
}
public synchronized List<String> generateNatTable(
List<SecurityGroup> securityGroups, List<String> existingChains) {
LOG.debug(String.format("generateNatTable(%s, %s)", securityGroups, existingChains));
lines = new ArrayList<String>();
if (!existingChains.contains(PI_PREROUTING)) {
lines.add(ipTablesHelper.addChain(PI_PREROUTING));
lines.add(ipTablesHelper.appendForwardChainToChain(PREROUTING, PI_PREROUTING));
}
if (!existingChains.contains(PI_OUTPUT)) {
lines.add(ipTablesHelper.addChain(PI_OUTPUT));
lines.add(ipTablesHelper.appendForwardChainToChain(OUTPUT, PI_OUTPUT));
}
for (SecurityGroup securityGroup : securityGroups) {
if (!validateSecurityGroup(securityGroup)) {
LOG.debug(String.format(SKIPPING_SECURITY_GROUP_S_AS_IT_IS_NOT_POPULATED, securityGroup));
continue;
}
LOG.debug(String.format("Processing Security group: %s for nat rules.", securityGroup));
String postChainName =
getChainNameForSecurityGroup(POST_PREFIX, securityGroup.getSecurityGroupId());
String destinationNetwork =
String.format(S_SLASH_S, securityGroup.getNetworkAddress(), securityGroup.getSlashnet());
if (!existingChains.contains(postChainName))
lines.add(ipTablesHelper.addChain(postChainName));
lines.add(
ipTablesHelper.insertChainForwardToChain(POSTROUTING, destinationNetwork, postChainName));
// TODO: Handle case where rule name specified instead of network
for (Entry<String, InstanceAddress> addressEntry : securityGroup.getInstances().entrySet()) {
LOG.debug(String.format("Processing addressEntry: %s for nat rules.", addressEntry));
InstanceAddress address = addressEntry.getValue();
if (null == address) {
LOG.warn(
String.format("null entry in instance table for instance %s", addressEntry.getKey()));
continue;
}
if (address.getPublicIpAddress() != null && address.getPrivateIpAddress() != null) {
addNatRule(postChainName, address.getPublicIpAddress(), address.getPrivateIpAddress());
} else {
LOG.debug(
String.format(
"Skipping NAT rule within group %s as one or both addresses are null (%s / %s)",
securityGroup.getSecurityGroupId(),
address.getPublicIpAddress(),
address.getPrivateIpAddress()));
}
}
}
logFlushRules("Generated nat iptables:\n%s", lines);
return lines;
}
@Override
public void handleResult(Instance result) {
LOG.debug(String.format("handleResult(PiEntity - %s)", result));
if (result != null) {
if (result.getNodeId() != null) {
LOG.debug(
String.format(
"Sending termination message for inst %s to node %s",
result.getInstanceId(), result.getNodeId()));
PId instanceRecordId = piIdBuilder.getNodeIdFromNodeId(result.getNodeId());
MessageContext instanceMessageContext = messageContextFactory.newMessageContext();
instanceMessageContext.routePiMessageToApplication(
instanceRecordId,
EntityMethod.DELETE,
result,
InstanceManagerApplication.APPLICATION_NAME);
} else {
LOG.debug(
String.format(
"Null node id detected, not sending termination message for %s to instance manager",
result.getInstanceId()));
}
int instanceGlobalAvzCode =
piIdBuilder.getGlobalAvailabilityZoneCodeFromEc2Id(result.getInstanceId());
PId securityGroupId =
piIdBuilder
.getPId(SecurityGroup.getUrl(result.getUserId(), result.getSecurityGroupName()))
.forGlobalAvailablityZoneCode(instanceGlobalAvzCode);
LOG.debug(
String.format(
"Sending termination message for inst %s to network app node %s for sec group %s:%s",
result.getInstanceId(),
result.getUserId(),
securityGroupId.toStringFull(),
result.getSecurityGroupName()));
MessageContext securityGroupMessageContext = messageContextFactory.newMessageContext();
securityGroupMessageContext.routePiMessageToApplication(
securityGroupId, EntityMethod.DELETE, result, NetworkManagerApplication.APPLICATION_NAME);
InstanceStateTransition transition = instanceStatusMap.get(result.getInstanceId());
if (transition != null) {
transition.setNextState(result.getState());
}
} else {
LOG.warn("Expected a non-null instance record");
}
}
@Before
public void setup() {
when(piIdBuilder.getPIdForEc2AvailabilityZone(uri)).thenReturn(instancePId);
when(piIdBuilder.getGlobalAvailabilityZoneCodeFromEc2Id(instanceId)).thenReturn(globalAvzCode);
when(piIdBuilder.getPId(SecurityGroup.getUrl(userId, securityGroupName)))
.thenReturn(securityGroupPId);
when(piIdBuilder.getPiQueuePId(PiQueue.INSTANCE_NETWORK_MANAGER_TEARDOWN))
.thenReturn(instanceNetworkManagerTeardownQueueId);
when(securityGroupPId.forGlobalAvailablityZoneCode(globalAvzCode)).thenReturn(securityGroupPId);
when(instanceNetworkManagerTeardownQueueId.forLocalScope(
PiQueue.TERMINATE_INSTANCE.getNodeScope()))
.thenReturn(instanceNetworkManagerTeardownQueueId);
when(messageContextFactory.newMessageContext()).thenReturn(messageContext);
when(dhtClientFactory.createReader()).thenReturn(dhtReader);
when(resultInstance.getInstanceId()).thenReturn(instanceId);
when(resultInstance.getNodeId()).thenReturn(instanceNodeId);
when(resultInstance.getUserId()).thenReturn(userId);
when(resultInstance.getSecurityGroupName()).thenReturn(securityGroupName);
}
private boolean validateSecurityGroup(SecurityGroup securityGroup) {
if (null == securityGroup.getNetmask() || null == securityGroup.getVlanId()) return false;
return true;
}
public ReservationInstances runInstances(final Reservation reservation) {
LOG.debug(String.format("runInstances(%s)", reservation));
String securityGroupUrl =
SecurityGroup.getUrl(reservation.getUserId(), reservation.getSecurityGroupName());
PId securityGroupId = getPiIdBuilder().getPId(securityGroupUrl).forLocalRegion();
SecurityGroup securityGroup =
(SecurityGroup) getDhtClientFactory().createBlockingReader().get(securityGroupId);
validateReservation(reservation, securityGroup);
reservation.setReservationId(getIdFactory().createNewReservationId());
AvailabilityZone availabilityZone;
if (StringUtils.isNotEmpty(reservation.getAvailabilityZone())) {
try {
availabilityZone = getAvailabilityZoneByName(reservation.getAvailabilityZone());
} catch (AvailabilityZoneNotFoundException e) {
throw new IllegalArgumentException(
String.format("Unknown availability zone: %s", reservation.getAvailabilityZone()));
}
} else {
availabilityZone = getLocalAvailabilityZone();
reservation.setAvailabilityZone(availabilityZone.getAvailabilityZoneName());
}
// setup return object
ReservationInstances reservationInstances = new ReservationInstances();
reservationInstances.setReservation(reservation);
for (int i = 0; i < reservation.getMaxCount(); i++) {
String instanceId =
getIdFactory().createNewInstanceId(availabilityZone.getGlobalAvailabilityZoneCode());
// create instance
Instance instance = new Instance(reservation);
instance.setInstanceType(reservation.getInstanceType());
instance.setUserId(reservation.getUserId());
instance.setInstanceId(instanceId);
instance.setState(InstanceState.PENDING);
instance.setLaunchTime(System.currentTimeMillis());
instance.setAvailabilityZoneCode(availabilityZone.getAvailabilityZoneCodeWithinRegion());
instance.setRegionCode(availabilityZone.getRegionCode());
LOG.info(String.format("Requesting new %s", instance));
reservationInstances.getInstances().add(instance);
// create instance in dht
PId instanceDhtId =
getPiIdBuilder().getPIdForEc2AvailabilityZone(Instance.getUrl(instanceId));
BlockingDhtWriter blockingDhtWriter = getDhtClientFactory().createBlockingWriter();
AddNewInstanceResolver addNewInstanceResolver = new AddNewInstanceResolver();
blockingDhtWriter.update(instanceDhtId, instance, addNewInstanceResolver);
reservation.addInstanceId(instance.getInstanceId());
}
getUserService()
.addInstancesToUser(
reservation.getUserId(), reservation.getInstanceIds(), reservation.getInstanceType());
// write security group to DHT
getDhtClientFactory()
.createBlockingWriter()
.update(
securityGroupId,
null,
new AddInstanceToSecurityGroupResolver(reservation.getInstanceIds()));
// add to task processing queue
PId runInstanceQueueId =
getPiIdBuilder()
.getPId(PiQueue.RUN_INSTANCE.getUrl())
.forGlobalAvailablityZoneCode(availabilityZone.getGlobalAvailabilityZoneCode());
for (String instanceId : reservation.getInstanceIds()) {
getTaskProcessingQueueHelper()
.addUrlToQueue(runInstanceQueueId, Instance.getUrl(instanceId), instanceTaskQueueRetries);
}
// anycast message
PubSubMessageContext pubSubMessageContext =
getApiApplicationManager()
.newPubSubMessageContextFromGlobalAvzCode(
PiTopics.RUN_INSTANCE, availabilityZone.getGlobalAvailabilityZoneCode());
pubSubMessageContext.randomAnycast(EntityMethod.CREATE, reservation);
return reservationInstances;
}
private int getSecurityGroupCapacity(SecurityGroup securityGroup) {
String netmask = securityGroup.getNetmask();
int addrsInSlashnet = IpAddressUtils.addrsInSlashnet(IpAddressUtils.netmaskToSlashnet(netmask));
return addrsInSlashnet;
}
