@SuppressWarnings("unchecked")
private void setupInstancesInDht() {
instance123 = setupInstanceWithReservationId("i-123", "r-123", ownerId);
instance456 = setupInstanceWithReservationId("i-456", "r-123", ownerId);
// setup dht reader to return the
doAnswer(
new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
PiScatterGatherContinuation<Instance> piContinuation =
(PiScatterGatherContinuation<Instance>) invocation.getArguments()[1];
if (invocation.getArguments()[0].equals(id123)) {
piContinuation.receiveResult(instance123);
} else piContinuation.receiveResult(instance456);
return null;
}
})
.when(dhtReader)
.getAnyAsync(or(eq(id123), eq(id456)), isA(PiScatterGatherContinuation.class));
instanceIds = Arrays.asList(new String[] {"i-123", "i-456"});
when(instance123.getNodeId()).thenReturn(nodeIdStr);
when(instance456.getNodeId()).thenReturn(nodeIdStr);
}
@Test
public void shouldRebootMultipleInstances() {
// setup
final Collection<String> instanceIds = Arrays.asList(new String[] {"i-123", "i-456"});
when(instance123.getNodeId()).thenReturn(nodeIdStr);
when(instance456.getNodeId()).thenReturn(nodeIdStr);
when(piIdBuilder.getNodeIdFromNodeId(anyString())).thenReturn(nodeId);
setupScatterGather(instanceIds);
// act
rebootInstanceServiceHelper.rebootInstances(ownerId, instanceIds, apiApplicationManager);
// verify
verify(instance123).setRestartRequested(eq(true));
verify(instance456).setRestartRequested(eq(true));
verify(instanceManagerMessageContext)
.routePiMessageToApplication(
eq(nodeId),
eq(EntityMethod.UPDATE),
eq(instance123),
eq(InstanceManagerApplication.APPLICATION_NAME));
verify(secondMessageContext)
.routePiMessageToApplication(
eq(nodeId),
eq(EntityMethod.UPDATE),
eq(instance456),
eq(InstanceManagerApplication.APPLICATION_NAME));
}
private Instance setupInstanceWithReservationId(
String instanceId, String reservationId, String userId) {
Instance instance = mock(Instance.class);
when(instance.getInstanceId()).thenReturn(instanceId);
when(instance.getReservationId()).thenReturn(reservationId);
when(instance.getUserId()).thenReturn(userId);
when(instance.getLastHeartbeatTimestamp()).thenReturn(System.currentTimeMillis());
return instance;
}
@Override
public Instance update(Instance existingEntity, Instance requestedEntity) {
LOG.debug(
String.format(
"update(Existing Instance - %s,Requested Instance - %s)",
existingEntity, requestedEntity));
InstanceStateTransition transition = new InstanceStateTransition();
Instance entityToReturn = null;
if (existingEntity != null) {
transition.setPreviousState(existingEntity.getState());
// we set this here in case we can't write the state want.
// this way the user doesn't get back a null. :D.
transition.setNextState(existingEntity.getState());
instanceStatusMap.put(existingEntity.getInstanceId(), transition);
if (existingEntity.getNodeId() != null
&& existingEntity.getState() != InstanceState.SHUTTING_DOWN
&& existingEntity.getState() != InstanceState.TERMINATED) {
existingEntity.setState(InstanceState.SHUTTING_DOWN);
entityToReturn = existingEntity;
} else if (existingEntity.getState() != InstanceState.TERMINATED) {
LOG.info(
String.format(
"Instance %s is provisionally being terminated before an Instance Manager has claimed it by stamping a node id onto it",
existingEntity.getInstanceId()));
existingEntity.setState(InstanceState.TERMINATED);
entityToReturn = existingEntity;
}
} else {
LOG.warn(String.format("Expected non-null instance"));
}
return entityToReturn;
}
protected void assertThatBridgeIfCommandIsExecuted() {
String[] expectedCommand =
new String[] {
"brctl",
"addif",
String.format("pibr%d", instance.getVlanId()),
String.format(
"%s.%d", properties.getProperty("vnet.private.interface"), instance.getVlanId())
};
assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
}
private void assertThatMoveCommandIsExecuted(String instanceId) {
String[] expectedCommand =
new String[] {
"mv",
String.format(
"%s/%s/%s",
properties.getProperty("instances.directory"), instance.getUserId(), instanceId),
String.format(
"%s/%s/%s-terminated-%s",
properties.getProperty("instances.directory"),
instance.getUserId(),
instanceId,
new SimpleDateFormat("yyyyMMdd").format(new Date()))
};
assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
}
protected void assertThatBridgeDownCommandIsExecuted() {
String[] expectedCommand =
new String[] {
"ip", "link", "set", "dev", String.format("pibr%d", instance.getVlanId()), "down"
};
assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
}
@Before
public void seedTheDht() throws Exception {
instanceId = String.format("i-07VO7Y%d", instanceIdIndex);
System.err.println("instanceId: " + instanceId);
instanceIdIndex++;
getBeanReferences();
seed();
putImageIntoDht();
putUserIntoDht();
putInstanceIntoDht(instanceId);
putInstanceTypeIntoDht();
instancePath =
String.format(
"%s/%s/%s",
properties.getProperty("instances.directory"), instance.getUserId(), instanceId);
instanceImagePath = String.format("%s/%s", instancePath, IMAGE_ID);
instanceManagerApplication.becomeActive();
AnycastHandler anycastHandler = (AnycastHandler) context.getBean("anycastHandler");
anycastHandler.refreshInstanceTypes();
Thread.sleep(3000); // to allow async initialisations etc
}
protected void putInstanceTypeIntoDht() {
instanceTypeConfiguration =
new InstanceTypeConfiguration(instance.getInstanceType(), 1, 256, 1);
InstanceTypes instanceTypes = new InstanceTypes();
instanceTypes.addInstanceType(instanceTypeConfiguration);
PId instanceTypesId = piIdBuilder.getPId(instanceTypes);
putPiEntityIntoDht(instanceTypes, instanceTypesId);
}
protected void assertThatVconfigRemoveVlanCommandIsExecuted() {
String[] expectedCommand =
new String[] {
"vconfig",
"rem",
String.format(
"%s.%d", properties.getProperty("vnet.private.interface"), instance.getVlanId())
};
assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
}
protected void assertThatDomainXmlContainsNecessaryFields(String domainXml) {
String errorMessage = String.format("Instance: %s, XML: %s", instance, domainXml);
assertTrue(errorMessage, domainXml.contains(String.format("<name>%s</name>", instanceId)));
assertTrue(
errorMessage, domainXml.contains(String.format("<type>%s</type>", instance.getPlatform())));
assertTrue(
errorMessage,
domainXml.contains(
String.format(
"<kernel>%s/%s/%s/%s</kernel>",
properties.getProperty("instances.directory"),
instance.getUserId(),
instanceId,
KERNEL_ID)));
assertTrue(errorMessage, domainXml.contains("<cmdline> ro</cmdline>"));
assertTrue(errorMessage, domainXml.contains("<root>/dev/sda1</root>"));
assertTrue(
errorMessage,
domainXml.contains(
String.format(
"<source file='%s/%s/%s/%s'/>",
properties.getProperty("instances.directory"),
instance.getUserId(),
instanceId,
instance.getImageId())));
assertTrue(
errorMessage,
domainXml.contains(
String.format(
"<source file='%s/%s/%s/ephemeral'/>",
properties.getProperty("instances.directory"), instance.getUserId(), instanceId)));
assertTrue(
errorMessage,
domainXml.contains(
String.format(
"<source file='%s/%s/%s/swap'/>",
properties.getProperty("instances.directory"), instance.getUserId(), instanceId)));
assertTrue(errorMessage, domainXml.contains("<target dev='sda1'/>"));
assertTrue(errorMessage, domainXml.contains("<target dev='sda2'/>"));
assertTrue(errorMessage, domainXml.contains("<target dev='sda3'/>"));
assertTrue(errorMessage, domainXml.contains("<interface type='bridge'>"));
assertTrue(
errorMessage,
domainXml.contains(String.format("<mac address='%s'/>", instance.getPrivateMacAddress())));
assertTrue(
errorMessage,
domainXml.contains(
String.format(
"<source bridge='%s'/>",
VlanAddressUtils.getBridgeNameForVlan(instance.getVlanId()))));
assertTrue(errorMessage, domainXml.contains("<script path='/etc/xen/scripts/vif-bridge'/>"));
}
@Override
public void handleResult(final Instance instance) {
LOG.debug("Received Instance:" + instance.getUrl() + " from Network Manager");
executor.execute(
new Runnable() {
public void run() {
try {
DhtWriter dhtWriter = dhtClientFactory.createWriter();
dhtWriter.update(
piIdBuilder.getPIdForEc2AvailabilityZone(instance),
instance,
new UpdateResolvingPiContinuation<Instance>() {
@Override
public Instance update(Instance existingEntity, Instance requestedEntity) {
String hostname = "";
try {
hostname = InetAddress.getLocalHost().getHostName();
existingEntity.setHostname(hostname);
} catch (UnknownHostException e) {
LOG.error("Unable to retrieve hostname", e);
}
LOG.debug(
String.format(
"Updating instance: %s hostname: %s and nodeId: %s",
existingEntity.getInstanceId(), hostname, nodeId));
existingEntity.setNodeId(nodeId);
existingEntity.setSourceImagePath(instance.getSourceImagePath());
existingEntity.setSourceKernelPath(instance.getSourceKernelPath());
existingEntity.setSourceRamdiskPath(instance.getSourceRamdiskPath());
existingEntity.setKernelId(instance.getKernelId());
existingEntity.setRamdiskId(instance.getRamdiskId());
existingEntity.setPlatform(instance.getPlatform());
return existingEntity;
}
@Override
public void handleResult(Instance result) {
LOG.debug(String.format("Run instances state update result: %s", result));
}
});
runInstanceHandler.startInstance(instance);
} catch (Throwable t) {
LOG.error(t.getMessage(), t);
}
}
});
}
@Before
public void setup() {
when(piIdBuilder.getPIdForEc2AvailabilityZone(uri)).thenReturn(instancePId);
when(piIdBuilder.getGlobalAvailabilityZoneCodeFromEc2Id(instanceId)).thenReturn(globalAvzCode);
when(piIdBuilder.getPId(SecurityGroup.getUrl(userId, securityGroupName)))
.thenReturn(securityGroupPId);
when(piIdBuilder.getPiQueuePId(PiQueue.INSTANCE_NETWORK_MANAGER_TEARDOWN))
.thenReturn(instanceNetworkManagerTeardownQueueId);
when(securityGroupPId.forGlobalAvailablityZoneCode(globalAvzCode)).thenReturn(securityGroupPId);
when(instanceNetworkManagerTeardownQueueId.forLocalScope(
PiQueue.TERMINATE_INSTANCE.getNodeScope()))
.thenReturn(instanceNetworkManagerTeardownQueueId);
when(messageContextFactory.newMessageContext()).thenReturn(messageContext);
when(dhtClientFactory.createReader()).thenReturn(dhtReader);
when(resultInstance.getInstanceId()).thenReturn(instanceId);
when(resultInstance.getNodeId()).thenReturn(instanceNodeId);
when(resultInstance.getUserId()).thenReturn(userId);
when(resultInstance.getSecurityGroupName()).thenReturn(securityGroupName);
}
protected void assertThatEphemeralDDCommandIsRun() {
int ephemeralDiskSize =
instance.getImageSizeInMB() - (int) (new File(instanceImagePath).length() / 1024 / 1024);
String[] expectedCommand =
new String[] {
"/bin/sh",
"-c",
String.format(
"ionice -c3 dd bs=1M count=0 seek=%d if=/dev/zero of=%s/ephemeral 2>/dev/null",
ephemeralDiskSize, instancePath)
};
assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
}
protected void assertThatBringInterfaceDownCommandIsExecuted() {
String[] expectedCommand =
new String[] {
"ip",
"link",
"set",
"dev",
String.format(
"%s.%d", properties.getProperty("vnet.private.interface"), instance.getVlanId()),
"down"
};
assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
}
public ReservationInstances runInstances(final Reservation reservation) {
LOG.debug(String.format("runInstances(%s)", reservation));
String securityGroupUrl =
SecurityGroup.getUrl(reservation.getUserId(), reservation.getSecurityGroupName());
PId securityGroupId = getPiIdBuilder().getPId(securityGroupUrl).forLocalRegion();
SecurityGroup securityGroup =
(SecurityGroup) getDhtClientFactory().createBlockingReader().get(securityGroupId);
validateReservation(reservation, securityGroup);
reservation.setReservationId(getIdFactory().createNewReservationId());
AvailabilityZone availabilityZone;
if (StringUtils.isNotEmpty(reservation.getAvailabilityZone())) {
try {
availabilityZone = getAvailabilityZoneByName(reservation.getAvailabilityZone());
} catch (AvailabilityZoneNotFoundException e) {
throw new IllegalArgumentException(
String.format("Unknown availability zone: %s", reservation.getAvailabilityZone()));
}
} else {
availabilityZone = getLocalAvailabilityZone();
reservation.setAvailabilityZone(availabilityZone.getAvailabilityZoneName());
}
// setup return object
ReservationInstances reservationInstances = new ReservationInstances();
reservationInstances.setReservation(reservation);
for (int i = 0; i < reservation.getMaxCount(); i++) {
String instanceId =
getIdFactory().createNewInstanceId(availabilityZone.getGlobalAvailabilityZoneCode());
// create instance
Instance instance = new Instance(reservation);
instance.setInstanceType(reservation.getInstanceType());
instance.setUserId(reservation.getUserId());
instance.setInstanceId(instanceId);
instance.setState(InstanceState.PENDING);
instance.setLaunchTime(System.currentTimeMillis());
instance.setAvailabilityZoneCode(availabilityZone.getAvailabilityZoneCodeWithinRegion());
instance.setRegionCode(availabilityZone.getRegionCode());
LOG.info(String.format("Requesting new %s", instance));
reservationInstances.getInstances().add(instance);
// create instance in dht
PId instanceDhtId =
getPiIdBuilder().getPIdForEc2AvailabilityZone(Instance.getUrl(instanceId));
BlockingDhtWriter blockingDhtWriter = getDhtClientFactory().createBlockingWriter();
AddNewInstanceResolver addNewInstanceResolver = new AddNewInstanceResolver();
blockingDhtWriter.update(instanceDhtId, instance, addNewInstanceResolver);
reservation.addInstanceId(instance.getInstanceId());
}
getUserService()
.addInstancesToUser(
reservation.getUserId(), reservation.getInstanceIds(), reservation.getInstanceType());
// write security group to DHT
getDhtClientFactory()
.createBlockingWriter()
.update(
securityGroupId,
null,
new AddInstanceToSecurityGroupResolver(reservation.getInstanceIds()));
// add to task processing queue
PId runInstanceQueueId =
getPiIdBuilder()
.getPId(PiQueue.RUN_INSTANCE.getUrl())
.forGlobalAvailablityZoneCode(availabilityZone.getGlobalAvailabilityZoneCode());
for (String instanceId : reservation.getInstanceIds()) {
getTaskProcessingQueueHelper()
.addUrlToQueue(runInstanceQueueId, Instance.getUrl(instanceId), instanceTaskQueueRetries);
}
// anycast message
PubSubMessageContext pubSubMessageContext =
getApiApplicationManager()
.newPubSubMessageContextFromGlobalAvzCode(
PiTopics.RUN_INSTANCE, availabilityZone.getGlobalAvailabilityZoneCode());
pubSubMessageContext.randomAnycast(EntityMethod.CREATE, reservation);
return reservationInstances;
}
protected Instance putInstanceIntoDht(String instanceID) {
int globalAvailabilityZoneCode = piIdBuilder.getGlobalAvailabilityZoneCodeFromEc2Id(instanceID);
instance = new Instance(instanceID, "test", "default", ImagePlatform.linux);
instance.setAvailabilityZoneCode(
PId.getAvailabilityZoneCodeWithinRegionFromGlobalAvailabilityZoneCode(
globalAvailabilityZoneCode));
instance.setRegionCode(
PId.getRegionCodeFromGlobalAvailabilityZoneCode(globalAvailabilityZoneCode));
instance.setImageId(IMAGE_ID);
instance.setKeyName(KEY);
instance.setImageSizeInMB(1024);
instance.setPrivateMacAddress("D0:0D:12:34:56:78");
instance.setMemoryInKB("512");
instance.setVcpus(1);
instance.setVlanId(getRandomVlanId());
instance.setInstanceType("m1.small");
instance.setUserId(USER_NAME);
instance.setState(InstanceState.PENDING);
instancePastryId = piIdBuilder.getPIdForEc2AvailabilityZone(instance);
System.err.println(
"Instance pastryId: "
+ instancePastryId
+ " gAvz code embeded in instanceID: "
+ globalAvailabilityZoneCode);
System.err.println("Instance pastryId: " + instancePastryId.toStringFull());
putPiEntityIntoDht(instance, instancePastryId);
return instance;
}
@Override
public void handleResult(Instance result) {
LOG.debug(String.format("handleResult(PiEntity - %s)", result));
if (result != null) {
if (result.getNodeId() != null) {
LOG.debug(
String.format(
"Sending termination message for inst %s to node %s",
result.getInstanceId(), result.getNodeId()));
PId instanceRecordId = piIdBuilder.getNodeIdFromNodeId(result.getNodeId());
MessageContext instanceMessageContext = messageContextFactory.newMessageContext();
instanceMessageContext.routePiMessageToApplication(
instanceRecordId,
EntityMethod.DELETE,
result,
InstanceManagerApplication.APPLICATION_NAME);
} else {
LOG.debug(
String.format(
"Null node id detected, not sending termination message for %s to instance manager",
result.getInstanceId()));
}
int instanceGlobalAvzCode =
piIdBuilder.getGlobalAvailabilityZoneCodeFromEc2Id(result.getInstanceId());
PId securityGroupId =
piIdBuilder
.getPId(SecurityGroup.getUrl(result.getUserId(), result.getSecurityGroupName()))
.forGlobalAvailablityZoneCode(instanceGlobalAvzCode);
LOG.debug(
String.format(
"Sending termination message for inst %s to network app node %s for sec group %s:%s",
result.getInstanceId(),
result.getUserId(),
securityGroupId.toStringFull(),
result.getSecurityGroupName()));
MessageContext securityGroupMessageContext = messageContextFactory.newMessageContext();
securityGroupMessageContext.routePiMessageToApplication(
securityGroupId, EntityMethod.DELETE, result, NetworkManagerApplication.APPLICATION_NAME);
InstanceStateTransition transition = instanceStatusMap.get(result.getInstanceId());
if (transition != null) {
transition.setNextState(result.getState());
}
} else {
LOG.warn("Expected a non-null instance record");
}
}
protected void assertThatNewBridgeCommandIsExecuted() {
String[] expectedCommand =
new String[] {"brctl", "addbr", String.format("pibr%d", instance.getVlanId())};
assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
}