@SuppressWarnings("unchecked")
  private void setupInstancesInDht() {
    instance123 = setupInstanceWithReservationId("i-123", "r-123", ownerId);
    instance456 = setupInstanceWithReservationId("i-456", "r-123", ownerId);

    // setup dht reader to return the
    doAnswer(
            new Answer<Object>() {
              @Override
              public Object answer(InvocationOnMock invocation) throws Throwable {
                PiScatterGatherContinuation<Instance> piContinuation =
                    (PiScatterGatherContinuation<Instance>) invocation.getArguments()[1];

                if (invocation.getArguments()[0].equals(id123)) {
                  piContinuation.receiveResult(instance123);
                } else piContinuation.receiveResult(instance456);
                return null;
              }
            })
        .when(dhtReader)
        .getAnyAsync(or(eq(id123), eq(id456)), isA(PiScatterGatherContinuation.class));

    instanceIds = Arrays.asList(new String[] {"i-123", "i-456"});
    when(instance123.getNodeId()).thenReturn(nodeIdStr);
    when(instance456.getNodeId()).thenReturn(nodeIdStr);
  }
  @Test
  public void shouldRebootMultipleInstances() {
    // setup
    final Collection<String> instanceIds = Arrays.asList(new String[] {"i-123", "i-456"});
    when(instance123.getNodeId()).thenReturn(nodeIdStr);
    when(instance456.getNodeId()).thenReturn(nodeIdStr);
    when(piIdBuilder.getNodeIdFromNodeId(anyString())).thenReturn(nodeId);
    setupScatterGather(instanceIds);

    // act
    rebootInstanceServiceHelper.rebootInstances(ownerId, instanceIds, apiApplicationManager);

    // verify
    verify(instance123).setRestartRequested(eq(true));
    verify(instance456).setRestartRequested(eq(true));
    verify(instanceManagerMessageContext)
        .routePiMessageToApplication(
            eq(nodeId),
            eq(EntityMethod.UPDATE),
            eq(instance123),
            eq(InstanceManagerApplication.APPLICATION_NAME));
    verify(secondMessageContext)
        .routePiMessageToApplication(
            eq(nodeId),
            eq(EntityMethod.UPDATE),
            eq(instance456),
            eq(InstanceManagerApplication.APPLICATION_NAME));
  }
 private Instance setupInstanceWithReservationId(
     String instanceId, String reservationId, String userId) {
   Instance instance = mock(Instance.class);
   when(instance.getInstanceId()).thenReturn(instanceId);
   when(instance.getReservationId()).thenReturn(reservationId);
   when(instance.getUserId()).thenReturn(userId);
   when(instance.getLastHeartbeatTimestamp()).thenReturn(System.currentTimeMillis());
   return instance;
 }
 @Override
 public Instance update(Instance existingEntity, Instance requestedEntity) {
   LOG.debug(
       String.format(
           "update(Existing Instance - %s,Requested Instance - %s)",
           existingEntity, requestedEntity));
   InstanceStateTransition transition = new InstanceStateTransition();
   Instance entityToReturn = null;
   if (existingEntity != null) {
     transition.setPreviousState(existingEntity.getState());
     // we set this here in case we can't write the state want.
     // this way the user doesn't get back a null. :D.
     transition.setNextState(existingEntity.getState());
     instanceStatusMap.put(existingEntity.getInstanceId(), transition);
     if (existingEntity.getNodeId() != null
         && existingEntity.getState() != InstanceState.SHUTTING_DOWN
         && existingEntity.getState() != InstanceState.TERMINATED) {
       existingEntity.setState(InstanceState.SHUTTING_DOWN);
       entityToReturn = existingEntity;
     } else if (existingEntity.getState() != InstanceState.TERMINATED) {
       LOG.info(
           String.format(
               "Instance %s is provisionally being terminated before an Instance Manager has claimed it by stamping a node id onto it",
               existingEntity.getInstanceId()));
       existingEntity.setState(InstanceState.TERMINATED);
       entityToReturn = existingEntity;
     }
   } else {
     LOG.warn(String.format("Expected non-null instance"));
   }
   return entityToReturn;
 }
 protected void assertThatBridgeIfCommandIsExecuted() {
   String[] expectedCommand =
       new String[] {
         "brctl",
         "addif",
         String.format("pibr%d", instance.getVlanId()),
         String.format(
             "%s.%d", properties.getProperty("vnet.private.interface"), instance.getVlanId())
       };
   assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
 }
 private void assertThatMoveCommandIsExecuted(String instanceId) {
   String[] expectedCommand =
       new String[] {
         "mv",
         String.format(
             "%s/%s/%s",
             properties.getProperty("instances.directory"), instance.getUserId(), instanceId),
         String.format(
             "%s/%s/%s-terminated-%s",
             properties.getProperty("instances.directory"),
             instance.getUserId(),
             instanceId,
             new SimpleDateFormat("yyyyMMdd").format(new Date()))
       };
   assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
 }
 protected void assertThatBridgeDownCommandIsExecuted() {
   String[] expectedCommand =
       new String[] {
         "ip", "link", "set", "dev", String.format("pibr%d", instance.getVlanId()), "down"
       };
   assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
 }
  @Before
  public void seedTheDht() throws Exception {
    instanceId = String.format("i-07VO7Y%d", instanceIdIndex);
    System.err.println("instanceId: " + instanceId);
    instanceIdIndex++;

    getBeanReferences();
    seed();
    putImageIntoDht();
    putUserIntoDht();
    putInstanceIntoDht(instanceId);
    putInstanceTypeIntoDht();

    instancePath =
        String.format(
            "%s/%s/%s",
            properties.getProperty("instances.directory"), instance.getUserId(), instanceId);
    instanceImagePath = String.format("%s/%s", instancePath, IMAGE_ID);

    instanceManagerApplication.becomeActive();

    AnycastHandler anycastHandler = (AnycastHandler) context.getBean("anycastHandler");
    anycastHandler.refreshInstanceTypes();

    Thread.sleep(3000); // to allow async initialisations etc
  }
 protected void putInstanceTypeIntoDht() {
   instanceTypeConfiguration =
       new InstanceTypeConfiguration(instance.getInstanceType(), 1, 256, 1);
   InstanceTypes instanceTypes = new InstanceTypes();
   instanceTypes.addInstanceType(instanceTypeConfiguration);
   PId instanceTypesId = piIdBuilder.getPId(instanceTypes);
   putPiEntityIntoDht(instanceTypes, instanceTypesId);
 }
 protected void assertThatVconfigRemoveVlanCommandIsExecuted() {
   String[] expectedCommand =
       new String[] {
         "vconfig",
         "rem",
         String.format(
             "%s.%d", properties.getProperty("vnet.private.interface"), instance.getVlanId())
       };
   assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
 }
 protected void assertThatDomainXmlContainsNecessaryFields(String domainXml) {
   String errorMessage = String.format("Instance: %s, XML: %s", instance, domainXml);
   assertTrue(errorMessage, domainXml.contains(String.format("<name>%s</name>", instanceId)));
   assertTrue(
       errorMessage, domainXml.contains(String.format("<type>%s</type>", instance.getPlatform())));
   assertTrue(
       errorMessage,
       domainXml.contains(
           String.format(
               "<kernel>%s/%s/%s/%s</kernel>",
               properties.getProperty("instances.directory"),
               instance.getUserId(),
               instanceId,
               KERNEL_ID)));
   assertTrue(errorMessage, domainXml.contains("<cmdline> ro</cmdline>"));
   assertTrue(errorMessage, domainXml.contains("<root>/dev/sda1</root>"));
   assertTrue(
       errorMessage,
       domainXml.contains(
           String.format(
               "<source file='%s/%s/%s/%s'/>",
               properties.getProperty("instances.directory"),
               instance.getUserId(),
               instanceId,
               instance.getImageId())));
   assertTrue(
       errorMessage,
       domainXml.contains(
           String.format(
               "<source file='%s/%s/%s/ephemeral'/>",
               properties.getProperty("instances.directory"), instance.getUserId(), instanceId)));
   assertTrue(
       errorMessage,
       domainXml.contains(
           String.format(
               "<source file='%s/%s/%s/swap'/>",
               properties.getProperty("instances.directory"), instance.getUserId(), instanceId)));
   assertTrue(errorMessage, domainXml.contains("<target dev='sda1'/>"));
   assertTrue(errorMessage, domainXml.contains("<target dev='sda2'/>"));
   assertTrue(errorMessage, domainXml.contains("<target dev='sda3'/>"));
   assertTrue(errorMessage, domainXml.contains("<interface type='bridge'>"));
   assertTrue(
       errorMessage,
       domainXml.contains(String.format("<mac address='%s'/>", instance.getPrivateMacAddress())));
   assertTrue(
       errorMessage,
       domainXml.contains(
           String.format(
               "<source bridge='%s'/>",
               VlanAddressUtils.getBridgeNameForVlan(instance.getVlanId()))));
   assertTrue(errorMessage, domainXml.contains("<script path='/etc/xen/scripts/vif-bridge'/>"));
 }
  @Override
  public void handleResult(final Instance instance) {
    LOG.debug("Received Instance:" + instance.getUrl() + " from Network Manager");

    executor.execute(
        new Runnable() {
          public void run() {
            try {
              DhtWriter dhtWriter = dhtClientFactory.createWriter();
              dhtWriter.update(
                  piIdBuilder.getPIdForEc2AvailabilityZone(instance),
                  instance,
                  new UpdateResolvingPiContinuation<Instance>() {
                    @Override
                    public Instance update(Instance existingEntity, Instance requestedEntity) {

                      String hostname = "";

                      try {
                        hostname = InetAddress.getLocalHost().getHostName();
                        existingEntity.setHostname(hostname);
                      } catch (UnknownHostException e) {
                        LOG.error("Unable to retrieve hostname", e);
                      }

                      LOG.debug(
                          String.format(
                              "Updating instance: %s hostname: %s and nodeId: %s",
                              existingEntity.getInstanceId(), hostname, nodeId));

                      existingEntity.setNodeId(nodeId);
                      existingEntity.setSourceImagePath(instance.getSourceImagePath());
                      existingEntity.setSourceKernelPath(instance.getSourceKernelPath());
                      existingEntity.setSourceRamdiskPath(instance.getSourceRamdiskPath());
                      existingEntity.setKernelId(instance.getKernelId());
                      existingEntity.setRamdiskId(instance.getRamdiskId());
                      existingEntity.setPlatform(instance.getPlatform());
                      return existingEntity;
                    }

                    @Override
                    public void handleResult(Instance result) {
                      LOG.debug(String.format("Run instances state update result: %s", result));
                    }
                  });

              runInstanceHandler.startInstance(instance);

            } catch (Throwable t) {
              LOG.error(t.getMessage(), t);
            }
          }
        });
  }
  @Before
  public void setup() {
    when(piIdBuilder.getPIdForEc2AvailabilityZone(uri)).thenReturn(instancePId);
    when(piIdBuilder.getGlobalAvailabilityZoneCodeFromEc2Id(instanceId)).thenReturn(globalAvzCode);
    when(piIdBuilder.getPId(SecurityGroup.getUrl(userId, securityGroupName)))
        .thenReturn(securityGroupPId);
    when(piIdBuilder.getPiQueuePId(PiQueue.INSTANCE_NETWORK_MANAGER_TEARDOWN))
        .thenReturn(instanceNetworkManagerTeardownQueueId);
    when(securityGroupPId.forGlobalAvailablityZoneCode(globalAvzCode)).thenReturn(securityGroupPId);

    when(instanceNetworkManagerTeardownQueueId.forLocalScope(
            PiQueue.TERMINATE_INSTANCE.getNodeScope()))
        .thenReturn(instanceNetworkManagerTeardownQueueId);
    when(messageContextFactory.newMessageContext()).thenReturn(messageContext);
    when(dhtClientFactory.createReader()).thenReturn(dhtReader);

    when(resultInstance.getInstanceId()).thenReturn(instanceId);
    when(resultInstance.getNodeId()).thenReturn(instanceNodeId);
    when(resultInstance.getUserId()).thenReturn(userId);
    when(resultInstance.getSecurityGroupName()).thenReturn(securityGroupName);
  }
 protected void assertThatEphemeralDDCommandIsRun() {
   int ephemeralDiskSize =
       instance.getImageSizeInMB() - (int) (new File(instanceImagePath).length() / 1024 / 1024);
   String[] expectedCommand =
       new String[] {
         "/bin/sh",
         "-c",
         String.format(
             "ionice -c3 dd bs=1M count=0 seek=%d if=/dev/zero of=%s/ephemeral 2>/dev/null",
             ephemeralDiskSize, instancePath)
       };
   assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
 }
 protected void assertThatBringInterfaceDownCommandIsExecuted() {
   String[] expectedCommand =
       new String[] {
         "ip",
         "link",
         "set",
         "dev",
         String.format(
             "%s.%d", properties.getProperty("vnet.private.interface"), instance.getVlanId()),
         "down"
       };
   assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
 }
  public ReservationInstances runInstances(final Reservation reservation) {
    LOG.debug(String.format("runInstances(%s)", reservation));

    String securityGroupUrl =
        SecurityGroup.getUrl(reservation.getUserId(), reservation.getSecurityGroupName());
    PId securityGroupId = getPiIdBuilder().getPId(securityGroupUrl).forLocalRegion();
    SecurityGroup securityGroup =
        (SecurityGroup) getDhtClientFactory().createBlockingReader().get(securityGroupId);

    validateReservation(reservation, securityGroup);

    reservation.setReservationId(getIdFactory().createNewReservationId());

    AvailabilityZone availabilityZone;
    if (StringUtils.isNotEmpty(reservation.getAvailabilityZone())) {
      try {
        availabilityZone = getAvailabilityZoneByName(reservation.getAvailabilityZone());
      } catch (AvailabilityZoneNotFoundException e) {
        throw new IllegalArgumentException(
            String.format("Unknown availability zone: %s", reservation.getAvailabilityZone()));
      }
    } else {
      availabilityZone = getLocalAvailabilityZone();
      reservation.setAvailabilityZone(availabilityZone.getAvailabilityZoneName());
    }

    // setup return object
    ReservationInstances reservationInstances = new ReservationInstances();
    reservationInstances.setReservation(reservation);

    for (int i = 0; i < reservation.getMaxCount(); i++) {
      String instanceId =
          getIdFactory().createNewInstanceId(availabilityZone.getGlobalAvailabilityZoneCode());
      // create instance
      Instance instance = new Instance(reservation);
      instance.setInstanceType(reservation.getInstanceType());
      instance.setUserId(reservation.getUserId());
      instance.setInstanceId(instanceId);
      instance.setState(InstanceState.PENDING);
      instance.setLaunchTime(System.currentTimeMillis());
      instance.setAvailabilityZoneCode(availabilityZone.getAvailabilityZoneCodeWithinRegion());
      instance.setRegionCode(availabilityZone.getRegionCode());
      LOG.info(String.format("Requesting new %s", instance));
      reservationInstances.getInstances().add(instance);

      // create instance in dht
      PId instanceDhtId =
          getPiIdBuilder().getPIdForEc2AvailabilityZone(Instance.getUrl(instanceId));
      BlockingDhtWriter blockingDhtWriter = getDhtClientFactory().createBlockingWriter();
      AddNewInstanceResolver addNewInstanceResolver = new AddNewInstanceResolver();
      blockingDhtWriter.update(instanceDhtId, instance, addNewInstanceResolver);

      reservation.addInstanceId(instance.getInstanceId());
    }

    getUserService()
        .addInstancesToUser(
            reservation.getUserId(), reservation.getInstanceIds(), reservation.getInstanceType());

    // write security group to DHT
    getDhtClientFactory()
        .createBlockingWriter()
        .update(
            securityGroupId,
            null,
            new AddInstanceToSecurityGroupResolver(reservation.getInstanceIds()));

    // add to task processing queue
    PId runInstanceQueueId =
        getPiIdBuilder()
            .getPId(PiQueue.RUN_INSTANCE.getUrl())
            .forGlobalAvailablityZoneCode(availabilityZone.getGlobalAvailabilityZoneCode());
    for (String instanceId : reservation.getInstanceIds()) {
      getTaskProcessingQueueHelper()
          .addUrlToQueue(runInstanceQueueId, Instance.getUrl(instanceId), instanceTaskQueueRetries);
    }

    // anycast message
    PubSubMessageContext pubSubMessageContext =
        getApiApplicationManager()
            .newPubSubMessageContextFromGlobalAvzCode(
                PiTopics.RUN_INSTANCE, availabilityZone.getGlobalAvailabilityZoneCode());
    pubSubMessageContext.randomAnycast(EntityMethod.CREATE, reservation);

    return reservationInstances;
  }
  protected Instance putInstanceIntoDht(String instanceID) {
    int globalAvailabilityZoneCode = piIdBuilder.getGlobalAvailabilityZoneCodeFromEc2Id(instanceID);
    instance = new Instance(instanceID, "test", "default", ImagePlatform.linux);
    instance.setAvailabilityZoneCode(
        PId.getAvailabilityZoneCodeWithinRegionFromGlobalAvailabilityZoneCode(
            globalAvailabilityZoneCode));
    instance.setRegionCode(
        PId.getRegionCodeFromGlobalAvailabilityZoneCode(globalAvailabilityZoneCode));
    instance.setImageId(IMAGE_ID);
    instance.setKeyName(KEY);
    instance.setImageSizeInMB(1024);
    instance.setPrivateMacAddress("D0:0D:12:34:56:78");
    instance.setMemoryInKB("512");
    instance.setVcpus(1);
    instance.setVlanId(getRandomVlanId());
    instance.setInstanceType("m1.small");
    instance.setUserId(USER_NAME);
    instance.setState(InstanceState.PENDING);

    instancePastryId = piIdBuilder.getPIdForEc2AvailabilityZone(instance);
    System.err.println(
        "Instance pastryId: "
            + instancePastryId
            + " gAvz code embeded in instanceID: "
            + globalAvailabilityZoneCode);
    System.err.println("Instance pastryId: " + instancePastryId.toStringFull());
    putPiEntityIntoDht(instance, instancePastryId);

    return instance;
  }
  @Override
  public void handleResult(Instance result) {
    LOG.debug(String.format("handleResult(PiEntity - %s)", result));
    if (result != null) {
      if (result.getNodeId() != null) {
        LOG.debug(
            String.format(
                "Sending termination message for inst %s to node %s",
                result.getInstanceId(), result.getNodeId()));
        PId instanceRecordId = piIdBuilder.getNodeIdFromNodeId(result.getNodeId());
        MessageContext instanceMessageContext = messageContextFactory.newMessageContext();
        instanceMessageContext.routePiMessageToApplication(
            instanceRecordId,
            EntityMethod.DELETE,
            result,
            InstanceManagerApplication.APPLICATION_NAME);
      } else {
        LOG.debug(
            String.format(
                "Null node id detected, not sending termination message for %s to instance manager",
                result.getInstanceId()));
      }

      int instanceGlobalAvzCode =
          piIdBuilder.getGlobalAvailabilityZoneCodeFromEc2Id(result.getInstanceId());
      PId securityGroupId =
          piIdBuilder
              .getPId(SecurityGroup.getUrl(result.getUserId(), result.getSecurityGroupName()))
              .forGlobalAvailablityZoneCode(instanceGlobalAvzCode);
      LOG.debug(
          String.format(
              "Sending termination message for inst %s to network app node %s for sec group %s:%s",
              result.getInstanceId(),
              result.getUserId(),
              securityGroupId.toStringFull(),
              result.getSecurityGroupName()));
      MessageContext securityGroupMessageContext = messageContextFactory.newMessageContext();
      securityGroupMessageContext.routePiMessageToApplication(
          securityGroupId, EntityMethod.DELETE, result, NetworkManagerApplication.APPLICATION_NAME);
      InstanceStateTransition transition = instanceStatusMap.get(result.getInstanceId());
      if (transition != null) {
        transition.setNextState(result.getState());
      }
    } else {
      LOG.warn("Expected a non-null instance record");
    }
  }
 protected void assertThatNewBridgeCommandIsExecuted() {
   String[] expectedCommand =
       new String[] {"brctl", "addbr", String.format("pibr%d", instance.getVlanId())};
   assertTrue(stubCommandExecutor.assertCommand(expectedCommand));
 }