Exemplo n.º 1
0
  /**
   * Handles paxos prepare messages
   *
   * @param msg the incomming message
   * @return a response message
   */
  public FleaseMessage handlePREPARE(FleaseMessage msg) {

    final FleaseAcceptorCell cc = getCell(msg);
    cc.touch();

    if ((cc.getPrepared() != null) && (cc.getPrepared().after(msg))) {
      if (Logging.isDebug() && config.isDebugPrintMessages()) {
        final String preped =
            (cc.getPrepared() == null)
                ? ProposalNumber.EMPTY_PROPOSAL_NUMBER.toString()
                : cc.getPrepared().getProposalNo().toString();
        Logging.logMessage(
            Logging.LEVEL_DEBUG,
            Category.replication,
            this,
            "A prepare NACK p:" + preped + " is after " + msg.getProposalNo() + "");
      }
      FleaseMessage reject = new FleaseMessage(FleaseMessage.MsgType.MSG_PREPARE_NACK, msg);
      reject.setPrevProposalNo(cc.getPrepared().getProposalNo());
      reject.setLeaseHolder(null);
      reject.setLeaseTimeout(0);
      reject.setSendTimestamp(TimeSync.getGlobalTime());
      return reject;
    } else {
      if (Logging.isDebug() && config.isDebugPrintMessages()) {
        final String preped =
            (cc.getPrepared() == null)
                ? ProposalNumber.EMPTY_PROPOSAL_NUMBER.toString()
                : cc.getPrepared().getProposalNo().toString();
        final String acced =
            (cc.getAccepted() == null)
                ? ProposalNumber.EMPTY_PROPOSAL_NUMBER.toString()
                : cc.getAccepted().getProposalNo()
                    + "="
                    + cc.getAccepted().getLeaseHolder()
                    + "/"
                    + cc.getAccepted().getLeaseTimeout();
        Logging.logMessage(
            Logging.LEVEL_DEBUG,
            Category.replication,
            this,
            "A prepare ACK  p:" + preped + " -> " + msg.getProposalNo() + " a:" + acced);
      }
      // lastPrepared = msg;
      cc.setPrepared(msg);
      // FIXME:Persistently write to disk
      FleaseMessage response = new FleaseMessage(FleaseMessage.MsgType.MSG_PREPARE_ACK, msg);

      if (cc.getAccepted() != null) {

        response.setPrevProposalNo(cc.getAccepted().getProposalNo());
        response.setLeaseHolder(cc.getAccepted().getLeaseHolder());
        assert (response.getLeaseHolder() != null);
        response.setLeaseTimeout(cc.getAccepted().getLeaseTimeout());
      }
      response.setSendTimestamp(TimeSync.getGlobalTime());
      return response;
    }
  }
Exemplo n.º 2
0
  @BeforeClass
  public static void setUpClass() throws Exception {
    Logging.start(Logging.LEVEL_WARN, Category.all);
    TimeSync.initializeLocal(50);

    cfg = new FleaseConfig(10000, 500, 500, new InetSocketAddress(12345), "localhost:12345", 5);
    testDir = new File("/tmp/xtreemfs-test/");
  }
Exemplo n.º 3
0
  @Override
  public void startRequest(MRCRequest rq) throws Throwable {

    final removexattrRequest rqArgs = (removexattrRequest) rq.getRequestArgs();

    final VolumeManager vMan = master.getVolumeManager();
    final FileAccessManager faMan = master.getFileAccessManager();

    Path p = new Path(rqArgs.getVolumeName(), rqArgs.getPath());

    validateContext(rq);

    StorageManager sMan = vMan.getStorageManagerByName(p.getComp(0));
    PathResolver res = new PathResolver(sMan, p);

    // check whether the path prefix is searchable
    faMan.checkSearchPermission(
        sMan, res, rq.getDetails().userId, rq.getDetails().superUser, rq.getDetails().groupIds);

    // check whether file exists
    res.checkIfFileDoesNotExist();

    // retrieve and prepare the metadata to return
    FileMetadata file = res.getFile();

    AtomicDBUpdate update = sMan.createAtomicDBUpdate(master, rq);

    // if the attribute is a system attribute, set it

    final String attrKey = rqArgs.getName();

    // set a system attribute
    if (attrKey.startsWith("xtreemfs.")) {

      // check whether the user has privileged permissions to set
      // system attributes
      faMan.checkPrivilegedPermissions(
          sMan, file, rq.getDetails().userId, rq.getDetails().superUser, rq.getDetails().groupIds);

      MRCHelper.setSysAttrValue(
          sMan, vMan, faMan, res.getParentDirId(), file, attrKey.substring(9), "", update);
    }

    // set a user attribute
    else {

      sMan.setXAttr(file.getId(), rq.getDetails().userId, attrKey, null, update);
    }

    // update POSIX timestamps
    int time = (int) (TimeSync.getGlobalTime() / 1000);
    MRCHelper.updateFileTimes(res.getParentDirId(), file, false, true, false, sMan, time, update);

    // set the response
    rq.setResponse(timestampResponse.newBuilder().setTimestampS(time).build());

    update.execute();
  }
  private void closeClientConnection() {
    if (client != null) {
      LOGGER.info("Closing XtreemFS-Connection completely!");
      try {
        client.stop();
        client = null;

        // Time Synchronization is a Singleton - it has to be closed on its own
        TimeSync.close();
      } catch (Exception e) {
        LOGGER.error("Could not close XtreemFS connection!", e);
      }
    }
  }
Exemplo n.º 5
0
  @Override
  public void startRequest(MRCRequest rq) throws Throwable {

    final XCap xcap = (XCap) rq.getRequestArgs();

    // perform master redirect if necessary due to DB operation
    if (master.getReplMasterUUID() != null
        && !master.getReplMasterUUID().equals(master.getConfig().getUUID().toString()))
      throw new DatabaseException(ExceptionType.REDIRECT);

    // create a capability object to verify the capability
    Capability cap = new Capability(xcap, master.getConfig().getCapabilitySecret());

    // check whether the capability has a valid signature
    if (!cap.hasValidSignature())
      throw new UserException(
          POSIXErrno.POSIX_ERROR_EPERM, cap + " does not have a valid signature");

    // check whether the capability has expired
    if (cap.hasExpired() && !renewTimedOutCaps)
      throw new UserException(POSIXErrno.POSIX_ERROR_EPERM, cap + " has expired");

    Capability newCap =
        new Capability(
            cap.getFileId(),
            cap.getAccessMode(),
            master.getConfig().getCapabilityTimeout(),
            TimeSync.getGlobalTime() / 1000 + master.getConfig().getCapabilityTimeout(),
            cap.getClientIdentity(),
            cap.getEpochNo(),
            cap.isReplicateOnClose(),
            cap.getSnapConfig(),
            cap.getSnapTimestamp(),
            cap.getTraceConfig().getTraceRequests(),
            cap.getTraceConfig().getTracingPolicy(),
            cap.getTraceConfig().getTracingPolicyConfig(),
            cap.getVoucherSize(),
            0L,
            master.getConfig().getCapabilitySecret());

    // set the response
    rq.setResponse(newCap.getXCap());
    finishRequest(rq);
  }
Exemplo n.º 6
0
  /** Test of createTimer method, of class FleaseStage. */
  @Test
  public void testCreateTimer() throws Exception {
    FleaseStage fs =
        new FleaseStage(
            cfg,
            "/tmp/xtreemfs-test/",
            new FleaseMessageSenderInterface() {

              @Override
              public void sendMessage(FleaseMessage messages, InetSocketAddress recipient) {
                // ignore me
              }
            },
            true,
            new FleaseViewChangeListenerInterface() {

              @Override
              public void viewIdChangeEvent(ASCIIString cellId, int viewId) {}
            },
            new FleaseStatusListener() {

              @Override
              public void statusChanged(ASCIIString cellId, Flease lease) {
                throw new UnsupportedOperationException("Not supported yet.");
              }

              @Override
              public void leaseFailed(ASCIIString cellId, FleaseException error) {
                throw new UnsupportedOperationException("Not supported yet.");
              }
            },
            null);

    FleaseMessage msg = new FleaseMessage(FleaseMessage.MsgType.EVENT_RESTART);
    msg.setCellId(new ASCIIString("testcell"));

    fs.start();
    fs.waitForStartup();
    fs.createTimer(msg, TimeSync.getLocalSystemTime() + 20);
    Thread.sleep(100);
    fs.shutdown();
    fs.waitForShutdown();
  }
Exemplo n.º 7
0
 /** @throws java.lang.Exception */
 @AfterClass
 public static void tearDownAfterClass() throws Exception {
   TimeSync ts = TimeSync.getInstance();
   ts.shutdown();
   ts.waitForShutdown();
 }
Exemplo n.º 8
0
 /** @throws java.lang.Exception */
 @BeforeClass
 public static void setUpBeforeClass() throws Exception {
   Logging.start(Logging.LEVEL_ERROR, Category.all);
   TimeSync.initializeLocal(TIMESYNC_LOCAL);
 }
  /**
   * Instantiates a storage manager by creating a new database.
   *
   * @param dbs the database system
   * @param volumeId the volume ID
   */
  public BabuDBStorageManager(
      BabuDB dbs,
      String volumeId,
      String volumeName,
      short fileAccessPolicyId,
      short[] osdPolicy,
      short[] replPolicy,
      String ownerId,
      String owningGroupId,
      int perms,
      ACLEntry[] acl,
      org.xtreemfs.pbrpc.generatedinterfaces.GlobalTypes.StripingPolicy rootDirDefSp,
      boolean allowSnaps,
      long volumeQuota,
      Map<String, String> attrs)
      throws DatabaseException {

    this.dbMan = dbs.getDatabaseManager();
    this.snapMan = dbs.getSnapshotManager();
    this.vcListeners = new LinkedList<VolumeChangeListener>();
    this.volume = new BabuDBVolumeInfo();

    TransactionalBabuDBUpdate update = new TransactionalBabuDBUpdate(dbMan);
    update.createDatabase(volumeId, 5);

    // atime, ctime, mtime
    int time = (int) (TimeSync.getGlobalTime() / 1000);

    // create the root directory; the name is the database name
    createDir(1, 0, volumeName, time, time, time, ownerId, owningGroupId, perms, 0, true, update);
    setLastFileId(1, update);

    volume.init(
        this,
        update.getDatabaseName(),
        volumeName,
        osdPolicy,
        replPolicy,
        fileAccessPolicyId,
        allowSnaps,
        volumeQuota,
        update);

    // set the default striping policy
    if (rootDirDefSp != null) setDefaultStripingPolicy(1, rootDirDefSp, true, update);

    if (acl != null)
      for (ACLEntry entry : acl) setACLEntry(1L, entry.getEntity(), entry.getRights(), update);

    if (attrs != null)
      for (Entry<String, String> attr : attrs.entrySet())
        setXAttr(
            1L,
            SYSTEM_UID,
            "xtreemfs.volattr." + attr.getKey(),
            attr.getValue().getBytes(),
            true,
            update);

    update.execute();

    try {
      database = dbMan.getDatabase(update.getDatabaseName());
    } catch (Exception exc) {
      throw new DatabaseException(exc);
    }

    notifyVolumeChange(volume);
  }
Exemplo n.º 10
0
  /** @param args the command line arguments */
  public static void main(String[] args) {
    // TODO code application logic here

    Logging.start(Logging.LEVEL_DEBUG);
    TimeSync ts = TimeSync.initializeLocal(50);

    try {
      ts.waitForStartup();

      TCPClient com =
          new TCPClient(
              3334,
              null,
              new NIOServer() {

                public void onAccept(NIOConnection connection) {
                  onConnect(connection);
                }

                public void onConnect(NIOConnection connection) {
                  System.out.println("connected to " + connection.getEndpoint());
                  connection.read(BufferPool.allocate(1024));
                  connection.setContext(new AtomicBoolean(false));
                }

                public void onRead(NIOConnection connection, ReusableBuffer buffer) {
                  System.out.println("read from " + connection);
                  buffer.flip();
                  byte[] data = new byte[buffer.remaining()];
                  buffer.get(data);
                  String contents = new String(data);
                  BufferPool.free(buffer);
                  connection.read(BufferPool.allocate(1024));
                  System.out.println(">> " + contents);
                }

                public void onClose(NIOConnection connection) {
                  System.out.println("connection from " + connection.getEndpoint() + " closed ");
                }

                public void onWriteFailed(IOException exception, Object context) {
                  System.out.println("could not write, context: " + context);
                }

                public void onConnectFailed(
                    InetSocketAddress endpoint, IOException exception, Object context) {
                  System.out.println("could not connect to: " + endpoint + ", context: " + context);
                }
              });
      com.start();
      com.waitForStartup();

      ReusableBuffer data = ReusableBuffer.wrap("Hello world!\n".getBytes());
      com.write(new InetSocketAddress("localhost", 3333), data, "Yagg");

      Thread.sleep(100);

      data = ReusableBuffer.wrap("Hello world!\n".getBytes());
      com.write(new InetSocketAddress("localhost", 3333), data, "Yagga");

      Thread.sleep(30000);

      data = ReusableBuffer.wrap("YaggaYagga!\n".getBytes());
      com.write(new InetSocketAddress("localhost", 3333), data, null);

      Thread.sleep(2000);
      com.shutdown();
      ts.close();

    } catch (Exception ex) {
      ex.printStackTrace();
      System.exit(1);
    }
  }
Exemplo n.º 11
0
  /** main loop */
  public FleaseMessage processMessage(FleaseMessage msg) {

    assert (!quit);

    /*if (Logging.isDebug())
    Logging.logMessage(Logging.LEVEL_DEBUG,this,"received %s",msg.toString());*/

    final long now = TimeSync.getLocalSystemTime();
    if (msg.getSendTimestamp() + config.getMessageTimeout() < TimeSync.getGlobalTime()) {
      // old message, ignore
      if (Logging.isDebug() && config.isDebugPrintMessages())
        Logging.logMessage(
            Logging.LEVEL_DEBUG,
            Category.replication,
            this,
            "A outdated message discarded: %s",
            msg.toString());
      return null;
    }
    if (this.waitUntilTimestamp_ms >= now) {
      if (Logging.isDebug() && config.isDebugPrintMessages())
        Logging.logMessage(
            Logging.LEVEL_DEBUG,
            Category.replication,
            this,
            "A message discarded, acceptor is still in recovery period");
      return null;
    }

    assert (msg.getCellId() != null);
    final FleaseAcceptorCell cc = getCell(msg.getCellId());

    if (cc.getViewId() < msg.getViewId()) {
      // If the local view is lower than the delivered one, the view listener has to be informed to
      // update
      // the local view. But the request can still be answered.
      viewListener.viewIdChangeEvent(msg.getCellId(), msg.getViewId());

    } else if (cc.getViewId() > msg.getViewId()
        || (cc.getViewId() == msg.getViewId() && cc.isViewInvalidated())) {
      // If the request is from an older view, or the a view that has been invalidated on this
      // AcceptorCell, the request has to be aborted.
      FleaseMessage response = new FleaseMessage(FleaseMessage.MsgType.MSG_WRONG_VIEW, msg);
      response.setViewId(cc.getViewId());
      return response;
    }

    FleaseMessage response = null;
    if (msg.getMsgType() == FleaseMessage.MsgType.MSG_PREPARE) response = handlePREPARE(msg);
    else if (msg.getMsgType() == FleaseMessage.MsgType.MSG_ACCEPT) response = handleACCEPT(msg);
    else if (msg.getMsgType() == FleaseMessage.MsgType.MSG_LEARN) handleLEARN(msg);
    /*else if (msg.getMsgType() == FleaseMessage.MsgType.MSG_GET_LEASE)
        response = handleGETLEASE(msg);
    else if (msg.getMsgType() == FleaseMessage.MsgType.MSG_RENEW_LEASE)
        response = handleRENEWINSTANCE(msg);*/
    else
      Logging.logMessage(
          Logging.LEVEL_ERROR,
          Category.replication,
          this,
          "A invalid message type received: %s",
          msg.toString());

    /*if (Logging.isDebug())
    Logging.logMessage(Logging.LEVEL_DEBUG,this,"response %s",(response != null) ? response.toString() : "<empty>");*/

    return response;
  }
Exemplo n.º 12
0
  /**
   * Handles paxos accept (vote) messages.
   *
   * @param msg incomming message
   * @return a response message or null
   */
  public FleaseMessage handleACCEPT(FleaseMessage msg) {

    final FleaseAcceptorCell cc = getCell(msg);

    cc.touch();
    if ((cc.getPrepared() != null) && (cc.getPrepared().after(msg))) {
      // reject the request
      if (Logging.isDebug() && config.isDebugPrintMessages()) {
        final String preped =
            (cc.getPrepared() == null)
                ? ProposalNumber.EMPTY_PROPOSAL_NUMBER.toString()
                : cc.getPrepared().getProposalNo().toString();
        Logging.logMessage(
            Logging.LEVEL_DEBUG,
            Category.replication,
            this,
            "A accept  NACK p:" + preped + " is after " + msg.getProposalNo() + "");
      }
      FleaseMessage tmp = new FleaseMessage(FleaseMessage.MsgType.MSG_ACCEPT_NACK, msg);
      tmp.setSendTimestamp(TimeSync.getGlobalTime());
      tmp.setLeaseHolder(null);
      tmp.setLeaseTimeout(0);
      tmp.setPrevProposalNo(cc.getPrepared().getProposalNo());
      return tmp;
    } else {
      // okay accept it
      if (Logging.isDebug() && config.isDebugPrintMessages()) {
        final String preped =
            (cc.getPrepared() == null)
                ? ProposalNumber.EMPTY_PROPOSAL_NUMBER.toString()
                : cc.getPrepared().getProposalNo().toString();
        final String acced =
            (cc.getAccepted() == null)
                ? ProposalNumber.EMPTY_PROPOSAL_NUMBER.toString()
                : cc.getAccepted().getProposalNo()
                    + "="
                    + cc.getAccepted().getLeaseHolder()
                    + "/"
                    + cc.getAccepted().getLeaseTimeout();
        Logging.logMessage(
            Logging.LEVEL_DEBUG,
            Category.replication,
            this,
            "A accept  ACK  p:"
                + preped
                + " a: "
                + acced
                + " -> "
                + msg.getProposalNo()
                + "="
                + msg.getLeaseHolder()
                + "/"
                + msg.getLeaseTimeout());
      }
      assert (msg.getLeaseHolder() != null);
      cc.setAccepted(msg);
      cc.setPrepared(msg);

      FleaseMessage tmp = new FleaseMessage(FleaseMessage.MsgType.MSG_ACCEPT_ACK, msg);
      tmp.setSendTimestamp(TimeSync.getGlobalTime());
      return tmp;
    }
  }
Exemplo n.º 13
0
  public void shutdown() {
    Logging.logMessage(Logging.LEVEL_DEBUG, this, "shutting down testEnv...");

    if (enabledServs.contains(Services.MRC) && mrc != null) {
      try {
        mrc.shutdown();
      } catch (Throwable th) {
        th.printStackTrace();
      }
    }

    if (enabledServs.contains(Services.OSD)) {
      try {
        for (TestOSD osd : osds.values()) {
          if (osd != null && osd.started) {
            osd.shutdown();
          }
        }
      } catch (Throwable th) {
        th.printStackTrace();
      }
    }

    if (enabledServs.contains(Services.UUID_RESOLVER)) {
      try {
        UUIDResolver.shutdown();
      } catch (Throwable th) {
      }
    }

    if (enabledServs.contains(Services.DIR_SERVICE) && dirService != null) {
      try {
        dirService.shutdown();
        dirService.waitForShutdown();
      } catch (Throwable th) {
        th.printStackTrace();
      }
    }

    try {
      getRpcClient().shutdown();
      getRpcClient().waitForShutdown();
    } catch (Throwable th) {
      th.printStackTrace();
    }

    if (enabledServs.contains(Services.TIME_SYNC)) {
      try {
        tsInstance = TimeSync.getInstance();
        if (tsInstance != null) {
          tsInstance.shutdown();
          tsInstance.waitForShutdown();
        }
      } catch (Throwable th) {
      }
    }

    // cleanup
    File testDir = new File(SetupUtils.TEST_DIR);
    // FSUtils.delTree(testDir);
  }
Exemplo n.º 14
0
  public void start() throws Exception {
    try {
      // ensure that TEST_DIR is empty
      File testDir = new File(SetupUtils.TEST_DIR);
      FSUtils.delTree(testDir);
      testDir.mkdirs();

      rpcClient = SetupUtils.createRPCClient(10000);
      getRpcClient().start();
      getRpcClient().waitForStartup();

      dirClient = SetupUtils.createDIRClient(getRpcClient());

      if (enabledServs.contains(Services.DIR_SERVICE)) {
        dirService =
            new DIRRequestDispatcher(SetupUtils.createDIRConfig(), SetupUtils.createDIRdbsConfig());
        dirService.startup();
        dirService.waitForStartup();
        Logging.logMessage(Logging.LEVEL_DEBUG, this, "DIR running");
      }

      if (enabledServs.contains(Services.TIME_SYNC) || enabledServs.contains(Services.MOCKUP_OSD)) {
        tsInstance = TimeSync.initializeLocal(50);
        tsInstance.waitForStartup();
      }

      if (enabledServs.contains(Services.UUID_RESOLVER)) {
        DIRClient dc =
            new DIRClient(dirClient, new InetSocketAddress[] {getDIRAddress()}, 10, 1000 * 5);
        UUIDResolver.start(dc, 1000, 10 * 10 * 1000);
        SetupUtils.localResolver();
      }

      if (enabledServs.contains(Services.MOCKUP_OSD)) {
        Map<String, String> dmap = new HashMap();
        dmap.put("free", "1000000000");
        dmap.put("total", "1000000000");
        dmap.put("load", "0");
        dmap.put("totalRAM", "1000000000");
        dmap.put("usedRAM", "0");
        dmap.put("proto_version", "" + OSDServiceConstants.INTERFACE_ID);
        Service reg =
            Service.newBuilder()
                .setType(ServiceType.SERVICE_TYPE_OSD)
                .setName("mockUpOSD")
                .setUuid("mockUpOSD")
                .setVersion(0)
                .setLastUpdatedS(0)
                .setData(ServiceDataMap.newBuilder().addAllData(KeyValuePairs.fromMap(dmap)))
                .build();
        RPCResponse<serviceRegisterResponse> response =
            dirClient.xtreemfs_service_register(
                null, RPCAuthentication.authNone, RPCAuthentication.userService, reg);
        response.get();
        response.freeBuffers();

        UUIDResolver.addLocalMapping("mockUpOSD", 11111, Schemes.SCHEME_PBRPC);
      }

      if (enabledServs.contains(Services.MOCKUP_OSD2)) {
        Map<String, String> dmap = new HashMap();
        dmap.put("free", "1000000000");
        dmap.put("total", "1000000000");
        dmap.put("load", "0");
        dmap.put("totalRAM", "1000000000");
        dmap.put("usedRAM", "0");
        dmap.put("proto_version", "" + OSDServiceConstants.INTERFACE_ID);
        Service reg =
            Service.newBuilder()
                .setType(ServiceType.SERVICE_TYPE_OSD)
                .setName("mockUpOSD2")
                .setUuid("mockUpOSD2")
                .setVersion(0)
                .setLastUpdatedS(0)
                .setData(ServiceDataMap.newBuilder().addAllData(KeyValuePairs.fromMap(dmap)))
                .build();
        RPCResponse<serviceRegisterResponse> response =
            dirClient.xtreemfs_service_register(
                null, RPCAuthentication.authNone, RPCAuthentication.userService, reg);
        response.get();
        response.freeBuffers();

        UUIDResolver.addLocalMapping("mockUpOSD2", 11111, Schemes.SCHEME_PBRPC);
      }

      if (enabledServs.contains(Services.MOCKUP_OSD3)) {
        Map<String, String> dmap = new HashMap();
        dmap.put("free", "1000000000");
        dmap.put("total", "1000000000");
        dmap.put("load", "0");
        dmap.put("totalRAM", "1000000000");
        dmap.put("usedRAM", "0");
        dmap.put("proto_version", "" + OSDServiceConstants.INTERFACE_ID);
        Service reg =
            Service.newBuilder()
                .setType(ServiceType.SERVICE_TYPE_OSD)
                .setName("mockUpOSD3")
                .setUuid("mockUpOSD3")
                .setVersion(0)
                .setLastUpdatedS(0)
                .setData(ServiceDataMap.newBuilder().addAllData(KeyValuePairs.fromMap(dmap)))
                .build();
        RPCResponse<serviceRegisterResponse> response =
            dirClient.xtreemfs_service_register(
                null, RPCAuthentication.authNone, RPCAuthentication.userService, reg);
        response.get();
        response.freeBuffers();

        UUIDResolver.addLocalMapping("mockUpOSD3", 11111, Schemes.SCHEME_PBRPC);
      }

      if (enabledServs.contains(Services.OSD)) {
        int osdCount = Collections.frequency(enabledServs, Services.OSD);
        osds = new HashMap<String, TestOSD>(osdCount);
        osdConfigs = SetupUtils.createMultipleOSDConfigs(osdCount);
        for (OSDConfig config : osdConfigs) {
          TestOSD osd = new TestOSD(new OSDRequestDispatcher(config));
          osd.start();
          osds.put(config.getUUID().toString(), osd);
        }

        // Save address of first OSD for getOSDAdress method.
        firstOSDAddress = osdConfigs[0].getUUID().getAddress();

        Logging.logMessage(Logging.LEVEL_DEBUG, this, "OSDs 1-" + osdCount + " running");
      }

      if (enabledServs.contains(Services.MRC)) {
        mrc =
            new MRCRequestDispatcher(
                SetupUtils.createMRC1Config(), SetupUtils.createMRC1dbsConfig());
        mrc.startup();
        Logging.logMessage(Logging.LEVEL_DEBUG, this, "MRC running");
      }

      if (enabledServs.contains(Services.MRC_CLIENT)) {
        mrcClient = new MRCServiceClient(rpcClient, null);
      }

      if (enabledServs.contains(Services.OSD_CLIENT)) {
        osdClient = new OSDServiceClient(rpcClient, null);
      }
    } catch (Exception ex) {
      ex.printStackTrace();
      // Shutdown servers which were already started or they will block ports.
      shutdown();

      // After shutdown, log remaining threads in case of blocked ports to debug the issue.
      if (ex instanceof BindException && ex.getMessage().contains("Address already in use")) {
        Logging.logMessage(
            Logging.LEVEL_ERROR,
            this,
            "TestEnvironment could not be started because: "
                + ex.getMessage()
                + " Please examine the following dump of threads to check if a previous test method did not correctly stop all servers.");
        StringBuilder threadStates = new StringBuilder();
        CrashReporter.reportThreadStates(threadStates);
        Logging.logMessage(Logging.LEVEL_ERROR, this, "Thread States: %s", threadStates.toString());
      }

      throw ex;
    }
  }