/**
  * Removes the containers from the cluster.
  *
  * @param containers
  */
 @Override
 public void removeFromCluster(List<String> containers, CreateEnsembleOptions options) {
   try {
     List<String> current = getEnsembleContainers();
     current.removeAll(containers);
     createCluster(current, options);
   } catch (Exception e) {
     throw new FabricException(
         "Unable to remove containers to fabric ensemble: " + e.getMessage(), e);
   }
 }
  public void createCluster(List<String> containers, CreateEnsembleOptions options) {
    try {
      if (options.getZookeeperPassword() != null) {
        // do nothing
      } else if (System.getProperties().containsKey(SystemProperties.ZOOKEEPER_PASSWORD)) {
        options.setZookeeperPassword(System.getProperty(SystemProperties.ZOOKEEPER_PASSWORD));
      } else {
        options.setZookeeperPassword(ZooKeeperRetriableUtils.generatePassword());
      }

      if (containers == null || containers.size() == 2) {
        throw new IllegalArgumentException(
            "One or at least 3 containers must be used to create a zookeeper ensemble");
      }
      Configuration config =
          configurationAdmin.getConfiguration("org.fusesource.fabric.zookeeper", null);
      String zooKeeperUrl =
          config != null && config.getProperties() != null
              ? (String) config.getProperties().get("zookeeper.url")
              : null;
      if (zooKeeperUrl == null) {
        if (containers.size() != 1
            || !containers.get(0).equals(System.getProperty(SystemProperties.KARAF_NAME))) {
          throw new FabricException(
              "The first zookeeper cluster must be configured on this container only.");
        }
        createLocalServer(2181, options);
        return;
      }

      String version = getDataStore().getDefaultVersion();

      for (String container : containers) {
        Container c = fabricService.getContainer(container);
        if (ZooKeeperRetriableUtils.exists(zooKeeper, ZkPath.CONTAINER_ALIVE.getPath(container))
            == null) {
          throw new FabricException("The container " + container + " is not alive");
        }
        String containerVersion =
            zooKeeper.getStringData(ZkPath.CONFIG_CONTAINER.getPath(container));
        if (!version.equals(containerVersion)) {
          throw new FabricException(
              "The container " + container + " is not using the default-version:" + version);
        }
      }

      // Find used zookeeper ports
      Map<String, List<Integer>> usedPorts = new HashMap<String, List<Integer>>();
      String oldClusterId =
          ZooKeeperRetriableUtils.get(zooKeeper, ZkPath.CONFIG_ENSEMBLES.getPath());
      if (oldClusterId != null) {
        String profile = "fabric-ensemble-" + oldClusterId;
        String pid = "org.fusesource.fabric.zookeeper.server-" + oldClusterId;

        Map<String, String> p = getDataStore().getConfiguration(version, profile, pid);

        if (p == null) {
          throw new FabricException(
              "Failed to find old cluster configuration for ID " + oldClusterId);
        }

        for (Object n : p.keySet()) {
          String node = (String) n;
          if (node.startsWith("server.")) {
            String data =
                ZooKeeperRetriableUtils.getSubstitutedPath(
                    zooKeeper,
                    ZkPath.CONFIG_ENSEMBLE_PROFILE.getPath("fabric-ensemble-" + oldClusterId)
                        + "/org.fusesource.fabric.zookeeper.server-"
                        + oldClusterId
                        + ".properties#"
                        + node);
            addUsedPorts(usedPorts, data);
          }
        }

        Map<String, String> zkConfig =
            getDataStore().getConfiguration(version, "default", "org.fusesource.fabric.zookeeper");
        if (zkConfig == null) {
          throw new FabricException(
              "Failed to find old zookeeper configuration in default profile");
        }
        String datas =
            ZooKeeperRetriableUtils.getSubstitutedPath(zooKeeper, zkConfig.get("zookeeper.url"));
        for (String data : datas.split(",")) {
          addUsedPorts(usedPorts, data);
        }
      }

      String newClusterId;
      if (oldClusterId == null) {
        newClusterId = "0000";
      } else {
        newClusterId = new DecimalFormat("0000").format(Integer.parseInt(oldClusterId) + 1);
      }

      // create new ensemble
      String ensembleProfile =
          getDataStore().getProfile(version, "fabric-ensemble-" + newClusterId, true);
      getDataStore().setProfileAttribute(version, ensembleProfile, "abstract", "true");
      getDataStore().setProfileAttribute(version, ensembleProfile, "hidden", "true");

      Properties ensembleProperties = new Properties();
      ensembleProperties.put("tickTime", "2000");
      ensembleProperties.put("initLimit", "10");
      ensembleProperties.put("syncLimit", "5");
      ensembleProperties.put("dataDir", "data/zookeeper/" + newClusterId);

      int index = 1;
      String connectionUrl = "";
      String realConnectionUrl = "";
      String containerList = "";
      for (String container : containers) {
        String ip =
            ZooKeeperRetriableUtils.getSubstitutedPath(
                zooKeeper, ZkPath.CONTAINER_IP.getPath(container));

        String minimumPort = String.valueOf(Ports.MIN_PORT_NUMBER);
        String maximumPort = String.valueOf(Ports.MAX_PORT_NUMBER);

        if (zooKeeper.exists(ZkPath.CONTAINER_PORT_MIN.getPath(container)) != null) {
          minimumPort =
              ZooKeeperRetriableUtils.getSubstitutedPath(
                  zooKeeper, ZkPath.CONTAINER_PORT_MIN.getPath(container));
        }

        if (zooKeeper.exists(ZkPath.CONTAINER_PORT_MAX.getPath(container)) != null) {
          maximumPort =
              ZooKeeperRetriableUtils.getSubstitutedPath(
                  zooKeeper, ZkPath.CONTAINER_PORT_MAX.getPath(container));
        }

        String ensembleMemberPid =
            "org.fusesource.fabric.zookeeper.server-" + newClusterId + ".properties";
        Properties ensembleMemberProperties = new Properties();

        // configure this server in the ensemble
        String ensembleMemberProfile =
            getDataStore()
                .getProfile(
                    version,
                    "fabric-ensemble-" + newClusterId + "-" + Integer.toString(index),
                    true);
        getDataStore().setProfileAttribute(version, ensembleMemberProfile, "hidden", "true");
        getDataStore()
            .setProfileAttribute(version, ensembleMemberProfile, "parents", ensembleProfile);

        String port1 =
            Integer.toString(
                findPort(
                    usedPorts,
                    ip,
                    mapPortToRange(Ports.DEFAULT_ZOOKEEPER_SERVER_PORT, minimumPort, maximumPort)));
        if (containers.size() > 1) {
          String port2 =
              Integer.toString(
                  findPort(
                      usedPorts,
                      ip,
                      mapPortToRange(Ports.DEFAULT_ZOOKEEPER_PEER_PORT, minimumPort, maximumPort)));
          String port3 =
              Integer.toString(
                  findPort(
                      usedPorts,
                      ip,
                      mapPortToRange(
                          Ports.DEFAULT_ZOOKEEPER_ELECTION_PORT, minimumPort, maximumPort)));
          ensembleProperties.put(
              "server." + Integer.toString(index),
              "${zk:" + container + "/ip}:" + port2 + ":" + port3);
          ensembleMemberProperties.put("server.id", Integer.toString(index));
        }
        ensembleMemberProperties.put("clientPort", port1);

        getDataStore()
            .setFileConfiguration(
                version,
                ensembleMemberProfile,
                ensembleMemberPid,
                DataStoreHelpers.toBytes(ensembleMemberProperties));

        // add this container to the ensemble
        ZooKeeperRetriableUtils.add(
            zooKeeper,
            "/fabric/configs/versions/" + version + "/containers/" + container,
            "fabric-ensemble-" + newClusterId + "-" + Integer.toString(index));
        if (connectionUrl.length() > 0) {
          connectionUrl += ",";
          realConnectionUrl += ",";
        }
        connectionUrl += "${zk:" + container + "/ip}:" + port1;
        realConnectionUrl += ip + ":" + port1;
        if (containerList.length() > 0) {
          containerList += ",";
        }
        containerList += container;
        index++;
      }

      String ensembleConfigName = "org.fusesource.fabric.zookeeper.server-" + newClusterId;
      getDataStore()
          .setFileConfiguration(
              version,
              ensembleProfile,
              ensembleConfigName,
              DataStoreHelpers.toBytes(ensembleProperties));

      if (oldClusterId != null) {
        Properties properties =
            DataStoreHelpers.toProperties(
                getDataStore()
                    .getConfiguration(version, "default", "org.fusesource.fabric.zookeeper"));
        properties.put(
            "zookeeper.url",
            ZooKeeperRetriableUtils.getSubstitutedData(zooKeeper, realConnectionUrl));
        properties.put("zookeeper.password", options.getZookeeperPassword());
        OsgiZkClient dst = new OsgiZkClient();
        dst.updated(properties);
        try {
          dst.waitForConnected(new Timespan(30, Timespan.TimeUnit.SECOND));

          ZooKeeperRetriableUtils.copy(zooKeeper, dst, "/fabric/registry");
          ZooKeeperRetriableUtils.copy(zooKeeper, dst, "/fabric/authentication");
          ZooKeeperRetriableUtils.copy(zooKeeper, dst, "/fabric/configs");

          // Make sure that the alive zndoe is deleted for each container.
          for (String container : containers) {
            ZookeeperCommandBuilder.delete("/fabric/registry/containers/alive/" + container)
                .execute(dst);
          }

          ZooKeeperRetriableUtils.set(dst, ZkPath.CONFIG_ENSEMBLES.getPath(), newClusterId);
          ZooKeeperRetriableUtils.set(
              dst, ZkPath.CONFIG_ENSEMBLE.getPath(newClusterId), containerList);

          ZooKeeperRetriableUtils.set(dst, ZkPath.CONFIG_ENSEMBLE_URL.getPath(), connectionUrl);
          ZooKeeperRetriableUtils.set(
              dst, ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath(), options.getZookeeperPassword());
          ZooKeeperRetriableUtils.set(
              zooKeeper, ZkPath.CONFIG_ENSEMBLE_URL.getPath(), connectionUrl);
          ZooKeeperRetriableUtils.set(
              zooKeeper, ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath(), options.getZookeeperPassword());

          for (String v : zooKeeper.getChildren("/fabric/configs/versions/")) {
            for (String container :
                dst.getChildren("/fabric/configs/versions/" + v + "/containers")) {
              ZooKeeperRetriableUtils.remove(
                  dst,
                  "/fabric/configs/versions/" + v + "/containers/" + container,
                  "fabric-ensemble-" + oldClusterId + "-.*");
            }
            setConfigProperty(
                dst,
                "/fabric/configs/versions/"
                    + v
                    + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
                "zookeeper.password",
                "${zk:" + ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath() + "}");
            setConfigProperty(
                dst,
                "/fabric/configs/versions/"
                    + v
                    + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
                "zookeeper.url",
                "${zk:" + ZkPath.CONFIG_ENSEMBLE_URL.getPath() + "}");
            setConfigProperty(
                zooKeeper,
                "/fabric/configs/versions/"
                    + v
                    + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
                "zookeeper.password",
                "${zk:" + ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath() + "}");
            setConfigProperty(
                zooKeeper,
                "/fabric/configs/versions/"
                    + v
                    + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
                "zookeeper.url",
                "${zk:" + ZkPath.CONFIG_ENSEMBLE_URL.getPath() + "}");
          }

        } finally {
          dst.close();
        }
      } else {
        setConfigProperty(
            zooKeeper,
            "/fabric/configs/versions/"
                + version
                + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
            "zookeeper.password",
            "${zk:" + ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath() + "}");
        setConfigProperty(
            zooKeeper,
            "/fabric/configs/versions/"
                + version
                + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
            "zookeeper.url",
            "${zk:" + ZkPath.CONFIG_ENSEMBLE_URL.getPath() + "}");
      }
    } catch (Exception e) {
      throw new FabricException("Unable to create zookeeper quorum: " + e.getMessage(), e);
    }
  }