public static Properties getProperties(IZKClient client, String file, Properties defaultValue)
     throws InterruptedException, KeeperException, IOException {
   try {
     String v = ZooKeeperRetriableUtils.get(client, file);
     if (v != null) {
       return DataStoreHelpers.toProperties(v);
     } else {
       return defaultValue;
     }
   } catch (KeeperException.NoNodeException e) {
     return defaultValue;
   }
 }
  public void createCluster(List<String> containers, CreateEnsembleOptions options) {
    try {
      if (options.getZookeeperPassword() != null) {
        // do nothing
      } else if (System.getProperties().containsKey(SystemProperties.ZOOKEEPER_PASSWORD)) {
        options.setZookeeperPassword(System.getProperty(SystemProperties.ZOOKEEPER_PASSWORD));
      } else {
        options.setZookeeperPassword(ZooKeeperRetriableUtils.generatePassword());
      }

      if (containers == null || containers.size() == 2) {
        throw new IllegalArgumentException(
            "One or at least 3 containers must be used to create a zookeeper ensemble");
      }
      Configuration config =
          configurationAdmin.getConfiguration("org.fusesource.fabric.zookeeper", null);
      String zooKeeperUrl =
          config != null && config.getProperties() != null
              ? (String) config.getProperties().get("zookeeper.url")
              : null;
      if (zooKeeperUrl == null) {
        if (containers.size() != 1
            || !containers.get(0).equals(System.getProperty(SystemProperties.KARAF_NAME))) {
          throw new FabricException(
              "The first zookeeper cluster must be configured on this container only.");
        }
        createLocalServer(2181, options);
        return;
      }

      String version = getDataStore().getDefaultVersion();

      for (String container : containers) {
        Container c = fabricService.getContainer(container);
        if (ZooKeeperRetriableUtils.exists(zooKeeper, ZkPath.CONTAINER_ALIVE.getPath(container))
            == null) {
          throw new FabricException("The container " + container + " is not alive");
        }
        String containerVersion =
            zooKeeper.getStringData(ZkPath.CONFIG_CONTAINER.getPath(container));
        if (!version.equals(containerVersion)) {
          throw new FabricException(
              "The container " + container + " is not using the default-version:" + version);
        }
      }

      // Find used zookeeper ports
      Map<String, List<Integer>> usedPorts = new HashMap<String, List<Integer>>();
      String oldClusterId =
          ZooKeeperRetriableUtils.get(zooKeeper, ZkPath.CONFIG_ENSEMBLES.getPath());
      if (oldClusterId != null) {
        String profile = "fabric-ensemble-" + oldClusterId;
        String pid = "org.fusesource.fabric.zookeeper.server-" + oldClusterId;

        Map<String, String> p = getDataStore().getConfiguration(version, profile, pid);

        if (p == null) {
          throw new FabricException(
              "Failed to find old cluster configuration for ID " + oldClusterId);
        }

        for (Object n : p.keySet()) {
          String node = (String) n;
          if (node.startsWith("server.")) {
            String data =
                ZooKeeperRetriableUtils.getSubstitutedPath(
                    zooKeeper,
                    ZkPath.CONFIG_ENSEMBLE_PROFILE.getPath("fabric-ensemble-" + oldClusterId)
                        + "/org.fusesource.fabric.zookeeper.server-"
                        + oldClusterId
                        + ".properties#"
                        + node);
            addUsedPorts(usedPorts, data);
          }
        }

        Map<String, String> zkConfig =
            getDataStore().getConfiguration(version, "default", "org.fusesource.fabric.zookeeper");
        if (zkConfig == null) {
          throw new FabricException(
              "Failed to find old zookeeper configuration in default profile");
        }
        String datas =
            ZooKeeperRetriableUtils.getSubstitutedPath(zooKeeper, zkConfig.get("zookeeper.url"));
        for (String data : datas.split(",")) {
          addUsedPorts(usedPorts, data);
        }
      }

      String newClusterId;
      if (oldClusterId == null) {
        newClusterId = "0000";
      } else {
        newClusterId = new DecimalFormat("0000").format(Integer.parseInt(oldClusterId) + 1);
      }

      // create new ensemble
      String ensembleProfile =
          getDataStore().getProfile(version, "fabric-ensemble-" + newClusterId, true);
      getDataStore().setProfileAttribute(version, ensembleProfile, "abstract", "true");
      getDataStore().setProfileAttribute(version, ensembleProfile, "hidden", "true");

      Properties ensembleProperties = new Properties();
      ensembleProperties.put("tickTime", "2000");
      ensembleProperties.put("initLimit", "10");
      ensembleProperties.put("syncLimit", "5");
      ensembleProperties.put("dataDir", "data/zookeeper/" + newClusterId);

      int index = 1;
      String connectionUrl = "";
      String realConnectionUrl = "";
      String containerList = "";
      for (String container : containers) {
        String ip =
            ZooKeeperRetriableUtils.getSubstitutedPath(
                zooKeeper, ZkPath.CONTAINER_IP.getPath(container));

        String minimumPort = String.valueOf(Ports.MIN_PORT_NUMBER);
        String maximumPort = String.valueOf(Ports.MAX_PORT_NUMBER);

        if (zooKeeper.exists(ZkPath.CONTAINER_PORT_MIN.getPath(container)) != null) {
          minimumPort =
              ZooKeeperRetriableUtils.getSubstitutedPath(
                  zooKeeper, ZkPath.CONTAINER_PORT_MIN.getPath(container));
        }

        if (zooKeeper.exists(ZkPath.CONTAINER_PORT_MAX.getPath(container)) != null) {
          maximumPort =
              ZooKeeperRetriableUtils.getSubstitutedPath(
                  zooKeeper, ZkPath.CONTAINER_PORT_MAX.getPath(container));
        }

        String ensembleMemberPid =
            "org.fusesource.fabric.zookeeper.server-" + newClusterId + ".properties";
        Properties ensembleMemberProperties = new Properties();

        // configure this server in the ensemble
        String ensembleMemberProfile =
            getDataStore()
                .getProfile(
                    version,
                    "fabric-ensemble-" + newClusterId + "-" + Integer.toString(index),
                    true);
        getDataStore().setProfileAttribute(version, ensembleMemberProfile, "hidden", "true");
        getDataStore()
            .setProfileAttribute(version, ensembleMemberProfile, "parents", ensembleProfile);

        String port1 =
            Integer.toString(
                findPort(
                    usedPorts,
                    ip,
                    mapPortToRange(Ports.DEFAULT_ZOOKEEPER_SERVER_PORT, minimumPort, maximumPort)));
        if (containers.size() > 1) {
          String port2 =
              Integer.toString(
                  findPort(
                      usedPorts,
                      ip,
                      mapPortToRange(Ports.DEFAULT_ZOOKEEPER_PEER_PORT, minimumPort, maximumPort)));
          String port3 =
              Integer.toString(
                  findPort(
                      usedPorts,
                      ip,
                      mapPortToRange(
                          Ports.DEFAULT_ZOOKEEPER_ELECTION_PORT, minimumPort, maximumPort)));
          ensembleProperties.put(
              "server." + Integer.toString(index),
              "${zk:" + container + "/ip}:" + port2 + ":" + port3);
          ensembleMemberProperties.put("server.id", Integer.toString(index));
        }
        ensembleMemberProperties.put("clientPort", port1);

        getDataStore()
            .setFileConfiguration(
                version,
                ensembleMemberProfile,
                ensembleMemberPid,
                DataStoreHelpers.toBytes(ensembleMemberProperties));

        // add this container to the ensemble
        ZooKeeperRetriableUtils.add(
            zooKeeper,
            "/fabric/configs/versions/" + version + "/containers/" + container,
            "fabric-ensemble-" + newClusterId + "-" + Integer.toString(index));
        if (connectionUrl.length() > 0) {
          connectionUrl += ",";
          realConnectionUrl += ",";
        }
        connectionUrl += "${zk:" + container + "/ip}:" + port1;
        realConnectionUrl += ip + ":" + port1;
        if (containerList.length() > 0) {
          containerList += ",";
        }
        containerList += container;
        index++;
      }

      String ensembleConfigName = "org.fusesource.fabric.zookeeper.server-" + newClusterId;
      getDataStore()
          .setFileConfiguration(
              version,
              ensembleProfile,
              ensembleConfigName,
              DataStoreHelpers.toBytes(ensembleProperties));

      if (oldClusterId != null) {
        Properties properties =
            DataStoreHelpers.toProperties(
                getDataStore()
                    .getConfiguration(version, "default", "org.fusesource.fabric.zookeeper"));
        properties.put(
            "zookeeper.url",
            ZooKeeperRetriableUtils.getSubstitutedData(zooKeeper, realConnectionUrl));
        properties.put("zookeeper.password", options.getZookeeperPassword());
        OsgiZkClient dst = new OsgiZkClient();
        dst.updated(properties);
        try {
          dst.waitForConnected(new Timespan(30, Timespan.TimeUnit.SECOND));

          ZooKeeperRetriableUtils.copy(zooKeeper, dst, "/fabric/registry");
          ZooKeeperRetriableUtils.copy(zooKeeper, dst, "/fabric/authentication");
          ZooKeeperRetriableUtils.copy(zooKeeper, dst, "/fabric/configs");

          // Make sure that the alive zndoe is deleted for each container.
          for (String container : containers) {
            ZookeeperCommandBuilder.delete("/fabric/registry/containers/alive/" + container)
                .execute(dst);
          }

          ZooKeeperRetriableUtils.set(dst, ZkPath.CONFIG_ENSEMBLES.getPath(), newClusterId);
          ZooKeeperRetriableUtils.set(
              dst, ZkPath.CONFIG_ENSEMBLE.getPath(newClusterId), containerList);

          ZooKeeperRetriableUtils.set(dst, ZkPath.CONFIG_ENSEMBLE_URL.getPath(), connectionUrl);
          ZooKeeperRetriableUtils.set(
              dst, ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath(), options.getZookeeperPassword());
          ZooKeeperRetriableUtils.set(
              zooKeeper, ZkPath.CONFIG_ENSEMBLE_URL.getPath(), connectionUrl);
          ZooKeeperRetriableUtils.set(
              zooKeeper, ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath(), options.getZookeeperPassword());

          for (String v : zooKeeper.getChildren("/fabric/configs/versions/")) {
            for (String container :
                dst.getChildren("/fabric/configs/versions/" + v + "/containers")) {
              ZooKeeperRetriableUtils.remove(
                  dst,
                  "/fabric/configs/versions/" + v + "/containers/" + container,
                  "fabric-ensemble-" + oldClusterId + "-.*");
            }
            setConfigProperty(
                dst,
                "/fabric/configs/versions/"
                    + v
                    + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
                "zookeeper.password",
                "${zk:" + ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath() + "}");
            setConfigProperty(
                dst,
                "/fabric/configs/versions/"
                    + v
                    + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
                "zookeeper.url",
                "${zk:" + ZkPath.CONFIG_ENSEMBLE_URL.getPath() + "}");
            setConfigProperty(
                zooKeeper,
                "/fabric/configs/versions/"
                    + v
                    + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
                "zookeeper.password",
                "${zk:" + ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath() + "}");
            setConfigProperty(
                zooKeeper,
                "/fabric/configs/versions/"
                    + v
                    + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
                "zookeeper.url",
                "${zk:" + ZkPath.CONFIG_ENSEMBLE_URL.getPath() + "}");
          }

        } finally {
          dst.close();
        }
      } else {
        setConfigProperty(
            zooKeeper,
            "/fabric/configs/versions/"
                + version
                + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
            "zookeeper.password",
            "${zk:" + ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath() + "}");
        setConfigProperty(
            zooKeeper,
            "/fabric/configs/versions/"
                + version
                + "/profiles/default/org.fusesource.fabric.zookeeper.properties",
            "zookeeper.url",
            "${zk:" + ZkPath.CONFIG_ENSEMBLE_URL.getPath() + "}");
      }
    } catch (Exception e) {
      throw new FabricException("Unable to create zookeeper quorum: " + e.getMessage(), e);
    }
  }
  public void createLocalServer(int port, CreateEnsembleOptions options) {
    try {
      IZKClient client;
      Hashtable<String, Object> properties;
      String version = ZkDefs.DEFAULT_VERSION;
      String karafName = System.getProperty(SystemProperties.KARAF_NAME);
      String minimumPort = System.getProperty(ZkDefs.MINIMUM_PORT);
      String maximumPort = System.getProperty(ZkDefs.MAXIMUM_PORT);
      int mappedPort = mapPortToRange(port, minimumPort, maximumPort);

      if (options.getZookeeperPassword() != null) {
        // do nothing
      } else if (System.getProperties().containsKey(SystemProperties.ZOOKEEPER_PASSWORD)) {
        options.setZookeeperPassword(System.getProperty(SystemProperties.ZOOKEEPER_PASSWORD));
      } else {
        options.setZookeeperPassword(ZooKeeperRetriableUtils.generatePassword());
      }

      // Install or stop the fabric-configadmin bridge
      Bundle bundleFabricAgent =
          findAndStopBundle(bundleContext, "org.fusesource.fabric.fabric-agent");
      Bundle bundleFabricConfigAdmin =
          installOrStopBundle(
              bundleContext,
              "org.fusesource.fabric.fabric-configadmin",
              "mvn:org.fusesource.fabric/fabric-configadmin/" + FabricConstants.FABRIC_VERSION);
      Bundle bundleFabricZooKeeper =
          installOrStopBundle(
              bundleContext,
              "org.fusesource.fabric.fabric-zookeeper",
              "mvn:org.fusesource.fabric/fabric-zookeeper/" + FabricConstants.FABRIC_VERSION);
      Bundle bundleFabricJaas =
          installOrStopBundle(
              bundleContext,
              "org.fusesource.fabric.fabric-jaas  ",
              "mvn:org.fusesource.fabric/fabric-jaas/" + FabricConstants.FABRIC_VERSION);
      Bundle bundleFabricCommands =
          installOrStopBundle(
              bundleContext,
              "org.fusesource.fabric.fabric-commands  ",
              "mvn:org.fusesource.fabric/fabric-commands/" + FabricConstants.FABRIC_VERSION);
      Bundle bundleFabricMavenProxy =
          installOrStopBundle(
              bundleContext,
              "org.fusesource.fabric.fabric-commands  ",
              "mvn:org.fusesource.fabric/fabric-maven-proxy/" + FabricConstants.FABRIC_VERSION);

      // Create configuration
      String connectionUrl = HostUtils.getLocalHostName() + ":" + Integer.toString(mappedPort);

      String autoImportFrom = System.getProperty(SystemProperties.PROFILES_AUTOIMPORT_PATH);

      Configuration config =
          configurationAdmin.createFactoryConfiguration("org.fusesource.fabric.zookeeper.server");
      properties = new Hashtable<String, Object>();
      if (autoImportFrom != null) {
        loadPropertiesFrom(
            properties,
            autoImportFrom
                + "/fabric/configs/versions/1.0/profiles/default/org.fusesource.fabric.zookeeper.server.properties");
      }
      properties.put("tickTime", "2000");
      properties.put("initLimit", "10");
      properties.put("syncLimit", "5");
      properties.put("dataDir", "data/zookeeper/0000");
      properties.put("clientPort", Integer.toString(mappedPort));
      properties.put("fabric.zookeeper.pid", "org.fusesource.fabric.zookeeper.server-0000");
      config.setBundleLocation(null);
      config.update(properties);

      // Update the client configuration
      config = configurationAdmin.getConfiguration("org.fusesource.fabric.zookeeper");
      properties = new Hashtable<String, Object>();
      if (autoImportFrom != null) {
        loadPropertiesFrom(
            properties,
            autoImportFrom
                + "/fabric/configs/versions/1.0/profiles/default/org.fusesource.fabric.zookeeper.properties");
      }
      properties.put("zookeeper.url", connectionUrl);
      properties.put(
          "zookeeper.timeout",
          System.getProperties().containsKey("zookeeper.timeout")
              ? System.getProperties().getProperty("zookeeper.timeout")
              : "30000");
      properties.put("fabric.zookeeper.pid", "org.fusesource.fabric.zookeeper");
      properties.put("zookeeper.password", options.getZookeeperPassword());
      config.setBundleLocation(null);
      config.update(properties);

      // Start fabric-zookeeper bundle
      bundleFabricZooKeeper.start();

      // Wait for the client to be available
      ServiceTracker tracker = new ServiceTracker(bundleContext, IZKClient.class.getName(), null);
      tracker.open();
      client = (IZKClient) tracker.waitForService(5000);
      if (client == null) {
        throw new IllegalStateException("Timeout waiting for ZooKeeper client to be registered");
      }
      tracker.close();
      client.waitForConnected();

      // Import data into zookeeper
      if (autoImportFrom != null) {
        getDataStore().importFromFileSystem(autoImportFrom);
      }

      getDataStore().setDefaultVersion(version);

      // configure default profile
      String defaultProfile = getDataStore().getProfile(version, "default", true);

      ZooKeeperRetriableUtils.set(
          client,
          ZkPath.CONFIG_ENSEMBLE_URL.getPath(),
          "${zk:" + karafName + "/ip}:" + Integer.toString(mappedPort));
      ZooKeeperRetriableUtils.set(
          client, ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath(), options.getZookeeperPassword());

      Properties zkProps = new Properties();
      zkProps.setProperty("zookeeper.url", "${zk:" + ZkPath.CONFIG_ENSEMBLE_URL.getPath() + "}");
      zkProps.setProperty(
          "zookeeper.password", "${zk:" + ZkPath.CONFIG_ENSEMBLE_PASSWORD.getPath() + "}");
      getDataStore()
          .setFileConfiguration(
              version,
              defaultProfile,
              "org.fusesource.fabric.zookeeper.properties",
              DataStoreHelpers.toBytes(zkProps));

      // configure the ensemble
      String ensembleProfile = getDataStore().getProfile(version, "fabric-ensemble-0000", true);
      getDataStore().setProfileAttribute(version, ensembleProfile, "abstract", "true");
      getDataStore().setProfileAttribute(version, ensembleProfile, "hidden", "true");

      Properties ensembleProps = new Properties();
      ensembleProps.put("tickTime", "2000");
      ensembleProps.put("initLimit", "10");
      ensembleProps.put("syncLimit", "5");
      ensembleProps.put("dataDir", "data/zookeeper/0000");

      loadPropertiesFrom(
          ensembleProps,
          autoImportFrom
              + "/fabric/configs/versions/1.0/profiles/default/org.fusesource.fabric.zookeeper.server.properties");
      getDataStore()
          .setFileConfiguration(
              version,
              ensembleProfile,
              "org.fusesource.fabric.zookeeper.server-0000.properties",
              DataStoreHelpers.toBytes(ensembleProps));

      // configure this server in the ensemble
      String ensembleServerProfile =
          getDataStore().getProfile(version, "fabric-ensemble-0000-1", true);
      getDataStore().setProfileAttribute(version, ensembleServerProfile, "hidden", "true");
      getDataStore()
          .setProfileAttribute(version, ensembleServerProfile, "parents", ensembleProfile);
      Properties serverProps = new Properties();
      serverProps.put("clientPort", String.valueOf(mappedPort));
      getDataStore()
          .setFileConfiguration(
              version,
              ensembleServerProfile,
              "org.fusesource.fabric.zookeeper.server-0000.properties",
              DataStoreHelpers.toBytes(serverProps));

      ZooKeeperRetriableUtils.set(client, ZkPath.CONFIG_ENSEMBLES.getPath(), "0000");
      ZooKeeperRetriableUtils.set(client, ZkPath.CONFIG_ENSEMBLE.getPath("0000"), karafName);

      // configure fabric profile
      String fabricProfile = getDataStore().getProfile(version, "fabric", true);
      Properties agentProps =
          DataStoreHelpers.toProperties(
              getDataStore()
                  .getFileConfiguration(
                      version, fabricProfile, "org.fusesource.fabric.agent.properties"));
      agentProps.put("feature.fabric-commands", "fabric-commands");
      getDataStore()
          .setFileConfiguration(
              version,
              "fabric",
              "org.fusesource.fabric.agent.properties",
              DataStoreHelpers.toBytes(agentProps));

      ZooKeeperRetriableUtils.createDefault(
          client, ZkPath.CONFIG_CONTAINER.getPath(karafName), version);
      String assignedProfile = System.getProperty(SystemProperties.PROFILE);
      if (assignedProfile != null
          && !assignedProfile.isEmpty()
          && !"fabric".equals(assignedProfile)) {
        ZooKeeperRetriableUtils.createDefault(
            client,
            ZkPath.CONFIG_VERSIONS_CONTAINER.getPath(version, karafName),
            "fabric fabric-ensemble-0000-1 " + assignedProfile);
      } else {
        ZooKeeperRetriableUtils.createDefault(
            client,
            ZkPath.CONFIG_VERSIONS_CONTAINER.getPath(version, karafName),
            "fabric fabric-ensemble-0000-1");
      }

      // add auth
      Map<String, String> configs = new HashMap<String, String>();
      configs.put("encryption.enabled", "${zk:/fabric/authentication/encryption.enabled}");
      getDataStore()
          .setConfiguration(version, defaultProfile, "org.fusesource.fabric.jaas", configs);

      // outside of the profile storage area, so we'll keep these in zk
      ZooKeeperRetriableUtils.createDefault(
          client, "/fabric/authentication/encryption.enabled", "true");
      ZooKeeperRetriableUtils.createDefault(client, "/fabric/authentication/domain", "karaf");
      addUsersToZookeeper(client, options.getUsers());

      ZooKeeperRetriableUtils.createDefault(
          client, ZkPath.AUTHENTICATION_CRYPT_ALGORITHM.getPath(), "PBEWithMD5AndDES");
      ZooKeeperRetriableUtils.createDefault(
          client, ZkPath.AUTHENTICATION_CRYPT_PASSWORD.getPath(), options.getZookeeperPassword());

      // Fix acls
      ZookeeperCommandBuilder.fixAcls("/", true).execute(client);

      // Reset the autostart flag
      if (ensembleAutoStart) {
        System.setProperty(SystemProperties.ENSEMBLE_AUTOSTART, Boolean.FALSE.toString());
        File file = new File(System.getProperty("karaf.base") + "/etc/system.properties");
        org.apache.felix.utils.properties.Properties props =
            new org.apache.felix.utils.properties.Properties(file);
        props.put(SystemProperties.ENSEMBLE_AUTOSTART, Boolean.FALSE.toString());
        props.save();
      }

      // Restart fabric-configadmin bridge
      bundleFabricConfigAdmin.start();
      bundleFabricJaas.start();
      bundleFabricCommands.start();
      bundleFabricMavenProxy.start();

      // Check if the agent is configured to auto start.
      if (!System.getProperties().containsKey(SystemProperties.AGENT_AUTOSTART)
          || Boolean.parseBoolean(System.getProperty(SystemProperties.AGENT_AUTOSTART))) {
        bundleFabricAgent =
            findOrInstallBundle(
                bundleContext,
                "org.fusesource.fabric.fabric-agent  ",
                "mvn:org.fusesource.fabric/fabric-agent/" + FabricConstants.FABRIC_VERSION);
        bundleFabricAgent.start();
      }
    } catch (Exception e) {
      throw new FabricException("Unable to create zookeeper server configuration", e);
    }
  }