/**
* This constructor's argument is from cassandral's yaml configuration. Here is an example
* seed_provider: - class_name: com.emc.storageos.db.server.impl.SeedProviderImpl parameters: -
* coordinators: "coordinator://127.0.0.1:2181, coordinator://127.0.0.1:3181,
* coordinator://127.0.0.1:4181" id: "db-one
*
* @param args
* @throws Exception
*/
public SeedProviderImpl(Map<String, String> args) throws Exception {
// get current node id
_id = args.get(ID);
if (_id == null) {
throw new IllegalArgumentException(ID);
}
// seed nodes in remote data centers
String seedsArg = args.get(SEEDS);
String[] seedIPs = null;
if (seedsArg != null && !seedsArg.trim().isEmpty()) {
seedIPs = seedsArg.split(",", -1);
}
if (seedIPs != null) {
// multiple site - assume seeds in other site is available
// so just pick from config file
for (String ip : seedIPs) {
extraSeeds.add(ip);
}
}
// setup zk connection
String coordinatorArg = args.get(COORDINATORS);
if (coordinatorArg == null || coordinatorArg.trim().isEmpty()) {
throw new IllegalArgumentException(COORDINATORS);
}
String[] coordinators = coordinatorArg.split(",", -1);
List<URI> uri = new ArrayList<URI>(coordinators.length);
for (String coord : coordinators) {
if (!coord.trim().isEmpty()) {
uri.add(URI.create(coord.trim()));
}
}
ZkConnection connection = new ZkConnection();
connection.setServer(uri);
String siteId = args.get(Constants.SITE_ID_FILE);
connection.setSiteIdFile(siteId);
connection.build();
CoordinatorClientImpl client = new CoordinatorClientImpl();
client.setZkConnection(connection);
ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext("/nodeaddrmap-var.xml");
CoordinatorClientInetAddressMap inetAddressMap =
(CoordinatorClientInetAddressMap) ctx.getBean("inetAddessLookupMap");
if (inetAddressMap == null) {
_logger.error(
"CoordinatorClientInetAddressMap is not initialized. Node address lookup will fail.");
}
client.setInetAddessLookupMap(inetAddressMap); // HARCODE FOR NOW
client.start();
drUtil = new DrUtil(client);
isDrActiveSite = drUtil.isActiveSite();
_client = client;
}
public static DbClientImpl newDBClient() throws Exception {
ZkConnection zkConnection = new ZkConnection();
zkConnection.setServer(Lists.newArrayList(new URI("coordinator://localhost:2181")));
zkConnection.build();
DualInetAddress dualInetAddress = DualInetAddress.fromAddresses("127.0.0.1", "::1");
Map<String, DualInetAddress> addresses = Maps.newHashMap();
addresses.put("localhost", dualInetAddress);
CoordinatorClientInetAddressMap map = new CoordinatorClientInetAddressMap();
map.setNodeId("standalone");
map.setDualInetAddress(dualInetAddress);
map.setControllerNodeIPLookupMap(addresses);
CoordinatorClientImpl coordinatorClient = new CoordinatorClientImpl();
coordinatorClient.setZkConnection(zkConnection);
coordinatorClient.setInetAddessLookupMap(map);
coordinatorClient.start();
DbClientContext localContext = new DbClientContext();
localContext.setKeyspaceName("StorageOS");
localContext.setClusterName("StorageOs");
DbClientContext geoContext = new DbClientContext();
geoContext.setKeyspaceName("GeoStorageOs");
geoContext.setClusterName("GeoStorageOs");
DbVersionInfo versionInfo = new DbVersionInfo();
versionInfo.setSchemaVersion("2.0");
DbClientImpl client = new DbClientImpl();
client.setDbVersionInfo(versionInfo);
client.setLocalContext(localContext);
client.setGeoContext(geoContext);
client.setCoordinatorClient(coordinatorClient);
client.setLocalContext(new DbClientContext());
client.start();
VdcUtil.setDbClient(client);
return client;
}
private InterProcessReadWriteLock getLock() throws ClientGeneralException {
try {
if (lock == null) {
String lockName =
ComputeSessionUtil.Constants.COMPUTE_SESSION_BASE_PATH.toString() + "-" + hostAddress;
lock = coordinator.getReadWriteLock(lockName);
}
} catch (Exception e) {
throw new ClientGeneralException(ClientMessageKeys.DISTRIBUTED_LOCK_ERROR, e);
}
return lock;
}
private void initializeSession() {
distributedDataManager =
coordinator.createDistributedDataManager(
ComputeSessionUtil.Constants.COMPUTE_SESSION_BASE_PATH.toString());
}
/**
* We select seeds based on the following rules - For DR standby sites, use all nodes in active
* site as seeds For DR active site, use local nodes as seeds. The rule to select local seed is -
* first boot node(AUTOBOOT = false) uses itself as seed nodes so that it could boot and
* initialize schema - subsquent node(AUTOBOOT = true) uses other successfully booted(JOINED =
* true) nodes as seeds
*/
@Override
public List<InetAddress> getSeeds() {
try {
CoordinatorClientInetAddressMap nodeMap = _client.getInetAddessLookupMap();
List<Configuration> configs =
_client.queryAllConfiguration(_client.getSiteId(), Constants.DB_CONFIG);
List<InetAddress> seeds = new ArrayList<>();
// If we are upgrading from pre-2.5 releases, dbconfig exists in zk global area
List<Configuration> leftoverConfig = _client.queryAllConfiguration(Constants.DB_CONFIG);
configs.addAll(leftoverConfig);
// Add extra seeds - seeds from remote sites
for (String seed : extraSeeds) {
if (StringUtils.isNotEmpty(seed)) {
seeds.add(InetAddress.getByName(seed));
}
}
// On DR standby site, only use seeds from active site. On active site
// we use local seeds
if (isDrActiveSite) {
for (int i = 0; i < configs.size(); i++) {
Configuration config = configs.get(i);
// Bypasses item of "global" and folders of "version", just check db configurations.
if (config.getId() == null || config.getId().equals(Constants.GLOBAL_ID)) {
continue;
}
String nodeIndex = config.getId().split("-")[1];
String nodeId = config.getConfig(DbConfigConstants.NODE_ID);
if (nodeId == null) {
// suppose that they are existing znodes from a previous version
// set the NODE_ID config, id is like db-x
nodeId = "vipr" + nodeIndex;
config.setConfig(DbConfigConstants.NODE_ID, nodeId);
config.removeConfig(DbConfigConstants.DB_IP);
_client.persistServiceConfiguration(_client.getSiteId(), config);
}
if (!Boolean.parseBoolean(config.getConfig(DbConfigConstants.AUTOBOOT))
|| (!config.getId().equals(_id)
&& Boolean.parseBoolean(config.getConfig(DbConfigConstants.JOINED)))) {
// all non autobootstrap nodes + other nodes are used as seeds
InetAddress ip = null;
if (nodeMap != null) {
String ipAddress = nodeMap.getConnectableInternalAddress(nodeId);
_logger.debug("ip[" + i + "]: " + ipAddress);
ip = InetAddress.getByName(ipAddress);
} else {
ip = InetAddress.getByName(nodeId);
}
seeds.add(ip);
_logger.info("Seed {}", ip);
}
}
}
_logger.info("Seeds list {}", StringUtils.join(seeds.toArray(), ","));
return seeds;
} catch (Exception e) {
throw new IllegalStateException(e);
}
}