public ClusterClient buildClusterClient() {
String clusterName = _senseiConf.getString(SENSEI_CLUSTER_NAME);
String clusterClientName = _senseiConf.getString(SENSEI_CLUSTER_CLIENT_NAME, clusterName);
String zkUrl = _senseiConf.getString(SENSEI_CLUSTER_URL);
int zkTimeout = _senseiConf.getInt(SENSEI_CLUSTER_TIMEOUT, 300000);
ClusterClient clusterClient =
new ZooKeeperClusterClient(clusterClientName, clusterName, zkUrl, zkTimeout);
logger.info("Connecting to cluster: " + clusterName + " ...");
clusterClient.awaitConnectionUninterruptibly();
logger.info("Cluster: " + clusterName + " successfully connected ");
return clusterClient;
}
private static void shutdownSensei() {
try {
broker.shutdown();
} catch (Throwable t) {
}
try {
httpRestSenseiService.shutdown();
} catch (Throwable t) {
}
try {
node1.shutdown();
} catch (Throwable t) {
}
try {
httpServer1.stop();
} catch (Throwable t) {
}
try {
node2.shutdown();
} catch (Throwable t) {
}
try {
httpServer2.stop();
} catch (Throwable t) {
}
try {
networkClient.shutdown();
} catch (Throwable t) {
}
try {
clusterClient.shutdown();
} catch (Throwable t) {
}
rmrf(new File(IndexDir));
}
@Override
public String getNodeStatistics() {
StringBuilder str = new StringBuilder("Nodes:");
for (Node node : clusterClient.getNodes()) {
str.append("[nodeId=" + node.getId() + ", partitions=[" + node.getPartitionIds() + "]],");
}
return str.toString();
}
public void shutdown() {
try {
logger.info("shutting down node...");
try {
_core.shutdown();
_clusterClient.removeNode(_id);
_clusterClient.shutdown();
_serverNode = null;
pluginRegistry.stop();
_core.getPluggableSearchEngineManager().close();
} catch (Exception e) {
logger.warn(e.getMessage());
} finally {
if (_networkServer != null) {
_networkServer.shutdown();
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
public boolean isAvailable() {
if (_serverNode != null && _serverNode.isAvailable() == _available) return _available;
try {
Thread.sleep(1000);
_serverNode = _clusterClient.getNodeWithId(_id);
if (_serverNode != null && _serverNode.isAvailable() == _available) return _available;
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
_available = (_serverNode != null ? _serverNode.isAvailable() : false);
return _available;
}
public SenseiBroker(
PartitionedNetworkClient<String> networkClient,
ClusterClient clusterClient,
boolean allowPartialMerge)
throws NorbertException {
super(networkClient, CoreSenseiServiceImpl.JAVA_SERIALIZER);
this.clusterClient = clusterClient;
this.allowPartialMerge = allowPartialMerge;
MBeanServer platformMBeanServer = ManagementFactory.getPlatformMBeanServer();
ObjectName name;
try {
name = new ObjectName("com.senseidb.search.node:type=SenseiBroker");
platformMBeanServer.registerMBean(this, name);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
clusterClient.addListener(this);
logger.info("created broker instance " + networkClient + " " + clusterClient);
}
@Override
public int getNumberOfNodes() {
return clusterClient.getNodes().size();
}
public void start(boolean available) throws Exception {
_core.start();
// ClusterClient clusterClient = ClusterClientFactory.newInstance().newZookeeperClient();
String clusterName = _clusterClient.getServiceName();
logger.info("Cluster Name: " + clusterName);
logger.info("Cluster info: " + _clusterClient.toString());
AbstractSenseiCoreService coreSenseiService = new CoreSenseiServiceImpl(_core);
AbstractSenseiCoreService sysSenseiCoreService = new SysSenseiCoreServiceImpl(_core);
// create the zookeeper cluster client
// SenseiClusterClientImpl senseiClusterClient = new SenseiClusterClientImpl(clusterName,
// zookeeperURL, zookeeperTimeout, false);
SenseiCoreServiceMessageHandler senseiMsgHandler =
new SenseiCoreServiceMessageHandler(coreSenseiService);
SenseiCoreServiceMessageHandler senseiSysMsgHandler =
new SenseiCoreServiceMessageHandler(sysSenseiCoreService);
_networkServer.registerHandler(senseiMsgHandler, coreSenseiService.getSerializer());
_networkServer.registerHandler(senseiSysMsgHandler, sysSenseiCoreService.getSerializer());
_networkServer.registerHandler(senseiMsgHandler, CoreSenseiServiceImpl.JAVA_SERIALIZER);
_networkServer.registerHandler(senseiSysMsgHandler, SysSenseiCoreServiceImpl.JAVA_SERIALIZER);
if (_externalSvc != null) {
for (AbstractSenseiCoreService svc : _externalSvc) {
_networkServer.registerHandler(
new SenseiCoreServiceMessageHandler(svc), svc.getSerializer());
}
}
HashSet<Integer> partition = new HashSet<Integer>();
for (int partId : _partitions) {
partition.add(partId);
}
boolean nodeExists = false;
try {
logger.info("waiting to connect to cluster...");
_clusterClient.awaitConnectionUninterruptibly();
_serverNode = _clusterClient.getNodeWithId(_id);
nodeExists = (_serverNode != null);
if (!nodeExists) {
String ipAddr = getLocalIpAddress();
logger.info("Node id : " + _id + " IP address : " + ipAddr);
_serverNode = _clusterClient.addNode(_id, ipAddr, partition);
logger.info("added node id: " + _id);
} else {
// node exists
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
throw e;
}
try {
logger.info("binding server ...");
_networkServer.bind(_id, available);
// exponential backoff
Thread.sleep(1000);
_available = available;
logger.info("started [markAvailable=" + available + "] ...");
if (nodeExists) {
logger.warn("existing node found, will try to overwrite.");
try {
// remove node above
_clusterClient.removeNode(_id);
_serverNode = null;
} catch (Exception e) {
logger.error("problem removing old node: " + e.getMessage(), e);
}
String ipAddr = getLocalIpAddress();
_serverNode = _clusterClient.addNode(_id, ipAddr, partition);
Thread.sleep(1000);
logger.info("added node id: " + _id);
}
} catch (NetworkingException e) {
logger.error(e.getMessage(), e);
try {
if (!nodeExists) {
_clusterClient.removeNode(_id);
_serverNode = null;
}
} catch (Exception ex) {
logger.warn(ex.getMessage());
} finally {
try {
_networkServer.shutdown();
_networkServer = null;
} finally {
_clusterClient.shutdown();
_clusterClient = null;
}
}
throw e;
}
SenseiServerAdminMBean senseiAdminMBean = getAdminMBean();
StandardMBean bean = new StandardMBean(senseiAdminMBean, SenseiServerAdminMBean.class);
JmxUtil.registerMBean(bean, "name", "sensei-server-" + _id);
}
/** Will start the new Sensei instance once per process */
public static synchronized void start(String confDir1, String confDir2) {
if (started) {
logger.warn("The server had been already started");
return;
}
try {
JmxSenseiMBeanServer.registerCustomMBeanServer();
ConfDir1 = new File(SenseiStarter.class.getClassLoader().getResource(confDir1).toURI());
ConfDir2 = new File(SenseiStarter.class.getClassLoader().getResource(confDir2).toURI());
org.apache.log4j.PropertyConfigurator.configure("resources/log4j.properties");
loadFromSpringContext();
boolean removeSuccessful = rmrf(new File(IndexDir));
if (!removeSuccessful) {
throw new IllegalStateException("The index dir " + IndexDir + " coulnd't be purged");
}
SenseiServerBuilder senseiServerBuilder1 = null;
senseiServerBuilder1 = new SenseiServerBuilder(ConfDir1, null);
node1 = senseiServerBuilder1.buildServer();
httpServer1 = senseiServerBuilder1.buildHttpRestServer();
logger.info("Node 1 created.");
SenseiServerBuilder senseiServerBuilder2 = null;
senseiServerBuilder2 = new SenseiServerBuilder(ConfDir2, null);
node2 = senseiServerBuilder2.buildServer();
httpServer2 = senseiServerBuilder2.buildHttpRestServer();
logger.info("Node 2 created.");
broker = null;
try {
broker = new SenseiBroker(networkClient, clusterClient, loadBalancerFactory);
broker.setTimeoutMillis(0);
mapReduceBroker = new MapReduceBroker(networkClient, clusterClient, loadBalancerFactory);
broker.setTimeoutMillis(0);
} catch (NorbertException ne) {
logger.info("shutting down cluster...", ne);
clusterClient.shutdown();
throw ne;
}
httpRestSenseiService = new HttpRestSenseiServiceImpl("http", "localhost", 8079, "/sensei");
logger.info("Cluster client started");
Runtime.getRuntime()
.addShutdownHook(
new Thread() {
@Override
public void run() {
shutdownSensei();
}
});
node1.start(true);
httpServer1.start();
logger.info("Node 1 started");
node2.start(true);
httpServer2.start();
logger.info("Node 2 started");
SenseiUrl = new URL("http://localhost:8079/sensei");
waitTillServerStarts();
} catch (Throwable ex) {
logger.error("Could not start the sensei", ex);
throw new RuntimeException(ex);
} finally {
started = true;
}
}