public SenseiServerBuilder(File confDir, Map<String, Object> properties) throws Exception {
if (properties != null) {
_senseiConfFile = null;
_senseiConf = new MapConfiguration(properties);
((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
} else {
_senseiConfFile = new File(confDir, SENSEI_PROPERTIES);
if (!_senseiConfFile.exists()) {
throw new ConfigurationException(
"configuration file: " + _senseiConfFile.getAbsolutePath() + " does not exist.");
}
_senseiConf = new PropertiesConfiguration();
((PropertiesConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
((PropertiesConfiguration) _senseiConf).load(_senseiConfFile);
}
pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
pluginRegistry.start();
processRelevanceFunctionPlugins(pluginRegistry);
processRelevanceExternalObjectPlugins(pluginRegistry);
_gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);
_schemaDoc = loadSchema(confDir);
_senseiSchema = SenseiSchema.build(_schemaDoc);
}
@Override
public void init(ServletConfig config) throws ServletException {
ServletContext ctx = config.getServletContext();
String confDirName =
ctx.getInitParameter(SenseiConfigServletContextListener.SENSEI_CONF_DIR_PARAM);
if (confDirName == null) {
throw new ServletException(
"configuration not specified, "
+ SenseiConfigServletContextListener.SENSEI_CONF_DIR_PARAM
+ " not set");
}
SenseiServerBuilder builder;
try {
builder = new SenseiServerBuilder(new File(confDirName), null);
ctx.setAttribute("sensei.search.configuration", builder.getConfiguration());
ctx.setAttribute("sensei.search.version.comparator", builder.getVersionComparator());
SenseiPluginRegistry pluginRegistry = builder.getPluginRegistry();
PartitionedLoadBalancerFactory<String> routerFactory =
pluginRegistry.getBeanByFullPrefix(
SenseiConfParams.SERVER_SEARCH_ROUTER_FACTORY, PartitionedLoadBalancerFactory.class);
if (routerFactory == null) {
routerFactory = new SenseiPartitionedLoadBalancerFactory(50);
}
ctx.setAttribute("sensei.search.router.factory", routerFactory);
_senseiServer = builder.buildServer();
_senseiServer.start(true);
super.init(config);
} catch (Exception e) {
throw new ServletException(e.getMessage(), e);
}
}
public SenseiServerBuilder(Resource confDir, Map<String, Object> properties) throws Exception {
_senseiConfFile = null;
_senseiConf = new MapConfiguration(properties);
((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
pluginRegistry.start();
processRelevanceFunctionPlugins(pluginRegistry);
_gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);
_schemaDoc = loadSchema(confDir);
_senseiSchema = SenseiSchema.build(_schemaDoc);
}
@BeforeClass
public static void init() throws Exception {
final ShutdownRegistryImpl shutdownRegistry = new ShutdownRegistryImpl();
try {
zkTestServer =
new ZooKeeperTestServer(0, shutdownRegistry, ZooKeeperTestServer.DEFAULT_SESSION_TIMEOUT);
port = zkTestServer.startNetwork();
} catch (Exception e) {
throw new RuntimeException(e);
}
Properties kafkaProps = new Properties();
kafkaProps.setProperty("num.partitions", "1");
kafkaProps.setProperty("port", "9092");
kafkaProps.setProperty("broker.id", "0");
kafkaProps.setProperty("log.dir", "/tmp/sensei-gateway-test-kafka-logs");
// override to the local running zk server
kafkaProps.setProperty("zookeeper.connect", "localhost:" + port);
kafkaLogFile = new File(kafkaProps.getProperty("log.dir"));
FileUtils.deleteDirectory(kafkaLogFile);
KafkaConfig kafkaConfig = new KafkaConfig(kafkaProps);
kafkaServer = new KafkaServerStartable(kafkaConfig);
kafkaServer.startup();
Configuration config = new PropertiesConfiguration();
config.addProperty(
"sensei.gateway.class", "com.senseidb.gateway.kafka.KafkaDataProviderBuilder");
config.addProperty("sensei.gateway.kafka.group.id", "1");
config.addProperty("sensei.gateway.kafka.zookeeper.connect", "localhost:" + port);
config.addProperty("sensei.gateway.kafka.auto.offset.reset", "smallest");
config.addProperty("sensei.gateway.kafka.topic", "test");
config.addProperty("sensei.gateway.provider.batchSize", "1");
pluginRegistry = SenseiPluginRegistry.build(config);
pluginRegistry.start();
kafkaGateway = pluginRegistry.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
kafkaGateway.start();
config = new PropertiesConfiguration();
config.addProperty("sensei.gateway.class", "com.senseidb.gateway.kafka.SimpleKafkaGateway");
config.addProperty("sensei.gateway.kafka.host", "localhost");
config.addProperty("sensei.gateway.kafka.port", "9092");
config.addProperty("sensei.gateway.kafka.topic", "test");
config.addProperty("sensei.gateway.kafka.timeout", "3000");
config.addProperty("sensei.gateway.provider.batchSize", "1");
pluginRegistry2 = SenseiPluginRegistry.build(config);
pluginRegistry2.start();
simpleKafkaGateway = pluginRegistry2.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
simpleKafkaGateway.start();
Properties props = new Properties();
props.put("metadata.broker.list", "localhost:9092");
props.put("serializer.class", "kafka.serializer.StringEncoder");
ProducerConfig producerConfig = new ProducerConfig(props);
Producer<String, String> kafkaProducer = new Producer<String, String>(producerConfig);
for (JSONObject jsonObj : BaseGatewayTestUtil.dataList) {
KeyedMessage<String, String> data =
new KeyedMessage<String, String>("test", jsonObj.toString());
kafkaProducer.send(data);
}
}
@SuppressWarnings("rawtypes")
private SenseiZoieFactory<?> constructZoieFactory(
ZoieConfig zoieConfig,
List<FacetHandler<?>> facetHandlers,
List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories,
ZoieIndexableInterpreter interpreter)
throws ConfigurationException {
String indexerType = _senseiConf.getString(SENSEI_INDEXER_TYPE, "zoie");
decorator = new SenseiIndexReaderDecorator(facetHandlers, runtimeFacetHandlerFactories);
File idxDir = new File(_senseiConf.getString(SENSEI_INDEX_DIR));
SenseiZoieFactory<?> zoieSystemFactory = null;
DIRECTORY_MODE dirMode;
String modeValue = _senseiConf.getString(SENSEI_INDEXER_MODE, "SIMPLE");
if ("SIMPLE".equalsIgnoreCase(modeValue)) {
dirMode = DIRECTORY_MODE.SIMPLE;
} else if ("NIO".equalsIgnoreCase(modeValue)) {
dirMode = DIRECTORY_MODE.NIO;
} else if ("MMAP".equalsIgnoreCase(modeValue)) {
dirMode = DIRECTORY_MODE.MMAP;
} else {
logger.error("directory mode " + modeValue + " is not supported, SIMPLE is used.");
dirMode = DIRECTORY_MODE.SIMPLE;
}
if (SENSEI_INDEXER_TYPE_ZOIE.equals(indexerType)) {
SenseiZoieSystemFactory senseiZoieFactory =
new SenseiZoieSystemFactory(idxDir, dirMode, interpreter, decorator, zoieConfig);
int retentionDays = _senseiConf.getInt(SENSEI_ZOIE_RETENTION_DAYS, -1);
if (retentionDays > 0) {
RetentionFilterFactory retentionFilterFactory =
pluginRegistry.getBeanByFullPrefix(
SENSEI_ZOIE_RETENTION_CLASS, RetentionFilterFactory.class);
Filter purgeFilter = null;
if (retentionFilterFactory != null) {
purgeFilter = retentionFilterFactory.buildRetentionFilter(retentionDays);
} else {
String timeColumn = _senseiConf.getString(SENSEI_ZOIE_RETENTION_COLUMN, null);
if (timeColumn == null) {
throw new ConfigurationException("Retention specified without a time column");
}
String unitString = _senseiConf.getString(SENSEI_ZOIE_RETENTION_TIMEUNIT, "seconds");
TimeUnit unit = TimeUnit.valueOf(unitString.toUpperCase());
if (unit == null) {
throw new ConfigurationException("Invalid timeunit for retention: " + unitString);
}
purgeFilter = new TimeRetentionFilter(timeColumn, retentionDays, unit);
}
if (purgeFilter != null && pluggableSearchEngineManager != null) {
purgeFilter = new PurgeFilterWrapper(purgeFilter, pluggableSearchEngineManager);
}
senseiZoieFactory.setPurgeFilter(purgeFilter);
}
zoieSystemFactory = senseiZoieFactory;
} else if (SENSEI_INDEXER_TYPE_HOURGLASS.equals(indexerType)) {
String schedule = _senseiConf.getString(SENSEI_HOURGLASS_SCHEDULE, "");
int trimThreshold = _senseiConf.getInt(SENSEI_HOURGLASS_TRIMTHRESHOLD, 14);
String frequencyString = _senseiConf.getString(SENSEI_HOURGLASS_FREQUENCY, "day");
FREQUENCY frequency;
if (SENSEI_HOURGLASS_FREQUENCY_MIN.equals(frequencyString)) {
frequency = FREQUENCY.MINUTELY;
} else if (SENSEI_HOURGLASS_FREQUENCY_HOUR.equals(frequencyString)) {
frequency = FREQUENCY.HOURLY;
} else if (SENSEI_HOURGLASS_FREQUENCY_DAY.equals(frequencyString)) {
frequency = FREQUENCY.DAILY;
} else {
throw new ConfigurationException("unsupported frequency setting: " + frequencyString);
}
boolean appendOnly = _senseiConf.getBoolean(SENSEI_HOURGLASS_APPENDONLY, true);
zoieSystemFactory =
new SenseiHourglassFactory(
idxDir,
dirMode,
interpreter,
decorator,
zoieConfig,
schedule,
appendOnly,
trimThreshold,
frequency,
pluggableSearchEngineManager != null
? Arrays.asList(pluggableSearchEngineManager)
: Collections.EMPTY_LIST);
} else {
ZoieFactoryFactory zoieFactoryFactory =
pluginRegistry.getBeanByFullPrefix(indexerType, ZoieFactoryFactory.class);
if (zoieFactoryFactory == null) {
throw new ConfigurationException(indexerType + " not defined");
}
zoieSystemFactory =
zoieFactoryFactory.getZoieFactory(idxDir, interpreter, decorator, zoieConfig);
}
String indexerCopier = _senseiConf.getString(SENSEI_INDEXER_COPIER);
IndexCopier copier =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEXER_COPIER, IndexCopier.class);
if (copier != null) {
zoieSystemFactory =
new SenseiPairFactory(
idxDir, dirMode, copier, interpreter, decorator, zoieConfig, zoieSystemFactory);
} else if (SENSEI_INDEXER_COPIER_HDFS.equals(indexerCopier)) {
zoieSystemFactory =
new SenseiPairFactory(
idxDir,
dirMode,
new HDFSIndexCopier(),
interpreter,
decorator,
zoieConfig,
zoieSystemFactory);
} else {
// do not support bootstrap index from other sources.
}
return zoieSystemFactory;
}
public SenseiCore buildCore() throws ConfigurationException {
SenseiUncaughtExceptionHandler.setAsDefaultForAllThreads();
int nodeid = _senseiConf.getInt(NODE_ID);
String partStr = _senseiConf.getString(PARTITIONS);
String[] partitionArray = partStr.split("[,\\s]+");
int[] partitions = buildPartitions(partitionArray);
logger.info("partitions to serve: " + Arrays.toString(partitions));
// Analyzer from configuration:
Analyzer analyzer = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_ANALYZER, Analyzer.class);
if (analyzer == null) {
analyzer = new StandardAnalyzer(Version.LUCENE_35);
}
// Similarity from configuration:
Similarity similarity =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_SIMILARITY, Similarity.class);
if (similarity == null) {
similarity = new DefaultSimilarity();
}
ZoieConfig zoieConfig;
if (_gateway != null) {
zoieConfig = new ZoieConfig(_gateway.getVersionComparator());
} else {
zoieConfig = new ZoieConfig();
}
zoieConfig.setAnalyzer(analyzer);
zoieConfig.setSimilarity(similarity);
zoieConfig.setBatchSize(
_senseiConf.getInt(SENSEI_INDEX_BATCH_SIZE, ZoieConfig.DEFAULT_SETTING_BATCHSIZE));
zoieConfig.setBatchDelay(
_senseiConf.getLong(SENSEI_INDEX_BATCH_DELAY, ZoieConfig.DEFAULT_SETTING_BATCHDELAY));
zoieConfig.setMaxBatchSize(
_senseiConf.getInt(SENSEI_INDEX_BATCH_MAXSIZE, ZoieConfig.DEFAULT_MAX_BATCH_SIZE));
zoieConfig.setRtIndexing(
_senseiConf.getBoolean(SENSEI_INDEX_REALTIME, ZoieConfig.DEFAULT_SETTING_REALTIME));
zoieConfig.setSkipBadRecord(_senseiConf.getBoolean(SENSEI_SKIP_BAD_RECORDS, false));
int delay = _senseiConf.getInt(SENSEI_INDEX_FRESHNESS, 10);
ReaderCacheFactory readercachefactory;
if (delay > 0) {
readercachefactory = DefaultReaderCache.FACTORY;
zoieConfig.setFreshness(delay * 1000);
} else {
readercachefactory = SimpleReaderCache.FACTORY;
}
zoieConfig.setReadercachefactory(readercachefactory);
ShardingStrategy strategy =
pluginRegistry.getBeanByFullPrefix(SENSEI_SHARDING_STRATEGY, ShardingStrategy.class);
if (strategy == null) {
strategy = new ShardingStrategy.FieldModShardingStrategy(_senseiSchema.getUidField());
}
pluggableSearchEngineManager = new PluggableSearchEngineManager();
pluggableSearchEngineManager.init(
_senseiConf.getString(SENSEI_INDEX_DIR),
nodeid,
_senseiSchema,
zoieConfig.getVersionComparator(),
pluginRegistry,
strategy);
List<FacetHandler<?>> facetHandlers = new LinkedList<FacetHandler<?>>();
List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories =
new LinkedList<RuntimeFacetHandlerFactory<?, ?>>();
SenseiSystemInfo sysInfo = null;
try {
sysInfo =
SenseiFacetHandlerBuilder.buildFacets(
_schemaDoc,
pluginRegistry,
facetHandlers,
runtimeFacetHandlerFactories,
pluggableSearchEngineManager);
} catch (JSONException jse) {
throw new ConfigurationException(jse.getMessage(), jse);
}
if (sysInfo != null) {
sysInfo.setSchema(_schemaDoc.toString());
try {
List<SenseiSystemInfo.SenseiNodeInfo> clusterInfo = new ArrayList(1);
String addr = NetUtil.getHostAddress();
clusterInfo.add(
new SenseiSystemInfo.SenseiNodeInfo(
nodeid,
partitions,
String.format("%s:%d", addr, _senseiConf.getInt(SERVER_PORT)),
String.format("http://%s:%d", addr, _senseiConf.getInt(SERVER_BROKER_PORT))));
sysInfo.setClusterInfo(clusterInfo);
} catch (Exception e) {
throw new ConfigurationException(e.getMessage(), e);
}
}
ZoieIndexableInterpreter interpreter =
pluginRegistry.getBeanByFullPrefix(
SENSEI_INDEX_INTERPRETER, ZoieIndexableInterpreter.class);
if (interpreter == null) {
DefaultJsonSchemaInterpreter defaultInterpreter =
new DefaultJsonSchemaInterpreter(_senseiSchema, pluggableSearchEngineManager);
interpreter = defaultInterpreter;
CustomIndexingPipeline customIndexingPipeline =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_CUSTOM, CustomIndexingPipeline.class);
if (customIndexingPipeline != null) {
try {
defaultInterpreter.setCustomIndexingPipeline(customIndexingPipeline);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
SenseiZoieFactory<?> zoieSystemFactory =
constructZoieFactory(zoieConfig, facetHandlers, runtimeFacetHandlerFactories, interpreter);
SenseiIndexingManager<?> indexingManager =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_MANAGER, SenseiIndexingManager.class);
if (indexingManager == null) {
indexingManager =
new DefaultStreamingIndexingManager(
_senseiSchema,
_senseiConf,
pluginRegistry,
_gateway,
strategy,
pluggableSearchEngineManager);
}
SenseiQueryBuilderFactory queryBuilderFactory =
pluginRegistry.getBeanByFullPrefix(
SENSEI_QUERY_BUILDER_FACTORY, SenseiQueryBuilderFactory.class);
if (queryBuilderFactory == null) {
QueryParser queryParser = new QueryParser(Version.LUCENE_35, "contents", analyzer);
queryBuilderFactory = new DefaultJsonQueryBuilderFactory(queryParser);
}
SenseiCore senseiCore =
new SenseiCore(
nodeid, partitions, zoieSystemFactory, indexingManager, queryBuilderFactory, decorator);
senseiCore.setSystemInfo(sysInfo);
SenseiIndexPruner indexPruner =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_PRUNER, SenseiIndexPruner.class);
if (indexPruner != null) {
senseiCore.setIndexPruner(indexPruner);
}
if (pluggableSearchEngineManager != null) {
senseiCore.setPluggableSearchEngineManager(pluggableSearchEngineManager);
}
return senseiCore;
}
public Server buildHttpRestServer() throws Exception {
int port = _senseiConf.getInt(SERVER_BROKER_PORT);
String webappPath =
_senseiConf.getString(SERVER_BROKER_WEBAPP_PATH, "sensei-core/src/main/webapp");
Server server = new Server();
QueuedThreadPool threadPool = new QueuedThreadPool();
threadPool.setName("Sensei Broker(jetty) threads");
threadPool.setMinThreads(_senseiConf.getInt(SERVER_BROKER_MINTHREAD, 20));
threadPool.setMaxThreads(_senseiConf.getInt(SERVER_BROKER_MAXTHREAD, 50));
threadPool.setMaxIdleTimeMs(_senseiConf.getInt(SERVER_BROKER_MAXWAIT, 2000));
// threadPool.start();
server.setThreadPool(threadPool);
logger.info("request threadpool started.");
SelectChannelConnector connector = new SelectChannelConnector();
connector.setPort(port);
server.addConnector(connector);
DefaultSenseiJSONServlet senseiServlet = new DefaultSenseiJSONServlet();
ServletHolder senseiServletHolder = new ServletHolder(senseiServlet);
SenseiHttpInvokerServiceServlet springServlet = new SenseiHttpInvokerServiceServlet();
ServletHolder springServletHolder = new ServletHolder(springServlet);
AgentServlet jmxServlet = new AgentServlet();
ServletHolder jmxServletHolder = new ServletHolder(jmxServlet);
WebAppContext senseiApp = new WebAppContext();
senseiApp.addFilter(GzipFilter.class, "/" + SENSEI_CONTEXT_PATH + "/*", 1);
// HashMap<String, String> initParam = new HashMap<String, String>();
// if (_senseiConfFile != null) {
// logger.info("Broker Configuration file: "+_senseiConfFile.getAbsolutePath());
// initParam.put("config.file", _senseiConfFile.getAbsolutePath());
// }
// senseiApp.setInitParams(initParam);
senseiApp.setAttribute("sensei.search.configuration", _senseiConf);
senseiApp.setAttribute(
SenseiConfigServletContextListener.SENSEI_CONF_PLUGIN_REGISTRY, pluginRegistry);
senseiApp.setAttribute(
"sensei.search.version.comparator",
_gateway != null ? _gateway.getVersionComparator() : ZoieConfig.DEFAULT_VERSION_COMPARATOR);
PartitionedLoadBalancerFactory<String> routerFactory =
pluginRegistry.getBeanByFullPrefix(
SenseiConfParams.SERVER_SEARCH_ROUTER_FACTORY, PartitionedLoadBalancerFactory.class);
if (routerFactory == null) {
routerFactory = new SenseiPartitionedLoadBalancerFactory(50);
}
senseiApp.setAttribute("sensei.search.router.factory", routerFactory);
senseiApp.addEventListener(new SenseiConfigServletContextListener());
senseiApp.addServlet(senseiServletHolder, "/" + SENSEI_CONTEXT_PATH + "/*");
senseiApp.setResourceBase(webappPath);
senseiApp.addServlet(springServletHolder, "/sensei-rpc/SenseiSpringRPCService");
senseiApp.addServlet(jmxServletHolder, "/admin/jmx/*");
server.setHandler(senseiApp);
server.setStopAtShutdown(true);
return server;
}