@Override
public void init(ServletConfig config) throws ServletException {
ServletContext ctx = config.getServletContext();
String confDirName =
ctx.getInitParameter(SenseiConfigServletContextListener.SENSEI_CONF_DIR_PARAM);
if (confDirName == null) {
throw new ServletException(
"configuration not specified, "
+ SenseiConfigServletContextListener.SENSEI_CONF_DIR_PARAM
+ " not set");
}
SenseiServerBuilder builder;
try {
builder = new SenseiServerBuilder(new File(confDirName), null);
ctx.setAttribute("sensei.search.configuration", builder.getConfiguration());
ctx.setAttribute("sensei.search.version.comparator", builder.getVersionComparator());
SenseiPluginRegistry pluginRegistry = builder.getPluginRegistry();
PartitionedLoadBalancerFactory<String> routerFactory =
pluginRegistry.getBeanByFullPrefix(
SenseiConfParams.SERVER_SEARCH_ROUTER_FACTORY, PartitionedLoadBalancerFactory.class);
if (routerFactory == null) {
routerFactory = new SenseiPartitionedLoadBalancerFactory(50);
}
ctx.setAttribute("sensei.search.router.factory", routerFactory);
_senseiServer = builder.buildServer();
_senseiServer.start(true);
super.init(config);
} catch (Exception e) {
throw new ServletException(e.getMessage(), e);
}
}
public SenseiServerBuilder(File confDir, Map<String, Object> properties) throws Exception {
if (properties != null) {
_senseiConfFile = null;
_senseiConf = new MapConfiguration(properties);
((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
} else {
_senseiConfFile = new File(confDir, SENSEI_PROPERTIES);
if (!_senseiConfFile.exists()) {
throw new ConfigurationException(
"configuration file: " + _senseiConfFile.getAbsolutePath() + " does not exist.");
}
_senseiConf = new PropertiesConfiguration();
((PropertiesConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
((PropertiesConfiguration) _senseiConf).load(_senseiConfFile);
}
pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
pluginRegistry.start();
processRelevanceFunctionPlugins(pluginRegistry);
processRelevanceExternalObjectPlugins(pluginRegistry);
_gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);
_schemaDoc = loadSchema(confDir);
_senseiSchema = SenseiSchema.build(_schemaDoc);
}
public SenseiServerBuilder(Resource confDir, Map<String, Object> properties) throws Exception {
_senseiConfFile = null;
_senseiConf = new MapConfiguration(properties);
((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
pluginRegistry.start();
processRelevanceFunctionPlugins(pluginRegistry);
_gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);
_schemaDoc = loadSchema(confDir);
_senseiSchema = SenseiSchema.build(_schemaDoc);
}
public static synchronized SenseiPluginRegistry build(Configuration conf) {
if (cachedRegistries.containsKey(conf)) {
return cachedRegistries.get(conf);
}
SenseiPluginRegistry ret = new SenseiPluginRegistry();
ret.configuration = conf;
Iterator<?> keysIterator = conf.getKeys();
while (keysIterator.hasNext()) {
String key = (String) keysIterator.next();
if (key.endsWith(".class")) {
String prefix = key.substring(0, key.indexOf(".class"));
String pluginName = getNameByPrefix(prefix);
String pluginClass = conf.getString(key);
ret.plugins.add(new PluginHolder(ret, pluginClass, pluginName, prefix));
}
if (key.endsWith(".instance")) {
String prefix = key.substring(0, key.indexOf(".instance"));
String pluginName = getNameByPrefix(prefix);
Object pluginInstance = conf.getProperty(key);
ret.plugins.add(new PluginHolder(ret, pluginInstance, pluginName, prefix));
}
}
for (PluginHolder pluginHolder : ret.plugins) {
ret.pluginsByPrefix.put(pluginHolder.fullPrefix, pluginHolder);
ret.pluginsByNames.put(pluginHolder.pluginName, pluginHolder);
Iterator<?> propertyIterator = conf.getKeys(pluginHolder.fullPrefix);
while (propertyIterator.hasNext()) {
String propertyName = (String) propertyIterator.next();
if (propertyName.endsWith(".class")) {
continue;
}
String property = propertyName;
if (propertyName.contains(pluginHolder.fullPrefix)) {
property = propertyName.substring(pluginHolder.fullPrefix.length() + 1);
}
pluginHolder.properties.put(property, conf.getProperty(propertyName).toString());
}
}
cachedRegistries.put(conf, ret);
return ret;
}
private void processRelevanceFunctionPlugins(SenseiPluginRegistry pluginRegistry) {
Map<String, CustomRelevanceFunctionFactory> map =
pluginRegistry.getNamedBeansByType(CustomRelevanceFunctionFactory.class);
Iterator<String> it = map.keySet().iterator();
while (it.hasNext()) {
String name = it.next();
CustomRelevanceFunctionFactory crf = map.get(name);
ModelStorage.injectPreloadedModel(name, crf);
}
}
@AfterClass
public static void shutdown() {
kafkaGateway.stop();
pluginRegistry.stop();
simpleKafkaGateway.stop();
pluginRegistry2.stop();
try {
if (kafkaServer != null) {
kafkaServer.shutdown();
kafkaServer.awaitShutdown();
}
zkTestServer.shutdownNetwork();
} finally {
try {
FileUtils.deleteDirectory(kafkaLogFile);
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Override
public void init(
String indexDirectory,
int nodeId,
SenseiSchema senseiSchema,
Comparator<String> versionComparator,
SenseiPluginRegistry pluginRegistry,
ShardingStrategy shardingStrategy) {
this.indexDirectory = indexDirectory;
this.nodeId = nodeId;
this.versionComparator = versionComparator;
this.shardingStrategy = shardingStrategy;
maxPartition =
pluginRegistry.getConfiguration().getInt("sensei.index.manager.default.maxpartition.id")
+ 1;
}
public SenseiServer buildServer() throws ConfigurationException {
int port = _senseiConf.getInt(SERVER_PORT);
JmxSenseiMBeanServer.registerCustomMBeanServer();
ClusterClient clusterClient = buildClusterClient();
NetworkServer networkServer = buildNetworkServer(_senseiConf, clusterClient);
SenseiCore core = buildCore();
List<AbstractSenseiCoreService<AbstractSenseiRequest, AbstractSenseiResult>> svcList =
(List)
pluginRegistry.resolveBeansByListKey(
SENSEI_PLUGIN_SVCS, AbstractSenseiCoreService.class);
return new SenseiServer(port, networkServer, clusterClient, core, svcList, pluginRegistry);
}
public PersistentCacheManager createpersistentCacheManager() {
PersistentCacheManager cacheManager = new PersistentCacheManager();
Map map = new HashMap();
map.put("sensei.index.manager.default.maxpartition.id", 0);
cacheManager.init(
indexDir.getAbsolutePath(),
1,
null,
ZoieConfig.DEFAULT_VERSION_COMPARATOR,
SenseiPluginRegistry.build(new MapConfiguration(map)),
new ShardingStrategy() {
@Override
public int caculateShard(int maxShardId, JSONObject dataObj) throws JSONException {
return 0;
}
});
cache = new PersistentCache(indexDir, ZoieConfig.DEFAULT_VERSION_COMPARATOR);
cacheManager.getPersistentCaches().put(0, cache);
return cacheManager;
}
public void shutdown() {
try {
logger.info("shutting down node...");
try {
_core.shutdown();
_clusterClient.removeNode(_id);
_clusterClient.shutdown();
_serverNode = null;
pluginRegistry.stop();
_core.getPluggableSearchEngineManager().close();
} catch (Exception e) {
logger.warn(e.getMessage());
} finally {
if (_networkServer != null) {
_networkServer.shutdown();
}
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
public static SenseiSystemInfo buildFacets(
JSONObject schemaObj,
SenseiPluginRegistry pluginRegistry,
List<FacetHandler<?>> facets,
List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacets,
PluggableSearchEngineManager pluggableSearchEngineManager)
throws JSONException, ConfigurationException {
Set<String> pluggableSearchEngineFacetNames = pluggableSearchEngineManager.getFacetNames();
SenseiSystemInfo sysInfo = new SenseiSystemInfo();
JSONArray facetsList = schemaObj.optJSONArray("facets");
int count = 0;
if (facetsList != null) {
count = facetsList.length();
}
if (count <= 0) {
return sysInfo;
}
JSONObject table = schemaObj.optJSONObject("table");
if (table == null) {
throw new ConfigurationException("Empty schema");
}
JSONArray columns = table.optJSONArray("columns");
Map<String, JSONObject> columnMap = new HashMap<String, JSONObject>();
for (int i = 0; i < columns.length(); ++i) {
JSONObject column = columns.getJSONObject(i);
try {
String name = column.getString("name");
columnMap.put(name, column);
} catch (Exception e) {
throw new ConfigurationException("Error parsing schema: ", e);
}
}
Map<String, TermListFactory<?>> termListFactoryMap = getPredefinedTermListFactoryMap(schemaObj);
Set<SenseiSystemInfo.SenseiFacetInfo> facetInfos =
new HashSet<SenseiSystemInfo.SenseiFacetInfo>();
for (int i = 0; i < count; ++i) {
JSONObject facet = facetsList.getJSONObject(i);
try {
String name = facet.getString("name");
if (UID_FACET_NAME.equals(name)) {
logger.error("facet name: " + UID_FACET_NAME + " is reserved, skipping...");
continue;
}
String type = facet.getString("type");
String fieldName = facet.optString("column", name);
Set<String> dependSet = new HashSet<String>();
JSONArray dependsArray = facet.optJSONArray("depends");
if (dependsArray != null) {
int depCount = dependsArray.length();
for (int k = 0; k < depCount; ++k) {
dependSet.add(dependsArray.getString(k));
}
}
SenseiSystemInfo.SenseiFacetInfo facetInfo = new SenseiSystemInfo.SenseiFacetInfo(name);
Map<String, String> facetProps = new HashMap<String, String>();
facetProps.put("type", type);
facetProps.put("column", fieldName);
JSONObject column = columnMap.get(fieldName);
String columnType = (column == null) ? "" : column.optString("type", "");
if (column != null && column.opt("activity") != null && column.optBoolean("activity")) {
if (columnType.equalsIgnoreCase("int")) {
columnType = "aint";
} else if (columnType.equalsIgnoreCase("long")) {
columnType = "along";
} else if (columnType.equalsIgnoreCase("float")) {
columnType = "afloat";
}
}
facetProps.put("column_type", columnType);
facetProps.put("depends", dependSet.toString());
JSONArray paramList = facet.optJSONArray("params");
Map<String, List<String>> paramMap = parseParams(paramList);
for (Entry<String, List<String>> entry : paramMap.entrySet()) {
facetProps.put(entry.getKey(), entry.getValue().toString());
}
facetInfo.setProps(facetProps);
facetInfos.add(facetInfo);
if (pluggableSearchEngineFacetNames.contains(name)) {
continue;
}
FacetHandler<?> facetHandler = null;
if (type.equals("simple")) {
facetHandler =
buildSimpleFacetHandler(
name, fieldName, dependSet, termListFactoryMap.get(fieldName));
} else if (type.equals("path")) {
facetHandler = buildPathHandler(name, fieldName, paramMap, false);
} else if (type.equals("multi-path")) {
facetHandler = buildPathHandler(name, fieldName, paramMap, true);
} else if (type.equals("range")) {
if (column.optBoolean("multi")) {
facetHandler =
new MultiRangeFacetHandler(
name,
fieldName,
null,
termListFactoryMap.get(fieldName),
buildPredefinedRanges(paramMap));
} else {
facetHandler =
buildRangeHandler(name, fieldName, termListFactoryMap.get(fieldName), paramMap);
}
} else if (type.equals("multi")) {
facetHandler =
buildMultiHandler(name, fieldName, termListFactoryMap.get(fieldName), dependSet);
} else if (type.equals("compact-multi")) {
facetHandler =
buildCompactMultiHandler(
name, fieldName, dependSet, termListFactoryMap.get(fieldName));
} else if (type.equals("weighted-multi")) {
facetHandler =
buildWeightedMultiHandler(
name, fieldName, termListFactoryMap.get(fieldName), dependSet);
} else if (type.equals("attribute")) {
facetHandler =
new AttributesFacetHandler(
name, fieldName, termListFactoryMap.get(fieldName), null, facetProps);
} else if (type.equals("histogram")) {
// A histogram facet handler is always dynamic
RuntimeFacetHandlerFactory<?, ?> runtimeFacetFactory =
getHistogramFacetHandlerFactory(facet, name, paramMap);
runtimeFacets.add(runtimeFacetFactory);
facetInfo.setRunTime(true);
} else if (type.equals("dynamicTimeRange")) {
if (dependSet.isEmpty()) {
Assert.isTrue(
fieldName != null && fieldName.length() > 0,
"Facet handler " + name + " requires either depends or column attributes");
RangeFacetHandler internalFacet =
new RangeFacetHandler(
name + "_internal",
fieldName,
new PredefinedTermListFactory<Long>(
Long.class, DynamicTimeRangeFacetHandler.NUMBER_FORMAT),
null);
facets.add(internalFacet);
dependSet.add(internalFacet.getName());
}
RuntimeFacetHandlerFactory<?, ?> runtimeFacetFactory =
getDynamicTimeFacetHandlerFactory(name, fieldName, dependSet, paramMap);
runtimeFacets.add(runtimeFacetFactory);
facetInfo.setRunTime(true);
} else if (type.equals("custom")) {
boolean isDynamic = facet.optBoolean("dynamic");
// Load from custom-facets spring configuration.
if (isDynamic) {
RuntimeFacetHandlerFactory<?, ?> runtimeFacetFactory =
pluginRegistry.getRuntimeFacet(name);
runtimeFacets.add(runtimeFacetFactory);
facetInfo.setRunTime(true);
} else {
facetHandler = pluginRegistry.getFacet(name);
}
} else {
throw new IllegalArgumentException("type not supported: " + type);
}
if (facetHandler != null) {
facets.add(facetHandler);
}
} catch (Exception e) {
throw new ConfigurationException("Error parsing schema: " + facet, e);
}
}
facets.addAll(pluggableSearchEngineManager.createFacetHandlers());
// uid facet handler to be added by default
UIDFacetHandler uidHandler = new UIDFacetHandler(UID_FACET_NAME);
SumGroupByFacetHandler sumGroupByFacetHandler =
new SumGroupByFacetHandler(SUM_GROUP_BY_FACET_NAME);
facets.add(uidHandler);
facets.add(sumGroupByFacetHandler);
sysInfo.setFacetInfos(facetInfos);
return sysInfo;
}
@BeforeClass
public static void init() throws Exception {
final ShutdownRegistryImpl shutdownRegistry = new ShutdownRegistryImpl();
try {
zkTestServer =
new ZooKeeperTestServer(0, shutdownRegistry, ZooKeeperTestServer.DEFAULT_SESSION_TIMEOUT);
port = zkTestServer.startNetwork();
} catch (Exception e) {
throw new RuntimeException(e);
}
Properties kafkaProps = new Properties();
kafkaProps.setProperty("num.partitions", "1");
kafkaProps.setProperty("port", "9092");
kafkaProps.setProperty("broker.id", "0");
kafkaProps.setProperty("log.dir", "/tmp/sensei-gateway-test-kafka-logs");
// override to the local running zk server
kafkaProps.setProperty("zookeeper.connect", "localhost:" + port);
kafkaLogFile = new File(kafkaProps.getProperty("log.dir"));
FileUtils.deleteDirectory(kafkaLogFile);
KafkaConfig kafkaConfig = new KafkaConfig(kafkaProps);
kafkaServer = new KafkaServerStartable(kafkaConfig);
kafkaServer.startup();
Configuration config = new PropertiesConfiguration();
config.addProperty(
"sensei.gateway.class", "com.senseidb.gateway.kafka.KafkaDataProviderBuilder");
config.addProperty("sensei.gateway.kafka.group.id", "1");
config.addProperty("sensei.gateway.kafka.zookeeper.connect", "localhost:" + port);
config.addProperty("sensei.gateway.kafka.auto.offset.reset", "smallest");
config.addProperty("sensei.gateway.kafka.topic", "test");
config.addProperty("sensei.gateway.provider.batchSize", "1");
pluginRegistry = SenseiPluginRegistry.build(config);
pluginRegistry.start();
kafkaGateway = pluginRegistry.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
kafkaGateway.start();
config = new PropertiesConfiguration();
config.addProperty("sensei.gateway.class", "com.senseidb.gateway.kafka.SimpleKafkaGateway");
config.addProperty("sensei.gateway.kafka.host", "localhost");
config.addProperty("sensei.gateway.kafka.port", "9092");
config.addProperty("sensei.gateway.kafka.topic", "test");
config.addProperty("sensei.gateway.kafka.timeout", "3000");
config.addProperty("sensei.gateway.provider.batchSize", "1");
pluginRegistry2 = SenseiPluginRegistry.build(config);
pluginRegistry2.start();
simpleKafkaGateway = pluginRegistry2.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
simpleKafkaGateway.start();
Properties props = new Properties();
props.put("metadata.broker.list", "localhost:9092");
props.put("serializer.class", "kafka.serializer.StringEncoder");
ProducerConfig producerConfig = new ProducerConfig(props);
Producer<String, String> kafkaProducer = new Producer<String, String>(producerConfig);
for (JSONObject jsonObj : BaseGatewayTestUtil.dataList) {
KeyedMessage<String, String> data =
new KeyedMessage<String, String>("test", jsonObj.toString());
kafkaProducer.send(data);
}
}
@SuppressWarnings("rawtypes")
private SenseiZoieFactory<?> constructZoieFactory(
ZoieConfig zoieConfig,
List<FacetHandler<?>> facetHandlers,
List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories,
ZoieIndexableInterpreter interpreter)
throws ConfigurationException {
String indexerType = _senseiConf.getString(SENSEI_INDEXER_TYPE, "zoie");
decorator = new SenseiIndexReaderDecorator(facetHandlers, runtimeFacetHandlerFactories);
File idxDir = new File(_senseiConf.getString(SENSEI_INDEX_DIR));
SenseiZoieFactory<?> zoieSystemFactory = null;
DIRECTORY_MODE dirMode;
String modeValue = _senseiConf.getString(SENSEI_INDEXER_MODE, "SIMPLE");
if ("SIMPLE".equalsIgnoreCase(modeValue)) {
dirMode = DIRECTORY_MODE.SIMPLE;
} else if ("NIO".equalsIgnoreCase(modeValue)) {
dirMode = DIRECTORY_MODE.NIO;
} else if ("MMAP".equalsIgnoreCase(modeValue)) {
dirMode = DIRECTORY_MODE.MMAP;
} else {
logger.error("directory mode " + modeValue + " is not supported, SIMPLE is used.");
dirMode = DIRECTORY_MODE.SIMPLE;
}
if (SENSEI_INDEXER_TYPE_ZOIE.equals(indexerType)) {
SenseiZoieSystemFactory senseiZoieFactory =
new SenseiZoieSystemFactory(idxDir, dirMode, interpreter, decorator, zoieConfig);
int retentionDays = _senseiConf.getInt(SENSEI_ZOIE_RETENTION_DAYS, -1);
if (retentionDays > 0) {
RetentionFilterFactory retentionFilterFactory =
pluginRegistry.getBeanByFullPrefix(
SENSEI_ZOIE_RETENTION_CLASS, RetentionFilterFactory.class);
Filter purgeFilter = null;
if (retentionFilterFactory != null) {
purgeFilter = retentionFilterFactory.buildRetentionFilter(retentionDays);
} else {
String timeColumn = _senseiConf.getString(SENSEI_ZOIE_RETENTION_COLUMN, null);
if (timeColumn == null) {
throw new ConfigurationException("Retention specified without a time column");
}
String unitString = _senseiConf.getString(SENSEI_ZOIE_RETENTION_TIMEUNIT, "seconds");
TimeUnit unit = TimeUnit.valueOf(unitString.toUpperCase());
if (unit == null) {
throw new ConfigurationException("Invalid timeunit for retention: " + unitString);
}
purgeFilter = new TimeRetentionFilter(timeColumn, retentionDays, unit);
}
if (purgeFilter != null && pluggableSearchEngineManager != null) {
purgeFilter = new PurgeFilterWrapper(purgeFilter, pluggableSearchEngineManager);
}
senseiZoieFactory.setPurgeFilter(purgeFilter);
}
zoieSystemFactory = senseiZoieFactory;
} else if (SENSEI_INDEXER_TYPE_HOURGLASS.equals(indexerType)) {
String schedule = _senseiConf.getString(SENSEI_HOURGLASS_SCHEDULE, "");
int trimThreshold = _senseiConf.getInt(SENSEI_HOURGLASS_TRIMTHRESHOLD, 14);
String frequencyString = _senseiConf.getString(SENSEI_HOURGLASS_FREQUENCY, "day");
FREQUENCY frequency;
if (SENSEI_HOURGLASS_FREQUENCY_MIN.equals(frequencyString)) {
frequency = FREQUENCY.MINUTELY;
} else if (SENSEI_HOURGLASS_FREQUENCY_HOUR.equals(frequencyString)) {
frequency = FREQUENCY.HOURLY;
} else if (SENSEI_HOURGLASS_FREQUENCY_DAY.equals(frequencyString)) {
frequency = FREQUENCY.DAILY;
} else {
throw new ConfigurationException("unsupported frequency setting: " + frequencyString);
}
boolean appendOnly = _senseiConf.getBoolean(SENSEI_HOURGLASS_APPENDONLY, true);
zoieSystemFactory =
new SenseiHourglassFactory(
idxDir,
dirMode,
interpreter,
decorator,
zoieConfig,
schedule,
appendOnly,
trimThreshold,
frequency,
pluggableSearchEngineManager != null
? Arrays.asList(pluggableSearchEngineManager)
: Collections.EMPTY_LIST);
} else {
ZoieFactoryFactory zoieFactoryFactory =
pluginRegistry.getBeanByFullPrefix(indexerType, ZoieFactoryFactory.class);
if (zoieFactoryFactory == null) {
throw new ConfigurationException(indexerType + " not defined");
}
zoieSystemFactory =
zoieFactoryFactory.getZoieFactory(idxDir, interpreter, decorator, zoieConfig);
}
String indexerCopier = _senseiConf.getString(SENSEI_INDEXER_COPIER);
IndexCopier copier =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEXER_COPIER, IndexCopier.class);
if (copier != null) {
zoieSystemFactory =
new SenseiPairFactory(
idxDir, dirMode, copier, interpreter, decorator, zoieConfig, zoieSystemFactory);
} else if (SENSEI_INDEXER_COPIER_HDFS.equals(indexerCopier)) {
zoieSystemFactory =
new SenseiPairFactory(
idxDir,
dirMode,
new HDFSIndexCopier(),
interpreter,
decorator,
zoieConfig,
zoieSystemFactory);
} else {
// do not support bootstrap index from other sources.
}
return zoieSystemFactory;
}
public SenseiCore buildCore() throws ConfigurationException {
SenseiUncaughtExceptionHandler.setAsDefaultForAllThreads();
int nodeid = _senseiConf.getInt(NODE_ID);
String partStr = _senseiConf.getString(PARTITIONS);
String[] partitionArray = partStr.split("[,\\s]+");
int[] partitions = buildPartitions(partitionArray);
logger.info("partitions to serve: " + Arrays.toString(partitions));
// Analyzer from configuration:
Analyzer analyzer = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_ANALYZER, Analyzer.class);
if (analyzer == null) {
analyzer = new StandardAnalyzer(Version.LUCENE_35);
}
// Similarity from configuration:
Similarity similarity =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_SIMILARITY, Similarity.class);
if (similarity == null) {
similarity = new DefaultSimilarity();
}
ZoieConfig zoieConfig;
if (_gateway != null) {
zoieConfig = new ZoieConfig(_gateway.getVersionComparator());
} else {
zoieConfig = new ZoieConfig();
}
zoieConfig.setAnalyzer(analyzer);
zoieConfig.setSimilarity(similarity);
zoieConfig.setBatchSize(
_senseiConf.getInt(SENSEI_INDEX_BATCH_SIZE, ZoieConfig.DEFAULT_SETTING_BATCHSIZE));
zoieConfig.setBatchDelay(
_senseiConf.getLong(SENSEI_INDEX_BATCH_DELAY, ZoieConfig.DEFAULT_SETTING_BATCHDELAY));
zoieConfig.setMaxBatchSize(
_senseiConf.getInt(SENSEI_INDEX_BATCH_MAXSIZE, ZoieConfig.DEFAULT_MAX_BATCH_SIZE));
zoieConfig.setRtIndexing(
_senseiConf.getBoolean(SENSEI_INDEX_REALTIME, ZoieConfig.DEFAULT_SETTING_REALTIME));
zoieConfig.setSkipBadRecord(_senseiConf.getBoolean(SENSEI_SKIP_BAD_RECORDS, false));
int delay = _senseiConf.getInt(SENSEI_INDEX_FRESHNESS, 10);
ReaderCacheFactory readercachefactory;
if (delay > 0) {
readercachefactory = DefaultReaderCache.FACTORY;
zoieConfig.setFreshness(delay * 1000);
} else {
readercachefactory = SimpleReaderCache.FACTORY;
}
zoieConfig.setReadercachefactory(readercachefactory);
ShardingStrategy strategy =
pluginRegistry.getBeanByFullPrefix(SENSEI_SHARDING_STRATEGY, ShardingStrategy.class);
if (strategy == null) {
strategy = new ShardingStrategy.FieldModShardingStrategy(_senseiSchema.getUidField());
}
pluggableSearchEngineManager = new PluggableSearchEngineManager();
pluggableSearchEngineManager.init(
_senseiConf.getString(SENSEI_INDEX_DIR),
nodeid,
_senseiSchema,
zoieConfig.getVersionComparator(),
pluginRegistry,
strategy);
List<FacetHandler<?>> facetHandlers = new LinkedList<FacetHandler<?>>();
List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories =
new LinkedList<RuntimeFacetHandlerFactory<?, ?>>();
SenseiSystemInfo sysInfo = null;
try {
sysInfo =
SenseiFacetHandlerBuilder.buildFacets(
_schemaDoc,
pluginRegistry,
facetHandlers,
runtimeFacetHandlerFactories,
pluggableSearchEngineManager);
} catch (JSONException jse) {
throw new ConfigurationException(jse.getMessage(), jse);
}
if (sysInfo != null) {
sysInfo.setSchema(_schemaDoc.toString());
try {
List<SenseiSystemInfo.SenseiNodeInfo> clusterInfo = new ArrayList(1);
String addr = NetUtil.getHostAddress();
clusterInfo.add(
new SenseiSystemInfo.SenseiNodeInfo(
nodeid,
partitions,
String.format("%s:%d", addr, _senseiConf.getInt(SERVER_PORT)),
String.format("http://%s:%d", addr, _senseiConf.getInt(SERVER_BROKER_PORT))));
sysInfo.setClusterInfo(clusterInfo);
} catch (Exception e) {
throw new ConfigurationException(e.getMessage(), e);
}
}
ZoieIndexableInterpreter interpreter =
pluginRegistry.getBeanByFullPrefix(
SENSEI_INDEX_INTERPRETER, ZoieIndexableInterpreter.class);
if (interpreter == null) {
DefaultJsonSchemaInterpreter defaultInterpreter =
new DefaultJsonSchemaInterpreter(_senseiSchema, pluggableSearchEngineManager);
interpreter = defaultInterpreter;
CustomIndexingPipeline customIndexingPipeline =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_CUSTOM, CustomIndexingPipeline.class);
if (customIndexingPipeline != null) {
try {
defaultInterpreter.setCustomIndexingPipeline(customIndexingPipeline);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
SenseiZoieFactory<?> zoieSystemFactory =
constructZoieFactory(zoieConfig, facetHandlers, runtimeFacetHandlerFactories, interpreter);
SenseiIndexingManager<?> indexingManager =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_MANAGER, SenseiIndexingManager.class);
if (indexingManager == null) {
indexingManager =
new DefaultStreamingIndexingManager(
_senseiSchema,
_senseiConf,
pluginRegistry,
_gateway,
strategy,
pluggableSearchEngineManager);
}
SenseiQueryBuilderFactory queryBuilderFactory =
pluginRegistry.getBeanByFullPrefix(
SENSEI_QUERY_BUILDER_FACTORY, SenseiQueryBuilderFactory.class);
if (queryBuilderFactory == null) {
QueryParser queryParser = new QueryParser(Version.LUCENE_35, "contents", analyzer);
queryBuilderFactory = new DefaultJsonQueryBuilderFactory(queryParser);
}
SenseiCore senseiCore =
new SenseiCore(
nodeid, partitions, zoieSystemFactory, indexingManager, queryBuilderFactory, decorator);
senseiCore.setSystemInfo(sysInfo);
SenseiIndexPruner indexPruner =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_PRUNER, SenseiIndexPruner.class);
if (indexPruner != null) {
senseiCore.setIndexPruner(indexPruner);
}
if (pluggableSearchEngineManager != null) {
senseiCore.setPluggableSearchEngineManager(pluggableSearchEngineManager);
}
return senseiCore;
}
private void processRelevanceExternalObjectPlugins(SenseiPluginRegistry pluginRegistry) {
List<RelevanceObjPlugin> relObjPlugins =
pluginRegistry.getBeansByType(RelevanceObjPlugin.class);
for (RelevanceObjPlugin rop : relObjPlugins) ExternalRelevanceDataStorage.putObj(rop);
}
public Server buildHttpRestServer() throws Exception {
int port = _senseiConf.getInt(SERVER_BROKER_PORT);
String webappPath =
_senseiConf.getString(SERVER_BROKER_WEBAPP_PATH, "sensei-core/src/main/webapp");
Server server = new Server();
QueuedThreadPool threadPool = new QueuedThreadPool();
threadPool.setName("Sensei Broker(jetty) threads");
threadPool.setMinThreads(_senseiConf.getInt(SERVER_BROKER_MINTHREAD, 20));
threadPool.setMaxThreads(_senseiConf.getInt(SERVER_BROKER_MAXTHREAD, 50));
threadPool.setMaxIdleTimeMs(_senseiConf.getInt(SERVER_BROKER_MAXWAIT, 2000));
// threadPool.start();
server.setThreadPool(threadPool);
logger.info("request threadpool started.");
SelectChannelConnector connector = new SelectChannelConnector();
connector.setPort(port);
server.addConnector(connector);
DefaultSenseiJSONServlet senseiServlet = new DefaultSenseiJSONServlet();
ServletHolder senseiServletHolder = new ServletHolder(senseiServlet);
SenseiHttpInvokerServiceServlet springServlet = new SenseiHttpInvokerServiceServlet();
ServletHolder springServletHolder = new ServletHolder(springServlet);
AgentServlet jmxServlet = new AgentServlet();
ServletHolder jmxServletHolder = new ServletHolder(jmxServlet);
WebAppContext senseiApp = new WebAppContext();
senseiApp.addFilter(GzipFilter.class, "/" + SENSEI_CONTEXT_PATH + "/*", 1);
// HashMap<String, String> initParam = new HashMap<String, String>();
// if (_senseiConfFile != null) {
// logger.info("Broker Configuration file: "+_senseiConfFile.getAbsolutePath());
// initParam.put("config.file", _senseiConfFile.getAbsolutePath());
// }
// senseiApp.setInitParams(initParam);
senseiApp.setAttribute("sensei.search.configuration", _senseiConf);
senseiApp.setAttribute(
SenseiConfigServletContextListener.SENSEI_CONF_PLUGIN_REGISTRY, pluginRegistry);
senseiApp.setAttribute(
"sensei.search.version.comparator",
_gateway != null ? _gateway.getVersionComparator() : ZoieConfig.DEFAULT_VERSION_COMPARATOR);
PartitionedLoadBalancerFactory<String> routerFactory =
pluginRegistry.getBeanByFullPrefix(
SenseiConfParams.SERVER_SEARCH_ROUTER_FACTORY, PartitionedLoadBalancerFactory.class);
if (routerFactory == null) {
routerFactory = new SenseiPartitionedLoadBalancerFactory(50);
}
senseiApp.setAttribute("sensei.search.router.factory", routerFactory);
senseiApp.addEventListener(new SenseiConfigServletContextListener());
senseiApp.addServlet(senseiServletHolder, "/" + SENSEI_CONTEXT_PATH + "/*");
senseiApp.setResourceBase(webappPath);
senseiApp.addServlet(springServletHolder, "/sensei-rpc/SenseiSpringRPCService");
senseiApp.addServlet(jmxServletHolder, "/admin/jmx/*");
server.setHandler(senseiApp);
server.setStopAtShutdown(true);
return server;
}