@SuppressWarnings("rawtypes")
@Override
public void start(SenseiCore senseiCore) {
persistentCaches = new HashMap<Integer, PersistentCache>(senseiCore.getPartitions().length);
for (int partition : senseiCore.getPartitions()) {
Zoie zoie = (Zoie) senseiCore.getIndexReaderFactory(partition);
PersistentCache persistentCache =
new PersistentCache(getPath(indexDirectory, nodeId, partition), versionComparator);
PersistentCache.registerAsListener(persistentCache, zoie);
persistentCaches.put(partition, persistentCache);
}
}
public AbstractSenseiCoreService(SenseiCore core) {
_core = core;
int[] partitions = _core.getPartitions();
}
public final Res execute(final Req senseiReq) {
SearchCounter.mark();
Set<Integer> partitions = senseiReq == null ? null : senseiReq.getPartitions();
if (partitions == null) {
partitions = new HashSet<Integer>();
int[] containsPart = _core.getPartitions();
if (containsPart != null) {
for (int part : containsPart) {
partitions.add(part);
}
}
}
Res finalResult;
if (partitions != null && partitions.size() > 0) {
if (logger.isDebugEnabled()) {
logger.debug("serving partitions: " + partitions.toString());
}
final ArrayList<Res> resultList = new ArrayList<Res>(partitions.size());
Future<Res>[] futures = new Future[partitions.size() - 1];
int i = 0;
for (final int partition : partitions) {
final long start = System.currentTimeMillis();
final IndexReaderFactory<ZoieIndexReader<BoboIndexReader>> readerFactory =
_core.getIndexReaderFactory(partition);
if (i < partitions.size() - 1) // Search simultaneously.
{
try {
futures[i] =
(Future<Res>)
_executorService.submit(
new Callable<Res>() {
public Res call() throws Exception {
Timer timer = getTimer(partition);
Res res =
timer.time(
new Callable<Res>() {
@Override
public Res call() throws Exception {
return handleRequest(
senseiReq,
readerFactory,
_core.getQueryBuilderFactory());
}
});
long end = System.currentTimeMillis();
res.setTime(end - start);
logger.info(
"searching partition: "
+ partition
+ " browse took: "
+ res.getTime());
return res;
}
});
} catch (Exception e) {
senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError));
logger.error(e.getMessage(), e);
}
} else // Reuse current thread.
{
try {
Timer timer = getTimer(partition);
Res res =
timer.time(
new Callable<Res>() {
@Override
public Res call() throws Exception {
return handleRequest(
senseiReq, readerFactory, _core.getQueryBuilderFactory());
}
});
resultList.add(res);
long end = System.currentTimeMillis();
res.setTime(end - start);
logger.info("searching partition: " + partition + " browse took: " + res.getTime());
} catch (Exception e) {
logger.error(e.getMessage(), e);
senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError));
resultList.add(getEmptyResultInstance(e));
}
}
++i;
}
for (i = 0; i < futures.length; ++i) {
try {
Res res = futures[i].get(_timeout, TimeUnit.MILLISECONDS);
resultList.add(res);
} catch (Exception e) {
logger.error(e.getMessage(), e);
if (e instanceof TimeoutException) {
senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.ExecutionTimeout));
} else {
senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError));
}
resultList.add(getEmptyResultInstance(e));
}
}
try {
finalResult =
MergeTimer.time(
new Callable<Res>() {
public Res call() throws Exception {
return mergePartitionedResults(senseiReq, resultList);
}
});
} catch (Exception e) {
logger.error(e.getMessage(), e);
finalResult = getEmptyResultInstance(null);
finalResult.addError(new SenseiError(e.getMessage(), ErrorType.MergePartitionError));
}
} else {
if (logger.isInfoEnabled()) {
logger.info("no partitions specified");
}
finalResult = getEmptyResultInstance(null);
finalResult.addError(
new SenseiError("no partitions specified", ErrorType.PartitionCallError));
}
if (logger.isInfoEnabled()) {
logger.info(
"searching partitions "
+ String.valueOf(partitions)
+ " took: "
+ finalResult.getTime());
}
return finalResult;
}
public SenseiCore buildCore() throws ConfigurationException {
SenseiUncaughtExceptionHandler.setAsDefaultForAllThreads();
int nodeid = _senseiConf.getInt(NODE_ID);
String partStr = _senseiConf.getString(PARTITIONS);
String[] partitionArray = partStr.split("[,\\s]+");
int[] partitions = buildPartitions(partitionArray);
logger.info("partitions to serve: " + Arrays.toString(partitions));
// Analyzer from configuration:
Analyzer analyzer = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_ANALYZER, Analyzer.class);
if (analyzer == null) {
analyzer = new StandardAnalyzer(Version.LUCENE_35);
}
// Similarity from configuration:
Similarity similarity =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_SIMILARITY, Similarity.class);
if (similarity == null) {
similarity = new DefaultSimilarity();
}
ZoieConfig zoieConfig;
if (_gateway != null) {
zoieConfig = new ZoieConfig(_gateway.getVersionComparator());
} else {
zoieConfig = new ZoieConfig();
}
zoieConfig.setAnalyzer(analyzer);
zoieConfig.setSimilarity(similarity);
zoieConfig.setBatchSize(
_senseiConf.getInt(SENSEI_INDEX_BATCH_SIZE, ZoieConfig.DEFAULT_SETTING_BATCHSIZE));
zoieConfig.setBatchDelay(
_senseiConf.getLong(SENSEI_INDEX_BATCH_DELAY, ZoieConfig.DEFAULT_SETTING_BATCHDELAY));
zoieConfig.setMaxBatchSize(
_senseiConf.getInt(SENSEI_INDEX_BATCH_MAXSIZE, ZoieConfig.DEFAULT_MAX_BATCH_SIZE));
zoieConfig.setRtIndexing(
_senseiConf.getBoolean(SENSEI_INDEX_REALTIME, ZoieConfig.DEFAULT_SETTING_REALTIME));
zoieConfig.setSkipBadRecord(_senseiConf.getBoolean(SENSEI_SKIP_BAD_RECORDS, false));
int delay = _senseiConf.getInt(SENSEI_INDEX_FRESHNESS, 10);
ReaderCacheFactory readercachefactory;
if (delay > 0) {
readercachefactory = DefaultReaderCache.FACTORY;
zoieConfig.setFreshness(delay * 1000);
} else {
readercachefactory = SimpleReaderCache.FACTORY;
}
zoieConfig.setReadercachefactory(readercachefactory);
ShardingStrategy strategy =
pluginRegistry.getBeanByFullPrefix(SENSEI_SHARDING_STRATEGY, ShardingStrategy.class);
if (strategy == null) {
strategy = new ShardingStrategy.FieldModShardingStrategy(_senseiSchema.getUidField());
}
pluggableSearchEngineManager = new PluggableSearchEngineManager();
pluggableSearchEngineManager.init(
_senseiConf.getString(SENSEI_INDEX_DIR),
nodeid,
_senseiSchema,
zoieConfig.getVersionComparator(),
pluginRegistry,
strategy);
List<FacetHandler<?>> facetHandlers = new LinkedList<FacetHandler<?>>();
List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories =
new LinkedList<RuntimeFacetHandlerFactory<?, ?>>();
SenseiSystemInfo sysInfo = null;
try {
sysInfo =
SenseiFacetHandlerBuilder.buildFacets(
_schemaDoc,
pluginRegistry,
facetHandlers,
runtimeFacetHandlerFactories,
pluggableSearchEngineManager);
} catch (JSONException jse) {
throw new ConfigurationException(jse.getMessage(), jse);
}
if (sysInfo != null) {
sysInfo.setSchema(_schemaDoc.toString());
try {
List<SenseiSystemInfo.SenseiNodeInfo> clusterInfo = new ArrayList(1);
String addr = NetUtil.getHostAddress();
clusterInfo.add(
new SenseiSystemInfo.SenseiNodeInfo(
nodeid,
partitions,
String.format("%s:%d", addr, _senseiConf.getInt(SERVER_PORT)),
String.format("http://%s:%d", addr, _senseiConf.getInt(SERVER_BROKER_PORT))));
sysInfo.setClusterInfo(clusterInfo);
} catch (Exception e) {
throw new ConfigurationException(e.getMessage(), e);
}
}
ZoieIndexableInterpreter interpreter =
pluginRegistry.getBeanByFullPrefix(
SENSEI_INDEX_INTERPRETER, ZoieIndexableInterpreter.class);
if (interpreter == null) {
DefaultJsonSchemaInterpreter defaultInterpreter =
new DefaultJsonSchemaInterpreter(_senseiSchema, pluggableSearchEngineManager);
interpreter = defaultInterpreter;
CustomIndexingPipeline customIndexingPipeline =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_CUSTOM, CustomIndexingPipeline.class);
if (customIndexingPipeline != null) {
try {
defaultInterpreter.setCustomIndexingPipeline(customIndexingPipeline);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
}
SenseiZoieFactory<?> zoieSystemFactory =
constructZoieFactory(zoieConfig, facetHandlers, runtimeFacetHandlerFactories, interpreter);
SenseiIndexingManager<?> indexingManager =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_MANAGER, SenseiIndexingManager.class);
if (indexingManager == null) {
indexingManager =
new DefaultStreamingIndexingManager(
_senseiSchema,
_senseiConf,
pluginRegistry,
_gateway,
strategy,
pluggableSearchEngineManager);
}
SenseiQueryBuilderFactory queryBuilderFactory =
pluginRegistry.getBeanByFullPrefix(
SENSEI_QUERY_BUILDER_FACTORY, SenseiQueryBuilderFactory.class);
if (queryBuilderFactory == null) {
QueryParser queryParser = new QueryParser(Version.LUCENE_35, "contents", analyzer);
queryBuilderFactory = new DefaultJsonQueryBuilderFactory(queryParser);
}
SenseiCore senseiCore =
new SenseiCore(
nodeid, partitions, zoieSystemFactory, indexingManager, queryBuilderFactory, decorator);
senseiCore.setSystemInfo(sysInfo);
SenseiIndexPruner indexPruner =
pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_PRUNER, SenseiIndexPruner.class);
if (indexPruner != null) {
senseiCore.setIndexPruner(indexPruner);
}
if (pluggableSearchEngineManager != null) {
senseiCore.setPluggableSearchEngineManager(pluggableSearchEngineManager);
}
return senseiCore;
}