@SuppressWarnings("rawtypes") @Override public void start(SenseiCore senseiCore) { persistentCaches = new HashMap<Integer, PersistentCache>(senseiCore.getPartitions().length); for (int partition : senseiCore.getPartitions()) { Zoie zoie = (Zoie) senseiCore.getIndexReaderFactory(partition); PersistentCache persistentCache = new PersistentCache(getPath(indexDirectory, nodeId, partition), versionComparator); PersistentCache.registerAsListener(persistentCache, zoie); persistentCaches.put(partition, persistentCache); } }
public AbstractSenseiCoreService(SenseiCore core) { _core = core; int[] partitions = _core.getPartitions(); }
public final Res execute(final Req senseiReq) { SearchCounter.mark(); Set<Integer> partitions = senseiReq == null ? null : senseiReq.getPartitions(); if (partitions == null) { partitions = new HashSet<Integer>(); int[] containsPart = _core.getPartitions(); if (containsPart != null) { for (int part : containsPart) { partitions.add(part); } } } Res finalResult; if (partitions != null && partitions.size() > 0) { if (logger.isDebugEnabled()) { logger.debug("serving partitions: " + partitions.toString()); } final ArrayList<Res> resultList = new ArrayList<Res>(partitions.size()); Future<Res>[] futures = new Future[partitions.size() - 1]; int i = 0; for (final int partition : partitions) { final long start = System.currentTimeMillis(); final IndexReaderFactory<ZoieIndexReader<BoboIndexReader>> readerFactory = _core.getIndexReaderFactory(partition); if (i < partitions.size() - 1) // Search simultaneously. { try { futures[i] = (Future<Res>) _executorService.submit( new Callable<Res>() { public Res call() throws Exception { Timer timer = getTimer(partition); Res res = timer.time( new Callable<Res>() { @Override public Res call() throws Exception { return handleRequest( senseiReq, readerFactory, _core.getQueryBuilderFactory()); } }); long end = System.currentTimeMillis(); res.setTime(end - start); logger.info( "searching partition: " + partition + " browse took: " + res.getTime()); return res; } }); } catch (Exception e) { senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError)); logger.error(e.getMessage(), e); } } else // Reuse current thread. { try { Timer timer = getTimer(partition); Res res = timer.time( new Callable<Res>() { @Override public Res call() throws Exception { return handleRequest( senseiReq, readerFactory, _core.getQueryBuilderFactory()); } }); resultList.add(res); long end = System.currentTimeMillis(); res.setTime(end - start); logger.info("searching partition: " + partition + " browse took: " + res.getTime()); } catch (Exception e) { logger.error(e.getMessage(), e); senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError)); resultList.add(getEmptyResultInstance(e)); } } ++i; } for (i = 0; i < futures.length; ++i) { try { Res res = futures[i].get(_timeout, TimeUnit.MILLISECONDS); resultList.add(res); } catch (Exception e) { logger.error(e.getMessage(), e); if (e instanceof TimeoutException) { senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.ExecutionTimeout)); } else { senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError)); } resultList.add(getEmptyResultInstance(e)); } } try { finalResult = MergeTimer.time( new Callable<Res>() { public Res call() throws Exception { return mergePartitionedResults(senseiReq, resultList); } }); } catch (Exception e) { logger.error(e.getMessage(), e); finalResult = getEmptyResultInstance(null); finalResult.addError(new SenseiError(e.getMessage(), ErrorType.MergePartitionError)); } } else { if (logger.isInfoEnabled()) { logger.info("no partitions specified"); } finalResult = getEmptyResultInstance(null); finalResult.addError( new SenseiError("no partitions specified", ErrorType.PartitionCallError)); } if (logger.isInfoEnabled()) { logger.info( "searching partitions " + String.valueOf(partitions) + " took: " + finalResult.getTime()); } return finalResult; }
public SenseiCore buildCore() throws ConfigurationException { SenseiUncaughtExceptionHandler.setAsDefaultForAllThreads(); int nodeid = _senseiConf.getInt(NODE_ID); String partStr = _senseiConf.getString(PARTITIONS); String[] partitionArray = partStr.split("[,\\s]+"); int[] partitions = buildPartitions(partitionArray); logger.info("partitions to serve: " + Arrays.toString(partitions)); // Analyzer from configuration: Analyzer analyzer = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_ANALYZER, Analyzer.class); if (analyzer == null) { analyzer = new StandardAnalyzer(Version.LUCENE_35); } // Similarity from configuration: Similarity similarity = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_SIMILARITY, Similarity.class); if (similarity == null) { similarity = new DefaultSimilarity(); } ZoieConfig zoieConfig; if (_gateway != null) { zoieConfig = new ZoieConfig(_gateway.getVersionComparator()); } else { zoieConfig = new ZoieConfig(); } zoieConfig.setAnalyzer(analyzer); zoieConfig.setSimilarity(similarity); zoieConfig.setBatchSize( _senseiConf.getInt(SENSEI_INDEX_BATCH_SIZE, ZoieConfig.DEFAULT_SETTING_BATCHSIZE)); zoieConfig.setBatchDelay( _senseiConf.getLong(SENSEI_INDEX_BATCH_DELAY, ZoieConfig.DEFAULT_SETTING_BATCHDELAY)); zoieConfig.setMaxBatchSize( _senseiConf.getInt(SENSEI_INDEX_BATCH_MAXSIZE, ZoieConfig.DEFAULT_MAX_BATCH_SIZE)); zoieConfig.setRtIndexing( _senseiConf.getBoolean(SENSEI_INDEX_REALTIME, ZoieConfig.DEFAULT_SETTING_REALTIME)); zoieConfig.setSkipBadRecord(_senseiConf.getBoolean(SENSEI_SKIP_BAD_RECORDS, false)); int delay = _senseiConf.getInt(SENSEI_INDEX_FRESHNESS, 10); ReaderCacheFactory readercachefactory; if (delay > 0) { readercachefactory = DefaultReaderCache.FACTORY; zoieConfig.setFreshness(delay * 1000); } else { readercachefactory = SimpleReaderCache.FACTORY; } zoieConfig.setReadercachefactory(readercachefactory); ShardingStrategy strategy = pluginRegistry.getBeanByFullPrefix(SENSEI_SHARDING_STRATEGY, ShardingStrategy.class); if (strategy == null) { strategy = new ShardingStrategy.FieldModShardingStrategy(_senseiSchema.getUidField()); } pluggableSearchEngineManager = new PluggableSearchEngineManager(); pluggableSearchEngineManager.init( _senseiConf.getString(SENSEI_INDEX_DIR), nodeid, _senseiSchema, zoieConfig.getVersionComparator(), pluginRegistry, strategy); List<FacetHandler<?>> facetHandlers = new LinkedList<FacetHandler<?>>(); List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories = new LinkedList<RuntimeFacetHandlerFactory<?, ?>>(); SenseiSystemInfo sysInfo = null; try { sysInfo = SenseiFacetHandlerBuilder.buildFacets( _schemaDoc, pluginRegistry, facetHandlers, runtimeFacetHandlerFactories, pluggableSearchEngineManager); } catch (JSONException jse) { throw new ConfigurationException(jse.getMessage(), jse); } if (sysInfo != null) { sysInfo.setSchema(_schemaDoc.toString()); try { List<SenseiSystemInfo.SenseiNodeInfo> clusterInfo = new ArrayList(1); String addr = NetUtil.getHostAddress(); clusterInfo.add( new SenseiSystemInfo.SenseiNodeInfo( nodeid, partitions, String.format("%s:%d", addr, _senseiConf.getInt(SERVER_PORT)), String.format("http://%s:%d", addr, _senseiConf.getInt(SERVER_BROKER_PORT)))); sysInfo.setClusterInfo(clusterInfo); } catch (Exception e) { throw new ConfigurationException(e.getMessage(), e); } } ZoieIndexableInterpreter interpreter = pluginRegistry.getBeanByFullPrefix( SENSEI_INDEX_INTERPRETER, ZoieIndexableInterpreter.class); if (interpreter == null) { DefaultJsonSchemaInterpreter defaultInterpreter = new DefaultJsonSchemaInterpreter(_senseiSchema, pluggableSearchEngineManager); interpreter = defaultInterpreter; CustomIndexingPipeline customIndexingPipeline = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_CUSTOM, CustomIndexingPipeline.class); if (customIndexingPipeline != null) { try { defaultInterpreter.setCustomIndexingPipeline(customIndexingPipeline); } catch (Exception e) { logger.error(e.getMessage(), e); } } } SenseiZoieFactory<?> zoieSystemFactory = constructZoieFactory(zoieConfig, facetHandlers, runtimeFacetHandlerFactories, interpreter); SenseiIndexingManager<?> indexingManager = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_MANAGER, SenseiIndexingManager.class); if (indexingManager == null) { indexingManager = new DefaultStreamingIndexingManager( _senseiSchema, _senseiConf, pluginRegistry, _gateway, strategy, pluggableSearchEngineManager); } SenseiQueryBuilderFactory queryBuilderFactory = pluginRegistry.getBeanByFullPrefix( SENSEI_QUERY_BUILDER_FACTORY, SenseiQueryBuilderFactory.class); if (queryBuilderFactory == null) { QueryParser queryParser = new QueryParser(Version.LUCENE_35, "contents", analyzer); queryBuilderFactory = new DefaultJsonQueryBuilderFactory(queryParser); } SenseiCore senseiCore = new SenseiCore( nodeid, partitions, zoieSystemFactory, indexingManager, queryBuilderFactory, decorator); senseiCore.setSystemInfo(sysInfo); SenseiIndexPruner indexPruner = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_PRUNER, SenseiIndexPruner.class); if (indexPruner != null) { senseiCore.setIndexPruner(indexPruner); } if (pluggableSearchEngineManager != null) { senseiCore.setPluggableSearchEngineManager(pluggableSearchEngineManager); } return senseiCore; }