示例#1
0
 /**
  * @param idxDir
  * @param realtime
  * @param delay delay for interpreter (simulating a slow interpreter)
  * @param analyzer
  * @param docidMapperFactory
  * @param zoieVersionFactory
  * @return
  */
 protected static ZoieSystem<IndexReader, String> createZoie(
     File idxDir,
     boolean realtime,
     long delay,
     Analyzer analyzer,
     DocIDMapperFactory docidMapperFactory,
     Comparator<String> versionComparator,
     boolean immediateRefresh) {
   ZoieConfig config = new ZoieConfig();
   config.setDocidMapperFactory(docidMapperFactory);
   config.setBatchSize(50);
   config.setBatchDelay(2000);
   config.setRtIndexing(realtime);
   config.setVersionComparator(versionComparator);
   config.setSimilarity(null);
   config.setAnalyzer(null);
   if (immediateRefresh) {
     config.setReadercachefactory(SimpleReaderCache.FACTORY);
   }
   ZoieSystem<IndexReader, String> idxSystem =
       new ZoieSystem<IndexReader, String>(
           idxDir,
           new DataInterpreterForTests(delay, analyzer),
           new TestIndexReaderDecorator(),
           config);
   return idxSystem;
 }
示例#2
0
 protected static ZoieSystem<IndexReader, String> createInRangeZoie(
     File idxDir,
     boolean realtime,
     InRangeDocIDMapperFactory docidMapperFactory,
     Comparator<String> versionComparator) {
   ZoieConfig config = new ZoieConfig();
   config.setDocidMapperFactory(docidMapperFactory);
   config.setBatchSize(50);
   config.setBatchDelay(2000);
   config.setRtIndexing(realtime);
   config.setVersionComparator(versionComparator);
   config.setSimilarity(null);
   config.setAnalyzer(null);
   // config.setReadercachefactory(SimpleReaderCache.FACTORY);
   ZoieSystem<IndexReader, String> idxSystem =
       new ZoieSystem<IndexReader, String>(
           idxDir,
           new InRangeDataInterpreterForTests(20, null),
           new TestIndexReaderDecorator(),
           config);
   return idxSystem;
 }
示例#3
0
  public SenseiCore buildCore() throws ConfigurationException {
    SenseiUncaughtExceptionHandler.setAsDefaultForAllThreads();
    int nodeid = _senseiConf.getInt(NODE_ID);
    String partStr = _senseiConf.getString(PARTITIONS);
    String[] partitionArray = partStr.split("[,\\s]+");
    int[] partitions = buildPartitions(partitionArray);
    logger.info("partitions to serve: " + Arrays.toString(partitions));
    // Analyzer from configuration:
    Analyzer analyzer = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_ANALYZER, Analyzer.class);
    if (analyzer == null) {
      analyzer = new StandardAnalyzer(Version.LUCENE_35);
    }
    // Similarity from configuration:
    Similarity similarity =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_SIMILARITY, Similarity.class);
    if (similarity == null) {
      similarity = new DefaultSimilarity();
    }
    ZoieConfig zoieConfig;
    if (_gateway != null) {
      zoieConfig = new ZoieConfig(_gateway.getVersionComparator());
    } else {
      zoieConfig = new ZoieConfig();
    }

    zoieConfig.setAnalyzer(analyzer);
    zoieConfig.setSimilarity(similarity);
    zoieConfig.setBatchSize(
        _senseiConf.getInt(SENSEI_INDEX_BATCH_SIZE, ZoieConfig.DEFAULT_SETTING_BATCHSIZE));
    zoieConfig.setBatchDelay(
        _senseiConf.getLong(SENSEI_INDEX_BATCH_DELAY, ZoieConfig.DEFAULT_SETTING_BATCHDELAY));
    zoieConfig.setMaxBatchSize(
        _senseiConf.getInt(SENSEI_INDEX_BATCH_MAXSIZE, ZoieConfig.DEFAULT_MAX_BATCH_SIZE));
    zoieConfig.setRtIndexing(
        _senseiConf.getBoolean(SENSEI_INDEX_REALTIME, ZoieConfig.DEFAULT_SETTING_REALTIME));
    zoieConfig.setSkipBadRecord(_senseiConf.getBoolean(SENSEI_SKIP_BAD_RECORDS, false));
    int delay = _senseiConf.getInt(SENSEI_INDEX_FRESHNESS, 10);
    ReaderCacheFactory readercachefactory;
    if (delay > 0) {
      readercachefactory = DefaultReaderCache.FACTORY;
      zoieConfig.setFreshness(delay * 1000);
    } else {
      readercachefactory = SimpleReaderCache.FACTORY;
    }
    zoieConfig.setReadercachefactory(readercachefactory);
    ShardingStrategy strategy =
        pluginRegistry.getBeanByFullPrefix(SENSEI_SHARDING_STRATEGY, ShardingStrategy.class);
    if (strategy == null) {
      strategy = new ShardingStrategy.FieldModShardingStrategy(_senseiSchema.getUidField());
    }

    pluggableSearchEngineManager = new PluggableSearchEngineManager();
    pluggableSearchEngineManager.init(
        _senseiConf.getString(SENSEI_INDEX_DIR),
        nodeid,
        _senseiSchema,
        zoieConfig.getVersionComparator(),
        pluginRegistry,
        strategy);

    List<FacetHandler<?>> facetHandlers = new LinkedList<FacetHandler<?>>();
    List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories =
        new LinkedList<RuntimeFacetHandlerFactory<?, ?>>();

    SenseiSystemInfo sysInfo = null;

    try {
      sysInfo =
          SenseiFacetHandlerBuilder.buildFacets(
              _schemaDoc,
              pluginRegistry,
              facetHandlers,
              runtimeFacetHandlerFactories,
              pluggableSearchEngineManager);
    } catch (JSONException jse) {
      throw new ConfigurationException(jse.getMessage(), jse);
    }

    if (sysInfo != null) {
      sysInfo.setSchema(_schemaDoc.toString());

      try {
        List<SenseiSystemInfo.SenseiNodeInfo> clusterInfo = new ArrayList(1);
        String addr = NetUtil.getHostAddress();
        clusterInfo.add(
            new SenseiSystemInfo.SenseiNodeInfo(
                nodeid,
                partitions,
                String.format("%s:%d", addr, _senseiConf.getInt(SERVER_PORT)),
                String.format("http://%s:%d", addr, _senseiConf.getInt(SERVER_BROKER_PORT))));
        sysInfo.setClusterInfo(clusterInfo);
      } catch (Exception e) {
        throw new ConfigurationException(e.getMessage(), e);
      }
    }
    ZoieIndexableInterpreter interpreter =
        pluginRegistry.getBeanByFullPrefix(
            SENSEI_INDEX_INTERPRETER, ZoieIndexableInterpreter.class);
    if (interpreter == null) {
      DefaultJsonSchemaInterpreter defaultInterpreter =
          new DefaultJsonSchemaInterpreter(_senseiSchema, pluggableSearchEngineManager);
      interpreter = defaultInterpreter;
      CustomIndexingPipeline customIndexingPipeline =
          pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_CUSTOM, CustomIndexingPipeline.class);
      if (customIndexingPipeline != null) {
        try {
          defaultInterpreter.setCustomIndexingPipeline(customIndexingPipeline);
        } catch (Exception e) {
          logger.error(e.getMessage(), e);
        }
      }
    }
    SenseiZoieFactory<?> zoieSystemFactory =
        constructZoieFactory(zoieConfig, facetHandlers, runtimeFacetHandlerFactories, interpreter);
    SenseiIndexingManager<?> indexingManager =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_MANAGER, SenseiIndexingManager.class);

    if (indexingManager == null) {
      indexingManager =
          new DefaultStreamingIndexingManager(
              _senseiSchema,
              _senseiConf,
              pluginRegistry,
              _gateway,
              strategy,
              pluggableSearchEngineManager);
    }
    SenseiQueryBuilderFactory queryBuilderFactory =
        pluginRegistry.getBeanByFullPrefix(
            SENSEI_QUERY_BUILDER_FACTORY, SenseiQueryBuilderFactory.class);
    if (queryBuilderFactory == null) {
      QueryParser queryParser = new QueryParser(Version.LUCENE_35, "contents", analyzer);
      queryBuilderFactory = new DefaultJsonQueryBuilderFactory(queryParser);
    }
    SenseiCore senseiCore =
        new SenseiCore(
            nodeid, partitions, zoieSystemFactory, indexingManager, queryBuilderFactory, decorator);
    senseiCore.setSystemInfo(sysInfo);
    SenseiIndexPruner indexPruner =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_PRUNER, SenseiIndexPruner.class);
    if (indexPruner != null) {
      senseiCore.setIndexPruner(indexPruner);
    }
    if (pluggableSearchEngineManager != null) {
      senseiCore.setPluggableSearchEngineManager(pluggableSearchEngineManager);
    }
    return senseiCore;
  }