Beispiel #1
0
  public SenseiServerBuilder(File confDir, Map<String, Object> properties) throws Exception {
    if (properties != null) {
      _senseiConfFile = null;
      _senseiConf = new MapConfiguration(properties);
      ((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
    } else {
      _senseiConfFile = new File(confDir, SENSEI_PROPERTIES);
      if (!_senseiConfFile.exists()) {
        throw new ConfigurationException(
            "configuration file: " + _senseiConfFile.getAbsolutePath() + " does not exist.");
      }
      _senseiConf = new PropertiesConfiguration();
      ((PropertiesConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
      ((PropertiesConfiguration) _senseiConf).load(_senseiConfFile);
    }

    pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
    pluginRegistry.start();

    processRelevanceFunctionPlugins(pluginRegistry);
    processRelevanceExternalObjectPlugins(pluginRegistry);

    _gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);
    _schemaDoc = loadSchema(confDir);
    _senseiSchema = SenseiSchema.build(_schemaDoc);
  }
Beispiel #2
0
  @Override
  public void init(ServletConfig config) throws ServletException {

    ServletContext ctx = config.getServletContext();

    String confDirName =
        ctx.getInitParameter(SenseiConfigServletContextListener.SENSEI_CONF_DIR_PARAM);
    if (confDirName == null) {
      throw new ServletException(
          "configuration not specified, "
              + SenseiConfigServletContextListener.SENSEI_CONF_DIR_PARAM
              + " not set");
    }

    SenseiServerBuilder builder;
    try {
      builder = new SenseiServerBuilder(new File(confDirName), null);
      ctx.setAttribute("sensei.search.configuration", builder.getConfiguration());
      ctx.setAttribute("sensei.search.version.comparator", builder.getVersionComparator());
      SenseiPluginRegistry pluginRegistry = builder.getPluginRegistry();
      PartitionedLoadBalancerFactory<String> routerFactory =
          pluginRegistry.getBeanByFullPrefix(
              SenseiConfParams.SERVER_SEARCH_ROUTER_FACTORY, PartitionedLoadBalancerFactory.class);
      if (routerFactory == null) {
        routerFactory = new SenseiPartitionedLoadBalancerFactory(50);
      }
      ctx.setAttribute("sensei.search.router.factory", routerFactory);

      _senseiServer = builder.buildServer();
      _senseiServer.start(true);
      super.init(config);
    } catch (Exception e) {
      throw new ServletException(e.getMessage(), e);
    }
  }
Beispiel #3
0
  public SenseiServerBuilder(Resource confDir, Map<String, Object> properties) throws Exception {
    _senseiConfFile = null;

    _senseiConf = new MapConfiguration(properties);
    ((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);

    pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
    pluginRegistry.start();

    processRelevanceFunctionPlugins(pluginRegistry);

    _gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);

    _schemaDoc = loadSchema(confDir);
    _senseiSchema = SenseiSchema.build(_schemaDoc);
  }
Beispiel #4
0
  @BeforeClass
  public static void init() throws Exception {
    final ShutdownRegistryImpl shutdownRegistry = new ShutdownRegistryImpl();

    try {
      zkTestServer =
          new ZooKeeperTestServer(0, shutdownRegistry, ZooKeeperTestServer.DEFAULT_SESSION_TIMEOUT);
      port = zkTestServer.startNetwork();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }

    Properties kafkaProps = new Properties();
    kafkaProps.setProperty("num.partitions", "1");
    kafkaProps.setProperty("port", "9092");
    kafkaProps.setProperty("broker.id", "0");
    kafkaProps.setProperty("log.dir", "/tmp/sensei-gateway-test-kafka-logs");
    // override to the local running zk server
    kafkaProps.setProperty("zookeeper.connect", "localhost:" + port);

    kafkaLogFile = new File(kafkaProps.getProperty("log.dir"));
    FileUtils.deleteDirectory(kafkaLogFile);

    KafkaConfig kafkaConfig = new KafkaConfig(kafkaProps);
    kafkaServer = new KafkaServerStartable(kafkaConfig);

    kafkaServer.startup();

    Configuration config = new PropertiesConfiguration();
    config.addProperty(
        "sensei.gateway.class", "com.senseidb.gateway.kafka.KafkaDataProviderBuilder");
    config.addProperty("sensei.gateway.kafka.group.id", "1");
    config.addProperty("sensei.gateway.kafka.zookeeper.connect", "localhost:" + port);
    config.addProperty("sensei.gateway.kafka.auto.offset.reset", "smallest");
    config.addProperty("sensei.gateway.kafka.topic", "test");
    config.addProperty("sensei.gateway.provider.batchSize", "1");
    pluginRegistry = SenseiPluginRegistry.build(config);
    pluginRegistry.start();

    kafkaGateway = pluginRegistry.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
    kafkaGateway.start();

    config = new PropertiesConfiguration();
    config.addProperty("sensei.gateway.class", "com.senseidb.gateway.kafka.SimpleKafkaGateway");
    config.addProperty("sensei.gateway.kafka.host", "localhost");
    config.addProperty("sensei.gateway.kafka.port", "9092");
    config.addProperty("sensei.gateway.kafka.topic", "test");
    config.addProperty("sensei.gateway.kafka.timeout", "3000");
    config.addProperty("sensei.gateway.provider.batchSize", "1");
    pluginRegistry2 = SenseiPluginRegistry.build(config);
    pluginRegistry2.start();

    simpleKafkaGateway = pluginRegistry2.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
    simpleKafkaGateway.start();

    Properties props = new Properties();
    props.put("metadata.broker.list", "localhost:9092");
    props.put("serializer.class", "kafka.serializer.StringEncoder");

    ProducerConfig producerConfig = new ProducerConfig(props);
    Producer<String, String> kafkaProducer = new Producer<String, String>(producerConfig);

    for (JSONObject jsonObj : BaseGatewayTestUtil.dataList) {
      KeyedMessage<String, String> data =
          new KeyedMessage<String, String>("test", jsonObj.toString());
      kafkaProducer.send(data);
    }
  }
Beispiel #5
0
  @SuppressWarnings("rawtypes")
  private SenseiZoieFactory<?> constructZoieFactory(
      ZoieConfig zoieConfig,
      List<FacetHandler<?>> facetHandlers,
      List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories,
      ZoieIndexableInterpreter interpreter)
      throws ConfigurationException {
    String indexerType = _senseiConf.getString(SENSEI_INDEXER_TYPE, "zoie");
    decorator = new SenseiIndexReaderDecorator(facetHandlers, runtimeFacetHandlerFactories);
    File idxDir = new File(_senseiConf.getString(SENSEI_INDEX_DIR));
    SenseiZoieFactory<?> zoieSystemFactory = null;

    DIRECTORY_MODE dirMode;
    String modeValue = _senseiConf.getString(SENSEI_INDEXER_MODE, "SIMPLE");
    if ("SIMPLE".equalsIgnoreCase(modeValue)) {
      dirMode = DIRECTORY_MODE.SIMPLE;
    } else if ("NIO".equalsIgnoreCase(modeValue)) {
      dirMode = DIRECTORY_MODE.NIO;
    } else if ("MMAP".equalsIgnoreCase(modeValue)) {
      dirMode = DIRECTORY_MODE.MMAP;
    } else {
      logger.error("directory mode " + modeValue + " is not supported, SIMPLE is used.");
      dirMode = DIRECTORY_MODE.SIMPLE;
    }

    if (SENSEI_INDEXER_TYPE_ZOIE.equals(indexerType)) {
      SenseiZoieSystemFactory senseiZoieFactory =
          new SenseiZoieSystemFactory(idxDir, dirMode, interpreter, decorator, zoieConfig);
      int retentionDays = _senseiConf.getInt(SENSEI_ZOIE_RETENTION_DAYS, -1);
      if (retentionDays > 0) {
        RetentionFilterFactory retentionFilterFactory =
            pluginRegistry.getBeanByFullPrefix(
                SENSEI_ZOIE_RETENTION_CLASS, RetentionFilterFactory.class);
        Filter purgeFilter = null;
        if (retentionFilterFactory != null) {
          purgeFilter = retentionFilterFactory.buildRetentionFilter(retentionDays);
        } else {
          String timeColumn = _senseiConf.getString(SENSEI_ZOIE_RETENTION_COLUMN, null);
          if (timeColumn == null) {
            throw new ConfigurationException("Retention specified without a time column");
          }
          String unitString = _senseiConf.getString(SENSEI_ZOIE_RETENTION_TIMEUNIT, "seconds");
          TimeUnit unit = TimeUnit.valueOf(unitString.toUpperCase());
          if (unit == null) {
            throw new ConfigurationException("Invalid timeunit for retention: " + unitString);
          }
          purgeFilter = new TimeRetentionFilter(timeColumn, retentionDays, unit);
        }
        if (purgeFilter != null && pluggableSearchEngineManager != null) {
          purgeFilter = new PurgeFilterWrapper(purgeFilter, pluggableSearchEngineManager);
        }
        senseiZoieFactory.setPurgeFilter(purgeFilter);
      }
      zoieSystemFactory = senseiZoieFactory;
    } else if (SENSEI_INDEXER_TYPE_HOURGLASS.equals(indexerType)) {
      String schedule = _senseiConf.getString(SENSEI_HOURGLASS_SCHEDULE, "");
      int trimThreshold = _senseiConf.getInt(SENSEI_HOURGLASS_TRIMTHRESHOLD, 14);
      String frequencyString = _senseiConf.getString(SENSEI_HOURGLASS_FREQUENCY, "day");

      FREQUENCY frequency;

      if (SENSEI_HOURGLASS_FREQUENCY_MIN.equals(frequencyString)) {
        frequency = FREQUENCY.MINUTELY;
      } else if (SENSEI_HOURGLASS_FREQUENCY_HOUR.equals(frequencyString)) {
        frequency = FREQUENCY.HOURLY;
      } else if (SENSEI_HOURGLASS_FREQUENCY_DAY.equals(frequencyString)) {
        frequency = FREQUENCY.DAILY;
      } else {
        throw new ConfigurationException("unsupported frequency setting: " + frequencyString);
      }

      boolean appendOnly = _senseiConf.getBoolean(SENSEI_HOURGLASS_APPENDONLY, true);
      zoieSystemFactory =
          new SenseiHourglassFactory(
              idxDir,
              dirMode,
              interpreter,
              decorator,
              zoieConfig,
              schedule,
              appendOnly,
              trimThreshold,
              frequency,
              pluggableSearchEngineManager != null
                  ? Arrays.asList(pluggableSearchEngineManager)
                  : Collections.EMPTY_LIST);
    } else {
      ZoieFactoryFactory zoieFactoryFactory =
          pluginRegistry.getBeanByFullPrefix(indexerType, ZoieFactoryFactory.class);
      if (zoieFactoryFactory == null) {
        throw new ConfigurationException(indexerType + " not defined");
      }
      zoieSystemFactory =
          zoieFactoryFactory.getZoieFactory(idxDir, interpreter, decorator, zoieConfig);
    }
    String indexerCopier = _senseiConf.getString(SENSEI_INDEXER_COPIER);
    IndexCopier copier =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEXER_COPIER, IndexCopier.class);
    if (copier != null) {
      zoieSystemFactory =
          new SenseiPairFactory(
              idxDir, dirMode, copier, interpreter, decorator, zoieConfig, zoieSystemFactory);
    } else if (SENSEI_INDEXER_COPIER_HDFS.equals(indexerCopier)) {
      zoieSystemFactory =
          new SenseiPairFactory(
              idxDir,
              dirMode,
              new HDFSIndexCopier(),
              interpreter,
              decorator,
              zoieConfig,
              zoieSystemFactory);
    } else {
      // do not support bootstrap index from other sources.

    }
    return zoieSystemFactory;
  }
Beispiel #6
0
  public SenseiCore buildCore() throws ConfigurationException {
    SenseiUncaughtExceptionHandler.setAsDefaultForAllThreads();
    int nodeid = _senseiConf.getInt(NODE_ID);
    String partStr = _senseiConf.getString(PARTITIONS);
    String[] partitionArray = partStr.split("[,\\s]+");
    int[] partitions = buildPartitions(partitionArray);
    logger.info("partitions to serve: " + Arrays.toString(partitions));
    // Analyzer from configuration:
    Analyzer analyzer = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_ANALYZER, Analyzer.class);
    if (analyzer == null) {
      analyzer = new StandardAnalyzer(Version.LUCENE_35);
    }
    // Similarity from configuration:
    Similarity similarity =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_SIMILARITY, Similarity.class);
    if (similarity == null) {
      similarity = new DefaultSimilarity();
    }
    ZoieConfig zoieConfig;
    if (_gateway != null) {
      zoieConfig = new ZoieConfig(_gateway.getVersionComparator());
    } else {
      zoieConfig = new ZoieConfig();
    }

    zoieConfig.setAnalyzer(analyzer);
    zoieConfig.setSimilarity(similarity);
    zoieConfig.setBatchSize(
        _senseiConf.getInt(SENSEI_INDEX_BATCH_SIZE, ZoieConfig.DEFAULT_SETTING_BATCHSIZE));
    zoieConfig.setBatchDelay(
        _senseiConf.getLong(SENSEI_INDEX_BATCH_DELAY, ZoieConfig.DEFAULT_SETTING_BATCHDELAY));
    zoieConfig.setMaxBatchSize(
        _senseiConf.getInt(SENSEI_INDEX_BATCH_MAXSIZE, ZoieConfig.DEFAULT_MAX_BATCH_SIZE));
    zoieConfig.setRtIndexing(
        _senseiConf.getBoolean(SENSEI_INDEX_REALTIME, ZoieConfig.DEFAULT_SETTING_REALTIME));
    zoieConfig.setSkipBadRecord(_senseiConf.getBoolean(SENSEI_SKIP_BAD_RECORDS, false));
    int delay = _senseiConf.getInt(SENSEI_INDEX_FRESHNESS, 10);
    ReaderCacheFactory readercachefactory;
    if (delay > 0) {
      readercachefactory = DefaultReaderCache.FACTORY;
      zoieConfig.setFreshness(delay * 1000);
    } else {
      readercachefactory = SimpleReaderCache.FACTORY;
    }
    zoieConfig.setReadercachefactory(readercachefactory);
    ShardingStrategy strategy =
        pluginRegistry.getBeanByFullPrefix(SENSEI_SHARDING_STRATEGY, ShardingStrategy.class);
    if (strategy == null) {
      strategy = new ShardingStrategy.FieldModShardingStrategy(_senseiSchema.getUidField());
    }

    pluggableSearchEngineManager = new PluggableSearchEngineManager();
    pluggableSearchEngineManager.init(
        _senseiConf.getString(SENSEI_INDEX_DIR),
        nodeid,
        _senseiSchema,
        zoieConfig.getVersionComparator(),
        pluginRegistry,
        strategy);

    List<FacetHandler<?>> facetHandlers = new LinkedList<FacetHandler<?>>();
    List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories =
        new LinkedList<RuntimeFacetHandlerFactory<?, ?>>();

    SenseiSystemInfo sysInfo = null;

    try {
      sysInfo =
          SenseiFacetHandlerBuilder.buildFacets(
              _schemaDoc,
              pluginRegistry,
              facetHandlers,
              runtimeFacetHandlerFactories,
              pluggableSearchEngineManager);
    } catch (JSONException jse) {
      throw new ConfigurationException(jse.getMessage(), jse);
    }

    if (sysInfo != null) {
      sysInfo.setSchema(_schemaDoc.toString());

      try {
        List<SenseiSystemInfo.SenseiNodeInfo> clusterInfo = new ArrayList(1);
        String addr = NetUtil.getHostAddress();
        clusterInfo.add(
            new SenseiSystemInfo.SenseiNodeInfo(
                nodeid,
                partitions,
                String.format("%s:%d", addr, _senseiConf.getInt(SERVER_PORT)),
                String.format("http://%s:%d", addr, _senseiConf.getInt(SERVER_BROKER_PORT))));
        sysInfo.setClusterInfo(clusterInfo);
      } catch (Exception e) {
        throw new ConfigurationException(e.getMessage(), e);
      }
    }
    ZoieIndexableInterpreter interpreter =
        pluginRegistry.getBeanByFullPrefix(
            SENSEI_INDEX_INTERPRETER, ZoieIndexableInterpreter.class);
    if (interpreter == null) {
      DefaultJsonSchemaInterpreter defaultInterpreter =
          new DefaultJsonSchemaInterpreter(_senseiSchema, pluggableSearchEngineManager);
      interpreter = defaultInterpreter;
      CustomIndexingPipeline customIndexingPipeline =
          pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_CUSTOM, CustomIndexingPipeline.class);
      if (customIndexingPipeline != null) {
        try {
          defaultInterpreter.setCustomIndexingPipeline(customIndexingPipeline);
        } catch (Exception e) {
          logger.error(e.getMessage(), e);
        }
      }
    }
    SenseiZoieFactory<?> zoieSystemFactory =
        constructZoieFactory(zoieConfig, facetHandlers, runtimeFacetHandlerFactories, interpreter);
    SenseiIndexingManager<?> indexingManager =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_MANAGER, SenseiIndexingManager.class);

    if (indexingManager == null) {
      indexingManager =
          new DefaultStreamingIndexingManager(
              _senseiSchema,
              _senseiConf,
              pluginRegistry,
              _gateway,
              strategy,
              pluggableSearchEngineManager);
    }
    SenseiQueryBuilderFactory queryBuilderFactory =
        pluginRegistry.getBeanByFullPrefix(
            SENSEI_QUERY_BUILDER_FACTORY, SenseiQueryBuilderFactory.class);
    if (queryBuilderFactory == null) {
      QueryParser queryParser = new QueryParser(Version.LUCENE_35, "contents", analyzer);
      queryBuilderFactory = new DefaultJsonQueryBuilderFactory(queryParser);
    }
    SenseiCore senseiCore =
        new SenseiCore(
            nodeid, partitions, zoieSystemFactory, indexingManager, queryBuilderFactory, decorator);
    senseiCore.setSystemInfo(sysInfo);
    SenseiIndexPruner indexPruner =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_PRUNER, SenseiIndexPruner.class);
    if (indexPruner != null) {
      senseiCore.setIndexPruner(indexPruner);
    }
    if (pluggableSearchEngineManager != null) {
      senseiCore.setPluggableSearchEngineManager(pluggableSearchEngineManager);
    }
    return senseiCore;
  }
Beispiel #7
0
  public Server buildHttpRestServer() throws Exception {
    int port = _senseiConf.getInt(SERVER_BROKER_PORT);

    String webappPath =
        _senseiConf.getString(SERVER_BROKER_WEBAPP_PATH, "sensei-core/src/main/webapp");

    Server server = new Server();

    QueuedThreadPool threadPool = new QueuedThreadPool();
    threadPool.setName("Sensei Broker(jetty) threads");
    threadPool.setMinThreads(_senseiConf.getInt(SERVER_BROKER_MINTHREAD, 20));
    threadPool.setMaxThreads(_senseiConf.getInt(SERVER_BROKER_MAXTHREAD, 50));
    threadPool.setMaxIdleTimeMs(_senseiConf.getInt(SERVER_BROKER_MAXWAIT, 2000));
    // threadPool.start();
    server.setThreadPool(threadPool);

    logger.info("request threadpool started.");
    SelectChannelConnector connector = new SelectChannelConnector();
    connector.setPort(port);
    server.addConnector(connector);

    DefaultSenseiJSONServlet senseiServlet = new DefaultSenseiJSONServlet();
    ServletHolder senseiServletHolder = new ServletHolder(senseiServlet);

    SenseiHttpInvokerServiceServlet springServlet = new SenseiHttpInvokerServiceServlet();
    ServletHolder springServletHolder = new ServletHolder(springServlet);

    AgentServlet jmxServlet = new AgentServlet();
    ServletHolder jmxServletHolder = new ServletHolder(jmxServlet);

    WebAppContext senseiApp = new WebAppContext();
    senseiApp.addFilter(GzipFilter.class, "/" + SENSEI_CONTEXT_PATH + "/*", 1);

    // HashMap<String, String> initParam = new HashMap<String, String>();
    // if (_senseiConfFile != null) {
    // logger.info("Broker Configuration file: "+_senseiConfFile.getAbsolutePath());
    // initParam.put("config.file", _senseiConfFile.getAbsolutePath());
    // }
    // senseiApp.setInitParams(initParam);
    senseiApp.setAttribute("sensei.search.configuration", _senseiConf);
    senseiApp.setAttribute(
        SenseiConfigServletContextListener.SENSEI_CONF_PLUGIN_REGISTRY, pluginRegistry);
    senseiApp.setAttribute(
        "sensei.search.version.comparator",
        _gateway != null ? _gateway.getVersionComparator() : ZoieConfig.DEFAULT_VERSION_COMPARATOR);

    PartitionedLoadBalancerFactory<String> routerFactory =
        pluginRegistry.getBeanByFullPrefix(
            SenseiConfParams.SERVER_SEARCH_ROUTER_FACTORY, PartitionedLoadBalancerFactory.class);
    if (routerFactory == null) {
      routerFactory = new SenseiPartitionedLoadBalancerFactory(50);
    }

    senseiApp.setAttribute("sensei.search.router.factory", routerFactory);
    senseiApp.addEventListener(new SenseiConfigServletContextListener());
    senseiApp.addServlet(senseiServletHolder, "/" + SENSEI_CONTEXT_PATH + "/*");
    senseiApp.setResourceBase(webappPath);
    senseiApp.addServlet(springServletHolder, "/sensei-rpc/SenseiSpringRPCService");
    senseiApp.addServlet(jmxServletHolder, "/admin/jmx/*");

    server.setHandler(senseiApp);
    server.setStopAtShutdown(true);

    return server;
  }