@SuppressWarnings("rawtypes")
 @Override
 public void start(SenseiCore senseiCore) {
   persistentCaches = new HashMap<Integer, PersistentCache>(senseiCore.getPartitions().length);
   for (int partition : senseiCore.getPartitions()) {
     Zoie zoie = (Zoie) senseiCore.getIndexReaderFactory(partition);
     PersistentCache persistentCache =
         new PersistentCache(getPath(indexDirectory, nodeId, partition), versionComparator);
     PersistentCache.registerAsListener(persistentCache, zoie);
     persistentCaches.put(partition, persistentCache);
   }
 }
 public AbstractSenseiCoreService(SenseiCore core) {
   _core = core;
   int[] partitions = _core.getPartitions();
 }
  public final Res execute(final Req senseiReq) {
    SearchCounter.mark();
    Set<Integer> partitions = senseiReq == null ? null : senseiReq.getPartitions();
    if (partitions == null) {
      partitions = new HashSet<Integer>();
      int[] containsPart = _core.getPartitions();
      if (containsPart != null) {
        for (int part : containsPart) {
          partitions.add(part);
        }
      }
    }
    Res finalResult;
    if (partitions != null && partitions.size() > 0) {
      if (logger.isDebugEnabled()) {
        logger.debug("serving partitions: " + partitions.toString());
      }
      final ArrayList<Res> resultList = new ArrayList<Res>(partitions.size());
      Future<Res>[] futures = new Future[partitions.size() - 1];
      int i = 0;
      for (final int partition : partitions) {
        final long start = System.currentTimeMillis();
        final IndexReaderFactory<ZoieIndexReader<BoboIndexReader>> readerFactory =
            _core.getIndexReaderFactory(partition);

        if (i < partitions.size() - 1) // Search simultaneously.
        {
          try {
            futures[i] =
                (Future<Res>)
                    _executorService.submit(
                        new Callable<Res>() {
                          public Res call() throws Exception {
                            Timer timer = getTimer(partition);

                            Res res =
                                timer.time(
                                    new Callable<Res>() {

                                      @Override
                                      public Res call() throws Exception {
                                        return handleRequest(
                                            senseiReq,
                                            readerFactory,
                                            _core.getQueryBuilderFactory());
                                      }
                                    });

                            long end = System.currentTimeMillis();
                            res.setTime(end - start);
                            logger.info(
                                "searching partition: "
                                    + partition
                                    + " browse took: "
                                    + res.getTime());

                            return res;
                          }
                        });
          } catch (Exception e) {
            senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError));
            logger.error(e.getMessage(), e);
          }
        } else // Reuse current thread.
        {
          try {
            Timer timer = getTimer(partition);
            Res res =
                timer.time(
                    new Callable<Res>() {

                      @Override
                      public Res call() throws Exception {
                        return handleRequest(
                            senseiReq, readerFactory, _core.getQueryBuilderFactory());
                      }
                    });

            resultList.add(res);
            long end = System.currentTimeMillis();
            res.setTime(end - start);
            logger.info("searching partition: " + partition + " browse took: " + res.getTime());
          } catch (Exception e) {
            logger.error(e.getMessage(), e);
            senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError));

            resultList.add(getEmptyResultInstance(e));
          }
        }
        ++i;
      }

      for (i = 0; i < futures.length; ++i) {
        try {
          Res res = futures[i].get(_timeout, TimeUnit.MILLISECONDS);
          resultList.add(res);
        } catch (Exception e) {

          logger.error(e.getMessage(), e);
          if (e instanceof TimeoutException) {
            senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.ExecutionTimeout));
          } else {
            senseiReq.addError(new SenseiError(e.getMessage(), ErrorType.BoboExecutionError));
          }
          resultList.add(getEmptyResultInstance(e));
        }
      }

      try {
        finalResult =
            MergeTimer.time(
                new Callable<Res>() {
                  public Res call() throws Exception {
                    return mergePartitionedResults(senseiReq, resultList);
                  }
                });
      } catch (Exception e) {
        logger.error(e.getMessage(), e);
        finalResult = getEmptyResultInstance(null);
        finalResult.addError(new SenseiError(e.getMessage(), ErrorType.MergePartitionError));
      }
    } else {
      if (logger.isInfoEnabled()) {
        logger.info("no partitions specified");
      }
      finalResult = getEmptyResultInstance(null);
      finalResult.addError(
          new SenseiError("no partitions specified", ErrorType.PartitionCallError));
    }
    if (logger.isInfoEnabled()) {
      logger.info(
          "searching partitions  "
              + String.valueOf(partitions)
              + " took: "
              + finalResult.getTime());
    }
    return finalResult;
  }
Example #4
0
  public SenseiCore buildCore() throws ConfigurationException {
    SenseiUncaughtExceptionHandler.setAsDefaultForAllThreads();
    int nodeid = _senseiConf.getInt(NODE_ID);
    String partStr = _senseiConf.getString(PARTITIONS);
    String[] partitionArray = partStr.split("[,\\s]+");
    int[] partitions = buildPartitions(partitionArray);
    logger.info("partitions to serve: " + Arrays.toString(partitions));
    // Analyzer from configuration:
    Analyzer analyzer = pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_ANALYZER, Analyzer.class);
    if (analyzer == null) {
      analyzer = new StandardAnalyzer(Version.LUCENE_35);
    }
    // Similarity from configuration:
    Similarity similarity =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_SIMILARITY, Similarity.class);
    if (similarity == null) {
      similarity = new DefaultSimilarity();
    }
    ZoieConfig zoieConfig;
    if (_gateway != null) {
      zoieConfig = new ZoieConfig(_gateway.getVersionComparator());
    } else {
      zoieConfig = new ZoieConfig();
    }

    zoieConfig.setAnalyzer(analyzer);
    zoieConfig.setSimilarity(similarity);
    zoieConfig.setBatchSize(
        _senseiConf.getInt(SENSEI_INDEX_BATCH_SIZE, ZoieConfig.DEFAULT_SETTING_BATCHSIZE));
    zoieConfig.setBatchDelay(
        _senseiConf.getLong(SENSEI_INDEX_BATCH_DELAY, ZoieConfig.DEFAULT_SETTING_BATCHDELAY));
    zoieConfig.setMaxBatchSize(
        _senseiConf.getInt(SENSEI_INDEX_BATCH_MAXSIZE, ZoieConfig.DEFAULT_MAX_BATCH_SIZE));
    zoieConfig.setRtIndexing(
        _senseiConf.getBoolean(SENSEI_INDEX_REALTIME, ZoieConfig.DEFAULT_SETTING_REALTIME));
    zoieConfig.setSkipBadRecord(_senseiConf.getBoolean(SENSEI_SKIP_BAD_RECORDS, false));
    int delay = _senseiConf.getInt(SENSEI_INDEX_FRESHNESS, 10);
    ReaderCacheFactory readercachefactory;
    if (delay > 0) {
      readercachefactory = DefaultReaderCache.FACTORY;
      zoieConfig.setFreshness(delay * 1000);
    } else {
      readercachefactory = SimpleReaderCache.FACTORY;
    }
    zoieConfig.setReadercachefactory(readercachefactory);
    ShardingStrategy strategy =
        pluginRegistry.getBeanByFullPrefix(SENSEI_SHARDING_STRATEGY, ShardingStrategy.class);
    if (strategy == null) {
      strategy = new ShardingStrategy.FieldModShardingStrategy(_senseiSchema.getUidField());
    }

    pluggableSearchEngineManager = new PluggableSearchEngineManager();
    pluggableSearchEngineManager.init(
        _senseiConf.getString(SENSEI_INDEX_DIR),
        nodeid,
        _senseiSchema,
        zoieConfig.getVersionComparator(),
        pluginRegistry,
        strategy);

    List<FacetHandler<?>> facetHandlers = new LinkedList<FacetHandler<?>>();
    List<RuntimeFacetHandlerFactory<?, ?>> runtimeFacetHandlerFactories =
        new LinkedList<RuntimeFacetHandlerFactory<?, ?>>();

    SenseiSystemInfo sysInfo = null;

    try {
      sysInfo =
          SenseiFacetHandlerBuilder.buildFacets(
              _schemaDoc,
              pluginRegistry,
              facetHandlers,
              runtimeFacetHandlerFactories,
              pluggableSearchEngineManager);
    } catch (JSONException jse) {
      throw new ConfigurationException(jse.getMessage(), jse);
    }

    if (sysInfo != null) {
      sysInfo.setSchema(_schemaDoc.toString());

      try {
        List<SenseiSystemInfo.SenseiNodeInfo> clusterInfo = new ArrayList(1);
        String addr = NetUtil.getHostAddress();
        clusterInfo.add(
            new SenseiSystemInfo.SenseiNodeInfo(
                nodeid,
                partitions,
                String.format("%s:%d", addr, _senseiConf.getInt(SERVER_PORT)),
                String.format("http://%s:%d", addr, _senseiConf.getInt(SERVER_BROKER_PORT))));
        sysInfo.setClusterInfo(clusterInfo);
      } catch (Exception e) {
        throw new ConfigurationException(e.getMessage(), e);
      }
    }
    ZoieIndexableInterpreter interpreter =
        pluginRegistry.getBeanByFullPrefix(
            SENSEI_INDEX_INTERPRETER, ZoieIndexableInterpreter.class);
    if (interpreter == null) {
      DefaultJsonSchemaInterpreter defaultInterpreter =
          new DefaultJsonSchemaInterpreter(_senseiSchema, pluggableSearchEngineManager);
      interpreter = defaultInterpreter;
      CustomIndexingPipeline customIndexingPipeline =
          pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_CUSTOM, CustomIndexingPipeline.class);
      if (customIndexingPipeline != null) {
        try {
          defaultInterpreter.setCustomIndexingPipeline(customIndexingPipeline);
        } catch (Exception e) {
          logger.error(e.getMessage(), e);
        }
      }
    }
    SenseiZoieFactory<?> zoieSystemFactory =
        constructZoieFactory(zoieConfig, facetHandlers, runtimeFacetHandlerFactories, interpreter);
    SenseiIndexingManager<?> indexingManager =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_MANAGER, SenseiIndexingManager.class);

    if (indexingManager == null) {
      indexingManager =
          new DefaultStreamingIndexingManager(
              _senseiSchema,
              _senseiConf,
              pluginRegistry,
              _gateway,
              strategy,
              pluggableSearchEngineManager);
    }
    SenseiQueryBuilderFactory queryBuilderFactory =
        pluginRegistry.getBeanByFullPrefix(
            SENSEI_QUERY_BUILDER_FACTORY, SenseiQueryBuilderFactory.class);
    if (queryBuilderFactory == null) {
      QueryParser queryParser = new QueryParser(Version.LUCENE_35, "contents", analyzer);
      queryBuilderFactory = new DefaultJsonQueryBuilderFactory(queryParser);
    }
    SenseiCore senseiCore =
        new SenseiCore(
            nodeid, partitions, zoieSystemFactory, indexingManager, queryBuilderFactory, decorator);
    senseiCore.setSystemInfo(sysInfo);
    SenseiIndexPruner indexPruner =
        pluginRegistry.getBeanByFullPrefix(SENSEI_INDEX_PRUNER, SenseiIndexPruner.class);
    if (indexPruner != null) {
      senseiCore.setIndexPruner(indexPruner);
    }
    if (pluggableSearchEngineManager != null) {
      senseiCore.setPluggableSearchEngineManager(pluggableSearchEngineManager);
    }
    return senseiCore;
  }