@Override
 public void setUp() throws Exception {
   super.setUp();
   originalMaxClauses = BooleanQuery.getMaxClauseCount();
 }
/**
 * @author Brian Wing Shun Chan
 * @author Harry Mark
 * @author Bruno Farache
 * @author Shuyang Zhou
 * @author Tina Tian
 * @author Hugo Huijser
 * @author Andrea Di Giorgi
 */
public class LuceneHelperImpl implements LuceneHelper {

  @Override
  public void addDocument(long companyId, Document document) throws IOException {

    IndexAccessor indexAccessor = getIndexAccessor(companyId);

    indexAccessor.addDocument(document);
  }

  @Override
  public void addExactTerm(BooleanQuery booleanQuery, String field, String value) {

    addTerm(booleanQuery, field, value, false);
  }

  @Override
  public void addNumericRangeTerm(
      BooleanQuery booleanQuery, String field, Integer startValue, Integer endValue) {

    NumericRangeQuery<?> numericRangeQuery =
        NumericRangeQuery.newIntRange(field, startValue, endValue, true, true);

    booleanQuery.add(numericRangeQuery, BooleanClause.Occur.SHOULD);
  }

  @Override
  public void addNumericRangeTerm(
      BooleanQuery booleanQuery, String field, Long startValue, Long endValue) {

    NumericRangeQuery<?> numericRangeQuery =
        NumericRangeQuery.newLongRange(field, startValue, endValue, true, true);

    booleanQuery.add(numericRangeQuery, BooleanClause.Occur.SHOULD);
  }

  /**
   * @deprecated As of 6.2.0, replaced by {@link #addNumericRangeTerm(BooleanQuery, String, Long,
   *     Long)}
   */
  @Deprecated
  @Override
  public void addNumericRangeTerm(
      BooleanQuery booleanQuery, String field, String startValue, String endValue) {

    addNumericRangeTerm(
        booleanQuery, field, GetterUtil.getLong(startValue), GetterUtil.getLong(endValue));
  }

  @Override
  public void addRangeTerm(
      BooleanQuery booleanQuery, String field, String startValue, String endValue) {

    boolean includesLower = true;

    if ((startValue != null) && startValue.equals(StringPool.STAR)) {
      includesLower = false;
    }

    boolean includesUpper = true;

    if ((endValue != null) && endValue.equals(StringPool.STAR)) {
      includesUpper = false;
    }

    TermRangeQuery termRangeQuery =
        new TermRangeQuery(field, startValue, endValue, includesLower, includesUpper);

    booleanQuery.add(termRangeQuery, BooleanClause.Occur.SHOULD);
  }

  @Override
  public void addRequiredTerm(BooleanQuery booleanQuery, String field, String value, boolean like) {

    addRequiredTerm(booleanQuery, field, new String[] {value}, like);
  }

  @Override
  public void addRequiredTerm(
      BooleanQuery booleanQuery, String field, String[] values, boolean like) {

    if (values == null) {
      return;
    }

    BooleanQuery query = new BooleanQuery();

    for (String value : values) {
      addTerm(query, field, value, like);
    }

    booleanQuery.add(query, BooleanClause.Occur.MUST);
  }

  @Override
  public void addTerm(BooleanQuery booleanQuery, String field, String value, boolean like) {

    addTerm(booleanQuery, field, value, like, BooleanClauseOccur.SHOULD);
  }

  @Override
  public void addTerm(
      BooleanQuery booleanQuery,
      String field,
      String value,
      boolean like,
      BooleanClauseOccur booleanClauseOccur) {

    if (Validator.isNull(value)) {
      return;
    }

    Analyzer analyzer = getAnalyzer();

    if (analyzer instanceof PerFieldAnalyzer) {
      PerFieldAnalyzer perFieldAnalyzer = (PerFieldAnalyzer) analyzer;

      Analyzer fieldAnalyzer = perFieldAnalyzer.getAnalyzer(field);

      if (fieldAnalyzer instanceof LikeKeywordAnalyzer) {
        like = true;
      }
    }

    if (like) {
      value = StringUtil.replace(value, StringPool.PERCENT, StringPool.BLANK);
    }

    try {
      QueryParser queryParser = new QueryParser(getVersion(), field, analyzer);

      Query query = queryParser.parse(value);

      BooleanClause.Occur occur = null;

      if (booleanClauseOccur.equals(BooleanClauseOccur.MUST)) {
        occur = BooleanClause.Occur.MUST;
      } else if (booleanClauseOccur.equals(BooleanClauseOccur.MUST_NOT)) {
        occur = BooleanClause.Occur.MUST_NOT;
      } else {
        occur = BooleanClause.Occur.SHOULD;
      }

      _includeIfUnique(booleanQuery, like, queryParser, query, occur);
    } catch (Exception e) {
      if (_log.isWarnEnabled()) {
        _log.warn(e, e);
      }
    }
  }

  @Override
  public void addTerm(BooleanQuery booleanQuery, String field, String[] values, boolean like) {

    for (String value : values) {
      addTerm(booleanQuery, field, value, like);
    }
  }

  /** @deprecated As of 7.0.0, replaced by {@link #releaseIndexSearcher(long, IndexSearcher)} */
  @Deprecated
  @Override
  public void cleanUp(IndexSearcher indexSearcher) {
    if (indexSearcher == null) {
      return;
    }

    try {
      indexSearcher.close();

      IndexReader indexReader = indexSearcher.getIndexReader();

      if (indexReader != null) {
        indexReader.close();
      }
    } catch (IOException ioe) {
      _log.error(ioe, ioe);
    }
  }

  @Override
  public int countScoredFieldNames(Query query, String[] filedNames) {
    int count = 0;

    for (String fieldName : filedNames) {
      WeightedTerm[] weightedTerms = QueryTermExtractor.getTerms(query, false, fieldName);

      if ((weightedTerms.length > 0)
          && !ArrayUtil.contains(Field.UNSCORED_FIELD_NAMES, fieldName)) {

        count++;
      }
    }

    return count;
  }

  @Override
  public void delete(long companyId) {
    IndexAccessor indexAccessor = _indexAccessors.get(companyId);

    if (indexAccessor == null) {
      return;
    }

    indexAccessor.delete();
  }

  @Override
  public void deleteDocuments(long companyId, Term term) throws IOException {
    IndexAccessor indexAccessor = _indexAccessors.get(companyId);

    if (indexAccessor == null) {
      return;
    }

    indexAccessor.deleteDocuments(term);
  }

  @Override
  public void dumpIndex(long companyId, OutputStream outputStream) throws IOException {

    long lastGeneration = getLastGeneration(companyId);

    if (lastGeneration == IndexAccessor.DEFAULT_LAST_GENERATION) {
      if (_log.isDebugEnabled()) {
        _log.debug("Dump index from cluster is not enabled for " + companyId);
      }

      return;
    }

    IndexAccessor indexAccessor = _indexAccessors.get(companyId);

    if (indexAccessor == null) {
      return;
    }

    indexAccessor.dumpIndex(outputStream);
  }

  @Override
  public Analyzer getAnalyzer() {
    return _analyzer;
  }

  @Override
  public IndexAccessor getIndexAccessor(long companyId) {
    IndexAccessor indexAccessor = _indexAccessors.get(companyId);

    if (indexAccessor != null) {
      return indexAccessor;
    }

    synchronized (this) {
      indexAccessor = _indexAccessors.get(companyId);

      if (indexAccessor == null) {
        indexAccessor = new IndexAccessorImpl(companyId);

        if (isLoadIndexFromClusterEnabled()) {
          indexAccessor = new SynchronizedIndexAccessorImpl(indexAccessor);

          boolean clusterForwardMessage =
              GetterUtil.getBoolean(
                  MessageValuesThreadLocal.getValue(ClusterLink.CLUSTER_FORWARD_MESSAGE));

          if (clusterForwardMessage) {
            if (_log.isInfoEnabled()) {
              _log.info(
                  "Skip Luncene index files cluster loading "
                      + "since this is a manual reindex request");
            }
          } else {
            try {
              _loadIndexFromCluster(indexAccessor, indexAccessor.getLastGeneration());
            } catch (Exception e) {
              _log.error("Unable to load index for company " + indexAccessor.getCompanyId(), e);
            }
          }
        }

        _indexAccessors.put(companyId, indexAccessor);
      }
    }

    return indexAccessor;
  }

  @Override
  public IndexSearcher getIndexSearcher(long companyId) throws IOException {
    IndexAccessor indexAccessor = getIndexAccessor(companyId);

    return indexAccessor.acquireIndexSearcher();
  }

  @Override
  public long getLastGeneration(long companyId) {
    if (!isLoadIndexFromClusterEnabled()) {
      return IndexAccessor.DEFAULT_LAST_GENERATION;
    }

    IndexAccessor indexAccessor = _indexAccessors.get(companyId);

    if (indexAccessor == null) {
      return IndexAccessor.DEFAULT_LAST_GENERATION;
    }

    return indexAccessor.getLastGeneration();
  }

  @Override
  public InputStream getLoadIndexesInputStreamFromCluster(long companyId, Address bootupAddress) {

    if (!isLoadIndexFromClusterEnabled()) {
      return null;
    }

    InputStream inputStream = null;

    try {
      ObjectValuePair<String, URL> bootupClusterNodeObjectValuePair =
          _getBootupClusterNodeObjectValuePair(bootupAddress);

      URL url = bootupClusterNodeObjectValuePair.getValue();

      URLConnection urlConnection = url.openConnection();

      urlConnection.setDoOutput(true);

      UnsyncPrintWriter unsyncPrintWriter =
          UnsyncPrintWriterPool.borrow(urlConnection.getOutputStream());

      unsyncPrintWriter.write("transientToken=");
      unsyncPrintWriter.write(bootupClusterNodeObjectValuePair.getKey());
      unsyncPrintWriter.write("&companyId=");
      unsyncPrintWriter.write(String.valueOf(companyId));

      unsyncPrintWriter.close();

      inputStream = urlConnection.getInputStream();

      return inputStream;
    } catch (IOException ioe) {
      throw new SystemException(ioe);
    }
  }

  @Override
  public Set<String> getQueryTerms(Query query) {
    String queryString = StringUtil.replace(query.toString(), StringPool.STAR, StringPool.BLANK);

    Query tempQuery = null;

    try {
      QueryParser queryParser = new QueryParser(getVersion(), StringPool.BLANK, getAnalyzer());

      tempQuery = queryParser.parse(queryString);
    } catch (Exception e) {
      if (_log.isWarnEnabled()) {
        _log.warn("Unable to parse " + queryString);
      }

      tempQuery = query;
    }

    WeightedTerm[] weightedTerms = null;

    for (String fieldName : Field.KEYWORDS) {
      weightedTerms = QueryTermExtractor.getTerms(tempQuery, false, fieldName);

      if (weightedTerms.length > 0) {
        break;
      }
    }

    Set<String> queryTerms = new HashSet<String>();

    for (WeightedTerm weightedTerm : weightedTerms) {
      queryTerms.add(weightedTerm.getTerm());
    }

    return queryTerms;
  }

  /** @deprecated As of 7.0.0, replaced by {@link #getIndexSearcher(long)} */
  @Deprecated
  @Override
  public IndexSearcher getSearcher(long companyId, boolean readOnly) throws IOException {

    IndexAccessor indexAccessor = getIndexAccessor(companyId);

    IndexReader indexReader = IndexReader.open(indexAccessor.getLuceneDir(), readOnly);

    IndexSearcher indexSearcher = new IndexSearcher(indexReader);

    indexSearcher.setDefaultFieldSortScoring(true, false);
    indexSearcher.setSimilarity(new FieldWeightSimilarity());

    return indexSearcher;
  }

  @Override
  public String getSnippet(
      Query query,
      String field,
      String s,
      int maxNumFragments,
      int fragmentLength,
      String fragmentSuffix,
      Formatter formatter)
      throws IOException {

    QueryScorer queryScorer = new QueryScorer(query, field);

    Highlighter highlighter = new Highlighter(formatter, queryScorer);

    highlighter.setTextFragmenter(new SimpleFragmenter(fragmentLength));

    TokenStream tokenStream = getAnalyzer().tokenStream(field, new UnsyncStringReader(s));

    try {
      String snippet =
          highlighter.getBestFragments(tokenStream, s, maxNumFragments, fragmentSuffix);

      if (Validator.isNotNull(snippet)
          && !StringUtil.endsWith(snippet, fragmentSuffix)
          && !s.equals(snippet)) {

        snippet = snippet.concat(fragmentSuffix);
      }

      return snippet;
    } catch (InvalidTokenOffsetsException itoe) {
      throw new IOException(itoe);
    }
  }

  @Override
  public Version getVersion() {
    return _version;
  }

  @Override
  public boolean isLoadIndexFromClusterEnabled() {
    if (PropsValues.CLUSTER_LINK_ENABLED && PropsValues.LUCENE_REPLICATE_WRITE) {

      return true;
    }

    if (_log.isDebugEnabled()) {
      _log.debug("Load index from cluster is not enabled");
    }

    return false;
  }

  @Override
  public void loadIndex(long companyId, InputStream inputStream) throws IOException {

    if (!isLoadIndexFromClusterEnabled()) {
      return;
    }

    IndexAccessor indexAccessor = _indexAccessors.get(companyId);

    if (indexAccessor == null) {
      if (_log.isInfoEnabled()) {
        _log.info(
            "Skip loading Lucene index files for company "
                + companyId
                + " in favor of lazy loading");
      }

      return;
    }

    StopWatch stopWatch = new StopWatch();

    stopWatch.start();

    if (_log.isInfoEnabled()) {
      _log.info("Start loading Lucene index files for company " + companyId);
    }

    indexAccessor.loadIndex(inputStream);

    if (_log.isInfoEnabled()) {
      _log.info(
          "Finished loading index files for company "
              + companyId
              + " in "
              + stopWatch.getTime()
              + " ms");
    }
  }

  @Override
  public void loadIndexesFromCluster(long companyId) {
    if (!isLoadIndexFromClusterEnabled()) {
      return;
    }

    IndexAccessor indexAccessor = _indexAccessors.get(companyId);

    if (indexAccessor == null) {
      return;
    }

    long localLastGeneration = getLastGeneration(companyId);

    _loadIndexFromCluster(indexAccessor, localLastGeneration);
  }

  @Override
  public void releaseIndexSearcher(long companyId, IndexSearcher indexSearcher) throws IOException {

    IndexAccessor indexAccessor = getIndexAccessor(companyId);

    indexAccessor.releaseIndexSearcher(indexSearcher);
  }

  public void setAnalyzer(Analyzer analyzer) {
    _analyzer = analyzer;
  }

  public void setVersion(Version version) {
    _version = version;
  }

  @Override
  public void shutdown() {
    if (_luceneIndexThreadPoolExecutor != null) {
      _luceneIndexThreadPoolExecutor.shutdownNow();

      try {
        _luceneIndexThreadPoolExecutor.awaitTermination(60, TimeUnit.SECONDS);
      } catch (InterruptedException ie) {
        _log.error("Lucene indexer shutdown interrupted", ie);
      }
    }

    if (isLoadIndexFromClusterEnabled()) {
      ClusterExecutorUtil.removeClusterEventListener(_loadIndexClusterEventListener);
    }

    MessageBus messageBus = MessageBusUtil.getMessageBus();

    for (String searchEngineId : SearchEngineUtil.getSearchEngineIds()) {
      String searchWriterDestinationName =
          SearchEngineUtil.getSearchWriterDestinationName(searchEngineId);

      Destination searchWriteDestination = messageBus.getDestination(searchWriterDestinationName);

      if (searchWriteDestination != null) {
        ThreadPoolExecutor threadPoolExecutor =
            PortalExecutorManagerUtil.getPortalExecutor(searchWriterDestinationName);

        int maxPoolSize = threadPoolExecutor.getMaxPoolSize();

        CountDownLatch countDownLatch = new CountDownLatch(maxPoolSize);

        ShutdownSyncJob shutdownSyncJob = new ShutdownSyncJob(countDownLatch);

        for (int i = 0; i < maxPoolSize; i++) {
          threadPoolExecutor.submit(shutdownSyncJob);
        }

        try {
          countDownLatch.await();
        } catch (InterruptedException ie) {
          _log.error("Shutdown waiting interrupted", ie);
        }

        List<Runnable> runnables = threadPoolExecutor.shutdownNow();

        if (_log.isDebugEnabled()) {
          _log.debug("Cancelled appending indexing jobs: " + runnables);
        }

        searchWriteDestination.close(true);
      }
    }

    for (IndexAccessor indexAccessor : _indexAccessors.values()) {
      indexAccessor.close();
    }
  }

  @Override
  public void shutdown(long companyId) {
    IndexAccessor indexAccessor = getIndexAccessor(companyId);

    _indexAccessors.remove(companyId);

    indexAccessor.close();
  }

  @Override
  public void startup(long companyId) {
    if (!PropsValues.INDEX_ON_STARTUP) {
      return;
    }

    if (_log.isInfoEnabled()) {
      _log.info("Indexing Lucene on startup");
    }

    LuceneIndexer luceneIndexer = new LuceneIndexer(companyId);

    if (PropsValues.INDEX_WITH_THREAD) {
      if (_luceneIndexThreadPoolExecutor == null) {

        // This should never be null except for the case where
        // VerifyProcessUtil#_verifyProcess(boolean) sets
        // PropsValues#INDEX_ON_STARTUP to true.

        _luceneIndexThreadPoolExecutor =
            PortalExecutorManagerUtil.getPortalExecutor(LuceneHelperImpl.class.getName());
      }

      _luceneIndexThreadPoolExecutor.execute(luceneIndexer);
    } else {
      luceneIndexer.reindex();
    }
  }

  @Override
  public void updateDocument(long companyId, Term term, Document document) throws IOException {

    IndexAccessor indexAccessor = getIndexAccessor(companyId);

    indexAccessor.updateDocument(term, document);
  }

  private LuceneHelperImpl() {
    if (PropsValues.INDEX_ON_STARTUP && PropsValues.INDEX_WITH_THREAD) {
      _luceneIndexThreadPoolExecutor =
          PortalExecutorManagerUtil.getPortalExecutor(LuceneHelperImpl.class.getName());
    }

    if (isLoadIndexFromClusterEnabled()) {
      _loadIndexClusterEventListener = new LoadIndexClusterEventListener();

      ClusterExecutorUtil.addClusterEventListener(_loadIndexClusterEventListener);
    }

    BooleanQuery.setMaxClauseCount(_LUCENE_BOOLEAN_QUERY_CLAUSE_MAX_SIZE);

    if (StringUtil.equalsIgnoreCase(Http.HTTPS, PropsValues.WEB_SERVER_PROTOCOL)) {

      _protocol = Http.HTTPS;
    } else {
      _protocol = Http.HTTP;
    }
  }

  private ObjectValuePair<String, URL> _getBootupClusterNodeObjectValuePair(Address bootupAddress) {

    ClusterRequest clusterRequest =
        ClusterRequest.createUnicastRequest(
            new MethodHandler(_createTokenMethodKey, _CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT),
            bootupAddress);

    FutureClusterResponses futureClusterResponses = ClusterExecutorUtil.execute(clusterRequest);

    BlockingQueue<ClusterNodeResponse> clusterNodeResponses =
        futureClusterResponses.getPartialResults();

    try {
      ClusterNodeResponse clusterNodeResponse =
          clusterNodeResponses.poll(
              _CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);

      ClusterNode clusterNode = clusterNodeResponse.getClusterNode();

      InetSocketAddress inetSocketAddress = clusterNode.getPortalInetSocketAddress();

      if (inetSocketAddress == null) {
        StringBundler sb = new StringBundler(6);

        sb.append("Invalid cluster node InetSocketAddress ");
        sb.append(". The InetSocketAddress is set by the first ");
        sb.append("request or configured in portal.properties by the");
        sb.append("properties ");
        sb.append("\"portal.instance.http.inet.socket.address\" and ");
        sb.append("\"portal.instance.https.inet.socket.address\".");

        throw new Exception(sb.toString());
      }

      InetAddress inetAddress = inetSocketAddress.getAddress();

      String fileName = PortalUtil.getPathContext();

      if (!fileName.endsWith(StringPool.SLASH)) {
        fileName = fileName.concat(StringPool.SLASH);
      }

      fileName = fileName.concat("lucene/dump");

      URL url =
          new URL(_protocol, inetAddress.getHostAddress(), inetSocketAddress.getPort(), fileName);

      String transientToken = (String) clusterNodeResponse.getResult();

      return new ObjectValuePair<String, URL>(transientToken, url);
    } catch (Exception e) {
      throw new SystemException(e);
    }
  }

  private void _includeIfUnique(
      BooleanQuery booleanQuery,
      boolean like,
      QueryParser queryParser,
      Query query,
      BooleanClause.Occur occur) {

    if (query instanceof TermQuery) {
      Set<Term> terms = new HashSet<Term>();

      TermQuery termQuery = (TermQuery) query;

      termQuery.extractTerms(terms);

      float boost = termQuery.getBoost();

      for (Term term : terms) {
        String termValue = term.text();

        if (like) {
          termValue = termValue.toLowerCase(queryParser.getLocale());

          term = term.createTerm(StringPool.STAR.concat(termValue).concat(StringPool.STAR));

          query = new WildcardQuery(term);
        } else {
          query = new TermQuery(term);
        }

        query.setBoost(boost);

        boolean included = false;

        for (BooleanClause booleanClause : booleanQuery.getClauses()) {
          if (query.equals(booleanClause.getQuery())) {
            included = true;
          }
        }

        if (!included) {
          booleanQuery.add(query, occur);
        }
      }
    } else if (query instanceof BooleanQuery) {
      BooleanQuery curBooleanQuery = (BooleanQuery) query;

      BooleanQuery containerBooleanQuery = new BooleanQuery();

      for (BooleanClause booleanClause : curBooleanQuery.getClauses()) {
        _includeIfUnique(
            containerBooleanQuery,
            like,
            queryParser,
            booleanClause.getQuery(),
            booleanClause.getOccur());
      }

      if (containerBooleanQuery.getClauses().length > 0) {
        booleanQuery.add(containerBooleanQuery, occur);
      }
    } else {
      boolean included = false;

      for (BooleanClause booleanClause : booleanQuery.getClauses()) {
        if (query.equals(booleanClause.getQuery())) {
          included = true;
        }
      }

      if (!included) {
        booleanQuery.add(query, occur);
      }
    }
  }

  private void _loadIndexFromCluster(IndexAccessor indexAccessor, long localLastGeneration) {

    List<Address> clusterNodeAddresses = ClusterExecutorUtil.getClusterNodeAddresses();

    int clusterNodeAddressesCount = clusterNodeAddresses.size();

    if (clusterNodeAddressesCount <= 1) {
      if (_log.isDebugEnabled()) {
        _log.debug(
            "Do not load indexes because there is either one portal "
                + "instance or no portal instances in the cluster");
      }

      return;
    }

    ClusterRequest clusterRequest =
        ClusterRequest.createMulticastRequest(
            new MethodHandler(_getLastGenerationMethodKey, indexAccessor.getCompanyId()), true);

    ClusterExecutorUtil.execute(
        clusterRequest,
        new LoadIndexClusterResponseCallback(
            indexAccessor, clusterNodeAddressesCount, localLastGeneration));
  }

  private static final long _CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT =
      PropsValues.CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT;

  private static final int _LUCENE_BOOLEAN_QUERY_CLAUSE_MAX_SIZE =
      GetterUtil.getInteger(
          PropsUtil.get(PropsKeys.LUCENE_BOOLEAN_QUERY_CLAUSE_MAX_SIZE),
          BooleanQuery.getMaxClauseCount());

  private static Log _log = LogFactoryUtil.getLog(LuceneHelperImpl.class);

  private static MethodKey _createTokenMethodKey =
      new MethodKey(TransientTokenUtil.class, "createToken", long.class);
  private static MethodKey _getLastGenerationMethodKey =
      new MethodKey(LuceneHelperUtil.class, "getLastGeneration", long.class);

  private Analyzer _analyzer;
  private Map<Long, IndexAccessor> _indexAccessors = new ConcurrentHashMap<Long, IndexAccessor>();
  private LoadIndexClusterEventListener _loadIndexClusterEventListener;
  private ThreadPoolExecutor _luceneIndexThreadPoolExecutor;
  private String _protocol;
  private Version _version;

  private static class ShutdownSyncJob implements Runnable {

    public ShutdownSyncJob(CountDownLatch countDownLatch) {
      _countDownLatch = countDownLatch;
    }

    @Override
    public void run() {
      _countDownLatch.countDown();

      try {
        synchronized (this) {
          wait();
        }
      } catch (InterruptedException ie) {
      }
    }

    private final CountDownLatch _countDownLatch;
  }

  private class LoadIndexClusterEventListener implements ClusterEventListener {

    @Override
    public void processClusterEvent(ClusterEvent clusterEvent) {
      ClusterEventType clusterEventType = clusterEvent.getClusterEventType();

      if (!clusterEventType.equals(ClusterEventType.JOIN)) {
        return;
      }

      List<Address> clusterNodeAddresses = ClusterExecutorUtil.getClusterNodeAddresses();
      List<ClusterNode> clusterNodes = clusterEvent.getClusterNodes();

      if ((clusterNodeAddresses.size() - clusterNodes.size()) > 1) {
        if (_log.isDebugEnabled()) {
          _log.debug("Number of original cluster members is greater than " + "one");
        }

        return;
      }

      long[] companyIds = PortalInstances.getCompanyIds();

      for (long companyId : companyIds) {
        loadIndexes(companyId);
      }

      loadIndexes(CompanyConstants.SYSTEM);
    }

    private void loadIndexes(long companyId) {
      long lastGeneration = getLastGeneration(companyId);

      if (lastGeneration == IndexAccessor.DEFAULT_LAST_GENERATION) {
        return;
      }

      try {
        LuceneClusterUtil.loadIndexesFromCluster(companyId);
      } catch (Exception e) {
        _log.error("Unable to load indexes for company " + companyId, e);
      }
    }
  }

  private class LoadIndexClusterResponseCallback extends BaseClusterResponseCallback {

    public LoadIndexClusterResponseCallback(
        IndexAccessor indexAccessor, int clusterNodeAddressesCount, long localLastGeneration) {

      _indexAccessor = indexAccessor;
      _clusterNodeAddressesCount = clusterNodeAddressesCount;
      _localLastGeneration = localLastGeneration;

      _companyId = _indexAccessor.getCompanyId();
    }

    @Override
    public void callback(BlockingQueue<ClusterNodeResponse> blockingQueue) {
      Address bootupAddress = null;

      do {
        _clusterNodeAddressesCount--;

        ClusterNodeResponse clusterNodeResponse = null;

        try {
          clusterNodeResponse =
              blockingQueue.poll(_CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);
        } catch (Exception e) {
          _log.error("Unable to get cluster node response", e);
        }

        if (clusterNodeResponse == null) {
          if (_log.isDebugEnabled()) {
            _log.debug(
                "Unable to get cluster node response in "
                    + _CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT
                    + TimeUnit.MILLISECONDS);
          }

          continue;
        }

        ClusterNode clusterNode = clusterNodeResponse.getClusterNode();

        if (clusterNode.getPortalInetSocketAddress() != null) {
          try {
            long remoteLastGeneration = (Long) clusterNodeResponse.getResult();

            if (remoteLastGeneration > _localLastGeneration) {
              bootupAddress = clusterNodeResponse.getAddress();

              break;
            }
          } catch (Exception e) {
            if (_log.isDebugEnabled()) {
              _log.debug("Suppress exception caused by remote method " + "invocation", e);
            }

            continue;
          }
        } else if (_log.isDebugEnabled()) {
          _log.debug("Cluster node " + clusterNode + " has invalid InetSocketAddress");
        }
      } while ((bootupAddress == null) && (_clusterNodeAddressesCount > 1));

      if (bootupAddress == null) {
        return;
      }

      if (_log.isInfoEnabled()) {
        _log.info("Start loading lucene index files from cluster node " + bootupAddress);
      }

      InputStream inputStream = null;

      try {
        inputStream = getLoadIndexesInputStreamFromCluster(_companyId, bootupAddress);

        _indexAccessor.loadIndex(inputStream);

        if (_log.isInfoEnabled()) {
          _log.info("Lucene index files loaded successfully");
        }
      } catch (Exception e) {
        _log.error("Unable to load index for company " + _companyId, e);
      } finally {
        if (inputStream != null) {
          try {
            inputStream.close();
          } catch (IOException ioe) {
            _log.error("Unable to close input stream for company " + _companyId, ioe);
          }
        }
      }
    }

    @Override
    public void processTimeoutException(TimeoutException timeoutException) {
      _log.error("Unable to load index for company " + _companyId, timeoutException);
    }

    private int _clusterNodeAddressesCount;
    private long _companyId;
    private IndexAccessor _indexAccessor;
    private long _localLastGeneration;
  }
}
Beispiel #3
0
  /**
   * Creates a configuration instance from a resource loader, a configuration name and a stream. If
   * the stream is null, the resource loader will open the configuration stream. If the stream is
   * not null, no attempt to load the resource will occur (the name is not used).
   *
   * @param loader the resource loader
   * @param name the configuration name
   * @param is the configuration stream
   */
  public SolrConfig(SolrResourceLoader loader, String name, InputSource is)
      throws ParserConfigurationException, IOException, SAXException {
    super(loader, name, is, "/config/");
    getOverlay(); // just in case it is not initialized
    getRequestParams();
    initLibs();
    luceneMatchVersion = getLuceneVersion("luceneMatchVersion");
    String indexConfigPrefix;

    // Old indexDefaults and mainIndex sections are deprecated and fails fast for
    // luceneMatchVersion=>LUCENE_4_0_0.
    // For older solrconfig.xml's we allow the old sections, but never mixed with the new
    // <indexConfig>
    boolean hasDeprecatedIndexConfig =
        (getNode("indexDefaults", false) != null) || (getNode("mainIndex", false) != null);
    if (hasDeprecatedIndexConfig) {
      throw new SolrException(
          ErrorCode.FORBIDDEN,
          "<indexDefaults> and <mainIndex> configuration sections are discontinued. Use <indexConfig> instead.");
    } else {
      defaultIndexConfig = mainIndexConfig = null;
      indexConfigPrefix = "indexConfig";
    }
    assertWarnOrFail(
        "The <nrtMode> config has been discontinued and NRT mode is always used by Solr."
            + " This config will be removed in future versions.",
        getNode(indexConfigPrefix + "/nrtMode", false) == null,
        true);
    assertWarnOrFail(
        "Solr no longer supports forceful unlocking via the 'unlockOnStartup' option.  "
            + "This is no longer neccessary for the default lockType except in situations where "
            + "it would be dangerous and should not be done.  For other lockTypes and/or "
            + "directoryFactory options it may also be dangerous and users must resolve "
            + "problematic locks manually.",
        null == getNode(indexConfigPrefix + "/unlockOnStartup", false),
        true // 'fail' in trunk
        );

    // Parse indexConfig section, using mainIndex as backup in case old config is used
    indexConfig = new SolrIndexConfig(this, "indexConfig", mainIndexConfig);

    booleanQueryMaxClauseCount =
        getInt("query/maxBooleanClauses", BooleanQuery.getMaxClauseCount());
    log.info("Using Lucene MatchVersion: " + luceneMatchVersion);

    // Warn about deprecated / discontinued parameters
    // boolToFilterOptimizer has had no effect since 3.1
    if (get("query/boolTofilterOptimizer", null) != null)
      log.warn(
          "solrconfig.xml: <boolTofilterOptimizer> is currently not implemented and has no effect.");
    if (get("query/HashDocSet", null) != null)
      log.warn("solrconfig.xml: <HashDocSet> is deprecated and no longer recommended used.");

    // TODO: Old code - in case somebody wants to re-enable. Also see SolrIndexSearcher#search()
    //    filtOptEnabled = getBool("query/boolTofilterOptimizer/@enabled", false);
    //    filtOptCacheSize = getInt("query/boolTofilterOptimizer/@cacheSize",32);
    //    filtOptThreshold = getFloat("query/boolTofilterOptimizer/@threshold",.05f);

    useFilterForSortedQuery = getBool("query/useFilterForSortedQuery", false);
    queryResultWindowSize = Math.max(1, getInt("query/queryResultWindowSize", 1));
    queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE);
    enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false);

    filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache");
    queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache");
    documentCacheConfig = CacheConfig.getConfig(this, "query/documentCache");
    CacheConfig conf = CacheConfig.getConfig(this, "query/fieldValueCache");
    if (conf == null) {
      Map<String, String> args = new HashMap<>();
      args.put(NAME, "fieldValueCache");
      args.put("size", "10000");
      args.put("initialSize", "10");
      args.put("showItems", "-1");
      conf = new CacheConfig(FastLRUCache.class, args, null);
    }
    fieldValueCacheConfig = conf;
    useColdSearcher = getBool("query/useColdSearcher", false);
    dataDir = get("dataDir", null);
    if (dataDir != null && dataDir.length() == 0) dataDir = null;

    userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");

    org.apache.solr.search.SolrIndexSearcher.initRegenerators(this);

    hashSetInverseLoadFactor = 1.0f / getFloat("//HashDocSet/@loadFactor", 0.75f);
    hashDocSetMaxSize = getInt("//HashDocSet/@maxSize", 3000);

    httpCachingConfig = new HttpCachingConfig(this);

    Node jmx = getNode("jmx", false);
    if (jmx != null) {
      jmxConfig =
          new JmxConfiguration(
              true,
              get("jmx/@agentId", null),
              get("jmx/@serviceUrl", null),
              get("jmx/@rootName", null));

    } else {
      jmxConfig = new JmxConfiguration(false, null, null, null);
    }
    maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE);
    slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
    for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);
    updateHandlerInfo = loadUpdatehandlerInfo();

    multipartUploadLimitKB =
        getInt("requestDispatcher/requestParsers/@multipartUploadLimitInKB", 2048);

    formUploadLimitKB = getInt("requestDispatcher/requestParsers/@formdataUploadLimitInKB", 2048);

    enableRemoteStreams = getBool("requestDispatcher/requestParsers/@enableRemoteStreaming", false);

    // Let this filter take care of /select?xxx format
    handleSelect = getBool("requestDispatcher/@handleSelect", true);

    addHttpRequestToContext =
        getBool("requestDispatcher/requestParsers/@addHttpRequestToContext", false);

    List<PluginInfo> argsInfos = getPluginInfos(InitParams.class.getName());
    if (argsInfos != null) {
      Map<String, InitParams> argsMap = new HashMap<>();
      for (PluginInfo p : argsInfos) {
        InitParams args = new InitParams(p);
        argsMap.put(args.name == null ? String.valueOf(args.hashCode()) : args.name, args);
      }
      this.initParams = Collections.unmodifiableMap(argsMap);
    }

    solrRequestParsers = new SolrRequestParsers(this);
    Config.log.info("Loaded SolrConfig: " + name);
  }
 public static int getMaxClauseCount() {
   return BooleanQuery.getMaxClauseCount();
 }
Beispiel #5
0
  /**
   * Creates a configuration instance from a resource loader, a configuration name and a stream. If
   * the stream is null, the resource loader will open the configuration stream. If the stream is
   * not null, no attempt to load the resource will occur (the name is not used).
   *
   * @param loader the resource loader
   * @param name the configuration name
   * @param is the configuration stream
   */
  public SolrConfig(SolrResourceLoader loader, String name, InputSource is)
      throws ParserConfigurationException, IOException, SAXException {
    super(loader, name, is, "/config/");
    initLibs();
    luceneMatchVersion = getLuceneVersion("luceneMatchVersion");
    String indexConfigPrefix;

    // Old indexDefaults and mainIndex sections are deprecated and fails fast for
    // luceneMatchVersion=>LUCENE_40.
    // For older solrconfig.xml's we allow the old sections, but never mixed with the new
    // <indexConfig>
    boolean hasDeprecatedIndexConfig =
        get("indexDefaults/text()", null) != null || get("mainIndex/text()", null) != null;
    boolean hasNewIndexConfig = get("indexConfig/text()", null) != null;
    if (hasDeprecatedIndexConfig) {
      if (luceneMatchVersion.onOrAfter(Version.LUCENE_40)) {
        throw new SolrException(
            ErrorCode.FORBIDDEN,
            "<indexDefaults> and <mainIndex> configuration sections are discontinued. Use <indexConfig> instead.");
      } else {
        // Still allow the old sections for older LuceneMatchVersion's
        if (hasNewIndexConfig) {
          throw new SolrException(
              ErrorCode.FORBIDDEN,
              "Cannot specify both <indexDefaults>, <mainIndex> and <indexConfig> at the same time. Please use <indexConfig> only.");
        }
        log.warn(
            "<indexDefaults> and <mainIndex> configuration sections are deprecated and will fail for luceneMatchVersion=LUCENE_40 and later. Please use <indexConfig> instead.");
        defaultIndexConfig = new SolrIndexConfig(this, "indexDefaults", null);
        mainIndexConfig = new SolrIndexConfig(this, "mainIndex", defaultIndexConfig);
        indexConfigPrefix = "mainIndex";
      }
    } else {
      defaultIndexConfig = mainIndexConfig = null;
      indexConfigPrefix = "indexConfig";
    }
    reopenReaders = getBool(indexConfigPrefix + "/reopenReaders", true);
    // Parse indexConfig section, using mainIndex as backup in case old config is used
    indexConfig = new SolrIndexConfig(this, "indexConfig", mainIndexConfig);

    booleanQueryMaxClauseCount =
        getInt("query/maxBooleanClauses", BooleanQuery.getMaxClauseCount());
    log.info("Using Lucene MatchVersion: " + luceneMatchVersion);

    // Warn about deprecated / discontinued parameters
    // boolToFilterOptimizer has had no effect since 3.1
    if (get("query/boolTofilterOptimizer", null) != null)
      log.warn(
          "solrconfig.xml: <boolTofilterOptimizer> is currently not implemented and has no effect.");
    if (get("query/HashDocSet", null) != null)
      log.warn("solrconfig.xml: <HashDocSet> is deprecated and no longer recommended used.");

    // TODO: Old code - in case somebody wants to re-enable. Also see SolrIndexSearcher#search()
    //    filtOptEnabled = getBool("query/boolTofilterOptimizer/@enabled", false);
    //    filtOptCacheSize = getInt("query/boolTofilterOptimizer/@cacheSize",32);
    //    filtOptThreshold = getFloat("query/boolTofilterOptimizer/@threshold",.05f);

    useFilterForSortedQuery = getBool("query/useFilterForSortedQuery", false);
    queryResultWindowSize = Math.max(1, getInt("query/queryResultWindowSize", 1));
    queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE);
    enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false);

    filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache");
    queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache");
    documentCacheConfig = CacheConfig.getConfig(this, "query/documentCache");
    CacheConfig conf = CacheConfig.getConfig(this, "query/fieldValueCache");
    if (conf == null) {
      Map<String, String> args = new HashMap<String, String>();
      args.put("name", "fieldValueCache");
      args.put("size", "10000");
      args.put("initialSize", "10");
      args.put("showItems", "-1");
      conf = new CacheConfig(FastLRUCache.class, args, null);
    }
    fieldValueCacheConfig = conf;
    unlockOnStartup = getBool(indexConfigPrefix + "/unlockOnStartup", false);
    useColdSearcher = getBool("query/useColdSearcher", false);
    dataDir = get("dataDir", null);
    if (dataDir != null && dataDir.length() == 0) dataDir = null;

    userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");

    org.apache.solr.search.SolrIndexSearcher.initRegenerators(this);

    hashSetInverseLoadFactor = 1.0f / getFloat("//HashDocSet/@loadFactor", 0.75f);
    hashDocSetMaxSize = getInt("//HashDocSet/@maxSize", 3000);

    httpCachingConfig = new HttpCachingConfig(this);

    Node jmx = getNode("jmx", false);
    if (jmx != null) {
      jmxConfig =
          new JmxConfiguration(
              true,
              get("jmx/@agentId", null),
              get("jmx/@serviceUrl", null),
              get("jmx/@rootName", null));

    } else {
      jmxConfig = new JmxConfiguration(false, null, null, null);
    }
    maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE);

    loadPluginInfo(SolrRequestHandler.class, "requestHandler", true, true);
    loadPluginInfo(QParserPlugin.class, "queryParser", true, true);
    loadPluginInfo(QueryResponseWriter.class, "queryResponseWriter", true, true);
    loadPluginInfo(ValueSourceParser.class, "valueSourceParser", true, true);
    loadPluginInfo(TransformerFactory.class, "transformer", true, true);
    loadPluginInfo(SearchComponent.class, "searchComponent", true, true);
    loadPluginInfo(QueryConverter.class, "queryConverter", true, true);

    // this is hackish, since it picks up all SolrEventListeners,
    // regardless of when/how/why they are used (or even if they are
    // declared outside of the appropriate context) but there's no nice
    // way around that in the PluginInfo framework
    loadPluginInfo(SolrEventListener.class, "//listener", false, true);

    loadPluginInfo(DirectoryFactory.class, "directoryFactory", false, true);
    loadPluginInfo(IndexDeletionPolicy.class, indexConfigPrefix + "/deletionPolicy", false, true);
    loadPluginInfo(CodecFactory.class, "codecFactory", false, false);
    loadPluginInfo(IndexReaderFactory.class, "indexReaderFactory", false, true);
    loadPluginInfo(UpdateRequestProcessorChain.class, "updateRequestProcessorChain", false, false);
    loadPluginInfo(UpdateLog.class, "updateHandler/updateLog", false, false);
    loadPluginInfo(IndexSchemaFactory.class, "schemaFactory", false, true);

    updateHandlerInfo = loadUpdatehandlerInfo();

    Config.log.info("Loaded SolrConfig: " + name);
  }