@Override
public void setUp() throws Exception {
super.setUp();
originalMaxClauses = BooleanQuery.getMaxClauseCount();
}
/**
* @author Brian Wing Shun Chan
* @author Harry Mark
* @author Bruno Farache
* @author Shuyang Zhou
* @author Tina Tian
* @author Hugo Huijser
* @author Andrea Di Giorgi
*/
public class LuceneHelperImpl implements LuceneHelper {
@Override
public void addDocument(long companyId, Document document) throws IOException {
IndexAccessor indexAccessor = getIndexAccessor(companyId);
indexAccessor.addDocument(document);
}
@Override
public void addExactTerm(BooleanQuery booleanQuery, String field, String value) {
addTerm(booleanQuery, field, value, false);
}
@Override
public void addNumericRangeTerm(
BooleanQuery booleanQuery, String field, Integer startValue, Integer endValue) {
NumericRangeQuery<?> numericRangeQuery =
NumericRangeQuery.newIntRange(field, startValue, endValue, true, true);
booleanQuery.add(numericRangeQuery, BooleanClause.Occur.SHOULD);
}
@Override
public void addNumericRangeTerm(
BooleanQuery booleanQuery, String field, Long startValue, Long endValue) {
NumericRangeQuery<?> numericRangeQuery =
NumericRangeQuery.newLongRange(field, startValue, endValue, true, true);
booleanQuery.add(numericRangeQuery, BooleanClause.Occur.SHOULD);
}
/**
* @deprecated As of 6.2.0, replaced by {@link #addNumericRangeTerm(BooleanQuery, String, Long,
* Long)}
*/
@Deprecated
@Override
public void addNumericRangeTerm(
BooleanQuery booleanQuery, String field, String startValue, String endValue) {
addNumericRangeTerm(
booleanQuery, field, GetterUtil.getLong(startValue), GetterUtil.getLong(endValue));
}
@Override
public void addRangeTerm(
BooleanQuery booleanQuery, String field, String startValue, String endValue) {
boolean includesLower = true;
if ((startValue != null) && startValue.equals(StringPool.STAR)) {
includesLower = false;
}
boolean includesUpper = true;
if ((endValue != null) && endValue.equals(StringPool.STAR)) {
includesUpper = false;
}
TermRangeQuery termRangeQuery =
new TermRangeQuery(field, startValue, endValue, includesLower, includesUpper);
booleanQuery.add(termRangeQuery, BooleanClause.Occur.SHOULD);
}
@Override
public void addRequiredTerm(BooleanQuery booleanQuery, String field, String value, boolean like) {
addRequiredTerm(booleanQuery, field, new String[] {value}, like);
}
@Override
public void addRequiredTerm(
BooleanQuery booleanQuery, String field, String[] values, boolean like) {
if (values == null) {
return;
}
BooleanQuery query = new BooleanQuery();
for (String value : values) {
addTerm(query, field, value, like);
}
booleanQuery.add(query, BooleanClause.Occur.MUST);
}
@Override
public void addTerm(BooleanQuery booleanQuery, String field, String value, boolean like) {
addTerm(booleanQuery, field, value, like, BooleanClauseOccur.SHOULD);
}
@Override
public void addTerm(
BooleanQuery booleanQuery,
String field,
String value,
boolean like,
BooleanClauseOccur booleanClauseOccur) {
if (Validator.isNull(value)) {
return;
}
Analyzer analyzer = getAnalyzer();
if (analyzer instanceof PerFieldAnalyzer) {
PerFieldAnalyzer perFieldAnalyzer = (PerFieldAnalyzer) analyzer;
Analyzer fieldAnalyzer = perFieldAnalyzer.getAnalyzer(field);
if (fieldAnalyzer instanceof LikeKeywordAnalyzer) {
like = true;
}
}
if (like) {
value = StringUtil.replace(value, StringPool.PERCENT, StringPool.BLANK);
}
try {
QueryParser queryParser = new QueryParser(getVersion(), field, analyzer);
Query query = queryParser.parse(value);
BooleanClause.Occur occur = null;
if (booleanClauseOccur.equals(BooleanClauseOccur.MUST)) {
occur = BooleanClause.Occur.MUST;
} else if (booleanClauseOccur.equals(BooleanClauseOccur.MUST_NOT)) {
occur = BooleanClause.Occur.MUST_NOT;
} else {
occur = BooleanClause.Occur.SHOULD;
}
_includeIfUnique(booleanQuery, like, queryParser, query, occur);
} catch (Exception e) {
if (_log.isWarnEnabled()) {
_log.warn(e, e);
}
}
}
@Override
public void addTerm(BooleanQuery booleanQuery, String field, String[] values, boolean like) {
for (String value : values) {
addTerm(booleanQuery, field, value, like);
}
}
/** @deprecated As of 7.0.0, replaced by {@link #releaseIndexSearcher(long, IndexSearcher)} */
@Deprecated
@Override
public void cleanUp(IndexSearcher indexSearcher) {
if (indexSearcher == null) {
return;
}
try {
indexSearcher.close();
IndexReader indexReader = indexSearcher.getIndexReader();
if (indexReader != null) {
indexReader.close();
}
} catch (IOException ioe) {
_log.error(ioe, ioe);
}
}
@Override
public int countScoredFieldNames(Query query, String[] filedNames) {
int count = 0;
for (String fieldName : filedNames) {
WeightedTerm[] weightedTerms = QueryTermExtractor.getTerms(query, false, fieldName);
if ((weightedTerms.length > 0)
&& !ArrayUtil.contains(Field.UNSCORED_FIELD_NAMES, fieldName)) {
count++;
}
}
return count;
}
@Override
public void delete(long companyId) {
IndexAccessor indexAccessor = _indexAccessors.get(companyId);
if (indexAccessor == null) {
return;
}
indexAccessor.delete();
}
@Override
public void deleteDocuments(long companyId, Term term) throws IOException {
IndexAccessor indexAccessor = _indexAccessors.get(companyId);
if (indexAccessor == null) {
return;
}
indexAccessor.deleteDocuments(term);
}
@Override
public void dumpIndex(long companyId, OutputStream outputStream) throws IOException {
long lastGeneration = getLastGeneration(companyId);
if (lastGeneration == IndexAccessor.DEFAULT_LAST_GENERATION) {
if (_log.isDebugEnabled()) {
_log.debug("Dump index from cluster is not enabled for " + companyId);
}
return;
}
IndexAccessor indexAccessor = _indexAccessors.get(companyId);
if (indexAccessor == null) {
return;
}
indexAccessor.dumpIndex(outputStream);
}
@Override
public Analyzer getAnalyzer() {
return _analyzer;
}
@Override
public IndexAccessor getIndexAccessor(long companyId) {
IndexAccessor indexAccessor = _indexAccessors.get(companyId);
if (indexAccessor != null) {
return indexAccessor;
}
synchronized (this) {
indexAccessor = _indexAccessors.get(companyId);
if (indexAccessor == null) {
indexAccessor = new IndexAccessorImpl(companyId);
if (isLoadIndexFromClusterEnabled()) {
indexAccessor = new SynchronizedIndexAccessorImpl(indexAccessor);
boolean clusterForwardMessage =
GetterUtil.getBoolean(
MessageValuesThreadLocal.getValue(ClusterLink.CLUSTER_FORWARD_MESSAGE));
if (clusterForwardMessage) {
if (_log.isInfoEnabled()) {
_log.info(
"Skip Luncene index files cluster loading "
+ "since this is a manual reindex request");
}
} else {
try {
_loadIndexFromCluster(indexAccessor, indexAccessor.getLastGeneration());
} catch (Exception e) {
_log.error("Unable to load index for company " + indexAccessor.getCompanyId(), e);
}
}
}
_indexAccessors.put(companyId, indexAccessor);
}
}
return indexAccessor;
}
@Override
public IndexSearcher getIndexSearcher(long companyId) throws IOException {
IndexAccessor indexAccessor = getIndexAccessor(companyId);
return indexAccessor.acquireIndexSearcher();
}
@Override
public long getLastGeneration(long companyId) {
if (!isLoadIndexFromClusterEnabled()) {
return IndexAccessor.DEFAULT_LAST_GENERATION;
}
IndexAccessor indexAccessor = _indexAccessors.get(companyId);
if (indexAccessor == null) {
return IndexAccessor.DEFAULT_LAST_GENERATION;
}
return indexAccessor.getLastGeneration();
}
@Override
public InputStream getLoadIndexesInputStreamFromCluster(long companyId, Address bootupAddress) {
if (!isLoadIndexFromClusterEnabled()) {
return null;
}
InputStream inputStream = null;
try {
ObjectValuePair<String, URL> bootupClusterNodeObjectValuePair =
_getBootupClusterNodeObjectValuePair(bootupAddress);
URL url = bootupClusterNodeObjectValuePair.getValue();
URLConnection urlConnection = url.openConnection();
urlConnection.setDoOutput(true);
UnsyncPrintWriter unsyncPrintWriter =
UnsyncPrintWriterPool.borrow(urlConnection.getOutputStream());
unsyncPrintWriter.write("transientToken=");
unsyncPrintWriter.write(bootupClusterNodeObjectValuePair.getKey());
unsyncPrintWriter.write("&companyId=");
unsyncPrintWriter.write(String.valueOf(companyId));
unsyncPrintWriter.close();
inputStream = urlConnection.getInputStream();
return inputStream;
} catch (IOException ioe) {
throw new SystemException(ioe);
}
}
@Override
public Set<String> getQueryTerms(Query query) {
String queryString = StringUtil.replace(query.toString(), StringPool.STAR, StringPool.BLANK);
Query tempQuery = null;
try {
QueryParser queryParser = new QueryParser(getVersion(), StringPool.BLANK, getAnalyzer());
tempQuery = queryParser.parse(queryString);
} catch (Exception e) {
if (_log.isWarnEnabled()) {
_log.warn("Unable to parse " + queryString);
}
tempQuery = query;
}
WeightedTerm[] weightedTerms = null;
for (String fieldName : Field.KEYWORDS) {
weightedTerms = QueryTermExtractor.getTerms(tempQuery, false, fieldName);
if (weightedTerms.length > 0) {
break;
}
}
Set<String> queryTerms = new HashSet<String>();
for (WeightedTerm weightedTerm : weightedTerms) {
queryTerms.add(weightedTerm.getTerm());
}
return queryTerms;
}
/** @deprecated As of 7.0.0, replaced by {@link #getIndexSearcher(long)} */
@Deprecated
@Override
public IndexSearcher getSearcher(long companyId, boolean readOnly) throws IOException {
IndexAccessor indexAccessor = getIndexAccessor(companyId);
IndexReader indexReader = IndexReader.open(indexAccessor.getLuceneDir(), readOnly);
IndexSearcher indexSearcher = new IndexSearcher(indexReader);
indexSearcher.setDefaultFieldSortScoring(true, false);
indexSearcher.setSimilarity(new FieldWeightSimilarity());
return indexSearcher;
}
@Override
public String getSnippet(
Query query,
String field,
String s,
int maxNumFragments,
int fragmentLength,
String fragmentSuffix,
Formatter formatter)
throws IOException {
QueryScorer queryScorer = new QueryScorer(query, field);
Highlighter highlighter = new Highlighter(formatter, queryScorer);
highlighter.setTextFragmenter(new SimpleFragmenter(fragmentLength));
TokenStream tokenStream = getAnalyzer().tokenStream(field, new UnsyncStringReader(s));
try {
String snippet =
highlighter.getBestFragments(tokenStream, s, maxNumFragments, fragmentSuffix);
if (Validator.isNotNull(snippet)
&& !StringUtil.endsWith(snippet, fragmentSuffix)
&& !s.equals(snippet)) {
snippet = snippet.concat(fragmentSuffix);
}
return snippet;
} catch (InvalidTokenOffsetsException itoe) {
throw new IOException(itoe);
}
}
@Override
public Version getVersion() {
return _version;
}
@Override
public boolean isLoadIndexFromClusterEnabled() {
if (PropsValues.CLUSTER_LINK_ENABLED && PropsValues.LUCENE_REPLICATE_WRITE) {
return true;
}
if (_log.isDebugEnabled()) {
_log.debug("Load index from cluster is not enabled");
}
return false;
}
@Override
public void loadIndex(long companyId, InputStream inputStream) throws IOException {
if (!isLoadIndexFromClusterEnabled()) {
return;
}
IndexAccessor indexAccessor = _indexAccessors.get(companyId);
if (indexAccessor == null) {
if (_log.isInfoEnabled()) {
_log.info(
"Skip loading Lucene index files for company "
+ companyId
+ " in favor of lazy loading");
}
return;
}
StopWatch stopWatch = new StopWatch();
stopWatch.start();
if (_log.isInfoEnabled()) {
_log.info("Start loading Lucene index files for company " + companyId);
}
indexAccessor.loadIndex(inputStream);
if (_log.isInfoEnabled()) {
_log.info(
"Finished loading index files for company "
+ companyId
+ " in "
+ stopWatch.getTime()
+ " ms");
}
}
@Override
public void loadIndexesFromCluster(long companyId) {
if (!isLoadIndexFromClusterEnabled()) {
return;
}
IndexAccessor indexAccessor = _indexAccessors.get(companyId);
if (indexAccessor == null) {
return;
}
long localLastGeneration = getLastGeneration(companyId);
_loadIndexFromCluster(indexAccessor, localLastGeneration);
}
@Override
public void releaseIndexSearcher(long companyId, IndexSearcher indexSearcher) throws IOException {
IndexAccessor indexAccessor = getIndexAccessor(companyId);
indexAccessor.releaseIndexSearcher(indexSearcher);
}
public void setAnalyzer(Analyzer analyzer) {
_analyzer = analyzer;
}
public void setVersion(Version version) {
_version = version;
}
@Override
public void shutdown() {
if (_luceneIndexThreadPoolExecutor != null) {
_luceneIndexThreadPoolExecutor.shutdownNow();
try {
_luceneIndexThreadPoolExecutor.awaitTermination(60, TimeUnit.SECONDS);
} catch (InterruptedException ie) {
_log.error("Lucene indexer shutdown interrupted", ie);
}
}
if (isLoadIndexFromClusterEnabled()) {
ClusterExecutorUtil.removeClusterEventListener(_loadIndexClusterEventListener);
}
MessageBus messageBus = MessageBusUtil.getMessageBus();
for (String searchEngineId : SearchEngineUtil.getSearchEngineIds()) {
String searchWriterDestinationName =
SearchEngineUtil.getSearchWriterDestinationName(searchEngineId);
Destination searchWriteDestination = messageBus.getDestination(searchWriterDestinationName);
if (searchWriteDestination != null) {
ThreadPoolExecutor threadPoolExecutor =
PortalExecutorManagerUtil.getPortalExecutor(searchWriterDestinationName);
int maxPoolSize = threadPoolExecutor.getMaxPoolSize();
CountDownLatch countDownLatch = new CountDownLatch(maxPoolSize);
ShutdownSyncJob shutdownSyncJob = new ShutdownSyncJob(countDownLatch);
for (int i = 0; i < maxPoolSize; i++) {
threadPoolExecutor.submit(shutdownSyncJob);
}
try {
countDownLatch.await();
} catch (InterruptedException ie) {
_log.error("Shutdown waiting interrupted", ie);
}
List<Runnable> runnables = threadPoolExecutor.shutdownNow();
if (_log.isDebugEnabled()) {
_log.debug("Cancelled appending indexing jobs: " + runnables);
}
searchWriteDestination.close(true);
}
}
for (IndexAccessor indexAccessor : _indexAccessors.values()) {
indexAccessor.close();
}
}
@Override
public void shutdown(long companyId) {
IndexAccessor indexAccessor = getIndexAccessor(companyId);
_indexAccessors.remove(companyId);
indexAccessor.close();
}
@Override
public void startup(long companyId) {
if (!PropsValues.INDEX_ON_STARTUP) {
return;
}
if (_log.isInfoEnabled()) {
_log.info("Indexing Lucene on startup");
}
LuceneIndexer luceneIndexer = new LuceneIndexer(companyId);
if (PropsValues.INDEX_WITH_THREAD) {
if (_luceneIndexThreadPoolExecutor == null) {
// This should never be null except for the case where
// VerifyProcessUtil#_verifyProcess(boolean) sets
// PropsValues#INDEX_ON_STARTUP to true.
_luceneIndexThreadPoolExecutor =
PortalExecutorManagerUtil.getPortalExecutor(LuceneHelperImpl.class.getName());
}
_luceneIndexThreadPoolExecutor.execute(luceneIndexer);
} else {
luceneIndexer.reindex();
}
}
@Override
public void updateDocument(long companyId, Term term, Document document) throws IOException {
IndexAccessor indexAccessor = getIndexAccessor(companyId);
indexAccessor.updateDocument(term, document);
}
private LuceneHelperImpl() {
if (PropsValues.INDEX_ON_STARTUP && PropsValues.INDEX_WITH_THREAD) {
_luceneIndexThreadPoolExecutor =
PortalExecutorManagerUtil.getPortalExecutor(LuceneHelperImpl.class.getName());
}
if (isLoadIndexFromClusterEnabled()) {
_loadIndexClusterEventListener = new LoadIndexClusterEventListener();
ClusterExecutorUtil.addClusterEventListener(_loadIndexClusterEventListener);
}
BooleanQuery.setMaxClauseCount(_LUCENE_BOOLEAN_QUERY_CLAUSE_MAX_SIZE);
if (StringUtil.equalsIgnoreCase(Http.HTTPS, PropsValues.WEB_SERVER_PROTOCOL)) {
_protocol = Http.HTTPS;
} else {
_protocol = Http.HTTP;
}
}
private ObjectValuePair<String, URL> _getBootupClusterNodeObjectValuePair(Address bootupAddress) {
ClusterRequest clusterRequest =
ClusterRequest.createUnicastRequest(
new MethodHandler(_createTokenMethodKey, _CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT),
bootupAddress);
FutureClusterResponses futureClusterResponses = ClusterExecutorUtil.execute(clusterRequest);
BlockingQueue<ClusterNodeResponse> clusterNodeResponses =
futureClusterResponses.getPartialResults();
try {
ClusterNodeResponse clusterNodeResponse =
clusterNodeResponses.poll(
_CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);
ClusterNode clusterNode = clusterNodeResponse.getClusterNode();
InetSocketAddress inetSocketAddress = clusterNode.getPortalInetSocketAddress();
if (inetSocketAddress == null) {
StringBundler sb = new StringBundler(6);
sb.append("Invalid cluster node InetSocketAddress ");
sb.append(". The InetSocketAddress is set by the first ");
sb.append("request or configured in portal.properties by the");
sb.append("properties ");
sb.append("\"portal.instance.http.inet.socket.address\" and ");
sb.append("\"portal.instance.https.inet.socket.address\".");
throw new Exception(sb.toString());
}
InetAddress inetAddress = inetSocketAddress.getAddress();
String fileName = PortalUtil.getPathContext();
if (!fileName.endsWith(StringPool.SLASH)) {
fileName = fileName.concat(StringPool.SLASH);
}
fileName = fileName.concat("lucene/dump");
URL url =
new URL(_protocol, inetAddress.getHostAddress(), inetSocketAddress.getPort(), fileName);
String transientToken = (String) clusterNodeResponse.getResult();
return new ObjectValuePair<String, URL>(transientToken, url);
} catch (Exception e) {
throw new SystemException(e);
}
}
private void _includeIfUnique(
BooleanQuery booleanQuery,
boolean like,
QueryParser queryParser,
Query query,
BooleanClause.Occur occur) {
if (query instanceof TermQuery) {
Set<Term> terms = new HashSet<Term>();
TermQuery termQuery = (TermQuery) query;
termQuery.extractTerms(terms);
float boost = termQuery.getBoost();
for (Term term : terms) {
String termValue = term.text();
if (like) {
termValue = termValue.toLowerCase(queryParser.getLocale());
term = term.createTerm(StringPool.STAR.concat(termValue).concat(StringPool.STAR));
query = new WildcardQuery(term);
} else {
query = new TermQuery(term);
}
query.setBoost(boost);
boolean included = false;
for (BooleanClause booleanClause : booleanQuery.getClauses()) {
if (query.equals(booleanClause.getQuery())) {
included = true;
}
}
if (!included) {
booleanQuery.add(query, occur);
}
}
} else if (query instanceof BooleanQuery) {
BooleanQuery curBooleanQuery = (BooleanQuery) query;
BooleanQuery containerBooleanQuery = new BooleanQuery();
for (BooleanClause booleanClause : curBooleanQuery.getClauses()) {
_includeIfUnique(
containerBooleanQuery,
like,
queryParser,
booleanClause.getQuery(),
booleanClause.getOccur());
}
if (containerBooleanQuery.getClauses().length > 0) {
booleanQuery.add(containerBooleanQuery, occur);
}
} else {
boolean included = false;
for (BooleanClause booleanClause : booleanQuery.getClauses()) {
if (query.equals(booleanClause.getQuery())) {
included = true;
}
}
if (!included) {
booleanQuery.add(query, occur);
}
}
}
private void _loadIndexFromCluster(IndexAccessor indexAccessor, long localLastGeneration) {
List<Address> clusterNodeAddresses = ClusterExecutorUtil.getClusterNodeAddresses();
int clusterNodeAddressesCount = clusterNodeAddresses.size();
if (clusterNodeAddressesCount <= 1) {
if (_log.isDebugEnabled()) {
_log.debug(
"Do not load indexes because there is either one portal "
+ "instance or no portal instances in the cluster");
}
return;
}
ClusterRequest clusterRequest =
ClusterRequest.createMulticastRequest(
new MethodHandler(_getLastGenerationMethodKey, indexAccessor.getCompanyId()), true);
ClusterExecutorUtil.execute(
clusterRequest,
new LoadIndexClusterResponseCallback(
indexAccessor, clusterNodeAddressesCount, localLastGeneration));
}
private static final long _CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT =
PropsValues.CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT;
private static final int _LUCENE_BOOLEAN_QUERY_CLAUSE_MAX_SIZE =
GetterUtil.getInteger(
PropsUtil.get(PropsKeys.LUCENE_BOOLEAN_QUERY_CLAUSE_MAX_SIZE),
BooleanQuery.getMaxClauseCount());
private static Log _log = LogFactoryUtil.getLog(LuceneHelperImpl.class);
private static MethodKey _createTokenMethodKey =
new MethodKey(TransientTokenUtil.class, "createToken", long.class);
private static MethodKey _getLastGenerationMethodKey =
new MethodKey(LuceneHelperUtil.class, "getLastGeneration", long.class);
private Analyzer _analyzer;
private Map<Long, IndexAccessor> _indexAccessors = new ConcurrentHashMap<Long, IndexAccessor>();
private LoadIndexClusterEventListener _loadIndexClusterEventListener;
private ThreadPoolExecutor _luceneIndexThreadPoolExecutor;
private String _protocol;
private Version _version;
private static class ShutdownSyncJob implements Runnable {
public ShutdownSyncJob(CountDownLatch countDownLatch) {
_countDownLatch = countDownLatch;
}
@Override
public void run() {
_countDownLatch.countDown();
try {
synchronized (this) {
wait();
}
} catch (InterruptedException ie) {
}
}
private final CountDownLatch _countDownLatch;
}
private class LoadIndexClusterEventListener implements ClusterEventListener {
@Override
public void processClusterEvent(ClusterEvent clusterEvent) {
ClusterEventType clusterEventType = clusterEvent.getClusterEventType();
if (!clusterEventType.equals(ClusterEventType.JOIN)) {
return;
}
List<Address> clusterNodeAddresses = ClusterExecutorUtil.getClusterNodeAddresses();
List<ClusterNode> clusterNodes = clusterEvent.getClusterNodes();
if ((clusterNodeAddresses.size() - clusterNodes.size()) > 1) {
if (_log.isDebugEnabled()) {
_log.debug("Number of original cluster members is greater than " + "one");
}
return;
}
long[] companyIds = PortalInstances.getCompanyIds();
for (long companyId : companyIds) {
loadIndexes(companyId);
}
loadIndexes(CompanyConstants.SYSTEM);
}
private void loadIndexes(long companyId) {
long lastGeneration = getLastGeneration(companyId);
if (lastGeneration == IndexAccessor.DEFAULT_LAST_GENERATION) {
return;
}
try {
LuceneClusterUtil.loadIndexesFromCluster(companyId);
} catch (Exception e) {
_log.error("Unable to load indexes for company " + companyId, e);
}
}
}
private class LoadIndexClusterResponseCallback extends BaseClusterResponseCallback {
public LoadIndexClusterResponseCallback(
IndexAccessor indexAccessor, int clusterNodeAddressesCount, long localLastGeneration) {
_indexAccessor = indexAccessor;
_clusterNodeAddressesCount = clusterNodeAddressesCount;
_localLastGeneration = localLastGeneration;
_companyId = _indexAccessor.getCompanyId();
}
@Override
public void callback(BlockingQueue<ClusterNodeResponse> blockingQueue) {
Address bootupAddress = null;
do {
_clusterNodeAddressesCount--;
ClusterNodeResponse clusterNodeResponse = null;
try {
clusterNodeResponse =
blockingQueue.poll(_CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);
} catch (Exception e) {
_log.error("Unable to get cluster node response", e);
}
if (clusterNodeResponse == null) {
if (_log.isDebugEnabled()) {
_log.debug(
"Unable to get cluster node response in "
+ _CLUSTER_LINK_NODE_BOOTUP_RESPONSE_TIMEOUT
+ TimeUnit.MILLISECONDS);
}
continue;
}
ClusterNode clusterNode = clusterNodeResponse.getClusterNode();
if (clusterNode.getPortalInetSocketAddress() != null) {
try {
long remoteLastGeneration = (Long) clusterNodeResponse.getResult();
if (remoteLastGeneration > _localLastGeneration) {
bootupAddress = clusterNodeResponse.getAddress();
break;
}
} catch (Exception e) {
if (_log.isDebugEnabled()) {
_log.debug("Suppress exception caused by remote method " + "invocation", e);
}
continue;
}
} else if (_log.isDebugEnabled()) {
_log.debug("Cluster node " + clusterNode + " has invalid InetSocketAddress");
}
} while ((bootupAddress == null) && (_clusterNodeAddressesCount > 1));
if (bootupAddress == null) {
return;
}
if (_log.isInfoEnabled()) {
_log.info("Start loading lucene index files from cluster node " + bootupAddress);
}
InputStream inputStream = null;
try {
inputStream = getLoadIndexesInputStreamFromCluster(_companyId, bootupAddress);
_indexAccessor.loadIndex(inputStream);
if (_log.isInfoEnabled()) {
_log.info("Lucene index files loaded successfully");
}
} catch (Exception e) {
_log.error("Unable to load index for company " + _companyId, e);
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException ioe) {
_log.error("Unable to close input stream for company " + _companyId, ioe);
}
}
}
}
@Override
public void processTimeoutException(TimeoutException timeoutException) {
_log.error("Unable to load index for company " + _companyId, timeoutException);
}
private int _clusterNodeAddressesCount;
private long _companyId;
private IndexAccessor _indexAccessor;
private long _localLastGeneration;
}
}
/**
* Creates a configuration instance from a resource loader, a configuration name and a stream. If
* the stream is null, the resource loader will open the configuration stream. If the stream is
* not null, no attempt to load the resource will occur (the name is not used).
*
* @param loader the resource loader
* @param name the configuration name
* @param is the configuration stream
*/
public SolrConfig(SolrResourceLoader loader, String name, InputSource is)
throws ParserConfigurationException, IOException, SAXException {
super(loader, name, is, "/config/");
getOverlay(); // just in case it is not initialized
getRequestParams();
initLibs();
luceneMatchVersion = getLuceneVersion("luceneMatchVersion");
String indexConfigPrefix;
// Old indexDefaults and mainIndex sections are deprecated and fails fast for
// luceneMatchVersion=>LUCENE_4_0_0.
// For older solrconfig.xml's we allow the old sections, but never mixed with the new
// <indexConfig>
boolean hasDeprecatedIndexConfig =
(getNode("indexDefaults", false) != null) || (getNode("mainIndex", false) != null);
if (hasDeprecatedIndexConfig) {
throw new SolrException(
ErrorCode.FORBIDDEN,
"<indexDefaults> and <mainIndex> configuration sections are discontinued. Use <indexConfig> instead.");
} else {
defaultIndexConfig = mainIndexConfig = null;
indexConfigPrefix = "indexConfig";
}
assertWarnOrFail(
"The <nrtMode> config has been discontinued and NRT mode is always used by Solr."
+ " This config will be removed in future versions.",
getNode(indexConfigPrefix + "/nrtMode", false) == null,
true);
assertWarnOrFail(
"Solr no longer supports forceful unlocking via the 'unlockOnStartup' option. "
+ "This is no longer neccessary for the default lockType except in situations where "
+ "it would be dangerous and should not be done. For other lockTypes and/or "
+ "directoryFactory options it may also be dangerous and users must resolve "
+ "problematic locks manually.",
null == getNode(indexConfigPrefix + "/unlockOnStartup", false),
true // 'fail' in trunk
);
// Parse indexConfig section, using mainIndex as backup in case old config is used
indexConfig = new SolrIndexConfig(this, "indexConfig", mainIndexConfig);
booleanQueryMaxClauseCount =
getInt("query/maxBooleanClauses", BooleanQuery.getMaxClauseCount());
log.info("Using Lucene MatchVersion: " + luceneMatchVersion);
// Warn about deprecated / discontinued parameters
// boolToFilterOptimizer has had no effect since 3.1
if (get("query/boolTofilterOptimizer", null) != null)
log.warn(
"solrconfig.xml: <boolTofilterOptimizer> is currently not implemented and has no effect.");
if (get("query/HashDocSet", null) != null)
log.warn("solrconfig.xml: <HashDocSet> is deprecated and no longer recommended used.");
// TODO: Old code - in case somebody wants to re-enable. Also see SolrIndexSearcher#search()
// filtOptEnabled = getBool("query/boolTofilterOptimizer/@enabled", false);
// filtOptCacheSize = getInt("query/boolTofilterOptimizer/@cacheSize",32);
// filtOptThreshold = getFloat("query/boolTofilterOptimizer/@threshold",.05f);
useFilterForSortedQuery = getBool("query/useFilterForSortedQuery", false);
queryResultWindowSize = Math.max(1, getInt("query/queryResultWindowSize", 1));
queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE);
enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false);
filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache");
queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache");
documentCacheConfig = CacheConfig.getConfig(this, "query/documentCache");
CacheConfig conf = CacheConfig.getConfig(this, "query/fieldValueCache");
if (conf == null) {
Map<String, String> args = new HashMap<>();
args.put(NAME, "fieldValueCache");
args.put("size", "10000");
args.put("initialSize", "10");
args.put("showItems", "-1");
conf = new CacheConfig(FastLRUCache.class, args, null);
}
fieldValueCacheConfig = conf;
useColdSearcher = getBool("query/useColdSearcher", false);
dataDir = get("dataDir", null);
if (dataDir != null && dataDir.length() == 0) dataDir = null;
userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");
org.apache.solr.search.SolrIndexSearcher.initRegenerators(this);
hashSetInverseLoadFactor = 1.0f / getFloat("//HashDocSet/@loadFactor", 0.75f);
hashDocSetMaxSize = getInt("//HashDocSet/@maxSize", 3000);
httpCachingConfig = new HttpCachingConfig(this);
Node jmx = getNode("jmx", false);
if (jmx != null) {
jmxConfig =
new JmxConfiguration(
true,
get("jmx/@agentId", null),
get("jmx/@serviceUrl", null),
get("jmx/@rootName", null));
} else {
jmxConfig = new JmxConfiguration(false, null, null, null);
}
maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE);
slowQueryThresholdMillis = getInt("query/slowQueryThresholdMillis", -1);
for (SolrPluginInfo plugin : plugins) loadPluginInfo(plugin);
updateHandlerInfo = loadUpdatehandlerInfo();
multipartUploadLimitKB =
getInt("requestDispatcher/requestParsers/@multipartUploadLimitInKB", 2048);
formUploadLimitKB = getInt("requestDispatcher/requestParsers/@formdataUploadLimitInKB", 2048);
enableRemoteStreams = getBool("requestDispatcher/requestParsers/@enableRemoteStreaming", false);
// Let this filter take care of /select?xxx format
handleSelect = getBool("requestDispatcher/@handleSelect", true);
addHttpRequestToContext =
getBool("requestDispatcher/requestParsers/@addHttpRequestToContext", false);
List<PluginInfo> argsInfos = getPluginInfos(InitParams.class.getName());
if (argsInfos != null) {
Map<String, InitParams> argsMap = new HashMap<>();
for (PluginInfo p : argsInfos) {
InitParams args = new InitParams(p);
argsMap.put(args.name == null ? String.valueOf(args.hashCode()) : args.name, args);
}
this.initParams = Collections.unmodifiableMap(argsMap);
}
solrRequestParsers = new SolrRequestParsers(this);
Config.log.info("Loaded SolrConfig: " + name);
}
public static int getMaxClauseCount() {
return BooleanQuery.getMaxClauseCount();
}
/**
* Creates a configuration instance from a resource loader, a configuration name and a stream. If
* the stream is null, the resource loader will open the configuration stream. If the stream is
* not null, no attempt to load the resource will occur (the name is not used).
*
* @param loader the resource loader
* @param name the configuration name
* @param is the configuration stream
*/
public SolrConfig(SolrResourceLoader loader, String name, InputSource is)
throws ParserConfigurationException, IOException, SAXException {
super(loader, name, is, "/config/");
initLibs();
luceneMatchVersion = getLuceneVersion("luceneMatchVersion");
String indexConfigPrefix;
// Old indexDefaults and mainIndex sections are deprecated and fails fast for
// luceneMatchVersion=>LUCENE_40.
// For older solrconfig.xml's we allow the old sections, but never mixed with the new
// <indexConfig>
boolean hasDeprecatedIndexConfig =
get("indexDefaults/text()", null) != null || get("mainIndex/text()", null) != null;
boolean hasNewIndexConfig = get("indexConfig/text()", null) != null;
if (hasDeprecatedIndexConfig) {
if (luceneMatchVersion.onOrAfter(Version.LUCENE_40)) {
throw new SolrException(
ErrorCode.FORBIDDEN,
"<indexDefaults> and <mainIndex> configuration sections are discontinued. Use <indexConfig> instead.");
} else {
// Still allow the old sections for older LuceneMatchVersion's
if (hasNewIndexConfig) {
throw new SolrException(
ErrorCode.FORBIDDEN,
"Cannot specify both <indexDefaults>, <mainIndex> and <indexConfig> at the same time. Please use <indexConfig> only.");
}
log.warn(
"<indexDefaults> and <mainIndex> configuration sections are deprecated and will fail for luceneMatchVersion=LUCENE_40 and later. Please use <indexConfig> instead.");
defaultIndexConfig = new SolrIndexConfig(this, "indexDefaults", null);
mainIndexConfig = new SolrIndexConfig(this, "mainIndex", defaultIndexConfig);
indexConfigPrefix = "mainIndex";
}
} else {
defaultIndexConfig = mainIndexConfig = null;
indexConfigPrefix = "indexConfig";
}
reopenReaders = getBool(indexConfigPrefix + "/reopenReaders", true);
// Parse indexConfig section, using mainIndex as backup in case old config is used
indexConfig = new SolrIndexConfig(this, "indexConfig", mainIndexConfig);
booleanQueryMaxClauseCount =
getInt("query/maxBooleanClauses", BooleanQuery.getMaxClauseCount());
log.info("Using Lucene MatchVersion: " + luceneMatchVersion);
// Warn about deprecated / discontinued parameters
// boolToFilterOptimizer has had no effect since 3.1
if (get("query/boolTofilterOptimizer", null) != null)
log.warn(
"solrconfig.xml: <boolTofilterOptimizer> is currently not implemented and has no effect.");
if (get("query/HashDocSet", null) != null)
log.warn("solrconfig.xml: <HashDocSet> is deprecated and no longer recommended used.");
// TODO: Old code - in case somebody wants to re-enable. Also see SolrIndexSearcher#search()
// filtOptEnabled = getBool("query/boolTofilterOptimizer/@enabled", false);
// filtOptCacheSize = getInt("query/boolTofilterOptimizer/@cacheSize",32);
// filtOptThreshold = getFloat("query/boolTofilterOptimizer/@threshold",.05f);
useFilterForSortedQuery = getBool("query/useFilterForSortedQuery", false);
queryResultWindowSize = Math.max(1, getInt("query/queryResultWindowSize", 1));
queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE);
enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false);
filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache");
queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache");
documentCacheConfig = CacheConfig.getConfig(this, "query/documentCache");
CacheConfig conf = CacheConfig.getConfig(this, "query/fieldValueCache");
if (conf == null) {
Map<String, String> args = new HashMap<String, String>();
args.put("name", "fieldValueCache");
args.put("size", "10000");
args.put("initialSize", "10");
args.put("showItems", "-1");
conf = new CacheConfig(FastLRUCache.class, args, null);
}
fieldValueCacheConfig = conf;
unlockOnStartup = getBool(indexConfigPrefix + "/unlockOnStartup", false);
useColdSearcher = getBool("query/useColdSearcher", false);
dataDir = get("dataDir", null);
if (dataDir != null && dataDir.length() == 0) dataDir = null;
userCacheConfigs = CacheConfig.getMultipleConfigs(this, "query/cache");
org.apache.solr.search.SolrIndexSearcher.initRegenerators(this);
hashSetInverseLoadFactor = 1.0f / getFloat("//HashDocSet/@loadFactor", 0.75f);
hashDocSetMaxSize = getInt("//HashDocSet/@maxSize", 3000);
httpCachingConfig = new HttpCachingConfig(this);
Node jmx = getNode("jmx", false);
if (jmx != null) {
jmxConfig =
new JmxConfiguration(
true,
get("jmx/@agentId", null),
get("jmx/@serviceUrl", null),
get("jmx/@rootName", null));
} else {
jmxConfig = new JmxConfiguration(false, null, null, null);
}
maxWarmingSearchers = getInt("query/maxWarmingSearchers", Integer.MAX_VALUE);
loadPluginInfo(SolrRequestHandler.class, "requestHandler", true, true);
loadPluginInfo(QParserPlugin.class, "queryParser", true, true);
loadPluginInfo(QueryResponseWriter.class, "queryResponseWriter", true, true);
loadPluginInfo(ValueSourceParser.class, "valueSourceParser", true, true);
loadPluginInfo(TransformerFactory.class, "transformer", true, true);
loadPluginInfo(SearchComponent.class, "searchComponent", true, true);
loadPluginInfo(QueryConverter.class, "queryConverter", true, true);
// this is hackish, since it picks up all SolrEventListeners,
// regardless of when/how/why they are used (or even if they are
// declared outside of the appropriate context) but there's no nice
// way around that in the PluginInfo framework
loadPluginInfo(SolrEventListener.class, "//listener", false, true);
loadPluginInfo(DirectoryFactory.class, "directoryFactory", false, true);
loadPluginInfo(IndexDeletionPolicy.class, indexConfigPrefix + "/deletionPolicy", false, true);
loadPluginInfo(CodecFactory.class, "codecFactory", false, false);
loadPluginInfo(IndexReaderFactory.class, "indexReaderFactory", false, true);
loadPluginInfo(UpdateRequestProcessorChain.class, "updateRequestProcessorChain", false, false);
loadPluginInfo(UpdateLog.class, "updateHandler/updateLog", false, false);
loadPluginInfo(IndexSchemaFactory.class, "schemaFactory", false, true);
updateHandlerInfo = loadUpdatehandlerInfo();
Config.log.info("Loaded SolrConfig: " + name);
}
