@NotNull
@ThreadSafe
private static QueryWrapper createQuery(@NotNull String queryString) throws SearchException {
PhraseDetectingQueryParser queryParser =
new PhraseDetectingQueryParser(
IndexRegistry.LUCENE_VERSION, Fields.CONTENT.key(), IndexRegistry.getAnalyzer());
queryParser.setAllowLeadingWildcard(true);
RewriteMethod rewriteMethod = MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE;
queryParser.setMultiTermRewriteMethod(rewriteMethod);
if (!SettingsConf.Bool.UseOrOperator.get())
queryParser.setDefaultOperator(QueryParser.AND_OPERATOR);
try {
Query query = queryParser.parse(queryString);
boolean isPhraseQuery = queryParser.isPhraseQuery();
return new QueryWrapper(query, isPhraseQuery);
} catch (IllegalArgumentException e) {
/*
* This happens for example when you enter a fuzzy search with
* similarity >= 1, e.g. "fuzzy~1".
*/
String msg = Msg.invalid_query.get() + "\n\n" + e.getMessage();
throw new SearchException(msg);
} catch (ParseException e) {
String msg = Msg.invalid_query.get() + "\n\n" + e.getMessage();
throw new SearchException(msg);
}
}
public void addTerm(String field, String value) throws ParseException {
try {
LuceneHelperUtil.addTerm(_booleanQuery, field, value);
} catch (org.apache.lucene.queryParser.ParseException pe) {
throw new ParseException(pe.getMessage());
}
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override
public List<AbstractPermissionsOwner> search(
String queryString, boolean withUsers, boolean withGroups) {
List<AbstractPermissionsOwner> results = new ArrayList<AbstractPermissionsOwner>();
// No query should be realized while re-indexing resources.
if (!inhibitSearch) {
// Gets the Hibernate search object to performs queries.
FullTextEntityManager fullTextEntityManager = Search.getFullTextEntityManager(entityManager);
// Parse the the queryString.
MultiFieldQueryParser parser =
new MultiFieldQueryParser(
Version.LUCENE_30,
new String[] {"name", "firstName", "lastName", "email", "login"},
new StandardAnalyzer(Version.LUCENE_31));
parser.setDefaultOperator(Operator.OR);
try {
Query luceneQuery = parser.parse(queryString);
FullTextQuery query = null;
// Because of the poor design of the Hibernate Search API and the usage of varagrs, we must
// have this
// if-else algorihm. TODO refactor with reflection.
if (withUsers && withGroups) {
query = fullTextEntityManager.createFullTextQuery(luceneQuery, User.class, Group.class);
} else if (withUsers) {
query = fullTextEntityManager.createFullTextQuery(luceneQuery, User.class);
} else if (withGroups) {
query = fullTextEntityManager.createFullTextQuery(luceneQuery, Group.class);
}
// Finally execute the query.
if (query != null) {
List<AbstractPermissionsOwner> found = query.getResultList();
// Keeps only distinct results.
for (AbstractPermissionsOwner foundObject : found) {
if (!results.contains(foundObject)) {
// TODO Remove this Hibernate specific block.
// Sometimes hibernate Search returns Javassist proxies, which can't be properly
// deserialize by Jackson.
if (foundObject instanceof HibernateProxy) {
HibernateProxy h = (HibernateProxy) foundObject;
foundObject =
(AbstractPermissionsOwner) h.getHibernateLazyInitializer().getImplementation();
}
results.add(foundObject);
}
}
}
} catch (ParseException exc) {
// Handle parsing failure
String error = "Misformatted queryString '" + queryString + "': " + exc.getMessage();
logger.debug("[search] " + error);
throw new IllegalArgumentException(error, exc);
}
}
return results;
} // search().
public void add(Query query, BooleanClauseOccur booleanClauseOccur) throws ParseException {
try {
_booleanQuery.add(
QueryTranslator.translate(query),
BooleanClauseOccurTranslator.translate(booleanClauseOccur));
} catch (org.apache.lucene.queryParser.ParseException pe) {
throw new ParseException(pe.getMessage());
}
}
public SolrSearchResultSet getSearchResultSet(
SlingHttpServletRequest request, Query query, boolean asAnon) throws SolrSearchException {
try {
SolrSearchResultSet rs = null;
if (query.getType() == Type.SOLR) {
rs = processSolrQuery(request, query, asAnon);
} else if (query.getType() == Type.SPARSE) {
rs = processSparseQuery(request, query, asAnon);
}
return rs;
} catch (SolrServerException e) {
LOGGER.warn(e.getMessage(), e);
throw new SolrSearchException(500, e.getMessage());
} catch (ParseException e) {
LOGGER.warn(e.getMessage(), e);
throw new SolrSearchException(500, e.getMessage());
} catch (StorageClientException e) {
LOGGER.warn(e.getMessage(), e);
throw new SolrSearchException(500, e.getMessage());
} catch (AccessDeniedException e) {
LOGGER.warn(e.getMessage(), e);
throw new SolrSearchException(403, e.getMessage());
}
}
/**
* Creates a new instance; Passes the query directly on to the Lucene parser.
*
* @param values
* @param query
* @param analyzer
*/
public FullText(SearchValues values, String query, Class<? extends Analyzer> analyzer) {
super(values);
Assert.notNull(analyzer, "Analyzer required");
this.analyzer = analyzer;
if (values.onlyTypes == null || values.onlyTypes.size() != 1) {
throw new ApiUsageException(
"Searches by full text are currently limited to a single type.\n"
+ "Plese use Search.onlyType()");
}
if (query == null || query.length() < 1) {
throw new IllegalArgumentException("Query string must be non-empty");
}
if ((query.startsWith("*") || query.startsWith("?")) && !values.leadingWildcard) {
throw new ApiUsageException(
"Searches starting with a leading "
+ "wildcard (*,?) can be slow.\nPlease use "
+ "setAllowLeadingWildcard() to permit this usage.");
}
if (query.equals("*")) {
throw new ApiUsageException(
"Wildcard searches (*) must contain more than a single wildcard. ");
}
this.queryStr = query;
try {
final Analyzer a = analyzer.newInstance();
final QueryParser parser = new /*Analyzing*/ QueryParser("combined_fields", a);
parser.setAllowLeadingWildcard(values.leadingWildcard);
q = parser.parse(queryStr);
} catch (ParseException pe) {
final String msg = queryStr + " caused a parse exception: " + pe.getMessage();
// No longer logging these, since it's a simple user error
ApiUsageException aue = new ApiUsageException(msg);
throw aue;
} catch (InstantiationException e) {
ApiUsageException aue =
new ApiUsageException(analyzer.getName() + " cannot be instantiated.");
throw aue;
} catch (IllegalAccessException e) {
ApiUsageException aue =
new ApiUsageException(analyzer.getName() + " cannot be instantiated.");
throw aue;
}
}
private void doQuery(Scanner scanner) {
scanner.nextLine();
Query query = null;
while (query == null) {
System.out.println("Enter a query:");
String queryLine = scanner.nextLine();
try {
query = actions.parseQuery(queryLine);
} catch (ParseException e) {
System.out.println("Wrong syntax in query: " + e.getMessage());
System.out.println("type it again: ");
}
}
List<String> listMatches = actions.listStoredValuesMatchingQuery(query);
printResult(listMatches);
}
@RequestMapping(value = "/lucene/o_create.do")
public void create(
Integer siteId,
Integer channelId,
Date startDate,
Date endDate,
Integer startId,
Integer max,
HttpServletRequest request,
HttpServletResponse response,
ModelMap model)
throws JSONException {
try {
Integer lastId =
luceneContentSvc.createIndex(siteId, channelId, startDate, endDate, startId, max);
JSONObject json = new JSONObject();
json.put("success", true);
if (lastId != null) {
json.put("lastId", lastId);
}
ResponseUtils.renderJson(response, json.toString());
} catch (CorruptIndexException e) {
JSONObject json = new JSONObject();
json.put("success", false).put("msg", e.getMessage());
ResponseUtils.renderJson(response, json.toString());
log.error("", e);
} catch (LockObtainFailedException e) {
JSONObject json = new JSONObject();
json.put("success", false).put("msg", e.getMessage());
ResponseUtils.renderJson(response, json.toString());
log.error("", e);
} catch (IOException e) {
JSONObject json = new JSONObject();
json.put("success", false).put("msg", e.getMessage());
ResponseUtils.renderJson(response, json.toString());
log.error("", e);
} catch (ParseException e) {
JSONObject json = new JSONObject();
json.put("success", false).put("msg", e.getMessage());
ResponseUtils.renderJson(response, json.toString());
log.error("", e);
}
}
@SuppressWarnings("unchecked")
public List<Book> freeTextSeachEntities(
String searchWord,
String[] targetFields,
String orderBy,
boolean reverseOrder,
int startIndex,
int maxResult) {
FullTextEntityManager fullTextEntityManager = Search.getFullTextEntityManager(entityManager);
QueryParser parser =
new MultiFieldQueryParser(
org.apache.lucene.util.Version.LUCENE_29,
targetFields,
new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_29));
parser.setAllowLeadingWildcard(true);
org.apache.lucene.search.Query luceneQuery = null;
try {
luceneQuery = parser.parse(searchWord);
System.out.println("@@@luceneQuery : " + luceneQuery.toString());
} catch (ParseException e) {
System.out.println("@@@ParseEcxetpion : " + e.getMessage());
}
FullTextQuery fullTextQuery =
fullTextEntityManager.createFullTextQuery(luceneQuery, Book.class);
if (orderBy != null) {
Sort sort = new Sort(new SortField(orderBy, SortField.STRING_VAL, reverseOrder));
fullTextQuery.setSort(sort);
}
if (startIndex > 0 && maxResult > 0) {
fullTextQuery.setFirstResult(startIndex);
fullTextQuery.setMaxResults(maxResult);
}
Query jpaQuery = fullTextQuery;
List<Book> resultEntities = jpaQuery.getResultList();
return resultEntities;
}
/**
* Constructs a new instance; Builds a Lucence query with the provided arguments and passes it on
* the Lucene parser
*
* @param values
* @param fields Comma separated field names (name, description, etc.)
* @param from Date range from in form YYYYMMDD
* @param to Date range to in form YYYYMMDD
* @param dateType Type of date {@link ome.api.Search#DATE_TYPE_ACQUISITION} or {@link
* ome.api.Search#DATE_TYPE_IMPORT}
* @param query The terms to search for
* @param analyzer
*/
public FullText(
SearchValues values,
String fields,
String from,
String to,
String dateType,
String query,
Class<? extends Analyzer> analyzer) {
super(values);
Assert.notNull(analyzer, "Analyzer required");
this.analyzer = analyzer;
if (values.onlyTypes == null || values.onlyTypes.size() != 1) {
throw new ApiUsageException(
"Searches by full text are currently limited to a single type.\n"
+ "Plese use Search.onlyType()");
}
if ((query == null || query.length() < 1)
&& (from == null || from.length() < 1)
&& (to == null || to.length() < 1)) {
throw new IllegalArgumentException(
"Query string must be non-empty if no date range is provided");
}
if ((query.startsWith("*") || query.startsWith("?")) && !values.leadingWildcard) {
throw new ApiUsageException(
"Searches starting with a leading "
+ "wildcard (*,?) can be slow.\nPlease use "
+ "setAllowLeadingWildcard() to permit this usage.");
}
if (query.equals("*")) {
throw new ApiUsageException(
"Wildcard searches (*) must contain more than a single wildcard. ");
}
List<String> fieldsArray = new ArrayList<String>();
String[] tmp = fields.split("\\,");
for (String t : tmp) {
t = t.trim();
if (t.length() > 0) fieldsArray.add(t);
}
Date dFrom;
Date dTo;
try {
dFrom = (from != null && from.trim().length() > 0) ? DATEFORMAT.parse(from) : null;
dTo = (to != null && to.trim().length() > 0) ? DATEFORMAT.parse(to) : null;
} catch (java.text.ParseException e1) {
throw new ApiUsageException("Invalid date format, dates must be in format YYYYMMDD.");
}
if (LuceneQueryBuilder.DATE_ACQUISITION.equals(dateType)
&& !values.onlyTypes.contains(Image.class)) {
// Use import for non-images
dateType = LuceneQueryBuilder.DATE_IMPORT;
}
try {
this.queryStr = LuceneQueryBuilder.buildLuceneQuery(fieldsArray, dFrom, dTo, dateType, query);
if (this.queryStr.isEmpty()) {
q = null;
log.info("Generated empty Lucene query");
return; // EARLY EXIT!
} else {
log.info("Generated Lucene query: " + this.queryStr);
}
} catch (InvalidQueryException e1) {
throw new ApiUsageException("Invalid query: " + e1.getMessage());
}
try {
final Analyzer a = analyzer.newInstance();
final QueryParser parser = new /*Analyzing*/ QueryParser("combined_fields", a);
parser.setAllowLeadingWildcard(values.leadingWildcard);
q = parser.parse(queryStr);
} catch (ParseException pe) {
final String msg = queryStr + " caused a parse exception: " + pe.getMessage();
// No longer logging these, since it's a simple user error
ApiUsageException aue = new ApiUsageException(msg);
throw aue;
} catch (InstantiationException e) {
ApiUsageException aue =
new ApiUsageException(analyzer.getName() + " cannot be instantiated.");
throw aue;
} catch (IllegalAccessException e) {
ApiUsageException aue =
new ApiUsageException(analyzer.getName() + " cannot be instantiated.");
throw aue;
}
}
/**
* Process properties to query sparse content directly.
*
* @param request
* @param query
* @param asAnon
* @return
* @throws StorageClientException
* @throws AccessDeniedException
*/
public SolrSearchResultSet processQuery(
SlingHttpServletRequest request, Query query, boolean asAnon) throws SolrSearchException {
try {
// use solr parsing to get the terms from the query string
QueryParser parser =
new QueryParser(Version.LUCENE_40, "id", new TextField().getQueryAnalyzer());
org.apache.lucene.search.Query luceneQuery = parser.parse(query.getQueryString());
Map<String, Object> props = Maps.newHashMap();
if (luceneQuery instanceof BooleanQuery) {
BooleanQuery boolLucQuery = (BooleanQuery) luceneQuery;
int orCount = 0;
List<BooleanClause> clauses = boolLucQuery.clauses();
for (BooleanClause clause : clauses) {
org.apache.lucene.search.Query clauseQuery = clause.getQuery();
Map<String, Object> subOrs = Maps.newHashMap();
// we support 1 level of nesting for OR clauses
if (clauseQuery instanceof BooleanQuery) {
for (BooleanClause subclause : ((BooleanQuery) clauseQuery).clauses()) {
org.apache.lucene.search.Query subclauseQuery = subclause.getQuery();
extractTerms(subclause, subclauseQuery, props, subOrs);
}
props.put("orset" + orCount, subOrs);
orCount++;
} else {
extractTerms(clause, clauseQuery, props, subOrs);
if (!subOrs.isEmpty()) {
props.put("orset" + orCount, subOrs);
orCount++;
}
}
}
} else {
extractTerms(null, luceneQuery, props, null);
}
// add the options to the parameters but prepend _ to avoid collision
for (Entry<String, String> option : query.getOptions().entrySet()) {
props.put("_" + option.getKey(), option.getValue());
}
Session session =
StorageClientUtils.adaptToSession(
request.getResourceResolver().adaptTo(javax.jcr.Session.class));
ContentManager cm = session.getContentManager();
long tquery = System.currentTimeMillis();
Iterable<Content> items = cm.find(props);
tquery = System.currentTimeMillis() - tquery;
try {
if (tquery > verySlowQueryThreshold) {
SLOW_QUERY_LOGGER.error(
"Very slow sparse query {} ms {} ",
tquery,
URLDecoder.decode(query.toString(), "UTF-8"));
} else if (tquery > slowQueryThreshold) {
SLOW_QUERY_LOGGER.warn(
"Slow sparse query {} ms {} ", tquery, URLDecoder.decode(query.toString(), "UTF-8"));
}
} catch (UnsupportedEncodingException e) {
// quietly swallow this exception
LOGGER.debug(e.getLocalizedMessage(), e);
}
SolrSearchResultSet rs = new SparseSearchResultSet(items, defaultMaxResults);
return rs;
} catch (AccessDeniedException e) {
throw new SolrSearchException(500, e.getMessage());
} catch (StorageClientException e) {
throw new SolrSearchException(500, e.getMessage());
} catch (ParseException e) {
throw new SolrSearchException(500, e.getMessage());
}
}
private Query getQuery(
VitroRequest request, PortalFlag portalState, Analyzer analyzer, String querystr)
throws SearchException, ParseException {
Query query = null;
try {
// String querystr = request.getParameter(VitroQuery.QUERY_PARAMETER_NAME);
if (querystr == null) {
log.error(
"There was no Parameter '" + VitroQuery.QUERY_PARAMETER_NAME + "' in the request.");
return null;
} else if (querystr.length() > MAX_QUERY_LENGTH) {
log.debug("The search was too long. The maximum " + "query length is " + MAX_QUERY_LENGTH);
return null;
}
QueryParser parser = getQueryParser(analyzer);
query = parser.parse(querystr);
String alpha = request.getParameter("alpha");
if (alpha != null && !"".equals(alpha) && alpha.length() == 1) {
BooleanQuery boolQuery = new BooleanQuery();
boolQuery.add(query, BooleanClause.Occur.MUST);
boolQuery.add(
new WildcardQuery(new Term(Entity2LuceneDoc.term.NAME, alpha + '*')),
BooleanClause.Occur.MUST);
query = boolQuery;
}
// check if this is classgroup filtered
Object param = request.getParameter("classgroup");
if (param != null && !"".equals(param)) {
BooleanQuery boolQuery = new BooleanQuery();
boolQuery.add(query, BooleanClause.Occur.MUST);
boolQuery.add(
new TermQuery(new Term(Entity2LuceneDoc.term.CLASSGROUP_URI, (String) param)),
BooleanClause.Occur.MUST);
query = boolQuery;
}
// check if this is rdf:type filtered
param = request.getParameter("type");
if (param != null && !"".equals(param)) {
BooleanQuery boolQuery = new BooleanQuery();
boolQuery.add(query, BooleanClause.Occur.MUST);
boolQuery.add(
new TermQuery(new Term(Entity2LuceneDoc.term.RDFTYPE, (String) param)),
BooleanClause.Occur.MUST);
query = boolQuery;
}
// if we have a flag/portal query then we add
// it by making a BooelanQuery.
Query flagQuery = makeFlagQuery(portalState);
if (flagQuery != null) {
BooleanQuery boolQuery = new BooleanQuery();
boolQuery.add(query, BooleanClause.Occur.MUST);
boolQuery.add(flagQuery, BooleanClause.Occur.MUST);
query = boolQuery;
}
log.debug("Query: " + query);
} catch (ParseException e) {
throw new ParseException(e.getMessage());
} catch (Exception ex) {
throw new SearchException(ex.getMessage());
}
return query;
}
/**
* Searches pages using a particular combination of flags.
*
* @param query The query to perform in Lucene query language
* @param flags A set of flags
* @return A Collection of SearchResult instances
* @throws ProviderException if there is a problem with the backend
*/
public Collection findPages(String query, int flags) throws ProviderException {
IndexSearcher searcher = null;
ArrayList<SearchResult> list = null;
Highlighter highlighter = null;
try {
String[] queryfields = {
LUCENE_PAGE_CONTENTS, LUCENE_PAGE_NAME, LUCENE_AUTHOR, LUCENE_ATTACHMENTS
};
QueryParser qp =
new MultiFieldQueryParser(Version.LUCENE_36, queryfields, getLuceneAnalyzer());
// QueryParser qp = new QueryParser( LUCENE_PAGE_CONTENTS, getLuceneAnalyzer() );
Query luceneQuery = qp.parse(query);
if ((flags & FLAG_CONTEXTS) != 0) {
highlighter =
new Highlighter(
new SimpleHTMLFormatter("<span class=\"searchmatch\">", "</span>"),
new SimpleHTMLEncoder(),
new QueryScorer(luceneQuery));
}
try {
File dir = new File(m_luceneDirectory);
Directory luceneDir = new SimpleFSDirectory(dir, null);
IndexReader reader = IndexReader.open(luceneDir);
searcher = new IndexSearcher(reader);
} catch (Exception ex) {
log.info("Lucene not yet ready; indexing not started", ex);
return null;
}
ScoreDoc[] hits = searcher.search(luceneQuery, MAX_SEARCH_HITS).scoreDocs;
list = new ArrayList<SearchResult>(hits.length);
for (int curr = 0; curr < hits.length; curr++) {
int docID = hits[curr].doc;
Document doc = searcher.doc(docID);
String pageName = doc.get(LUCENE_ID);
WikiPage page = m_engine.getPage(pageName, WikiPageProvider.LATEST_VERSION);
if (page != null) {
if (page instanceof Attachment) {
// Currently attachments don't look nice on the search-results page
// When the search-results are cleaned up this can be enabled again.
}
int score = (int) (hits[curr].score * 100);
// Get highlighted search contexts
String text = doc.get(LUCENE_PAGE_CONTENTS);
String[] fragments = new String[0];
if (text != null && highlighter != null) {
TokenStream tokenStream =
getLuceneAnalyzer().tokenStream(LUCENE_PAGE_CONTENTS, new StringReader(text));
fragments = highlighter.getBestFragments(tokenStream, text, MAX_FRAGMENTS);
}
SearchResult result = new SearchResultImpl(page, score, fragments);
list.add(result);
} else {
log.error(
"Lucene found a result page '"
+ pageName
+ "' that could not be loaded, removing from Lucene cache");
pageRemoved(new WikiPage(m_engine, pageName));
}
}
} catch (IOException e) {
log.error("Failed during lucene search", e);
} catch (ParseException e) {
log.info("Broken query; cannot parse query ", e);
throw new ProviderException(
"You have entered a query Lucene cannot process: " + e.getMessage());
} catch (InvalidTokenOffsetsException e) {
log.error("Tokens are incompatible with provided text ", e);
} finally {
if (searcher != null) {
try {
searcher.close();
} catch (IOException e) {
log.error(e);
}
}
}
return list;
}
