/** @return number of non-trivial (num sub-ops > 1) Combining operations (joins/unions) */
int countNontrivialCombiningOperations() {
if (operation == null) {
return 0;
}
CountCombiningOperations count = new CountCombiningOperations();
operation.depthFirstRecurse(count);
return count.num;
}
/** Returns number of text parts of this query. */
private int getTextOperationCount(QueryOperation op) {
if (op == null) {
return 0;
}
CountTextOperations count = new CountTextOperations();
op.depthFirstRecurse(count);
return count.num;
}
/**
* Runs the search and gets an open result set.
*
* <p>WARNING: You **MUST** call {@link ZimbraQueryResults#close()} when you are done with them!
*/
public ZimbraQueryResults execute() throws ServiceException {
compile();
Set<QueryTarget> targets = operation.getQueryTargets();
assert (operation instanceof UnionQueryOperation
|| QueryTarget.getExplicitTargetCount(targets) <= 1);
assert (targets.size() > 1
|| !QueryTarget.hasExternalTarget(targets)
|| operation instanceof RemoteQueryOperation);
ZimbraLog.search.debug("OPERATION: %s", operation);
int chunkSize = (int) Math.min((long) params.getOffset() + (long) params.getLimit(), 1000L);
ZimbraQueryResults results = null;
try {
results = operation.run(mailbox, params, chunkSize);
if (((!params.getIncludeTagDeleted() || !params.getIncludeTagMuted())
&& params.getFetchMode() != SearchParams.Fetch.IDS)
|| params.getAllowableTaskStatuses() != null) {
// we have to do some filtering of the result set
FilteredQueryResults filtered = new FilteredQueryResults(results, params);
if (!params.getIncludeTagDeleted()) {
filtered.setFilterTagDeleted(true);
}
if (!params.getIncludeTagMuted()) {
filtered.setFilterTagMuted(true);
}
if (params.getAllowableTaskStatuses() != null) {
filtered.setAllowedTaskStatuses(params.getAllowableTaskStatuses());
}
results = filtered;
}
return results;
} catch (RuntimeException e) {
Closeables.closeQuietly(results);
Closeables.closeQuietly(operation);
throw e;
}
}
public String toQueryString() throws ServiceException {
if (operation == null) {
compile();
}
return operation.toQueryString();
}
/**
* For the local targets: - exclude all the not-visible folders from the query - look at all the
* text-operations and figure out if private appointments need to be excluded
*/
private static UnionQueryOperation handleLocalPermissionChecks(
UnionQueryOperation union, Set<Folder> visibleFolders, boolean allowPrivateAccess) {
// Since optimize() has already been run, we know that each of our ops only has one target (or
// none). Find those
// operations which have an external target and wrap them in RemoteQueryOperations.
// iterate backwards so we can remove/add w/o screwing iteration
for (int i = union.operations.size() - 1; i >= 0; i--) {
QueryOperation op = union.operations.get(i);
Set<QueryTarget> targets = op.getQueryTargets();
// this assertion is OK because we have already distributed multi-target query ops
// during the optimize() step
assert (QueryTarget.getExplicitTargetCount(targets) <= 1);
// the assertion above is critical: the code below all assumes
// that we only have ONE target (ie we've already distributed if necessary)
if (!QueryTarget.hasExternalTarget(targets)) {
// local target
if (!allowPrivateAccess) op.depthFirstRecurse(new excludePrivateCalendarItems());
if (visibleFolders != null) {
if (visibleFolders.isEmpty()) {
union.operations.remove(i);
ZimbraLog.search.debug("Query changed to NULL_QUERY_OPERATION, no visible folders");
union.operations.add(i, new NoResultsQueryOperation());
} else {
union.operations.remove(i);
// build a "and (in:visible1 or in:visible2 or in:visible3...)" query tree here!
IntersectionQueryOperation intersect = new IntersectionQueryOperation();
intersect.addQueryOp(op);
UnionQueryOperation newUnion = new UnionQueryOperation();
intersect.addQueryOp(newUnion);
// if one or more target folders are specified, use those which are visible.
Set<Folder> targetFolders = null;
if (op instanceof DBQueryOperation) {
DBQueryOperation dbOp = (DBQueryOperation) op;
targetFolders = dbOp.getTargetFolders();
}
for (Folder folder : visibleFolders) {
// exclude remote folders
if (!(folder instanceof Mountpoint) || ((Mountpoint) folder).isLocal()) {
if (targetFolders != null
&& targetFolders.size() > 0
&& !targetFolders.contains(folder)) {
continue; // don't bother searching other visible folders if the query asked for
// specific folders...
}
DBQueryOperation newOp = new DBQueryOperation();
newUnion.add(newOp);
newOp.addInFolder(folder, true);
}
}
union.operations.add(i, intersect);
}
}
}
}
return union;
}
private void compile() throws ServiceException {
assert clauses != null;
if (clauses.isEmpty()) {
return;
}
// Convert list of Queries into list of QueryOperations, then optimize them.
// this generates all of the query operations
operation = parseTree.compile(mailbox);
ZimbraLog.search.debug("OP=%s", operation);
// expand the is:local and is:remote parts into in:(LIST)'s
operation = operation.expandLocalRemotePart(mailbox);
ZimbraLog.search.debug("AFTEREXP=%s", operation);
// optimize the query down
operation = operation.optimize(mailbox);
ZimbraLog.search.debug("OPTIMIZED=%s", operation);
if (operation == null || operation instanceof NoTermQueryOperation) {
operation = new NoResultsQueryOperation();
return;
}
// Use the OperationContext to update the set of visible referenced folders, local AND remote.
Set<QueryTarget> targets = operation.getQueryTargets();
assert (operation instanceof UnionQueryOperation
|| QueryTarget.getExplicitTargetCount(targets) <= 1);
// easiest to treat the query two unions: one the LOCAL and one REMOTE parts
UnionQueryOperation remoteOps = new UnionQueryOperation();
UnionQueryOperation localOps = new UnionQueryOperation();
if (operation instanceof UnionQueryOperation) {
UnionQueryOperation union = (UnionQueryOperation) operation;
// separate out the LOCAL vs REMOTE parts...
for (QueryOperation op : union.operations) {
Set<QueryTarget> set = op.getQueryTargets();
// this assertion OK because we have already distributed multi-target query ops
// during the optimize() step
assert (QueryTarget.getExplicitTargetCount(set) <= 1);
// the assertion above is critical: the code below all assumes
// that we only have ONE target (ie we've already distributed if necessary)
if (QueryTarget.hasExternalTarget(set)) {
remoteOps.add(op);
} else {
localOps.add(op);
}
}
} else {
// single target: might be local, might be remote
Set<QueryTarget> set = operation.getQueryTargets();
// this assertion OK because we have already distributed multi-target query ops
// during the optimize() step
assert (QueryTarget.getExplicitTargetCount(set) <= 1);
if (QueryTarget.hasExternalTarget(set)) {
remoteOps.add(operation);
} else {
localOps.add(operation);
}
}
// Handle the REMOTE side:
if (!remoteOps.operations.isEmpty()) {
// Since optimize() has already been run, we know that each of our ops
// only has one target (or none). Find those operations which have
// an external target and wrap them in RemoteQueryOperations
// iterate backwards so we can remove/add w/o screwing iteration
for (int i = remoteOps.operations.size() - 1; i >= 0; i--) {
QueryOperation op = remoteOps.operations.get(i);
Set<QueryTarget> set = op.getQueryTargets();
// this assertion OK because we have already distributed multi-target query ops
// during the optimize() step
assert (QueryTarget.getExplicitTargetCount(set) <= 1);
// the assertion above is critical: the code below all assumes
// that we only have ONE target (ie we've already distributed if necessary)
if (QueryTarget.hasExternalTarget(set)) {
remoteOps.operations.remove(i);
boolean foundOne = false;
// find a remoteOp to add this one to
for (QueryOperation remoteOp : remoteOps.operations) {
if (remoteOp instanceof RemoteQueryOperation) {
if (((RemoteQueryOperation) remoteOp).tryAddOredOperation(op)) {
foundOne = true;
break;
}
}
}
if (!foundOne) {
RemoteQueryOperation remoteOp = new RemoteQueryOperation();
remoteOp.tryAddOredOperation(op);
remoteOps.operations.add(i, remoteOp);
}
}
}
// ...we need to call setup on every RemoteQueryOperation we end up with...
for (QueryOperation remoteOp : remoteOps.operations) {
assert (remoteOp instanceof RemoteQueryOperation);
try {
((RemoteQueryOperation) remoteOp).setup(protocol, octxt.getAuthToken(), params);
} catch (Exception e) {
ZimbraLog.search.info("Ignoring %s during RemoteQuery generation for %s", e, remoteOps);
}
}
}
// For the LOCAL parts of the query, do permission checks, do trash/spam exclusion
if (!localOps.operations.isEmpty()) {
ZimbraLog.search.debug("LOCAL_IN=%s", localOps);
Account authAcct = octxt != null ? octxt.getAuthenticatedUser() : mailbox.getAccount();
// Now, for all the LOCAL PARTS of the query, add the trash/spam exclusion part
boolean includeTrash = false;
boolean includeSpam = false;
if (params.inDumpster()) {
// Always include trash and spam for dumpster searches. Excluding spam is a client side
// choice.
includeTrash = true;
includeSpam = true;
if (!mailbox.hasFullAdminAccess(
octxt)) { // If the requester is not an admin, limit to recent date range.
long now = octxt != null ? octxt.getTimestamp() : System.currentTimeMillis();
long mdate = now - authAcct.getDumpsterUserVisibleAge();
IntersectionQueryOperation and = new IntersectionQueryOperation();
DBQueryOperation db = new DBQueryOperation();
db.addMDateRange(mdate, false, -1L, false, true);
and.addQueryOp((QueryOperation) localOps.clone());
and.addQueryOp(db);
localOps.operations.clear();
localOps.operations.add(and);
}
} else {
includeTrash = authAcct.isPrefIncludeTrashInSearch();
;
includeSpam = authAcct.isPrefIncludeSpamInSearch();
}
if (!includeTrash || !includeSpam) {
// First check that we aren't specifically looking for items in one of these.
// For instance, in ZWC, if "Include Shared Items" is selected, the Trash list may currently
// use a
// search similar to : 'in:"trash" (inid:565 OR is:local)'
// where 565 is the folder ID for a shared folder. We don't want to end up doing a search
// for items
// that are both in "trash" and NOT in "trash"...
for (Query q : clauses) {
if (q.toString().equalsIgnoreCase("Q(IN:Trash)")) {
includeTrash = true;
}
if (q.toString().equalsIgnoreCase("Q(IN:Junk)")) {
includeSpam = true;
}
}
}
if (!includeTrash || !includeSpam) {
List<QueryOperation> toAdd = new ArrayList<QueryOperation>();
for (Iterator<QueryOperation> iter = localOps.operations.iterator(); iter.hasNext(); ) {
QueryOperation cur = iter.next();
if (!cur.hasSpamTrashSetting()) {
QueryOperation newOp = cur.ensureSpamTrashSetting(mailbox, includeTrash, includeSpam);
if (newOp != cur) {
iter.remove();
toAdd.add(newOp);
}
}
}
localOps.operations.addAll(toAdd);
ZimbraLog.search.debug("LOCAL_AFTER_TRASH/SPAM/DUMPSTER=%s", localOps);
}
// Check to see if we need to filter out private appointment data
boolean allowPrivateAccess = true;
if (octxt != null) {
allowPrivateAccess =
AccessManager.getInstance()
.allowPrivateAccess(
octxt.getAuthenticatedUser(),
mailbox.getAccount(),
octxt.isUsingAdminPrivileges());
}
//
// bug 28892 - ACL.RIGHT_PRIVATE support:
//
// Basically, if ACL.RIGHT_PRIVATE is set somewhere, and if we're excluding private items from
// search, then we need to run the query twice -- once over the whole mailbox with
// private items excluded and then UNION it with a second run, this time only in the
// RIGHT_PRIVATE enabled folders, with private items enabled.
//
UnionQueryOperation clonedLocal = null;
Set<Folder> hasFolderRightPrivateSet = new HashSet<Folder>();
// ...don't do any of this if they aren't asking for a calendar type...
Set<MailItem.Type> types = params.getTypes();
boolean hasCalendarType =
types.contains(MailItem.Type.APPOINTMENT) || types.contains(MailItem.Type.TASK);
if (hasCalendarType && !allowPrivateAccess && getTextOperationCount(localOps) > 0) {
// the searcher is NOT allowed to see private items globally....lets check
// to see if there are any individual folders that they DO have rights to...
// if there are any, then we'll need to run special searches in those
// folders
Set<Folder> allVisibleFolders = mailbox.getVisibleFolders(octxt);
if (allVisibleFolders == null) {
allVisibleFolders = new HashSet<Folder>();
allVisibleFolders.addAll(mailbox.getFolderList(octxt, SortBy.NONE));
}
for (Folder f : allVisibleFolders) {
if (f.getType() == MailItem.Type.FOLDER
&& CalendarItem.allowPrivateAccess(f, authAcct, false)) {
hasFolderRightPrivateSet.add(f);
}
}
if (!hasFolderRightPrivateSet.isEmpty()) {
clonedLocal = (UnionQueryOperation) localOps.clone();
}
}
Set<Folder> visibleFolders = mailbox.getVisibleFolders(octxt);
localOps = handleLocalPermissionChecks(localOps, visibleFolders, allowPrivateAccess);
ZimbraLog.search.debug("LOCAL_AFTER_PERM_CHECKS=%s", localOps);
if (!hasFolderRightPrivateSet.isEmpty()) {
ZimbraLog.search.debug("CLONED_LOCAL_BEFORE_PERM=%s", clonedLocal);
// now we're going to setup the clonedLocal tree
// to run with private access ALLOWED, over the set of folders
// that have RIGHT_PRIVATE (note that we build this list from the visible
// folder list, so we are
clonedLocal = handleLocalPermissionChecks(clonedLocal, hasFolderRightPrivateSet, true);
ZimbraLog.search.debug("CLONED_LOCAL_AFTER_PERM=%s", clonedLocal);
// clonedLocal should only have the single INTERSECT in it
assert (clonedLocal.operations.size() == 1);
QueryOperation optimizedClonedLocal = clonedLocal.optimize(mailbox);
ZimbraLog.search.debug("CLONED_LOCAL_AFTER_OPTIMIZE=%s", optimizedClonedLocal);
UnionQueryOperation withPrivateExcluded = localOps;
localOps = new UnionQueryOperation();
localOps.add(withPrivateExcluded);
localOps.add(optimizedClonedLocal);
ZimbraLog.search.debug("LOCAL_WITH_CLONED=%s", localOps);
//
// we should end up with:
//
// localOps =
// UNION(withPrivateExcluded,
// UNION(INTERSECT(clonedLocal,
// UNION(hasFolderRightPrivateList)
// )
// )
// )
//
}
}
UnionQueryOperation union = new UnionQueryOperation();
union.add(localOps);
union.add(remoteOps);
ZimbraLog.search.debug("BEFORE_FINAL_OPT=%s", union);
operation = union.optimize(mailbox);
ZimbraLog.search.debug("COMPILED=%s", operation);
}