@Override
public ItemId fileInto(String folderPath, Collection<ActionFlag> flagActions, String[] tags)
throws ServiceException {
Message source = getMessage();
// See if the message is already in the target folder.
Folder targetFolder = null;
try {
targetFolder = mailbox.getFolderByPath(octxt, folderPath);
} catch (NoSuchItemException ignored) {
}
if (targetFolder != null && source.getFolderId() == targetFolder.getId()) {
ZimbraLog.filter.debug(
"Ignoring fileinto action for message %d. It is already in %s.", messageId, folderPath);
updateTagsAndFlagsIfNecessary(source, flagActions, tags);
return null;
}
ZimbraLog.filter.info("Copying existing message %d to folder %s.", messageId, folderPath);
if (isLocalExistingFolder(folderPath)) {
// Copy item into to a local folder.
Folder target = mailbox.getFolderByPath(octxt, folderPath);
Message newMsg =
(Message) mailbox.copy(octxt, messageId, MailItem.Type.MESSAGE, target.getId());
filtered = true;
filed = true;
// Apply flags and tags
mailbox.setTags(
octxt,
newMsg.getId(),
MailItem.Type.MESSAGE,
FilterUtil.getFlagBitmask(flagActions, source.getFlagBitmask()),
FilterUtil.getTagsUnion(source.getTags(), tags));
return new ItemId(mailbox, messageId);
}
ItemId id =
FilterUtil.addMessage(
new DeliveryContext(),
mailbox,
getParsedMessage(),
mailbox.getAccount().getName(),
folderPath,
false,
FilterUtil.getFlagBitmask(flagActions, source.getFlagBitmask()),
tags,
Mailbox.ID_AUTO_INCREMENT,
octxt);
if (id != null) {
filtered = true;
filed = true;
}
return id;
}
private boolean dedupe(ParsedMessage pm, Mailbox mbox) throws ServiceException {
if (pm == null || mbox == null || !mbox.getAccount().isPrefMessageIdDedupingEnabled())
return false;
checkDedupeCacheSize();
String msgid = getMessageID(pm);
if (msgid == null || msgid.equals("")) return false;
synchronized (ZimbraLmtpBackend.class) {
Set<Integer> mboxIds = receivedMessageIDs.get(msgid);
if (mboxIds != null && mboxIds.contains(mbox.getId())) {
return true;
}
}
return false;
}
@Override
public Element handle(Element request, Map<String, Object> context) throws ServiceException {
ZimbraSoapContext zsc = getZimbraSoapContext(context);
Mailbox mbox = getRequestedMailbox(zsc);
Account account = getRequestedAccount(zsc);
OperationContext octxt = getOperationContext(zsc, context);
SearchRequest req = JaxbUtil.elementToJaxb(request);
if (Objects.firstNonNull(req.getWarmup(), false)) {
mbox.index.getIndexStore().warmup();
return zsc.createElement(MailConstants.SEARCH_RESPONSE);
}
SearchParams params = SearchParams.parse(req, zsc, account.getPrefMailInitialSearch());
if (params.getLocale() == null) {
params.setLocale(mbox.getAccount().getLocale());
}
if (params.inDumpster() && params.getTypes().contains(MailItem.Type.CONVERSATION)) {
throw ServiceException.INVALID_REQUEST("cannot search for conversations in dumpster", null);
}
if (LC.calendar_cache_enabled.booleanValue()) {
List<String> apptFolderIds = getFolderIdListIfSimpleAppointmentsQuery(params, zsc);
if (apptFolderIds != null) {
Account authAcct = getAuthenticatedAccount(zsc);
Element response = zsc.createElement(MailConstants.SEARCH_RESPONSE);
runSimpleAppointmentQuery(response, params, octxt, zsc, authAcct, mbox, apptFolderIds);
return response;
}
}
ZimbraQueryResults results = mbox.index.search(zsc.getResponseProtocol(), octxt, params);
try {
// create the XML response Element
Element response = zsc.createElement(MailConstants.SEARCH_RESPONSE);
// must use results.getSortBy() because the results might have ignored our sortBy
// request and used something else...
response.addAttribute(MailConstants.A_SORTBY, results.getSortBy().toString());
putHits(zsc, octxt, response, results, params);
return response;
} finally {
Closeables.closeQuietly(results);
}
}
@Override
public Void call() {
ZimbraLog.clearContext();
ZimbraLog.addMboxToContext(getMailboxId());
ZimbraLog.datasource.debug("Running scheduled import for DataSource %s", getDataSourceId());
Mailbox mbox = null;
try {
// Look up mailbox, account and data source
mbox = MailboxManager.getInstance().getMailboxById(getMailboxId());
Account account = mbox.getAccount();
ZimbraLog.addAccountNameToContext(account.getName());
Provisioning prov = Provisioning.getInstance();
DataSource ds = prov.get(account, Key.DataSourceBy.id, getDataSourceId());
if (ds != null) {
ZimbraLog.addDataSourceNameToContext(ds.getName());
if (!ds.isEnabled()) {
ZimbraLog.datasource.info("DataSource is disabled. Cancelling future tasks.");
DataSourceManager.cancelTask(mbox, getDataSourceId());
return null;
}
// Do the work
DataSourceManager.importData(ds);
} else {
ZimbraLog.datasource.info(
"DataSource %s was deleted. Cancelling future tasks.", getDataSourceId());
DataSourceManager.cancelTask(mbox, getDataSourceId());
}
} catch (ServiceException e) {
ZimbraLog.datasource.warn("Scheduled DataSource import failed.", e);
return null;
}
ZimbraLog.clearContext();
return null;
}
static SetCalendarItemData getSetCalendarItemData(
ZimbraSoapContext zsc,
OperationContext octxt,
Account acct,
Mailbox mbox,
Element e,
ParseMimeMessage.InviteParser parser)
throws ServiceException {
String partStatStr =
e.getAttribute(MailConstants.A_CAL_PARTSTAT, IcalXmlStrMap.PARTSTAT_NEEDS_ACTION);
// <M>
Element msgElem = e.getElement(MailConstants.E_MSG);
// check to see whether the entire message has been uploaded under separate cover
String attachmentId = msgElem.getAttribute(MailConstants.A_ATTACHMENT_ID, null);
Element contentElement = msgElem.getOptionalElement(MailConstants.E_CONTENT);
InviteParserResult ipr = null;
MimeMessage mm = null;
if (attachmentId != null) {
ParseMimeMessage.MimeMessageData mimeData = new ParseMimeMessage.MimeMessageData();
mm = SendMsg.parseUploadedMessage(zsc, attachmentId, mimeData);
} else if (contentElement != null) {
mm = ParseMimeMessage.importMsgSoap(msgElem);
} else {
CalSendData dat = handleMsgElement(zsc, octxt, msgElem, acct, mbox, parser);
mm = dat.mMm;
ipr = parser.getResult();
}
if (ipr == null && msgElem.getOptionalElement(MailConstants.E_INVITE) != null) {
ipr = parser.parse(zsc, octxt, mbox.getAccount(), msgElem.getElement(MailConstants.E_INVITE));
// Get description texts out of the MimeMessage and set in the parsed invite. Do it only if
// the MimeMessage has text parts. This prevents discarding texts when they're specified only
// in the <inv> but not in mime parts.
if (ipr != null && ipr.mInvite != null && mm != null) {
String desc = Invite.getDescription(mm, MimeConstants.CT_TEXT_PLAIN);
String descHtml = Invite.getDescription(mm, MimeConstants.CT_TEXT_HTML);
if ((desc != null && desc.length() > 0) || (descHtml != null && descHtml.length() > 0))
ipr.mInvite.setDescription(desc, descHtml);
}
}
ParsedMessage pm = new ParsedMessage(mm, mbox.attachmentsIndexingEnabled());
Invite inv = (ipr == null ? null : ipr.mInvite);
if (inv == null || inv.getDTStamp() == -1) { // zdsync if -1 for 4.5 back compat
CalendarPartInfo cpi = pm.getCalendarPartInfo();
ZVCalendar cal = null;
if (cpi != null && CalendarItem.isAcceptableInvite(mbox.getAccount(), cpi)) cal = cpi.cal;
if (cal == null)
throw ServiceException.FAILURE("SetCalendarItem could not build an iCalendar object", null);
boolean sentByMe = false; // not applicable in the SetCalendarItem case
Invite iCalInv = Invite.createFromCalendar(acct, pm.getFragment(), cal, sentByMe).get(0);
if (inv == null) {
inv = iCalInv;
} else {
inv.setDtStamp(iCalInv.getDTStamp()); // zdsync
inv.setFragment(iCalInv.getFragment()); // zdsync
}
}
inv.setPartStat(partStatStr);
SetCalendarItemData sadata = new SetCalendarItemData();
sadata.invite = inv;
sadata.message = pm;
return sadata;
}
private void compile() throws ServiceException {
assert clauses != null;
if (clauses.isEmpty()) {
return;
}
// Convert list of Queries into list of QueryOperations, then optimize them.
// this generates all of the query operations
operation = parseTree.compile(mailbox);
ZimbraLog.search.debug("OP=%s", operation);
// expand the is:local and is:remote parts into in:(LIST)'s
operation = operation.expandLocalRemotePart(mailbox);
ZimbraLog.search.debug("AFTEREXP=%s", operation);
// optimize the query down
operation = operation.optimize(mailbox);
ZimbraLog.search.debug("OPTIMIZED=%s", operation);
if (operation == null || operation instanceof NoTermQueryOperation) {
operation = new NoResultsQueryOperation();
return;
}
// Use the OperationContext to update the set of visible referenced folders, local AND remote.
Set<QueryTarget> targets = operation.getQueryTargets();
assert (operation instanceof UnionQueryOperation
|| QueryTarget.getExplicitTargetCount(targets) <= 1);
// easiest to treat the query two unions: one the LOCAL and one REMOTE parts
UnionQueryOperation remoteOps = new UnionQueryOperation();
UnionQueryOperation localOps = new UnionQueryOperation();
if (operation instanceof UnionQueryOperation) {
UnionQueryOperation union = (UnionQueryOperation) operation;
// separate out the LOCAL vs REMOTE parts...
for (QueryOperation op : union.operations) {
Set<QueryTarget> set = op.getQueryTargets();
// this assertion OK because we have already distributed multi-target query ops
// during the optimize() step
assert (QueryTarget.getExplicitTargetCount(set) <= 1);
// the assertion above is critical: the code below all assumes
// that we only have ONE target (ie we've already distributed if necessary)
if (QueryTarget.hasExternalTarget(set)) {
remoteOps.add(op);
} else {
localOps.add(op);
}
}
} else {
// single target: might be local, might be remote
Set<QueryTarget> set = operation.getQueryTargets();
// this assertion OK because we have already distributed multi-target query ops
// during the optimize() step
assert (QueryTarget.getExplicitTargetCount(set) <= 1);
if (QueryTarget.hasExternalTarget(set)) {
remoteOps.add(operation);
} else {
localOps.add(operation);
}
}
// Handle the REMOTE side:
if (!remoteOps.operations.isEmpty()) {
// Since optimize() has already been run, we know that each of our ops
// only has one target (or none). Find those operations which have
// an external target and wrap them in RemoteQueryOperations
// iterate backwards so we can remove/add w/o screwing iteration
for (int i = remoteOps.operations.size() - 1; i >= 0; i--) {
QueryOperation op = remoteOps.operations.get(i);
Set<QueryTarget> set = op.getQueryTargets();
// this assertion OK because we have already distributed multi-target query ops
// during the optimize() step
assert (QueryTarget.getExplicitTargetCount(set) <= 1);
// the assertion above is critical: the code below all assumes
// that we only have ONE target (ie we've already distributed if necessary)
if (QueryTarget.hasExternalTarget(set)) {
remoteOps.operations.remove(i);
boolean foundOne = false;
// find a remoteOp to add this one to
for (QueryOperation remoteOp : remoteOps.operations) {
if (remoteOp instanceof RemoteQueryOperation) {
if (((RemoteQueryOperation) remoteOp).tryAddOredOperation(op)) {
foundOne = true;
break;
}
}
}
if (!foundOne) {
RemoteQueryOperation remoteOp = new RemoteQueryOperation();
remoteOp.tryAddOredOperation(op);
remoteOps.operations.add(i, remoteOp);
}
}
}
// ...we need to call setup on every RemoteQueryOperation we end up with...
for (QueryOperation remoteOp : remoteOps.operations) {
assert (remoteOp instanceof RemoteQueryOperation);
try {
((RemoteQueryOperation) remoteOp).setup(protocol, octxt.getAuthToken(), params);
} catch (Exception e) {
ZimbraLog.search.info("Ignoring %s during RemoteQuery generation for %s", e, remoteOps);
}
}
}
// For the LOCAL parts of the query, do permission checks, do trash/spam exclusion
if (!localOps.operations.isEmpty()) {
ZimbraLog.search.debug("LOCAL_IN=%s", localOps);
Account authAcct = octxt != null ? octxt.getAuthenticatedUser() : mailbox.getAccount();
// Now, for all the LOCAL PARTS of the query, add the trash/spam exclusion part
boolean includeTrash = false;
boolean includeSpam = false;
if (params.inDumpster()) {
// Always include trash and spam for dumpster searches. Excluding spam is a client side
// choice.
includeTrash = true;
includeSpam = true;
if (!mailbox.hasFullAdminAccess(
octxt)) { // If the requester is not an admin, limit to recent date range.
long now = octxt != null ? octxt.getTimestamp() : System.currentTimeMillis();
long mdate = now - authAcct.getDumpsterUserVisibleAge();
IntersectionQueryOperation and = new IntersectionQueryOperation();
DBQueryOperation db = new DBQueryOperation();
db.addMDateRange(mdate, false, -1L, false, true);
and.addQueryOp((QueryOperation) localOps.clone());
and.addQueryOp(db);
localOps.operations.clear();
localOps.operations.add(and);
}
} else {
includeTrash = authAcct.isPrefIncludeTrashInSearch();
;
includeSpam = authAcct.isPrefIncludeSpamInSearch();
}
if (!includeTrash || !includeSpam) {
// First check that we aren't specifically looking for items in one of these.
// For instance, in ZWC, if "Include Shared Items" is selected, the Trash list may currently
// use a
// search similar to : 'in:"trash" (inid:565 OR is:local)'
// where 565 is the folder ID for a shared folder. We don't want to end up doing a search
// for items
// that are both in "trash" and NOT in "trash"...
for (Query q : clauses) {
if (q.toString().equalsIgnoreCase("Q(IN:Trash)")) {
includeTrash = true;
}
if (q.toString().equalsIgnoreCase("Q(IN:Junk)")) {
includeSpam = true;
}
}
}
if (!includeTrash || !includeSpam) {
List<QueryOperation> toAdd = new ArrayList<QueryOperation>();
for (Iterator<QueryOperation> iter = localOps.operations.iterator(); iter.hasNext(); ) {
QueryOperation cur = iter.next();
if (!cur.hasSpamTrashSetting()) {
QueryOperation newOp = cur.ensureSpamTrashSetting(mailbox, includeTrash, includeSpam);
if (newOp != cur) {
iter.remove();
toAdd.add(newOp);
}
}
}
localOps.operations.addAll(toAdd);
ZimbraLog.search.debug("LOCAL_AFTER_TRASH/SPAM/DUMPSTER=%s", localOps);
}
// Check to see if we need to filter out private appointment data
boolean allowPrivateAccess = true;
if (octxt != null) {
allowPrivateAccess =
AccessManager.getInstance()
.allowPrivateAccess(
octxt.getAuthenticatedUser(),
mailbox.getAccount(),
octxt.isUsingAdminPrivileges());
}
//
// bug 28892 - ACL.RIGHT_PRIVATE support:
//
// Basically, if ACL.RIGHT_PRIVATE is set somewhere, and if we're excluding private items from
// search, then we need to run the query twice -- once over the whole mailbox with
// private items excluded and then UNION it with a second run, this time only in the
// RIGHT_PRIVATE enabled folders, with private items enabled.
//
UnionQueryOperation clonedLocal = null;
Set<Folder> hasFolderRightPrivateSet = new HashSet<Folder>();
// ...don't do any of this if they aren't asking for a calendar type...
Set<MailItem.Type> types = params.getTypes();
boolean hasCalendarType =
types.contains(MailItem.Type.APPOINTMENT) || types.contains(MailItem.Type.TASK);
if (hasCalendarType && !allowPrivateAccess && getTextOperationCount(localOps) > 0) {
// the searcher is NOT allowed to see private items globally....lets check
// to see if there are any individual folders that they DO have rights to...
// if there are any, then we'll need to run special searches in those
// folders
Set<Folder> allVisibleFolders = mailbox.getVisibleFolders(octxt);
if (allVisibleFolders == null) {
allVisibleFolders = new HashSet<Folder>();
allVisibleFolders.addAll(mailbox.getFolderList(octxt, SortBy.NONE));
}
for (Folder f : allVisibleFolders) {
if (f.getType() == MailItem.Type.FOLDER
&& CalendarItem.allowPrivateAccess(f, authAcct, false)) {
hasFolderRightPrivateSet.add(f);
}
}
if (!hasFolderRightPrivateSet.isEmpty()) {
clonedLocal = (UnionQueryOperation) localOps.clone();
}
}
Set<Folder> visibleFolders = mailbox.getVisibleFolders(octxt);
localOps = handleLocalPermissionChecks(localOps, visibleFolders, allowPrivateAccess);
ZimbraLog.search.debug("LOCAL_AFTER_PERM_CHECKS=%s", localOps);
if (!hasFolderRightPrivateSet.isEmpty()) {
ZimbraLog.search.debug("CLONED_LOCAL_BEFORE_PERM=%s", clonedLocal);
// now we're going to setup the clonedLocal tree
// to run with private access ALLOWED, over the set of folders
// that have RIGHT_PRIVATE (note that we build this list from the visible
// folder list, so we are
clonedLocal = handleLocalPermissionChecks(clonedLocal, hasFolderRightPrivateSet, true);
ZimbraLog.search.debug("CLONED_LOCAL_AFTER_PERM=%s", clonedLocal);
// clonedLocal should only have the single INTERSECT in it
assert (clonedLocal.operations.size() == 1);
QueryOperation optimizedClonedLocal = clonedLocal.optimize(mailbox);
ZimbraLog.search.debug("CLONED_LOCAL_AFTER_OPTIMIZE=%s", optimizedClonedLocal);
UnionQueryOperation withPrivateExcluded = localOps;
localOps = new UnionQueryOperation();
localOps.add(withPrivateExcluded);
localOps.add(optimizedClonedLocal);
ZimbraLog.search.debug("LOCAL_WITH_CLONED=%s", localOps);
//
// we should end up with:
//
// localOps =
// UNION(withPrivateExcluded,
// UNION(INTERSECT(clonedLocal,
// UNION(hasFolderRightPrivateList)
// )
// )
// )
//
}
}
UnionQueryOperation union = new UnionQueryOperation();
union.add(localOps);
union.add(remoteOps);
ZimbraLog.search.debug("BEFORE_FINAL_OPT=%s", union);
operation = union.optimize(mailbox);
ZimbraLog.search.debug("COMPILED=%s", operation);
}
