@Override public ItemId fileInto(String folderPath, Collection<ActionFlag> flagActions, String[] tags) throws ServiceException { Message source = getMessage(); // See if the message is already in the target folder. Folder targetFolder = null; try { targetFolder = mailbox.getFolderByPath(octxt, folderPath); } catch (NoSuchItemException ignored) { } if (targetFolder != null && source.getFolderId() == targetFolder.getId()) { ZimbraLog.filter.debug( "Ignoring fileinto action for message %d. It is already in %s.", messageId, folderPath); updateTagsAndFlagsIfNecessary(source, flagActions, tags); return null; } ZimbraLog.filter.info("Copying existing message %d to folder %s.", messageId, folderPath); if (isLocalExistingFolder(folderPath)) { // Copy item into to a local folder. Folder target = mailbox.getFolderByPath(octxt, folderPath); Message newMsg = (Message) mailbox.copy(octxt, messageId, MailItem.Type.MESSAGE, target.getId()); filtered = true; filed = true; // Apply flags and tags mailbox.setTags( octxt, newMsg.getId(), MailItem.Type.MESSAGE, FilterUtil.getFlagBitmask(flagActions, source.getFlagBitmask()), FilterUtil.getTagsUnion(source.getTags(), tags)); return new ItemId(mailbox, messageId); } ItemId id = FilterUtil.addMessage( new DeliveryContext(), mailbox, getParsedMessage(), mailbox.getAccount().getName(), folderPath, false, FilterUtil.getFlagBitmask(flagActions, source.getFlagBitmask()), tags, Mailbox.ID_AUTO_INCREMENT, octxt); if (id != null) { filtered = true; filed = true; } return id; }
private boolean dedupe(ParsedMessage pm, Mailbox mbox) throws ServiceException { if (pm == null || mbox == null || !mbox.getAccount().isPrefMessageIdDedupingEnabled()) return false; checkDedupeCacheSize(); String msgid = getMessageID(pm); if (msgid == null || msgid.equals("")) return false; synchronized (ZimbraLmtpBackend.class) { Set<Integer> mboxIds = receivedMessageIDs.get(msgid); if (mboxIds != null && mboxIds.contains(mbox.getId())) { return true; } } return false; }
@Override public Element handle(Element request, Map<String, Object> context) throws ServiceException { ZimbraSoapContext zsc = getZimbraSoapContext(context); Mailbox mbox = getRequestedMailbox(zsc); Account account = getRequestedAccount(zsc); OperationContext octxt = getOperationContext(zsc, context); SearchRequest req = JaxbUtil.elementToJaxb(request); if (Objects.firstNonNull(req.getWarmup(), false)) { mbox.index.getIndexStore().warmup(); return zsc.createElement(MailConstants.SEARCH_RESPONSE); } SearchParams params = SearchParams.parse(req, zsc, account.getPrefMailInitialSearch()); if (params.getLocale() == null) { params.setLocale(mbox.getAccount().getLocale()); } if (params.inDumpster() && params.getTypes().contains(MailItem.Type.CONVERSATION)) { throw ServiceException.INVALID_REQUEST("cannot search for conversations in dumpster", null); } if (LC.calendar_cache_enabled.booleanValue()) { List<String> apptFolderIds = getFolderIdListIfSimpleAppointmentsQuery(params, zsc); if (apptFolderIds != null) { Account authAcct = getAuthenticatedAccount(zsc); Element response = zsc.createElement(MailConstants.SEARCH_RESPONSE); runSimpleAppointmentQuery(response, params, octxt, zsc, authAcct, mbox, apptFolderIds); return response; } } ZimbraQueryResults results = mbox.index.search(zsc.getResponseProtocol(), octxt, params); try { // create the XML response Element Element response = zsc.createElement(MailConstants.SEARCH_RESPONSE); // must use results.getSortBy() because the results might have ignored our sortBy // request and used something else... response.addAttribute(MailConstants.A_SORTBY, results.getSortBy().toString()); putHits(zsc, octxt, response, results, params); return response; } finally { Closeables.closeQuietly(results); } }
@Override public Void call() { ZimbraLog.clearContext(); ZimbraLog.addMboxToContext(getMailboxId()); ZimbraLog.datasource.debug("Running scheduled import for DataSource %s", getDataSourceId()); Mailbox mbox = null; try { // Look up mailbox, account and data source mbox = MailboxManager.getInstance().getMailboxById(getMailboxId()); Account account = mbox.getAccount(); ZimbraLog.addAccountNameToContext(account.getName()); Provisioning prov = Provisioning.getInstance(); DataSource ds = prov.get(account, Key.DataSourceBy.id, getDataSourceId()); if (ds != null) { ZimbraLog.addDataSourceNameToContext(ds.getName()); if (!ds.isEnabled()) { ZimbraLog.datasource.info("DataSource is disabled. Cancelling future tasks."); DataSourceManager.cancelTask(mbox, getDataSourceId()); return null; } // Do the work DataSourceManager.importData(ds); } else { ZimbraLog.datasource.info( "DataSource %s was deleted. Cancelling future tasks.", getDataSourceId()); DataSourceManager.cancelTask(mbox, getDataSourceId()); } } catch (ServiceException e) { ZimbraLog.datasource.warn("Scheduled DataSource import failed.", e); return null; } ZimbraLog.clearContext(); return null; }
static SetCalendarItemData getSetCalendarItemData( ZimbraSoapContext zsc, OperationContext octxt, Account acct, Mailbox mbox, Element e, ParseMimeMessage.InviteParser parser) throws ServiceException { String partStatStr = e.getAttribute(MailConstants.A_CAL_PARTSTAT, IcalXmlStrMap.PARTSTAT_NEEDS_ACTION); // <M> Element msgElem = e.getElement(MailConstants.E_MSG); // check to see whether the entire message has been uploaded under separate cover String attachmentId = msgElem.getAttribute(MailConstants.A_ATTACHMENT_ID, null); Element contentElement = msgElem.getOptionalElement(MailConstants.E_CONTENT); InviteParserResult ipr = null; MimeMessage mm = null; if (attachmentId != null) { ParseMimeMessage.MimeMessageData mimeData = new ParseMimeMessage.MimeMessageData(); mm = SendMsg.parseUploadedMessage(zsc, attachmentId, mimeData); } else if (contentElement != null) { mm = ParseMimeMessage.importMsgSoap(msgElem); } else { CalSendData dat = handleMsgElement(zsc, octxt, msgElem, acct, mbox, parser); mm = dat.mMm; ipr = parser.getResult(); } if (ipr == null && msgElem.getOptionalElement(MailConstants.E_INVITE) != null) { ipr = parser.parse(zsc, octxt, mbox.getAccount(), msgElem.getElement(MailConstants.E_INVITE)); // Get description texts out of the MimeMessage and set in the parsed invite. Do it only if // the MimeMessage has text parts. This prevents discarding texts when they're specified only // in the <inv> but not in mime parts. if (ipr != null && ipr.mInvite != null && mm != null) { String desc = Invite.getDescription(mm, MimeConstants.CT_TEXT_PLAIN); String descHtml = Invite.getDescription(mm, MimeConstants.CT_TEXT_HTML); if ((desc != null && desc.length() > 0) || (descHtml != null && descHtml.length() > 0)) ipr.mInvite.setDescription(desc, descHtml); } } ParsedMessage pm = new ParsedMessage(mm, mbox.attachmentsIndexingEnabled()); Invite inv = (ipr == null ? null : ipr.mInvite); if (inv == null || inv.getDTStamp() == -1) { // zdsync if -1 for 4.5 back compat CalendarPartInfo cpi = pm.getCalendarPartInfo(); ZVCalendar cal = null; if (cpi != null && CalendarItem.isAcceptableInvite(mbox.getAccount(), cpi)) cal = cpi.cal; if (cal == null) throw ServiceException.FAILURE("SetCalendarItem could not build an iCalendar object", null); boolean sentByMe = false; // not applicable in the SetCalendarItem case Invite iCalInv = Invite.createFromCalendar(acct, pm.getFragment(), cal, sentByMe).get(0); if (inv == null) { inv = iCalInv; } else { inv.setDtStamp(iCalInv.getDTStamp()); // zdsync inv.setFragment(iCalInv.getFragment()); // zdsync } } inv.setPartStat(partStatStr); SetCalendarItemData sadata = new SetCalendarItemData(); sadata.invite = inv; sadata.message = pm; return sadata; }
private void compile() throws ServiceException { assert clauses != null; if (clauses.isEmpty()) { return; } // Convert list of Queries into list of QueryOperations, then optimize them. // this generates all of the query operations operation = parseTree.compile(mailbox); ZimbraLog.search.debug("OP=%s", operation); // expand the is:local and is:remote parts into in:(LIST)'s operation = operation.expandLocalRemotePart(mailbox); ZimbraLog.search.debug("AFTEREXP=%s", operation); // optimize the query down operation = operation.optimize(mailbox); ZimbraLog.search.debug("OPTIMIZED=%s", operation); if (operation == null || operation instanceof NoTermQueryOperation) { operation = new NoResultsQueryOperation(); return; } // Use the OperationContext to update the set of visible referenced folders, local AND remote. Set<QueryTarget> targets = operation.getQueryTargets(); assert (operation instanceof UnionQueryOperation || QueryTarget.getExplicitTargetCount(targets) <= 1); // easiest to treat the query two unions: one the LOCAL and one REMOTE parts UnionQueryOperation remoteOps = new UnionQueryOperation(); UnionQueryOperation localOps = new UnionQueryOperation(); if (operation instanceof UnionQueryOperation) { UnionQueryOperation union = (UnionQueryOperation) operation; // separate out the LOCAL vs REMOTE parts... for (QueryOperation op : union.operations) { Set<QueryTarget> set = op.getQueryTargets(); // this assertion OK because we have already distributed multi-target query ops // during the optimize() step assert (QueryTarget.getExplicitTargetCount(set) <= 1); // the assertion above is critical: the code below all assumes // that we only have ONE target (ie we've already distributed if necessary) if (QueryTarget.hasExternalTarget(set)) { remoteOps.add(op); } else { localOps.add(op); } } } else { // single target: might be local, might be remote Set<QueryTarget> set = operation.getQueryTargets(); // this assertion OK because we have already distributed multi-target query ops // during the optimize() step assert (QueryTarget.getExplicitTargetCount(set) <= 1); if (QueryTarget.hasExternalTarget(set)) { remoteOps.add(operation); } else { localOps.add(operation); } } // Handle the REMOTE side: if (!remoteOps.operations.isEmpty()) { // Since optimize() has already been run, we know that each of our ops // only has one target (or none). Find those operations which have // an external target and wrap them in RemoteQueryOperations // iterate backwards so we can remove/add w/o screwing iteration for (int i = remoteOps.operations.size() - 1; i >= 0; i--) { QueryOperation op = remoteOps.operations.get(i); Set<QueryTarget> set = op.getQueryTargets(); // this assertion OK because we have already distributed multi-target query ops // during the optimize() step assert (QueryTarget.getExplicitTargetCount(set) <= 1); // the assertion above is critical: the code below all assumes // that we only have ONE target (ie we've already distributed if necessary) if (QueryTarget.hasExternalTarget(set)) { remoteOps.operations.remove(i); boolean foundOne = false; // find a remoteOp to add this one to for (QueryOperation remoteOp : remoteOps.operations) { if (remoteOp instanceof RemoteQueryOperation) { if (((RemoteQueryOperation) remoteOp).tryAddOredOperation(op)) { foundOne = true; break; } } } if (!foundOne) { RemoteQueryOperation remoteOp = new RemoteQueryOperation(); remoteOp.tryAddOredOperation(op); remoteOps.operations.add(i, remoteOp); } } } // ...we need to call setup on every RemoteQueryOperation we end up with... for (QueryOperation remoteOp : remoteOps.operations) { assert (remoteOp instanceof RemoteQueryOperation); try { ((RemoteQueryOperation) remoteOp).setup(protocol, octxt.getAuthToken(), params); } catch (Exception e) { ZimbraLog.search.info("Ignoring %s during RemoteQuery generation for %s", e, remoteOps); } } } // For the LOCAL parts of the query, do permission checks, do trash/spam exclusion if (!localOps.operations.isEmpty()) { ZimbraLog.search.debug("LOCAL_IN=%s", localOps); Account authAcct = octxt != null ? octxt.getAuthenticatedUser() : mailbox.getAccount(); // Now, for all the LOCAL PARTS of the query, add the trash/spam exclusion part boolean includeTrash = false; boolean includeSpam = false; if (params.inDumpster()) { // Always include trash and spam for dumpster searches. Excluding spam is a client side // choice. includeTrash = true; includeSpam = true; if (!mailbox.hasFullAdminAccess( octxt)) { // If the requester is not an admin, limit to recent date range. long now = octxt != null ? octxt.getTimestamp() : System.currentTimeMillis(); long mdate = now - authAcct.getDumpsterUserVisibleAge(); IntersectionQueryOperation and = new IntersectionQueryOperation(); DBQueryOperation db = new DBQueryOperation(); db.addMDateRange(mdate, false, -1L, false, true); and.addQueryOp((QueryOperation) localOps.clone()); and.addQueryOp(db); localOps.operations.clear(); localOps.operations.add(and); } } else { includeTrash = authAcct.isPrefIncludeTrashInSearch(); ; includeSpam = authAcct.isPrefIncludeSpamInSearch(); } if (!includeTrash || !includeSpam) { // First check that we aren't specifically looking for items in one of these. // For instance, in ZWC, if "Include Shared Items" is selected, the Trash list may currently // use a // search similar to : 'in:"trash" (inid:565 OR is:local)' // where 565 is the folder ID for a shared folder. We don't want to end up doing a search // for items // that are both in "trash" and NOT in "trash"... for (Query q : clauses) { if (q.toString().equalsIgnoreCase("Q(IN:Trash)")) { includeTrash = true; } if (q.toString().equalsIgnoreCase("Q(IN:Junk)")) { includeSpam = true; } } } if (!includeTrash || !includeSpam) { List<QueryOperation> toAdd = new ArrayList<QueryOperation>(); for (Iterator<QueryOperation> iter = localOps.operations.iterator(); iter.hasNext(); ) { QueryOperation cur = iter.next(); if (!cur.hasSpamTrashSetting()) { QueryOperation newOp = cur.ensureSpamTrashSetting(mailbox, includeTrash, includeSpam); if (newOp != cur) { iter.remove(); toAdd.add(newOp); } } } localOps.operations.addAll(toAdd); ZimbraLog.search.debug("LOCAL_AFTER_TRASH/SPAM/DUMPSTER=%s", localOps); } // Check to see if we need to filter out private appointment data boolean allowPrivateAccess = true; if (octxt != null) { allowPrivateAccess = AccessManager.getInstance() .allowPrivateAccess( octxt.getAuthenticatedUser(), mailbox.getAccount(), octxt.isUsingAdminPrivileges()); } // // bug 28892 - ACL.RIGHT_PRIVATE support: // // Basically, if ACL.RIGHT_PRIVATE is set somewhere, and if we're excluding private items from // search, then we need to run the query twice -- once over the whole mailbox with // private items excluded and then UNION it with a second run, this time only in the // RIGHT_PRIVATE enabled folders, with private items enabled. // UnionQueryOperation clonedLocal = null; Set<Folder> hasFolderRightPrivateSet = new HashSet<Folder>(); // ...don't do any of this if they aren't asking for a calendar type... Set<MailItem.Type> types = params.getTypes(); boolean hasCalendarType = types.contains(MailItem.Type.APPOINTMENT) || types.contains(MailItem.Type.TASK); if (hasCalendarType && !allowPrivateAccess && getTextOperationCount(localOps) > 0) { // the searcher is NOT allowed to see private items globally....lets check // to see if there are any individual folders that they DO have rights to... // if there are any, then we'll need to run special searches in those // folders Set<Folder> allVisibleFolders = mailbox.getVisibleFolders(octxt); if (allVisibleFolders == null) { allVisibleFolders = new HashSet<Folder>(); allVisibleFolders.addAll(mailbox.getFolderList(octxt, SortBy.NONE)); } for (Folder f : allVisibleFolders) { if (f.getType() == MailItem.Type.FOLDER && CalendarItem.allowPrivateAccess(f, authAcct, false)) { hasFolderRightPrivateSet.add(f); } } if (!hasFolderRightPrivateSet.isEmpty()) { clonedLocal = (UnionQueryOperation) localOps.clone(); } } Set<Folder> visibleFolders = mailbox.getVisibleFolders(octxt); localOps = handleLocalPermissionChecks(localOps, visibleFolders, allowPrivateAccess); ZimbraLog.search.debug("LOCAL_AFTER_PERM_CHECKS=%s", localOps); if (!hasFolderRightPrivateSet.isEmpty()) { ZimbraLog.search.debug("CLONED_LOCAL_BEFORE_PERM=%s", clonedLocal); // now we're going to setup the clonedLocal tree // to run with private access ALLOWED, over the set of folders // that have RIGHT_PRIVATE (note that we build this list from the visible // folder list, so we are clonedLocal = handleLocalPermissionChecks(clonedLocal, hasFolderRightPrivateSet, true); ZimbraLog.search.debug("CLONED_LOCAL_AFTER_PERM=%s", clonedLocal); // clonedLocal should only have the single INTERSECT in it assert (clonedLocal.operations.size() == 1); QueryOperation optimizedClonedLocal = clonedLocal.optimize(mailbox); ZimbraLog.search.debug("CLONED_LOCAL_AFTER_OPTIMIZE=%s", optimizedClonedLocal); UnionQueryOperation withPrivateExcluded = localOps; localOps = new UnionQueryOperation(); localOps.add(withPrivateExcluded); localOps.add(optimizedClonedLocal); ZimbraLog.search.debug("LOCAL_WITH_CLONED=%s", localOps); // // we should end up with: // // localOps = // UNION(withPrivateExcluded, // UNION(INTERSECT(clonedLocal, // UNION(hasFolderRightPrivateList) // ) // ) // ) // } } UnionQueryOperation union = new UnionQueryOperation(); union.add(localOps); union.add(remoteOps); ZimbraLog.search.debug("BEFORE_FINAL_OPT=%s", union); operation = union.optimize(mailbox); ZimbraLog.search.debug("COMPILED=%s", operation); }