/** Verifies serializing, deserializing, and replaying for folder. */ @Test public void redoFolder() throws Exception { Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID); // Create folder. Folder folder = mbox.createFolder( null, "/redo", new Folder.FolderOptions().setDefaultView(MailItem.Type.MESSAGE)); assertEquals(0, folder.getRetentionPolicy().getKeepPolicy().size()); assertEquals(0, folder.getRetentionPolicy().getPurgePolicy().size()); // Create RedoableOp. RetentionPolicy rp = new RetentionPolicy( Arrays.asList(Policy.newSystemPolicy("123")), Arrays.asList(Policy.newUserPolicy("45m"))); SetRetentionPolicy redoPlayer = new SetRetentionPolicy(mbox.getId(), MailItem.Type.FOLDER, folder.getId(), rp); // Serialize, deserialize, and redo. byte[] data = redoPlayer.testSerialize(); redoPlayer = new SetRetentionPolicy(); redoPlayer.setMailboxId(mbox.getId()); redoPlayer.testDeserialize(data); redoPlayer.redo(); folder = mbox.getFolderById(null, folder.getId()); assertEquals(1, folder.getRetentionPolicy().getKeepPolicy().size()); assertEquals(1, folder.getRetentionPolicy().getPurgePolicy().size()); assertEquals("45m", folder.getRetentionPolicy().getPurgePolicy().get(0).getLifetime()); assertEquals("123", folder.getRetentionPolicy().getKeepPolicy().get(0).getId()); }
/** Verifies serializing, deserializing, and replaying for tag. */ @Test public void redoTag() throws Exception { Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID); // Create folder. Tag tag = mbox.createTag(null, "tag", (byte) 0); assertEquals(0, tag.getRetentionPolicy().getKeepPolicy().size()); assertEquals(0, tag.getRetentionPolicy().getPurgePolicy().size()); // Create RedoableOp. RetentionPolicy rp = new RetentionPolicy( Arrays.asList(Policy.newSystemPolicy("123")), Arrays.asList(Policy.newUserPolicy("45m"))); SetRetentionPolicy redoPlayer = new SetRetentionPolicy(mbox.getId(), MailItem.Type.TAG, tag.getId(), rp); // Serialize, deserialize, and redo. byte[] data = redoPlayer.testSerialize(); redoPlayer = new SetRetentionPolicy(); redoPlayer.setMailboxId(mbox.getId()); redoPlayer.testDeserialize(data); redoPlayer.redo(); tag = mbox.getTagById(null, tag.getId()); assertEquals(1, tag.getRetentionPolicy().getKeepPolicy().size()); assertEquals(1, tag.getRetentionPolicy().getPurgePolicy().size()); assertEquals("45m", tag.getRetentionPolicy().getPurgePolicy().get(0).getLifetime()); assertEquals("123", tag.getRetentionPolicy().getKeepPolicy().get(0).getId()); }
private void removeFromDedupeCache(String msgid, Mailbox mbox) { if (mbox == null || Strings.isNullOrEmpty(msgid)) return; synchronized (ZimbraLmtpBackend.class) { Set<Integer> mboxIds = receivedMessageIDs.get(msgid); if (mboxIds != null) { mboxIds.remove(mbox.getId()); } } }
private void addToDedupeCache(ParsedMessage pm, Mailbox mbox) { if (pm == null || mbox == null) return; String msgid = getMessageID(pm); if (msgid == null || msgid.equals("")) return; synchronized (ZimbraLmtpBackend.class) { Set<Integer> mboxIds = receivedMessageIDs.get(msgid); if (mboxIds == null) { mboxIds = new HashSet<Integer>(); receivedMessageIDs.put(msgid, mboxIds); } mboxIds.add(mbox.getId()); } }
private boolean dedupe(ParsedMessage pm, Mailbox mbox) throws ServiceException { if (pm == null || mbox == null || !mbox.getAccount().isPrefMessageIdDedupingEnabled()) return false; checkDedupeCacheSize(); String msgid = getMessageID(pm); if (msgid == null || msgid.equals("")) return false; synchronized (ZimbraLmtpBackend.class) { Set<Integer> mboxIds = receivedMessageIDs.get(msgid); if (mboxIds != null && mboxIds.contains(mbox.getId())) { return true; } } return false; }
@Override public Element handle(Element request, Map<String, Object> context) throws ServiceException { ZimbraSoapContext zsc = getZimbraSoapContext(context); Element mreq = request.getElement(AdminConstants.E_MAILBOX); String accountId = mreq.getAttribute(AdminConstants.A_ACCOUNTID); Account account = Provisioning.getInstance().get(AccountBy.id, accountId, zsc.getAuthToken()); if (account == null) { // Note: isDomainAdminOnly *always* returns false for pure ACL based AccessManager if (isDomainAdminOnly(zsc)) { throw ServiceException.PERM_DENIED( "account doesn't exist, unable to determine authorization"); } // still need to check right, since we don't have an account, the // last resort is checking the global grant. Do this for now until // there is complain. checkRight(zsc, context, null, Admin.R_deleteAccount); ZimbraLog.account.warn( "DeleteMailbox: account doesn't exist: " + accountId + " (still deleting mailbox)"); } else { checkAccountRight(zsc, account, Admin.R_deleteAccount); } Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(accountId, false); int mailboxId = -1; if (mbox != null) { mailboxId = mbox.getId(); mbox.deleteMailbox(); } String idString = (mbox == null) ? "<no mailbox for account " + accountId + ">" : Integer.toString(mailboxId); ZimbraLog.security.info( ZimbraLog.encodeAttrs(new String[] {"cmd", "DeleteMailbox", "id", idString})); Element response = zsc.createElement(AdminConstants.DELETE_MAILBOX_RESPONSE); if (mbox != null) response .addElement(AdminConstants.E_MAILBOX) .addAttribute(AdminConstants.A_MAILBOXID, mailboxId); return response; }
/** * initialize the Pop3Mailbox, without keeping a reference to either the Mailbox object or any of * the Message objects in the inbox. */ Pop3Mailbox(Mailbox mbox, Account acct, String query) throws ServiceException { id = mbox.getId(); numDeleted = 0; opContext = new OperationContext(acct); deleteOption = acct.getPrefPop3DeleteOption(); if (Strings.isNullOrEmpty(query)) { Set<Integer> folderIds = acct.isPrefPop3IncludeSpam() ? ImmutableSet.of(Mailbox.ID_FOLDER_INBOX, Mailbox.ID_FOLDER_SPAM) : Collections.singleton(Mailbox.ID_FOLDER_INBOX); String dateConstraint = acct.getAttr(Provisioning.A_zimbraPrefPop3DownloadSince); Date popSince = dateConstraint == null ? null : DateUtil.parseGeneralizedTime(dateConstraint); messages = mbox.openPop3Folder(opContext, folderIds, popSince); for (Pop3Message p3m : messages) { totalSize += p3m.getSize(); } } else { ZimbraQueryResults results = null; messages = new ArrayList<Pop3Message>(500); try { results = mbox.index.search(opContext, query, POP3_TYPES, SortBy.DATE_DESC, 500); while (results.hasNext()) { ZimbraHit hit = results.getNext(); if (hit instanceof MessageHit) { MessageHit mh = (MessageHit) hit; Message msg = mh.getMessage(); if (!msg.isTagged(Flag.FlagInfo.POPPED)) { totalSize += msg.getSize(); messages.add(new Pop3Message(msg)); } } } } finally { Closeables.closeQuietly(results); } } }
@Test public void mdate() throws Exception { Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID); DbConnection conn = DbPool.getConnection(); DbUtil.executeUpdate( conn, "INSERT INTO mboxgroup1.mail_item " + "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) " + "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)", mbox.getId(), 101, Mailbox.ID_FOLDER_INBOX, MailItem.Type.MESSAGE.toByte(), 100, 1000); DbUtil.executeUpdate( conn, "INSERT INTO mboxgroup1.mail_item " + "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) " + "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)", mbox.getId(), 102, Mailbox.ID_FOLDER_INBOX, MailItem.Type.MESSAGE.toByte(), 200, 2000); DbUtil.executeUpdate( conn, "INSERT INTO mboxgroup1.mail_item " + "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) " + "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)", mbox.getId(), 103, Mailbox.ID_FOLDER_INBOX, MailItem.Type.MESSAGE.toByte(), 300, 3000); DbUtil.executeUpdate( conn, "INSERT INTO mboxgroup1.mail_item " + "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) " + "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)", mbox.getId(), 104, Mailbox.ID_FOLDER_INBOX, MailItem.Type.MESSAGE.toByte(), 400, 4000); DbUtil.executeUpdate( conn, "INSERT INTO mboxgroup1.mail_item " + "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) " + "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)", mbox.getId(), 105, Mailbox.ID_FOLDER_INBOX, MailItem.Type.MESSAGE.toByte(), 500, 5000); conn.commit(); conn.closeQuietly(); SearchParams params = new SearchParams(); params.setQueryString("mdate:>3000000"); params.setSortBy(SortBy.DATE_ASC); params.setTypes(EnumSet.of(MailItem.Type.MESSAGE)); params.setFetchMode(SearchParams.Fetch.IDS); ZimbraQuery query = new ZimbraQuery(new OperationContext(mbox), SoapProtocol.Soap12, mbox, params); Assert.assertEquals("ZQ: Q(DATE:MDATE,197001010050-196912312359)", query.toString()); ZimbraQueryResults result = query.execute(); Assert.assertEquals(104, result.getNext().getItemId()); Assert.assertEquals(105, result.getNext().getItemId()); Assert.assertEquals(null, result.getNext()); Closeables.closeQuietly(result); }
private void deliverMessageToLocalMailboxes( Blob blob, BlobInputStream bis, byte[] data, MimeMessage mm, LmtpEnvelope env) throws ServiceException, IOException { List<LmtpAddress> recipients = env.getLocalRecipients(); String envSender = env.getSender().getEmailAddress(); boolean shared = recipients.size() > 1; List<Integer> targetMailboxIds = new ArrayList<Integer>(recipients.size()); Map<LmtpAddress, RecipientDetail> rcptMap = new HashMap<LmtpAddress, RecipientDetail>(recipients.size()); try { // Examine attachments indexing option for all recipients and // prepare ParsedMessage versions needed. Parsing is done before // attempting delivery to any recipient. Therefore, parse error // will result in non-delivery to all recipients. // ParsedMessage for users with attachments indexing ParsedMessage pmAttachIndex = null; // ParsedMessage for users without attachments indexing ParsedMessage pmNoAttachIndex = null; // message id for logging String msgId = null; for (LmtpAddress recipient : recipients) { String rcptEmail = recipient.getEmailAddress(); Account account; Mailbox mbox; boolean attachmentsIndexingEnabled; try { account = Provisioning.getInstance().get(AccountBy.name, rcptEmail); if (account == null) { ZimbraLog.mailbox.warn("No account found delivering mail to " + rcptEmail); continue; } mbox = MailboxManager.getInstance().getMailboxByAccount(account); if (mbox == null) { ZimbraLog.mailbox.warn("No mailbox found delivering mail to " + rcptEmail); continue; } attachmentsIndexingEnabled = mbox.attachmentsIndexingEnabled(); } catch (ServiceException se) { if (se.isReceiversFault()) { ZimbraLog.mailbox.info("Recoverable exception getting mailbox for " + rcptEmail, se); rcptMap.put( recipient, new RecipientDetail(null, null, null, false, DeliveryAction.defer)); } else { ZimbraLog.mailbox.warn("Unrecoverable exception getting mailbox for " + rcptEmail, se); } continue; } if (account != null && mbox != null) { ParsedMessageOptions pmo; if (mm != null) { pmo = new ParsedMessageOptions() .setContent(mm) .setDigest(blob.getDigest()) .setSize(blob.getRawSize()); } else { pmo = new ParsedMessageOptions(blob, data); } ParsedMessage pm; if (attachmentsIndexingEnabled) { if (pmAttachIndex == null) { pmo.setAttachmentIndexing(true); ZimbraLog.lmtp.debug( "Creating ParsedMessage from %s with attachment indexing enabled", data == null ? "file" : "memory"); pmAttachIndex = new ParsedMessage(pmo); } pm = pmAttachIndex; } else { if (pmNoAttachIndex == null) { pmo.setAttachmentIndexing(false); ZimbraLog.lmtp.debug( "Creating ParsedMessage from %s with attachment indexing disabled", data == null ? "file" : "memory"); pmNoAttachIndex = new ParsedMessage(pmo); } pm = pmNoAttachIndex; } msgId = pm.getMessageID(); if (account.isPrefMailLocalDeliveryDisabled()) { ZimbraLog.lmtp.debug("Local delivery disabled for account %s", rcptEmail); rcptMap.put( recipient, new RecipientDetail(account, mbox, pm, false, DeliveryAction.discard)); continue; } // For non-shared delivery (i.e. only one recipient), // always deliver regardless of backup mode. DeliveryAction da = DeliveryAction.deliver; boolean endSharedDelivery = false; if (shared) { if (mbox.beginSharedDelivery()) { endSharedDelivery = true; } else { // Skip delivery to mailboxes in backup mode. da = DeliveryAction.defer; } } rcptMap.put(recipient, new RecipientDetail(account, mbox, pm, endSharedDelivery, da)); if (da == DeliveryAction.deliver) { targetMailboxIds.add(mbox.getId()); } } } ZimbraLog.removeAccountFromContext(); if (ZimbraLog.lmtp.isInfoEnabled()) { ZimbraLog.lmtp.info( "Delivering message: size=%s, nrcpts=%d, sender=%s, msgid=%s", env.getSize() == 0 ? "unspecified" : Integer.toString(env.getSize()) + " bytes", recipients.size(), env.getSender(), msgId == null ? "" : msgId); } DeliveryContext sharedDeliveryCtxt = new DeliveryContext(shared, targetMailboxIds); sharedDeliveryCtxt.setIncomingBlob(blob); // We now know which addresses are valid and which ParsedMessage // version each recipient needs. Deliver! for (LmtpAddress recipient : recipients) { String rcptEmail = recipient.getEmailAddress(); LmtpReply reply = LmtpReply.TEMPORARY_FAILURE; RecipientDetail rd = rcptMap.get(recipient); if (rd.account != null) ZimbraLog.addAccountNameToContext(rd.account.getName()); if (rd.mbox != null) ZimbraLog.addMboxToContext(rd.mbox.getId()); boolean success = false; try { if (rd != null) { switch (rd.action) { case discard: ZimbraLog.lmtp.info( "accepted and discarded message from=%s,to=%s: local delivery is disabled", envSender, rcptEmail); if (rd.account.getPrefMailForwardingAddress() != null) { // mail forwarding is set up for (LmtpCallback callback : callbacks) { ZimbraLog.lmtp.debug("Executing callback %s", callback.getClass().getName()); callback.forwardWithoutDelivery( rd.account, rd.mbox, envSender, rcptEmail, rd.pm); } } reply = LmtpReply.DELIVERY_OK; break; case deliver: Account account = rd.account; Mailbox mbox = rd.mbox; ParsedMessage pm = rd.pm; List<ItemId> addedMessageIds = null; ReentrantLock lock = mailboxDeliveryLocks.get(mbox.getId()); boolean acquiredLock; try { // Wait for the lock, up to the timeout acquiredLock = lock.tryLock(LC.zimbra_mailbox_lock_timeout.intValue(), TimeUnit.SECONDS); } catch (InterruptedException e) { acquiredLock = false; } if (!acquiredLock) { ZimbraLog.lmtp.info( "try again for message from=%s,to=%s: another mail delivery in progress.", envSender, rcptEmail); reply = LmtpReply.TEMPORARY_FAILURE; break; } try { if (dedupe(pm, mbox)) { // message was already delivered to this mailbox ZimbraLog.lmtp.info( "Not delivering message with duplicate Message-ID %s", pm.getMessageID()); } else if (recipient.getSkipFilters()) { msgId = pm.getMessageID(); int folderId = Mailbox.ID_FOLDER_INBOX; if (recipient.getFolder() != null) { try { Folder folder = mbox.getFolderByPath(null, recipient.getFolder()); folderId = folder.getId(); } catch (ServiceException se) { if (se.getCode().equals(MailServiceException.NO_SUCH_FOLDER)) { Folder folder = mbox.createFolder( null, recipient.getFolder(), new Folder.FolderOptions().setDefaultView(MailItem.Type.MESSAGE)); folderId = folder.getId(); } else { throw se; } } } int flags = Flag.BITMASK_UNREAD; if (recipient.getFlags() != null) { flags = Flag.toBitmask(recipient.getFlags()); } DeliveryOptions dopt = new DeliveryOptions().setFolderId(folderId); dopt.setFlags(flags).setTags(recipient.getTags()).setRecipientEmail(rcptEmail); Message msg = mbox.addMessage(null, pm, dopt, sharedDeliveryCtxt); addedMessageIds = Lists.newArrayList(new ItemId(msg)); } else if (!DebugConfig.disableIncomingFilter) { // Get msgid first, to avoid having to reopen and reparse the blob // file if Mailbox.addMessageInternal() closes it. pm.getMessageID(); addedMessageIds = RuleManager.applyRulesToIncomingMessage( null, mbox, pm, (int) blob.getRawSize(), rcptEmail, sharedDeliveryCtxt, Mailbox.ID_FOLDER_INBOX, false); } else { pm.getMessageID(); DeliveryOptions dopt = new DeliveryOptions().setFolderId(Mailbox.ID_FOLDER_INBOX); dopt.setFlags(Flag.BITMASK_UNREAD).setRecipientEmail(rcptEmail); Message msg = mbox.addMessage(null, pm, dopt, sharedDeliveryCtxt); addedMessageIds = Lists.newArrayList(new ItemId(msg)); } success = true; if (addedMessageIds != null && addedMessageIds.size() > 0) { addToDedupeCache(pm, mbox); } } finally { lock.unlock(); } if (addedMessageIds != null && addedMessageIds.size() > 0) { // Execute callbacks for (LmtpCallback callback : callbacks) { for (ItemId id : addedMessageIds) { if (id.belongsTo(mbox)) { // Message was added to the local mailbox, as opposed to a mountpoint. ZimbraLog.lmtp.debug( "Executing callback %s", callback.getClass().getName()); try { Message msg = mbox.getMessageById(null, id.getId()); callback.afterDelivery(account, mbox, envSender, rcptEmail, msg); } catch (Throwable t) { if (t instanceof OutOfMemoryError) { Zimbra.halt("LMTP callback failed", t); } else { ZimbraLog.lmtp.warn("LMTP callback threw an exception", t); } } } } } } reply = LmtpReply.DELIVERY_OK; break; case defer: // Delivery to mailbox skipped. Let MTA retry again later. // This case happens for shared delivery to a mailbox in // backup mode. ZimbraLog.lmtp.info( "try again for message from=%s,to=%s: mailbox skipped", envSender, rcptEmail); reply = LmtpReply.TEMPORARY_FAILURE; break; } } else { // Account or mailbox not found. ZimbraLog.lmtp.info( "rejecting message from=%s,to=%s: account or mailbox not found", envSender, rcptEmail); reply = LmtpReply.PERMANENT_FAILURE; } } catch (ServiceException e) { if (e.getCode().equals(MailServiceException.QUOTA_EXCEEDED)) { ZimbraLog.lmtp.info("rejecting message from=%s,to=%s: overquota", envSender, rcptEmail); if (config.isPermanentFailureWhenOverQuota()) { reply = LmtpReply.PERMANENT_FAILURE_OVER_QUOTA; } else { reply = LmtpReply.TEMPORARY_FAILURE_OVER_QUOTA; } } else if (e.isReceiversFault()) { ZimbraLog.lmtp.info("try again for message from=%s,to=%s", envSender, rcptEmail, e); reply = LmtpReply.TEMPORARY_FAILURE; } else { ZimbraLog.lmtp.info("rejecting message from=%s,to=%s", envSender, rcptEmail, e); reply = LmtpReply.PERMANENT_FAILURE; } } catch (Exception e) { reply = LmtpReply.TEMPORARY_FAILURE; ZimbraLog.lmtp.warn("try again for message from=%s,to=%s", envSender, rcptEmail, e); } finally { if (rd.action == DeliveryAction.deliver && !success) { // Message was not delivered. Remove it from the dedupe // cache so we don't dedupe it on LMTP retry. removeFromDedupeCache(msgId, rd.mbox); } recipient.setDeliveryStatus(reply); if (shared && rd != null && rd.esd) { rd.mbox.endSharedDelivery(); rd.esd = false; } } } // If this message is being streamed from disk, cache it ParsedMessage mimeSource = pmAttachIndex != null ? pmAttachIndex : pmNoAttachIndex; MailboxBlob mblob = sharedDeliveryCtxt.getMailboxBlob(); if (mblob != null && mimeSource != null) { if (bis == null) { bis = mimeSource.getBlobInputStream(); } if (bis != null) { try { // Update the MimeMessage with the blob that's stored inside the mailbox, // since the incoming blob will be deleted. Blob storedBlob = mblob.getLocalBlob(); bis.fileMoved(storedBlob.getFile()); MessageCache.cacheMessage( mblob.getDigest(), mimeSource.getOriginalMessage(), mimeSource.getMimeMessage()); } catch (IOException e) { ZimbraLog.lmtp.warn("Unable to cache message for " + mblob, e); } } } } finally { // If there were any stray exceptions after the call to // beginSharedDelivery that caused endSharedDelivery to be not // called, we check and fix those here. if (shared) { for (RecipientDetail rd : rcptMap.values()) { if (rd.esd && rd.mbox != null) rd.mbox.endSharedDelivery(); } } } }