/** Verifies serializing, deserializing, and replaying for folder. */
@Test
public void redoFolder() throws Exception {
Mailbox mbox =
MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
// Create folder.
Folder folder =
mbox.createFolder(
null, "/redo", new Folder.FolderOptions().setDefaultView(MailItem.Type.MESSAGE));
assertEquals(0, folder.getRetentionPolicy().getKeepPolicy().size());
assertEquals(0, folder.getRetentionPolicy().getPurgePolicy().size());
// Create RedoableOp.
RetentionPolicy rp =
new RetentionPolicy(
Arrays.asList(Policy.newSystemPolicy("123")),
Arrays.asList(Policy.newUserPolicy("45m")));
SetRetentionPolicy redoPlayer =
new SetRetentionPolicy(mbox.getId(), MailItem.Type.FOLDER, folder.getId(), rp);
// Serialize, deserialize, and redo.
byte[] data = redoPlayer.testSerialize();
redoPlayer = new SetRetentionPolicy();
redoPlayer.setMailboxId(mbox.getId());
redoPlayer.testDeserialize(data);
redoPlayer.redo();
folder = mbox.getFolderById(null, folder.getId());
assertEquals(1, folder.getRetentionPolicy().getKeepPolicy().size());
assertEquals(1, folder.getRetentionPolicy().getPurgePolicy().size());
assertEquals("45m", folder.getRetentionPolicy().getPurgePolicy().get(0).getLifetime());
assertEquals("123", folder.getRetentionPolicy().getKeepPolicy().get(0).getId());
}
/** Verifies serializing, deserializing, and replaying for tag. */
@Test
public void redoTag() throws Exception {
Mailbox mbox =
MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
// Create folder.
Tag tag = mbox.createTag(null, "tag", (byte) 0);
assertEquals(0, tag.getRetentionPolicy().getKeepPolicy().size());
assertEquals(0, tag.getRetentionPolicy().getPurgePolicy().size());
// Create RedoableOp.
RetentionPolicy rp =
new RetentionPolicy(
Arrays.asList(Policy.newSystemPolicy("123")),
Arrays.asList(Policy.newUserPolicy("45m")));
SetRetentionPolicy redoPlayer =
new SetRetentionPolicy(mbox.getId(), MailItem.Type.TAG, tag.getId(), rp);
// Serialize, deserialize, and redo.
byte[] data = redoPlayer.testSerialize();
redoPlayer = new SetRetentionPolicy();
redoPlayer.setMailboxId(mbox.getId());
redoPlayer.testDeserialize(data);
redoPlayer.redo();
tag = mbox.getTagById(null, tag.getId());
assertEquals(1, tag.getRetentionPolicy().getKeepPolicy().size());
assertEquals(1, tag.getRetentionPolicy().getPurgePolicy().size());
assertEquals("45m", tag.getRetentionPolicy().getPurgePolicy().get(0).getLifetime());
assertEquals("123", tag.getRetentionPolicy().getKeepPolicy().get(0).getId());
}
private void removeFromDedupeCache(String msgid, Mailbox mbox) {
if (mbox == null || Strings.isNullOrEmpty(msgid)) return;
synchronized (ZimbraLmtpBackend.class) {
Set<Integer> mboxIds = receivedMessageIDs.get(msgid);
if (mboxIds != null) {
mboxIds.remove(mbox.getId());
}
}
}
private void addToDedupeCache(ParsedMessage pm, Mailbox mbox) {
if (pm == null || mbox == null) return;
String msgid = getMessageID(pm);
if (msgid == null || msgid.equals("")) return;
synchronized (ZimbraLmtpBackend.class) {
Set<Integer> mboxIds = receivedMessageIDs.get(msgid);
if (mboxIds == null) {
mboxIds = new HashSet<Integer>();
receivedMessageIDs.put(msgid, mboxIds);
}
mboxIds.add(mbox.getId());
}
}
private boolean dedupe(ParsedMessage pm, Mailbox mbox) throws ServiceException {
if (pm == null || mbox == null || !mbox.getAccount().isPrefMessageIdDedupingEnabled())
return false;
checkDedupeCacheSize();
String msgid = getMessageID(pm);
if (msgid == null || msgid.equals("")) return false;
synchronized (ZimbraLmtpBackend.class) {
Set<Integer> mboxIds = receivedMessageIDs.get(msgid);
if (mboxIds != null && mboxIds.contains(mbox.getId())) {
return true;
}
}
return false;
}
@Override
public Element handle(Element request, Map<String, Object> context) throws ServiceException {
ZimbraSoapContext zsc = getZimbraSoapContext(context);
Element mreq = request.getElement(AdminConstants.E_MAILBOX);
String accountId = mreq.getAttribute(AdminConstants.A_ACCOUNTID);
Account account = Provisioning.getInstance().get(AccountBy.id, accountId, zsc.getAuthToken());
if (account == null) {
// Note: isDomainAdminOnly *always* returns false for pure ACL based AccessManager
if (isDomainAdminOnly(zsc)) {
throw ServiceException.PERM_DENIED(
"account doesn't exist, unable to determine authorization");
}
// still need to check right, since we don't have an account, the
// last resort is checking the global grant. Do this for now until
// there is complain.
checkRight(zsc, context, null, Admin.R_deleteAccount);
ZimbraLog.account.warn(
"DeleteMailbox: account doesn't exist: " + accountId + " (still deleting mailbox)");
} else {
checkAccountRight(zsc, account, Admin.R_deleteAccount);
}
Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(accountId, false);
int mailboxId = -1;
if (mbox != null) {
mailboxId = mbox.getId();
mbox.deleteMailbox();
}
String idString =
(mbox == null) ? "<no mailbox for account " + accountId + ">" : Integer.toString(mailboxId);
ZimbraLog.security.info(
ZimbraLog.encodeAttrs(new String[] {"cmd", "DeleteMailbox", "id", idString}));
Element response = zsc.createElement(AdminConstants.DELETE_MAILBOX_RESPONSE);
if (mbox != null)
response
.addElement(AdminConstants.E_MAILBOX)
.addAttribute(AdminConstants.A_MAILBOXID, mailboxId);
return response;
}
/**
* initialize the Pop3Mailbox, without keeping a reference to either the Mailbox object or any of
* the Message objects in the inbox.
*/
Pop3Mailbox(Mailbox mbox, Account acct, String query) throws ServiceException {
id = mbox.getId();
numDeleted = 0;
opContext = new OperationContext(acct);
deleteOption = acct.getPrefPop3DeleteOption();
if (Strings.isNullOrEmpty(query)) {
Set<Integer> folderIds =
acct.isPrefPop3IncludeSpam()
? ImmutableSet.of(Mailbox.ID_FOLDER_INBOX, Mailbox.ID_FOLDER_SPAM)
: Collections.singleton(Mailbox.ID_FOLDER_INBOX);
String dateConstraint = acct.getAttr(Provisioning.A_zimbraPrefPop3DownloadSince);
Date popSince = dateConstraint == null ? null : DateUtil.parseGeneralizedTime(dateConstraint);
messages = mbox.openPop3Folder(opContext, folderIds, popSince);
for (Pop3Message p3m : messages) {
totalSize += p3m.getSize();
}
} else {
ZimbraQueryResults results = null;
messages = new ArrayList<Pop3Message>(500);
try {
results = mbox.index.search(opContext, query, POP3_TYPES, SortBy.DATE_DESC, 500);
while (results.hasNext()) {
ZimbraHit hit = results.getNext();
if (hit instanceof MessageHit) {
MessageHit mh = (MessageHit) hit;
Message msg = mh.getMessage();
if (!msg.isTagged(Flag.FlagInfo.POPPED)) {
totalSize += msg.getSize();
messages.add(new Pop3Message(msg));
}
}
}
} finally {
Closeables.closeQuietly(results);
}
}
}
@Test
public void mdate() throws Exception {
Mailbox mbox =
MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
DbConnection conn = DbPool.getConnection();
DbUtil.executeUpdate(
conn,
"INSERT INTO mboxgroup1.mail_item "
+ "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) "
+ "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)",
mbox.getId(),
101,
Mailbox.ID_FOLDER_INBOX,
MailItem.Type.MESSAGE.toByte(),
100,
1000);
DbUtil.executeUpdate(
conn,
"INSERT INTO mboxgroup1.mail_item "
+ "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) "
+ "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)",
mbox.getId(),
102,
Mailbox.ID_FOLDER_INBOX,
MailItem.Type.MESSAGE.toByte(),
200,
2000);
DbUtil.executeUpdate(
conn,
"INSERT INTO mboxgroup1.mail_item "
+ "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) "
+ "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)",
mbox.getId(),
103,
Mailbox.ID_FOLDER_INBOX,
MailItem.Type.MESSAGE.toByte(),
300,
3000);
DbUtil.executeUpdate(
conn,
"INSERT INTO mboxgroup1.mail_item "
+ "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) "
+ "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)",
mbox.getId(),
104,
Mailbox.ID_FOLDER_INBOX,
MailItem.Type.MESSAGE.toByte(),
400,
4000);
DbUtil.executeUpdate(
conn,
"INSERT INTO mboxgroup1.mail_item "
+ "(mailbox_id, id, folder_id, type, flags, date, change_date, size, tags, mod_metadata, mod_content) "
+ "VALUES(?, ?, ?, ?, 0, ?, ?, 0, 0, 0, 0)",
mbox.getId(),
105,
Mailbox.ID_FOLDER_INBOX,
MailItem.Type.MESSAGE.toByte(),
500,
5000);
conn.commit();
conn.closeQuietly();
SearchParams params = new SearchParams();
params.setQueryString("mdate:>3000000");
params.setSortBy(SortBy.DATE_ASC);
params.setTypes(EnumSet.of(MailItem.Type.MESSAGE));
params.setFetchMode(SearchParams.Fetch.IDS);
ZimbraQuery query =
new ZimbraQuery(new OperationContext(mbox), SoapProtocol.Soap12, mbox, params);
Assert.assertEquals("ZQ: Q(DATE:MDATE,197001010050-196912312359)", query.toString());
ZimbraQueryResults result = query.execute();
Assert.assertEquals(104, result.getNext().getItemId());
Assert.assertEquals(105, result.getNext().getItemId());
Assert.assertEquals(null, result.getNext());
Closeables.closeQuietly(result);
}
private void deliverMessageToLocalMailboxes(
Blob blob, BlobInputStream bis, byte[] data, MimeMessage mm, LmtpEnvelope env)
throws ServiceException, IOException {
List<LmtpAddress> recipients = env.getLocalRecipients();
String envSender = env.getSender().getEmailAddress();
boolean shared = recipients.size() > 1;
List<Integer> targetMailboxIds = new ArrayList<Integer>(recipients.size());
Map<LmtpAddress, RecipientDetail> rcptMap =
new HashMap<LmtpAddress, RecipientDetail>(recipients.size());
try {
// Examine attachments indexing option for all recipients and
// prepare ParsedMessage versions needed. Parsing is done before
// attempting delivery to any recipient. Therefore, parse error
// will result in non-delivery to all recipients.
// ParsedMessage for users with attachments indexing
ParsedMessage pmAttachIndex = null;
// ParsedMessage for users without attachments indexing
ParsedMessage pmNoAttachIndex = null;
// message id for logging
String msgId = null;
for (LmtpAddress recipient : recipients) {
String rcptEmail = recipient.getEmailAddress();
Account account;
Mailbox mbox;
boolean attachmentsIndexingEnabled;
try {
account = Provisioning.getInstance().get(AccountBy.name, rcptEmail);
if (account == null) {
ZimbraLog.mailbox.warn("No account found delivering mail to " + rcptEmail);
continue;
}
mbox = MailboxManager.getInstance().getMailboxByAccount(account);
if (mbox == null) {
ZimbraLog.mailbox.warn("No mailbox found delivering mail to " + rcptEmail);
continue;
}
attachmentsIndexingEnabled = mbox.attachmentsIndexingEnabled();
} catch (ServiceException se) {
if (se.isReceiversFault()) {
ZimbraLog.mailbox.info("Recoverable exception getting mailbox for " + rcptEmail, se);
rcptMap.put(
recipient, new RecipientDetail(null, null, null, false, DeliveryAction.defer));
} else {
ZimbraLog.mailbox.warn("Unrecoverable exception getting mailbox for " + rcptEmail, se);
}
continue;
}
if (account != null && mbox != null) {
ParsedMessageOptions pmo;
if (mm != null) {
pmo =
new ParsedMessageOptions()
.setContent(mm)
.setDigest(blob.getDigest())
.setSize(blob.getRawSize());
} else {
pmo = new ParsedMessageOptions(blob, data);
}
ParsedMessage pm;
if (attachmentsIndexingEnabled) {
if (pmAttachIndex == null) {
pmo.setAttachmentIndexing(true);
ZimbraLog.lmtp.debug(
"Creating ParsedMessage from %s with attachment indexing enabled",
data == null ? "file" : "memory");
pmAttachIndex = new ParsedMessage(pmo);
}
pm = pmAttachIndex;
} else {
if (pmNoAttachIndex == null) {
pmo.setAttachmentIndexing(false);
ZimbraLog.lmtp.debug(
"Creating ParsedMessage from %s with attachment indexing disabled",
data == null ? "file" : "memory");
pmNoAttachIndex = new ParsedMessage(pmo);
}
pm = pmNoAttachIndex;
}
msgId = pm.getMessageID();
if (account.isPrefMailLocalDeliveryDisabled()) {
ZimbraLog.lmtp.debug("Local delivery disabled for account %s", rcptEmail);
rcptMap.put(
recipient, new RecipientDetail(account, mbox, pm, false, DeliveryAction.discard));
continue;
}
// For non-shared delivery (i.e. only one recipient),
// always deliver regardless of backup mode.
DeliveryAction da = DeliveryAction.deliver;
boolean endSharedDelivery = false;
if (shared) {
if (mbox.beginSharedDelivery()) {
endSharedDelivery = true;
} else {
// Skip delivery to mailboxes in backup mode.
da = DeliveryAction.defer;
}
}
rcptMap.put(recipient, new RecipientDetail(account, mbox, pm, endSharedDelivery, da));
if (da == DeliveryAction.deliver) {
targetMailboxIds.add(mbox.getId());
}
}
}
ZimbraLog.removeAccountFromContext();
if (ZimbraLog.lmtp.isInfoEnabled()) {
ZimbraLog.lmtp.info(
"Delivering message: size=%s, nrcpts=%d, sender=%s, msgid=%s",
env.getSize() == 0 ? "unspecified" : Integer.toString(env.getSize()) + " bytes",
recipients.size(),
env.getSender(),
msgId == null ? "" : msgId);
}
DeliveryContext sharedDeliveryCtxt = new DeliveryContext(shared, targetMailboxIds);
sharedDeliveryCtxt.setIncomingBlob(blob);
// We now know which addresses are valid and which ParsedMessage
// version each recipient needs. Deliver!
for (LmtpAddress recipient : recipients) {
String rcptEmail = recipient.getEmailAddress();
LmtpReply reply = LmtpReply.TEMPORARY_FAILURE;
RecipientDetail rd = rcptMap.get(recipient);
if (rd.account != null) ZimbraLog.addAccountNameToContext(rd.account.getName());
if (rd.mbox != null) ZimbraLog.addMboxToContext(rd.mbox.getId());
boolean success = false;
try {
if (rd != null) {
switch (rd.action) {
case discard:
ZimbraLog.lmtp.info(
"accepted and discarded message from=%s,to=%s: local delivery is disabled",
envSender, rcptEmail);
if (rd.account.getPrefMailForwardingAddress() != null) {
// mail forwarding is set up
for (LmtpCallback callback : callbacks) {
ZimbraLog.lmtp.debug("Executing callback %s", callback.getClass().getName());
callback.forwardWithoutDelivery(
rd.account, rd.mbox, envSender, rcptEmail, rd.pm);
}
}
reply = LmtpReply.DELIVERY_OK;
break;
case deliver:
Account account = rd.account;
Mailbox mbox = rd.mbox;
ParsedMessage pm = rd.pm;
List<ItemId> addedMessageIds = null;
ReentrantLock lock = mailboxDeliveryLocks.get(mbox.getId());
boolean acquiredLock;
try {
// Wait for the lock, up to the timeout
acquiredLock =
lock.tryLock(LC.zimbra_mailbox_lock_timeout.intValue(), TimeUnit.SECONDS);
} catch (InterruptedException e) {
acquiredLock = false;
}
if (!acquiredLock) {
ZimbraLog.lmtp.info(
"try again for message from=%s,to=%s: another mail delivery in progress.",
envSender, rcptEmail);
reply = LmtpReply.TEMPORARY_FAILURE;
break;
}
try {
if (dedupe(pm, mbox)) {
// message was already delivered to this mailbox
ZimbraLog.lmtp.info(
"Not delivering message with duplicate Message-ID %s", pm.getMessageID());
} else if (recipient.getSkipFilters()) {
msgId = pm.getMessageID();
int folderId = Mailbox.ID_FOLDER_INBOX;
if (recipient.getFolder() != null) {
try {
Folder folder = mbox.getFolderByPath(null, recipient.getFolder());
folderId = folder.getId();
} catch (ServiceException se) {
if (se.getCode().equals(MailServiceException.NO_SUCH_FOLDER)) {
Folder folder =
mbox.createFolder(
null,
recipient.getFolder(),
new Folder.FolderOptions().setDefaultView(MailItem.Type.MESSAGE));
folderId = folder.getId();
} else {
throw se;
}
}
}
int flags = Flag.BITMASK_UNREAD;
if (recipient.getFlags() != null) {
flags = Flag.toBitmask(recipient.getFlags());
}
DeliveryOptions dopt = new DeliveryOptions().setFolderId(folderId);
dopt.setFlags(flags).setTags(recipient.getTags()).setRecipientEmail(rcptEmail);
Message msg = mbox.addMessage(null, pm, dopt, sharedDeliveryCtxt);
addedMessageIds = Lists.newArrayList(new ItemId(msg));
} else if (!DebugConfig.disableIncomingFilter) {
// Get msgid first, to avoid having to reopen and reparse the blob
// file if Mailbox.addMessageInternal() closes it.
pm.getMessageID();
addedMessageIds =
RuleManager.applyRulesToIncomingMessage(
null,
mbox,
pm,
(int) blob.getRawSize(),
rcptEmail,
sharedDeliveryCtxt,
Mailbox.ID_FOLDER_INBOX,
false);
} else {
pm.getMessageID();
DeliveryOptions dopt =
new DeliveryOptions().setFolderId(Mailbox.ID_FOLDER_INBOX);
dopt.setFlags(Flag.BITMASK_UNREAD).setRecipientEmail(rcptEmail);
Message msg = mbox.addMessage(null, pm, dopt, sharedDeliveryCtxt);
addedMessageIds = Lists.newArrayList(new ItemId(msg));
}
success = true;
if (addedMessageIds != null && addedMessageIds.size() > 0) {
addToDedupeCache(pm, mbox);
}
} finally {
lock.unlock();
}
if (addedMessageIds != null && addedMessageIds.size() > 0) {
// Execute callbacks
for (LmtpCallback callback : callbacks) {
for (ItemId id : addedMessageIds) {
if (id.belongsTo(mbox)) {
// Message was added to the local mailbox, as opposed to a mountpoint.
ZimbraLog.lmtp.debug(
"Executing callback %s", callback.getClass().getName());
try {
Message msg = mbox.getMessageById(null, id.getId());
callback.afterDelivery(account, mbox, envSender, rcptEmail, msg);
} catch (Throwable t) {
if (t instanceof OutOfMemoryError) {
Zimbra.halt("LMTP callback failed", t);
} else {
ZimbraLog.lmtp.warn("LMTP callback threw an exception", t);
}
}
}
}
}
}
reply = LmtpReply.DELIVERY_OK;
break;
case defer:
// Delivery to mailbox skipped. Let MTA retry again later.
// This case happens for shared delivery to a mailbox in
// backup mode.
ZimbraLog.lmtp.info(
"try again for message from=%s,to=%s: mailbox skipped", envSender, rcptEmail);
reply = LmtpReply.TEMPORARY_FAILURE;
break;
}
} else {
// Account or mailbox not found.
ZimbraLog.lmtp.info(
"rejecting message from=%s,to=%s: account or mailbox not found",
envSender, rcptEmail);
reply = LmtpReply.PERMANENT_FAILURE;
}
} catch (ServiceException e) {
if (e.getCode().equals(MailServiceException.QUOTA_EXCEEDED)) {
ZimbraLog.lmtp.info("rejecting message from=%s,to=%s: overquota", envSender, rcptEmail);
if (config.isPermanentFailureWhenOverQuota()) {
reply = LmtpReply.PERMANENT_FAILURE_OVER_QUOTA;
} else {
reply = LmtpReply.TEMPORARY_FAILURE_OVER_QUOTA;
}
} else if (e.isReceiversFault()) {
ZimbraLog.lmtp.info("try again for message from=%s,to=%s", envSender, rcptEmail, e);
reply = LmtpReply.TEMPORARY_FAILURE;
} else {
ZimbraLog.lmtp.info("rejecting message from=%s,to=%s", envSender, rcptEmail, e);
reply = LmtpReply.PERMANENT_FAILURE;
}
} catch (Exception e) {
reply = LmtpReply.TEMPORARY_FAILURE;
ZimbraLog.lmtp.warn("try again for message from=%s,to=%s", envSender, rcptEmail, e);
} finally {
if (rd.action == DeliveryAction.deliver && !success) {
// Message was not delivered. Remove it from the dedupe
// cache so we don't dedupe it on LMTP retry.
removeFromDedupeCache(msgId, rd.mbox);
}
recipient.setDeliveryStatus(reply);
if (shared && rd != null && rd.esd) {
rd.mbox.endSharedDelivery();
rd.esd = false;
}
}
}
// If this message is being streamed from disk, cache it
ParsedMessage mimeSource = pmAttachIndex != null ? pmAttachIndex : pmNoAttachIndex;
MailboxBlob mblob = sharedDeliveryCtxt.getMailboxBlob();
if (mblob != null && mimeSource != null) {
if (bis == null) {
bis = mimeSource.getBlobInputStream();
}
if (bis != null) {
try {
// Update the MimeMessage with the blob that's stored inside the mailbox,
// since the incoming blob will be deleted.
Blob storedBlob = mblob.getLocalBlob();
bis.fileMoved(storedBlob.getFile());
MessageCache.cacheMessage(
mblob.getDigest(), mimeSource.getOriginalMessage(), mimeSource.getMimeMessage());
} catch (IOException e) {
ZimbraLog.lmtp.warn("Unable to cache message for " + mblob, e);
}
}
}
} finally {
// If there were any stray exceptions after the call to
// beginSharedDelivery that caused endSharedDelivery to be not
// called, we check and fix those here.
if (shared) {
for (RecipientDetail rd : rcptMap.values()) {
if (rd.esd && rd.mbox != null) rd.mbox.endSharedDelivery();
}
}
}
}
