@Override
protected boolean executeBasic(MailAdapter mail, Arguments arguments, SieveContext context) {
if (!(mail instanceof ZimbraMailAdapter)) return false;
ParsedMessage pm = ((ZimbraMailAdapter) mail).getParsedMessage();
if (pm == null) {
return false;
} else {
return pm.hasAttachments();
}
}
private void verifyParsedMessage(ParsedMessage pm, ExpectedResults expected) throws Exception {
// Run tests multiple times to make sure the API's don't alter the state of the ParsedMessage
for (int i = 1; i < 3; i++) {
// Test accessors.
assertEquals(expected.rawContent, new String(pm.getRawData()));
assertEquals(expected.convertedSubject, pm.getSubject());
// Test sender and recipient
String sender = TestUtil.getAddress(SENDER_NAME);
String recipient = TestUtil.getAddress(RECIPIENT_NAME);
assertTrue(pm.getSender().contains(sender));
assertEquals(sender, pm.getSenderEmail());
assertTrue(pm.getRecipients().contains(recipient));
// Test InputStream accessor
String contentFromStream =
new String(ByteUtil.getContent(pm.getRawInputStream(), expected.rawContent.length()));
assertEquals(expected.rawContent, contentFromStream);
// Test MimeMessage accessor
assertTrue(getContent(pm.getMimeMessage()).contains(expected.convertedSubject));
// Test mutated status
assertEquals(expected.wasMutated, pm.wasMutated());
pm.analyzeFully();
}
}
private void runContentTests(String originalMsg, ParsedMessage pm) throws Exception {
int size = originalMsg.length();
// Test InputStream accessor
String msg = new String(ByteUtil.getContent(pm.getRawInputStream(), size));
assertEquals("expected: " + originalMsg + "\ngot: " + msg, originalMsg, msg);
// Test byte[] accessor
msg = new String(pm.getRawData());
assertEquals("expected: " + originalMsg + "\ngot: " + msg, originalMsg, msg);
}
private void verifyMutatedMessage(ParsedMessage pm, String substring, boolean wasMutated)
throws Exception {
assertEquals(wasMutated, pm.wasMutated());
assertTrue(pm.getSubject().contains(substring));
assertTrue((new String(pm.getRawData()).contains(substring)));
byte[] data = ByteUtil.getContent(pm.getRawInputStream(), 0);
assertTrue((new String(data)).contains(substring));
data = pm.getRawData();
assertTrue((new String(data)).contains(substring));
}
/**
* Returns the value of the {@code Message-ID} header, or the most recent {@code
* Resent-Message-ID} header, if set.
*/
private String getMessageID(ParsedMessage pm) {
try {
String id = pm.getMimeMessage().getHeader("Resent-Message-ID", null);
if (!Strings.isNullOrEmpty(id)) {
ZimbraLog.lmtp.debug("Resent-Message-ID=%s", id);
return id;
}
} catch (MessagingException e) {
ZimbraLog.lmtp.warn("Unable to determine Resent-Message-ID header value", e);
}
String id = pm.getMessageID();
ZimbraLog.lmtp.debug("Resent-Message-ID not found. Message-ID=%s", id);
return id;
}
public void testMimeConverter() throws Exception {
MimeVisitor.registerConverter(TestMimeVisitor.class);
ExpectedResults expected = new ExpectedResults();
String subject = NAME_PREFIX + " testMimeConverter oldsubject";
expected.convertedSubject = NAME_PREFIX + " testMimeConverter newsubject";
expected.rawContent = TestUtil.getTestMessage(subject, RECIPIENT_NAME, SENDER_NAME, null);
expected.wasMutated = false;
// Test ParsedMessage created from byte[]
ParsedMessage pm = new ParsedMessage(expected.rawContent.getBytes(), false);
verifyParsedMessage(pm, expected);
pm = new ParsedMessage(expected.rawContent.getBytes(), true);
verifyParsedMessage(pm, expected);
// Test ParsedMessage created from File
mFile = File.createTempFile("TestParsedMessage", ".msg");
FileOutputStream out = new FileOutputStream(mFile);
out.write(expected.rawContent.getBytes());
out.close();
pm = new ParsedMessage(mFile, null, false);
verifyParsedMessage(pm, expected);
pm = new ParsedMessage(mFile, null, true);
verifyParsedMessage(pm, expected);
// Test ParsedMessage created from MimeMessage. Can't verify entire content
// because JavaMail mangles the headers.
MimeMessage mimeMsg =
new ZMimeMessage(
JMSession.getSession(), new SharedByteArrayInputStream(expected.rawContent.getBytes()));
pm = new ParsedMessage(mimeMsg, false);
assertTrue((new String(pm.getRawData()).contains("oldsubject")));
assertTrue(getContent(pm.getMimeMessage()).contains("newsubject"));
assertTrue(pm.getSubject().contains("newsubject"));
pm = new ParsedMessage(mimeMsg, true);
assertTrue((new String(pm.getRawData()).contains("oldsubject")));
assertTrue(getContent(pm.getMimeMessage()).contains("newsubject"));
assertTrue(pm.getSubject().contains("newsubject"));
}
static SetCalendarItemData getSetCalendarItemData(
ZimbraSoapContext zsc,
OperationContext octxt,
Account acct,
Mailbox mbox,
Element e,
ParseMimeMessage.InviteParser parser)
throws ServiceException {
String partStatStr =
e.getAttribute(MailConstants.A_CAL_PARTSTAT, IcalXmlStrMap.PARTSTAT_NEEDS_ACTION);
// <M>
Element msgElem = e.getElement(MailConstants.E_MSG);
// check to see whether the entire message has been uploaded under separate cover
String attachmentId = msgElem.getAttribute(MailConstants.A_ATTACHMENT_ID, null);
Element contentElement = msgElem.getOptionalElement(MailConstants.E_CONTENT);
InviteParserResult ipr = null;
MimeMessage mm = null;
if (attachmentId != null) {
ParseMimeMessage.MimeMessageData mimeData = new ParseMimeMessage.MimeMessageData();
mm = SendMsg.parseUploadedMessage(zsc, attachmentId, mimeData);
} else if (contentElement != null) {
mm = ParseMimeMessage.importMsgSoap(msgElem);
} else {
CalSendData dat = handleMsgElement(zsc, octxt, msgElem, acct, mbox, parser);
mm = dat.mMm;
ipr = parser.getResult();
}
if (ipr == null && msgElem.getOptionalElement(MailConstants.E_INVITE) != null) {
ipr = parser.parse(zsc, octxt, mbox.getAccount(), msgElem.getElement(MailConstants.E_INVITE));
// Get description texts out of the MimeMessage and set in the parsed invite. Do it only if
// the MimeMessage has text parts. This prevents discarding texts when they're specified only
// in the <inv> but not in mime parts.
if (ipr != null && ipr.mInvite != null && mm != null) {
String desc = Invite.getDescription(mm, MimeConstants.CT_TEXT_PLAIN);
String descHtml = Invite.getDescription(mm, MimeConstants.CT_TEXT_HTML);
if ((desc != null && desc.length() > 0) || (descHtml != null && descHtml.length() > 0))
ipr.mInvite.setDescription(desc, descHtml);
}
}
ParsedMessage pm = new ParsedMessage(mm, mbox.attachmentsIndexingEnabled());
Invite inv = (ipr == null ? null : ipr.mInvite);
if (inv == null || inv.getDTStamp() == -1) { // zdsync if -1 for 4.5 back compat
CalendarPartInfo cpi = pm.getCalendarPartInfo();
ZVCalendar cal = null;
if (cpi != null && CalendarItem.isAcceptableInvite(mbox.getAccount(), cpi)) cal = cpi.cal;
if (cal == null)
throw ServiceException.FAILURE("SetCalendarItem could not build an iCalendar object", null);
boolean sentByMe = false; // not applicable in the SetCalendarItem case
Invite iCalInv = Invite.createFromCalendar(acct, pm.getFragment(), cal, sentByMe).get(0);
if (inv == null) {
inv = iCalInv;
} else {
inv.setDtStamp(iCalInv.getDTStamp()); // zdsync
inv.setFragment(iCalInv.getFragment()); // zdsync
}
}
inv.setPartStat(partStatStr);
SetCalendarItemData sadata = new SetCalendarItemData();
sadata.invite = inv;
sadata.message = pm;
return sadata;
}
private void deliverMessageToLocalMailboxes(
Blob blob, BlobInputStream bis, byte[] data, MimeMessage mm, LmtpEnvelope env)
throws ServiceException, IOException {
List<LmtpAddress> recipients = env.getLocalRecipients();
String envSender = env.getSender().getEmailAddress();
boolean shared = recipients.size() > 1;
List<Integer> targetMailboxIds = new ArrayList<Integer>(recipients.size());
Map<LmtpAddress, RecipientDetail> rcptMap =
new HashMap<LmtpAddress, RecipientDetail>(recipients.size());
try {
// Examine attachments indexing option for all recipients and
// prepare ParsedMessage versions needed. Parsing is done before
// attempting delivery to any recipient. Therefore, parse error
// will result in non-delivery to all recipients.
// ParsedMessage for users with attachments indexing
ParsedMessage pmAttachIndex = null;
// ParsedMessage for users without attachments indexing
ParsedMessage pmNoAttachIndex = null;
// message id for logging
String msgId = null;
for (LmtpAddress recipient : recipients) {
String rcptEmail = recipient.getEmailAddress();
Account account;
Mailbox mbox;
boolean attachmentsIndexingEnabled;
try {
account = Provisioning.getInstance().get(AccountBy.name, rcptEmail);
if (account == null) {
ZimbraLog.mailbox.warn("No account found delivering mail to " + rcptEmail);
continue;
}
mbox = MailboxManager.getInstance().getMailboxByAccount(account);
if (mbox == null) {
ZimbraLog.mailbox.warn("No mailbox found delivering mail to " + rcptEmail);
continue;
}
attachmentsIndexingEnabled = mbox.attachmentsIndexingEnabled();
} catch (ServiceException se) {
if (se.isReceiversFault()) {
ZimbraLog.mailbox.info("Recoverable exception getting mailbox for " + rcptEmail, se);
rcptMap.put(
recipient, new RecipientDetail(null, null, null, false, DeliveryAction.defer));
} else {
ZimbraLog.mailbox.warn("Unrecoverable exception getting mailbox for " + rcptEmail, se);
}
continue;
}
if (account != null && mbox != null) {
ParsedMessageOptions pmo;
if (mm != null) {
pmo =
new ParsedMessageOptions()
.setContent(mm)
.setDigest(blob.getDigest())
.setSize(blob.getRawSize());
} else {
pmo = new ParsedMessageOptions(blob, data);
}
ParsedMessage pm;
if (attachmentsIndexingEnabled) {
if (pmAttachIndex == null) {
pmo.setAttachmentIndexing(true);
ZimbraLog.lmtp.debug(
"Creating ParsedMessage from %s with attachment indexing enabled",
data == null ? "file" : "memory");
pmAttachIndex = new ParsedMessage(pmo);
}
pm = pmAttachIndex;
} else {
if (pmNoAttachIndex == null) {
pmo.setAttachmentIndexing(false);
ZimbraLog.lmtp.debug(
"Creating ParsedMessage from %s with attachment indexing disabled",
data == null ? "file" : "memory");
pmNoAttachIndex = new ParsedMessage(pmo);
}
pm = pmNoAttachIndex;
}
msgId = pm.getMessageID();
if (account.isPrefMailLocalDeliveryDisabled()) {
ZimbraLog.lmtp.debug("Local delivery disabled for account %s", rcptEmail);
rcptMap.put(
recipient, new RecipientDetail(account, mbox, pm, false, DeliveryAction.discard));
continue;
}
// For non-shared delivery (i.e. only one recipient),
// always deliver regardless of backup mode.
DeliveryAction da = DeliveryAction.deliver;
boolean endSharedDelivery = false;
if (shared) {
if (mbox.beginSharedDelivery()) {
endSharedDelivery = true;
} else {
// Skip delivery to mailboxes in backup mode.
da = DeliveryAction.defer;
}
}
rcptMap.put(recipient, new RecipientDetail(account, mbox, pm, endSharedDelivery, da));
if (da == DeliveryAction.deliver) {
targetMailboxIds.add(mbox.getId());
}
}
}
ZimbraLog.removeAccountFromContext();
if (ZimbraLog.lmtp.isInfoEnabled()) {
ZimbraLog.lmtp.info(
"Delivering message: size=%s, nrcpts=%d, sender=%s, msgid=%s",
env.getSize() == 0 ? "unspecified" : Integer.toString(env.getSize()) + " bytes",
recipients.size(),
env.getSender(),
msgId == null ? "" : msgId);
}
DeliveryContext sharedDeliveryCtxt = new DeliveryContext(shared, targetMailboxIds);
sharedDeliveryCtxt.setIncomingBlob(blob);
// We now know which addresses are valid and which ParsedMessage
// version each recipient needs. Deliver!
for (LmtpAddress recipient : recipients) {
String rcptEmail = recipient.getEmailAddress();
LmtpReply reply = LmtpReply.TEMPORARY_FAILURE;
RecipientDetail rd = rcptMap.get(recipient);
if (rd.account != null) ZimbraLog.addAccountNameToContext(rd.account.getName());
if (rd.mbox != null) ZimbraLog.addMboxToContext(rd.mbox.getId());
boolean success = false;
try {
if (rd != null) {
switch (rd.action) {
case discard:
ZimbraLog.lmtp.info(
"accepted and discarded message from=%s,to=%s: local delivery is disabled",
envSender, rcptEmail);
if (rd.account.getPrefMailForwardingAddress() != null) {
// mail forwarding is set up
for (LmtpCallback callback : callbacks) {
ZimbraLog.lmtp.debug("Executing callback %s", callback.getClass().getName());
callback.forwardWithoutDelivery(
rd.account, rd.mbox, envSender, rcptEmail, rd.pm);
}
}
reply = LmtpReply.DELIVERY_OK;
break;
case deliver:
Account account = rd.account;
Mailbox mbox = rd.mbox;
ParsedMessage pm = rd.pm;
List<ItemId> addedMessageIds = null;
ReentrantLock lock = mailboxDeliveryLocks.get(mbox.getId());
boolean acquiredLock;
try {
// Wait for the lock, up to the timeout
acquiredLock =
lock.tryLock(LC.zimbra_mailbox_lock_timeout.intValue(), TimeUnit.SECONDS);
} catch (InterruptedException e) {
acquiredLock = false;
}
if (!acquiredLock) {
ZimbraLog.lmtp.info(
"try again for message from=%s,to=%s: another mail delivery in progress.",
envSender, rcptEmail);
reply = LmtpReply.TEMPORARY_FAILURE;
break;
}
try {
if (dedupe(pm, mbox)) {
// message was already delivered to this mailbox
ZimbraLog.lmtp.info(
"Not delivering message with duplicate Message-ID %s", pm.getMessageID());
} else if (recipient.getSkipFilters()) {
msgId = pm.getMessageID();
int folderId = Mailbox.ID_FOLDER_INBOX;
if (recipient.getFolder() != null) {
try {
Folder folder = mbox.getFolderByPath(null, recipient.getFolder());
folderId = folder.getId();
} catch (ServiceException se) {
if (se.getCode().equals(MailServiceException.NO_SUCH_FOLDER)) {
Folder folder =
mbox.createFolder(
null,
recipient.getFolder(),
new Folder.FolderOptions().setDefaultView(MailItem.Type.MESSAGE));
folderId = folder.getId();
} else {
throw se;
}
}
}
int flags = Flag.BITMASK_UNREAD;
if (recipient.getFlags() != null) {
flags = Flag.toBitmask(recipient.getFlags());
}
DeliveryOptions dopt = new DeliveryOptions().setFolderId(folderId);
dopt.setFlags(flags).setTags(recipient.getTags()).setRecipientEmail(rcptEmail);
Message msg = mbox.addMessage(null, pm, dopt, sharedDeliveryCtxt);
addedMessageIds = Lists.newArrayList(new ItemId(msg));
} else if (!DebugConfig.disableIncomingFilter) {
// Get msgid first, to avoid having to reopen and reparse the blob
// file if Mailbox.addMessageInternal() closes it.
pm.getMessageID();
addedMessageIds =
RuleManager.applyRulesToIncomingMessage(
null,
mbox,
pm,
(int) blob.getRawSize(),
rcptEmail,
sharedDeliveryCtxt,
Mailbox.ID_FOLDER_INBOX,
false);
} else {
pm.getMessageID();
DeliveryOptions dopt =
new DeliveryOptions().setFolderId(Mailbox.ID_FOLDER_INBOX);
dopt.setFlags(Flag.BITMASK_UNREAD).setRecipientEmail(rcptEmail);
Message msg = mbox.addMessage(null, pm, dopt, sharedDeliveryCtxt);
addedMessageIds = Lists.newArrayList(new ItemId(msg));
}
success = true;
if (addedMessageIds != null && addedMessageIds.size() > 0) {
addToDedupeCache(pm, mbox);
}
} finally {
lock.unlock();
}
if (addedMessageIds != null && addedMessageIds.size() > 0) {
// Execute callbacks
for (LmtpCallback callback : callbacks) {
for (ItemId id : addedMessageIds) {
if (id.belongsTo(mbox)) {
// Message was added to the local mailbox, as opposed to a mountpoint.
ZimbraLog.lmtp.debug(
"Executing callback %s", callback.getClass().getName());
try {
Message msg = mbox.getMessageById(null, id.getId());
callback.afterDelivery(account, mbox, envSender, rcptEmail, msg);
} catch (Throwable t) {
if (t instanceof OutOfMemoryError) {
Zimbra.halt("LMTP callback failed", t);
} else {
ZimbraLog.lmtp.warn("LMTP callback threw an exception", t);
}
}
}
}
}
}
reply = LmtpReply.DELIVERY_OK;
break;
case defer:
// Delivery to mailbox skipped. Let MTA retry again later.
// This case happens for shared delivery to a mailbox in
// backup mode.
ZimbraLog.lmtp.info(
"try again for message from=%s,to=%s: mailbox skipped", envSender, rcptEmail);
reply = LmtpReply.TEMPORARY_FAILURE;
break;
}
} else {
// Account or mailbox not found.
ZimbraLog.lmtp.info(
"rejecting message from=%s,to=%s: account or mailbox not found",
envSender, rcptEmail);
reply = LmtpReply.PERMANENT_FAILURE;
}
} catch (ServiceException e) {
if (e.getCode().equals(MailServiceException.QUOTA_EXCEEDED)) {
ZimbraLog.lmtp.info("rejecting message from=%s,to=%s: overquota", envSender, rcptEmail);
if (config.isPermanentFailureWhenOverQuota()) {
reply = LmtpReply.PERMANENT_FAILURE_OVER_QUOTA;
} else {
reply = LmtpReply.TEMPORARY_FAILURE_OVER_QUOTA;
}
} else if (e.isReceiversFault()) {
ZimbraLog.lmtp.info("try again for message from=%s,to=%s", envSender, rcptEmail, e);
reply = LmtpReply.TEMPORARY_FAILURE;
} else {
ZimbraLog.lmtp.info("rejecting message from=%s,to=%s", envSender, rcptEmail, e);
reply = LmtpReply.PERMANENT_FAILURE;
}
} catch (Exception e) {
reply = LmtpReply.TEMPORARY_FAILURE;
ZimbraLog.lmtp.warn("try again for message from=%s,to=%s", envSender, rcptEmail, e);
} finally {
if (rd.action == DeliveryAction.deliver && !success) {
// Message was not delivered. Remove it from the dedupe
// cache so we don't dedupe it on LMTP retry.
removeFromDedupeCache(msgId, rd.mbox);
}
recipient.setDeliveryStatus(reply);
if (shared && rd != null && rd.esd) {
rd.mbox.endSharedDelivery();
rd.esd = false;
}
}
}
// If this message is being streamed from disk, cache it
ParsedMessage mimeSource = pmAttachIndex != null ? pmAttachIndex : pmNoAttachIndex;
MailboxBlob mblob = sharedDeliveryCtxt.getMailboxBlob();
if (mblob != null && mimeSource != null) {
if (bis == null) {
bis = mimeSource.getBlobInputStream();
}
if (bis != null) {
try {
// Update the MimeMessage with the blob that's stored inside the mailbox,
// since the incoming blob will be deleted.
Blob storedBlob = mblob.getLocalBlob();
bis.fileMoved(storedBlob.getFile());
MessageCache.cacheMessage(
mblob.getDigest(), mimeSource.getOriginalMessage(), mimeSource.getMimeMessage());
} catch (IOException e) {
ZimbraLog.lmtp.warn("Unable to cache message for " + mblob, e);
}
}
}
} finally {
// If there were any stray exceptions after the call to
// beginSharedDelivery that caused endSharedDelivery to be not
// called, we check and fix those here.
if (shared) {
for (RecipientDetail rd : rcptMap.values()) {
if (rd.esd && rd.mbox != null) rd.mbox.endSharedDelivery();
}
}
}
}
private void fetchAndAddMessage(int msgno, int size, String uid, boolean allowFilterToMountpoint)
throws ServiceException, IOException {
ContentInputStream cis = null;
MessageContent mc = null;
checkIsEnabled();
try {
cis = connection.getMessage(msgno);
mc = MessageContent.read(cis, size);
ParsedMessage pm = mc.getParsedMessage(null, indexAttachments);
if (pm == null) {
LOG.warn("Empty message body for UID %d. Must be ignored.", uid);
return;
}
Message msg = null;
// bug 47796: Set received date to sent date if available otherwise use current time
try {
Date sentDate = pm.getMimeMessage().getSentDate();
if (sentDate == null) {
LOG.warn(
"null sent date; probably due to parse error. Date header value: [%s]",
pm.getMimeMessage().getHeader("Date", null));
}
pm.setReceivedDate(sentDate != null ? sentDate.getTime() : System.currentTimeMillis());
} catch (MessagingException e) {
LOG.warn(
"unable to get sent date from parsed message due to exception, must use current time",
e);
pm.setReceivedDate(System.currentTimeMillis());
}
DeliveryContext dc = mc.getDeliveryContext();
if (isOffline()) {
msg = addMessage(null, pm, size, dataSource.getFolderId(), Flag.BITMASK_UNREAD, dc);
} else {
Integer localId =
getFirstLocalId(
RuleManager.applyRulesToIncomingMessage(
null,
mbox,
pm,
size,
dataSource.getEmailAddress(),
dc,
dataSource.getFolderId(),
true,
allowFilterToMountpoint));
if (localId != null) {
msg = mbox.getMessageById(null, localId);
}
}
if (msg != null && uid != null) {
PopMessage msgTracker = new PopMessage(dataSource, msg.getId(), uid);
msgTracker.add();
}
} catch (CommandFailedException e) {
LOG.warn("Error fetching message number %d: %s", msgno, e.getMessage());
} finally {
if (cis != null) {
try {
cis.close();
} catch (ParseException pe) {
LOG.error(
"ParseException while closing ContentInputStream. Assuming cis is effectively closed",
pe);
}
}
if (mc != null) {
mc.cleanup();
}
}
}
/**
* Returns the normalized subject of the conversation. This is done by taking the
* <tt>Subject:</tt> header of the first message and removing prefixes (e.g. <tt>"Re:"</tt>) and
* suffixes (e.g. <tt>"(fwd)"</tt>) and the like.
*
* @see ParsedMessage#normalizeSubject
*/
public String getNormalizedSubject() {
return ParsedMessage.normalize(getSubject());
}
