/**
* Method createMetaContents.
*
* @param message
* @return
*/
private String createMetaContents(IMessageContext message) {
Envelope envelope = message.getEnvelope();
Partition partition = message.getPartition();
StringBuffer buf = new StringBuffer(200);
buf.append("Subject: ").append(envelope.getSubject()).append(MetaMessageWriterStage.NEW_LINE);
Date sentDate = envelope.getSentDate();
if (sentDate != null) {
buf.append("Date: ")
.append(MetaMessageWriterStage.s_rfc822DateFormat.format(sentDate))
.append(MetaMessageWriterStage.NEW_LINE);
}
// If there is a header based sender that was found by the vault box, then use it.
// The header based sender is usually in a nicer format "Bob Smith" <*****@*****.**>
// instead of just the email address of the sender [email protected]
String from;
if (envelope.getHeaderBasedSender() != null && envelope.getHeaderBasedSender().length() > 3) {
from = envelope.getHeaderBasedSender();
} else {
from = envelope.getSender();
}
if (from != null) {
buf.append("From: ").append(from).append(MetaMessageWriterStage.NEW_LINE);
}
buf.append("Message-ID: ");
buf.append(envelope.getSize());
buf.append("__");
buf.append("&mid=");
buf.append(message.getInternalId());
buf.append("&svr=");
buf.append(partition.getServer());
buf.append("&p=");
buf.append(partition.getId());
buf.append(MetaMessageWriterStage.NEW_LINE);
return buf.toString();
}
/**
* Validates that the stage properly identifies when a message should be restored
*
* @throws Exception on error
*/
@Test
public void testRestoreMissingMessage() throws Exception {
List<IMessageContext> msgs = new ArrayList<IMessageContext>();
IMessageContext msg = EasyMock.createStrictMock(IMessageContext.class);
msgs.add(msg);
// isUpdate & does not need restorage == don't store!
EasyMock.expect(msg.shouldReplaceStorage()).andReturn(false);
EasyMock.expect(msg.isUpdate()).andReturn(true);
EasyMock.replay(msg);
List<IMessageContext> needsStorage = Itertools.filter(PartitionStoreStage.NEEDS_STORAGE, msgs);
assertTrue(needsStorage.isEmpty());
EasyMock.verify(msg);
EasyMock.reset(msg);
// should replace storage will bypass the other tests
EasyMock.expect(msg.shouldReplaceStorage()).andReturn(true);
EasyMock.replay(msg);
needsStorage = Itertools.filter(PartitionStoreStage.NEEDS_STORAGE, msgs);
assertEquals(1, needsStorage.size());
EasyMock.verify(msg);
EasyMock.reset(msg);
// !isUpdate & no filepath & does not need restorage == store!
EasyMock.expect(msg.shouldReplaceStorage()).andReturn(false);
EasyMock.expect(msg.isUpdate()).andReturn(false);
EasyMock.expect(msg.getStoreFileSubPath()).andReturn(null);
EasyMock.replay(msg);
needsStorage = Itertools.filter(PartitionStoreStage.NEEDS_STORAGE, msgs);
assertEquals(1, needsStorage.size());
EasyMock.verify(msg);
EasyMock.reset(msg);
}
protected void failMessage(IMessageContext message, Exception ex) {
if (ex != null) {
MessageProcessError mpe = new MessageProcessError(ex, getName());
message.addError(mpe);
}
m_importer.audit(message, getName(), null, ex);
m_importer.importFailed(message, ex);
}
/**
* Method writeMetaFiles. This uses the MailStoreFileManger API to store files remotely.
*
* @param message
*/
private boolean writeMetaFiles(IMessageContext message) {
String metaContents = createMetaContents(message);
String metaFileName = new File(message.getStoreFileSubPath()).getName();
Collection<MailStoreFileManager.MetaMessageDescriptor> msgs =
new ArrayList<MailStoreFileManager.MetaMessageDescriptor>();
if (!CollectionsUtils.isNullOrEmpty(message.getResolvedRecipients())) {
for (ReplicatedRecipient rr : message.getResolvedRecipients()) {
msgs.add(
new MailStoreFileManager.MetaMessageDescriptor(
message.getInternalId(),
rr.getMaildir(),
metaFileName,
message.getEnvelope().getSentDate(),
message.getEnvelope().getReceivedDate(),
false,
rr.getState().toInt()));
}
}
if (message.getResolvedSender() != null) {
ReplicatedRecipient rr = message.getResolvedSender();
msgs.add(
new MailStoreFileManager.MetaMessageDescriptor(
message.getInternalId(),
rr.getMaildir(),
metaFileName,
message.getEnvelope().getSentDate(),
message.getEnvelope().getReceivedDate(),
true,
rr.getState().toInt()));
}
IMailStoreFileManager imsfm = ManagementContainer.getInstance().getMailStoreFileManager();
if (imsfm.storeMetaMessage(msgs, metaContents)) {
m_importer.audit(message, getName(), metaFileName);
return true;
} else {
m_importer.getStatComponent().addToValue(STAT_META_STORE_FAILURES, 1);
m_importer.consumerImportFailed(message, true);
return false;
}
}
@Override
protected boolean processMessage(IMessageContext message) throws CapabilitiesLoadException {
Capabilities caps = message.getCustomer().getCapabilities();
if (caps == null) throw new CapabilitiesLoadException();
Capabilities.ActiveMailboxSupport amSupportCap =
caps.getEnumCapability(
Capabilities.CAP_ACTIVEMAILBOX_SUPPORT, Capabilities.ActiveMailboxSupport.NONE);
boolean useNFS =
ManagementContainer.getInstance()
.getConfiguration()
.getBooleanProperty(CONFIG_USE_NFS, DEFAULT_USE_NFS);
if (amSupportCap == Capabilities.ActiveMailboxSupport.AM_CUSTOMER_ENCRYPTION
|| message.getEnvelope().isCustomerEncrypted()) {
if (!useNFS) {
return writeMetaFiles(message);
} else {
// Direct nfs usage; doesn't occur in production (mmfs is used instead)
boolean success = true;
String metaContents = createMetaContents(message);
String metaFileName = new File(message.getStoreFileSubPath()).getName();
if (!CollectionsUtils.isNullOrEmpty(message.getResolvedRecipients())) {
for (Iterator<ReplicatedRecipient> it = message.getResolvedRecipients().iterator();
it.hasNext(); ) {
ReplicatedRecipient rr = it.next();
if (!writeMetaFile(rr, message, false, metaFileName, metaContents)) {
success = false;
}
}
}
if (message.getResolvedSender() != null) {
if (!writeMetaFile(
message.getResolvedSender(), message, true, metaFileName, metaContents)) {
success = false;
}
}
if (!success) {
m_importer.getStatComponent().addToValue(STAT_META_STORE_FAILURES, 1);
m_importer.consumerImportFailed(message, true);
return false;
}
}
}
return true;
}
/**
* Method writeMetaFile. This uses NFS to store files remotely. Not used in production, mmfs is
* used instead.
*
* @param recpt
* @param message
* @param isSender
* @param metaFileName
* @param metaContents
*/
private boolean writeMetaFile(
ReplicatedRecipient recpt,
IMessageContext message,
boolean isSender,
String metaFileName,
String metaContents) {
Envelope env = message.getEnvelope();
ActiveMailboxFolder folder =
ActiveMailboxFolder.getFolder(recpt.getMaildir(), env.getSentDate(), isSender);
folder.initialize();
File metaMsgDir;
CustomerState userState = recpt.getState();
if (userState == CustomerState.ACTIVE || userState == CustomerState.TEST) {
metaMsgDir = folder.getMeta();
} else {
metaMsgDir = folder.getCur();
}
if (!metaMsgDir.exists()) {
m_importer.auditError(
message, getName(), metaMsgDir + " doesn't exist while trying to create meta-file");
}
File metaMsgFile = new File(metaMsgDir, metaFileName);
BufferedWriter out = null;
try {
try {
out = new BufferedWriter(new FileWriter(metaMsgFile));
out.write(metaContents);
} finally {
if (out != null) out.close();
}
// preserve arrival timestamp
metaMsgFile.setLastModified(env.getReceivedDate().getTime());
m_importer.audit(message, getName(), metaMsgFile.getAbsolutePath());
} catch (IOException ex) {
m_logger.error("Error creating file " + metaMsgFile.getPath(), ex);
return false;
}
return true;
}
private List<IMessageContext> generateMockMessageContext(int start, int count, Customer cust)
throws Exception {
final String indexFilePrefix = "INDEX";
final String contentFilePrefix = "CONTENT";
int time = (int) new Date().getTime();
List<IMessageContext> arr = new ArrayList<IMessageContext>();
for (int x = 0; x < count; x++) {
File indexFile = new File(m_tmpDir, indexFilePrefix + x + ".ftp");
File contentFile = new File(m_tmpDir, contentFilePrefix + x + ".ftp");
if (!indexFile.exists()) {
assertTrue("Failed to create index file.", indexFile.createNewFile());
}
if (!contentFile.exists()) {
assertTrue("Failed to create content file.", contentFile.createNewFile());
}
IMessageContext mock = new MessageContext(indexFile, contentFile);
Envelope lope = new Envelope();
lope.setMessageID("booga" + x + new Date().getTime());
lope.setReceivedDate(new Date());
lope.setSentDate(new Date());
lope.setSize(10);
lope.setCompressedSize(5);
lope.setSubject("Test");
lope.setRecipients("bleh");
lope.setSftpTimeMS(100);
lope.setSender("test2");
mock.setEnvelope(lope);
mock.setResolvedRecipients(new ArrayList<ReplicatedRecipient>());
mock.setCustomer(cust);
mock.setPartition(m_part);
mock.setInternalId(start + 1 + x);
arr.add(mock);
}
return arr;
}
@Test
public void testFailureTracking() throws Exception {
final IPartitionProxy partitionProxy = createNiceMock(IPartitionProxy.class);
final IPartitionProxy partitionProxy2 = createNiceMock(IPartitionProxy.class);
long msgId = 2L;
IFailureTrackingManager failMgr = ManagementContainer.getInstance().getFailureTrackingManager();
StorageImporter importer = new MockStorageImporter();
Customer customer = TestUtils.createTestCustomer();
try {
PartitionStoreStage stage = new PartitionStoreStage(importer);
Pipeline pipeline =
PipelineBuilder.start(
ManagementContainer.getInstance().getConfiguration(),
MessageImporter.PIPELINE_CONFIG_SECTION)
.singleton("store-stage", stage)
.get();
File envelopeFile =
File.createTempFile(
"PartitionStoreTest", "test", ManagementContainer.getInstance().getNfsRoot());
IMessageContext msgCtx = new MessageContext(envelopeFile, null);
msgCtx.setCustomer(customer);
msgCtx.setIndexMessage(true);
msgCtx.setInternalId(msgId);
msgCtx.setPartition(m_part);
msgCtx.setPartitionID(m_part.getId());
Collection<IMessageContext> msgs = Collections.singletonList(msgCtx);
Collection<IMessageContext> emptyResult = Collections.emptyList();
expect(partitionProxy.storeMessages(msgs)).andReturn(emptyResult).anyTimes();
replay(partitionProxy);
PartitionManager pm =
new PartitionManager(
ManagementContainer.getInstance().getPool(ManagementContainer.AM_POOL_NAME)) {
@Override
public IPartitionProxy getContentProxy(int id) {
return partitionProxy;
}
};
PartitionManagerTest.preparePartitionManager(pm);
stage.setPartitionManager(pm);
FailureRecord failureRecord =
FailureTrackingHelper.buildFailureRecord(msgCtx, FailureCategory.STORAGE, null, null);
failureRecord.setStoredOnPartition(false);
failMgr.insertMessageFailure(failureRecord);
msgCtx.setInternalFailureId(msgId);
pipeline.process(
new Pair<Partition, List<IMessageContext>>(m_part, Collections.singletonList(msgCtx)));
failureRecord = failMgr.getFailureRecord(msgId);
assertTrue("Should find stored on partition to be true", failureRecord.isStoredOnPartition());
Map<String, String> props = msgCtx.readEnvelope();
assertTrue(
"envelope should contain failure ID",
props.containsKey(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
assertEquals(
"message ID in envelope should match permanent ID",
String.valueOf(msgId),
props.get(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
// confirm that transient ID gets set back to real ID of message
// remove the existing failure record
failMgr.removeFailure(msgId);
// change the ID
msgCtx.setInternalFailureId(msgId + 1);
msgCtx.setInternalId(msgId + 1);
// insert the new failure record
failureRecord =
FailureTrackingHelper.buildFailureRecord(msgCtx, FailureCategory.STORAGE, null, null);
failMgr.insertMessageFailure(failureRecord);
// old record should be gone
failureRecord = failMgr.getFailureRecord(msgId);
assertNull("Should no longer find the failure record with permanent ID", failureRecord);
// check that the new record can be fetched
failureRecord = failMgr.getFailureRecord(msgId + 1);
assertNotNull("Should find transient ID failure record", failureRecord);
msgCtx.appendToEnvelope(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID, msgId + 1);
props = msgCtx.readEnvelope();
assertTrue(
"envelope should contain failure ID",
props.containsKey(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
assertEquals(
"message ID in envelope should match permanent ID",
String.valueOf(msgId + 1),
props.get(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
// now assign back the permanent ID and process through the pipeline
msgCtx.setInternalId(msgId);
pipeline.process(
new Pair<Partition, List<IMessageContext>>(m_part, Collections.singletonList(msgCtx)));
failureRecord = failMgr.getFailureRecord(msgId + 1);
assertNull("Should not find any record based on transient message ID", failureRecord);
failureRecord = failMgr.getFailureRecord(msgId);
assertNotNull("Should find failure record for permanent ID", failureRecord);
assertTrue("Should find stored on partition to be true", failureRecord.isStoredOnPartition());
// envelope should now reflect permanent ID
props = msgCtx.readEnvelope();
assertTrue(
"envelope should contain failure ID",
props.containsKey(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
assertEquals(
"message ID in envelope should match permanent ID",
String.valueOf(msgId),
props.get(ParseEnvelopeStage.INTERNAL_MESSAGE_FAILURE_ID));
// confirm that state does not change on failure
// force an exception on any call to storeMessages
pm =
new PartitionManager(
ManagementContainer.getInstance().getPool(ManagementContainer.AM_POOL_NAME)) {
@Override
public IPartitionProxy getContentProxy(int id) {
return partitionProxy2;
}
};
PartitionManagerTest.preparePartitionManager(pm);
stage.setPartitionManager(pm);
expect(partitionProxy2.storeMessages(msgs))
.andThrow(new IPartitionProxy.ServiceNotAvailableException(new Exception()))
.anyTimes();
replay(partitionProxy2);
failureRecord.setStoredOnPartition(false);
failMgr.insertMessageFailure(failureRecord);
pipeline.process(
new Pair<Partition, List<IMessageContext>>(m_part, Collections.singletonList(msgCtx)));
failureRecord = failMgr.getFailureRecord(msgId);
assertNotNull("Should find the failure record for message", failureRecord);
assertFalse(
"Should find stored on partition to be true", failureRecord.isStoredOnPartition());
} finally {
importer.stop();
TestUtils.quietDeleteCustomer(customer);
}
}
