@Override
protected void processSubDataEntity(
MultiValueHashMap<String, Object> subDataEntityInformation,
Metadata metadata,
ContentHandler handler2use4recursiveCall,
ParseContext context)
throws Exception {
URLName urlNameWithPassword =
(URLName) subDataEntityInformation.getFirst("urlNameWithPassword");
String strMessageId = (String) subDataEntityInformation.getFirst("Message-ID");
String strMessageFolder = (String) subDataEntityInformation.getFirst("folder");
String strEntityId = ImapURLStreamProvider.getEntityId(strMessageFolder, strMessageId);
// wir setzten die hier schon mal - die Daten haben wir in einem prefetching-Schritt schon
// effizient geladen. Wenn diese hier schon im
// Metadata-Objekt stehen, werden sie von der addFirstMetadata nicht nochmal geladen
metadata.set(Metadata.SOURCE, urlNameWithPassword.toString());
metadata.set(IncrementalCrawlingHistory.dataEntityId, strEntityId);
metadata.set(
IncrementalCrawlingHistory.dataEntityContentFingerprint,
ImapURLStreamProvider.getDataEntityContentFingerprint(strEntityId));
URLName urlNameWithoutPassword =
new URLName(
urlNameWithPassword.getProtocol(),
urlNameWithPassword.getHost(),
urlNameWithPassword.getPort(),
urlNameWithPassword.getFile(),
urlNameWithPassword.getUsername(),
"");
metadata.set(Metadata.RESOURCE_NAME_KEY, urlNameWithoutPassword.toString());
if (strMessageId == null)
metadata.set("Content-Type", DatasourceMediaTypes.IMAPFOLDER.toString());
else metadata.set("Content-Type", "message/rfc822");
metadata =
URLStreamProvider.getURLStreamProvider4Protocol(urlNameWithPassword.getProtocol())
.addFirstMetadata(urlNameWithPassword, metadata, context);
InputStream stream =
URLStreamProvider.getURLStreamProvider(urlNameWithPassword)
.getStream(urlNameWithPassword, metadata, context);
try {
if (m_leech == null) m_leech = new Leech();
// hier nimmt der dann bei einer message hoffentlich den Tika RFC822Parser
Parser parser = m_leech.getParser();
parser.parse(stream, handler2use4recursiveCall, metadata, context);
} finally {
if (stream != null) stream.close();
}
}
public static Store connect2Server(URLName url, ParseContext context) throws MessagingException {
ImapCrawlerContext imapCrawlerContext =
context.get(ImapCrawlerContext.class, new ImapCrawlerContext());
Properties properties = System.getProperties();
properties.setProperty("mail.store.protocol", url.getProtocol());
if (imapCrawlerContext.getIgnoreSSLCertificates()) {
properties.setProperty(
"mail.imaps.socketFactory.class", CertificateIgnoringSocketFactory.class.getName());
properties.setProperty("mail.imaps.socketFactory.fallback", "false");
}
if (!StringUtils.nullOrWhitespace(imapCrawlerContext.getSSLCertificateFilePath())
&& "imaps".equalsIgnoreCase(url.getProtocol())) {
properties.setProperty(
"javax.net.ssl.trustStore", imapCrawlerContext.getSSLCertificateFilePath());
properties.setProperty(
"javax.net.ssl.trustStorePassword", imapCrawlerContext.getSSLCertificateFilePassword());
}
Session session = Session.getDefaultInstance(properties);
Store mailStore = session.getStore(url.getProtocol());
String strUserName = imapCrawlerContext.getUserName();
if (strUserName == null) strUserName = url.getUsername();
String strPassword = imapCrawlerContext.getPassword();
if (strPassword == null) strPassword = url.getPassword();
if (!mailStore.isConnected())
mailStore.connect(url.getHost(), url.getPort(), strUserName, strPassword);
return mailStore;
}
/** We override toString() so we can display the store URLName without the password. */
public String toString() {
if (display == null) {
URLName url = store.getURLName();
if (url == null) {
display = store.toString();
} else {
// don't show the password
URLName too =
new URLName(
url.getProtocol(),
url.getHost(),
url.getPort(),
url.getFile(),
url.getUsername(),
null);
display = too.toString();
}
}
return display;
}
public Transport getTransport(URLName var1) throws NoSuchProviderException {
String var2 = var1.getProtocol();
Provider var3 = this.getProvider(var2);
return this.getTransport(var3, var1);
}
public Store getStore(URLName var1) throws NoSuchProviderException {
String var2 = var1.getProtocol();
Provider var3 = this.getProvider(var2);
return this.getStore(var3, var1);
}
@Override
protected Iterator<MultiValueHashMap<String, Object>> getSubDataEntitiesInformation(
InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context)
throws Exception {
// imap url schema: imap[s]://uname@hostname:port/folder;uidvalidity=385759045/;uid=20. Examples
// (incl. message-referenzierung)
// http://xml.resource.org/public/rfc/html/rfc2192.html#anchor10
// allerdings nimmt der Java ImapStore auch URLs mit Passwörtern an. Dann geht auch
// imap[s]://uname:pwd@hostname:port/folder;uidvalidity=385759045/;uid=20
CrawlerContext crawlerContext = context.get(CrawlerContext.class, new CrawlerContext());
String strContainerURL = metadata.get(Metadata.SOURCE);
URLName containerURLName = new URLName(strContainerURL);
if (m_mailStore == null) m_mailStore = connect2Server(containerURLName, context);
// wenn kein directory angegeben wird, dann crawlen wir einfach den default folder und die inbox
LinkedList<Folder> llFolderz2Crawl = new LinkedList<Folder>();
if (containerURLName.getFile() != null) {
Folder folder = m_mailStore.getFolder(containerURLName.getFile());
if (folder != null && folder.exists()) llFolderz2Crawl.add(folder);
else throw new FileNotFoundException("Can't find imap folder '" + folder.getFullName() + "'");
} else {
Folder folder = m_mailStore.getDefaultFolder();
if (folder != null && folder.exists()) llFolderz2Crawl.add(folder);
folder = m_mailStore.getFolder("INBOX");
if (folder != null && folder.exists()) llFolderz2Crawl.add(folder);
}
LinkedList<MultiValueHashMap<String, Object>> llEntityInfo =
new LinkedList<MultiValueHashMap<String, Object>>();
for (Folder folder2crawl : llFolderz2Crawl) {
// Jetzt haben wir die Containerobjekte - nun geben wir die Daten zu den SubEntities zurück
// die subfolder
boolean bFolderCanHaveSubFolders =
(folder2crawl.getType() & Folder.HOLDS_FOLDERS) == Folder.HOLDS_FOLDERS;
if (bFolderCanHaveSubFolders) {
folder2crawl.open(Folder.READ_ONLY);
Folder[] subFolders = folder2crawl.list();
for (Folder subFolder : subFolders) {
URLName urlName = subFolder.getURLName();
URLName urlNameWithPassword =
new URLName(
containerURLName.getProtocol(),
urlName.getHost(),
urlName.getPort(),
urlName.getFile(),
urlName.getUsername(),
containerURLName.getPassword());
if (!checkIfInConstraints(urlName.toString(), null, context)) continue;
MultiValueHashMap<String, Object> hsEntityInformation =
new MultiValueHashMap<String, Object>();
hsEntityInformation.add(CrawlerParser.SOURCEID, urlName);
hsEntityInformation.add("urlNameWithPassword", urlNameWithPassword);
hsEntityInformation.add("folder", subFolder.getFullName());
llEntityInfo.add(hsEntityInformation);
}
}
// die messages
boolean bFolderCanHaveMessages =
(folder2crawl.getType() & Folder.HOLDS_MESSAGES) == Folder.HOLDS_MESSAGES;
if (bFolderCanHaveMessages) {
if (!folder2crawl.isOpen()) folder2crawl.open(Folder.READ_ONLY);
// wir holen uns alle nicht-deleted messages, und werfen noch die raus, die 'expunged' sind
Message[] relevantMessagesOfFolder =
folder2crawl.search(new FlagTerm(new Flags(Flags.Flag.DELETED), false));
ArrayList<Message> nonDelNonExpungedMessages = new ArrayList<Message>();
for (Message message : relevantMessagesOfFolder)
if (!message.isExpunged()) nonDelNonExpungedMessages.add(message);
relevantMessagesOfFolder = nonDelNonExpungedMessages.toArray(new Message[0]);
// die Daten die wir später benötigen holen wir uns effizient in einem Rutsch - deswegen
// benötigen wir auch keinen Thread mit dem
// OneAfterOneIterator, um Speicher zu sparen (siehe DirectoryCrawlerParser). Das Array
// haben wir hier eh. Entweder oder.
FetchProfile profile = new FetchProfile();
profile.add(UIDFolder.FetchProfileItem.UID);
profile.add("Message-ID");
folder2crawl.fetch(relevantMessagesOfFolder, profile);
for (int i = 0;
i < relevantMessagesOfFolder.length && !crawlerContext.stopRequested();
i++) {
MimeMessage message = (MimeMessage) relevantMessagesOfFolder[i];
// hier brauchen wir noch eine URL mit und eine ohne Passwort
URLName urlName = getMessageUrl(folder2crawl, message);
URLName urlNameWithPassword =
new URLName(
containerURLName.getProtocol(),
urlName.getHost(),
urlName.getPort(),
urlName.getFile(),
urlName.getUsername(),
containerURLName.getPassword());
if (!checkIfInConstraints(urlName.toString(), message, context)) continue;
MultiValueHashMap<String, Object> hsEntityInformation =
new MultiValueHashMap<String, Object>();
hsEntityInformation.add(CrawlerParser.SOURCEID, urlName);
hsEntityInformation.add("urlNameWithPassword", urlNameWithPassword);
hsEntityInformation.add("Message-ID", message.getHeader("Message-ID")[0]);
hsEntityInformation.add("folder", folder2crawl.getFullName());
llEntityInfo.add(hsEntityInformation);
}
}
// wir haben die folder abgearbeitet, dann können wir diesen Speicher wieder frei geben
m_hsImapFolder2Stickyness.clear();
if (folder2crawl.isOpen()) folder2crawl.close(false);
}
return llEntityInfo.iterator();
}