예제 #1
0
  @Override
  protected void processSubDataEntity(
      MultiValueHashMap<String, Object> subDataEntityInformation,
      Metadata metadata,
      ContentHandler handler2use4recursiveCall,
      ParseContext context)
      throws Exception {

    URLName urlNameWithPassword =
        (URLName) subDataEntityInformation.getFirst("urlNameWithPassword");

    String strMessageId = (String) subDataEntityInformation.getFirst("Message-ID");
    String strMessageFolder = (String) subDataEntityInformation.getFirst("folder");

    String strEntityId = ImapURLStreamProvider.getEntityId(strMessageFolder, strMessageId);

    // wir setzten die hier schon mal - die Daten haben wir in einem prefetching-Schritt schon
    // effizient geladen. Wenn diese hier schon im
    // Metadata-Objekt stehen, werden sie von der addFirstMetadata nicht nochmal geladen
    metadata.set(Metadata.SOURCE, urlNameWithPassword.toString());
    metadata.set(IncrementalCrawlingHistory.dataEntityId, strEntityId);
    metadata.set(
        IncrementalCrawlingHistory.dataEntityContentFingerprint,
        ImapURLStreamProvider.getDataEntityContentFingerprint(strEntityId));
    URLName urlNameWithoutPassword =
        new URLName(
            urlNameWithPassword.getProtocol(),
            urlNameWithPassword.getHost(),
            urlNameWithPassword.getPort(),
            urlNameWithPassword.getFile(),
            urlNameWithPassword.getUsername(),
            "");
    metadata.set(Metadata.RESOURCE_NAME_KEY, urlNameWithoutPassword.toString());
    if (strMessageId == null)
      metadata.set("Content-Type", DatasourceMediaTypes.IMAPFOLDER.toString());
    else metadata.set("Content-Type", "message/rfc822");

    metadata =
        URLStreamProvider.getURLStreamProvider4Protocol(urlNameWithPassword.getProtocol())
            .addFirstMetadata(urlNameWithPassword, metadata, context);
    InputStream stream =
        URLStreamProvider.getURLStreamProvider(urlNameWithPassword)
            .getStream(urlNameWithPassword, metadata, context);

    try {

      if (m_leech == null) m_leech = new Leech();

      // hier nimmt der dann bei einer message hoffentlich den Tika RFC822Parser
      Parser parser = m_leech.getParser();

      parser.parse(stream, handler2use4recursiveCall, metadata, context);

    } finally {
      if (stream != null) stream.close();
    }
  }
예제 #2
0
  public static Store connect2Server(URLName url, ParseContext context) throws MessagingException {

    ImapCrawlerContext imapCrawlerContext =
        context.get(ImapCrawlerContext.class, new ImapCrawlerContext());

    Properties properties = System.getProperties();

    properties.setProperty("mail.store.protocol", url.getProtocol());

    if (imapCrawlerContext.getIgnoreSSLCertificates()) {
      properties.setProperty(
          "mail.imaps.socketFactory.class", CertificateIgnoringSocketFactory.class.getName());
      properties.setProperty("mail.imaps.socketFactory.fallback", "false");
    }

    if (!StringUtils.nullOrWhitespace(imapCrawlerContext.getSSLCertificateFilePath())
        && "imaps".equalsIgnoreCase(url.getProtocol())) {
      properties.setProperty(
          "javax.net.ssl.trustStore", imapCrawlerContext.getSSLCertificateFilePath());
      properties.setProperty(
          "javax.net.ssl.trustStorePassword", imapCrawlerContext.getSSLCertificateFilePassword());
    }

    Session session = Session.getDefaultInstance(properties);
    Store mailStore = session.getStore(url.getProtocol());

    String strUserName = imapCrawlerContext.getUserName();
    if (strUserName == null) strUserName = url.getUsername();

    String strPassword = imapCrawlerContext.getPassword();
    if (strPassword == null) strPassword = url.getPassword();

    if (!mailStore.isConnected())
      mailStore.connect(url.getHost(), url.getPort(), strUserName, strPassword);

    return mailStore;
  }
  /** We override toString() so we can display the store URLName without the password. */
  public String toString() {
    if (display == null) {
      URLName url = store.getURLName();
      if (url == null) {
        display = store.toString();
      } else {
        // don't show the password
        URLName too =
            new URLName(
                url.getProtocol(),
                url.getHost(),
                url.getPort(),
                url.getFile(),
                url.getUsername(),
                null);
        display = too.toString();
      }
    }

    return display;
  }
예제 #4
0
 public Transport getTransport(URLName var1) throws NoSuchProviderException {
   String var2 = var1.getProtocol();
   Provider var3 = this.getProvider(var2);
   return this.getTransport(var3, var1);
 }
예제 #5
0
 public Store getStore(URLName var1) throws NoSuchProviderException {
   String var2 = var1.getProtocol();
   Provider var3 = this.getProvider(var2);
   return this.getStore(var3, var1);
 }
예제 #6
0
  @Override
  protected Iterator<MultiValueHashMap<String, Object>> getSubDataEntitiesInformation(
      InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context)
      throws Exception {

    // imap url schema: imap[s]://uname@hostname:port/folder;uidvalidity=385759045/;uid=20. Examples
    // (incl. message-referenzierung)
    // http://xml.resource.org/public/rfc/html/rfc2192.html#anchor10
    // allerdings nimmt der Java ImapStore auch URLs mit Passwörtern an. Dann geht auch
    // imap[s]://uname:pwd@hostname:port/folder;uidvalidity=385759045/;uid=20

    CrawlerContext crawlerContext = context.get(CrawlerContext.class, new CrawlerContext());

    String strContainerURL = metadata.get(Metadata.SOURCE);

    URLName containerURLName = new URLName(strContainerURL);

    if (m_mailStore == null) m_mailStore = connect2Server(containerURLName, context);

    // wenn kein directory angegeben wird, dann crawlen wir einfach den default folder und die inbox
    LinkedList<Folder> llFolderz2Crawl = new LinkedList<Folder>();
    if (containerURLName.getFile() != null) {
      Folder folder = m_mailStore.getFolder(containerURLName.getFile());
      if (folder != null && folder.exists()) llFolderz2Crawl.add(folder);
      else throw new FileNotFoundException("Can't find imap folder '" + folder.getFullName() + "'");

    } else {
      Folder folder = m_mailStore.getDefaultFolder();
      if (folder != null && folder.exists()) llFolderz2Crawl.add(folder);

      folder = m_mailStore.getFolder("INBOX");
      if (folder != null && folder.exists()) llFolderz2Crawl.add(folder);
    }

    LinkedList<MultiValueHashMap<String, Object>> llEntityInfo =
        new LinkedList<MultiValueHashMap<String, Object>>();

    for (Folder folder2crawl : llFolderz2Crawl) {
      // Jetzt haben wir die Containerobjekte - nun geben wir die Daten zu den SubEntities zurück

      // die subfolder
      boolean bFolderCanHaveSubFolders =
          (folder2crawl.getType() & Folder.HOLDS_FOLDERS) == Folder.HOLDS_FOLDERS;

      if (bFolderCanHaveSubFolders) {
        folder2crawl.open(Folder.READ_ONLY);

        Folder[] subFolders = folder2crawl.list();
        for (Folder subFolder : subFolders) {
          URLName urlName = subFolder.getURLName();
          URLName urlNameWithPassword =
              new URLName(
                  containerURLName.getProtocol(),
                  urlName.getHost(),
                  urlName.getPort(),
                  urlName.getFile(),
                  urlName.getUsername(),
                  containerURLName.getPassword());

          if (!checkIfInConstraints(urlName.toString(), null, context)) continue;

          MultiValueHashMap<String, Object> hsEntityInformation =
              new MultiValueHashMap<String, Object>();

          hsEntityInformation.add(CrawlerParser.SOURCEID, urlName);
          hsEntityInformation.add("urlNameWithPassword", urlNameWithPassword);
          hsEntityInformation.add("folder", subFolder.getFullName());

          llEntityInfo.add(hsEntityInformation);
        }
      }

      // die messages
      boolean bFolderCanHaveMessages =
          (folder2crawl.getType() & Folder.HOLDS_MESSAGES) == Folder.HOLDS_MESSAGES;

      if (bFolderCanHaveMessages) {
        if (!folder2crawl.isOpen()) folder2crawl.open(Folder.READ_ONLY);

        // wir holen uns alle nicht-deleted messages, und werfen noch die raus, die 'expunged' sind
        Message[] relevantMessagesOfFolder =
            folder2crawl.search(new FlagTerm(new Flags(Flags.Flag.DELETED), false));
        ArrayList<Message> nonDelNonExpungedMessages = new ArrayList<Message>();
        for (Message message : relevantMessagesOfFolder)
          if (!message.isExpunged()) nonDelNonExpungedMessages.add(message);
        relevantMessagesOfFolder = nonDelNonExpungedMessages.toArray(new Message[0]);

        // die Daten die wir später benötigen holen wir uns effizient in einem Rutsch - deswegen
        // benötigen wir auch keinen Thread mit dem
        // OneAfterOneIterator, um Speicher zu sparen (siehe DirectoryCrawlerParser). Das Array
        // haben wir hier eh. Entweder oder.
        FetchProfile profile = new FetchProfile();
        profile.add(UIDFolder.FetchProfileItem.UID);
        profile.add("Message-ID");
        folder2crawl.fetch(relevantMessagesOfFolder, profile);

        for (int i = 0;
            i < relevantMessagesOfFolder.length && !crawlerContext.stopRequested();
            i++) {
          MimeMessage message = (MimeMessage) relevantMessagesOfFolder[i];

          // hier brauchen wir noch eine URL mit und eine ohne Passwort
          URLName urlName = getMessageUrl(folder2crawl, message);
          URLName urlNameWithPassword =
              new URLName(
                  containerURLName.getProtocol(),
                  urlName.getHost(),
                  urlName.getPort(),
                  urlName.getFile(),
                  urlName.getUsername(),
                  containerURLName.getPassword());

          if (!checkIfInConstraints(urlName.toString(), message, context)) continue;

          MultiValueHashMap<String, Object> hsEntityInformation =
              new MultiValueHashMap<String, Object>();

          hsEntityInformation.add(CrawlerParser.SOURCEID, urlName);
          hsEntityInformation.add("urlNameWithPassword", urlNameWithPassword);
          hsEntityInformation.add("Message-ID", message.getHeader("Message-ID")[0]);
          hsEntityInformation.add("folder", folder2crawl.getFullName());

          llEntityInfo.add(hsEntityInformation);
        }
      }

      // wir haben die folder abgearbeitet, dann können wir diesen Speicher wieder frei geben
      m_hsImapFolder2Stickyness.clear();

      if (folder2crawl.isOpen()) folder2crawl.close(false);
    }

    return llEntityInfo.iterator();
  }