コード例 #1
0
 public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
   return TerminatingBlocksFinder.INSTANCE.process(doc)
       | new DocumentTitleMatchClassifier(doc.getTitle()).process(doc)
       | NumWordsRulesClassifier.INSTANCE.process(doc)
       | IgnoreBlocksAfterContentFilter.DEFAULT_INSTANCE.process(doc)
       | BlockProximityFusion.MAX_DISTANCE_1.process(doc)
       | BoilerplateBlockFilter.INSTANCE.process(doc)
       | BlockProximityFusion.MAX_DISTANCE_1_CONTENT_ONLY.process(doc)
       | KeepLargestBlockFilter.INSTANCE.process(doc)
       | ExpandTitleToContentFilter.INSTANCE.process(doc);
 }
コード例 #2
0
 public boolean process(TextDocument doc) throws BoilerpipeProcessingException {
   return TerminatingBlocksFinder.INSTANCE.process(doc)
       | new DocumentTitleMatchClassifier(doc.getTitle()).process(doc)
       | NumWordsRulesClassifier.INSTANCE.process(doc)
       | IgnoreBlocksAfterContentFilter.DEFAULT_INSTANCE.process(doc)
       | TrailingHeadlineToBoilerplateFilter.INSTANCE.process(doc)
       | BlockProximityFusion.MAX_DISTANCE_1.process(doc)
       | BoilerplateBlockFilter.INSTANCE_KEEP_TITLE.process(doc)
       | BlockProximityFusion.MAX_DISTANCE_1_CONTENT_ONLY_SAME_TAGLEVEL.process(doc)
       | KeepLargestBlockFilter.INSTANCE_EXPAND_TO_SAME_TAGLEVEL_MIN_WORDS.process(doc)
       | ExpandTitleToContentFilter.INSTANCE.process(doc)
       | LargeBlockSameTagLevelToContentFilter.INSTANCE.process(doc)
       | ListAtEndFilter.INSTANCE.process(doc);
 }
コード例 #3
0
  @Override
  public void endDocument() throws SAXException {
    super.endDocument();

    TextDocument td = toTextDocument();
    try {
      extractor.process(td);
    } catch (BoilerpipeProcessingException e) {
      throw new SAXException(e);
    }

    Attributes emptyAttrs = new AttributesImpl();

    // At this point we have all the information we need to either emit N paragraphs
    // of plain text (if not including markup), or we have to replay our recorded elements
    // and only emit character runs that passed the boilerpipe filters.
    if (includeMarkup) {
      BitSet validCharacterRuns = new BitSet();
      for (TextBlock block : td.getTextBlocks()) {
        if (block.isContent()) {
          BitSet bs = block.getContainedTextElements();
          if (bs != null) {
            validCharacterRuns.or(bs);
          }
        }
      }

      // Now have bits set for all valid character runs. Replay our recorded elements,
      // but only emit character runs flagged as valid.
      int curCharsIndex = headerCharOffset;
      for (RecordedElement element : elements) {
        switch (element.getElementType()) {
          case START:
            delegate.startElement(
                element.getUri(), element.getLocalName(), element.getQName(), element.getAttrs());
            // Fall through

          case CONTINUE:
            // Now emit characters that are valid. Note that boilerpipe pre-increments the character
            // index, so
            // we have to follow suit.
            for (char[] chars : element.getCharacters()) {
              curCharsIndex++;

              if (validCharacterRuns.get(curCharsIndex)) {
                delegate.characters(chars, 0, chars.length);
              }
            }
            break;

          case END:
            delegate.endElement(element.getUri(), element.getLocalName(), element.getQName());
            break;

          default:
            throw new RuntimeException("Unhandled element type: " + element.getElementType());
        }
      }
    } else {
      for (TextBlock block : td.getTextBlocks()) {
        if (block.isContent()) {
          delegate.startElement(XHTMLContentHandler.XHTML, "p", "p", emptyAttrs);
          char[] chars = block.getText().toCharArray();
          delegate.characters(chars, 0, chars.length);
          delegate.endElement(XHTMLContentHandler.XHTML, "p", "p");
          delegate.ignorableWhitespace(NL, 0, NL.length);
        }
      }
    }

    delegate.endElement(XHTMLContentHandler.XHTML, "body", "body");
    delegate.endElement(XHTMLContentHandler.XHTML, "html", "html");

    // We defer ending any prefix mapping until here, which is why we don't pass this
    // through to the delegate in an overridden method.
    delegate.endPrefixMapping("");

    delegate.endDocument();
  }
コード例 #4
0
 /** getContent returns the boilerpipe extracted text. */
 @Override
 public String getContent() {
   TextDocument textDocument = _bpContentHandler.getTextDocument();
   return textDocument.getText(true, false);
 }