Пример #1
0
  @Override
  public JapaneseTokenizer init(final byte[] txt) {
    String source = string(txt);
    if (wc) { // convert wide-space to space
      source = source.replace('\u3000', '\u0020');
    }
    final ArrayList<?> morpheme = (ArrayList<?>) Reflect.invoke(parse, tagger, source);
    final ArrayList<Morpheme> list = new ArrayList<>();
    try {
      int prev = 0;
      final int ms = morpheme.size();
      for (int i = 0; i < ms; i++) {
        final Object m = morpheme.get(i);
        final String srfc = surface.get(m).toString();
        final String ftr = feature.get(m).toString();
        final int strt = start.getInt(m);
        if (i != 0) {
          final int l = strt - prev;
          if (l != 0) {
            list.add(new Morpheme(source.substring(strt - 1, strt + l - 1), KIGOU_FEATURE));
          }
        }
        prev = srfc.length() + strt;

        // separates continuous mark (ASCII)
        boolean cont = true;
        final ArrayList<Morpheme> marks = new ArrayList<>();
        final int sl = srfc.length();
        for (int s = 0; s < sl; s++) {
          final String c = String.valueOf(srfc.charAt(s));
          final byte[] t = token(c);
          if (t.length == 1) {
            if (letter(t[0]) || digit(t[0])) cont = false;
            else marks.add(new Morpheme(c, KIGOU_FEATURE));
          } else {
            cont = false;
          }
        }

        if (cont) list.addAll(marks);
        else list.add(new Morpheme(srfc, ftr));
      }
    } catch (final Exception ex) {
      Util.errln(Util.className(this) + ": " + ex);
    }
    tokenList = list;
    tokens = list.iterator();

    return this;
  }
Пример #2
0
  static {
    IOFile dic = null;
    if (Reflect.available(PATTERN)) {
      dic = new IOFile(LANG);
      if (!dic.exists()) {
        dic = new IOFile(Prop.HOME, "etc/" + LANG);
        if (!dic.exists()) {
          available = false;
        }
      }
    } else {
      available = false;
    }

    if (available) {
      Class<?> clz = Reflect.find(PATTERN);
      if (clz == null) {
        Util.debug("Could not initialize Igo Japanese lexer.");
      } else {
        /* Igo constructor. */
        final Constructor<?> tgr = Reflect.find(clz, String.class);
        tagger = Reflect.get(tgr, dic.path());
        if (tagger == null) {
          available = false;
          Util.debug("Could not initialize Igo Japanese lexer.");
        } else {
          parse = Reflect.method(clz, "parse", CharSequence.class);
          if (parse == null) {
            Util.debug("Could not initialize Igo lexer method.");
          }
          clz = Reflect.find("net.reduls.igo.Morpheme");
          surface = Reflect.field(clz, "surface");
          feature = Reflect.field(clz, "feature");
          start = Reflect.field(clz, "start");
        }
      }
    }
  }
Пример #3
0
  /**
   * Converts an HTML document to XML.
   *
   * @param io io reference
   * @param opts html options
   * @return parser
   * @throws IOException I/O exception
   */
  private static IO toXML(final IO io, final HtmlOptions opts) throws IOException {
    // reader could not be initialized; fall back to XML
    if (READER == null) return io;

    try {
      // tries to extract the encoding from the input
      final TextInput ti = new TextInput(io);
      String enc = ti.encoding();
      final byte[] content = ti.content();

      // looks for a charset definition
      final byte[] encoding = token("charset=");
      int cs = indexOf(content, encoding);
      if (cs > 0) {
        // extracts the encoding string
        cs += encoding.length;
        int ce = cs;
        final int cl = content.length;
        while (++ce < cl && content[ce] > 0x28) ;
        enc = string(substring(content, cs, ce));
      }

      // define input
      final InputSource is = new InputSource(new ArrayInput(content));
      is.setEncoding(supported(enc) ? normEncoding(enc) : UTF8);
      // define output
      final StringWriter sw = new StringWriter();
      final XMLReader reader = (XMLReader) Reflect.get(READER);
      final Object writer = Reflect.get(WRITER, sw);

      // set TagSoup options
      if (opts.get(HtmlOptions.HTML)) {
        reader.setFeature("http://xml.org/sax/features/namespaces", false);
        opt("method", "html");
        opt("omit-xml-declaration", "yes");
      }
      if (opts.get(HtmlOptions.NONS))
        reader.setFeature("http://xml.org/sax/features/namespaces", false);
      if (opts.get(HtmlOptions.OMITXML)) opt("omit-xml-declaration", "yes");
      if (opts.get(HtmlOptions.NOBOGONS)) reader.setFeature(FEATURES + "ignore-bogons", true);
      if (opts.get(HtmlOptions.NODEFAULTS))
        reader.setFeature(FEATURES + "default-attributes", false);
      if (opts.get(HtmlOptions.NOCOLONS)) reader.setFeature(FEATURES + "translate-colons", true);
      if (opts.get(HtmlOptions.NORESTART)) reader.setFeature(FEATURES + "restart-elements", false);
      if (opts.get(HtmlOptions.IGNORABLE))
        reader.setFeature(FEATURES + "ignorable-whitespace", true);
      if (opts.get(HtmlOptions.EMPTYBOGONS)) reader.setFeature(FEATURES + "bogons-empty", true);
      if (opts.get(HtmlOptions.ANY)) reader.setFeature(FEATURES + "bogons-empty", false);
      if (opts.get(HtmlOptions.NOROOTBOGONS)) reader.setFeature(FEATURES + "root-bogons", false);
      if (opts.get(HtmlOptions.NOCDATA)) reader.setFeature(FEATURES + "cdata-elements", false);
      if (opts.get(HtmlOptions.LEXICAL))
        reader.setProperty("http://xml.org/sax/properties/lexical-handler", writer);
      if (opts.contains(HtmlOptions.METHOD)) opt("method", opts.get(HtmlOptions.METHOD));
      if (opts.contains(HtmlOptions.DOCTYPESYS))
        opt("doctype-system", opts.get(HtmlOptions.DOCTYPESYS));
      if (opts.contains(HtmlOptions.DOCTYPEPUB))
        opt("doctype-public", opts.get(HtmlOptions.DOCTYPEPUB));
      if (opts.contains(HtmlOptions.ENCODING)) is.setEncoding(opts.get(HtmlOptions.ENCODING));
      // end TagSoup options

      reader.setContentHandler((ContentHandler) writer);
      reader.parse(is);
      return new IOContent(token(sw.toString()), io.name());

    } catch (final SAXException ex) {
      Util.errln(ex);
      return io;
    }
  }
Пример #4
0
/**
 * This class uses TagSoup to convert HTML input to well-formed XML. If TagSoup is not found in the
 * classpath, the original document is passed on.
 *
 * <p>TagSoup was written by John Cowan and is based on the Apache 2.0 License: {@code
 * http://home.ccil.org/~cowan/XML/tagsoup/}.
 *
 * @author BaseX Team 2005-14, BSD License
 * @author Christian Gruen
 */
public final class HtmlParser extends XMLParser {
  /** Name of HTML Parser. */
  private static final String NAME = "TagSoup";
  /** TagSoup URL. */
  private static final String FEATURES = "http://www.ccil.org/~cowan/tagsoup/features/";

  /** XML parser class string. */
  private static final String PCLASS = "org.ccil.cowan.tagsoup.Parser";
  /** XML writer class string. */
  private static final String WCLASS = "org.ccil.cowan.tagsoup.XMLWriter";
  /** HTML reader. */
  private static final Class<?> READER = Reflect.find(PCLASS);
  /** HTML writer. */
  private static final Constructor<?> WRITER = Reflect.find(Reflect.find(WCLASS), Writer.class);
  /** XML writer output property method. */
  private static final Method METHOD =
      Reflect.method(
          Reflect.find(WCLASS), "setOutputProperty", new Class[] {String.class, String.class});

  /**
   * Checks if a CatalogResolver is available.
   *
   * @return result of check
   */
  public static boolean available() {
    return READER != null;
  }

  /**
   * Returns the name of the parser, or an empty string.
   *
   * @return name of parser
   */
  public static String parser() {
    return available() ? NAME : "";
  }

  /**
   * Constructor.
   *
   * @param source document source
   * @param opts database options
   * @throws IOException I/O exception
   */
  public HtmlParser(final IO source, final MainOptions opts) throws IOException {
    this(source, opts, opts.get(MainOptions.HTMLPARSER));
  }

  /**
   * Constructor.
   *
   * @param source document source
   * @param opts database options
   * @param hopts html options
   * @throws IOException I/O exception
   */
  public HtmlParser(final IO source, final MainOptions opts, final HtmlOptions hopts)
      throws IOException {
    super(toXML(source, hopts), opts);
  }

  /**
   * Converts an HTML document to XML.
   *
   * @param io io reference
   * @param opts html options
   * @return parser
   * @throws IOException I/O exception
   */
  private static IO toXML(final IO io, final HtmlOptions opts) throws IOException {
    // reader could not be initialized; fall back to XML
    if (READER == null) return io;

    try {
      // tries to extract the encoding from the input
      final TextInput ti = new TextInput(io);
      String enc = ti.encoding();
      final byte[] content = ti.content();

      // looks for a charset definition
      final byte[] encoding = token("charset=");
      int cs = indexOf(content, encoding);
      if (cs > 0) {
        // extracts the encoding string
        cs += encoding.length;
        int ce = cs;
        final int cl = content.length;
        while (++ce < cl && content[ce] > 0x28) ;
        enc = string(substring(content, cs, ce));
      }

      // define input
      final InputSource is = new InputSource(new ArrayInput(content));
      is.setEncoding(supported(enc) ? normEncoding(enc) : UTF8);
      // define output
      final StringWriter sw = new StringWriter();
      final XMLReader reader = (XMLReader) Reflect.get(READER);
      final Object writer = Reflect.get(WRITER, sw);

      // set TagSoup options
      if (opts.get(HtmlOptions.HTML)) {
        reader.setFeature("http://xml.org/sax/features/namespaces", false);
        opt("method", "html");
        opt("omit-xml-declaration", "yes");
      }
      if (opts.get(HtmlOptions.NONS))
        reader.setFeature("http://xml.org/sax/features/namespaces", false);
      if (opts.get(HtmlOptions.OMITXML)) opt("omit-xml-declaration", "yes");
      if (opts.get(HtmlOptions.NOBOGONS)) reader.setFeature(FEATURES + "ignore-bogons", true);
      if (opts.get(HtmlOptions.NODEFAULTS))
        reader.setFeature(FEATURES + "default-attributes", false);
      if (opts.get(HtmlOptions.NOCOLONS)) reader.setFeature(FEATURES + "translate-colons", true);
      if (opts.get(HtmlOptions.NORESTART)) reader.setFeature(FEATURES + "restart-elements", false);
      if (opts.get(HtmlOptions.IGNORABLE))
        reader.setFeature(FEATURES + "ignorable-whitespace", true);
      if (opts.get(HtmlOptions.EMPTYBOGONS)) reader.setFeature(FEATURES + "bogons-empty", true);
      if (opts.get(HtmlOptions.ANY)) reader.setFeature(FEATURES + "bogons-empty", false);
      if (opts.get(HtmlOptions.NOROOTBOGONS)) reader.setFeature(FEATURES + "root-bogons", false);
      if (opts.get(HtmlOptions.NOCDATA)) reader.setFeature(FEATURES + "cdata-elements", false);
      if (opts.get(HtmlOptions.LEXICAL))
        reader.setProperty("http://xml.org/sax/properties/lexical-handler", writer);
      if (opts.contains(HtmlOptions.METHOD)) opt("method", opts.get(HtmlOptions.METHOD));
      if (opts.contains(HtmlOptions.DOCTYPESYS))
        opt("doctype-system", opts.get(HtmlOptions.DOCTYPESYS));
      if (opts.contains(HtmlOptions.DOCTYPEPUB))
        opt("doctype-public", opts.get(HtmlOptions.DOCTYPEPUB));
      if (opts.contains(HtmlOptions.ENCODING)) is.setEncoding(opts.get(HtmlOptions.ENCODING));
      // end TagSoup options

      reader.setContentHandler((ContentHandler) writer);
      reader.parse(is);
      return new IOContent(token(sw.toString()), io.name());

    } catch (final SAXException ex) {
      Util.errln(ex);
      return io;
    }
  }

  /**
   * Reflection invoke XMLWriter.setOutputProperty().
   *
   * @param name property
   * @param value value
   */
  private static void opt(final String name, final String value) {
    Reflect.invoke(METHOD, WRITER, name, value);
  }
}
Пример #5
0
 /**
  * Reflection invoke XMLWriter.setOutputProperty().
  *
  * @param name property
  * @param value value
  */
 private static void opt(final String name, final String value) {
   Reflect.invoke(METHOD, WRITER, name, value);
 }