public static void main(String[] args) throws Exception {
String address = "http://vnexpress.net/GL/Xa-hoi/2009/02/3BA0B4AB/";
webClient.setURL(address, new URL(address));
// String address = "http://vnmedia.vn/newsdetail.asp?NewsId=154558&CatId=58";
java.net.URL url = new java.net.URL(address);
HTMLDocument document = HTMLParser.createDocument(loadContent(address), "utf-8");
RefsDecoder decoder = new RefsDecoder();
NodeIterator iterator = document.getRoot().iterator();
while (iterator.hasNext()) {
HTMLNode node = iterator.next();
if (!node.isNode(Name.CONTENT)) continue;
char[] chars = node.getValue();
chars = decoder.decode(chars);
chars = CharsUtil.cutAndTrim(chars, 0, chars.length);
chars = java.text.Normalizer.normalize(new String(chars), Normalizer.Form.NFC).toCharArray();
node.setValue(chars);
}
loadCSS(address, document);
NodePath nodePath = pathParser.toPath("BODY");
HTMLNode body = extractor.lookNode(document.getRoot(), nodePath);
WebPageDataSearcher dataSearcher = new WebPageDataSearcher(document);
HTMLNode node = dataSearcher.search(body);
File file = new File("F:\\Temp2\\web\\output\\extract.htm");
byte[] bytes = new byte[0];
if (node != null) bytes = node.getTextValue().getBytes(Application.CHARSET);
RWData.getInstance().save(file, bytes);
}
private int countWord(HTMLNode node) {
if (node == null) return 0;
NodeIterator nodeIterator = node.iterator();
int word = 0;
while (nodeIterator.hasNext()) {
HTMLNode iterNode = nodeIterator.next();
if (getAncestor(iterNode, Name.A, 0, 5) != null) continue;
if (iterNode.isNode(Name.CONTENT)) {
String text = iterNode.getTextValue();
word += textCounter.countWord(text, 0, text.length());
}
}
return word;
}