public void writeTo(final Writer writer) throws IOException {
this.writer = writer;
if (segment instanceof Source) ((Source) segment).fullSequentialParse();
nextTag = segment.source.findNextTag(segment.begin);
index = segment.begin;
writeContent(segment.end, segment.getChildElements(), 0);
writer.flush();
}
public static void main(String[] args) throws Exception {
String sourceUrlString = null;
PrintWriter writer = null;
File Directory = new File("asd\\Web Pages\\HousingAndEnvironment");
if (Directory.isDirectory()) {
for (File f : Directory.listFiles()) {
sourceUrlString = Directory.getPath() + "\\" + f.getName();
if (sourceUrlString.indexOf(':') == -1) sourceUrlString = "file:" + sourceUrlString;
// reader = new BufferedReader(new FileReader(sourceUrlString));
writer = new PrintWriter("asd\\Web Pages\\HousingAndEnvironment" + f.getName() + ".txt");
Source source = new Source(new URL(sourceUrlString));
source.fullSequentialParse();
writer.print(source.getTextExtractor().setIncludeAttributes(true).toString());
writer.flush();
writer.close();
// MicrosoftConditionalCommentTagTypes.register();
// PHPTagTypes.register();
// PHPTagTypes.PHP_SHORT.deregister(); // remove PHP short tags for this example otherwise
// they override processing instructions
// MasonTagTypes.register();
// Call fullSequentialParse manually as most of the source will be parsed.
// System.out.println("Document title:");
// String title=getTitle(source);
// System.out.println(title==null ? "(none)" : title);
// System.out.println("\nDocument description:");
// String description=getMetaValue(source,"description");
// System.out.println(description==null ? "(none)" : description);
//
// System.out.println("\nDocument keywords:");
// String keywords=getMetaValue(source,"keywords");
// System.out.println(keywords==null ? "(none)" : keywords);
//
// System.out.println("\nLinks to other documents:");
// List<Element> linkElements=source.getAllElements(HTMLElementName.A);
// for (Element linkElement : linkElements) {
// String href=linkElement.getAttributeValue("href");
// if (href==null) continue;
// // A element can contain other tags so need to extract the text from it:
// String label=linkElement.getContent().getTextExtractor().toString();
// System.out.println(label+" <"+href+'>');
// }
// System.out.println("\nAll text from file (exluding content inside SCRIPT and STYLE
// elements):\n");
// System.out.println(source.getTextExtractor().setIncludeAttributes(true).toString());
// System.out.println("\nSame again but this time extend the TextExtractor class to also
// exclude text from P elements and any elements with class=\"control\":\n");
// TextExtractor textExtractor=new TextExtractor(source) {
// public boolean excludeElement(StartTag startTag) {
// return startTag.getName()==HTMLElementName.P ||
// "control".equalsIgnoreCase(startTag.getAttributeValue("class"));
// }
// };
// System.out.println(textExtractor.setIncludeAttributes(true).toString());
}
}
}
