private boolean processURL(URL url, String baseDir, StatusWindow status) throws IOException {
if (processedLinks.contains(url)) {
return false;
} else {
processedLinks.add(url);
}
URLConnection connection = url.openConnection();
InputStream in = new BufferedInputStream(connection.getInputStream());
ArrayList list = processPage(in, baseDir, url);
if ((status != null) && (list.size() > 0)) {
status.setMaximum(list.size());
}
for (int i = 0; i < list.size(); i++) {
if (status != null) {
status.setMessage(Utils.trimFileName(list.get(i).toString(), 40), i);
}
if ((!((String) list.get(i)).startsWith("RUN"))
&& (!((String) list.get(i)).startsWith("SAVE"))
&& (!((String) list.get(i)).startsWith("LOAD"))) {
processURL(
new URL(url.getProtocol(), url.getHost(), url.getPort(), (String) list.get(i)),
baseDir,
status);
}
}
in.close();
return true;
}
/**
* Scans the html string and checks for <A HREF= > tags, then adds them to a ArrayList and returns
* the modified html string.
*/
private String checkForLinks(String htmlStr, ArrayList urlList) {
String STARTTAG = "<A HREF=\"";
String ENDTAG = "\">";
String urlString;
int startTag = 0;
int endTag = 0;
while ((startTag != -1) && (endTag != -1)) {
startTag = htmlStr.toUpperCase().indexOf(STARTTAG, endTag);
endTag = htmlStr.indexOf(ENDTAG, startTag);
if ((startTag != -1) && (endTag != -1)) {
urlString = htmlStr.substring(startTag + STARTTAG.length(), endTag);
urlList.add(urlString);
// now nicify this link
StringBuffer htmlStrBuffer = new StringBuffer(htmlStr);
htmlStrBuffer.replace(startTag + STARTTAG.length(), endTag, niceifyLink(urlString));
htmlStr = htmlStrBuffer.toString();
endTag = htmlStr.indexOf(ENDTAG, startTag) + ENDTAG.length();
}
}
return htmlStr;
}