/*
** calculate, given the examine/ignore and save/refuse values, whether
** to examine and/or save s.
*/
private boolean[] EISR(
String s, String which, String examine[], String ignore[], String save[], String refuse[]) {
if (s == null) return (null);
logger.fine(s);
logger.fine(which);
logger.fine(java.util.Arrays.toString(examine));
logger.fine(java.util.Arrays.toString(ignore));
logger.fine(java.util.Arrays.toString(save));
logger.fine(java.util.Arrays.toString(refuse));
boolean E = Utils.blurf(examine, ignore, s, false);
boolean S = Utils.blurf(save, refuse, s, false);
if (args.PrintExamine && E) logger.info("Examining " + which + ": " + s);
if (args.PrintIgnore && !E) logger.info("Ignoring " + which + ": " + s);
if (args.PrintSave && S) logger.info("Saving " + which + ": " + s);
if (args.PrintRefuse && !S) logger.info("Refusing " + which + ": " + s);
boolean ret[] = new boolean[2];
ret[0] = E;
ret[1] = S;
return (ret);
}
private void addToURLs(String baseURL, String strings[]) {
logger.entering(baseURL);
logger.entering(strings);
for (int i = 0; i < strings.length; i++) {
String next = replaceAll(strings[i], args.URLFixUp);
String newBase = newBase(next);
if (newBase != null) {
logger.fine("Setting base to " + baseURL);
baseURL = newBase;
}
String possible[] = interesting(next);
for (int j = 0; j < args.Interesting.length; j++) {
if (possible[j] != null) {
URL u = makeURL(baseURL, possible[j]);
if (u == null) continue;
String total = u.toString();
String PathAccept[] = args.PathAccept;
String PathReject[] = args.PathReject;
boolean accept = Utils.blurf(PathAccept, PathReject, total, true);
if (args.PrintAccept && accept) logger.info("Accepting path: " + total);
if (args.PrintReject && !accept) logger.info("Rejecting path: " + total);
if (accept) {
if (args.URLRewrite != null) total = REplican.replaceAll(total, args.URLRewrite);
// if we don't already have it
if (urls.get(total) == null) {
if (args.PrintAdd) logger.info("Adding: " + total);
addOne(total);
}
}
}
}
}
}
private void fetch(String url) {
logger.entering(url);
boolean Path =
args.PathExamine != null
|| args.PathIgnore != null
|| args.PathSave != null
|| args.PathRefuse != null;
boolean MIME =
args.MIMEExamine != null
|| args.MIMEIgnore != null
|| args.MIMESave != null
|| args.MIMERefuse != null;
logger.fine("Path = " + Path);
logger.fine("MIME = " + MIME);
boolean tb[] =
EISR(url, "path", args.PathExamine, args.PathIgnore, args.PathSave, args.PathRefuse);
boolean Pexamine = tb[0];
boolean Psave = tb[1];
/*
* if there is no MIME, and the Path doesn't say to examine or save,
* we're done.
*/
if (!MIME && !Pexamine && !Psave) return;
/*
* otherwise, we need to Path examine or save, or we need the MIME
* header. in either case, we need an InputStream.
*/
InputStream is = null;
YouAreEll yrl = null;
for (int t = 0; t < args.Tries; t++) {
yrl = new YouAreEll(url, urls, cookies, args);
is = yrl.getInputStream();
if (is != null) break;
if (args.Tries > 1) logger.warning("Trying again");
}
if (is == null) return;
boolean Mexamine = false;
boolean Msave = false;
if (MIME && yrl.getContentType() != null) {
tb =
EISR(
yrl.getContentType(),
"MIME",
args.MIMEExamine,
args.MIMEIgnore,
args.MIMESave,
args.MIMERefuse);
Mexamine = tb[0];
Msave = tb[1];
}
// we've looked at both Path and now MIME and there's nothing to do
if (!Pexamine && !Psave && !Mexamine && !Msave) return;
fetchOne(Pexamine || Mexamine, Psave || Msave, yrl, is);
try {
is.close();
} catch (IOException IOE) {
logger.throwing(IOE);
}
}