public GatewayEngineImpl(Config config) {
if (config instanceof ConfigWeb) {
this.config = (ConfigWeb) config;
} else {
Resource root = config.getConfigDir().getRealResource("gatewayRoot");
root.mkdirs();
this.config = ((ConfigServerImpl) config).getConfigWeb();
}
this.log = ((ConfigImpl) config).getGatewayLogger();
}
private boolean isValid(Config config, String serverPassword) {
if (StringUtil.isEmpty(serverPassword, true)) {
try {
PageContextImpl pc = (PageContextImpl) ThreadLocalPageContext.get();
serverPassword = pc.getServerPassword();
} catch (Throwable t) {
}
}
config = ThreadLocalPageContext.getConfig(config);
if (config == null || StringUtil.isEmpty(serverPassword, true)) return false;
try {
config.getConfigServer(serverPassword);
return true;
} catch (PageException e) {
return false;
}
}
protected static void _parseChildren(
Log log,
StringBuffer content,
IndexWriter writer,
String root,
URL base,
List urlsDone,
String[] extensions,
boolean recurse,
int deep,
long timeout)
throws IOException {
if (recurse) {
List urls = htmlUtil.getURLS(content.toString(), base);
// loop through all children
int len = urls.size();
List childIndexer = len > 1 ? new ArrayList() : null;
ChildrenIndexer ci;
// print.out("getting content");
for (int i = 0; i < len; i++) {
URL url = (URL) urls.get(i);
/*if(url.toExternalForm().indexOf("80")!=-1){
SystemOut.printDate("base:"+base);
SystemOut.printDate("url:"+url);
}*/
url = translateURL(url);
if (urlsDone.contains(url.toExternalForm())) continue;
// urlsDone.add(url.toExternalForm());
String protocol = url.getProtocol().toLowerCase();
String file = url.getPath();
if ((protocol.equals("http") || protocol.equals("https"))
&& validExtension(extensions, file)
&& base.getHost().equalsIgnoreCase(url.getHost())) {
try {
ci =
new ChildrenIndexer(
log, writer, root, url, urlsDone, extensions, recurse, deep + 1, timeout);
childIndexer.add(ci);
ci.start();
} catch (Throwable t) {
// print.printST(t);
}
}
}
if (childIndexer != null && !childIndexer.isEmpty()) {
Iterator it = childIndexer.iterator();
while (it.hasNext()) {
ci = (ChildrenIndexer) it.next();
if (ci.isAlive()) {
try {
ci.join(timeout);
} catch (InterruptedException e) {
// print.printST(e);
}
}
// timeout exceptionif(ci.isAlive()) throw new IOException("timeout occur while invoking
// page ["+ci.url+"]");
if (ci.isAlive()) {
ci.interrupt();
Config config = ThreadLocalPageContext.getConfig();
SystemOut.printDate(
config != null ? config.getErrWriter() : new PrintWriter(System.err),
"timeout [" + timeout + " ms] occur while invoking page [" + ci.url + "]");
}
}
// print.out("exe child");
it = childIndexer.iterator();
while (it.hasNext()) {
ci = (ChildrenIndexer) it.next();
// print.out("exec-child:"+ci.url);
// print.out(content);
if (ci.content != null)
_parseChildren(
log,
ci.content,
writer,
root,
ci.url,
urlsDone,
extensions,
recurse,
deep,
timeout);
}
}
urls.clear();
}
// print.out("end:"+base);
}