private void checkRules() {
outputMessage("\nChecking " + m_baseUrl, TEST_SUMMARY_MESSAGE);
outputMessage(
"crawl depth: " + m_crawlDepth + " crawl delay: " + m_crawlDelay + " ms.",
PLAIN_MESSAGE);
TreeSet crawlList = new TreeSet();
TreeSet fetched = new TreeSet();
// inialize with the baseUrl
crawlList.add(m_baseUrl);
depth_incl = new int[m_crawlDepth];
depth_fetched = new int[m_crawlDepth];
depth_parsed = new int[m_crawlDepth];
long start_time = TimeBase.nowMs();
for (int depth = 1; depth <= m_crawlDepth; depth++) {
if (isInterrupted()) {
return;
}
m_curDepth = depth;
if (crawlList.isEmpty() && depth <= m_crawlDepth) {
outputMessage("\nNothing left to crawl, exiting after depth " + (depth - 1), PLAIN_MESSAGE);
break;
}
String[] urls = (String[]) crawlList.toArray(new String[0]);
crawlList.clear();
outputMessage("\nDepth " + depth, PLAIN_MESSAGE);
for (int ix = 0; ix < urls.length; ix++) {
if (isInterrupted()) {
return;
}
pauseBeforeFetch();
String urlstr = urls[ix];
m_incls.clear();
m_excls.clear();
// crawl the page
buildUrlSets(urlstr);
fetched.add(urlstr);
// output incl/excl results,
// add the new_incls to the crawlList for next crawl depth loop
crawlList.addAll(outputUrlResults(urlstr, m_incls, m_excls));
}
}
long elapsed_time = TimeBase.nowMs() - start_time;
outputSummary(m_baseUrl, fetched, crawlList, elapsed_time);
}
public void testPollDuration() throws Exception {
TimeBase.setSimulated(TimeBase.nowMs());
V3LcapMessage src = this.makePollMessage(6 * Constants.WEEK);
InputStream srcStream = src.getInputStream();
V3LcapMessage copy = new V3LcapMessage(srcStream, tempDir, theDaemon);
assertEqualMessages(src, copy);
assertEquals(6 * Constants.WEEK, copy.getDuration());
}
public void lockssRun() {
setPriority(PRIORITY_PARAM_SIZE_CALC, PRIORITY_DEFAULT_SIZE_CALC);
startWDog(WDOG_PARAM_SIZE_CALC, WDOG_DEFAULT_SIZE_CALC);
triggerWDogOnExit(true);
nowRunning();
while (goOn) {
try {
pokeWDog();
if (sizeCalcQueue.isEmpty()) {
Deadline timeout = Deadline.in(Constants.HOUR);
sizeCalcSem.take(timeout);
}
RepositoryNode node;
synchronized (sizeCalcQueue) {
node = (RepositoryNode) CollectionUtil.getAnElement(sizeCalcQueue);
}
if (node != null) {
long start = TimeBase.nowMs();
log.debug2("CalcSize start: " + node);
long dur = 0;
try {
doSizeCalc(node);
dur = TimeBase.nowMs() - start;
log.debug2("CalcSize finish (" + StringUtil.timeIntervalToString(dur) + "): " + node);
} catch (RuntimeException e) {
log.warning("doSizeCalc: " + node, e);
}
synchronized (sizeCalcQueue) {
sizeCalcQueue.remove(node);
}
pokeWDog();
long sleep = sleepTimeToAchieveLoad(dur, sizeCalcMaxLoad);
Deadline.in(sleep).sleep();
}
} catch (InterruptedException e) {
// just wakeup and check for exit
}
}
if (!goOn) {
triggerWDogOnExit(false);
}
}
private V3LcapMessage makePollMessage(long duration) {
V3LcapMessage msg =
new V3LcapMessage(
"ArchivalID_2",
"key",
"Plug42",
m_testBytes,
m_testBytes,
V3LcapMessage.MSG_POLL,
TimeBase.nowMs() + duration,
m_testID,
tempDir,
theDaemon);
return msg;
}
/** Common request handling. */
public void service(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
resetState();
boolean success = false;
HttpSession session = req.getSession(false);
try {
this.req = req;
this.resp = resp;
if (log.isDebug()) {
logParams();
}
resp.setContentType("text/html");
if (!mayPageBeCached()) {
resp.setHeader("pragma", "no-cache");
resp.setHeader("Cache-control", "no-cache");
}
reqURL = new URL(UrlUtil.getRequestURL(req));
clientAddr = getLocalIPAddr();
// check that current user has permission to run this servlet
if (!isServletAllowed(myServletDescr())) {
displayWarningInLieuOfPage("You are not authorized to use " + myServletDescr().heading);
return;
}
// check whether servlet is disabled
String reason = ServletUtil.servletDisabledReason(myServletDescr().getServletName());
if (reason != null) {
displayWarningInLieuOfPage("This function is disabled. " + reason);
return;
}
if (session != null) {
session.setAttribute(SESSION_KEY_RUNNING_SERVLET, getHeading());
String reqHost = req.getRemoteHost();
String forw = req.getHeader(HttpFields.__XForwardedFor);
if (!StringUtil.isNullString(forw)) {
reqHost += " (proxies for " + forw + ")";
}
session.setAttribute(SESSION_KEY_REQUEST_HOST, reqHost);
}
lockssHandleRequest();
success = (errMsg == null);
} catch (ServletException e) {
log.error("Servlet threw", e);
throw e;
} catch (IOException e) {
log.error("Servlet threw", e);
throw e;
} catch (RuntimeException e) {
log.error("Servlet threw", e);
throw e;
} finally {
if (session != null) {
session.setAttribute(SESSION_KEY_RUNNING_SERVLET, null);
session.setAttribute(LockssFormAuthenticator.__J_AUTH_ACTIVITY, TimeBase.nowMs());
}
if ("please".equalsIgnoreCase(req.getHeader("X-Lockss-Result"))) {
log.debug3("X-Lockss-Result: " + (success ? "Ok" : "Fail"));
resp.setHeader("X-Lockss-Result", success ? "Ok" : "Fail");
}
resetMyLocals();
resetLocals();
}
}