private void checkRules() { outputMessage("\nChecking " + m_baseUrl, TEST_SUMMARY_MESSAGE); outputMessage( "crawl depth: " + m_crawlDepth + " crawl delay: " + m_crawlDelay + " ms.", PLAIN_MESSAGE); TreeSet crawlList = new TreeSet(); TreeSet fetched = new TreeSet(); // inialize with the baseUrl crawlList.add(m_baseUrl); depth_incl = new int[m_crawlDepth]; depth_fetched = new int[m_crawlDepth]; depth_parsed = new int[m_crawlDepth]; long start_time = TimeBase.nowMs(); for (int depth = 1; depth <= m_crawlDepth; depth++) { if (isInterrupted()) { return; } m_curDepth = depth; if (crawlList.isEmpty() && depth <= m_crawlDepth) { outputMessage("\nNothing left to crawl, exiting after depth " + (depth - 1), PLAIN_MESSAGE); break; } String[] urls = (String[]) crawlList.toArray(new String[0]); crawlList.clear(); outputMessage("\nDepth " + depth, PLAIN_MESSAGE); for (int ix = 0; ix < urls.length; ix++) { if (isInterrupted()) { return; } pauseBeforeFetch(); String urlstr = urls[ix]; m_incls.clear(); m_excls.clear(); // crawl the page buildUrlSets(urlstr); fetched.add(urlstr); // output incl/excl results, // add the new_incls to the crawlList for next crawl depth loop crawlList.addAll(outputUrlResults(urlstr, m_incls, m_excls)); } } long elapsed_time = TimeBase.nowMs() - start_time; outputSummary(m_baseUrl, fetched, crawlList, elapsed_time); }
public void testPollDuration() throws Exception { TimeBase.setSimulated(TimeBase.nowMs()); V3LcapMessage src = this.makePollMessage(6 * Constants.WEEK); InputStream srcStream = src.getInputStream(); V3LcapMessage copy = new V3LcapMessage(srcStream, tempDir, theDaemon); assertEqualMessages(src, copy); assertEquals(6 * Constants.WEEK, copy.getDuration()); }
public void lockssRun() { setPriority(PRIORITY_PARAM_SIZE_CALC, PRIORITY_DEFAULT_SIZE_CALC); startWDog(WDOG_PARAM_SIZE_CALC, WDOG_DEFAULT_SIZE_CALC); triggerWDogOnExit(true); nowRunning(); while (goOn) { try { pokeWDog(); if (sizeCalcQueue.isEmpty()) { Deadline timeout = Deadline.in(Constants.HOUR); sizeCalcSem.take(timeout); } RepositoryNode node; synchronized (sizeCalcQueue) { node = (RepositoryNode) CollectionUtil.getAnElement(sizeCalcQueue); } if (node != null) { long start = TimeBase.nowMs(); log.debug2("CalcSize start: " + node); long dur = 0; try { doSizeCalc(node); dur = TimeBase.nowMs() - start; log.debug2("CalcSize finish (" + StringUtil.timeIntervalToString(dur) + "): " + node); } catch (RuntimeException e) { log.warning("doSizeCalc: " + node, e); } synchronized (sizeCalcQueue) { sizeCalcQueue.remove(node); } pokeWDog(); long sleep = sleepTimeToAchieveLoad(dur, sizeCalcMaxLoad); Deadline.in(sleep).sleep(); } } catch (InterruptedException e) { // just wakeup and check for exit } } if (!goOn) { triggerWDogOnExit(false); } }
private V3LcapMessage makePollMessage(long duration) { V3LcapMessage msg = new V3LcapMessage( "ArchivalID_2", "key", "Plug42", m_testBytes, m_testBytes, V3LcapMessage.MSG_POLL, TimeBase.nowMs() + duration, m_testID, tempDir, theDaemon); return msg; }
/** Common request handling. */ public void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { resetState(); boolean success = false; HttpSession session = req.getSession(false); try { this.req = req; this.resp = resp; if (log.isDebug()) { logParams(); } resp.setContentType("text/html"); if (!mayPageBeCached()) { resp.setHeader("pragma", "no-cache"); resp.setHeader("Cache-control", "no-cache"); } reqURL = new URL(UrlUtil.getRequestURL(req)); clientAddr = getLocalIPAddr(); // check that current user has permission to run this servlet if (!isServletAllowed(myServletDescr())) { displayWarningInLieuOfPage("You are not authorized to use " + myServletDescr().heading); return; } // check whether servlet is disabled String reason = ServletUtil.servletDisabledReason(myServletDescr().getServletName()); if (reason != null) { displayWarningInLieuOfPage("This function is disabled. " + reason); return; } if (session != null) { session.setAttribute(SESSION_KEY_RUNNING_SERVLET, getHeading()); String reqHost = req.getRemoteHost(); String forw = req.getHeader(HttpFields.__XForwardedFor); if (!StringUtil.isNullString(forw)) { reqHost += " (proxies for " + forw + ")"; } session.setAttribute(SESSION_KEY_REQUEST_HOST, reqHost); } lockssHandleRequest(); success = (errMsg == null); } catch (ServletException e) { log.error("Servlet threw", e); throw e; } catch (IOException e) { log.error("Servlet threw", e); throw e; } catch (RuntimeException e) { log.error("Servlet threw", e); throw e; } finally { if (session != null) { session.setAttribute(SESSION_KEY_RUNNING_SERVLET, null); session.setAttribute(LockssFormAuthenticator.__J_AUTH_ACTIVITY, TimeBase.nowMs()); } if ("please".equalsIgnoreCase(req.getHeader("X-Lockss-Result"))) { log.debug3("X-Lockss-Result: " + (success ? "Ok" : "Fail")); resp.setHeader("X-Lockss-Result", success ? "Ok" : "Fail"); } resetMyLocals(); resetLocals(); } }