private void checkRules() { outputMessage("\nChecking " + m_baseUrl, TEST_SUMMARY_MESSAGE); outputMessage( "crawl depth: " + m_crawlDepth + " crawl delay: " + m_crawlDelay + " ms.", PLAIN_MESSAGE); TreeSet crawlList = new TreeSet(); TreeSet fetched = new TreeSet(); // inialize with the baseUrl crawlList.add(m_baseUrl); depth_incl = new int[m_crawlDepth]; depth_fetched = new int[m_crawlDepth]; depth_parsed = new int[m_crawlDepth]; long start_time = TimeBase.nowMs(); for (int depth = 1; depth <= m_crawlDepth; depth++) { if (isInterrupted()) { return; } m_curDepth = depth; if (crawlList.isEmpty() && depth <= m_crawlDepth) { outputMessage("\nNothing left to crawl, exiting after depth " + (depth - 1), PLAIN_MESSAGE); break; } String[] urls = (String[]) crawlList.toArray(new String[0]); crawlList.clear(); outputMessage("\nDepth " + depth, PLAIN_MESSAGE); for (int ix = 0; ix < urls.length; ix++) { if (isInterrupted()) { return; } pauseBeforeFetch(); String urlstr = urls[ix]; m_incls.clear(); m_excls.clear(); // crawl the page buildUrlSets(urlstr); fetched.add(urlstr); // output incl/excl results, // add the new_incls to the crawlList for next crawl depth loop crawlList.addAll(outputUrlResults(urlstr, m_incls, m_excls)); } } long elapsed_time = TimeBase.nowMs() - start_time; outputSummary(m_baseUrl, fetched, crawlList, elapsed_time); }
public void testCrawlStarted() throws Exception { MyAuState aus = new MyAuState(mau, historyRepo); assertEquals(-1, aus.getLastCrawlTime()); assertEquals(-1, aus.getLastCrawlAttempt()); assertEquals(-1, aus.getLastCrawlResult()); assertFalse(aus.isCrawlActive()); assertFalse(aus.hasCrawled()); assertNull(historyRepo.theAuState); assertEquals(0, historyRepo.getAuStateStoreCount()); TimeBase.setSimulated(t1); aus.newCrawlStarted(); // these should now reflect the previoud crawl, not the active one assertEquals(-1, aus.getLastCrawlTime()); assertEquals(-1, aus.getLastCrawlAttempt()); assertEquals(-1, aus.getLastCrawlResult()); assertTrue(aus.isCrawlActive()); assertFalse(aus.hasCrawled()); assertNotNull(historyRepo.theAuState); assertEquals(1, historyRepo.getAuStateStoreCount()); TimeBase.setSimulated(t2); aus.newCrawlFinished(Crawler.STATUS_ERROR, "Plorg"); assertEquals(-1, aus.getLastCrawlTime()); assertEquals(t1, aus.getLastCrawlAttempt()); assertEquals(Crawler.STATUS_ERROR, aus.getLastCrawlResult()); assertEquals("Plorg", aus.getLastCrawlResultMsg()); assertFalse(aus.isCrawlActive()); assertFalse(aus.hasCrawled()); assertEquals(2, historyRepo.getAuStateStoreCount()); TimeBase.setSimulated(t3); aus.newCrawlFinished(Crawler.STATUS_SUCCESSFUL, "Syrah"); assertEquals(t3, aus.getLastCrawlTime()); assertEquals(t1, aus.getLastCrawlAttempt()); assertEquals(Crawler.STATUS_SUCCESSFUL, aus.getLastCrawlResult()); assertEquals("Syrah", aus.getLastCrawlResultMsg()); assertFalse(aus.isCrawlActive()); assertTrue(aus.hasCrawled()); assertEquals(3, historyRepo.getAuStateStoreCount()); aus = aus.simulateStoreLoad(); assertEquals(t3, aus.getLastCrawlTime()); assertEquals(t1, aus.getLastCrawlAttempt()); assertEquals(Crawler.STATUS_SUCCESSFUL, aus.getLastCrawlResult()); assertEquals("Syrah", aus.getLastCrawlResultMsg()); assertFalse(aus.isCrawlActive()); assertTrue(aus.hasCrawled()); TimeBase.setSimulated(t4); aus.newCrawlStarted(); assertEquals(t3, aus.getLastCrawlTime()); assertEquals(t1, aus.getLastCrawlAttempt()); assertEquals(Crawler.STATUS_SUCCESSFUL, aus.getLastCrawlResult()); assertEquals("Syrah", aus.getLastCrawlResultMsg()); assertTrue(aus.hasCrawled()); }
public void testStoreNodeState() throws Exception { TimeBase.setSimulated(100); CachedUrlSet mcus = new MockCachedUrlSet(mau, new RangeCachedUrlSetSpec("http://www.example.com")); CrawlState crawl = new CrawlState(1, 2, 123); List polls = new ArrayList(2); PollState poll1 = new PollState(1, "sdf", "jkl", 2, 123, Deadline.at(456), false); PollState poll2 = new PollState(2, "abc", "def", 3, 321, Deadline.at(654), false); polls.add(poll1); polls.add(poll2); NodeState nodeState = new NodeStateImpl(mcus, 123321, crawl, polls, repository); ((NodeStateImpl) nodeState).setState(NodeState.DAMAGE_AT_OR_BELOW); repository.storeNodeState(nodeState); String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau); filePath = LockssRepositoryImpl.mapUrlToFileLocation( filePath, "http://www.example.com/" + HistoryRepositoryImpl.NODE_FILE_NAME); File xmlFile = new File(filePath); assertTrue(xmlFile.exists()); nodeState = null; nodeState = repository.loadNodeState(mcus); assertSame(mcus, nodeState.getCachedUrlSet()); assertEquals(123321, nodeState.getAverageHashDuration()); assertEquals(1, nodeState.getCrawlState().getType()); assertEquals(2, nodeState.getCrawlState().getStatus()); assertEquals(123, nodeState.getCrawlState().getStartTime()); assertEquals(NodeState.DAMAGE_AT_OR_BELOW, nodeState.getState()); Iterator pollIt = nodeState.getActivePolls(); assertTrue(pollIt.hasNext()); PollState loadedPoll = (PollState) pollIt.next(); assertEquals(1, loadedPoll.getType()); assertEquals("sdf", loadedPoll.getLwrBound()); assertEquals("jkl", loadedPoll.getUprBound()); assertEquals(2, loadedPoll.getStatus()); assertEquals(123, loadedPoll.getStartTime()); assertEquals(456, loadedPoll.getDeadline().getExpirationTime()); assertTrue(pollIt.hasNext()); loadedPoll = (PollState) pollIt.next(); assertEquals(2, loadedPoll.getType()); assertEquals("abc", loadedPoll.getLwrBound()); assertEquals("def", loadedPoll.getUprBound()); assertEquals(3, loadedPoll.getStatus()); assertEquals(321, loadedPoll.getStartTime()); assertEquals(654, loadedPoll.getDeadline().getExpirationTime()); assertFalse(pollIt.hasNext()); TimeBase.setReal(); }
public void testScheduleOurHash() { V1Poll p = testV1polls[0]; p.m_pollstate = V1Poll.PS_WAIT_HASH; // no time has elapsed - so we should be able to schedule our hash assertTrue(p.scheduleOurHash()); // half the time has elapsed so we should be able to schedule our hash TimeBase.step(p.m_deadline.getRemainingTime() / 2); assertTrue(p.scheduleOurHash()); // all of the time has elapsed we should not be able to schedule our hash TimeBase.step(p.m_deadline.getRemainingTime() - 1000); assertFalse(p.scheduleOurHash()); p.m_pollstate = V1Poll.PS_COMPLETE; }
public void testLastContentChange() { TimeBase.setSimulated(10); AuState aus = new AuState(mau, historyRepo); aus.newCrawlStarted(); TimeBase.step(10); aus.contentChanged(); assertEquals(20, aus.getLastContentChange()); TimeBase.step(10); aus.contentChanged(); assertEquals(20, aus.getLastContentChange()); TimeBase.step(10); aus.newCrawlFinished(1, "foo"); TimeBase.step(10); aus.contentChanged(); assertEquals(50, aus.getLastContentChange()); }
public void testTreeWalkFinished() { AuState auState = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo); assertEquals(123, auState.getLastTreeWalkTime()); TimeBase.setSimulated(456); auState.setLastTreeWalkTime(); assertEquals(456, auState.getLastTreeWalkTime()); }
public void lockssRun() { setPriority(PRIORITY_PARAM_SIZE_CALC, PRIORITY_DEFAULT_SIZE_CALC); startWDog(WDOG_PARAM_SIZE_CALC, WDOG_DEFAULT_SIZE_CALC); triggerWDogOnExit(true); nowRunning(); while (goOn) { try { pokeWDog(); if (sizeCalcQueue.isEmpty()) { Deadline timeout = Deadline.in(Constants.HOUR); sizeCalcSem.take(timeout); } RepositoryNode node; synchronized (sizeCalcQueue) { node = (RepositoryNode) CollectionUtil.getAnElement(sizeCalcQueue); } if (node != null) { long start = TimeBase.nowMs(); log.debug2("CalcSize start: " + node); long dur = 0; try { doSizeCalc(node); dur = TimeBase.nowMs() - start; log.debug2("CalcSize finish (" + StringUtil.timeIntervalToString(dur) + "): " + node); } catch (RuntimeException e) { log.warning("doSizeCalc: " + node, e); } synchronized (sizeCalcQueue) { sizeCalcQueue.remove(node); } pokeWDog(); long sleep = sleepTimeToAchieveLoad(dur, sizeCalcMaxLoad); Deadline.in(sleep).sleep(); } } catch (InterruptedException e) { // just wakeup and check for exit } } if (!goOn) { triggerWDogOnExit(false); } }
public void setUp() throws Exception { super.setUp(); log.setLevel(Logger.LEVEL_DEBUG); if (dig == null) { dig = MessageDigest.getInstance(hashAlgorithm); } mau = new MockArchivalUnit(new MockPlugin()); cus = new MockCachedUrlSet(mau, null); cus.setHashItSource(Collections.EMPTY_LIST); TimeBase.setSimulated(); }
protected void setUp() throws Exception { super.setUp(); TimeBase.setSimulated(); initRequiredServices(); testau.setPlugin(new MockPlugin()); initTestPeerIDs(); initTestMsg(); initTestPolls(); }
/** * tearDown method for test case * * @throws Exception if removePoll failed */ public void tearDown() throws Exception { pollmanager.stopService(); theDaemon.getLockssRepository(testau).stopService(); theDaemon.getHashService().stopService(); theDaemon.getDatagramRouterManager().stopService(); theDaemon.getRouterManager().stopService(); theDaemon.getSystemMetrics().stopService(); TimeBase.setReal(); for (int i = 0; i < testV1msg.length; i++) { if (testV1msg[i] != null) pollmanager.removePoll(testV1msg[i].getKey()); } super.tearDown(); }
// test completion & callback public void testDone() throws Exception { HashQueue q = new HashQueue(); final List cookieList = new LinkedList(); final List eList = new LinkedList(); HashService.Callback cb = new HashService.Callback() { public void hashingFinished( CachedUrlSet urlset, long timeUsed, Object cookie, CachedUrlSetHasher hasher, Exception e) { cookieList.add(cookie); eList.add(e); } }; HashQueue.Request r1, r2, r3, r4, r5; r1 = req(2000, 0, 100, cb); r2 = req(10000, 0, 200, cb); r3 = req(20000, 0, 0, cb); r4 = req(50000, 0, 1, cb); assertTrue(q.insert(r1)); assertTrue(q.insert(r2)); assertTrue(q.insert(r4)); assertEquals(0, cookieList.size()); q.removeCompleted(); assertEquals(0, cookieList.size()); // make r1 timeout r1.deadline.expire(); q.removeCompleted(); List exp = ListUtil.list(r1); assertEquals(exp, cookieList); assertEquals(exp, q.getCompletedSnapshot()); // make r2 timeout TimeBase.step(11000); // r3 is finished assertTrue(q.insert(r3)); Exception r4e = new Exception(); // make r4 error r4.e = r4e; q.removeCompleted(); // check that they all finished, and in the right order Object exp2[] = {r1, r2, r3, r4}; assertIsomorphic(exp2, cookieList); assertIsomorphic(exp2, q.getCompletedSnapshot()); // check their exceptions assertTrue(eList.get(0) instanceof HashService.Timeout); assertTrue(eList.get(1) instanceof HashService.Timeout); assertSame(null, eList.get(2)); assertSame(r4e, eList.get(3)); }
public void testDaemonCrashedDuringCrawl() throws Exception { MyAuState aus = new MyAuState(mau, historyRepo); assertEquals(-1, aus.getLastCrawlTime()); assertEquals(-1, aus.getLastCrawlAttempt()); assertEquals(-1, aus.getLastCrawlResult()); assertFalse(aus.isCrawlActive()); assertNull(historyRepo.theAuState); TimeBase.setSimulated(t1); aus.newCrawlStarted(); // these should now reflect the previoud crawl, not the active one assertEquals(-1, aus.getLastCrawlTime()); assertEquals(-1, aus.getLastCrawlAttempt()); assertEquals(-1, aus.getLastCrawlResult()); assertTrue(aus.isCrawlActive()); assertNotNull(historyRepo.theAuState); TimeBase.setSimulated(t2); aus = aus.simulateStoreLoad(); assertEquals(-1, aus.getLastCrawlTime()); assertEquals(t1, aus.getLastCrawlAttempt()); assertEquals(Crawler.STATUS_RUNNING_AT_CRASH, aus.getLastCrawlResult()); assertFalse(aus.isCrawlActive()); TimeBase.setSimulated(t3); aus.newCrawlStarted(); assertEquals(-1, aus.getLastCrawlTime()); assertEquals(t1, aus.getLastCrawlAttempt()); assertEquals(Crawler.STATUS_RUNNING_AT_CRASH, aus.getLastCrawlResult()); TimeBase.setSimulated(t4); aus.newCrawlFinished(Crawler.STATUS_SUCCESSFUL, "Plorg"); assertEquals(t4, aus.getLastCrawlTime()); assertEquals(t3, aus.getLastCrawlAttempt()); assertEquals(Crawler.STATUS_SUCCESSFUL, aus.getLastCrawlResult()); }
/* * Abbreviate the verbose constructor */ private AuState makeAuState( ArchivalUnit au, long lastCrawlTime, long lastCrawlAttempt, long lastTopLevelPoll, long lastPollStart, long lastTreeWalk, HashSet crawlUrls, int clockssSubscriptionStatus, double v3Agreement, double highestV3Agreement, HistoryRepository historyRepo) { return new AuState( au, lastCrawlTime, lastCrawlAttempt, -1, null, lastTopLevelPoll, lastPollStart, -1, null, 0, lastTreeWalk, crawlUrls, null, clockssSubscriptionStatus, v3Agreement, highestV3Agreement, SubstanceChecker.State.Unknown, null, // substanceFeatureVersion null, // metadataFeatureVersion -1, // lastMetadataIndex TimeBase.nowMs(), // lastContentChange -1, -1, -1, -1, // numWillingRepairers -1, // numCurrentSuspectVersions 0, null, historyRepo); }
public void tearDown() throws Exception { TimeBase.setReal(); super.tearDown(); }
public void testPollTimeAndResult() throws Exception { MyAuState aus = new MyAuState(mau, historyRepo); assertEquals(-1, aus.getLastTopLevelPollTime()); assertEquals(-1, aus.getLastPollStart()); assertEquals(-1, aus.getLastPollResult()); assertEquals(null, aus.getLastPollResultMsg()); assertEquals(-1, aus.getLastPoPPoll()); assertEquals(-1, aus.getLastPoPPollResult()); assertEquals(null, aus.getLastPoPPollResultMsg()); assertEquals(-1, aus.getLastLocalHashScan()); assertEquals(-1, aus.getLastTimePollCompleted()); assertEquals(0, aus.getPollDuration()); assertNull(historyRepo.theAuState); TimeBase.setSimulated(t1); aus.pollStarted(); // running poll assertEquals(t1, aus.getLastPollStart()); // These haven't been updated yet assertEquals(-1, aus.getLastTopLevelPollTime()); assertEquals(-1, aus.getLastPollResult()); assertEquals(0, aus.getPollDuration()); assertEquals(-1, aus.getLastTimePollCompleted()); assertNotNull(historyRepo.theAuState); TimeBase.setSimulated(t2); aus.pollFinished(V3Poller.POLLER_STATUS_ERROR, PollVariant.PoR); assertEquals(-1, aus.getLastTopLevelPollTime()); assertEquals(t1, aus.getLastPollStart()); assertEquals(V3Poller.POLLER_STATUS_ERROR, aus.getLastPollResult()); assertEquals("Error", aus.getLastPollResultMsg()); assertEquals(t2, aus.getPollDuration()); assertEquals(-1, aus.getLastPoPPoll()); assertEquals(-1, aus.getLastPoPPollResult()); assertEquals(null, aus.getLastPoPPollResultMsg()); assertEquals(-1, aus.getLastLocalHashScan()); assertEquals(-1, aus.getLastTimePollCompleted()); TimeBase.setSimulated(t3); aus.pollFinished(V3Poller.POLLER_STATUS_COMPLETE, PollVariant.PoR); assertEquals(t3, aus.getLastTopLevelPollTime()); assertEquals(t1, aus.getLastPollStart()); assertEquals(V3Poller.POLLER_STATUS_COMPLETE, aus.getLastPollResult()); assertEquals("Complete", aus.getLastPollResultMsg()); assertEquals((t3 + t2) / 2, aus.getPollDuration()); assertEquals(-1, aus.getLastPoPPoll()); assertEquals(-1, aus.getLastPoPPollResult()); assertEquals(null, aus.getLastPoPPollResultMsg()); assertEquals(-1, aus.getLastLocalHashScan()); assertEquals(t3, aus.getLastTimePollCompleted()); TimeBase.setSimulated(t4); aus.pollFinished(V3Poller.POLLER_STATUS_NO_QUORUM, PollVariant.PoP); assertEquals(t3, aus.getLastTopLevelPollTime()); assertEquals(t1, aus.getLastPollStart()); assertEquals(V3Poller.POLLER_STATUS_COMPLETE, aus.getLastPollResult()); assertEquals("Complete", aus.getLastPollResultMsg()); assertEquals((t3 + t2) / 2, aus.getPollDuration()); assertEquals(-1, aus.getLastPoPPoll()); assertEquals(V3Poller.POLLER_STATUS_NO_QUORUM, aus.getLastPoPPollResult()); assertEquals(-1, aus.getLastLocalHashScan()); assertEquals("No Quorum", aus.getLastPoPPollResultMsg()); assertEquals(t3, aus.getLastTimePollCompleted()); TimeBase.setSimulated(t5); aus.pollFinished(V3Poller.POLLER_STATUS_COMPLETE, PollVariant.PoP); assertEquals(t3, aus.getLastTopLevelPollTime()); assertEquals(t1, aus.getLastPollStart()); assertEquals(V3Poller.POLLER_STATUS_COMPLETE, aus.getLastPollResult()); assertEquals("Complete", aus.getLastPollResultMsg()); assertEquals((t3 + t2) / 2, aus.getPollDuration()); assertEquals(t5, aus.getLastPoPPoll()); assertEquals(V3Poller.POLLER_STATUS_COMPLETE, aus.getLastPoPPollResult()); assertEquals("Complete", aus.getLastPoPPollResultMsg()); assertEquals(-1, aus.getLastLocalHashScan()); assertEquals(t5, aus.getLastTimePollCompleted()); aus.pollFinished(V3Poller.POLLER_STATUS_NO_QUORUM, PollVariant.PoP); TimeBase.setSimulated(t6); aus.pollFinished(V3Poller.POLLER_STATUS_COMPLETE, PollVariant.Local); assertEquals(t3, aus.getLastTopLevelPollTime()); assertEquals(t1, aus.getLastPollStart()); assertEquals(V3Poller.POLLER_STATUS_COMPLETE, aus.getLastPollResult()); assertEquals("Complete", aus.getLastPollResultMsg()); assertEquals((t3 + t2) / 2, aus.getPollDuration()); assertEquals(t5, aus.getLastPoPPoll()); assertEquals(V3Poller.POLLER_STATUS_NO_QUORUM, aus.getLastPoPPollResult()); assertEquals("No Quorum", aus.getLastPoPPollResultMsg()); assertEquals(t6, aus.getLastLocalHashScan()); assertEquals(t5, aus.getLastTimePollCompleted()); aus = aus.simulateStoreLoad(); assertEquals(t3, aus.getLastTopLevelPollTime()); assertEquals(t1, aus.getLastPollStart()); assertEquals(V3Poller.POLLER_STATUS_COMPLETE, aus.getLastPollResult()); TimeBase.setSimulated(t7); aus.pollStarted(); assertEquals(t3, aus.getLastTopLevelPollTime()); assertEquals(t7, aus.getLastPollStart()); assertEquals(V3Poller.POLLER_STATUS_COMPLETE, aus.getLastPollResult()); }
public void testStorePollHistories() throws Exception { TimeBase.setSimulated(123321); MockCachedUrlSetSpec mspec = new MockCachedUrlSetSpec("http://www.example.com", null); CachedUrlSet mcus = new MockCachedUrlSet(mau, mspec); NodeStateImpl nodeState = new NodeStateImpl(mcus, -1, null, null, repository); List histories = ListUtil.list( createPollHistoryBean(3), createPollHistoryBean(3), createPollHistoryBean(3), createPollHistoryBean(3), createPollHistoryBean(3)); /* * CASTOR: [summary] Rewrite test in non-Castor way * This is obviously not an appropriate way of writing this test, * Right now it creates sample data in Castor format, from legacy * code back when Castor was the built-in serialization engine. * TODO: Rewrite test in non-Castor way */ // nodeState.setPollHistoryBeanList(histories); nodeState.setPollHistoryList(NodeHistoryBean.fromBeanListToList(histories)); repository.storePollHistories(nodeState); String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau); filePath = LockssRepositoryImpl.mapUrlToFileLocation( filePath, "http://www.example.com/" + HistoryRepositoryImpl.HISTORY_FILE_NAME); File xmlFile = new File(filePath); assertTrue(xmlFile.exists()); nodeState.setPollHistoryList(new ArrayList()); repository.loadPollHistories(nodeState); List loadedHistory = nodeState.getPollHistoryList(); assertEquals(histories.size(), loadedHistory.size()); // CASTOR: some Castor-tailored stuff here // PollHistoryBean expect1 = (PollHistoryBean)histories.get(0); // PollHistoryBean elem1 = (PollHistoryBean)loadedHistory.get(0); PollHistory expect1 = (PollHistory) histories.get(0); PollHistory elem1 = (PollHistory) loadedHistory.get(0); assertEquals(expect1.type, elem1.type); assertEquals(expect1.lwrBound, elem1.lwrBound); assertEquals(expect1.uprBound, elem1.uprBound); assertEquals(expect1.status, elem1.status); assertEquals(expect1.startTime, elem1.startTime); assertEquals(expect1.duration, elem1.duration); // CASTOR: some Castor-tailored stuff here // List expectBeans = (List)expect1.getVoteBeans(); // List elemBeans = (List)elem1.getVoteBeans(); Iterator expectIter = (Iterator) expect1.getVotes(); Iterator elemIter = (Iterator) elem1.getVotes(); while (expectIter.hasNext() && elemIter.hasNext()) { Vote expectVote = (Vote) expectIter.next(); Vote elemVote = (Vote) elemIter.next(); assertEquals( expectVote.getVoterIdentity().getIdString(), elemVote.getVoterIdentity().getIdString()); assertEquals(expectVote.isAgreeVote(), elemVote.isAgreeVote()); assertEquals(expectVote.getChallengeString(), elemVote.getChallengeString()); assertEquals(expectVote.getVerifierString(), elemVote.getVerifierString()); assertEquals(expectVote.getHashString(), elemVote.getHashString()); } assertFalse(expectIter.hasNext()); assertFalse(expectIter.hasNext()); TimeBase.setReal(); }