private V3Poller makeV3Poller(String key) throws Exception { MockArchivalUnit mau = new MockArchivalUnit(); mau.setAuId("mock"); mau.getPlugin().initPlugin(daemon); PollSpec ps = new MockPollSpec(mau, "http://www.example.com/", null, null, Poll.V3_POLL); MockNodeManager nodeMgr = new MockNodeManager(); daemon.setNodeManager(nodeMgr, mau); return new V3Poller(ps, daemon, null, key, 20000, "SHA-1"); }
private Set parseSingleSource(String source) throws IOException { MockArchivalUnit mau = new MockArchivalUnit(); LinkExtractor ue = new RegexpCssLinkExtractor(); mau.setLinkExtractor("text/css", ue); MockCachedUrl mcu = new MockCachedUrl("http://www.example.com", mau); mcu.setContent(source); cb.reset(); extractor.extractUrls(mau, new StringInputStream(source), ENC, "http://www.example.com", cb); return cb.getFoundUrls(); }
public void testParsesOptionPositive() throws IOException { TypedEntryMap pMap = new TypedEntryMap(); pMap.setMapElement("html-parser-select-attrs", ListUtil.list("value")); mau.setPropertyMap(pMap); singleTagShouldParse( "http://www.example.com/web_link.jpg", "<option value=", "</option>", mau); singleTagShouldParse( "http://www.example.com/web_link.jpg", "<option a=b value=", "</option>", mau); }
protected void setUp() throws Exception { super.setUp(); TimeBase.setSimulated(); initRequiredServices(); testau.setPlugin(new MockPlugin()); initTestPeerIDs(); initTestMsg(); initTestPolls(); }
public void testStoreDamagedNodeSet() throws Exception { DamagedNodeSet damNodes = new DamagedNodeSet(mau, repository); damNodes.nodesWithDamage.add("test1"); damNodes.nodesWithDamage.add("test2"); damNodes.cusToRepair.put("cus1", ListUtil.list("cus1-1", "cus1-2")); damNodes.cusToRepair.put("cus2", ListUtil.list("cus2-1")); assertTrue(damNodes.containsWithDamage("test1")); assertTrue(damNodes.containsWithDamage("test2")); assertFalse(damNodes.containsWithDamage("test3")); repository.storeDamagedNodeSet(damNodes); String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau); filePath += HistoryRepositoryImpl.DAMAGED_NODES_FILE_NAME; File xmlFile = new File(filePath); assertTrue(xmlFile.exists()); damNodes = null; damNodes = repository.loadDamagedNodeSet(); // check damage assertTrue(damNodes.containsWithDamage("test1")); assertTrue(damNodes.containsWithDamage("test2")); assertFalse(damNodes.containsWithDamage("test3")); MockCachedUrlSet mcus1 = new MockCachedUrlSet("cus1"); MockCachedUrlSet mcus2 = new MockCachedUrlSet("cus2"); // check repairs assertTrue(damNodes.containsToRepair(mcus1, "cus1-1")); assertTrue(damNodes.containsToRepair(mcus1, "cus1-2")); assertFalse(damNodes.containsToRepair(mcus1, "cus2-1")); assertTrue(damNodes.containsToRepair(mcus2, "cus2-1")); assertEquals(mau.getAuId(), damNodes.theAu.getAuId()); // check remove damNodes.removeFromRepair(mcus1, "cus1-1"); assertFalse(damNodes.containsToRepair(mcus1, "cus1-1")); assertTrue(damNodes.containsToRepair(mcus1, "cus1-2")); damNodes.removeFromRepair(mcus1, "cus1-2"); assertFalse(damNodes.containsToRepair(mcus1, "cus1-2")); assertNull(damNodes.cusToRepair.get(mcus1)); // check remove from damaged nodes damNodes.removeFromDamage("test1"); damNodes.removeFromDamage("test2"); repository.storeDamagedNodeSet(damNodes); damNodes = repository.loadDamagedNodeSet(); assertNotNull(damNodes); assertFalse(damNodes.containsWithDamage("test1")); assertFalse(damNodes.containsWithDamage("test2")); }
public void testStoreAuEmptyState() throws Exception { HashSet strCol = new HashSet(); strCol.add("test"); AuState origState = new AuState(mau, repository); repository.storeAuState(origState); AuState loadedState = repository.loadAuState(); assertEquals(-1, loadedState.getLastCrawlTime()); assertEquals(-1, loadedState.getLastCrawlAttempt()); assertEquals(-1, loadedState.getLastCrawlResult()); assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg()); assertEquals(-1, loadedState.getLastTopLevelPollTime()); assertEquals(-1, loadedState.getLastPollStart()); assertEquals(-1, loadedState.getLastPollResult()); assertEquals(null, loadedState.getLastPollResultMsg()); assertEquals(-1, loadedState.getLastPoPPoll()); assertEquals(-1, loadedState.getLastPoPPollResult()); assertEquals(-1, loadedState.getLastLocalHashScan()); assertEquals(0, loadedState.getNumAgreePeersLastPoR()); assertEquals(0, loadedState.getNumWillingRepairers()); assertEquals(0, loadedState.getNumCurrentSuspectVersions()); assertEmpty(loadedState.getCdnStems()); loadedState.addCdnStem("http://this.is.new/"); assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems()); loadedState.addCdnStem("http://this.is.new/"); assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems()); assertEquals(0, loadedState.getPollDuration()); assertEquals(0, loadedState.getClockssSubscriptionStatus()); assertEquals(null, loadedState.getAccessType()); assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState()); assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance)); assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata)); assertEquals(-1, loadedState.getLastMetadataIndex()); assertEquals(0, loadedState.getLastContentChange()); assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId()); }
private V1NamePoll makeCompletedNamePoll(int numAgree, int numDisagree, int numDissenting) throws Exception { V1NamePoll np = null; V1LcapMessage agree_msg = null; V1LcapMessage disagree_msg1 = null; V1LcapMessage disagree_msg2 = null; Plugin plugin = testau.getPlugin(); PollSpec spec = new MockPollSpec(testau, rootV1urls[0], null, null, Poll.V1_NAME_POLL); ((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false); V1LcapMessage poll_msg = V1LcapMessage.makeRequestMsg( spec, null, ByteArray.makeRandomBytes(20), ByteArray.makeRandomBytes(20), V1LcapMessage.NAME_POLL_REQ, testduration, testID); // make our poll np = (V1NamePoll) new V1NamePoll( spec, pollmanager, poll_msg.getOriginatorId(), poll_msg.getChallenge(), poll_msg.getDuration(), poll_msg.getHashAlgorithm()); np.setMessage(poll_msg); // generate agree vote msg agree_msg = V1LcapMessage.makeReplyMsg( poll_msg, ByteArray.makeRandomBytes(20), poll_msg.getVerifier(), agree_entries, V1LcapMessage.NAME_POLL_REP, testduration, testID); // generate a disagree vote msg disagree_msg1 = V1LcapMessage.makeReplyMsg( poll_msg, ByteArray.makeRandomBytes(20), ByteArray.makeRandomBytes(20), disagree_entries, V1LcapMessage.NAME_POLL_REP, testduration, testID1); // generate a losing disagree vote msg disagree_msg2 = V1LcapMessage.makeReplyMsg( poll_msg, ByteArray.makeRandomBytes(20), ByteArray.makeRandomBytes(20), dissenting_entries, V1LcapMessage.NAME_POLL_REP, testduration, testID1); // add our vote V1LcapMessage msg = (V1LcapMessage) (np.getMessage()); PeerIdentity id = msg.getOriginatorId(); np.m_tally.addVote(np.makeNameVote(msg, true), id, true); // add the agree votes id = agree_msg.getOriginatorId(); for (int i = 0; i < numAgree; i++) { np.m_tally.addVote(np.makeNameVote(agree_msg, true), id, false); } // add the disagree votes id = disagree_msg1.getOriginatorId(); for (int i = 0; i < numDisagree; i++) { np.m_tally.addVote(np.makeNameVote(disagree_msg1, false), id, false); } // add dissenting disagree vote id = disagree_msg2.getOriginatorId(); for (int i = 0; i < numDissenting; i++) { np.m_tally.addVote(np.makeNameVote(disagree_msg2, false), id, false); } np.m_pollstate = V1Poll.PS_COMPLETE; np.m_tally.tallyVotes(); return np; }
public void testStoreOverwrite() throws Exception { AuState auState = new AuState( mau, 123, // lastCrawlTime 321, // lastCrawlAttempt -1, // lastCrawlResult null, // lastCrawlResultMsg, 321, // lastTopLevelPoll 333, // lastPollStart -1, // lastPollresult null, // lastPollresultMsg 0, // pollDuration -1, // lastTreeWalk null, // crawlUrls null, // accessType 1, // clockssSubscriptionState 1.0, // v3Agreement 1.0, // highestV3Agreement SubstanceChecker.State.Unknown, null, // substanceVersion null, // metadataVersion -1, // lastMetadataIndex 0, // lastContentChange 444, // lastPoPPoll 8, // lastPoPPollResult -1, // lastLocalHashScan 27, // numAgreePeersLastPoR 72, // numWillingRepirers 19, // numCurrentSuspectVersions ListUtil.list("http://foo/"), // cdnStems repository); repository.storeAuState(auState); String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau); filePath += HistoryRepositoryImpl.AU_FILE_NAME; File xmlFile = new File(filePath); FileInputStream fis = new FileInputStream(xmlFile); ByteArrayOutputStream baos = new ByteArrayOutputStream(); StreamUtil.copy(fis, baos); fis.close(); String expectedStr = baos.toString(); auState = new AuState( mau, 1234, // lastCrawlTime 4321, // lastCrawlAttempt -1, // lastCrawlResult null, // lastCrawlResultMsg, 4321, // lastTopLevelPoll 5555, // lastPollStart -1, // lastPollresult null, // lastPollresultMsg 0, // pollDuration -1, // lastTreeWalk null, // crawlUrls null, // accessType 1, // clockssSubscriptionState 1.0, // v3Agreement 1.0, // highestV3Agreement SubstanceChecker.State.Unknown, null, // substanceVersion null, // metadataVersion -1, // lastMetadataIndex 0, // lastContentChange -1, // lastPoPPoll -1, // lastPoPPollResult -1, // lastLocalHashScan 13, // numAgreePeersLastPoR 31, // numWillingRepairers 91, // numCurrentSuspectVersions ListUtil.list("http://foo/"), // cdnStems repository); repository.storeAuState(auState); assertEquals(1234, auState.getLastCrawlTime()); assertEquals(4321, auState.getLastCrawlAttempt()); assertEquals(4321, auState.getLastTopLevelPollTime()); assertEquals(5555, auState.getLastPollStart()); assertEquals(13, auState.getNumAgreePeersLastPoR()); assertEquals(31, auState.getNumWillingRepairers()); assertEquals(91, auState.getNumCurrentSuspectVersions()); assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId()); assertEquals(ListUtil.list("http://foo/"), auState.getCdnStems()); fis = new FileInputStream(xmlFile); baos = new ByteArrayOutputStream(expectedStr.length()); StreamUtil.copy(fis, baos); fis.close(); log.info(baos.toString()); auState = null; auState = repository.loadAuState(); assertEquals(1234, auState.getLastCrawlTime()); assertEquals(4321, auState.getLastCrawlAttempt()); assertEquals(4321, auState.getLastTopLevelPollTime()); assertEquals(5555, auState.getLastPollStart()); assertEquals(13, auState.getNumAgreePeersLastPoR()); assertEquals(31, auState.getNumWillingRepairers()); assertEquals(91, auState.getNumCurrentSuspectVersions()); assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId()); auState = new AuState( mau, 123, // lastCrawlTime 321, // lastCrawlAttempt -1, // lastCrawlResult null, // lastCrawlResultMsg, 321, // lastTopLevelPoll 333, // lastPollStart -1, // lastPollresult null, // lastPollresultMsg 0, // pollDuration -1, // lastTreeWalk null, // crawlUrls null, // accessType 1, // clockssSubscriptionState 1.0, // v3Agreement 1.0, // highestV3Agreement SubstanceChecker.State.Unknown, null, // substanceVersion null, // metadataVersion -1, // lastMetadataIndex 0, // lastContentChange 444, // lastPoPPoll 8, // lastPoPPollResult -1, // lastLocalHashScan 27, // numAgreePeersLastPoR 72, // numWillingRepairers 19, // numCurrentSuspectVersions ListUtil.list("http://foo/"), // cdnStems repository); repository.storeAuState(auState); fis = new FileInputStream(xmlFile); baos = new ByteArrayOutputStream(expectedStr.length()); StreamUtil.copy(fis, baos); fis.close(); assertEquals(expectedStr, baos.toString()); }
public void testStoreAuState() throws Exception { HashSet strCol = new HashSet(); strCol.add("test"); AuState origState = new AuState( mau, 123000, 123123, 41, "woop woop", 321000, 222000, 3, "pollres", 12345, 456000, strCol, AuState.AccessType.OpenAccess, 2, 1.0, 1.0, SubstanceChecker.State.Yes, "SubstVer3", "MetadatVer7", 111444, 12345, 111222, // lastPoPPoll 7, // lastPoPPollResult 222333, // lastLocalHashScan 444777, // numAgreePeersLastPoR 777444, // numWillingRepairers 747474, // numCurrentSuspectVersions ListUtil.list("http://hos.t/pa/th"), repository); assertEquals("SubstVer3", origState.getFeatureVersion(Plugin.Feature.Substance)); assertEquals("MetadatVer7", origState.getFeatureVersion(Plugin.Feature.Metadata)); assertEquals(111444, origState.getLastMetadataIndex()); repository.storeAuState(origState); String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau); filePath += HistoryRepositoryImpl.AU_FILE_NAME; File xmlFile = new File(filePath); assertTrue(xmlFile.exists()); origState = null; AuState loadedState = repository.loadAuState(); assertEquals(123000, loadedState.getLastCrawlTime()); assertEquals(123123, loadedState.getLastCrawlAttempt()); assertEquals(41, loadedState.getLastCrawlResult()); assertEquals("woop woop", loadedState.getLastCrawlResultMsg()); assertEquals(321000, loadedState.getLastTopLevelPollTime()); assertEquals(222000, loadedState.getLastPollStart()); assertEquals(3, loadedState.getLastPollResult()); assertEquals("Inviting Peers", loadedState.getLastPollResultMsg()); assertEquals(111222, loadedState.getLastPoPPoll()); assertEquals(7, loadedState.getLastPoPPollResult()); assertEquals(222333, loadedState.getLastLocalHashScan()); assertEquals(444777, loadedState.getNumAgreePeersLastPoR()); assertEquals(777444, loadedState.getNumWillingRepairers()); assertEquals(747474, loadedState.getNumCurrentSuspectVersions()); assertEquals(ListUtil.list("http://hos.t/pa/th"), loadedState.getCdnStems()); loadedState.addCdnStem("http://this.is.new/"); assertEquals( ListUtil.list("http://hos.t/pa/th", "http://this.is.new/"), loadedState.getCdnStems()); assertEquals(12345, loadedState.getPollDuration()); assertEquals(2, loadedState.getClockssSubscriptionStatus()); assertEquals(AuState.AccessType.OpenAccess, loadedState.getAccessType()); assertEquals(SubstanceChecker.State.Yes, loadedState.getSubstanceState()); assertEquals("SubstVer3", loadedState.getFeatureVersion(Plugin.Feature.Substance)); assertEquals("MetadatVer7", loadedState.getFeatureVersion(Plugin.Feature.Metadata)); assertEquals(111444, loadedState.getLastMetadataIndex()); assertEquals(12345, loadedState.getLastContentChange()); assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId()); // check crawl urls Collection col = loadedState.getCrawlUrls(); Iterator colIter = col.iterator(); assertTrue(colIter.hasNext()); assertEquals("test", colIter.next()); assertFalse(colIter.hasNext()); }