/**
* Make a basic TaylorAndFrancis test AU to which URLs can be added.
*
* @return a basic Ingenta test AU
* @throws ConfigurationException if can't set configuration
*/
MockArchivalUnit makeAu() throws ConfigurationException {
MockArchivalUnit mau = new MockArchivalUnit();
Configuration config = ConfigurationUtil.fromArgs("base_url", "http://www.xyz.com/");
mau.setConfiguration(config);
mau.setUrlStems(ListUtil.list("http://www.xyz.com/"));
return mau;
}
private V3Poller makeV3Poller(String key) throws Exception {
MockArchivalUnit mau = new MockArchivalUnit();
mau.setAuId("mock");
mau.getPlugin().initPlugin(daemon);
PollSpec ps = new MockPollSpec(mau, "http://www.example.com/", null, null, Poll.V3_POLL);
MockNodeManager nodeMgr = new MockNodeManager();
daemon.setNodeManager(nodeMgr, mau);
return new V3Poller(ps, daemon, null, key, 20000, "SHA-1");
}
private Set parseSingleSource(String source) throws IOException {
MockArchivalUnit mau = new MockArchivalUnit();
LinkExtractor ue = new RegexpCssLinkExtractor();
mau.setLinkExtractor("text/css", ue);
MockCachedUrl mcu = new MockCachedUrl("http://www.example.com", mau);
mcu.setContent(source);
cb.reset();
extractor.extractUrls(mau, new StringInputStream(source), ENC, "http://www.example.com", cb);
return cb.getFoundUrls();
}
public void testParsesOptionPositive() throws IOException {
TypedEntryMap pMap = new TypedEntryMap();
pMap.setMapElement("html-parser-select-attrs", ListUtil.list("value"));
mau.setPropertyMap(pMap);
singleTagShouldParse(
"http://www.example.com/web_link.jpg", "<option value=", "</option>", mau);
singleTagShouldParse(
"http://www.example.com/web_link.jpg", "<option a=b value=", "</option>", mau);
}
protected void setUp() throws Exception {
super.setUp();
TimeBase.setSimulated();
initRequiredServices();
testau.setPlugin(new MockPlugin());
initTestPeerIDs();
initTestMsg();
initTestPolls();
}
public void testStoreDamagedNodeSet() throws Exception {
DamagedNodeSet damNodes = new DamagedNodeSet(mau, repository);
damNodes.nodesWithDamage.add("test1");
damNodes.nodesWithDamage.add("test2");
damNodes.cusToRepair.put("cus1", ListUtil.list("cus1-1", "cus1-2"));
damNodes.cusToRepair.put("cus2", ListUtil.list("cus2-1"));
assertTrue(damNodes.containsWithDamage("test1"));
assertTrue(damNodes.containsWithDamage("test2"));
assertFalse(damNodes.containsWithDamage("test3"));
repository.storeDamagedNodeSet(damNodes);
String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
filePath += HistoryRepositoryImpl.DAMAGED_NODES_FILE_NAME;
File xmlFile = new File(filePath);
assertTrue(xmlFile.exists());
damNodes = null;
damNodes = repository.loadDamagedNodeSet();
// check damage
assertTrue(damNodes.containsWithDamage("test1"));
assertTrue(damNodes.containsWithDamage("test2"));
assertFalse(damNodes.containsWithDamage("test3"));
MockCachedUrlSet mcus1 = new MockCachedUrlSet("cus1");
MockCachedUrlSet mcus2 = new MockCachedUrlSet("cus2");
// check repairs
assertTrue(damNodes.containsToRepair(mcus1, "cus1-1"));
assertTrue(damNodes.containsToRepair(mcus1, "cus1-2"));
assertFalse(damNodes.containsToRepair(mcus1, "cus2-1"));
assertTrue(damNodes.containsToRepair(mcus2, "cus2-1"));
assertEquals(mau.getAuId(), damNodes.theAu.getAuId());
// check remove
damNodes.removeFromRepair(mcus1, "cus1-1");
assertFalse(damNodes.containsToRepair(mcus1, "cus1-1"));
assertTrue(damNodes.containsToRepair(mcus1, "cus1-2"));
damNodes.removeFromRepair(mcus1, "cus1-2");
assertFalse(damNodes.containsToRepair(mcus1, "cus1-2"));
assertNull(damNodes.cusToRepair.get(mcus1));
// check remove from damaged nodes
damNodes.removeFromDamage("test1");
damNodes.removeFromDamage("test2");
repository.storeDamagedNodeSet(damNodes);
damNodes = repository.loadDamagedNodeSet();
assertNotNull(damNodes);
assertFalse(damNodes.containsWithDamage("test1"));
assertFalse(damNodes.containsWithDamage("test2"));
}
public void testStoreAuEmptyState() throws Exception {
HashSet strCol = new HashSet();
strCol.add("test");
AuState origState = new AuState(mau, repository);
repository.storeAuState(origState);
AuState loadedState = repository.loadAuState();
assertEquals(-1, loadedState.getLastCrawlTime());
assertEquals(-1, loadedState.getLastCrawlAttempt());
assertEquals(-1, loadedState.getLastCrawlResult());
assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg());
assertEquals(-1, loadedState.getLastTopLevelPollTime());
assertEquals(-1, loadedState.getLastPollStart());
assertEquals(-1, loadedState.getLastPollResult());
assertEquals(null, loadedState.getLastPollResultMsg());
assertEquals(-1, loadedState.getLastPoPPoll());
assertEquals(-1, loadedState.getLastPoPPollResult());
assertEquals(-1, loadedState.getLastLocalHashScan());
assertEquals(0, loadedState.getNumAgreePeersLastPoR());
assertEquals(0, loadedState.getNumWillingRepairers());
assertEquals(0, loadedState.getNumCurrentSuspectVersions());
assertEmpty(loadedState.getCdnStems());
loadedState.addCdnStem("http://this.is.new/");
assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
loadedState.addCdnStem("http://this.is.new/");
assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
assertEquals(0, loadedState.getPollDuration());
assertEquals(0, loadedState.getClockssSubscriptionStatus());
assertEquals(null, loadedState.getAccessType());
assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState());
assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance));
assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata));
assertEquals(-1, loadedState.getLastMetadataIndex());
assertEquals(0, loadedState.getLastContentChange());
assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());
}
private V1NamePoll makeCompletedNamePoll(int numAgree, int numDisagree, int numDissenting)
throws Exception {
V1NamePoll np = null;
V1LcapMessage agree_msg = null;
V1LcapMessage disagree_msg1 = null;
V1LcapMessage disagree_msg2 = null;
Plugin plugin = testau.getPlugin();
PollSpec spec = new MockPollSpec(testau, rootV1urls[0], null, null, Poll.V1_NAME_POLL);
((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
V1LcapMessage poll_msg =
V1LcapMessage.makeRequestMsg(
spec,
null,
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
V1LcapMessage.NAME_POLL_REQ,
testduration,
testID);
// make our poll
np =
(V1NamePoll)
new V1NamePoll(
spec,
pollmanager,
poll_msg.getOriginatorId(),
poll_msg.getChallenge(),
poll_msg.getDuration(),
poll_msg.getHashAlgorithm());
np.setMessage(poll_msg);
// generate agree vote msg
agree_msg =
V1LcapMessage.makeReplyMsg(
poll_msg,
ByteArray.makeRandomBytes(20),
poll_msg.getVerifier(),
agree_entries,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID);
// generate a disagree vote msg
disagree_msg1 =
V1LcapMessage.makeReplyMsg(
poll_msg,
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
disagree_entries,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID1);
// generate a losing disagree vote msg
disagree_msg2 =
V1LcapMessage.makeReplyMsg(
poll_msg,
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
dissenting_entries,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID1);
// add our vote
V1LcapMessage msg = (V1LcapMessage) (np.getMessage());
PeerIdentity id = msg.getOriginatorId();
np.m_tally.addVote(np.makeNameVote(msg, true), id, true);
// add the agree votes
id = agree_msg.getOriginatorId();
for (int i = 0; i < numAgree; i++) {
np.m_tally.addVote(np.makeNameVote(agree_msg, true), id, false);
}
// add the disagree votes
id = disagree_msg1.getOriginatorId();
for (int i = 0; i < numDisagree; i++) {
np.m_tally.addVote(np.makeNameVote(disagree_msg1, false), id, false);
}
// add dissenting disagree vote
id = disagree_msg2.getOriginatorId();
for (int i = 0; i < numDissenting; i++) {
np.m_tally.addVote(np.makeNameVote(disagree_msg2, false), id, false);
}
np.m_pollstate = V1Poll.PS_COMPLETE;
np.m_tally.tallyVotes();
return np;
}
public void testStoreOverwrite() throws Exception {
AuState auState =
new AuState(
mau,
123, // lastCrawlTime
321, // lastCrawlAttempt
-1, // lastCrawlResult
null, // lastCrawlResultMsg,
321, // lastTopLevelPoll
333, // lastPollStart
-1, // lastPollresult
null, // lastPollresultMsg
0, // pollDuration
-1, // lastTreeWalk
null, // crawlUrls
null, // accessType
1, // clockssSubscriptionState
1.0, // v3Agreement
1.0, // highestV3Agreement
SubstanceChecker.State.Unknown,
null, // substanceVersion
null, // metadataVersion
-1, // lastMetadataIndex
0, // lastContentChange
444, // lastPoPPoll
8, // lastPoPPollResult
-1, // lastLocalHashScan
27, // numAgreePeersLastPoR
72, // numWillingRepirers
19, // numCurrentSuspectVersions
ListUtil.list("http://foo/"), // cdnStems
repository);
repository.storeAuState(auState);
String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
filePath += HistoryRepositoryImpl.AU_FILE_NAME;
File xmlFile = new File(filePath);
FileInputStream fis = new FileInputStream(xmlFile);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
StreamUtil.copy(fis, baos);
fis.close();
String expectedStr = baos.toString();
auState =
new AuState(
mau,
1234, // lastCrawlTime
4321, // lastCrawlAttempt
-1, // lastCrawlResult
null, // lastCrawlResultMsg,
4321, // lastTopLevelPoll
5555, // lastPollStart
-1, // lastPollresult
null, // lastPollresultMsg
0, // pollDuration
-1, // lastTreeWalk
null, // crawlUrls
null, // accessType
1, // clockssSubscriptionState
1.0, // v3Agreement
1.0, // highestV3Agreement
SubstanceChecker.State.Unknown,
null, // substanceVersion
null, // metadataVersion
-1, // lastMetadataIndex
0, // lastContentChange
-1, // lastPoPPoll
-1, // lastPoPPollResult
-1, // lastLocalHashScan
13, // numAgreePeersLastPoR
31, // numWillingRepairers
91, // numCurrentSuspectVersions
ListUtil.list("http://foo/"), // cdnStems
repository);
repository.storeAuState(auState);
assertEquals(1234, auState.getLastCrawlTime());
assertEquals(4321, auState.getLastCrawlAttempt());
assertEquals(4321, auState.getLastTopLevelPollTime());
assertEquals(5555, auState.getLastPollStart());
assertEquals(13, auState.getNumAgreePeersLastPoR());
assertEquals(31, auState.getNumWillingRepairers());
assertEquals(91, auState.getNumCurrentSuspectVersions());
assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId());
assertEquals(ListUtil.list("http://foo/"), auState.getCdnStems());
fis = new FileInputStream(xmlFile);
baos = new ByteArrayOutputStream(expectedStr.length());
StreamUtil.copy(fis, baos);
fis.close();
log.info(baos.toString());
auState = null;
auState = repository.loadAuState();
assertEquals(1234, auState.getLastCrawlTime());
assertEquals(4321, auState.getLastCrawlAttempt());
assertEquals(4321, auState.getLastTopLevelPollTime());
assertEquals(5555, auState.getLastPollStart());
assertEquals(13, auState.getNumAgreePeersLastPoR());
assertEquals(31, auState.getNumWillingRepairers());
assertEquals(91, auState.getNumCurrentSuspectVersions());
assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId());
auState =
new AuState(
mau,
123, // lastCrawlTime
321, // lastCrawlAttempt
-1, // lastCrawlResult
null, // lastCrawlResultMsg,
321, // lastTopLevelPoll
333, // lastPollStart
-1, // lastPollresult
null, // lastPollresultMsg
0, // pollDuration
-1, // lastTreeWalk
null, // crawlUrls
null, // accessType
1, // clockssSubscriptionState
1.0, // v3Agreement
1.0, // highestV3Agreement
SubstanceChecker.State.Unknown,
null, // substanceVersion
null, // metadataVersion
-1, // lastMetadataIndex
0, // lastContentChange
444, // lastPoPPoll
8, // lastPoPPollResult
-1, // lastLocalHashScan
27, // numAgreePeersLastPoR
72, // numWillingRepairers
19, // numCurrentSuspectVersions
ListUtil.list("http://foo/"), // cdnStems
repository);
repository.storeAuState(auState);
fis = new FileInputStream(xmlFile);
baos = new ByteArrayOutputStream(expectedStr.length());
StreamUtil.copy(fis, baos);
fis.close();
assertEquals(expectedStr, baos.toString());
}
public void testStoreAuState() throws Exception {
HashSet strCol = new HashSet();
strCol.add("test");
AuState origState =
new AuState(
mau,
123000,
123123,
41,
"woop woop",
321000,
222000,
3,
"pollres",
12345,
456000,
strCol,
AuState.AccessType.OpenAccess,
2,
1.0,
1.0,
SubstanceChecker.State.Yes,
"SubstVer3",
"MetadatVer7",
111444,
12345,
111222, // lastPoPPoll
7, // lastPoPPollResult
222333, // lastLocalHashScan
444777, // numAgreePeersLastPoR
777444, // numWillingRepairers
747474, // numCurrentSuspectVersions
ListUtil.list("http://hos.t/pa/th"),
repository);
assertEquals("SubstVer3", origState.getFeatureVersion(Plugin.Feature.Substance));
assertEquals("MetadatVer7", origState.getFeatureVersion(Plugin.Feature.Metadata));
assertEquals(111444, origState.getLastMetadataIndex());
repository.storeAuState(origState);
String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
filePath += HistoryRepositoryImpl.AU_FILE_NAME;
File xmlFile = new File(filePath);
assertTrue(xmlFile.exists());
origState = null;
AuState loadedState = repository.loadAuState();
assertEquals(123000, loadedState.getLastCrawlTime());
assertEquals(123123, loadedState.getLastCrawlAttempt());
assertEquals(41, loadedState.getLastCrawlResult());
assertEquals("woop woop", loadedState.getLastCrawlResultMsg());
assertEquals(321000, loadedState.getLastTopLevelPollTime());
assertEquals(222000, loadedState.getLastPollStart());
assertEquals(3, loadedState.getLastPollResult());
assertEquals("Inviting Peers", loadedState.getLastPollResultMsg());
assertEquals(111222, loadedState.getLastPoPPoll());
assertEquals(7, loadedState.getLastPoPPollResult());
assertEquals(222333, loadedState.getLastLocalHashScan());
assertEquals(444777, loadedState.getNumAgreePeersLastPoR());
assertEquals(777444, loadedState.getNumWillingRepairers());
assertEquals(747474, loadedState.getNumCurrentSuspectVersions());
assertEquals(ListUtil.list("http://hos.t/pa/th"), loadedState.getCdnStems());
loadedState.addCdnStem("http://this.is.new/");
assertEquals(
ListUtil.list("http://hos.t/pa/th", "http://this.is.new/"), loadedState.getCdnStems());
assertEquals(12345, loadedState.getPollDuration());
assertEquals(2, loadedState.getClockssSubscriptionStatus());
assertEquals(AuState.AccessType.OpenAccess, loadedState.getAccessType());
assertEquals(SubstanceChecker.State.Yes, loadedState.getSubstanceState());
assertEquals("SubstVer3", loadedState.getFeatureVersion(Plugin.Feature.Substance));
assertEquals("MetadatVer7", loadedState.getFeatureVersion(Plugin.Feature.Metadata));
assertEquals(111444, loadedState.getLastMetadataIndex());
assertEquals(12345, loadedState.getLastContentChange());
assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());
// check crawl urls
Collection col = loadedState.getCrawlUrls();
Iterator colIter = col.iterator();
assertTrue(colIter.hasNext());
assertEquals("test", colIter.next());
assertFalse(colIter.hasNext());
}