private void checkSubstance(ArchivalUnit au) {
SubstanceChecker subChecker = new SubstanceChecker(au);
if (!subChecker.isEnabled()) {
errMsg = "No substance patterns defined for plugin.";
return;
}
AuState auState = AuUtil.getAuState(au);
SubstanceChecker.State oldState = auState.getSubstanceState();
SubstanceChecker.State newState = subChecker.findSubstance();
String chtxt = (newState == oldState ? "(unchanged)" : "(was " + oldState.toString() + ")");
switch (newState) {
case Unknown:
log.error("Shouldn't happen: SubstanceChecker returned Unknown");
errMsg = "Error in SubstanceChecker; see log.";
break;
case Yes:
statusMsg = "AU has substance " + chtxt + ": " + au.getName();
auState.setSubstanceState(SubstanceChecker.State.Yes);
break;
case No:
statusMsg = "AU has no substance " + chtxt + ": " + au.getName();
auState.setSubstanceState(SubstanceChecker.State.No);
break;
}
}
public void testTreeWalkFinished() {
AuState auState = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo);
assertEquals(123, auState.getLastTreeWalkTime());
TimeBase.setSimulated(456);
auState.setLastTreeWalkTime();
assertEquals(456, auState.getLastTreeWalkTime());
}
private boolean startReindexingMetadata(ArchivalUnit au, boolean force) {
if (metadataMgr == null) {
errMsg = "Metadata processing is not enabled.";
return false;
}
if (!force) {
if (!AuUtil.hasCrawled(au)) {
errMsg = "Au has never crawled. Click again to reindex metadata";
showForceReindexMetadata = true;
return false;
}
AuState auState = AuUtil.getAuState(au);
switch (auState.getSubstanceState()) {
case No:
errMsg = "Au has no substance. Click again to reindex metadata";
showForceReindexMetadata = true;
return false;
case Unknown:
errMsg = "Unknown substance for Au. Click again to reindex metadata.";
showForceReindexMetadata = true;
return false;
case Yes:
// fall through
}
}
// Fully reindex metadata with the highest priority.
Connection conn = null;
PreparedStatement insertPendingAuBatchStatement = null;
try {
conn = dbMgr.getConnection();
insertPendingAuBatchStatement = metadataMgr.getPrioritizedInsertPendingAuBatchStatement(conn);
if (metadataMgr.enableAndAddAuToReindex(
au, conn, insertPendingAuBatchStatement, false, true)) {
statusMsg = "Reindexing metadata for " + au.getName();
return true;
}
} catch (DbException dbe) {
log.error("Cannot reindex metadata for " + au.getName(), dbe);
} finally {
DbManager.safeCloseStatement(insertPendingAuBatchStatement);
DbManager.safeRollbackAndClose(conn);
}
if (force) {
errMsg = "Still cannot reindex metadata for " + au.getName();
} else {
errMsg = "Cannot reindex metadata for " + au.getName();
}
return false;
}
// Ensure that fields added to AuState get their default value when
// deserialized from old files not containing the field
public void testNewField() throws Exception {
AuState aus = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo);
String ser = ser(aus);
String edser = ser.replaceAll(".*numAgreePeersLastPoR.*", "");
log.debug2("old: " + ser);
log.debug2("new: " + edser);
AuState newaus = deser(edser);
assertEquals(-1, newaus.getNumAgreePeersLastPoR());
assertEquals(Collections.EMPTY_LIST, newaus.getCdnStems());
}
public void testGetUrls() {
HashSet stringCollection = new HashSet();
stringCollection.add("test");
AuState auState =
makeAuState(mau, -1, -1, -1, -1, 123, stringCollection, 1, -1.0, 1.0, historyRepo);
Collection col = auState.getCrawlUrls();
Iterator colIter = col.iterator();
assertTrue(colIter.hasNext());
assertEquals("test", colIter.next());
assertFalse(colIter.hasNext());
}
public void testForceUpdateUrls() {
AuState auState = new AuState(mau, historyRepo);
assertNull(historyRepo.theAuState);
Collection col = auState.getCrawlUrls();
if (AuState.URL_UPDATE_LIMIT > 1) {
col.add("test");
auState.updatedCrawlUrls(true);
assertEquals(0, auState.urlUpdateCntr);
assertNotNull(historyRepo.theAuState);
}
}
public void testAccessType() {
AuState aus = new AuState(mau, historyRepo);
assertFalse(aus.isOpenAccess());
aus.setAccessType(AuState.AccessType.Subscription);
assertEquals(AuState.AccessType.Subscription, aus.getAccessType());
assertFalse(aus.isOpenAccess());
aus.setAccessType(AuState.AccessType.OpenAccess);
assertEquals(AuState.AccessType.OpenAccess, aus.getAccessType());
assertTrue(aus.isOpenAccess());
}
public void testLastContentChange() {
TimeBase.setSimulated(10);
AuState aus = new AuState(mau, historyRepo);
aus.newCrawlStarted();
TimeBase.step(10);
aus.contentChanged();
assertEquals(20, aus.getLastContentChange());
TimeBase.step(10);
aus.contentChanged();
assertEquals(20, aus.getLastContentChange());
TimeBase.step(10);
aus.newCrawlFinished(1, "foo");
TimeBase.step(10);
aus.contentChanged();
assertEquals(50, aus.getLastContentChange());
}
private List getSummaryInfo(CrawlerStatus status) {
List res = new ArrayList();
StatusTable.SummaryInfo statusSi =
new StatusTable.SummaryInfo(
"Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg());
ArchivalUnit au = status.getAu();
if (au != null) {
AuState aus = AuUtil.getAuState(au);
if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) {
statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS);
}
}
res.add(statusSi);
String sources = StringUtil.separatedString(status.getSources());
res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources));
String startUrls = StringUtil.separatedString(status.getStartUrls());
res.add(
new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls));
return res;
}
public void testUpdateUrls() {
AuState auState = new AuState(mau, historyRepo);
assertNull(historyRepo.theAuState);
Collection col = auState.getCrawlUrls();
for (int ii = 1; ii < AuState.URL_UPDATE_LIMIT; ii++) {
col.add("test" + ii);
auState.updatedCrawlUrls(false);
assertEquals(ii, auState.urlUpdateCntr);
assertNull(historyRepo.theAuState);
}
col.add("test-limit");
auState.updatedCrawlUrls(false);
assertEquals(0, auState.urlUpdateCntr);
assertNotNull(historyRepo.theAuState);
// clear, and check that counter is reset
historyRepo.theAuState = null;
if (AuState.URL_UPDATE_LIMIT > 1) {
col.add("test");
auState.updatedCrawlUrls(false);
assertNull(historyRepo.theAuState);
}
}
public void testStoreAuState() throws Exception {
HashSet strCol = new HashSet();
strCol.add("test");
AuState origState =
new AuState(
mau,
123000,
123123,
41,
"woop woop",
321000,
222000,
3,
"pollres",
12345,
456000,
strCol,
AuState.AccessType.OpenAccess,
2,
1.0,
1.0,
SubstanceChecker.State.Yes,
"SubstVer3",
"MetadatVer7",
111444,
12345,
111222, // lastPoPPoll
7, // lastPoPPollResult
222333, // lastLocalHashScan
444777, // numAgreePeersLastPoR
777444, // numWillingRepairers
747474, // numCurrentSuspectVersions
ListUtil.list("http://hos.t/pa/th"),
repository);
assertEquals("SubstVer3", origState.getFeatureVersion(Plugin.Feature.Substance));
assertEquals("MetadatVer7", origState.getFeatureVersion(Plugin.Feature.Metadata));
assertEquals(111444, origState.getLastMetadataIndex());
repository.storeAuState(origState);
String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
filePath += HistoryRepositoryImpl.AU_FILE_NAME;
File xmlFile = new File(filePath);
assertTrue(xmlFile.exists());
origState = null;
AuState loadedState = repository.loadAuState();
assertEquals(123000, loadedState.getLastCrawlTime());
assertEquals(123123, loadedState.getLastCrawlAttempt());
assertEquals(41, loadedState.getLastCrawlResult());
assertEquals("woop woop", loadedState.getLastCrawlResultMsg());
assertEquals(321000, loadedState.getLastTopLevelPollTime());
assertEquals(222000, loadedState.getLastPollStart());
assertEquals(3, loadedState.getLastPollResult());
assertEquals("Inviting Peers", loadedState.getLastPollResultMsg());
assertEquals(111222, loadedState.getLastPoPPoll());
assertEquals(7, loadedState.getLastPoPPollResult());
assertEquals(222333, loadedState.getLastLocalHashScan());
assertEquals(444777, loadedState.getNumAgreePeersLastPoR());
assertEquals(777444, loadedState.getNumWillingRepairers());
assertEquals(747474, loadedState.getNumCurrentSuspectVersions());
assertEquals(ListUtil.list("http://hos.t/pa/th"), loadedState.getCdnStems());
loadedState.addCdnStem("http://this.is.new/");
assertEquals(
ListUtil.list("http://hos.t/pa/th", "http://this.is.new/"), loadedState.getCdnStems());
assertEquals(12345, loadedState.getPollDuration());
assertEquals(2, loadedState.getClockssSubscriptionStatus());
assertEquals(AuState.AccessType.OpenAccess, loadedState.getAccessType());
assertEquals(SubstanceChecker.State.Yes, loadedState.getSubstanceState());
assertEquals("SubstVer3", loadedState.getFeatureVersion(Plugin.Feature.Substance));
assertEquals("MetadatVer7", loadedState.getFeatureVersion(Plugin.Feature.Metadata));
assertEquals(111444, loadedState.getLastMetadataIndex());
assertEquals(12345, loadedState.getLastContentChange());
assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());
// check crawl urls
Collection col = loadedState.getCrawlUrls();
Iterator colIter = col.iterator();
assertTrue(colIter.hasNext());
assertEquals("test", colIter.next());
assertFalse(colIter.hasNext());
}
public void testBatch() {
AuState aus = new AuState(mau, historyRepo);
assertEquals(0, historyRepo.getAuStateStoreCount());
aus.setNumAgreePeersLastPoR(1);
aus.setNumWillingRepairers(3);
aus.setNumCurrentSuspectVersions(5);
assertEquals(3, historyRepo.getAuStateStoreCount());
aus.batchSaves();
aus.setNumAgreePeersLastPoR(2);
aus.setNumWillingRepairers(4);
aus.setNumCurrentSuspectVersions(6);
assertEquals(3, historyRepo.getAuStateStoreCount());
aus.unBatchSaves();
assertEquals(4, historyRepo.getAuStateStoreCount());
aus.batchSaves();
aus.setNumAgreePeersLastPoR(4);
aus.batchSaves();
aus.setNumWillingRepairers(8);
aus.setNumCurrentSuspectVersions(12);
assertEquals(4, historyRepo.getAuStateStoreCount());
aus.unBatchSaves();
assertEquals(4, historyRepo.getAuStateStoreCount());
aus.unBatchSaves();
assertEquals(5, historyRepo.getAuStateStoreCount());
}
public void testCdnStems() {
AuState aus = new AuState(mau, historyRepo);
assertEquals(Collections.EMPTY_LIST, aus.getCdnStems());
aus.addCdnStem("http://fff.uselesstld");
assertClass(ArrayList.class, aus.getCdnStems());
assertEquals(ListUtil.list("http://fff.uselesstld"), aus.getCdnStems());
aus.addCdnStem("ccc");
assertEquals(ListUtil.list("http://fff.uselesstld", "ccc"), aus.getCdnStems());
aus.setCdnStems(new LinkedList(ListUtil.list("a", "b")));
assertClass(ArrayList.class, aus.getCdnStems());
assertEquals(ListUtil.list("a", "b"), aus.getCdnStems());
aus.setCdnStems(null);
assertEmpty(aus.getCdnStems());
aus.addCdnStem("https://a.b/");
aus.addCdnStem("https://b.a/");
assertEquals(ListUtil.list("https://a.b/", "https://b.a/"), aus.getCdnStems());
}
public void testNumCurrentSuspectVersions() {
MyMockLockssRepository repo = new MyMockLockssRepository();
MyAuSuspectUrlVersions asuv = new MyAuSuspectUrlVersions();
repo.setAsuv(asuv);
daemon.setLockssRepository(repo, mau);
AuState aus = new AuState(mau, historyRepo);
assertEquals(0, aus.getNumCurrentSuspectVersions());
// ensure this isn't automatically recomputed, as that would happen
// when historyRepo loads the object during startAuManagers, before the
// AU is fully created.
aus.setNumCurrentSuspectVersions(-1);
assertEquals(-1, aus.getNumCurrentSuspectVersions());
asuv.setCountResult(17);
aus.recomputeNumCurrentSuspectVersions();
assertEquals(17, aus.getNumCurrentSuspectVersions());
aus.incrementNumCurrentSuspectVersions(-1);
assertEquals(16, aus.getNumCurrentSuspectVersions());
aus.setNumCurrentSuspectVersions(-1);
asuv.setCountResult(6);
aus.incrementNumCurrentSuspectVersions(-1);
assertEquals(5, aus.getNumCurrentSuspectVersions());
}
public void testLastMetadataIndex() {
AuState aus = new AuState(mau, historyRepo);
assertEquals(-1, aus.getLastMetadataIndex());
aus.setLastMetadataIndex(123);
assertEquals(123, aus.getLastMetadataIndex());
}
public void testFeatureVersion() {
AuState aus = new AuState(mau, historyRepo);
assertNull(aus.getFeatureVersion(Plugin.Feature.Substance));
assertNull(aus.getFeatureVersion(Plugin.Feature.Metadata));
assertNull(aus.getFeatureVersion(Plugin.Feature.Poll));
aus.setFeatureVersion(Plugin.Feature.Metadata, "foo");
assertNull(aus.getFeatureVersion(Plugin.Feature.Substance));
assertEquals("foo", aus.getFeatureVersion(Plugin.Feature.Metadata));
aus.setFeatureVersion(Plugin.Feature.Substance, "sub_42");
assertEquals("sub_42", aus.getFeatureVersion(Plugin.Feature.Substance));
assertEquals("foo", aus.getFeatureVersion(Plugin.Feature.Metadata));
assertNull(aus.getFeatureVersion(Plugin.Feature.Poll));
}
public void testStoreAuEmptyState() throws Exception {
HashSet strCol = new HashSet();
strCol.add("test");
AuState origState = new AuState(mau, repository);
repository.storeAuState(origState);
AuState loadedState = repository.loadAuState();
assertEquals(-1, loadedState.getLastCrawlTime());
assertEquals(-1, loadedState.getLastCrawlAttempt());
assertEquals(-1, loadedState.getLastCrawlResult());
assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg());
assertEquals(-1, loadedState.getLastTopLevelPollTime());
assertEquals(-1, loadedState.getLastPollStart());
assertEquals(-1, loadedState.getLastPollResult());
assertEquals(null, loadedState.getLastPollResultMsg());
assertEquals(-1, loadedState.getLastPoPPoll());
assertEquals(-1, loadedState.getLastPoPPollResult());
assertEquals(-1, loadedState.getLastLocalHashScan());
assertEquals(0, loadedState.getNumAgreePeersLastPoR());
assertEquals(0, loadedState.getNumWillingRepairers());
assertEquals(0, loadedState.getNumCurrentSuspectVersions());
assertEmpty(loadedState.getCdnStems());
loadedState.addCdnStem("http://this.is.new/");
assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
loadedState.addCdnStem("http://this.is.new/");
assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
assertEquals(0, loadedState.getPollDuration());
assertEquals(0, loadedState.getClockssSubscriptionStatus());
assertEquals(null, loadedState.getAccessType());
assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState());
assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance));
assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata));
assertEquals(-1, loadedState.getLastMetadataIndex());
assertEquals(0, loadedState.getLastContentChange());
assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());
}
public void testStoreOverwrite() throws Exception {
AuState auState =
new AuState(
mau,
123, // lastCrawlTime
321, // lastCrawlAttempt
-1, // lastCrawlResult
null, // lastCrawlResultMsg,
321, // lastTopLevelPoll
333, // lastPollStart
-1, // lastPollresult
null, // lastPollresultMsg
0, // pollDuration
-1, // lastTreeWalk
null, // crawlUrls
null, // accessType
1, // clockssSubscriptionState
1.0, // v3Agreement
1.0, // highestV3Agreement
SubstanceChecker.State.Unknown,
null, // substanceVersion
null, // metadataVersion
-1, // lastMetadataIndex
0, // lastContentChange
444, // lastPoPPoll
8, // lastPoPPollResult
-1, // lastLocalHashScan
27, // numAgreePeersLastPoR
72, // numWillingRepirers
19, // numCurrentSuspectVersions
ListUtil.list("http://foo/"), // cdnStems
repository);
repository.storeAuState(auState);
String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
filePath += HistoryRepositoryImpl.AU_FILE_NAME;
File xmlFile = new File(filePath);
FileInputStream fis = new FileInputStream(xmlFile);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
StreamUtil.copy(fis, baos);
fis.close();
String expectedStr = baos.toString();
auState =
new AuState(
mau,
1234, // lastCrawlTime
4321, // lastCrawlAttempt
-1, // lastCrawlResult
null, // lastCrawlResultMsg,
4321, // lastTopLevelPoll
5555, // lastPollStart
-1, // lastPollresult
null, // lastPollresultMsg
0, // pollDuration
-1, // lastTreeWalk
null, // crawlUrls
null, // accessType
1, // clockssSubscriptionState
1.0, // v3Agreement
1.0, // highestV3Agreement
SubstanceChecker.State.Unknown,
null, // substanceVersion
null, // metadataVersion
-1, // lastMetadataIndex
0, // lastContentChange
-1, // lastPoPPoll
-1, // lastPoPPollResult
-1, // lastLocalHashScan
13, // numAgreePeersLastPoR
31, // numWillingRepairers
91, // numCurrentSuspectVersions
ListUtil.list("http://foo/"), // cdnStems
repository);
repository.storeAuState(auState);
assertEquals(1234, auState.getLastCrawlTime());
assertEquals(4321, auState.getLastCrawlAttempt());
assertEquals(4321, auState.getLastTopLevelPollTime());
assertEquals(5555, auState.getLastPollStart());
assertEquals(13, auState.getNumAgreePeersLastPoR());
assertEquals(31, auState.getNumWillingRepairers());
assertEquals(91, auState.getNumCurrentSuspectVersions());
assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId());
assertEquals(ListUtil.list("http://foo/"), auState.getCdnStems());
fis = new FileInputStream(xmlFile);
baos = new ByteArrayOutputStream(expectedStr.length());
StreamUtil.copy(fis, baos);
fis.close();
log.info(baos.toString());
auState = null;
auState = repository.loadAuState();
assertEquals(1234, auState.getLastCrawlTime());
assertEquals(4321, auState.getLastCrawlAttempt());
assertEquals(4321, auState.getLastTopLevelPollTime());
assertEquals(5555, auState.getLastPollStart());
assertEquals(13, auState.getNumAgreePeersLastPoR());
assertEquals(31, auState.getNumWillingRepairers());
assertEquals(91, auState.getNumCurrentSuspectVersions());
assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId());
auState =
new AuState(
mau,
123, // lastCrawlTime
321, // lastCrawlAttempt
-1, // lastCrawlResult
null, // lastCrawlResultMsg,
321, // lastTopLevelPoll
333, // lastPollStart
-1, // lastPollresult
null, // lastPollresultMsg
0, // pollDuration
-1, // lastTreeWalk
null, // crawlUrls
null, // accessType
1, // clockssSubscriptionState
1.0, // v3Agreement
1.0, // highestV3Agreement
SubstanceChecker.State.Unknown,
null, // substanceVersion
null, // metadataVersion
-1, // lastMetadataIndex
0, // lastContentChange
444, // lastPoPPoll
8, // lastPoPPollResult
-1, // lastLocalHashScan
27, // numAgreePeersLastPoR
72, // numWillingRepairers
19, // numCurrentSuspectVersions
ListUtil.list("http://foo/"), // cdnStems
repository);
repository.storeAuState(auState);
fis = new FileInputStream(xmlFile);
baos = new ByteArrayOutputStream(expectedStr.length());
StreamUtil.copy(fis, baos);
fis.close();
assertEquals(expectedStr, baos.toString());
}
public void testClockssSubscriptionStatus() {
AuState aus = new AuState(mau, historyRepo);
assertEquals(AuState.CLOCKSS_SUB_UNKNOWN, aus.getClockssSubscriptionStatus());
assertEquals("Unknown", aus.getClockssSubscriptionStatusString());
aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_YES);
assertEquals(AuState.CLOCKSS_SUB_YES, aus.getClockssSubscriptionStatus());
assertEquals("Yes", aus.getClockssSubscriptionStatusString());
aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_NO);
assertEquals(AuState.CLOCKSS_SUB_NO, aus.getClockssSubscriptionStatus());
assertEquals("No", aus.getClockssSubscriptionStatusString());
aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_INACCESSIBLE);
assertEquals(AuState.CLOCKSS_SUB_INACCESSIBLE, aus.getClockssSubscriptionStatus());
assertEquals("Inaccessible", aus.getClockssSubscriptionStatusString());
}
public void testSubstanceState() {
AuState aus = new AuState(mau, historyRepo);
assertEquals(SubstanceChecker.State.Unknown, aus.getSubstanceState());
assertFalse(aus.hasNoSubstance());
aus.setSubstanceState(SubstanceChecker.State.Yes);
assertEquals(1, historyRepo.getAuStateStoreCount());
assertEquals(SubstanceChecker.State.Yes, aus.getSubstanceState());
assertFalse(aus.hasNoSubstance());
aus.setSubstanceState(SubstanceChecker.State.No);
assertEquals(2, historyRepo.getAuStateStoreCount());
assertEquals(SubstanceChecker.State.No, aus.getSubstanceState());
assertTrue(aus.hasNoSubstance());
assertNotEquals("2", aus.getFeatureVersion(Plugin.Feature.Substance));
mplug.setFeatureVersionMap(MapUtil.map(Plugin.Feature.Substance, "2"));
aus.setSubstanceState(SubstanceChecker.State.Yes);
// changing both the substance state and feature version should store
// only once
assertEquals(3, historyRepo.getAuStateStoreCount());
assertEquals(SubstanceChecker.State.Yes, aus.getSubstanceState());
assertEquals("2", aus.getFeatureVersion(Plugin.Feature.Substance));
}
protected void setAuAccessType(Crawler.PermissionHelper pHelper, AuState.AccessType accessType) {
ArchivalUnit au = pHelper.getAu();
AuState aus = AuUtil.getAuState(au);
aus.setAccessType(accessType);
}