private void checkSubstance(ArchivalUnit au) { SubstanceChecker subChecker = new SubstanceChecker(au); if (!subChecker.isEnabled()) { errMsg = "No substance patterns defined for plugin."; return; } AuState auState = AuUtil.getAuState(au); SubstanceChecker.State oldState = auState.getSubstanceState(); SubstanceChecker.State newState = subChecker.findSubstance(); String chtxt = (newState == oldState ? "(unchanged)" : "(was " + oldState.toString() + ")"); switch (newState) { case Unknown: log.error("Shouldn't happen: SubstanceChecker returned Unknown"); errMsg = "Error in SubstanceChecker; see log."; break; case Yes: statusMsg = "AU has substance " + chtxt + ": " + au.getName(); auState.setSubstanceState(SubstanceChecker.State.Yes); break; case No: statusMsg = "AU has no substance " + chtxt + ": " + au.getName(); auState.setSubstanceState(SubstanceChecker.State.No); break; } }
public void testTreeWalkFinished() { AuState auState = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo); assertEquals(123, auState.getLastTreeWalkTime()); TimeBase.setSimulated(456); auState.setLastTreeWalkTime(); assertEquals(456, auState.getLastTreeWalkTime()); }
private boolean startReindexingMetadata(ArchivalUnit au, boolean force) { if (metadataMgr == null) { errMsg = "Metadata processing is not enabled."; return false; } if (!force) { if (!AuUtil.hasCrawled(au)) { errMsg = "Au has never crawled. Click again to reindex metadata"; showForceReindexMetadata = true; return false; } AuState auState = AuUtil.getAuState(au); switch (auState.getSubstanceState()) { case No: errMsg = "Au has no substance. Click again to reindex metadata"; showForceReindexMetadata = true; return false; case Unknown: errMsg = "Unknown substance for Au. Click again to reindex metadata."; showForceReindexMetadata = true; return false; case Yes: // fall through } } // Fully reindex metadata with the highest priority. Connection conn = null; PreparedStatement insertPendingAuBatchStatement = null; try { conn = dbMgr.getConnection(); insertPendingAuBatchStatement = metadataMgr.getPrioritizedInsertPendingAuBatchStatement(conn); if (metadataMgr.enableAndAddAuToReindex( au, conn, insertPendingAuBatchStatement, false, true)) { statusMsg = "Reindexing metadata for " + au.getName(); return true; } } catch (DbException dbe) { log.error("Cannot reindex metadata for " + au.getName(), dbe); } finally { DbManager.safeCloseStatement(insertPendingAuBatchStatement); DbManager.safeRollbackAndClose(conn); } if (force) { errMsg = "Still cannot reindex metadata for " + au.getName(); } else { errMsg = "Cannot reindex metadata for " + au.getName(); } return false; }
// Ensure that fields added to AuState get their default value when // deserialized from old files not containing the field public void testNewField() throws Exception { AuState aus = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo); String ser = ser(aus); String edser = ser.replaceAll(".*numAgreePeersLastPoR.*", ""); log.debug2("old: " + ser); log.debug2("new: " + edser); AuState newaus = deser(edser); assertEquals(-1, newaus.getNumAgreePeersLastPoR()); assertEquals(Collections.EMPTY_LIST, newaus.getCdnStems()); }
public void testForceUpdateUrls() { AuState auState = new AuState(mau, historyRepo); assertNull(historyRepo.theAuState); Collection col = auState.getCrawlUrls(); if (AuState.URL_UPDATE_LIMIT > 1) { col.add("test"); auState.updatedCrawlUrls(true); assertEquals(0, auState.urlUpdateCntr); assertNotNull(historyRepo.theAuState); } }
public void testGetUrls() { HashSet stringCollection = new HashSet(); stringCollection.add("test"); AuState auState = makeAuState(mau, -1, -1, -1, -1, 123, stringCollection, 1, -1.0, 1.0, historyRepo); Collection col = auState.getCrawlUrls(); Iterator colIter = col.iterator(); assertTrue(colIter.hasNext()); assertEquals("test", colIter.next()); assertFalse(colIter.hasNext()); }
public void testAccessType() { AuState aus = new AuState(mau, historyRepo); assertFalse(aus.isOpenAccess()); aus.setAccessType(AuState.AccessType.Subscription); assertEquals(AuState.AccessType.Subscription, aus.getAccessType()); assertFalse(aus.isOpenAccess()); aus.setAccessType(AuState.AccessType.OpenAccess); assertEquals(AuState.AccessType.OpenAccess, aus.getAccessType()); assertTrue(aus.isOpenAccess()); }
public void testLastContentChange() { TimeBase.setSimulated(10); AuState aus = new AuState(mau, historyRepo); aus.newCrawlStarted(); TimeBase.step(10); aus.contentChanged(); assertEquals(20, aus.getLastContentChange()); TimeBase.step(10); aus.contentChanged(); assertEquals(20, aus.getLastContentChange()); TimeBase.step(10); aus.newCrawlFinished(1, "foo"); TimeBase.step(10); aus.contentChanged(); assertEquals(50, aus.getLastContentChange()); }
private List getSummaryInfo(CrawlerStatus status) { List res = new ArrayList(); StatusTable.SummaryInfo statusSi = new StatusTable.SummaryInfo( "Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg()); ArchivalUnit au = status.getAu(); if (au != null) { AuState aus = AuUtil.getAuState(au); if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) { statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS); } } res.add(statusSi); String sources = StringUtil.separatedString(status.getSources()); res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources)); String startUrls = StringUtil.separatedString(status.getStartUrls()); res.add( new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls)); return res; }
public void testUpdateUrls() { AuState auState = new AuState(mau, historyRepo); assertNull(historyRepo.theAuState); Collection col = auState.getCrawlUrls(); for (int ii = 1; ii < AuState.URL_UPDATE_LIMIT; ii++) { col.add("test" + ii); auState.updatedCrawlUrls(false); assertEquals(ii, auState.urlUpdateCntr); assertNull(historyRepo.theAuState); } col.add("test-limit"); auState.updatedCrawlUrls(false); assertEquals(0, auState.urlUpdateCntr); assertNotNull(historyRepo.theAuState); // clear, and check that counter is reset historyRepo.theAuState = null; if (AuState.URL_UPDATE_LIMIT > 1) { col.add("test"); auState.updatedCrawlUrls(false); assertNull(historyRepo.theAuState); } }
public void testSubstanceState() { AuState aus = new AuState(mau, historyRepo); assertEquals(SubstanceChecker.State.Unknown, aus.getSubstanceState()); assertFalse(aus.hasNoSubstance()); aus.setSubstanceState(SubstanceChecker.State.Yes); assertEquals(1, historyRepo.getAuStateStoreCount()); assertEquals(SubstanceChecker.State.Yes, aus.getSubstanceState()); assertFalse(aus.hasNoSubstance()); aus.setSubstanceState(SubstanceChecker.State.No); assertEquals(2, historyRepo.getAuStateStoreCount()); assertEquals(SubstanceChecker.State.No, aus.getSubstanceState()); assertTrue(aus.hasNoSubstance()); assertNotEquals("2", aus.getFeatureVersion(Plugin.Feature.Substance)); mplug.setFeatureVersionMap(MapUtil.map(Plugin.Feature.Substance, "2")); aus.setSubstanceState(SubstanceChecker.State.Yes); // changing both the substance state and feature version should store // only once assertEquals(3, historyRepo.getAuStateStoreCount()); assertEquals(SubstanceChecker.State.Yes, aus.getSubstanceState()); assertEquals("2", aus.getFeatureVersion(Plugin.Feature.Substance)); }
public void testStoreOverwrite() throws Exception { AuState auState = new AuState( mau, 123, // lastCrawlTime 321, // lastCrawlAttempt -1, // lastCrawlResult null, // lastCrawlResultMsg, 321, // lastTopLevelPoll 333, // lastPollStart -1, // lastPollresult null, // lastPollresultMsg 0, // pollDuration -1, // lastTreeWalk null, // crawlUrls null, // accessType 1, // clockssSubscriptionState 1.0, // v3Agreement 1.0, // highestV3Agreement SubstanceChecker.State.Unknown, null, // substanceVersion null, // metadataVersion -1, // lastMetadataIndex 0, // lastContentChange 444, // lastPoPPoll 8, // lastPoPPollResult -1, // lastLocalHashScan 27, // numAgreePeersLastPoR 72, // numWillingRepirers 19, // numCurrentSuspectVersions ListUtil.list("http://foo/"), // cdnStems repository); repository.storeAuState(auState); String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau); filePath += HistoryRepositoryImpl.AU_FILE_NAME; File xmlFile = new File(filePath); FileInputStream fis = new FileInputStream(xmlFile); ByteArrayOutputStream baos = new ByteArrayOutputStream(); StreamUtil.copy(fis, baos); fis.close(); String expectedStr = baos.toString(); auState = new AuState( mau, 1234, // lastCrawlTime 4321, // lastCrawlAttempt -1, // lastCrawlResult null, // lastCrawlResultMsg, 4321, // lastTopLevelPoll 5555, // lastPollStart -1, // lastPollresult null, // lastPollresultMsg 0, // pollDuration -1, // lastTreeWalk null, // crawlUrls null, // accessType 1, // clockssSubscriptionState 1.0, // v3Agreement 1.0, // highestV3Agreement SubstanceChecker.State.Unknown, null, // substanceVersion null, // metadataVersion -1, // lastMetadataIndex 0, // lastContentChange -1, // lastPoPPoll -1, // lastPoPPollResult -1, // lastLocalHashScan 13, // numAgreePeersLastPoR 31, // numWillingRepairers 91, // numCurrentSuspectVersions ListUtil.list("http://foo/"), // cdnStems repository); repository.storeAuState(auState); assertEquals(1234, auState.getLastCrawlTime()); assertEquals(4321, auState.getLastCrawlAttempt()); assertEquals(4321, auState.getLastTopLevelPollTime()); assertEquals(5555, auState.getLastPollStart()); assertEquals(13, auState.getNumAgreePeersLastPoR()); assertEquals(31, auState.getNumWillingRepairers()); assertEquals(91, auState.getNumCurrentSuspectVersions()); assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId()); assertEquals(ListUtil.list("http://foo/"), auState.getCdnStems()); fis = new FileInputStream(xmlFile); baos = new ByteArrayOutputStream(expectedStr.length()); StreamUtil.copy(fis, baos); fis.close(); log.info(baos.toString()); auState = null; auState = repository.loadAuState(); assertEquals(1234, auState.getLastCrawlTime()); assertEquals(4321, auState.getLastCrawlAttempt()); assertEquals(4321, auState.getLastTopLevelPollTime()); assertEquals(5555, auState.getLastPollStart()); assertEquals(13, auState.getNumAgreePeersLastPoR()); assertEquals(31, auState.getNumWillingRepairers()); assertEquals(91, auState.getNumCurrentSuspectVersions()); assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId()); auState = new AuState( mau, 123, // lastCrawlTime 321, // lastCrawlAttempt -1, // lastCrawlResult null, // lastCrawlResultMsg, 321, // lastTopLevelPoll 333, // lastPollStart -1, // lastPollresult null, // lastPollresultMsg 0, // pollDuration -1, // lastTreeWalk null, // crawlUrls null, // accessType 1, // clockssSubscriptionState 1.0, // v3Agreement 1.0, // highestV3Agreement SubstanceChecker.State.Unknown, null, // substanceVersion null, // metadataVersion -1, // lastMetadataIndex 0, // lastContentChange 444, // lastPoPPoll 8, // lastPoPPollResult -1, // lastLocalHashScan 27, // numAgreePeersLastPoR 72, // numWillingRepairers 19, // numCurrentSuspectVersions ListUtil.list("http://foo/"), // cdnStems repository); repository.storeAuState(auState); fis = new FileInputStream(xmlFile); baos = new ByteArrayOutputStream(expectedStr.length()); StreamUtil.copy(fis, baos); fis.close(); assertEquals(expectedStr, baos.toString()); }
public void testBatch() { AuState aus = new AuState(mau, historyRepo); assertEquals(0, historyRepo.getAuStateStoreCount()); aus.setNumAgreePeersLastPoR(1); aus.setNumWillingRepairers(3); aus.setNumCurrentSuspectVersions(5); assertEquals(3, historyRepo.getAuStateStoreCount()); aus.batchSaves(); aus.setNumAgreePeersLastPoR(2); aus.setNumWillingRepairers(4); aus.setNumCurrentSuspectVersions(6); assertEquals(3, historyRepo.getAuStateStoreCount()); aus.unBatchSaves(); assertEquals(4, historyRepo.getAuStateStoreCount()); aus.batchSaves(); aus.setNumAgreePeersLastPoR(4); aus.batchSaves(); aus.setNumWillingRepairers(8); aus.setNumCurrentSuspectVersions(12); assertEquals(4, historyRepo.getAuStateStoreCount()); aus.unBatchSaves(); assertEquals(4, historyRepo.getAuStateStoreCount()); aus.unBatchSaves(); assertEquals(5, historyRepo.getAuStateStoreCount()); }
public void testCdnStems() { AuState aus = new AuState(mau, historyRepo); assertEquals(Collections.EMPTY_LIST, aus.getCdnStems()); aus.addCdnStem("http://fff.uselesstld"); assertClass(ArrayList.class, aus.getCdnStems()); assertEquals(ListUtil.list("http://fff.uselesstld"), aus.getCdnStems()); aus.addCdnStem("ccc"); assertEquals(ListUtil.list("http://fff.uselesstld", "ccc"), aus.getCdnStems()); aus.setCdnStems(new LinkedList(ListUtil.list("a", "b"))); assertClass(ArrayList.class, aus.getCdnStems()); assertEquals(ListUtil.list("a", "b"), aus.getCdnStems()); aus.setCdnStems(null); assertEmpty(aus.getCdnStems()); aus.addCdnStem("https://a.b/"); aus.addCdnStem("https://b.a/"); assertEquals(ListUtil.list("https://a.b/", "https://b.a/"), aus.getCdnStems()); }
public void testNumCurrentSuspectVersions() { MyMockLockssRepository repo = new MyMockLockssRepository(); MyAuSuspectUrlVersions asuv = new MyAuSuspectUrlVersions(); repo.setAsuv(asuv); daemon.setLockssRepository(repo, mau); AuState aus = new AuState(mau, historyRepo); assertEquals(0, aus.getNumCurrentSuspectVersions()); // ensure this isn't automatically recomputed, as that would happen // when historyRepo loads the object during startAuManagers, before the // AU is fully created. aus.setNumCurrentSuspectVersions(-1); assertEquals(-1, aus.getNumCurrentSuspectVersions()); asuv.setCountResult(17); aus.recomputeNumCurrentSuspectVersions(); assertEquals(17, aus.getNumCurrentSuspectVersions()); aus.incrementNumCurrentSuspectVersions(-1); assertEquals(16, aus.getNumCurrentSuspectVersions()); aus.setNumCurrentSuspectVersions(-1); asuv.setCountResult(6); aus.incrementNumCurrentSuspectVersions(-1); assertEquals(5, aus.getNumCurrentSuspectVersions()); }
public void testLastMetadataIndex() { AuState aus = new AuState(mau, historyRepo); assertEquals(-1, aus.getLastMetadataIndex()); aus.setLastMetadataIndex(123); assertEquals(123, aus.getLastMetadataIndex()); }
public void testStoreAuState() throws Exception { HashSet strCol = new HashSet(); strCol.add("test"); AuState origState = new AuState( mau, 123000, 123123, 41, "woop woop", 321000, 222000, 3, "pollres", 12345, 456000, strCol, AuState.AccessType.OpenAccess, 2, 1.0, 1.0, SubstanceChecker.State.Yes, "SubstVer3", "MetadatVer7", 111444, 12345, 111222, // lastPoPPoll 7, // lastPoPPollResult 222333, // lastLocalHashScan 444777, // numAgreePeersLastPoR 777444, // numWillingRepairers 747474, // numCurrentSuspectVersions ListUtil.list("http://hos.t/pa/th"), repository); assertEquals("SubstVer3", origState.getFeatureVersion(Plugin.Feature.Substance)); assertEquals("MetadatVer7", origState.getFeatureVersion(Plugin.Feature.Metadata)); assertEquals(111444, origState.getLastMetadataIndex()); repository.storeAuState(origState); String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau); filePath += HistoryRepositoryImpl.AU_FILE_NAME; File xmlFile = new File(filePath); assertTrue(xmlFile.exists()); origState = null; AuState loadedState = repository.loadAuState(); assertEquals(123000, loadedState.getLastCrawlTime()); assertEquals(123123, loadedState.getLastCrawlAttempt()); assertEquals(41, loadedState.getLastCrawlResult()); assertEquals("woop woop", loadedState.getLastCrawlResultMsg()); assertEquals(321000, loadedState.getLastTopLevelPollTime()); assertEquals(222000, loadedState.getLastPollStart()); assertEquals(3, loadedState.getLastPollResult()); assertEquals("Inviting Peers", loadedState.getLastPollResultMsg()); assertEquals(111222, loadedState.getLastPoPPoll()); assertEquals(7, loadedState.getLastPoPPollResult()); assertEquals(222333, loadedState.getLastLocalHashScan()); assertEquals(444777, loadedState.getNumAgreePeersLastPoR()); assertEquals(777444, loadedState.getNumWillingRepairers()); assertEquals(747474, loadedState.getNumCurrentSuspectVersions()); assertEquals(ListUtil.list("http://hos.t/pa/th"), loadedState.getCdnStems()); loadedState.addCdnStem("http://this.is.new/"); assertEquals( ListUtil.list("http://hos.t/pa/th", "http://this.is.new/"), loadedState.getCdnStems()); assertEquals(12345, loadedState.getPollDuration()); assertEquals(2, loadedState.getClockssSubscriptionStatus()); assertEquals(AuState.AccessType.OpenAccess, loadedState.getAccessType()); assertEquals(SubstanceChecker.State.Yes, loadedState.getSubstanceState()); assertEquals("SubstVer3", loadedState.getFeatureVersion(Plugin.Feature.Substance)); assertEquals("MetadatVer7", loadedState.getFeatureVersion(Plugin.Feature.Metadata)); assertEquals(111444, loadedState.getLastMetadataIndex()); assertEquals(12345, loadedState.getLastContentChange()); assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId()); // check crawl urls Collection col = loadedState.getCrawlUrls(); Iterator colIter = col.iterator(); assertTrue(colIter.hasNext()); assertEquals("test", colIter.next()); assertFalse(colIter.hasNext()); }
public void testClockssSubscriptionStatus() { AuState aus = new AuState(mau, historyRepo); assertEquals(AuState.CLOCKSS_SUB_UNKNOWN, aus.getClockssSubscriptionStatus()); assertEquals("Unknown", aus.getClockssSubscriptionStatusString()); aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_YES); assertEquals(AuState.CLOCKSS_SUB_YES, aus.getClockssSubscriptionStatus()); assertEquals("Yes", aus.getClockssSubscriptionStatusString()); aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_NO); assertEquals(AuState.CLOCKSS_SUB_NO, aus.getClockssSubscriptionStatus()); assertEquals("No", aus.getClockssSubscriptionStatusString()); aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_INACCESSIBLE); assertEquals(AuState.CLOCKSS_SUB_INACCESSIBLE, aus.getClockssSubscriptionStatus()); assertEquals("Inaccessible", aus.getClockssSubscriptionStatusString()); }
public void testFeatureVersion() { AuState aus = new AuState(mau, historyRepo); assertNull(aus.getFeatureVersion(Plugin.Feature.Substance)); assertNull(aus.getFeatureVersion(Plugin.Feature.Metadata)); assertNull(aus.getFeatureVersion(Plugin.Feature.Poll)); aus.setFeatureVersion(Plugin.Feature.Metadata, "foo"); assertNull(aus.getFeatureVersion(Plugin.Feature.Substance)); assertEquals("foo", aus.getFeatureVersion(Plugin.Feature.Metadata)); aus.setFeatureVersion(Plugin.Feature.Substance, "sub_42"); assertEquals("sub_42", aus.getFeatureVersion(Plugin.Feature.Substance)); assertEquals("foo", aus.getFeatureVersion(Plugin.Feature.Metadata)); assertNull(aus.getFeatureVersion(Plugin.Feature.Poll)); }
public void testStoreAuEmptyState() throws Exception { HashSet strCol = new HashSet(); strCol.add("test"); AuState origState = new AuState(mau, repository); repository.storeAuState(origState); AuState loadedState = repository.loadAuState(); assertEquals(-1, loadedState.getLastCrawlTime()); assertEquals(-1, loadedState.getLastCrawlAttempt()); assertEquals(-1, loadedState.getLastCrawlResult()); assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg()); assertEquals(-1, loadedState.getLastTopLevelPollTime()); assertEquals(-1, loadedState.getLastPollStart()); assertEquals(-1, loadedState.getLastPollResult()); assertEquals(null, loadedState.getLastPollResultMsg()); assertEquals(-1, loadedState.getLastPoPPoll()); assertEquals(-1, loadedState.getLastPoPPollResult()); assertEquals(-1, loadedState.getLastLocalHashScan()); assertEquals(0, loadedState.getNumAgreePeersLastPoR()); assertEquals(0, loadedState.getNumWillingRepairers()); assertEquals(0, loadedState.getNumCurrentSuspectVersions()); assertEmpty(loadedState.getCdnStems()); loadedState.addCdnStem("http://this.is.new/"); assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems()); loadedState.addCdnStem("http://this.is.new/"); assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems()); assertEquals(0, loadedState.getPollDuration()); assertEquals(0, loadedState.getClockssSubscriptionStatus()); assertEquals(null, loadedState.getAccessType()); assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState()); assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance)); assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata)); assertEquals(-1, loadedState.getLastMetadataIndex()); assertEquals(0, loadedState.getLastContentChange()); assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId()); }