Example #1
0
  public void testTreeWalkFinished() {
    AuState auState = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo);
    assertEquals(123, auState.getLastTreeWalkTime());

    TimeBase.setSimulated(456);
    auState.setLastTreeWalkTime();
    assertEquals(456, auState.getLastTreeWalkTime());
  }
Example #2
0
  // Ensure that fields added to AuState get their default value when
  // deserialized from old files not containing the field
  public void testNewField() throws Exception {
    AuState aus = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo);

    String ser = ser(aus);
    String edser = ser.replaceAll(".*numAgreePeersLastPoR.*", "");
    log.debug2("old: " + ser);
    log.debug2("new: " + edser);
    AuState newaus = deser(edser);
    assertEquals(-1, newaus.getNumAgreePeersLastPoR());
    assertEquals(Collections.EMPTY_LIST, newaus.getCdnStems());
  }
Example #3
0
  public void testForceUpdateUrls() {
    AuState auState = new AuState(mau, historyRepo);
    assertNull(historyRepo.theAuState);

    Collection col = auState.getCrawlUrls();
    if (AuState.URL_UPDATE_LIMIT > 1) {
      col.add("test");
      auState.updatedCrawlUrls(true);
      assertEquals(0, auState.urlUpdateCntr);
      assertNotNull(historyRepo.theAuState);
    }
  }
Example #4
0
  public void testGetUrls() {
    HashSet stringCollection = new HashSet();
    stringCollection.add("test");

    AuState auState =
        makeAuState(mau, -1, -1, -1, -1, 123, stringCollection, 1, -1.0, 1.0, historyRepo);
    Collection col = auState.getCrawlUrls();
    Iterator colIter = col.iterator();
    assertTrue(colIter.hasNext());
    assertEquals("test", colIter.next());
    assertFalse(colIter.hasNext());
  }
Example #5
0
 public void testAccessType() {
   AuState aus = new AuState(mau, historyRepo);
   assertFalse(aus.isOpenAccess());
   aus.setAccessType(AuState.AccessType.Subscription);
   assertEquals(AuState.AccessType.Subscription, aus.getAccessType());
   assertFalse(aus.isOpenAccess());
   aus.setAccessType(AuState.AccessType.OpenAccess);
   assertEquals(AuState.AccessType.OpenAccess, aus.getAccessType());
   assertTrue(aus.isOpenAccess());
 }
Example #6
0
 public void testLastContentChange() {
   TimeBase.setSimulated(10);
   AuState aus = new AuState(mau, historyRepo);
   aus.newCrawlStarted();
   TimeBase.step(10);
   aus.contentChanged();
   assertEquals(20, aus.getLastContentChange());
   TimeBase.step(10);
   aus.contentChanged();
   assertEquals(20, aus.getLastContentChange());
   TimeBase.step(10);
   aus.newCrawlFinished(1, "foo");
   TimeBase.step(10);
   aus.contentChanged();
   assertEquals(50, aus.getLastContentChange());
 }
 private List getSummaryInfo(CrawlerStatus status) {
   List res = new ArrayList();
   StatusTable.SummaryInfo statusSi =
       new StatusTable.SummaryInfo(
           "Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg());
   ArchivalUnit au = status.getAu();
   if (au != null) {
     AuState aus = AuUtil.getAuState(au);
     if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) {
       statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS);
     }
   }
   res.add(statusSi);
   String sources = StringUtil.separatedString(status.getSources());
   res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources));
   String startUrls = StringUtil.separatedString(status.getStartUrls());
   res.add(
       new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls));
   return res;
 }
Example #8
0
  public void testUpdateUrls() {
    AuState auState = new AuState(mau, historyRepo);
    assertNull(historyRepo.theAuState);

    Collection col = auState.getCrawlUrls();
    for (int ii = 1; ii < AuState.URL_UPDATE_LIMIT; ii++) {
      col.add("test" + ii);
      auState.updatedCrawlUrls(false);
      assertEquals(ii, auState.urlUpdateCntr);
      assertNull(historyRepo.theAuState);
    }
    col.add("test-limit");
    auState.updatedCrawlUrls(false);
    assertEquals(0, auState.urlUpdateCntr);
    assertNotNull(historyRepo.theAuState);

    // clear, and check that counter is reset
    historyRepo.theAuState = null;
    if (AuState.URL_UPDATE_LIMIT > 1) {
      col.add("test");
      auState.updatedCrawlUrls(false);
      assertNull(historyRepo.theAuState);
    }
  }
Example #9
0
  public void testBatch() {
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(0, historyRepo.getAuStateStoreCount());
    aus.setNumAgreePeersLastPoR(1);
    aus.setNumWillingRepairers(3);
    aus.setNumCurrentSuspectVersions(5);
    assertEquals(3, historyRepo.getAuStateStoreCount());

    aus.batchSaves();
    aus.setNumAgreePeersLastPoR(2);
    aus.setNumWillingRepairers(4);
    aus.setNumCurrentSuspectVersions(6);
    assertEquals(3, historyRepo.getAuStateStoreCount());
    aus.unBatchSaves();
    assertEquals(4, historyRepo.getAuStateStoreCount());

    aus.batchSaves();
    aus.setNumAgreePeersLastPoR(4);
    aus.batchSaves();
    aus.setNumWillingRepairers(8);
    aus.setNumCurrentSuspectVersions(12);
    assertEquals(4, historyRepo.getAuStateStoreCount());
    aus.unBatchSaves();
    assertEquals(4, historyRepo.getAuStateStoreCount());
    aus.unBatchSaves();
    assertEquals(5, historyRepo.getAuStateStoreCount());
  }
Example #10
0
  public void testCdnStems() {
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(Collections.EMPTY_LIST, aus.getCdnStems());
    aus.addCdnStem("http://fff.uselesstld");
    assertClass(ArrayList.class, aus.getCdnStems());
    assertEquals(ListUtil.list("http://fff.uselesstld"), aus.getCdnStems());
    aus.addCdnStem("ccc");
    assertEquals(ListUtil.list("http://fff.uselesstld", "ccc"), aus.getCdnStems());

    aus.setCdnStems(new LinkedList(ListUtil.list("a", "b")));
    assertClass(ArrayList.class, aus.getCdnStems());
    assertEquals(ListUtil.list("a", "b"), aus.getCdnStems());
    aus.setCdnStems(null);
    assertEmpty(aus.getCdnStems());
    aus.addCdnStem("https://a.b/");
    aus.addCdnStem("https://b.a/");
    assertEquals(ListUtil.list("https://a.b/", "https://b.a/"), aus.getCdnStems());
  }
Example #11
0
  public void testNumCurrentSuspectVersions() {
    MyMockLockssRepository repo = new MyMockLockssRepository();
    MyAuSuspectUrlVersions asuv = new MyAuSuspectUrlVersions();
    repo.setAsuv(asuv);

    daemon.setLockssRepository(repo, mau);
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(0, aus.getNumCurrentSuspectVersions());
    // ensure this isn't automatically recomputed, as that would happen
    // when historyRepo loads the object during startAuManagers, before the
    // AU is fully created.
    aus.setNumCurrentSuspectVersions(-1);
    assertEquals(-1, aus.getNumCurrentSuspectVersions());
    asuv.setCountResult(17);
    aus.recomputeNumCurrentSuspectVersions();
    assertEquals(17, aus.getNumCurrentSuspectVersions());
    aus.incrementNumCurrentSuspectVersions(-1);
    assertEquals(16, aus.getNumCurrentSuspectVersions());

    aus.setNumCurrentSuspectVersions(-1);
    asuv.setCountResult(6);
    aus.incrementNumCurrentSuspectVersions(-1);
    assertEquals(5, aus.getNumCurrentSuspectVersions());
  }
Example #12
0
 public void testLastMetadataIndex() {
   AuState aus = new AuState(mau, historyRepo);
   assertEquals(-1, aus.getLastMetadataIndex());
   aus.setLastMetadataIndex(123);
   assertEquals(123, aus.getLastMetadataIndex());
 }
Example #13
0
 public void testFeatureVersion() {
   AuState aus = new AuState(mau, historyRepo);
   assertNull(aus.getFeatureVersion(Plugin.Feature.Substance));
   assertNull(aus.getFeatureVersion(Plugin.Feature.Metadata));
   assertNull(aus.getFeatureVersion(Plugin.Feature.Poll));
   aus.setFeatureVersion(Plugin.Feature.Metadata, "foo");
   assertNull(aus.getFeatureVersion(Plugin.Feature.Substance));
   assertEquals("foo", aus.getFeatureVersion(Plugin.Feature.Metadata));
   aus.setFeatureVersion(Plugin.Feature.Substance, "sub_42");
   assertEquals("sub_42", aus.getFeatureVersion(Plugin.Feature.Substance));
   assertEquals("foo", aus.getFeatureVersion(Plugin.Feature.Metadata));
   assertNull(aus.getFeatureVersion(Plugin.Feature.Poll));
 }
  public void testStoreAuState() throws Exception {
    HashSet strCol = new HashSet();
    strCol.add("test");
    AuState origState =
        new AuState(
            mau,
            123000,
            123123,
            41,
            "woop woop",
            321000,
            222000,
            3,
            "pollres",
            12345,
            456000,
            strCol,
            AuState.AccessType.OpenAccess,
            2,
            1.0,
            1.0,
            SubstanceChecker.State.Yes,
            "SubstVer3",
            "MetadatVer7",
            111444,
            12345,
            111222, // lastPoPPoll
            7, // lastPoPPollResult
            222333, // lastLocalHashScan
            444777, // numAgreePeersLastPoR
            777444, // numWillingRepairers
            747474, // numCurrentSuspectVersions
            ListUtil.list("http://hos.t/pa/th"),
            repository);

    assertEquals("SubstVer3", origState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals("MetadatVer7", origState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(111444, origState.getLastMetadataIndex());

    repository.storeAuState(origState);

    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath += HistoryRepositoryImpl.AU_FILE_NAME;
    File xmlFile = new File(filePath);
    assertTrue(xmlFile.exists());

    origState = null;
    AuState loadedState = repository.loadAuState();
    assertEquals(123000, loadedState.getLastCrawlTime());
    assertEquals(123123, loadedState.getLastCrawlAttempt());
    assertEquals(41, loadedState.getLastCrawlResult());
    assertEquals("woop woop", loadedState.getLastCrawlResultMsg());
    assertEquals(321000, loadedState.getLastTopLevelPollTime());
    assertEquals(222000, loadedState.getLastPollStart());
    assertEquals(3, loadedState.getLastPollResult());
    assertEquals("Inviting Peers", loadedState.getLastPollResultMsg());

    assertEquals(111222, loadedState.getLastPoPPoll());
    assertEquals(7, loadedState.getLastPoPPollResult());
    assertEquals(222333, loadedState.getLastLocalHashScan());

    assertEquals(444777, loadedState.getNumAgreePeersLastPoR());
    assertEquals(777444, loadedState.getNumWillingRepairers());
    assertEquals(747474, loadedState.getNumCurrentSuspectVersions());
    assertEquals(ListUtil.list("http://hos.t/pa/th"), loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(
        ListUtil.list("http://hos.t/pa/th", "http://this.is.new/"), loadedState.getCdnStems());

    assertEquals(12345, loadedState.getPollDuration());
    assertEquals(2, loadedState.getClockssSubscriptionStatus());
    assertEquals(AuState.AccessType.OpenAccess, loadedState.getAccessType());
    assertEquals(SubstanceChecker.State.Yes, loadedState.getSubstanceState());
    assertEquals("SubstVer3", loadedState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals("MetadatVer7", loadedState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(111444, loadedState.getLastMetadataIndex());
    assertEquals(12345, loadedState.getLastContentChange());
    assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());

    // check crawl urls
    Collection col = loadedState.getCrawlUrls();
    Iterator colIter = col.iterator();
    assertTrue(colIter.hasNext());
    assertEquals("test", colIter.next());
    assertFalse(colIter.hasNext());
  }
Example #15
0
  public void testClockssSubscriptionStatus() {
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(AuState.CLOCKSS_SUB_UNKNOWN, aus.getClockssSubscriptionStatus());
    assertEquals("Unknown", aus.getClockssSubscriptionStatusString());

    aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_YES);
    assertEquals(AuState.CLOCKSS_SUB_YES, aus.getClockssSubscriptionStatus());
    assertEquals("Yes", aus.getClockssSubscriptionStatusString());

    aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_NO);
    assertEquals(AuState.CLOCKSS_SUB_NO, aus.getClockssSubscriptionStatus());
    assertEquals("No", aus.getClockssSubscriptionStatusString());

    aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_INACCESSIBLE);
    assertEquals(AuState.CLOCKSS_SUB_INACCESSIBLE, aus.getClockssSubscriptionStatus());
    assertEquals("Inaccessible", aus.getClockssSubscriptionStatusString());
  }
  public void testStoreOverwrite() throws Exception {
    AuState auState =
        new AuState(
            mau,
            123, // lastCrawlTime
            321, // lastCrawlAttempt
            -1, // lastCrawlResult
            null, // lastCrawlResultMsg,
            321, // lastTopLevelPoll
            333, // lastPollStart
            -1, // lastPollresult
            null, // lastPollresultMsg
            0, // pollDuration
            -1, // lastTreeWalk
            null, // crawlUrls
            null, // accessType
            1, // clockssSubscriptionState
            1.0, // v3Agreement
            1.0, // highestV3Agreement
            SubstanceChecker.State.Unknown,
            null, // substanceVersion
            null, // metadataVersion
            -1, // lastMetadataIndex
            0, // lastContentChange
            444, // lastPoPPoll
            8, // lastPoPPollResult
            -1, // lastLocalHashScan
            27, // numAgreePeersLastPoR
            72, // numWillingRepirers
            19, // numCurrentSuspectVersions
            ListUtil.list("http://foo/"), // cdnStems
            repository);

    repository.storeAuState(auState);
    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath += HistoryRepositoryImpl.AU_FILE_NAME;
    File xmlFile = new File(filePath);
    FileInputStream fis = new FileInputStream(xmlFile);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    StreamUtil.copy(fis, baos);
    fis.close();
    String expectedStr = baos.toString();

    auState =
        new AuState(
            mau,
            1234, // lastCrawlTime
            4321, // lastCrawlAttempt
            -1, // lastCrawlResult
            null, // lastCrawlResultMsg,
            4321, // lastTopLevelPoll
            5555, // lastPollStart
            -1, // lastPollresult
            null, // lastPollresultMsg
            0, // pollDuration
            -1, // lastTreeWalk
            null, // crawlUrls
            null, // accessType
            1, // clockssSubscriptionState
            1.0, // v3Agreement
            1.0, // highestV3Agreement
            SubstanceChecker.State.Unknown,
            null, // substanceVersion
            null, // metadataVersion
            -1, // lastMetadataIndex
            0, // lastContentChange
            -1, // lastPoPPoll
            -1, // lastPoPPollResult
            -1, // lastLocalHashScan
            13, // numAgreePeersLastPoR
            31, // numWillingRepairers
            91, // numCurrentSuspectVersions
            ListUtil.list("http://foo/"), // cdnStems
            repository);
    repository.storeAuState(auState);
    assertEquals(1234, auState.getLastCrawlTime());
    assertEquals(4321, auState.getLastCrawlAttempt());
    assertEquals(4321, auState.getLastTopLevelPollTime());
    assertEquals(5555, auState.getLastPollStart());
    assertEquals(13, auState.getNumAgreePeersLastPoR());
    assertEquals(31, auState.getNumWillingRepairers());
    assertEquals(91, auState.getNumCurrentSuspectVersions());
    assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId());
    assertEquals(ListUtil.list("http://foo/"), auState.getCdnStems());

    fis = new FileInputStream(xmlFile);
    baos = new ByteArrayOutputStream(expectedStr.length());
    StreamUtil.copy(fis, baos);
    fis.close();
    log.info(baos.toString());

    auState = null;
    auState = repository.loadAuState();
    assertEquals(1234, auState.getLastCrawlTime());
    assertEquals(4321, auState.getLastCrawlAttempt());
    assertEquals(4321, auState.getLastTopLevelPollTime());
    assertEquals(5555, auState.getLastPollStart());
    assertEquals(13, auState.getNumAgreePeersLastPoR());
    assertEquals(31, auState.getNumWillingRepairers());
    assertEquals(91, auState.getNumCurrentSuspectVersions());
    assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId());

    auState =
        new AuState(
            mau,
            123, // lastCrawlTime
            321, // lastCrawlAttempt
            -1, // lastCrawlResult
            null, // lastCrawlResultMsg,
            321, // lastTopLevelPoll
            333, // lastPollStart
            -1, // lastPollresult
            null, // lastPollresultMsg
            0, // pollDuration
            -1, // lastTreeWalk
            null, // crawlUrls
            null, // accessType
            1, // clockssSubscriptionState
            1.0, // v3Agreement
            1.0, // highestV3Agreement
            SubstanceChecker.State.Unknown,
            null, // substanceVersion
            null, // metadataVersion
            -1, // lastMetadataIndex
            0, // lastContentChange
            444, // lastPoPPoll
            8, // lastPoPPollResult
            -1, // lastLocalHashScan
            27, // numAgreePeersLastPoR
            72, // numWillingRepairers
            19, // numCurrentSuspectVersions
            ListUtil.list("http://foo/"), // cdnStems
            repository);
    repository.storeAuState(auState);
    fis = new FileInputStream(xmlFile);
    baos = new ByteArrayOutputStream(expectedStr.length());
    StreamUtil.copy(fis, baos);
    fis.close();
    assertEquals(expectedStr, baos.toString());
  }
Example #17
0
 public void testSubstanceState() {
   AuState aus = new AuState(mau, historyRepo);
   assertEquals(SubstanceChecker.State.Unknown, aus.getSubstanceState());
   assertFalse(aus.hasNoSubstance());
   aus.setSubstanceState(SubstanceChecker.State.Yes);
   assertEquals(1, historyRepo.getAuStateStoreCount());
   assertEquals(SubstanceChecker.State.Yes, aus.getSubstanceState());
   assertFalse(aus.hasNoSubstance());
   aus.setSubstanceState(SubstanceChecker.State.No);
   assertEquals(2, historyRepo.getAuStateStoreCount());
   assertEquals(SubstanceChecker.State.No, aus.getSubstanceState());
   assertTrue(aus.hasNoSubstance());
   assertNotEquals("2", aus.getFeatureVersion(Plugin.Feature.Substance));
   mplug.setFeatureVersionMap(MapUtil.map(Plugin.Feature.Substance, "2"));
   aus.setSubstanceState(SubstanceChecker.State.Yes);
   // changing both the substance state and feature version should store
   // only once
   assertEquals(3, historyRepo.getAuStateStoreCount());
   assertEquals(SubstanceChecker.State.Yes, aus.getSubstanceState());
   assertEquals("2", aus.getFeatureVersion(Plugin.Feature.Substance));
 }
  public void testStoreAuEmptyState() throws Exception {
    HashSet strCol = new HashSet();
    strCol.add("test");
    AuState origState = new AuState(mau, repository);
    repository.storeAuState(origState);
    AuState loadedState = repository.loadAuState();
    assertEquals(-1, loadedState.getLastCrawlTime());
    assertEquals(-1, loadedState.getLastCrawlAttempt());
    assertEquals(-1, loadedState.getLastCrawlResult());
    assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg());
    assertEquals(-1, loadedState.getLastTopLevelPollTime());
    assertEquals(-1, loadedState.getLastPollStart());
    assertEquals(-1, loadedState.getLastPollResult());
    assertEquals(null, loadedState.getLastPollResultMsg());

    assertEquals(-1, loadedState.getLastPoPPoll());
    assertEquals(-1, loadedState.getLastPoPPollResult());
    assertEquals(-1, loadedState.getLastLocalHashScan());
    assertEquals(0, loadedState.getNumAgreePeersLastPoR());
    assertEquals(0, loadedState.getNumWillingRepairers());
    assertEquals(0, loadedState.getNumCurrentSuspectVersions());
    assertEmpty(loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());

    assertEquals(0, loadedState.getPollDuration());
    assertEquals(0, loadedState.getClockssSubscriptionStatus());
    assertEquals(null, loadedState.getAccessType());
    assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState());
    assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(-1, loadedState.getLastMetadataIndex());
    assertEquals(0, loadedState.getLastContentChange());
    assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());
  }