Ejemplo n.º 1
0
 private void checkSubstance(ArchivalUnit au) {
   SubstanceChecker subChecker = new SubstanceChecker(au);
   if (!subChecker.isEnabled()) {
     errMsg = "No substance patterns defined for plugin.";
     return;
   }
   AuState auState = AuUtil.getAuState(au);
   SubstanceChecker.State oldState = auState.getSubstanceState();
   SubstanceChecker.State newState = subChecker.findSubstance();
   String chtxt = (newState == oldState ? "(unchanged)" : "(was " + oldState.toString() + ")");
   switch (newState) {
     case Unknown:
       log.error("Shouldn't happen: SubstanceChecker returned Unknown");
       errMsg = "Error in SubstanceChecker; see log.";
       break;
     case Yes:
       statusMsg = "AU has substance " + chtxt + ": " + au.getName();
       auState.setSubstanceState(SubstanceChecker.State.Yes);
       break;
     case No:
       statusMsg = "AU has no substance " + chtxt + ": " + au.getName();
       auState.setSubstanceState(SubstanceChecker.State.No);
       break;
   }
 }
Ejemplo n.º 2
0
  public void testTreeWalkFinished() {
    AuState auState = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo);
    assertEquals(123, auState.getLastTreeWalkTime());

    TimeBase.setSimulated(456);
    auState.setLastTreeWalkTime();
    assertEquals(456, auState.getLastTreeWalkTime());
  }
Ejemplo n.º 3
0
  private boolean startReindexingMetadata(ArchivalUnit au, boolean force) {
    if (metadataMgr == null) {
      errMsg = "Metadata processing is not enabled.";
      return false;
    }

    if (!force) {
      if (!AuUtil.hasCrawled(au)) {
        errMsg = "Au has never crawled. Click again to reindex metadata";
        showForceReindexMetadata = true;
        return false;
      }

      AuState auState = AuUtil.getAuState(au);
      switch (auState.getSubstanceState()) {
        case No:
          errMsg = "Au has no substance. Click again to reindex metadata";
          showForceReindexMetadata = true;
          return false;
        case Unknown:
          errMsg = "Unknown substance for Au. Click again to reindex metadata.";
          showForceReindexMetadata = true;
          return false;
        case Yes:
          // fall through
      }
    }

    // Fully reindex metadata with the highest priority.
    Connection conn = null;
    PreparedStatement insertPendingAuBatchStatement = null;

    try {
      conn = dbMgr.getConnection();
      insertPendingAuBatchStatement = metadataMgr.getPrioritizedInsertPendingAuBatchStatement(conn);

      if (metadataMgr.enableAndAddAuToReindex(
          au, conn, insertPendingAuBatchStatement, false, true)) {
        statusMsg = "Reindexing metadata for " + au.getName();
        return true;
      }
    } catch (DbException dbe) {
      log.error("Cannot reindex metadata for " + au.getName(), dbe);
    } finally {
      DbManager.safeCloseStatement(insertPendingAuBatchStatement);
      DbManager.safeRollbackAndClose(conn);
    }

    if (force) {
      errMsg = "Still cannot reindex metadata for " + au.getName();
    } else {
      errMsg = "Cannot reindex metadata for " + au.getName();
    }
    return false;
  }
Ejemplo n.º 4
0
  // Ensure that fields added to AuState get their default value when
  // deserialized from old files not containing the field
  public void testNewField() throws Exception {
    AuState aus = makeAuState(mau, -1, -1, -1, -1, 123, null, 1, -1.0, 1.0, historyRepo);

    String ser = ser(aus);
    String edser = ser.replaceAll(".*numAgreePeersLastPoR.*", "");
    log.debug2("old: " + ser);
    log.debug2("new: " + edser);
    AuState newaus = deser(edser);
    assertEquals(-1, newaus.getNumAgreePeersLastPoR());
    assertEquals(Collections.EMPTY_LIST, newaus.getCdnStems());
  }
Ejemplo n.º 5
0
  public void testForceUpdateUrls() {
    AuState auState = new AuState(mau, historyRepo);
    assertNull(historyRepo.theAuState);

    Collection col = auState.getCrawlUrls();
    if (AuState.URL_UPDATE_LIMIT > 1) {
      col.add("test");
      auState.updatedCrawlUrls(true);
      assertEquals(0, auState.urlUpdateCntr);
      assertNotNull(historyRepo.theAuState);
    }
  }
Ejemplo n.º 6
0
  public void testGetUrls() {
    HashSet stringCollection = new HashSet();
    stringCollection.add("test");

    AuState auState =
        makeAuState(mau, -1, -1, -1, -1, 123, stringCollection, 1, -1.0, 1.0, historyRepo);
    Collection col = auState.getCrawlUrls();
    Iterator colIter = col.iterator();
    assertTrue(colIter.hasNext());
    assertEquals("test", colIter.next());
    assertFalse(colIter.hasNext());
  }
Ejemplo n.º 7
0
 public void testAccessType() {
   AuState aus = new AuState(mau, historyRepo);
   assertFalse(aus.isOpenAccess());
   aus.setAccessType(AuState.AccessType.Subscription);
   assertEquals(AuState.AccessType.Subscription, aus.getAccessType());
   assertFalse(aus.isOpenAccess());
   aus.setAccessType(AuState.AccessType.OpenAccess);
   assertEquals(AuState.AccessType.OpenAccess, aus.getAccessType());
   assertTrue(aus.isOpenAccess());
 }
Ejemplo n.º 8
0
 public void testLastContentChange() {
   TimeBase.setSimulated(10);
   AuState aus = new AuState(mau, historyRepo);
   aus.newCrawlStarted();
   TimeBase.step(10);
   aus.contentChanged();
   assertEquals(20, aus.getLastContentChange());
   TimeBase.step(10);
   aus.contentChanged();
   assertEquals(20, aus.getLastContentChange());
   TimeBase.step(10);
   aus.newCrawlFinished(1, "foo");
   TimeBase.step(10);
   aus.contentChanged();
   assertEquals(50, aus.getLastContentChange());
 }
Ejemplo n.º 9
0
  public void testUpdateUrls() {
    AuState auState = new AuState(mau, historyRepo);
    assertNull(historyRepo.theAuState);

    Collection col = auState.getCrawlUrls();
    for (int ii = 1; ii < AuState.URL_UPDATE_LIMIT; ii++) {
      col.add("test" + ii);
      auState.updatedCrawlUrls(false);
      assertEquals(ii, auState.urlUpdateCntr);
      assertNull(historyRepo.theAuState);
    }
    col.add("test-limit");
    auState.updatedCrawlUrls(false);
    assertEquals(0, auState.urlUpdateCntr);
    assertNotNull(historyRepo.theAuState);

    // clear, and check that counter is reset
    historyRepo.theAuState = null;
    if (AuState.URL_UPDATE_LIMIT > 1) {
      col.add("test");
      auState.updatedCrawlUrls(false);
      assertNull(historyRepo.theAuState);
    }
  }
Ejemplo n.º 10
0
  public void testBatch() {
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(0, historyRepo.getAuStateStoreCount());
    aus.setNumAgreePeersLastPoR(1);
    aus.setNumWillingRepairers(3);
    aus.setNumCurrentSuspectVersions(5);
    assertEquals(3, historyRepo.getAuStateStoreCount());

    aus.batchSaves();
    aus.setNumAgreePeersLastPoR(2);
    aus.setNumWillingRepairers(4);
    aus.setNumCurrentSuspectVersions(6);
    assertEquals(3, historyRepo.getAuStateStoreCount());
    aus.unBatchSaves();
    assertEquals(4, historyRepo.getAuStateStoreCount());

    aus.batchSaves();
    aus.setNumAgreePeersLastPoR(4);
    aus.batchSaves();
    aus.setNumWillingRepairers(8);
    aus.setNumCurrentSuspectVersions(12);
    assertEquals(4, historyRepo.getAuStateStoreCount());
    aus.unBatchSaves();
    assertEquals(4, historyRepo.getAuStateStoreCount());
    aus.unBatchSaves();
    assertEquals(5, historyRepo.getAuStateStoreCount());
  }
Ejemplo n.º 11
0
  public void testCdnStems() {
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(Collections.EMPTY_LIST, aus.getCdnStems());
    aus.addCdnStem("http://fff.uselesstld");
    assertClass(ArrayList.class, aus.getCdnStems());
    assertEquals(ListUtil.list("http://fff.uselesstld"), aus.getCdnStems());
    aus.addCdnStem("ccc");
    assertEquals(ListUtil.list("http://fff.uselesstld", "ccc"), aus.getCdnStems());

    aus.setCdnStems(new LinkedList(ListUtil.list("a", "b")));
    assertClass(ArrayList.class, aus.getCdnStems());
    assertEquals(ListUtil.list("a", "b"), aus.getCdnStems());
    aus.setCdnStems(null);
    assertEmpty(aus.getCdnStems());
    aus.addCdnStem("https://a.b/");
    aus.addCdnStem("https://b.a/");
    assertEquals(ListUtil.list("https://a.b/", "https://b.a/"), aus.getCdnStems());
  }
Ejemplo n.º 12
0
  public void testNumCurrentSuspectVersions() {
    MyMockLockssRepository repo = new MyMockLockssRepository();
    MyAuSuspectUrlVersions asuv = new MyAuSuspectUrlVersions();
    repo.setAsuv(asuv);

    daemon.setLockssRepository(repo, mau);
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(0, aus.getNumCurrentSuspectVersions());
    // ensure this isn't automatically recomputed, as that would happen
    // when historyRepo loads the object during startAuManagers, before the
    // AU is fully created.
    aus.setNumCurrentSuspectVersions(-1);
    assertEquals(-1, aus.getNumCurrentSuspectVersions());
    asuv.setCountResult(17);
    aus.recomputeNumCurrentSuspectVersions();
    assertEquals(17, aus.getNumCurrentSuspectVersions());
    aus.incrementNumCurrentSuspectVersions(-1);
    assertEquals(16, aus.getNumCurrentSuspectVersions());

    aus.setNumCurrentSuspectVersions(-1);
    asuv.setCountResult(6);
    aus.incrementNumCurrentSuspectVersions(-1);
    assertEquals(5, aus.getNumCurrentSuspectVersions());
  }
Ejemplo n.º 13
0
 public void testLastMetadataIndex() {
   AuState aus = new AuState(mau, historyRepo);
   assertEquals(-1, aus.getLastMetadataIndex());
   aus.setLastMetadataIndex(123);
   assertEquals(123, aus.getLastMetadataIndex());
 }
Ejemplo n.º 14
0
 public void testFeatureVersion() {
   AuState aus = new AuState(mau, historyRepo);
   assertNull(aus.getFeatureVersion(Plugin.Feature.Substance));
   assertNull(aus.getFeatureVersion(Plugin.Feature.Metadata));
   assertNull(aus.getFeatureVersion(Plugin.Feature.Poll));
   aus.setFeatureVersion(Plugin.Feature.Metadata, "foo");
   assertNull(aus.getFeatureVersion(Plugin.Feature.Substance));
   assertEquals("foo", aus.getFeatureVersion(Plugin.Feature.Metadata));
   aus.setFeatureVersion(Plugin.Feature.Substance, "sub_42");
   assertEquals("sub_42", aus.getFeatureVersion(Plugin.Feature.Substance));
   assertEquals("foo", aus.getFeatureVersion(Plugin.Feature.Metadata));
   assertNull(aus.getFeatureVersion(Plugin.Feature.Poll));
 }
Ejemplo n.º 15
0
 public void testSubstanceState() {
   AuState aus = new AuState(mau, historyRepo);
   assertEquals(SubstanceChecker.State.Unknown, aus.getSubstanceState());
   assertFalse(aus.hasNoSubstance());
   aus.setSubstanceState(SubstanceChecker.State.Yes);
   assertEquals(1, historyRepo.getAuStateStoreCount());
   assertEquals(SubstanceChecker.State.Yes, aus.getSubstanceState());
   assertFalse(aus.hasNoSubstance());
   aus.setSubstanceState(SubstanceChecker.State.No);
   assertEquals(2, historyRepo.getAuStateStoreCount());
   assertEquals(SubstanceChecker.State.No, aus.getSubstanceState());
   assertTrue(aus.hasNoSubstance());
   assertNotEquals("2", aus.getFeatureVersion(Plugin.Feature.Substance));
   mplug.setFeatureVersionMap(MapUtil.map(Plugin.Feature.Substance, "2"));
   aus.setSubstanceState(SubstanceChecker.State.Yes);
   // changing both the substance state and feature version should store
   // only once
   assertEquals(3, historyRepo.getAuStateStoreCount());
   assertEquals(SubstanceChecker.State.Yes, aus.getSubstanceState());
   assertEquals("2", aus.getFeatureVersion(Plugin.Feature.Substance));
 }
Ejemplo n.º 16
0
  public void testClockssSubscriptionStatus() {
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(AuState.CLOCKSS_SUB_UNKNOWN, aus.getClockssSubscriptionStatus());
    assertEquals("Unknown", aus.getClockssSubscriptionStatusString());

    aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_YES);
    assertEquals(AuState.CLOCKSS_SUB_YES, aus.getClockssSubscriptionStatus());
    assertEquals("Yes", aus.getClockssSubscriptionStatusString());

    aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_NO);
    assertEquals(AuState.CLOCKSS_SUB_NO, aus.getClockssSubscriptionStatus());
    assertEquals("No", aus.getClockssSubscriptionStatusString());

    aus.setClockssSubscriptionStatus(AuState.CLOCKSS_SUB_INACCESSIBLE);
    assertEquals(AuState.CLOCKSS_SUB_INACCESSIBLE, aus.getClockssSubscriptionStatus());
    assertEquals("Inaccessible", aus.getClockssSubscriptionStatusString());
  }