/**
  * Make a basic TaylorAndFrancis test AU to which URLs can be added.
  *
  * @return a basic Ingenta test AU
  * @throws ConfigurationException if can't set configuration
  */
 MockArchivalUnit makeAu() throws ConfigurationException {
   MockArchivalUnit mau = new MockArchivalUnit();
   Configuration config = ConfigurationUtil.fromArgs("base_url", "http://www.xyz.com/");
   mau.setConfiguration(config);
   mau.setUrlStems(ListUtil.list("http://www.xyz.com/"));
   return mau;
 }
Esempio n. 2
0
 private V3Poller makeV3Poller(String key) throws Exception {
   MockArchivalUnit mau = new MockArchivalUnit();
   mau.setAuId("mock");
   mau.getPlugin().initPlugin(daemon);
   PollSpec ps = new MockPollSpec(mau, "http://www.example.com/", null, null, Poll.V3_POLL);
   MockNodeManager nodeMgr = new MockNodeManager();
   daemon.setNodeManager(nodeMgr, mau);
   return new V3Poller(ps, daemon, null, key, 20000, "SHA-1");
 }
  private Set parseSingleSource(String source) throws IOException {
    MockArchivalUnit mau = new MockArchivalUnit();
    LinkExtractor ue = new RegexpCssLinkExtractor();
    mau.setLinkExtractor("text/css", ue);
    MockCachedUrl mcu = new MockCachedUrl("http://www.example.com", mau);
    mcu.setContent(source);

    cb.reset();
    extractor.extractUrls(mau, new StringInputStream(source), ENC, "http://www.example.com", cb);
    return cb.getFoundUrls();
  }
 public void testParsesOptionPositive() throws IOException {
   TypedEntryMap pMap = new TypedEntryMap();
   pMap.setMapElement("html-parser-select-attrs", ListUtil.list("value"));
   mau.setPropertyMap(pMap);
   singleTagShouldParse(
       "http://www.example.com/web_link.jpg", "<option  value=", "</option>", mau);
   singleTagShouldParse(
       "http://www.example.com/web_link.jpg", "<option a=b value=", "</option>", mau);
 }
Esempio n. 5
0
  protected void setUp() throws Exception {
    super.setUp();
    TimeBase.setSimulated();

    initRequiredServices();

    testau.setPlugin(new MockPlugin());

    initTestPeerIDs();
    initTestMsg();
    initTestPolls();
  }
  public void testStoreDamagedNodeSet() throws Exception {
    DamagedNodeSet damNodes = new DamagedNodeSet(mau, repository);
    damNodes.nodesWithDamage.add("test1");
    damNodes.nodesWithDamage.add("test2");
    damNodes.cusToRepair.put("cus1", ListUtil.list("cus1-1", "cus1-2"));
    damNodes.cusToRepair.put("cus2", ListUtil.list("cus2-1"));
    assertTrue(damNodes.containsWithDamage("test1"));
    assertTrue(damNodes.containsWithDamage("test2"));
    assertFalse(damNodes.containsWithDamage("test3"));

    repository.storeDamagedNodeSet(damNodes);
    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath += HistoryRepositoryImpl.DAMAGED_NODES_FILE_NAME;
    File xmlFile = new File(filePath);
    assertTrue(xmlFile.exists());

    damNodes = null;
    damNodes = repository.loadDamagedNodeSet();
    // check damage
    assertTrue(damNodes.containsWithDamage("test1"));
    assertTrue(damNodes.containsWithDamage("test2"));
    assertFalse(damNodes.containsWithDamage("test3"));

    MockCachedUrlSet mcus1 = new MockCachedUrlSet("cus1");
    MockCachedUrlSet mcus2 = new MockCachedUrlSet("cus2");

    // check repairs
    assertTrue(damNodes.containsToRepair(mcus1, "cus1-1"));
    assertTrue(damNodes.containsToRepair(mcus1, "cus1-2"));
    assertFalse(damNodes.containsToRepair(mcus1, "cus2-1"));
    assertTrue(damNodes.containsToRepair(mcus2, "cus2-1"));
    assertEquals(mau.getAuId(), damNodes.theAu.getAuId());

    // check remove
    damNodes.removeFromRepair(mcus1, "cus1-1");
    assertFalse(damNodes.containsToRepair(mcus1, "cus1-1"));
    assertTrue(damNodes.containsToRepair(mcus1, "cus1-2"));
    damNodes.removeFromRepair(mcus1, "cus1-2");
    assertFalse(damNodes.containsToRepair(mcus1, "cus1-2"));
    assertNull(damNodes.cusToRepair.get(mcus1));

    // check remove from damaged nodes
    damNodes.removeFromDamage("test1");
    damNodes.removeFromDamage("test2");
    repository.storeDamagedNodeSet(damNodes);
    damNodes = repository.loadDamagedNodeSet();
    assertNotNull(damNodes);
    assertFalse(damNodes.containsWithDamage("test1"));
    assertFalse(damNodes.containsWithDamage("test2"));
  }
  public void testStoreAuEmptyState() throws Exception {
    HashSet strCol = new HashSet();
    strCol.add("test");
    AuState origState = new AuState(mau, repository);
    repository.storeAuState(origState);
    AuState loadedState = repository.loadAuState();
    assertEquals(-1, loadedState.getLastCrawlTime());
    assertEquals(-1, loadedState.getLastCrawlAttempt());
    assertEquals(-1, loadedState.getLastCrawlResult());
    assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg());
    assertEquals(-1, loadedState.getLastTopLevelPollTime());
    assertEquals(-1, loadedState.getLastPollStart());
    assertEquals(-1, loadedState.getLastPollResult());
    assertEquals(null, loadedState.getLastPollResultMsg());

    assertEquals(-1, loadedState.getLastPoPPoll());
    assertEquals(-1, loadedState.getLastPoPPollResult());
    assertEquals(-1, loadedState.getLastLocalHashScan());
    assertEquals(0, loadedState.getNumAgreePeersLastPoR());
    assertEquals(0, loadedState.getNumWillingRepairers());
    assertEquals(0, loadedState.getNumCurrentSuspectVersions());
    assertEmpty(loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());

    assertEquals(0, loadedState.getPollDuration());
    assertEquals(0, loadedState.getClockssSubscriptionStatus());
    assertEquals(null, loadedState.getAccessType());
    assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState());
    assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(-1, loadedState.getLastMetadataIndex());
    assertEquals(0, loadedState.getLastContentChange());
    assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());
  }
Esempio n. 8
0
  private V1NamePoll makeCompletedNamePoll(int numAgree, int numDisagree, int numDissenting)
      throws Exception {
    V1NamePoll np = null;
    V1LcapMessage agree_msg = null;
    V1LcapMessage disagree_msg1 = null;
    V1LcapMessage disagree_msg2 = null;

    Plugin plugin = testau.getPlugin();
    PollSpec spec = new MockPollSpec(testau, rootV1urls[0], null, null, Poll.V1_NAME_POLL);
    ((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
    V1LcapMessage poll_msg =
        V1LcapMessage.makeRequestMsg(
            spec,
            null,
            ByteArray.makeRandomBytes(20),
            ByteArray.makeRandomBytes(20),
            V1LcapMessage.NAME_POLL_REQ,
            testduration,
            testID);

    // make our poll
    np =
        (V1NamePoll)
            new V1NamePoll(
                spec,
                pollmanager,
                poll_msg.getOriginatorId(),
                poll_msg.getChallenge(),
                poll_msg.getDuration(),
                poll_msg.getHashAlgorithm());
    np.setMessage(poll_msg);

    // generate agree vote msg
    agree_msg =
        V1LcapMessage.makeReplyMsg(
            poll_msg,
            ByteArray.makeRandomBytes(20),
            poll_msg.getVerifier(),
            agree_entries,
            V1LcapMessage.NAME_POLL_REP,
            testduration,
            testID);

    // generate a disagree vote msg
    disagree_msg1 =
        V1LcapMessage.makeReplyMsg(
            poll_msg,
            ByteArray.makeRandomBytes(20),
            ByteArray.makeRandomBytes(20),
            disagree_entries,
            V1LcapMessage.NAME_POLL_REP,
            testduration,
            testID1);
    // generate a losing disagree vote msg
    disagree_msg2 =
        V1LcapMessage.makeReplyMsg(
            poll_msg,
            ByteArray.makeRandomBytes(20),
            ByteArray.makeRandomBytes(20),
            dissenting_entries,
            V1LcapMessage.NAME_POLL_REP,
            testduration,
            testID1);

    // add our vote
    V1LcapMessage msg = (V1LcapMessage) (np.getMessage());
    PeerIdentity id = msg.getOriginatorId();
    np.m_tally.addVote(np.makeNameVote(msg, true), id, true);

    // add the agree votes
    id = agree_msg.getOriginatorId();
    for (int i = 0; i < numAgree; i++) {
      np.m_tally.addVote(np.makeNameVote(agree_msg, true), id, false);
    }

    // add the disagree votes
    id = disagree_msg1.getOriginatorId();
    for (int i = 0; i < numDisagree; i++) {
      np.m_tally.addVote(np.makeNameVote(disagree_msg1, false), id, false);
    }

    // add dissenting disagree vote
    id = disagree_msg2.getOriginatorId();
    for (int i = 0; i < numDissenting; i++) {
      np.m_tally.addVote(np.makeNameVote(disagree_msg2, false), id, false);
    }
    np.m_pollstate = V1Poll.PS_COMPLETE;
    np.m_tally.tallyVotes();
    return np;
  }
  public void testStoreOverwrite() throws Exception {
    AuState auState =
        new AuState(
            mau,
            123, // lastCrawlTime
            321, // lastCrawlAttempt
            -1, // lastCrawlResult
            null, // lastCrawlResultMsg,
            321, // lastTopLevelPoll
            333, // lastPollStart
            -1, // lastPollresult
            null, // lastPollresultMsg
            0, // pollDuration
            -1, // lastTreeWalk
            null, // crawlUrls
            null, // accessType
            1, // clockssSubscriptionState
            1.0, // v3Agreement
            1.0, // highestV3Agreement
            SubstanceChecker.State.Unknown,
            null, // substanceVersion
            null, // metadataVersion
            -1, // lastMetadataIndex
            0, // lastContentChange
            444, // lastPoPPoll
            8, // lastPoPPollResult
            -1, // lastLocalHashScan
            27, // numAgreePeersLastPoR
            72, // numWillingRepirers
            19, // numCurrentSuspectVersions
            ListUtil.list("http://foo/"), // cdnStems
            repository);

    repository.storeAuState(auState);
    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath += HistoryRepositoryImpl.AU_FILE_NAME;
    File xmlFile = new File(filePath);
    FileInputStream fis = new FileInputStream(xmlFile);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    StreamUtil.copy(fis, baos);
    fis.close();
    String expectedStr = baos.toString();

    auState =
        new AuState(
            mau,
            1234, // lastCrawlTime
            4321, // lastCrawlAttempt
            -1, // lastCrawlResult
            null, // lastCrawlResultMsg,
            4321, // lastTopLevelPoll
            5555, // lastPollStart
            -1, // lastPollresult
            null, // lastPollresultMsg
            0, // pollDuration
            -1, // lastTreeWalk
            null, // crawlUrls
            null, // accessType
            1, // clockssSubscriptionState
            1.0, // v3Agreement
            1.0, // highestV3Agreement
            SubstanceChecker.State.Unknown,
            null, // substanceVersion
            null, // metadataVersion
            -1, // lastMetadataIndex
            0, // lastContentChange
            -1, // lastPoPPoll
            -1, // lastPoPPollResult
            -1, // lastLocalHashScan
            13, // numAgreePeersLastPoR
            31, // numWillingRepairers
            91, // numCurrentSuspectVersions
            ListUtil.list("http://foo/"), // cdnStems
            repository);
    repository.storeAuState(auState);
    assertEquals(1234, auState.getLastCrawlTime());
    assertEquals(4321, auState.getLastCrawlAttempt());
    assertEquals(4321, auState.getLastTopLevelPollTime());
    assertEquals(5555, auState.getLastPollStart());
    assertEquals(13, auState.getNumAgreePeersLastPoR());
    assertEquals(31, auState.getNumWillingRepairers());
    assertEquals(91, auState.getNumCurrentSuspectVersions());
    assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId());
    assertEquals(ListUtil.list("http://foo/"), auState.getCdnStems());

    fis = new FileInputStream(xmlFile);
    baos = new ByteArrayOutputStream(expectedStr.length());
    StreamUtil.copy(fis, baos);
    fis.close();
    log.info(baos.toString());

    auState = null;
    auState = repository.loadAuState();
    assertEquals(1234, auState.getLastCrawlTime());
    assertEquals(4321, auState.getLastCrawlAttempt());
    assertEquals(4321, auState.getLastTopLevelPollTime());
    assertEquals(5555, auState.getLastPollStart());
    assertEquals(13, auState.getNumAgreePeersLastPoR());
    assertEquals(31, auState.getNumWillingRepairers());
    assertEquals(91, auState.getNumCurrentSuspectVersions());
    assertEquals(mau.getAuId(), auState.getArchivalUnit().getAuId());

    auState =
        new AuState(
            mau,
            123, // lastCrawlTime
            321, // lastCrawlAttempt
            -1, // lastCrawlResult
            null, // lastCrawlResultMsg,
            321, // lastTopLevelPoll
            333, // lastPollStart
            -1, // lastPollresult
            null, // lastPollresultMsg
            0, // pollDuration
            -1, // lastTreeWalk
            null, // crawlUrls
            null, // accessType
            1, // clockssSubscriptionState
            1.0, // v3Agreement
            1.0, // highestV3Agreement
            SubstanceChecker.State.Unknown,
            null, // substanceVersion
            null, // metadataVersion
            -1, // lastMetadataIndex
            0, // lastContentChange
            444, // lastPoPPoll
            8, // lastPoPPollResult
            -1, // lastLocalHashScan
            27, // numAgreePeersLastPoR
            72, // numWillingRepairers
            19, // numCurrentSuspectVersions
            ListUtil.list("http://foo/"), // cdnStems
            repository);
    repository.storeAuState(auState);
    fis = new FileInputStream(xmlFile);
    baos = new ByteArrayOutputStream(expectedStr.length());
    StreamUtil.copy(fis, baos);
    fis.close();
    assertEquals(expectedStr, baos.toString());
  }
  public void testStoreAuState() throws Exception {
    HashSet strCol = new HashSet();
    strCol.add("test");
    AuState origState =
        new AuState(
            mau,
            123000,
            123123,
            41,
            "woop woop",
            321000,
            222000,
            3,
            "pollres",
            12345,
            456000,
            strCol,
            AuState.AccessType.OpenAccess,
            2,
            1.0,
            1.0,
            SubstanceChecker.State.Yes,
            "SubstVer3",
            "MetadatVer7",
            111444,
            12345,
            111222, // lastPoPPoll
            7, // lastPoPPollResult
            222333, // lastLocalHashScan
            444777, // numAgreePeersLastPoR
            777444, // numWillingRepairers
            747474, // numCurrentSuspectVersions
            ListUtil.list("http://hos.t/pa/th"),
            repository);

    assertEquals("SubstVer3", origState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals("MetadatVer7", origState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(111444, origState.getLastMetadataIndex());

    repository.storeAuState(origState);

    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath += HistoryRepositoryImpl.AU_FILE_NAME;
    File xmlFile = new File(filePath);
    assertTrue(xmlFile.exists());

    origState = null;
    AuState loadedState = repository.loadAuState();
    assertEquals(123000, loadedState.getLastCrawlTime());
    assertEquals(123123, loadedState.getLastCrawlAttempt());
    assertEquals(41, loadedState.getLastCrawlResult());
    assertEquals("woop woop", loadedState.getLastCrawlResultMsg());
    assertEquals(321000, loadedState.getLastTopLevelPollTime());
    assertEquals(222000, loadedState.getLastPollStart());
    assertEquals(3, loadedState.getLastPollResult());
    assertEquals("Inviting Peers", loadedState.getLastPollResultMsg());

    assertEquals(111222, loadedState.getLastPoPPoll());
    assertEquals(7, loadedState.getLastPoPPollResult());
    assertEquals(222333, loadedState.getLastLocalHashScan());

    assertEquals(444777, loadedState.getNumAgreePeersLastPoR());
    assertEquals(777444, loadedState.getNumWillingRepairers());
    assertEquals(747474, loadedState.getNumCurrentSuspectVersions());
    assertEquals(ListUtil.list("http://hos.t/pa/th"), loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(
        ListUtil.list("http://hos.t/pa/th", "http://this.is.new/"), loadedState.getCdnStems());

    assertEquals(12345, loadedState.getPollDuration());
    assertEquals(2, loadedState.getClockssSubscriptionStatus());
    assertEquals(AuState.AccessType.OpenAccess, loadedState.getAccessType());
    assertEquals(SubstanceChecker.State.Yes, loadedState.getSubstanceState());
    assertEquals("SubstVer3", loadedState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals("MetadatVer7", loadedState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(111444, loadedState.getLastMetadataIndex());
    assertEquals(12345, loadedState.getLastContentChange());
    assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());

    // check crawl urls
    Collection col = loadedState.getCrawlUrls();
    Iterator colIter = col.iterator();
    assertTrue(colIter.hasNext());
    assertEquals("test", colIter.next());
    assertFalse(colIter.hasNext());
  }