Example #1
0
  private V3LcapMessage makeTestVoteMessage(Collection voteBlocks) throws IOException {
    mPollMgr.setStateDir("key", tempDir);
    V3LcapMessage msg =
        new V3LcapMessage(
            "ArchivalID_2",
            "key",
            "Plug42",
            m_testBytes,
            m_testBytes,
            V3LcapMessage.MSG_VOTE,
            987654321,
            m_testID,
            tempDir,
            theDaemon);

    // Set msg vote blocks.
    for (Iterator ix = voteBlocks.iterator(); ix.hasNext(); ) {
      msg.addVoteBlock((VoteBlock) ix.next());
    }

    msg.setHashAlgorithm(LcapMessage.getDefaultHashAlgorithm());
    msg.setArchivalId(m_archivalID);
    msg.setPluginVersion("PlugVer42");
    return msg;
  }
 /** iterate over the mime-types makeRow for each */
 private List getRows(CrawlerStatus status, String key) {
   Collection mimeTypes = status.getMimeTypes();
   List rows = new ArrayList();
   if (mimeTypes != null) {
     String mimeType;
     for (Iterator it = mimeTypes.iterator(); it.hasNext(); ) {
       mimeType = (String) it.next();
       rows.add(makeRow(status, mimeType, key));
     }
   }
   return rows;
 }
Example #3
0
  public void testForceUpdateUrls() {
    AuState auState = new AuState(mau, historyRepo);
    assertNull(historyRepo.theAuState);

    Collection col = auState.getCrawlUrls();
    if (AuState.URL_UPDATE_LIMIT > 1) {
      col.add("test");
      auState.updatedCrawlUrls(true);
      assertEquals(0, auState.urlUpdateCntr);
      assertNotNull(historyRepo.theAuState);
    }
  }
Example #4
0
  public void testGetUrls() {
    HashSet stringCollection = new HashSet();
    stringCollection.add("test");

    AuState auState =
        makeAuState(mau, -1, -1, -1, -1, 123, stringCollection, 1, -1.0, 1.0, historyRepo);
    Collection col = auState.getCrawlUrls();
    Iterator colIter = col.iterator();
    assertTrue(colIter.hasNext());
    assertEquals("test", colIter.next());
    assertFalse(colIter.hasNext());
  }
Example #5
0
 /**
  * Unregister the au with this Tdb for its plugin.
  *
  * @param au the TdbAu
  * @return <code>false</code> if au was not registered, otherwise <code>true</code>
  */
 private boolean removeTdbAuForPlugin(TdbAu au) {
   // if can't add au to title, we need to undo the au
   // registration and re-throw the exception we just caught
   String pluginId = au.getPluginId();
   Collection<TdbAu.Id> c = pluginIdTdbAuIdsMap.get(pluginId);
   if (c.remove(au.getId())) {
     if (c.isEmpty()) {
       pluginIdTdbAuIdsMap.remove(c);
     }
     tdbAuCount--;
     return true;
   }
   return false;
 }
Example #6
0
  /**
   * Register the au with this Tdb for its plugin.
   *
   * @param au the TdbAu
   * @return <code>false</code> if already registered, otherwise <code>true</code>
   */
  private boolean addTdbAuForPlugin(TdbAu au) {
    // add AU to list for plugins
    String pluginId = au.getPluginId();
    Collection<TdbAu.Id> auids = pluginIdTdbAuIdsMap.get(pluginId);
    if (auids == null) {
      auids = new HashSet<TdbAu.Id>();
      pluginIdTdbAuIdsMap.put(pluginId, auids);
    }

    if (!auids.add(au.getId())) {
      return false;
    }

    // increment the total AU count;
    tdbAuCount++;
    return true;
  }
Example #7
0
 /**
  * Get the linked titles for the specified link type.
  *
  * @param linkType the link type {@see TdbTitle} for description of link types
  * @param title the TdbTitle with links
  * @return a collection of linked titles for the specified type
  */
 public Collection<TdbTitle> getLinkedTdbTitlesForType(
     TdbTitle.LinkType linkType, TdbTitle title) {
   if (linkType == null) {
     throw new IllegalArgumentException("linkType cannot be null");
   }
   if (title == null) {
     throw new IllegalArgumentException("title cannot be null");
   }
   Collection<String> titleIds = title.getLinkedTdbTitleIdsForType(linkType);
   if (titleIds.isEmpty()) {
     return Collections.emptyList();
   }
   ArrayList<TdbTitle> titles = new ArrayList<TdbTitle>();
   for (String titleId : titleIds) {
     TdbTitle aTitle = getTdbTitleById(titleId);
     if (aTitle != null) {
       titles.add(aTitle);
     }
   }
   titles.trimToSize();
   return titles;
 }
Example #8
0
  public void testUpdateUrls() {
    AuState auState = new AuState(mau, historyRepo);
    assertNull(historyRepo.theAuState);

    Collection col = auState.getCrawlUrls();
    for (int ii = 1; ii < AuState.URL_UPDATE_LIMIT; ii++) {
      col.add("test" + ii);
      auState.updatedCrawlUrls(false);
      assertEquals(ii, auState.urlUpdateCntr);
      assertNull(historyRepo.theAuState);
    }
    col.add("test-limit");
    auState.updatedCrawlUrls(false);
    assertEquals(0, auState.urlUpdateCntr);
    assertNotNull(historyRepo.theAuState);

    // clear, and check that counter is reset
    historyRepo.theAuState = null;
    if (AuState.URL_UPDATE_LIMIT > 1) {
      col.add("test");
      auState.updatedCrawlUrls(false);
      assertNull(historyRepo.theAuState);
    }
  }
  public void testStoreAuState() throws Exception {
    HashSet strCol = new HashSet();
    strCol.add("test");
    AuState origState =
        new AuState(
            mau,
            123000,
            123123,
            41,
            "woop woop",
            321000,
            222000,
            3,
            "pollres",
            12345,
            456000,
            strCol,
            AuState.AccessType.OpenAccess,
            2,
            1.0,
            1.0,
            SubstanceChecker.State.Yes,
            "SubstVer3",
            "MetadatVer7",
            111444,
            12345,
            111222, // lastPoPPoll
            7, // lastPoPPollResult
            222333, // lastLocalHashScan
            444777, // numAgreePeersLastPoR
            777444, // numWillingRepairers
            747474, // numCurrentSuspectVersions
            ListUtil.list("http://hos.t/pa/th"),
            repository);

    assertEquals("SubstVer3", origState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals("MetadatVer7", origState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(111444, origState.getLastMetadataIndex());

    repository.storeAuState(origState);

    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath += HistoryRepositoryImpl.AU_FILE_NAME;
    File xmlFile = new File(filePath);
    assertTrue(xmlFile.exists());

    origState = null;
    AuState loadedState = repository.loadAuState();
    assertEquals(123000, loadedState.getLastCrawlTime());
    assertEquals(123123, loadedState.getLastCrawlAttempt());
    assertEquals(41, loadedState.getLastCrawlResult());
    assertEquals("woop woop", loadedState.getLastCrawlResultMsg());
    assertEquals(321000, loadedState.getLastTopLevelPollTime());
    assertEquals(222000, loadedState.getLastPollStart());
    assertEquals(3, loadedState.getLastPollResult());
    assertEquals("Inviting Peers", loadedState.getLastPollResultMsg());

    assertEquals(111222, loadedState.getLastPoPPoll());
    assertEquals(7, loadedState.getLastPoPPollResult());
    assertEquals(222333, loadedState.getLastLocalHashScan());

    assertEquals(444777, loadedState.getNumAgreePeersLastPoR());
    assertEquals(777444, loadedState.getNumWillingRepairers());
    assertEquals(747474, loadedState.getNumCurrentSuspectVersions());
    assertEquals(ListUtil.list("http://hos.t/pa/th"), loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(
        ListUtil.list("http://hos.t/pa/th", "http://this.is.new/"), loadedState.getCdnStems());

    assertEquals(12345, loadedState.getPollDuration());
    assertEquals(2, loadedState.getClockssSubscriptionStatus());
    assertEquals(AuState.AccessType.OpenAccess, loadedState.getAccessType());
    assertEquals(SubstanceChecker.State.Yes, loadedState.getSubstanceState());
    assertEquals("SubstVer3", loadedState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals("MetadatVer7", loadedState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(111444, loadedState.getLastMetadataIndex());
    assertEquals(12345, loadedState.getLastContentChange());
    assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());

    // check crawl urls
    Collection col = loadedState.getCrawlUrls();
    Iterator colIter = col.iterator();
    assertTrue(colIter.hasNext());
    assertEquals("test", colIter.next());
    assertFalse(colIter.hasNext());
  }