Пример #1
0
  private void assertEqualMessages(V3LcapMessage a, V3LcapMessage b) throws Exception {
    assertTrue(a.getOriginatorId() == b.getOriginatorId());
    assertEquals(a.getOpcode(), b.getOpcode());
    assertEquals(a.getTargetUrl(), b.getTargetUrl());
    assertEquals(a.getArchivalId(), b.getArchivalId());
    assertEquals(a.getProtocolVersion(), b.getProtocolVersion());
    assertEquals(a.getPollerNonce(), b.getPollerNonce());
    assertEquals(a.getVoterNonce(), b.getVoterNonce());
    assertEquals(a.getVoterNonce2(), b.getVoterNonce2());
    assertEquals(a.getPluginVersion(), b.getPluginVersion());
    assertEquals(a.getHashAlgorithm(), b.getHashAlgorithm());
    assertEquals(a.isVoteComplete(), b.isVoteComplete());
    assertEquals(a.getRepairDataLength(), b.getRepairDataLength());
    assertEquals(a.getLastVoteBlockURL(), b.getLastVoteBlockURL());
    assertIsomorphic(a.getNominees(), b.getNominees());
    List aBlocks = new ArrayList();
    List bBlocks = new ArrayList();
    for (VoteBlocksIterator iter = a.getVoteBlockIterator(); iter.hasNext(); ) {
      aBlocks.add(iter.next());
    }
    for (VoteBlocksIterator iter = b.getVoteBlockIterator(); iter.hasNext(); ) {
      bBlocks.add(iter.next());
    }
    assertTrue(aBlocks.equals(bBlocks));

    //  TODO: Figure out how to test time.

  }
 public void setRepoMap(Map<String, PlatformUtil.DF> repoMap) {
   List repos = new ArrayList();
   this.repoMap = repoMap;
   for (String repo : repoMap.keySet()) {
     repos.add(repo);
   }
   setRepos(repos);
 }
 public void emitMetadata(ArticleFiles af, ArticleMetadata md) {
   if (log.isDebug3()) log.debug3("emit(" + af + ", " + md + ")");
   if (md != null) {
     log.debug3("add " + md + " to amlist");
     amlst.add(md);
   }
   ;
 }
Пример #4
0
 // make a Schedule with one chunk per task
 Schedule sched(List tasks) {
   List events = new ArrayList();
   for (Iterator iter = tasks.iterator(); iter.hasNext(); ) {
     Object obj = iter.next();
     if (obj instanceof Schedule.Event) {
       events.add(obj);
     } else {
       SchedulableTask task = (SchedulableTask) obj;
       if (task.isBackgroundTask()) {
         events.add(bEvent((BackgroundTask) task, Schedule.EventType.START));
       } else {
         events.add(chunk((StepTask) task));
       }
     }
   }
   Schedule s = new Schedule(events);
   return s;
 }
 List<String> auUrls(ArchivalUnit au) {
   List<String> res = new ArrayList<String>();
   for (Iterator iter = au.getAuCachedUrlSet().contentHashIterator(); iter.hasNext(); ) {
     CachedUrlSetNode cusn = (CachedUrlSetNode) iter.next();
     if (cusn.hasContent()) {
       res.add(cusn.getUrl());
     }
   }
   return res;
 }
 private List getSummaryInfo(CrawlerStatus status) {
   List res = new ArrayList();
   StatusTable.SummaryInfo statusSi =
       new StatusTable.SummaryInfo(
           "Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg());
   ArchivalUnit au = status.getAu();
   if (au != null) {
     AuState aus = AuUtil.getAuState(au);
     if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) {
       statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS);
     }
   }
   res.add(statusSi);
   String sources = StringUtil.separatedString(status.getSources());
   res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources));
   String startUrls = StringUtil.separatedString(status.getStartUrls());
   res.add(
       new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls));
   return res;
 }
Пример #7
0
  public void testRequestMessageCreation() throws Exception {
    V3LcapMessage reqMsg =
        new V3LcapMessage(
            "ArchivalID_2",
            "key",
            "Plug42",
            m_testBytes,
            m_testBytes,
            V3LcapMessage.MSG_REPAIR_REQ,
            987654321,
            m_testID,
            tempDir,
            theDaemon);
    reqMsg.setTargetUrl("http://foo.com/");

    for (Iterator ix = m_testVoteBlocks.iterator(); ix.hasNext(); ) {
      reqMsg.addVoteBlock((VoteBlock) ix.next());
    }

    assertEquals(3, reqMsg.getProtocolVersion());
    assertEquals("Plug42", reqMsg.getPluginVersion());
    assertTrue(m_testID == reqMsg.getOriginatorId());
    assertEquals(V3LcapMessage.MSG_REPAIR_REQ, reqMsg.getOpcode());
    assertEquals("ArchivalID_2", reqMsg.getArchivalId());
    assertEquals("http://foo.com/", reqMsg.getTargetUrl());
    assertEquals(m_testBytes, reqMsg.getPollerNonce());
    assertEquals(m_testBytes, reqMsg.getVoterNonce());
    assertEquals(null, reqMsg.getVoterNonce2());
    List aBlocks = new ArrayList();
    List bBlocks = new ArrayList();
    for (VoteBlocksIterator iter = m_testMsg.getVoteBlockIterator(); iter.hasNext(); ) {
      aBlocks.add(iter.next());
    }
    for (VoteBlocksIterator iter = reqMsg.getVoteBlockIterator(); iter.hasNext(); ) {
      bBlocks.add(iter.next());
    }
    assertEquals(aBlocks, bBlocks);

    // Actual size of test vote blocks is unpredictable
    assertTrue(reqMsg.getEstimatedEncodedLength() > V3LcapMessage.EST_ENCODED_HEADER_LENGTH);
  }
 /** iterate over the mime-types makeRow for each */
 private List getRows(CrawlerStatus status, String key) {
   Collection mimeTypes = status.getMimeTypes();
   List rows = new ArrayList();
   if (mimeTypes != null) {
     String mimeType;
     for (Iterator it = mimeTypes.iterator(); it.hasNext(); ) {
       mimeType = (String) it.next();
       rows.add(makeRow(status, mimeType, key));
     }
   }
   return rows;
 }
  public void testStoreNodeState() throws Exception {
    TimeBase.setSimulated(100);
    CachedUrlSet mcus =
        new MockCachedUrlSet(mau, new RangeCachedUrlSetSpec("http://www.example.com"));
    CrawlState crawl = new CrawlState(1, 2, 123);
    List polls = new ArrayList(2);
    PollState poll1 = new PollState(1, "sdf", "jkl", 2, 123, Deadline.at(456), false);
    PollState poll2 = new PollState(2, "abc", "def", 3, 321, Deadline.at(654), false);
    polls.add(poll1);
    polls.add(poll2);
    NodeState nodeState = new NodeStateImpl(mcus, 123321, crawl, polls, repository);
    ((NodeStateImpl) nodeState).setState(NodeState.DAMAGE_AT_OR_BELOW);
    repository.storeNodeState(nodeState);
    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath =
        LockssRepositoryImpl.mapUrlToFileLocation(
            filePath, "http://www.example.com/" + HistoryRepositoryImpl.NODE_FILE_NAME);
    File xmlFile = new File(filePath);
    assertTrue(xmlFile.exists());

    nodeState = null;
    nodeState = repository.loadNodeState(mcus);
    assertSame(mcus, nodeState.getCachedUrlSet());

    assertEquals(123321, nodeState.getAverageHashDuration());
    assertEquals(1, nodeState.getCrawlState().getType());
    assertEquals(2, nodeState.getCrawlState().getStatus());
    assertEquals(123, nodeState.getCrawlState().getStartTime());
    assertEquals(NodeState.DAMAGE_AT_OR_BELOW, nodeState.getState());

    Iterator pollIt = nodeState.getActivePolls();
    assertTrue(pollIt.hasNext());
    PollState loadedPoll = (PollState) pollIt.next();
    assertEquals(1, loadedPoll.getType());
    assertEquals("sdf", loadedPoll.getLwrBound());
    assertEquals("jkl", loadedPoll.getUprBound());
    assertEquals(2, loadedPoll.getStatus());
    assertEquals(123, loadedPoll.getStartTime());
    assertEquals(456, loadedPoll.getDeadline().getExpirationTime());

    assertTrue(pollIt.hasNext());
    loadedPoll = (PollState) pollIt.next();
    assertEquals(2, loadedPoll.getType());
    assertEquals("abc", loadedPoll.getLwrBound());
    assertEquals("def", loadedPoll.getUprBound());
    assertEquals(3, loadedPoll.getStatus());
    assertEquals(321, loadedPoll.getStartTime());
    assertEquals(654, loadedPoll.getDeadline().getExpirationTime());
    assertFalse(pollIt.hasNext());

    TimeBase.setReal();
  }
 private PollHistoryBean createPollHistoryBean(int voteCount) throws Exception {
   PollState state = new PollState(1, "lwr", "upr", 2, 5, null, false);
   List votes = new ArrayList(voteCount);
   for (int ii = 0; ii < voteCount; ii++) {
     VoteBean bean = new VoteBean();
     bean.setId(idKey);
     bean.setAgreeState(true);
     bean.setChallengeString("1234");
     bean.setHashString("2345");
     bean.setVerifierString("3456");
     votes.add(bean.getVote());
   }
   return new PollHistoryBean(new PollHistory(state, 0, votes));
 }
Пример #11
0
 public List findExistingRepositoriesFor(String auid) {
   List res = null;
   for (Iterator iter = getRepositoryList().iterator(); iter.hasNext(); ) {
     String repoName = (String) iter.next();
     String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
     if (LockssRepositoryImpl.doesAuDirExist(auid, path)) {
       if (res == null) {
         res = new ArrayList();
       }
       res.add(repoName);
     }
   }
   return res == null ? Collections.EMPTY_LIST : res;
 }
Пример #12
0
 private ExternalizableMap loadMap(String extMapName, ClassLoader loader)
     throws FileNotFoundException {
   String first = null;
   String next = extMapName;
   List<String> urls = new ArrayList<String>();
   ExternalizableMap res = null;
   while (next != null) {
     // convert the plugin class name to an xml file name
     String mapFile = next.replace('.', '/') + MAP_SUFFIX;
     URL url = loader.getResource(mapFile);
     if (url != null && urls.contains(url.toString())) {
       throw new PluginException.InvalidDefinition("Plugin inheritance loop: " + next);
     }
     // load into map
     ExternalizableMap oneMap = new ExternalizableMap();
     oneMap.loadMapFromResource(mapFile, loader);
     urls.add(url.toString());
     // apply overrides one plugin at a time in inheritance chain
     processOverrides(oneMap);
     if (res == null) {
       res = oneMap;
     } else {
       for (Map.Entry ent : oneMap.entrySet()) {
         String key = (String) ent.getKey();
         Object val = ent.getValue();
         if (!res.containsKey(key)) {
           res.setMapElement(key, val);
         }
       }
     }
     if (oneMap.containsKey(KEY_PLUGIN_PARENT)) {
       next = oneMap.getString(KEY_PLUGIN_PARENT);
     } else {
       next = null;
     }
   }
   loadedFromUrls = urls;
   return res;
 }
Пример #13
0
 public void setConfig(
     Configuration config, Configuration oldConfig, Configuration.Differences changedKeys) {
   //  Build list of repositories from list of disk (fs) paths).  Needs to
   //  be generalized if ever another repository implementation.
   if (changedKeys.contains(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST)) {
     List lst = new ArrayList();
     String dspace = config.get(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, "");
     List paths = StringUtil.breakAt(dspace, ';');
     if (paths != null) {
       for (Iterator iter = paths.iterator(); iter.hasNext(); ) {
         lst.add("local:" + (String) iter.next());
       }
     }
     repoList = lst;
   }
   if (changedKeys.contains(PARAM_MAX_PER_AU_CACHE_SIZE)) {
     paramNodeCacheSize =
         config.getInt(PARAM_MAX_PER_AU_CACHE_SIZE, DEFAULT_MAX_PER_AU_CACHE_SIZE);
     for (Iterator iter = getDaemon().getAllLockssRepositories().iterator(); iter.hasNext(); ) {
       LockssRepository repo = (LockssRepository) iter.next();
       if (repo instanceof LockssRepositoryImpl) {
         LockssRepositoryImpl repoImpl = (LockssRepositoryImpl) repo;
         repoImpl.setNodeCacheSize(paramNodeCacheSize);
       }
     }
   }
   if (changedKeys.contains(PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE)) {
     paramSuspectVersionsCacheSize =
         config.getInt(
             PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE, DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
     suspectVersionsCache.setMaxSize(paramSuspectVersionsCacheSize);
   }
   if (changedKeys.contains(GLOBAL_CACHE_PREFIX)) {
     paramIsGlobalNodeCache =
         config.getBoolean(PARAM_GLOBAL_CACHE_ENABLED, DEFAULT_GLOBAL_CACHE_ENABLED);
     if (paramIsGlobalNodeCache) {
       paramGlobalNodeCacheSize =
           config.getInt(PARAM_MAX_GLOBAL_CACHE_SIZE, DEFAULT_MAX_GLOBAL_CACHE_SIZE);
       log.debug("global node cache size: " + paramGlobalNodeCacheSize);
       globalNodeCache.setMaxSize(paramGlobalNodeCacheSize);
     }
   }
   if (changedKeys.contains(DISK_PREFIX)) {
     int minMB = config.getInt(PARAM_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_MB);
     double minPer =
         config.getPercentage(PARAM_DISK_WARN_FRRE_PERCENT, DEFAULT_DISK_WARN_FRRE_PERCENT);
     paramDFWarn = PlatformUtil.DF.makeThreshold(minMB, minPer);
     minMB = config.getInt(PARAM_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_MB);
     minPer = config.getPercentage(PARAM_DISK_FULL_FRRE_PERCENT, DEFAULT_DISK_FULL_FRRE_PERCENT);
     paramDFFull = PlatformUtil.DF.makeThreshold(minMB, minPer);
   }
   if (changedKeys.contains(PARAM_SIZE_CALC_MAX_LOAD)) {
     sizeCalcMaxLoad = config.getPercentage(PARAM_SIZE_CALC_MAX_LOAD, DEFAULT_SIZE_CALC_MAX_LOAD);
   }
   if (changedKeys.contains(PREFIX)) {
     maxUnusedDirSearch =
         config.getInt(PARAM_MAX_UNUSED_DIR_SEARCH, DEFAULT_MAX_UNUSED_DIR_SEARCH);
     isStatefulUnusedDirSearch =
         config.getBoolean(
             PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH, DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH);
     enableLongComponents =
         config.getBoolean(PARAM_ENABLE_LONG_COMPONENTS, DEFAULT_ENABLE_LONG_COMPONENTS);
     enableLongComponentsCompatibility =
         config.getBoolean(
             PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY,
             DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY);
     maxComponentLength = config.getInt(PARAM_MAX_COMPONENT_LENGTH, DEFAULT_MAX_COMPONENT_LENGTH);
     checkUnnormalized =
         (CheckUnnormalizedMode)
             config.getEnum(
                 CheckUnnormalizedMode.class,
                 PARAM_CHECK_UNNORMALIZED,
                 DEFAULT_CHECK_UNNORMALIZED);
   }
 }
 private void addIfNonZero(List res, String head, int val) {
   if (val != 0) {
     res.add(new StatusTable.SummaryInfo(head, ColumnDescriptor.TYPE_INT, new Long(val)));
   }
 }
Пример #15
0
 protected void recordExportFile(File file) {
   if (fileSet.add(file)) {
     fileList.add(file);
   }
 }
Пример #16
0
 protected void recordError(String msg) {
   log.error(msg);
   errors.add(msg);
 }
Пример #17
0
 protected void recordError(String msg, Throwable t) {
   log.error(msg, t);
   errors.add(msg + ": " + t.toString());
 }
Пример #18
0
 void doSizeCalc(RepositoryNode node) {
   TimerUtil.guaranteedSleep(10);
   nodes.add(node);
   sem.give();
 }