public void testArticleCountAndType() throws Exception { int expCount = 28; PluginTestUtil.crawlSimAu(sau); String pat1 = "branch(\\d+)/(\\d+file\\.html)"; String rep1 = "aps/journal/v123/n$1/full/$2"; PluginTestUtil.copyAu(sau, nau, ".*[^.][^p][^d][^f]$", pat1, rep1); String pat2 = "branch(\\d+)/(\\d+file\\.pdf)"; String rep2 = "aps/journal/v123/n$1/pdf/$2"; PluginTestUtil.copyAu(sau, nau, ".*\\.pdf$", pat2, rep2); // Remove some URLs int deleted = 0; for (Iterator it = nau.getAuCachedUrlSet().contentHashIterator(); it.hasNext(); ) { CachedUrlSetNode cusn = (CachedUrlSetNode) it.next(); if (cusn instanceof CachedUrl) { CachedUrl cu = (CachedUrl) cusn; String url = cu.getUrl(); if (url.contains("/journal/") && (url.endsWith("1file.html") || url.endsWith("2file.pdf"))) { deleteBlock(cu); ++deleted; } } } assertEquals(8, deleted); Iterator<ArticleFiles> it = nau.getArticleIterator(); int count = 0; int countHtmlOnly = 0; int countPdfOnly = 0; while (it.hasNext()) { ArticleFiles af = it.next(); log.info(af.toString()); CachedUrl cu = af.getFullTextCu(); String url = cu.getUrl(); assertNotNull(cu); String contentType = cu.getContentType(); log.debug("count " + count + " url " + url + " " + contentType); count++; if (af.getRoleUrl(ArticleFiles.ROLE_FULL_TEXT_PDF) == null) { ++countHtmlOnly; } if (af.getRoleUrl(ArticleFiles.ROLE_FULL_TEXT_PDF) == url) { ++countPdfOnly; } } log.debug("Article count is " + count); assertEquals(expCount, count); assertEquals(4, countHtmlOnly); assertEquals(4, countPdfOnly); }
public void setUp() throws Exception { super.setUp(); String tempDirPath = getTempDir().getAbsolutePath() + File.separator; ConfigurationUtil.setFromArgs(LockssRepositoryImpl.PARAM_CACHE_LOCATION, tempDirPath); theDaemon = getMockLockssDaemon(); theDaemon.getAlertManager(); theDaemon.getPluginManager().setLoadablePluginsReady(true); theDaemon.setDaemonInited(true); theDaemon.getPluginManager().startService(); theDaemon.getCrawlManager(); sau = PluginTestUtil.createAndStartSimAu(simAuConfig(tempDirPath)); nau = PluginTestUtil.createAndStartAu(PLUGIN_NAME, natureAuConfig()); }
public void setUp() throws Exception { super.setUp(); tempDirPath = getTempDir().getAbsolutePath() + File.separator; theDaemon = getMockLockssDaemon(); theDaemon.getAlertManager(); theDaemon.getPluginManager().setLoadablePluginsReady(true); theDaemon.getHashService(); MockSystemMetrics metrics = new MyMockSystemMetrics(); metrics.initService(theDaemon); theDaemon.setSystemMetrics(metrics); theDaemon.setDaemonInited(true); Properties props = new Properties(); props.setProperty(SystemMetrics.PARAM_HASH_TEST_DURATION, "1000"); props.setProperty(SystemMetrics.PARAM_HASH_TEST_BYTE_STEP, "1024"); props.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath); ConfigurationUtil.setCurrentConfigFromProps(props); pluginMgr = theDaemon.getPluginManager(); pluginMgr.startService(); theDaemon.getHashService().startService(); metrics.startService(); metrics.setHashSpeed(100); simPlugin = PluginTestUtil.findPlugin(SimulatedPlugin.class); }
protected ArchivalUnit createAu() throws ArchivalUnit.ConfigurationException { return PluginTestUtil.createAndStartAu( PLUGIN_NAME, ConfigurationUtil.fromArgs( "base_url", "http://pediatrics.aappublications.org/", "volume_name", "52", "journal_issn", "1098-4275")); }
protected ArchivalUnit createTarAu() throws ArchivalUnit.ConfigurationException { // in this directory this is file "test_elsevierdtd5.tdb" but it becomes xml try { ConfigurationUtil.addFromUrl(getResource("test_elsevierdtd5.xml")); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } Tdb tdb = ConfigManager.getCurrentConfig().getTdb(); TdbAu tdbau1 = tdb.getTdbAusLikeName("Elsevier Source Content 2014").get(0); assertNotNull("Didn't find named TdbAu", tdbau1); return PluginTestUtil.createAndStartAu(tdbau1); }
public void testCreateArticleFiles() throws Exception { PluginTestUtil.crawlSimAu(sau); String pat0 = "001file[.]html"; String rep0 = "52/1/S1"; PluginTestUtil.copyAu(sau, au, ".*[.]html$", pat0, rep0); String pat1 = "001file[.]pdf"; String rep1 = "52/1/S1.pdf"; PluginTestUtil.copyAu(sau, au, ".*[.]pdf$", pat1, rep1); String pdfurl = "http://pediatrics.aappublications.org/cgi/reprint/52/1/S1.pdf"; String url = "http://pediatrics.aappublications.org/cgi/reprint/52/1/S1"; au.makeCachedUrl(url); CachedUrl cu = au.makeCachedUrl(pdfurl); assertNotNull(cu); SubTreeArticleIterator artIter = createSubTreeIter(); assertNotNull(artIter); ArticleFiles af = artIter.next(); assertNotNull(af); System.out.println("article files::" + af); assertEquals(url, af.getRoleCu(ArticleFiles.ROLE_FULL_TEXT_PDF_LANDING_PAGE).getUrl()); assertEquals(pdfurl, af.getRoleCu(ArticleFiles.ROLE_FULL_TEXT_PDF).getUrl()); }
private void initRequiredServices() { theDaemon = getMockLockssDaemon(); pollmanager = new LocalPollManager(); pollmanager.initService(theDaemon); theDaemon.setPollManager(pollmanager); theDaemon.getPluginManager(); testau = PollTestPlugin.PTArchivalUnit.createFromListOfRootUrls(rootV1urls); PluginTestUtil.registerArchivalUnit(testau); String tempDirPath = null; try { tempDirPath = getTempDir().getAbsolutePath() + File.separator; } catch (IOException ex) { fail("unable to create a temporary directory"); } Properties p = new Properties(); p.setProperty(IdentityManager.PARAM_IDDB_DIR, tempDirPath + "iddb"); p.setProperty(LockssRepositoryImpl.PARAM_CACHE_LOCATION, tempDirPath); p.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath); p.setProperty(IdentityManager.PARAM_LOCAL_IP, "127.0.0.1"); p.setProperty(ConfigManager.PARAM_NEW_SCHEDULER, "false"); // XXX we need to disable verification of votes because the // voter isn't really there p.setProperty(V1Poll.PARAM_AGREE_VERIFY, "0"); p.setProperty(V1Poll.PARAM_DISAGREE_VERIFY, "0"); ConfigurationUtil.setCurrentConfigFromProps(p); idmgr = theDaemon.getIdentityManager(); idmgr.startService(); // theDaemon.getSchedService().startService(); theDaemon.getHashService().startService(); theDaemon.getDatagramRouterManager().startService(); theDaemon.getRouterManager().startService(); theDaemon.getSystemMetrics().startService(); theDaemon.getActivityRegulator(testau).startService(); theDaemon.setNodeManager(new MockNodeManager(), testau); pollmanager.startService(); }
public void setUp() throws Exception { super.setUp(); setUpDiskSpace(); // you need this to have startService work properly... theDaemon = getMockLockssDaemon(); theDaemon.getAlertManager(); theDaemon.getPluginManager().setLoadablePluginsReady(true); theDaemon.setDaemonInited(true); theDaemon.getPluginManager().startService(); theDaemon.getCrawlManager(); // in this directory this is file "test_baseatypon.tdb" but it becomes xml ConfigurationUtil.addFromUrl(getResource("test_baseatypon.xml")); Tdb tdb = ConfigManager.getCurrentConfig().getTdb(); TdbAu tdbau1 = tdb.getTdbAusLikeName(goodJournal + " Volume " + goodVolume).get(0); assertNotNull("Didn't find named TdbAu", tdbau1); bau1 = PluginTestUtil.createAndStartAu(tdbau1); assertNotNull(bau1); TypedEntryMap auConfig = bau1.getProperties(); assertEquals(BASE_URL, auConfig.getString(BASE_URL_KEY)); }
public void setUp() throws Exception { super.setUp(); String tempDirPath = setUpDiskSpace(); au = createAu(); sau = PluginTestUtil.createAndStartSimAu(simAuConfig(tempDirPath)); }
SimulatedArchivalUnit setupSimAu(Configuration auConfig) throws ArchivalUnit.ConfigurationException { ArchivalUnit au = PluginTestUtil.createAndStartAu(simPlugin, auConfig); return (SimulatedArchivalUnit) au; }