public void testArticleCountAndType() throws Exception {
int expCount = 28;
PluginTestUtil.crawlSimAu(sau);
String pat1 = "branch(\\d+)/(\\d+file\\.html)";
String rep1 = "aps/journal/v123/n$1/full/$2";
PluginTestUtil.copyAu(sau, nau, ".*[^.][^p][^d][^f]$", pat1, rep1);
String pat2 = "branch(\\d+)/(\\d+file\\.pdf)";
String rep2 = "aps/journal/v123/n$1/pdf/$2";
PluginTestUtil.copyAu(sau, nau, ".*\\.pdf$", pat2, rep2);
// Remove some URLs
int deleted = 0;
for (Iterator it = nau.getAuCachedUrlSet().contentHashIterator(); it.hasNext(); ) {
CachedUrlSetNode cusn = (CachedUrlSetNode) it.next();
if (cusn instanceof CachedUrl) {
CachedUrl cu = (CachedUrl) cusn;
String url = cu.getUrl();
if (url.contains("/journal/")
&& (url.endsWith("1file.html") || url.endsWith("2file.pdf"))) {
deleteBlock(cu);
++deleted;
}
}
}
assertEquals(8, deleted);
Iterator<ArticleFiles> it = nau.getArticleIterator();
int count = 0;
int countHtmlOnly = 0;
int countPdfOnly = 0;
while (it.hasNext()) {
ArticleFiles af = it.next();
log.info(af.toString());
CachedUrl cu = af.getFullTextCu();
String url = cu.getUrl();
assertNotNull(cu);
String contentType = cu.getContentType();
log.debug("count " + count + " url " + url + " " + contentType);
count++;
if (af.getRoleUrl(ArticleFiles.ROLE_FULL_TEXT_PDF) == null) {
++countHtmlOnly;
}
if (af.getRoleUrl(ArticleFiles.ROLE_FULL_TEXT_PDF) == url) {
++countPdfOnly;
}
}
log.debug("Article count is " + count);
assertEquals(expCount, count);
assertEquals(4, countHtmlOnly);
assertEquals(4, countPdfOnly);
}
public void setUp() throws Exception {
super.setUp();
String tempDirPath = getTempDir().getAbsolutePath() + File.separator;
ConfigurationUtil.setFromArgs(LockssRepositoryImpl.PARAM_CACHE_LOCATION, tempDirPath);
theDaemon = getMockLockssDaemon();
theDaemon.getAlertManager();
theDaemon.getPluginManager().setLoadablePluginsReady(true);
theDaemon.setDaemonInited(true);
theDaemon.getPluginManager().startService();
theDaemon.getCrawlManager();
sau = PluginTestUtil.createAndStartSimAu(simAuConfig(tempDirPath));
nau = PluginTestUtil.createAndStartAu(PLUGIN_NAME, natureAuConfig());
}
public void setUp() throws Exception {
super.setUp();
tempDirPath = getTempDir().getAbsolutePath() + File.separator;
theDaemon = getMockLockssDaemon();
theDaemon.getAlertManager();
theDaemon.getPluginManager().setLoadablePluginsReady(true);
theDaemon.getHashService();
MockSystemMetrics metrics = new MyMockSystemMetrics();
metrics.initService(theDaemon);
theDaemon.setSystemMetrics(metrics);
theDaemon.setDaemonInited(true);
Properties props = new Properties();
props.setProperty(SystemMetrics.PARAM_HASH_TEST_DURATION, "1000");
props.setProperty(SystemMetrics.PARAM_HASH_TEST_BYTE_STEP, "1024");
props.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath);
ConfigurationUtil.setCurrentConfigFromProps(props);
pluginMgr = theDaemon.getPluginManager();
pluginMgr.startService();
theDaemon.getHashService().startService();
metrics.startService();
metrics.setHashSpeed(100);
simPlugin = PluginTestUtil.findPlugin(SimulatedPlugin.class);
}
static String createPlugin(String name, Path structure, boolean createSecurityPolicyFile)
throws IOException {
PluginTestUtil.writeProperties(
structure,
"description",
"fake desc",
"name",
name,
"version",
"1.0",
"elasticsearch.version",
Version.CURRENT.toString(),
"java.version",
System.getProperty("java.specification.version"),
"classname",
"FakePlugin");
if (createSecurityPolicyFile) {
String securityPolicyContent =
"grant {\n permission java.lang.RuntimePermission \"setFactory\";\n};\n";
Files.write(
structure.resolve("plugin-security.policy"),
securityPolicyContent.getBytes(StandardCharsets.UTF_8));
}
writeJar(structure.resolve("plugin.jar"), "FakePlugin");
return writeZip(structure, "elasticsearch");
}
protected ArchivalUnit createAu() throws ArchivalUnit.ConfigurationException {
return PluginTestUtil.createAndStartAu(
PLUGIN_NAME,
ConfigurationUtil.fromArgs(
"base_url",
"http://pediatrics.aappublications.org/",
"volume_name",
"52",
"journal_issn",
"1098-4275"));
}
protected ArchivalUnit createTarAu() throws ArchivalUnit.ConfigurationException {
// in this directory this is file "test_elsevierdtd5.tdb" but it becomes xml
try {
ConfigurationUtil.addFromUrl(getResource("test_elsevierdtd5.xml"));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Tdb tdb = ConfigManager.getCurrentConfig().getTdb();
TdbAu tdbau1 = tdb.getTdbAusLikeName("Elsevier Source Content 2014").get(0);
assertNotNull("Didn't find named TdbAu", tdbau1);
return PluginTestUtil.createAndStartAu(tdbau1);
}
public void testCreateArticleFiles() throws Exception {
PluginTestUtil.crawlSimAu(sau);
String pat0 = "001file[.]html";
String rep0 = "52/1/S1";
PluginTestUtil.copyAu(sau, au, ".*[.]html$", pat0, rep0);
String pat1 = "001file[.]pdf";
String rep1 = "52/1/S1.pdf";
PluginTestUtil.copyAu(sau, au, ".*[.]pdf$", pat1, rep1);
String pdfurl = "http://pediatrics.aappublications.org/cgi/reprint/52/1/S1.pdf";
String url = "http://pediatrics.aappublications.org/cgi/reprint/52/1/S1";
au.makeCachedUrl(url);
CachedUrl cu = au.makeCachedUrl(pdfurl);
assertNotNull(cu);
SubTreeArticleIterator artIter = createSubTreeIter();
assertNotNull(artIter);
ArticleFiles af = artIter.next();
assertNotNull(af);
System.out.println("article files::" + af);
assertEquals(url, af.getRoleCu(ArticleFiles.ROLE_FULL_TEXT_PDF_LANDING_PAGE).getUrl());
assertEquals(pdfurl, af.getRoleCu(ArticleFiles.ROLE_FULL_TEXT_PDF).getUrl());
}
private void initRequiredServices() {
theDaemon = getMockLockssDaemon();
pollmanager = new LocalPollManager();
pollmanager.initService(theDaemon);
theDaemon.setPollManager(pollmanager);
theDaemon.getPluginManager();
testau = PollTestPlugin.PTArchivalUnit.createFromListOfRootUrls(rootV1urls);
PluginTestUtil.registerArchivalUnit(testau);
String tempDirPath = null;
try {
tempDirPath = getTempDir().getAbsolutePath() + File.separator;
} catch (IOException ex) {
fail("unable to create a temporary directory");
}
Properties p = new Properties();
p.setProperty(IdentityManager.PARAM_IDDB_DIR, tempDirPath + "iddb");
p.setProperty(LockssRepositoryImpl.PARAM_CACHE_LOCATION, tempDirPath);
p.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath);
p.setProperty(IdentityManager.PARAM_LOCAL_IP, "127.0.0.1");
p.setProperty(ConfigManager.PARAM_NEW_SCHEDULER, "false");
// XXX we need to disable verification of votes because the
// voter isn't really there
p.setProperty(V1Poll.PARAM_AGREE_VERIFY, "0");
p.setProperty(V1Poll.PARAM_DISAGREE_VERIFY, "0");
ConfigurationUtil.setCurrentConfigFromProps(p);
idmgr = theDaemon.getIdentityManager();
idmgr.startService();
// theDaemon.getSchedService().startService();
theDaemon.getHashService().startService();
theDaemon.getDatagramRouterManager().startService();
theDaemon.getRouterManager().startService();
theDaemon.getSystemMetrics().startService();
theDaemon.getActivityRegulator(testau).startService();
theDaemon.setNodeManager(new MockNodeManager(), testau);
pollmanager.startService();
}
public void setUp() throws Exception {
super.setUp();
setUpDiskSpace(); // you need this to have startService work properly...
theDaemon = getMockLockssDaemon();
theDaemon.getAlertManager();
theDaemon.getPluginManager().setLoadablePluginsReady(true);
theDaemon.setDaemonInited(true);
theDaemon.getPluginManager().startService();
theDaemon.getCrawlManager();
// in this directory this is file "test_baseatypon.tdb" but it becomes xml
ConfigurationUtil.addFromUrl(getResource("test_baseatypon.xml"));
Tdb tdb = ConfigManager.getCurrentConfig().getTdb();
TdbAu tdbau1 = tdb.getTdbAusLikeName(goodJournal + " Volume " + goodVolume).get(0);
assertNotNull("Didn't find named TdbAu", tdbau1);
bau1 = PluginTestUtil.createAndStartAu(tdbau1);
assertNotNull(bau1);
TypedEntryMap auConfig = bau1.getProperties();
assertEquals(BASE_URL, auConfig.getString(BASE_URL_KEY));
}
public void setUp() throws Exception {
super.setUp();
String tempDirPath = setUpDiskSpace();
au = createAu();
sau = PluginTestUtil.createAndStartSimAu(simAuConfig(tempDirPath));
}
SimulatedArchivalUnit setupSimAu(Configuration auConfig)
throws ArchivalUnit.ConfigurationException {
ArchivalUnit au = PluginTestUtil.createAndStartAu(simPlugin, auConfig);
return (SimulatedArchivalUnit) au;
}