@Override
protected ArticleFiles createArticleFiles(CachedUrl cu) {
String url = cu.getUrl();
Matcher mat = ABSTRACT_PATTERN.matcher(url);
if (mat.find()) {
return processAbstract(cu, mat);
}
log.warning("Mismatch between article iterator factory and article iterator: " + url);
return null;
}
@Override
protected ArticleFiles createArticleFiles(CachedUrl cu) {
String url = cu.getUrl();
log.debug3("iterating url: " + url);
Matcher mat = PATTERN.matcher(url);
if (mat.find()) {
return processFullTextPdf(cu, mat);
}
log.warning("Mismatch between article iterator factory and article iterator: " + url);
return null;
}
public void testBaseUrl() throws Exception {
sau1 = setupSimAu(simAuConfig(tempDirPath));
createContent(sau1);
crawlContent(sau1);
CachedUrlSet cus1 = sau1.getAuCachedUrlSet();
tempDirPath2 = getTempDir().getAbsolutePath() + File.separator;
Configuration config2 = simAuConfig(tempDirPath2);
config2.put("base_url", "http://anotherhost.org/");
SimulatedArchivalUnit sau2 = setupSimAu(config2);
createContent(sau2);
crawlContent(sau2);
CachedUrlSet cus2 = sau1.getAuCachedUrlSet();
List urls1 = auUrls(sau1);
List urls2 = auUrls(sau2);
Pattern pat = Pattern.compile("http://([^/]+)(/.*)$");
List<String> l1 = auUrls(sau1);
List<String> l2 = auUrls(sau2);
assertEquals(l1.size(), l2.size());
for (int ix = 0; ix < l1.size(); ix++) {
Matcher m1 = pat.matcher(l1.get(ix));
assertTrue(m1.matches());
Matcher m2 = pat.matcher(l2.get(ix));
assertTrue(m2.matches());
assertEquals("www.example.com", m1.group(1));
assertEquals("anotherhost.org", m2.group(1));
assertEquals(m1.group(2), m2.group(2));
}
}
/*
* This is comlicated. MOST AUs have articles that live below and issue level TOC
* that is,
* <blah>/<journal_id>/vol#/iss#/ is a toc with no relevant metadata
* <blah>/<journal_id>/vol#/iss#/xxx is an article with metadata
* (eg Economist Voice V1)
* BUT
* in some AUs there are issues with only 1 article, in which case
* <blah>/<journal_id>/vol#/iss#/ is an abstract with metadata
* (eg Rhodes Cook V4)
* and a few AUs with a mixture
* (eg Forum for Health Economics V5)
* So to identify ALL articles, we'll also have to capture issue level items and then look
* at the html and if it has article metadata in it, count it as an article.
*
*/
@Override
protected ArticleFiles createArticleFiles(CachedUrl cu) {
String url = cu.getUrl();
Matcher mat = pattern.matcher(url);
if (mat.find()) {
// we matched, but could this pattern potentially be a toc?
Matcher tocmat = TOC_pattern.matcher(url);
// if we could be a TOC then we must have metadata to be considered an article
if (tocmat.find()) {
if (hasArticleMetadata(cu)) {
return processUrl(cu, mat);
}
} else {
// we're not a potential TOC, so treat this as an article without checking
return processUrl(cu, mat);
}
return null; // this was a TOC, not an article
}
log.warning("Mismatch between article iterator factory and article iterator: " + url);
return null;
}
protected ArticleFiles processFullTextPdf(CachedUrl cu, Matcher mat) {
ArticleFiles af = new ArticleFiles();
af.setRoleCu(ArticleFiles.ROLE_FULL_TEXT_PDF, cu);
String abstractUrl = mat.replaceAll("$1aspx");
CachedUrl abstractCu = cu.getArchivalUnit().makeCachedUrl(abstractUrl);
if (abstractCu.hasContent()) af.setFullTextCu(abstractCu);
else af.setFullTextCu(cu);
log.debug3("returning full text: " + af.getFullTextUrl());
return af;
}