/* private support methods */
private List<ArticleMetadata> setupContentForAU(
ArchivalUnit au, String url, String content, boolean isHtmlExtractor)
throws IOException, PluginException {
FileMetadataExtractor me;
InputStream input = null;
CIProperties props = null;
if (isHtmlExtractor) {
input = IOUtils.toInputStream(content, "utf-8");
props = getContentHtmlProperties();
me =
new BaseAtyponHtmlMetadataExtractorFactory()
.createFileMetadataExtractor(MetadataTarget.Any(), "text/html");
} else {
input = IOUtils.toInputStream(content, "utf-8");
props = getContentRisProperties();
me =
new BaseAtyponRisMetadataExtractorFactory()
.createFileMetadataExtractor(MetadataTarget.Any(), "text/plain");
}
UrlData ud = new UrlData(input, props, url);
UrlCacher uc = au.makeUrlCacher(ud);
uc.storeContent();
CachedUrl cu = uc.getCachedUrl();
FileMetadataListExtractor mle = new FileMetadataListExtractor(me);
return mle.extract(MetadataTarget.Any(), cu);
}
public void testFunctionalFromTarHierarchy() throws Exception {
log.debug3("in testFromTarHierarchy");
// load the tarballs
InputStream file_input = null;
try {
file_input = getResourceAsStream(realTARFile_A);
// UrlCacher uc = au.makeUrlCacher(TAR_A_BASE);
// uc.storeContent(file_input, tarHeader);
UrlCacher uc = tarAu.makeUrlCacher(new UrlData(file_input, tarHeader, TAR_A_BASE));
uc.storeContent();
IOUtil.safeClose(file_input);
file_input = getResourceAsStream(realTARFile_B);
// uc = au.makeUrlCacher(TAR_B_BASE);
// uc.storeContent(file_input, tarHeader);
uc = tarAu.makeUrlCacher(new UrlData(file_input, tarHeader, TAR_B_BASE));
uc.storeContent();
IOUtil.safeClose(file_input);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
IOUtil.safeClose(file_input);
}
CachedUrlSet cus = tarAu.getAuCachedUrlSet();
for (CachedUrl cu : cus.getCuIterable()) {
log.debug3("AU - cu is: " + cu.getUrl());
cu.release();
}
// We need to start from the level of the ArticleMetadataExtractor
MyListEmitter emitter = new MyListEmitter();
ArticleMetadataExtractor amEx =
new ElsevierDeferredArticleMetadataExtractor(ArticleFiles.ROLE_ARTICLE_METADATA);
Iterator<ArticleFiles> it = tarAu.getArticleIterator(MetadataTarget.Any());
while (it.hasNext()) {
ArticleFiles af = it.next();
log.debug3("Metadata test - articlefiles " + af.toString());
// CachedUrl cu = af.getFullTextCu();
CachedUrl cu = af.getRoleCu(ArticleFiles.ROLE_ARTICLE_METADATA);
log.debug3("metadata cu is " + cu.getUrl());
// List<ArticleMetadata> mdlist = mle.extract(MetadataTarget.Any(), cu);
amEx.extract(MetadataTarget.Any(), af, emitter);
List<ArticleMetadata> returnList = emitter.getAmList();
assertNotNull(returnList);
log.debug3("size of returnList is " + returnList.size());
Iterator<ArticleMetadata> mdIt = returnList.iterator();
ArticleMetadata mdRecord = null;
while (mdIt.hasNext()) {
mdRecord = (ArticleMetadata) mdIt.next();
validateCompleteMetadataRecord(mdRecord);
}
}
}
/*
* hasArticleMetadata(CachedUrl cu)
* Given the CachedUrl for the potential abstract file, using the existing
* SimpleHtmlMetaTagMetadataExtractor to parse the file and
* retrieve any contained metadata. If a doi or author exists, it's an article
* NOT defining the Metadata Extractor here!
*/
private boolean hasArticleMetadata(CachedUrl cu) {
MetadataTarget at = new MetadataTarget(MetadataTarget.PURPOSE_ARTICLE);
ArticleMetadata am;
SimpleHtmlMetaTagMetadataExtractor ext = new SimpleHtmlMetaTagMetadataExtractor();
if (cu != null && cu.hasContent()) {
try {
at.setFormat("text/html");
am = ext.extract(at, cu);
if ((am.containsRawKey("bepress_citation_journal_title"))
|| (am.containsRawKey("bepress_citation_abstract_html_url"))
|| (am.containsRawKey("bepress_citation_doi"))
|| (am.containsRawKey("bepress_citation_author"))) {
return true;
}
} catch (IOException e) {
e.printStackTrace();
}
}
return false; // no reasonable metadata, probably a toc
}