private void deleteBlock(CachedUrl cu) throws IOException { log.info("deleting " + cu.getUrl()); CachedUrlSetSpec cuss = new SingleNodeCachedUrlSetSpec(cu.getUrl()); ArchivalUnit au = cu.getArchivalUnit(); CachedUrlSet cus = au.makeCachedUrlSet(cuss); NodeManager nm = au.getPlugin().getDaemon().getNodeManager(au); nm.deleteNode(cus); }
/** * Lookup the CU's archive file type in its AU's ArchiveFileTypes * * @return the file extension (including dot), or null if none found */ public static String getArchiveExtension(CachedUrl cu) { ArchiveFileTypes aft = cu.getArchivalUnit().getArchiveFileTypes(); if (aft == null) { return null; } try { return aft.getFromCu(cu); } catch (MalformedURLException e) { log.warning("isArchive(" + cu + ")", e); return null; } }
protected ArticleFiles processFullTextPdf(CachedUrl cu, Matcher mat) { ArticleFiles af = new ArticleFiles(); af.setRoleCu(ArticleFiles.ROLE_FULL_TEXT_PDF, cu); String abstractUrl = mat.replaceAll("$1aspx"); CachedUrl abstractCu = cu.getArchivalUnit().makeCachedUrl(abstractUrl); if (abstractCu.hasContent()) af.setFullTextCu(abstractCu); else af.setFullTextCu(cu); log.debug3("returning full text: " + af.getFullTextUrl()); return af; }
@Override public void extract(MetadataTarget target, CachedUrl cu, Emitter emitter) throws IOException { ArticleMetadata am = new SimpleHtmlMetaTagMetadataExtractor().extract(target, cu); am.cook(tagMap); String url = am.get(MetadataField.FIELD_ACCESS_URL); ArchivalUnit au = cu.getArchivalUnit(); if (url == null || url.isEmpty() || !au.makeCachedUrl(url).hasContent()) { url = cu.getUrl(); } am.replace( MetadataField.FIELD_ACCESS_URL, HttpToHttpsUtil.AuUtil.normalizeHttpHttpsFromBaseUrl(au, url)); emitter.emitMetadata(cu, am); }
@Override public ArticleMetadata extract(MetadataTarget target, CachedUrl cu) throws IOException { ArticleMetadata am = super.extract(target, cu); am.cook(tagMap); String url = am.get(MetadataField.FIELD_ACCESS_URL); if (url != null && !url.isEmpty()) { CachedUrl val = cu.getArchivalUnit().makeCachedUrl(url); if (!val.hasContent()) { am.replace(MetadataField.FIELD_ACCESS_URL, cu.getUrl()); } } else { am.replace(MetadataField.FIELD_ACCESS_URL, cu.getUrl()); } return am; }