protected void writeCu(CachedUrl cu) throws IOException {
String url = cu.getUrl();
long contentSize = cu.getContentSize();
CIProperties props = cu.getProperties();
long fetchTime = Long.parseLong(props.getProperty(CachedUrl.PROPERTY_FETCH_TIME));
InputStream contentIn = cu.getUnfilteredInputStream();
try {
if (isResponse) {
String hdrString = getHttpResponseString(cu);
long size = contentSize + hdrString.length();
InputStream headerIn = new ReaderInputStream(new StringReader(hdrString));
InputStream concat = new SequenceInputStream(headerIn, contentIn);
try {
aw.write(xlateFilename(url), cu.getContentType(), getHostIp(), fetchTime, size, concat);
} finally {
IOUtil.safeClose(concat);
}
} else {
aw.write(
xlateFilename(url),
cu.getContentType(),
getHostIp(),
fetchTime,
cu.getContentSize(),
contentIn);
}
} finally {
AuUtil.safeRelease(cu);
}
}
/**
* Return the archive file type corresponding to the CU's MIME type or filename extension, or null
* if none.
*/
public String getFromCu(CachedUrl cu) throws MalformedURLException {
String res = getFromMime(cu.getContentType());
if (res == null) {
res = getFromUrl(cu.getUrl());
}
return res;
}
public void testArticleCountAndType() throws Exception {
int expCount = 28;
PluginTestUtil.crawlSimAu(sau);
String pat1 = "branch(\\d+)/(\\d+file\\.html)";
String rep1 = "aps/journal/v123/n$1/full/$2";
PluginTestUtil.copyAu(sau, nau, ".*[^.][^p][^d][^f]$", pat1, rep1);
String pat2 = "branch(\\d+)/(\\d+file\\.pdf)";
String rep2 = "aps/journal/v123/n$1/pdf/$2";
PluginTestUtil.copyAu(sau, nau, ".*\\.pdf$", pat2, rep2);
// Remove some URLs
int deleted = 0;
for (Iterator it = nau.getAuCachedUrlSet().contentHashIterator(); it.hasNext(); ) {
CachedUrlSetNode cusn = (CachedUrlSetNode) it.next();
if (cusn instanceof CachedUrl) {
CachedUrl cu = (CachedUrl) cusn;
String url = cu.getUrl();
if (url.contains("/journal/")
&& (url.endsWith("1file.html") || url.endsWith("2file.pdf"))) {
deleteBlock(cu);
++deleted;
}
}
}
assertEquals(8, deleted);
Iterator<ArticleFiles> it = nau.getArticleIterator();
int count = 0;
int countHtmlOnly = 0;
int countPdfOnly = 0;
while (it.hasNext()) {
ArticleFiles af = it.next();
log.info(af.toString());
CachedUrl cu = af.getFullTextCu();
String url = cu.getUrl();
assertNotNull(cu);
String contentType = cu.getContentType();
log.debug("count " + count + " url " + url + " " + contentType);
count++;
if (af.getRoleUrl(ArticleFiles.ROLE_FULL_TEXT_PDF) == null) {
++countHtmlOnly;
}
if (af.getRoleUrl(ArticleFiles.ROLE_FULL_TEXT_PDF) == url) {
++countPdfOnly;
}
}
log.debug("Article count is " + count);
assertEquals(expCount, count);
assertEquals(4, countHtmlOnly);
assertEquals(4, countPdfOnly);
}
