public void testBaseUrlPath() throws Exception {
sau1 = setupSimAu(simAuConfig(tempDirPath));
createContent(sau1);
crawlContent(sau1);
CachedUrlSet cus1 = sau1.getAuCachedUrlSet();
tempDirPath2 = getTempDir().getAbsolutePath() + File.separator;
Configuration config2 = simAuConfig(tempDirPath2);
config2.put("base_url", "http://anotherhost.org/some/path/");
SimulatedArchivalUnit sau2 = setupSimAu(config2);
createContent(sau2);
crawlContent(sau2);
CachedUrlSet cus2 = sau1.getAuCachedUrlSet();
List urls1 = auUrls(sau1);
List urls2 = auUrls(sau2);
Pattern pat1 = Pattern.compile("http://www\\.example\\.com(/.*)$");
Pattern pat2 = Pattern.compile("http://anotherhost\\.org/some/path(/.*)$");
List<String> l1 = auUrls(sau1);
List<String> l2 = auUrls(sau2);
assertEquals(l1.size(), l2.size());
for (int ix = 0; ix < l1.size(); ix++) {
Matcher m1 = pat1.matcher(l1.get(ix));
assertTrue(m1.matches());
Matcher m2 = pat2.matcher(l2.get(ix));
assertTrue(m2.matches());
assertEquals(m1.group(1), m2.group(1));
}
}
protected void assertEqualTables(Object[][] a1, List lines) {
assertEquals("numrows", a1.length, lines.size() - NUM_HEADER_LINES);
for (int irow = 0; irow <= a1.length - 1; irow++) {
Object expRow[] = a1[irow];
List row = StringUtil.breakAt((String) lines.get(irow + NUM_HEADER_LINES), ',');
assertEquals("numcols", expRow.length, row.size());
assertEquals(("row " + irow), SetUtil.fromArray(expRow), new HashSet(row));
}
}
void assertPubKs(File file, String pass, List<String> hosts) throws Exception {
KeyStore ks = loadKeyStore("jceks", file, pass);
List aliases = ListUtil.fromIterator(new EnumerationIterator(ks.aliases()));
assertEquals(hosts.size(), aliases.size());
for (String host : hosts) {
String alias = host + ".crt";
Certificate cert = ks.getCertificate(alias);
assertNotNull(cert);
assertEquals("X.509", cert.getType());
}
}
// test completion & callback
public void testDone() throws Exception {
HashQueue q = new HashQueue();
final List cookieList = new LinkedList();
final List eList = new LinkedList();
HashService.Callback cb =
new HashService.Callback() {
public void hashingFinished(
CachedUrlSet urlset,
long timeUsed,
Object cookie,
CachedUrlSetHasher hasher,
Exception e) {
cookieList.add(cookie);
eList.add(e);
}
};
HashQueue.Request r1, r2, r3, r4, r5;
r1 = req(2000, 0, 100, cb);
r2 = req(10000, 0, 200, cb);
r3 = req(20000, 0, 0, cb);
r4 = req(50000, 0, 1, cb);
assertTrue(q.insert(r1));
assertTrue(q.insert(r2));
assertTrue(q.insert(r4));
assertEquals(0, cookieList.size());
q.removeCompleted();
assertEquals(0, cookieList.size());
// make r1 timeout
r1.deadline.expire();
q.removeCompleted();
List exp = ListUtil.list(r1);
assertEquals(exp, cookieList);
assertEquals(exp, q.getCompletedSnapshot());
// make r2 timeout
TimeBase.step(11000);
// r3 is finished
assertTrue(q.insert(r3));
Exception r4e = new Exception();
// make r4 error
r4.e = r4e;
q.removeCompleted();
// check that they all finished, and in the right order
Object exp2[] = {r1, r2, r3, r4};
assertIsomorphic(exp2, cookieList);
assertIsomorphic(exp2, q.getCompletedSnapshot());
// check their exceptions
assertTrue(eList.get(0) instanceof HashService.Timeout);
assertTrue(eList.get(1) instanceof HashService.Timeout);
assertSame(null, eList.get(2));
assertSame(r4e, eList.get(3));
}
public void testSimpleMainXML() throws Exception {
log.debug3("testSimpleMainXML");
String xml_url = TAR_A_BASE + SUBDIR + "01420615/v64sC/S0142061514004608/main.xml";
List<ArticleMetadata> mdList =
extractFromContent(xml_url, "text/xml", simpleMain, nocheck_mle, null);
assertEquals(1, mdList.size());
validateSingleMainMetadataRecord(mdList.get(0), "10.1016/j.jidx.2014.07.028", "article");
}
public void testFunctionalFromTarHierarchy() throws Exception {
log.debug3("in testFromTarHierarchy");
// load the tarballs
InputStream file_input = null;
try {
file_input = getResourceAsStream(realTARFile_A);
// UrlCacher uc = au.makeUrlCacher(TAR_A_BASE);
// uc.storeContent(file_input, tarHeader);
UrlCacher uc = tarAu.makeUrlCacher(new UrlData(file_input, tarHeader, TAR_A_BASE));
uc.storeContent();
IOUtil.safeClose(file_input);
file_input = getResourceAsStream(realTARFile_B);
// uc = au.makeUrlCacher(TAR_B_BASE);
// uc.storeContent(file_input, tarHeader);
uc = tarAu.makeUrlCacher(new UrlData(file_input, tarHeader, TAR_B_BASE));
uc.storeContent();
IOUtil.safeClose(file_input);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
IOUtil.safeClose(file_input);
}
CachedUrlSet cus = tarAu.getAuCachedUrlSet();
for (CachedUrl cu : cus.getCuIterable()) {
log.debug3("AU - cu is: " + cu.getUrl());
cu.release();
}
// We need to start from the level of the ArticleMetadataExtractor
MyListEmitter emitter = new MyListEmitter();
ArticleMetadataExtractor amEx =
new ElsevierDeferredArticleMetadataExtractor(ArticleFiles.ROLE_ARTICLE_METADATA);
Iterator<ArticleFiles> it = tarAu.getArticleIterator(MetadataTarget.Any());
while (it.hasNext()) {
ArticleFiles af = it.next();
log.debug3("Metadata test - articlefiles " + af.toString());
// CachedUrl cu = af.getFullTextCu();
CachedUrl cu = af.getRoleCu(ArticleFiles.ROLE_ARTICLE_METADATA);
log.debug3("metadata cu is " + cu.getUrl());
// List<ArticleMetadata> mdlist = mle.extract(MetadataTarget.Any(), cu);
amEx.extract(MetadataTarget.Any(), af, emitter);
List<ArticleMetadata> returnList = emitter.getAmList();
assertNotNull(returnList);
log.debug3("size of returnList is " + returnList.size());
Iterator<ArticleMetadata> mdIt = returnList.iterator();
ArticleMetadata mdRecord = null;
while (mdIt.hasNext()) {
mdRecord = (ArticleMetadata) mdIt.next();
validateCompleteMetadataRecord(mdRecord);
}
}
}
void assertPrivateKs(File file, String pass, String alias) throws Exception {
KeyStore ks = loadKeyStore("jceks", file, alias);
List aliases = ListUtil.fromIterator(new EnumerationIterator(ks.aliases()));
assertEquals(2, aliases.size());
Certificate cert = ks.getCertificate(alias + ".crt");
assertNotNull(cert);
assertEquals("X.509", cert.getType());
assertTrue(ks.isKeyEntry(alias + ".key"));
assertTrue(ks.isCertificateEntry(alias + ".crt"));
Key key = ks.getKey(alias + ".key", pass.toCharArray());
assertNotNull(key);
assertEquals("RSA", key.getAlgorithm());
}
public void testFindTaskToRunRemovesExpiredOverrunners() {
assertFalse(tr.findTaskToRun());
StepTask t1 = task(100, 200, 100);
StepTask t2 = task(100, 300, 50);
StepTask texp1 = task(0, 0, 50);
StepTask texp2 = task(0, 0, 49);
Schedule s = sched(ListUtil.list(t1, t2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
tr.addOverrunner(texp1);
tr.addOverrunner(texp2);
// if this fails, it might be because the sorted list/set is treating
// sort-order equivalence as object equality, which we don't want
assertEquals(2, tr.getOverrunTasks().size());
assertFalse(tr.findTaskToRun());
assertEquals(0, removedChunks.size());
assertEquals(2, removedTasks.size());
assertEquals(
SetUtil.set(texp1, texp2),
SetUtil.set((StepTask) removedTasks.get(0), (StepTask) removedTasks.get(1)));
}
public void testSimpleDatasetXML() throws Exception {
log.debug3("testSimpleDatasetXML");
String file_input = StringUtil.fromInputStream(getResourceAsStream(testDatasetFile));
String xml_url = TAR_A_BASE + SUBDIR + "dataset.xml";
List<ArticleMetadata> mdList =
extractFromContent(xml_url, "text/xml", file_input, nocheck_mle, null);
assertEquals(6, mdList.size());
Iterator<ArticleMetadata> mdIt = mdList.iterator();
ArticleMetadata mdRecord = null;
while (mdIt.hasNext()) {
mdRecord = (ArticleMetadata) mdIt.next();
validateDatasetMetadataRecord(mdRecord);
}
}
public void testFindTaskToRunRemovesExpiredChunks() {
assertFalse(tr.findTaskToRun());
StepTask t1 = task(100, 200, 100);
StepTask t2 = task(100, 300, 50);
StepTask texp1 = task(0, 0, 50);
StepTask texp2 = task(0, 0, 50);
Schedule s = sched(ListUtil.list(texp1, texp2, t1, t2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
assertFalse(tr.findTaskToRun());
assertEquals(2, removedChunks.size());
assertEquals(
SetUtil.set(texp1, texp2),
SetUtil.set(
((Schedule.Chunk) removedChunks.get(0)).getTask(),
((Schedule.Chunk) removedChunks.get(1)).getTask()));
}
public void testText() throws Exception {
MockStatusAccessor statusAccessor =
MockStatusAccessor.generateStatusAccessor(colArray1, rowArray1);
statusAccessor.setTitle("testtbl", null);
statSvc.registerStatusAccessor("testtbl", statusAccessor);
WebResponse resp = getTable("testtbl", true);
assertResponseOk(resp);
assertEquals("Content type", "text/plain", resp.getContentType());
log.debug(resp.getText());
List lines = getLines(resp);
assertEquals(rowArray1.length + 3, lines.size());
Map row0 = getRow((String) lines.get(0));
assertEquals("2.4.6.8", row0.get("host"));
Map row2 = getRow((String) lines.get(2));
assertEquals("testtbl", row2.get("table"));
assertEqualTables(table1, lines);
}
public void testNotifyThread() {
final List rec = new ArrayList();
final SimpleBinarySemaphore sem = new SimpleBinarySemaphore();
tr.setImmediateNotify(false);
TaskCallback cb =
new TaskCallback() {
public void taskEvent(SchedulableTask task, Schedule.EventType event) {
rec.add(new BERec(Deadline.in(0), (BackgroundTask) task, event));
sem.give();
}
};
BackgroundTask t1 = btask(100, 200, .1, cb);
BackgroundTask t2 = btask(100, 300, .2, cb);
tr.notify(t1, Schedule.EventType.START);
tr.notify(t1, Schedule.EventType.FINISH);
// 2nd finish event should not cause another callback
tr.notify(t1, Schedule.EventType.FINISH);
tr.notify(t2, Schedule.EventType.START);
Interrupter intr = null;
try {
intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
while (rec.size() < 3) {
sem.take();
}
assertEquals(
ListUtil.list(
new BERec(0, t1, Schedule.EventType.START),
new BERec(0, t1, Schedule.EventType.FINISH),
new BERec(0, t2, Schedule.EventType.START)),
rec);
intr.cancel();
} finally {
if (intr.did()) {
fail("Notifier didn't run callbacks");
}
}
}
public void testBackground() {
final List rec = new ArrayList();
TaskCallback cb =
new TaskCallback() {
public void taskEvent(SchedulableTask task, Schedule.EventType event) {
rec.add(new BERec(Deadline.in(0), (BackgroundTask) task, event));
}
};
assertFalse(tr.findTaskToRun());
BackgroundTask t1 = btask(100, 200, .1, cb);
BackgroundTask t2 = btask(100, 300, .2, cb);
BackgroundTask t3 = btask(150, 200, .4, cb);
Schedule s =
sched(
ListUtil.list(
bEvent(t1, Schedule.EventType.START),
bEvent(t2, Schedule.EventType.START),
bEvent(t3, Schedule.EventType.START),
bEvent(t1, Schedule.EventType.FINISH),
bEvent(t3, Schedule.EventType.FINISH),
bEvent(t2, Schedule.EventType.FINISH)));
fact.setResults(ListUtil.list(s, s, s));
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
assertTrue(tr.addToSchedule(t3));
assertEquals(3, tr.getAcceptedTasks().size());
assertIsomorphic(ListUtil.list(t1, t2, t3), tr.getAcceptedTasks());
assertFalse(tr.findTaskToRun());
assertEquals(0, rec.size());
assertEquals(0, tr.getBackgroundLoadFactor(), .005);
assertEquals(Deadline.at(100), tr.runningDeadline);
TimeBase.setSimulated(101);
assertFalse(tr.findTaskToRun());
assertEquals(2, rec.size());
assertEquals(.3, tr.getBackgroundLoadFactor(), .005);
TimeBase.setSimulated(151);
assertFalse(tr.findTaskToRun());
assertEquals(3, rec.size());
assertEquals(.7, tr.getBackgroundLoadFactor(), .005);
assertEquals(3, tr.getAcceptedTasks().size());
TimeBase.setSimulated(201);
assertFalse(tr.findTaskToRun());
assertEquals(5, rec.size());
assertEquals(.2, tr.getBackgroundLoadFactor(), .005);
assertEquals(1, tr.getAcceptedTasks().size());
t2.taskIsFinished();
TimeBase.setSimulated(202);
assertFalse(tr.findTaskToRun());
assertEquals(6, rec.size());
assertEquals(0, tr.getBackgroundLoadFactor(), .005);
assertEquals(0, tr.getAcceptedTasks().size());
TimeBase.setSimulated(301);
assertFalse(tr.findTaskToRun());
assertEquals(6, rec.size());
assertEquals(0, tr.getBackgroundLoadFactor(), .005);
List exp =
ListUtil.list(
new BERec(101, t1, Schedule.EventType.START),
new BERec(101, t2, Schedule.EventType.START),
new BERec(151, t3, Schedule.EventType.START),
new BERec(201, t1, Schedule.EventType.FINISH),
new BERec(201, t3, Schedule.EventType.FINISH),
new BERec(201, t2, Schedule.EventType.FINISH));
assertEquals(exp, rec);
}
public void testStorePollHistories() throws Exception {
TimeBase.setSimulated(123321);
MockCachedUrlSetSpec mspec = new MockCachedUrlSetSpec("http://www.example.com", null);
CachedUrlSet mcus = new MockCachedUrlSet(mau, mspec);
NodeStateImpl nodeState = new NodeStateImpl(mcus, -1, null, null, repository);
List histories =
ListUtil.list(
createPollHistoryBean(3),
createPollHistoryBean(3),
createPollHistoryBean(3),
createPollHistoryBean(3),
createPollHistoryBean(3));
/*
* CASTOR: [summary] Rewrite test in non-Castor way
* This is obviously not an appropriate way of writing this test,
* Right now it creates sample data in Castor format, from legacy
* code back when Castor was the built-in serialization engine.
* TODO: Rewrite test in non-Castor way
*/
// nodeState.setPollHistoryBeanList(histories);
nodeState.setPollHistoryList(NodeHistoryBean.fromBeanListToList(histories));
repository.storePollHistories(nodeState);
String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
filePath =
LockssRepositoryImpl.mapUrlToFileLocation(
filePath, "http://www.example.com/" + HistoryRepositoryImpl.HISTORY_FILE_NAME);
File xmlFile = new File(filePath);
assertTrue(xmlFile.exists());
nodeState.setPollHistoryList(new ArrayList());
repository.loadPollHistories(nodeState);
List loadedHistory = nodeState.getPollHistoryList();
assertEquals(histories.size(), loadedHistory.size());
// CASTOR: some Castor-tailored stuff here
// PollHistoryBean expect1 = (PollHistoryBean)histories.get(0);
// PollHistoryBean elem1 = (PollHistoryBean)loadedHistory.get(0);
PollHistory expect1 = (PollHistory) histories.get(0);
PollHistory elem1 = (PollHistory) loadedHistory.get(0);
assertEquals(expect1.type, elem1.type);
assertEquals(expect1.lwrBound, elem1.lwrBound);
assertEquals(expect1.uprBound, elem1.uprBound);
assertEquals(expect1.status, elem1.status);
assertEquals(expect1.startTime, elem1.startTime);
assertEquals(expect1.duration, elem1.duration);
// CASTOR: some Castor-tailored stuff here
// List expectBeans = (List)expect1.getVoteBeans();
// List elemBeans = (List)elem1.getVoteBeans();
Iterator expectIter = (Iterator) expect1.getVotes();
Iterator elemIter = (Iterator) elem1.getVotes();
while (expectIter.hasNext() && elemIter.hasNext()) {
Vote expectVote = (Vote) expectIter.next();
Vote elemVote = (Vote) elemIter.next();
assertEquals(
expectVote.getVoterIdentity().getIdString(), elemVote.getVoterIdentity().getIdString());
assertEquals(expectVote.isAgreeVote(), elemVote.isAgreeVote());
assertEquals(expectVote.getChallengeString(), elemVote.getChallengeString());
assertEquals(expectVote.getVerifierString(), elemVote.getVerifierString());
assertEquals(expectVote.getHashString(), elemVote.getHashString());
}
assertFalse(expectIter.hasNext());
assertFalse(expectIter.hasNext());
TimeBase.setReal();
}