// remove overrunnable task-ending chunk, before deadline,
public void testRemoveChunkTaskEndOver() {
final List finished = new ArrayList();
StepTask t1 =
task(
100,
200,
100,
new TaskCallback() {
public void taskEvent(SchedulableTask task, Schedule.EventType event) {
if (log.isDebug2()) {
log.debug2("testRemoveChunkTaskEndOver callback");
}
if (event == Schedule.EventType.FINISH) {
finished.add(task);
}
}
});
t1.setOverrunAllowed(true);
Schedule s = sched(ListUtil.list(t1));
fact.setResult(s);
assertTrue(tr.addToSchedule(t1));
Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
chunk.setTaskEnd();
tr.removeChunk(chunk);
assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
assertEmpty(finished);
assertIsomorphic(ListUtil.list(t1), tr.getAcceptedTasks());
assertIsomorphic(SetUtil.set(t1), tr.getOverrunTasks());
}
public void testGetRepositoryList() throws Exception {
assertEmpty(mgr.getRepositoryList());
String tempDirPath = setUpDiskSpace();
assertEquals(ListUtil.list("local:" + tempDirPath), mgr.getRepositoryList());
String tempdir2 = getTempDir().getAbsolutePath() + File.separator;
ConfigurationUtil.setFromArgs(
"org.lockss.platform.diskSpacePaths", tempdir2 + ";" + tempDirPath);
assertEquals(
ListUtil.list("local:" + tempdir2, "local:" + tempDirPath), mgr.getRepositoryList());
}
public void testRemoveChunk() {
StepTask t1 = task(100, 200, 100);
Schedule s = sched(ListUtil.list(t1));
fact.setResult(s);
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), tr.getAcceptedTasks());
Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
tr.removeChunk(chunk);
assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
}
public void testStoreDamagedNodeSet() throws Exception {
DamagedNodeSet damNodes = new DamagedNodeSet(mau, repository);
damNodes.nodesWithDamage.add("test1");
damNodes.nodesWithDamage.add("test2");
damNodes.cusToRepair.put("cus1", ListUtil.list("cus1-1", "cus1-2"));
damNodes.cusToRepair.put("cus2", ListUtil.list("cus2-1"));
assertTrue(damNodes.containsWithDamage("test1"));
assertTrue(damNodes.containsWithDamage("test2"));
assertFalse(damNodes.containsWithDamage("test3"));
repository.storeDamagedNodeSet(damNodes);
String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
filePath += HistoryRepositoryImpl.DAMAGED_NODES_FILE_NAME;
File xmlFile = new File(filePath);
assertTrue(xmlFile.exists());
damNodes = null;
damNodes = repository.loadDamagedNodeSet();
// check damage
assertTrue(damNodes.containsWithDamage("test1"));
assertTrue(damNodes.containsWithDamage("test2"));
assertFalse(damNodes.containsWithDamage("test3"));
MockCachedUrlSet mcus1 = new MockCachedUrlSet("cus1");
MockCachedUrlSet mcus2 = new MockCachedUrlSet("cus2");
// check repairs
assertTrue(damNodes.containsToRepair(mcus1, "cus1-1"));
assertTrue(damNodes.containsToRepair(mcus1, "cus1-2"));
assertFalse(damNodes.containsToRepair(mcus1, "cus2-1"));
assertTrue(damNodes.containsToRepair(mcus2, "cus2-1"));
assertEquals(mau.getAuId(), damNodes.theAu.getAuId());
// check remove
damNodes.removeFromRepair(mcus1, "cus1-1");
assertFalse(damNodes.containsToRepair(mcus1, "cus1-1"));
assertTrue(damNodes.containsToRepair(mcus1, "cus1-2"));
damNodes.removeFromRepair(mcus1, "cus1-2");
assertFalse(damNodes.containsToRepair(mcus1, "cus1-2"));
assertNull(damNodes.cusToRepair.get(mcus1));
// check remove from damaged nodes
damNodes.removeFromDamage("test1");
damNodes.removeFromDamage("test2");
repository.storeDamagedNodeSet(damNodes);
damNodes = repository.loadDamagedNodeSet();
assertNotNull(damNodes);
assertFalse(damNodes.containsWithDamage("test1"));
assertFalse(damNodes.containsWithDamage("test2"));
}
public void getGroup() throws Exception {
m_testMsg.setGroups(null);
assertNull(m_testMsg.getGroups());
assertNull(m_testMsg.getGroupList());
m_testMsg.setGroups("foo");
assertEquals("foo", m_testMsg.getGroups());
assertEquals(ListUtil.list("foo"), m_testMsg.getGroupList());
m_testMsg.setGroups("foo;bar");
assertEquals("foo;bar", m_testMsg.getGroups());
assertEquals(ListUtil.list("foo", "bar"), m_testMsg.getGroupList());
m_testMsg.setGroups("foo;bar;baz");
assertEquals("foo;bar;baz", m_testMsg.getGroups());
assertEquals(ListUtil.list("foo", "bar", "baz"), m_testMsg.getGroupList());
}
public void testIsTaskSchedulable() {
fact.setResult(null);
StepTask t1 = task(100, 200, 50);
assertFalse(tr.isTaskSchedulable(t1));
fact.setResult(sched(ListUtil.list(t1)));
assertTrue(tr.isTaskSchedulable(t1));
}
public void testRunStepsWithOverrunDisallowed() {
StepTask t1 = task(100, 300, 100, null, new MyMockStepper(15, -10));
// t1.setOverrunAllowed(true);
StepTask t2 = task(150, 250, 100, null, new MyMockStepper(10, -10));
Schedule s = sched(ListUtil.list(t1, t2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
TimeBase.setSimulated(101);
Interrupter intr = null;
try {
intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
while (tr.findTaskToRun()) {
tr.runSteps(new MutableBoolean(true), null);
}
intr.cancel();
} catch (Exception e) {
log.error("runSteps threw:", e);
} finally {
if (intr.did()) {
fail("runSteps looped");
}
}
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(removedTasks));
assertTrue(t1.e.toString(), t1.e instanceof SchedService.Overrun);
}
public void testStepperThrows() {
final List finished = new ArrayList();
TaskCallback cb =
new TaskCallback() {
public void taskEvent(SchedulableTask task, Schedule.EventType event) {
if (event == Schedule.EventType.FINISH) {
finished.add(task);
}
}
};
MyMockStepper stepper = new MyMockStepper(10, -10);
stepper.setWhenToThrow(5);
StepTask t1 = task(100, 200, 100, cb, stepper);
Schedule s = sched(ListUtil.list(t1));
fact.setResult(s);
assertTrue(tr.addToSchedule(t1));
TimeBase.setSimulated(101);
assertTrue(tr.findTaskToRun());
Interrupter intr = null;
try {
intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
tr.runSteps(new MutableBoolean(true), null);
intr.cancel();
} catch (Exception e) {
log.error("runSteps threw:", e);
} finally {
if (intr.did()) {
fail("runSteps looped");
}
}
assertSame(t1, finished.get(0));
assertTrue(t1.e instanceof ExpectedRuntimeException);
assertEquals(5, stepper.nSteps);
}
// remove task-ending chunk, past task deadline, s.b. Timeout error.
public void testRemoveChunkTaskEndTimeout() {
final List finished = new ArrayList();
StepTask t1 =
task(
100,
200,
100,
new TaskCallback() {
public void taskEvent(SchedulableTask task, Schedule.EventType event) {
if (log.isDebug2()) {
log.debug2("testRemoveChunkTaskEndTimeout callback");
}
if (event == Schedule.EventType.FINISH) {
finished.add(task);
}
}
});
Schedule s = sched(ListUtil.list(t1));
fact.setResult(s);
assertTrue(tr.addToSchedule(t1));
Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
chunk.setTaskEnd();
TimeBase.setSimulated(201);
tr.removeChunk(chunk);
assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
assertSame(t1, finished.get(0));
assertNotNull(t1.e);
assertTrue(t1.e.toString(), t1.e instanceof SchedService.Timeout);
assertEmpty(tr.getAcceptedTasks());
}
public void testSizeCalc() throws Exception {
SimpleBinarySemaphore sem = new SimpleBinarySemaphore();
mgr.setSem(sem);
RepositoryNode node1 = new RepositoryNodeImpl("url1", "testDir", null);
RepositoryNode node2 = new RepositoryNodeImpl("url2", "testDir", null);
RepositoryNode node3 = new RepositoryNodeImpl("url3", "testDir", null);
mgr.queueSizeCalc(node1);
assertTrue(sem.take(TIMEOUT_SHOULDNT));
assertEquals(ListUtil.list(node1), mgr.getNodes());
mgr.queueSizeCalc(node2);
mgr.queueSizeCalc(node3);
assertTrue(sem.take(TIMEOUT_SHOULDNT));
if (mgr.getNodes().size() < 3) {
assertTrue(sem.take(TIMEOUT_SHOULDNT));
}
assertSameElements(ListUtil.list(node1, node2, node3), mgr.getNodes());
}
// ensure addToSchedule updates structures if (Mock)Scheduler returns true
public void testAddToScheduleOk() {
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched = sched(ListUtil.list(t1, t2));
fact.setResult(sched);
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
fact.setResult(sched);
assertTrue(tr.addToSchedule(t2));
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks));
assertEquals(sched, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(2, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
}
public void testCdnStems() {
AuState aus = new AuState(mau, historyRepo);
assertEquals(Collections.EMPTY_LIST, aus.getCdnStems());
aus.addCdnStem("http://fff.uselesstld");
assertClass(ArrayList.class, aus.getCdnStems());
assertEquals(ListUtil.list("http://fff.uselesstld"), aus.getCdnStems());
aus.addCdnStem("ccc");
assertEquals(ListUtil.list("http://fff.uselesstld", "ccc"), aus.getCdnStems());
aus.setCdnStems(new LinkedList(ListUtil.list("a", "b")));
assertClass(ArrayList.class, aus.getCdnStems());
assertEquals(ListUtil.list("a", "b"), aus.getCdnStems());
aus.setCdnStems(null);
assertEmpty(aus.getCdnStems());
aus.addCdnStem("https://a.b/");
aus.addCdnStem("https://b.a/");
assertEquals(ListUtil.list("https://a.b/", "https://b.a/"), aus.getCdnStems());
}
public void testParsesOptionPositive() throws IOException {
TypedEntryMap pMap = new TypedEntryMap();
pMap.setMapElement("html-parser-select-attrs", ListUtil.list("value"));
mau.setPropertyMap(pMap);
singleTagShouldParse(
"http://www.example.com/web_link.jpg", "<option value=", "</option>", mau);
singleTagShouldParse(
"http://www.example.com/web_link.jpg", "<option a=b value=", "</option>", mau);
}
public void testCreateSharedPLNKeyStores() throws Exception {
List<String> hosts = ListUtil.list("host1", "host2.foo.bar", "host3");
List<String> hosts2 = ListUtil.list("host3", "host4");
File dir = getTempDir();
File pub = new File(dir, "pub.ks");
KeyStoreUtil.createSharedPLNKeyStores(
dir, hosts, pub, "pubpass", MiscTestUtil.getSecureRandom());
assertPubKs(pub, "pubpass", hosts);
for (String host : hosts) {
assertPrivateKs(
new File(dir, host + ".jceks"), StringUtil.fromFile(new File(dir, host + ".pass")), host);
}
KeyStore pubks1 = loadKeyStore("jceks", new File(dir, "pub.ks"), "pubpass");
Certificate host1cert1 = pubks1.getCertificate("host1.crt");
Certificate host3cert1 = pubks1.getCertificate("host3.crt");
String host1priv1 = StringUtil.fromFile(new File(dir, "host1.jceks"));
String host3priv1 = StringUtil.fromFile(new File(dir, "host3.jceks"));
// Now add host4 and generate a new key for host3
KeyStoreUtil.createSharedPLNKeyStores(
dir, hosts2, pub, "pubpass", MiscTestUtil.getSecureRandom());
List<String> both = ListUtils.sum(hosts, hosts2);
assertPubKs(pub, "pubpass", both);
for (String host : both) {
assertPrivateKs(
new File(dir, host + ".jceks"), StringUtil.fromFile(new File(dir, host + ".pass")), host);
}
KeyStore pubks2 = loadKeyStore("jceks", new File(dir, "pub.ks"), "pubpass");
// host1 should have the same cert, host3 not
Certificate host1cert2 = pubks2.getCertificate("host1.crt");
Certificate host3cert2 = pubks2.getCertificate("host3.crt");
assertEquals(host1cert1, host1cert2);
assertNotEquals(host3cert1, host3cert2);
// host1's private key file should be the same, host3's not
String host1priv2 = StringUtil.fromFile(new File(dir, "host1.jceks"));
String host3priv2 = StringUtil.fromFile(new File(dir, "host3.jceks"));
assertEquals(host1priv1, host1priv2);
assertNotEquals(host3priv1, host3priv2);
}
// test special (non-string) key in row doesn't throw
public void testTextNonStringKey() throws Exception {
MockStatusAccessor statusAccessor = new MockStatusAccessor();
List cols = ListUtil.list("foo", StatusTable.ROW_SEPARATOR);
statusAccessor.setRows(MockStatusAccessor.makeRowsFrom(cols, rowArray1), null);
statusAccessor.setColumnDescriptors(
MockStatusAccessor.makeColumnDescriptorsFrom(colArray1), null);
statSvc.registerStatusAccessor("testtbl", statusAccessor);
WebResponse resp = getTable("testtbl", true);
log.debug(resp.getText());
}
// one task to drop, two failed schedule tries
public void testAddToScheduleFailNoCleanup() {
ConfigurationUtil.addFromArgs(
TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched = sched(ListUtil.list(t1));
fact.setResult(sched);
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
assertFalse(tr.addToSchedule(t2));
assertEquals(
ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
fact.createArgs);
assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
assertEquals(sched, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(1, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
}
// one task to drop, succeeds after dropping it
public void testAddToScheduleOkAfterDrops() {
log.debug("testAddToScheduleOkAfterDrops()");
ConfigurationUtil.addFromArgs(
TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched1 = sched(ListUtil.list(t1));
Schedule sched2 = sched(ListUtil.list(t2));
fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2));
TimeBase.step(101);
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
assertTrue(tr.addToSchedule(t2));
assertEquals(
ListUtil.list(
ListUtil.list(t1),
ListUtil.list(t1, t2),
ListUtil.list(t1),
Collections.EMPTY_SET,
ListUtil.list(t1, t2)),
fact.createArgs);
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks));
assertTrue(t1.isDropped());
assertEquals(sched2, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks()));
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getOverrunTasks()));
assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(1, TaskRunner.STAT_DROPPED);
t2.cancel();
assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(0, TaskRunner.STAT_WAITING);
assertForegroundStat(1, TaskRunner.STAT_DROPPED);
assertForegroundStat(1, TaskRunner.STAT_CANCELLED);
}
// test completion & callback
public void testDone() throws Exception {
HashQueue q = new HashQueue();
final List cookieList = new LinkedList();
final List eList = new LinkedList();
HashService.Callback cb =
new HashService.Callback() {
public void hashingFinished(
CachedUrlSet urlset,
long timeUsed,
Object cookie,
CachedUrlSetHasher hasher,
Exception e) {
cookieList.add(cookie);
eList.add(e);
}
};
HashQueue.Request r1, r2, r3, r4, r5;
r1 = req(2000, 0, 100, cb);
r2 = req(10000, 0, 200, cb);
r3 = req(20000, 0, 0, cb);
r4 = req(50000, 0, 1, cb);
assertTrue(q.insert(r1));
assertTrue(q.insert(r2));
assertTrue(q.insert(r4));
assertEquals(0, cookieList.size());
q.removeCompleted();
assertEquals(0, cookieList.size());
// make r1 timeout
r1.deadline.expire();
q.removeCompleted();
List exp = ListUtil.list(r1);
assertEquals(exp, cookieList);
assertEquals(exp, q.getCompletedSnapshot());
// make r2 timeout
TimeBase.step(11000);
// r3 is finished
assertTrue(q.insert(r3));
Exception r4e = new Exception();
// make r4 error
r4.e = r4e;
q.removeCompleted();
// check that they all finished, and in the right order
Object exp2[] = {r1, r2, r3, r4};
assertIsomorphic(exp2, cookieList);
assertIsomorphic(exp2, q.getCompletedSnapshot());
// check their exceptions
assertTrue(eList.get(0) instanceof HashService.Timeout);
assertTrue(eList.get(1) instanceof HashService.Timeout);
assertSame(null, eList.get(2));
assertSame(r4e, eList.get(3));
}
public void testSortedRepairCandidates() throws Exception {
VersionCounts versionCounts = VersionCounts.make();
VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
addVersion(vb1, "content 1 for foo1");
VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
addVersion(vb2, "content 2 for foo1");
VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
addVersion(vb3, "content 3 for foo1");
addVersion(vb3, "content 2 for foo1");
versionCounts.vote(vb1, participant1);
versionCounts.vote(vb2, participant2);
versionCounts.vote(vb3, participant3);
Map<Integer, Collection<ParticipantUserData>> repairCandidates;
repairCandidates = versionCounts.getSortedRepairCandidatesMap(2);
assertEquals(SetUtil.set(2), repairCandidates.keySet());
assertSameElements(SetUtil.set(participant2), repairCandidates.get(2));
assertEquals(ListUtil.list(participant2), versionCounts.getSortedRepairCandidates(2));
repairCandidates = versionCounts.getSortedRepairCandidatesMap(1);
assertIsomorphic(ListUtil.list(2, 1), repairCandidates.keySet());
assertSameElements(SetUtil.set(participant2), repairCandidates.get(2));
assertSameElements(SetUtil.set(participant1, participant3), repairCandidates.get(1));
List<ParticipantUserData> lst = versionCounts.getSortedRepairCandidates(1);
assertTrue(
"" + lst,
(lst.equals(ListUtil.list(participant2, participant1, participant3))
|| lst.equals(ListUtil.list(participant2, participant3, participant1))));
assertEmpty(versionCounts.getSortedRepairCandidatesMap(4));
assertEmpty(versionCounts.getSortedRepairCandidates(4));
}
public void testStoreAuEmptyState() throws Exception {
HashSet strCol = new HashSet();
strCol.add("test");
AuState origState = new AuState(mau, repository);
repository.storeAuState(origState);
AuState loadedState = repository.loadAuState();
assertEquals(-1, loadedState.getLastCrawlTime());
assertEquals(-1, loadedState.getLastCrawlAttempt());
assertEquals(-1, loadedState.getLastCrawlResult());
assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg());
assertEquals(-1, loadedState.getLastTopLevelPollTime());
assertEquals(-1, loadedState.getLastPollStart());
assertEquals(-1, loadedState.getLastPollResult());
assertEquals(null, loadedState.getLastPollResultMsg());
assertEquals(-1, loadedState.getLastPoPPoll());
assertEquals(-1, loadedState.getLastPoPPollResult());
assertEquals(-1, loadedState.getLastLocalHashScan());
assertEquals(0, loadedState.getNumAgreePeersLastPoR());
assertEquals(0, loadedState.getNumWillingRepairers());
assertEquals(0, loadedState.getNumCurrentSuspectVersions());
assertEmpty(loadedState.getCdnStems());
loadedState.addCdnStem("http://this.is.new/");
assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
loadedState.addCdnStem("http://this.is.new/");
assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
assertEquals(0, loadedState.getPollDuration());
assertEquals(0, loadedState.getClockssSubscriptionStatus());
assertEquals(null, loadedState.getAccessType());
assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState());
assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance));
assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata));
assertEquals(-1, loadedState.getLastMetadataIndex());
assertEquals(0, loadedState.getLastContentChange());
assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());
}
public void testFromList1() {
assertEquals(MapUtil.map(), MapUtil.fromList(ListUtil.list()));
assertEquals(
MapUtil.map("FOO", "bar", "One", "Two"),
MapUtil.fromList(ListUtil.list("FOO", "bar", "One", "Two")));
assertEquals(
MapUtil.map("foo", "bar", "one", "two"),
MapUtil.fromList(ListUtil.list(ListUtil.list("foo", "bar"), ListUtil.list("one", "two"))));
try {
MapUtil.fromList(ListUtil.list("FOO", "bar", "One"));
fail("Odd length arg list should throw");
} catch (IllegalArgumentException e) {
}
try {
MapUtil.fromList(ListUtil.list(ListUtil.list("foo", "bar"), ListUtil.list("one")));
fail("Short sublist should throw");
} catch (IllegalArgumentException e) {
}
}
public void testFindOverrunTaskToRun() {
assertFalse(tr.findTaskToRun());
StepTask t1 = task(100, 200, 100);
Schedule s = sched(ListUtil.list(t1));
fact.setResult(s);
assertTrue(tr.addToSchedule(t1));
assertFalse(tr.findTaskToRun());
assertEquals(Deadline.at(100), tr.runningDeadline);
StepTask t2 = task(0, 300, 50);
tr.addOverrunner(t2);
assertTrue(tr.findTaskToRun());
assertEquals(t2, tr.runningTask);
assertEquals(Deadline.at(100), tr.runningDeadline);
assertNull(tr.runningChunk);
}
public void testFindChunkTaskToRun() {
assertFalse(tr.findTaskToRun());
StepTask t1 = task(100, 200, 100);
StepTask t2 = task(100, 300, 50);
Schedule s = sched(ListUtil.list(t1, t2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
assertFalse(tr.findTaskToRun());
assertEquals(Deadline.at(100), tr.runningDeadline);
TimeBase.setSimulated(101);
assertTrue(tr.findTaskToRun());
assertEquals(t1, tr.runningTask);
assertEquals(t1.getLatestFinish(), tr.runningDeadline);
assertEquals(s.getEvents().get(0), tr.runningChunk);
}
public void testFindTaskToRunRemovesExpiredChunks() {
assertFalse(tr.findTaskToRun());
StepTask t1 = task(100, 200, 100);
StepTask t2 = task(100, 300, 50);
StepTask texp1 = task(0, 0, 50);
StepTask texp2 = task(0, 0, 50);
Schedule s = sched(ListUtil.list(texp1, texp2, t1, t2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
assertFalse(tr.findTaskToRun());
assertEquals(2, removedChunks.size());
assertEquals(
SetUtil.set(texp1, texp2),
SetUtil.set(
((Schedule.Chunk) removedChunks.get(0)).getTask(),
((Schedule.Chunk) removedChunks.get(1)).getTask()));
}
public void testFindRunnableChunk() {
assertFalse(tr.findTaskToRun());
StepTask t1 = task(100, 200, 100);
StepTask t2 = task(10, 300, 50);
Schedule.Chunk c1 = new Schedule.Chunk(t1, Deadline.at(100), Deadline.at(200), 100);
Schedule.Chunk c2 = new Schedule.Chunk(t2, Deadline.at(200), Deadline.at(300), 100);
Schedule s = new Schedule(ListUtil.list(c1, c2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
assertFalse(tr.findTaskToRun());
assertEquals(Deadline.at(100), tr.runningDeadline);
TimeBase.setSimulated(11);
assertTrue(tr.findTaskToRun());
assertEquals(t2, tr.runningTask);
assertEquals(c2, tr.runningChunk);
assertEquals(Deadline.at(100), tr.runningDeadline);
assertEquals(s.getEvents().get(1), tr.runningChunk);
}
// one task not ready to start yet, so not droppable
public void testAddToScheduleFailNoDroppable() {
log.debug("testAddToScheduleOkAfterDrops()");
ConfigurationUtil.addFromArgs(
TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched1 = sched(ListUtil.list(t1));
Schedule sched2 = sched(ListUtil.list(t2));
fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2));
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertFalse(tr.addToSchedule(t2));
assertEquals(
ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
fact.createArgs);
assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
assertFalse(t1.isDropped());
assertEquals(sched1, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertEmpty(SetUtil.theSet(tr.getOverrunTasks()));
}
public void testNotifyThread() {
final List rec = new ArrayList();
final SimpleBinarySemaphore sem = new SimpleBinarySemaphore();
tr.setImmediateNotify(false);
TaskCallback cb =
new TaskCallback() {
public void taskEvent(SchedulableTask task, Schedule.EventType event) {
rec.add(new BERec(Deadline.in(0), (BackgroundTask) task, event));
sem.give();
}
};
BackgroundTask t1 = btask(100, 200, .1, cb);
BackgroundTask t2 = btask(100, 300, .2, cb);
tr.notify(t1, Schedule.EventType.START);
tr.notify(t1, Schedule.EventType.FINISH);
// 2nd finish event should not cause another callback
tr.notify(t1, Schedule.EventType.FINISH);
tr.notify(t2, Schedule.EventType.START);
Interrupter intr = null;
try {
intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
while (rec.size() < 3) {
sem.take();
}
assertEquals(
ListUtil.list(
new BERec(0, t1, Schedule.EventType.START),
new BERec(0, t1, Schedule.EventType.FINISH),
new BERec(0, t2, Schedule.EventType.START)),
rec);
intr.cancel();
} finally {
if (intr.did()) {
fail("Notifier didn't run callbacks");
}
}
}
public void testFindTaskToRunRemovesExpiredOverrunners() {
assertFalse(tr.findTaskToRun());
StepTask t1 = task(100, 200, 100);
StepTask t2 = task(100, 300, 50);
StepTask texp1 = task(0, 0, 50);
StepTask texp2 = task(0, 0, 49);
Schedule s = sched(ListUtil.list(t1, t2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
tr.addOverrunner(texp1);
tr.addOverrunner(texp2);
// if this fails, it might be because the sorted list/set is treating
// sort-order equivalence as object equality, which we don't want
assertEquals(2, tr.getOverrunTasks().size());
assertFalse(tr.findTaskToRun());
assertEquals(0, removedChunks.size());
assertEquals(2, removedTasks.size());
assertEquals(
SetUtil.set(texp1, texp2),
SetUtil.set((StepTask) removedTasks.get(0), (StepTask) removedTasks.get(1)));
}
public class SingleCrawlStatusAccessor implements StatusAccessor {
private static final String MIME_TYPE_NAME = "mime_type_name";
private static final String MIME_TYPE_NUM_URLS = "mime_type_num_urls";
private static final String MIMETYPES_URLS_KEY = "mime-type";
private static final String CRAWL_URLS_STATUS_ACCESSOR = CrawlManagerImpl.CRAWL_URLS_STATUS_TABLE;
private List colDescsMimeTypes =
ListUtil.fromArray(
new ColumnDescriptor[] {
new ColumnDescriptor(MIME_TYPE_NAME, "Mime Type", ColumnDescriptor.TYPE_STRING),
new ColumnDescriptor(
MIME_TYPE_NUM_URLS,
"URLs Found",
ColumnDescriptor.TYPE_INT,
"Number of pages of that mime type fetched during this crawl"),
});
private static final List statusSortRules =
ListUtil.list(new StatusTable.SortRule(MIME_TYPE_NAME, true));
private CrawlManager.StatusSource statusSource;
public SingleCrawlStatusAccessor(CrawlManager.StatusSource statusSource) {
this.statusSource = statusSource;
}
public void populateTable(StatusTable table) throws StatusService.NoSuchTableException {
if (table == null) {
throw new IllegalArgumentException("Called with null table");
} else if (table.getKey() == null) {
throw new IllegalArgumentException("SingleCrawlStatusAccessor requires a key");
}
String key = table.getKey();
CrawlerStatus status = statusSource.getStatus().getCrawlerStatus(key);
if (status == null) {
throw new StatusService.NoSuchTableException(
"Status info from that crawl is no longer available");
}
table.setDefaultSortRules(statusSortRules);
table.setColumnDescriptors(colDescsMimeTypes);
table.setTitle(getTableTitle(status));
table.setRows(getRows(status, key));
table.setSummaryInfo(getSummaryInfo(status));
}
private String getTableTitle(CrawlerStatus status) {
return "Status of crawl of " + status.getAuName();
}
/** iterate over the mime-types makeRow for each */
private List getRows(CrawlerStatus status, String key) {
Collection mimeTypes = status.getMimeTypes();
List rows = new ArrayList();
if (mimeTypes != null) {
String mimeType;
for (Iterator it = mimeTypes.iterator(); it.hasNext(); ) {
mimeType = (String) it.next();
rows.add(makeRow(status, mimeType, key));
}
}
return rows;
}
private Map makeRow(CrawlerStatus status, String mimeType, String key) {
Map row = new HashMap();
row.put(MIME_TYPE_NAME, mimeType);
row.put(
MIME_TYPE_NUM_URLS,
makeRefIfColl(status.getMimeTypeCtr(mimeType), key, MIMETYPES_URLS_KEY + ":" + mimeType));
return row;
}
/** Return a reference object to the table, displaying the value */
private Object makeRef(long value, String tableName, String key) {
return new StatusTable.Reference(new Long(value), tableName, key);
}
/** If the UrlCounter has a collection, return a reference to it, else just the count */
Object makeRefIfColl(CrawlerStatus.UrlCount ctr, String crawlKey, String subkey) {
if (ctr.hasCollection()) {
return makeRef(ctr.getCount(), CRAWL_URLS_STATUS_ACCESSOR, crawlKey + "." + subkey);
}
return new Long(ctr.getCount());
}
public String getDisplayName() {
throw new UnsupportedOperationException("No generic name for MimeTypeStatusCrawler");
}
public boolean requiresKey() {
return true;
}
public static final String FOOT_NO_SUBSTANCE_CRAWL_STATUS =
"Though the crawl finished successfully, no files containing substantial content were collected.";
private List getSummaryInfo(CrawlerStatus status) {
List res = new ArrayList();
StatusTable.SummaryInfo statusSi =
new StatusTable.SummaryInfo(
"Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg());
ArchivalUnit au = status.getAu();
if (au != null) {
AuState aus = AuUtil.getAuState(au);
if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) {
statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS);
}
}
res.add(statusSi);
String sources = StringUtil.separatedString(status.getSources());
res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources));
String startUrls = StringUtil.separatedString(status.getStartUrls());
res.add(
new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls));
return res;
}
private void addIfNonZero(List res, String head, int val) {
if (val != 0) {
res.add(new StatusTable.SummaryInfo(head, ColumnDescriptor.TYPE_INT, new Long(val)));
}
}
}
private ARCWriter makeARCWriter() {
return new ARCWriter(
serialNo, ListUtil.list(dir), prefix, compress, maxSize >= 0 ? maxSize : Long.MAX_VALUE);
}