// remove overrunnable task-ending chunk, before deadline, public void testRemoveChunkTaskEndOver() { final List finished = new ArrayList(); StepTask t1 = task( 100, 200, 100, new TaskCallback() { public void taskEvent(SchedulableTask task, Schedule.EventType event) { if (log.isDebug2()) { log.debug2("testRemoveChunkTaskEndOver callback"); } if (event == Schedule.EventType.FINISH) { finished.add(task); } } }); t1.setOverrunAllowed(true); Schedule s = sched(ListUtil.list(t1)); fact.setResult(s); assertTrue(tr.addToSchedule(t1)); Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0); assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk)); chunk.setTaskEnd(); tr.removeChunk(chunk); assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk)); assertEmpty(finished); assertIsomorphic(ListUtil.list(t1), tr.getAcceptedTasks()); assertIsomorphic(SetUtil.set(t1), tr.getOverrunTasks()); }
public void testGetRepositoryList() throws Exception { assertEmpty(mgr.getRepositoryList()); String tempDirPath = setUpDiskSpace(); assertEquals(ListUtil.list("local:" + tempDirPath), mgr.getRepositoryList()); String tempdir2 = getTempDir().getAbsolutePath() + File.separator; ConfigurationUtil.setFromArgs( "org.lockss.platform.diskSpacePaths", tempdir2 + ";" + tempDirPath); assertEquals( ListUtil.list("local:" + tempdir2, "local:" + tempDirPath), mgr.getRepositoryList()); }
public void testRemoveChunk() { StepTask t1 = task(100, 200, 100); Schedule s = sched(ListUtil.list(t1)); fact.setResult(s); assertTrue(tr.addToSchedule(t1)); assertIsomorphic(ListUtil.list(t1), tr.getAcceptedTasks()); Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0); assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk)); tr.removeChunk(chunk); assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk)); }
public void testStoreDamagedNodeSet() throws Exception { DamagedNodeSet damNodes = new DamagedNodeSet(mau, repository); damNodes.nodesWithDamage.add("test1"); damNodes.nodesWithDamage.add("test2"); damNodes.cusToRepair.put("cus1", ListUtil.list("cus1-1", "cus1-2")); damNodes.cusToRepair.put("cus2", ListUtil.list("cus2-1")); assertTrue(damNodes.containsWithDamage("test1")); assertTrue(damNodes.containsWithDamage("test2")); assertFalse(damNodes.containsWithDamage("test3")); repository.storeDamagedNodeSet(damNodes); String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau); filePath += HistoryRepositoryImpl.DAMAGED_NODES_FILE_NAME; File xmlFile = new File(filePath); assertTrue(xmlFile.exists()); damNodes = null; damNodes = repository.loadDamagedNodeSet(); // check damage assertTrue(damNodes.containsWithDamage("test1")); assertTrue(damNodes.containsWithDamage("test2")); assertFalse(damNodes.containsWithDamage("test3")); MockCachedUrlSet mcus1 = new MockCachedUrlSet("cus1"); MockCachedUrlSet mcus2 = new MockCachedUrlSet("cus2"); // check repairs assertTrue(damNodes.containsToRepair(mcus1, "cus1-1")); assertTrue(damNodes.containsToRepair(mcus1, "cus1-2")); assertFalse(damNodes.containsToRepair(mcus1, "cus2-1")); assertTrue(damNodes.containsToRepair(mcus2, "cus2-1")); assertEquals(mau.getAuId(), damNodes.theAu.getAuId()); // check remove damNodes.removeFromRepair(mcus1, "cus1-1"); assertFalse(damNodes.containsToRepair(mcus1, "cus1-1")); assertTrue(damNodes.containsToRepair(mcus1, "cus1-2")); damNodes.removeFromRepair(mcus1, "cus1-2"); assertFalse(damNodes.containsToRepair(mcus1, "cus1-2")); assertNull(damNodes.cusToRepair.get(mcus1)); // check remove from damaged nodes damNodes.removeFromDamage("test1"); damNodes.removeFromDamage("test2"); repository.storeDamagedNodeSet(damNodes); damNodes = repository.loadDamagedNodeSet(); assertNotNull(damNodes); assertFalse(damNodes.containsWithDamage("test1")); assertFalse(damNodes.containsWithDamage("test2")); }
public void getGroup() throws Exception { m_testMsg.setGroups(null); assertNull(m_testMsg.getGroups()); assertNull(m_testMsg.getGroupList()); m_testMsg.setGroups("foo"); assertEquals("foo", m_testMsg.getGroups()); assertEquals(ListUtil.list("foo"), m_testMsg.getGroupList()); m_testMsg.setGroups("foo;bar"); assertEquals("foo;bar", m_testMsg.getGroups()); assertEquals(ListUtil.list("foo", "bar"), m_testMsg.getGroupList()); m_testMsg.setGroups("foo;bar;baz"); assertEquals("foo;bar;baz", m_testMsg.getGroups()); assertEquals(ListUtil.list("foo", "bar", "baz"), m_testMsg.getGroupList()); }
public void testIsTaskSchedulable() { fact.setResult(null); StepTask t1 = task(100, 200, 50); assertFalse(tr.isTaskSchedulable(t1)); fact.setResult(sched(ListUtil.list(t1))); assertTrue(tr.isTaskSchedulable(t1)); }
public void testRunStepsWithOverrunDisallowed() { StepTask t1 = task(100, 300, 100, null, new MyMockStepper(15, -10)); // t1.setOverrunAllowed(true); StepTask t2 = task(150, 250, 100, null, new MyMockStepper(10, -10)); Schedule s = sched(ListUtil.list(t1, t2)); fact.setResults(s, s); assertTrue(tr.addToSchedule(t1)); assertTrue(tr.addToSchedule(t2)); TimeBase.setSimulated(101); Interrupter intr = null; try { intr = interruptMeIn(TIMEOUT_SHOULDNT, true); while (tr.findTaskToRun()) { tr.runSteps(new MutableBoolean(true), null); } intr.cancel(); } catch (Exception e) { log.error("runSteps threw:", e); } finally { if (intr.did()) { fail("runSteps looped"); } } assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(removedTasks)); assertTrue(t1.e.toString(), t1.e instanceof SchedService.Overrun); }
public void testStepperThrows() { final List finished = new ArrayList(); TaskCallback cb = new TaskCallback() { public void taskEvent(SchedulableTask task, Schedule.EventType event) { if (event == Schedule.EventType.FINISH) { finished.add(task); } } }; MyMockStepper stepper = new MyMockStepper(10, -10); stepper.setWhenToThrow(5); StepTask t1 = task(100, 200, 100, cb, stepper); Schedule s = sched(ListUtil.list(t1)); fact.setResult(s); assertTrue(tr.addToSchedule(t1)); TimeBase.setSimulated(101); assertTrue(tr.findTaskToRun()); Interrupter intr = null; try { intr = interruptMeIn(TIMEOUT_SHOULDNT, true); tr.runSteps(new MutableBoolean(true), null); intr.cancel(); } catch (Exception e) { log.error("runSteps threw:", e); } finally { if (intr.did()) { fail("runSteps looped"); } } assertSame(t1, finished.get(0)); assertTrue(t1.e instanceof ExpectedRuntimeException); assertEquals(5, stepper.nSteps); }
// remove task-ending chunk, past task deadline, s.b. Timeout error. public void testRemoveChunkTaskEndTimeout() { final List finished = new ArrayList(); StepTask t1 = task( 100, 200, 100, new TaskCallback() { public void taskEvent(SchedulableTask task, Schedule.EventType event) { if (log.isDebug2()) { log.debug2("testRemoveChunkTaskEndTimeout callback"); } if (event == Schedule.EventType.FINISH) { finished.add(task); } } }); Schedule s = sched(ListUtil.list(t1)); fact.setResult(s); assertTrue(tr.addToSchedule(t1)); Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0); assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk)); chunk.setTaskEnd(); TimeBase.setSimulated(201); tr.removeChunk(chunk); assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk)); assertSame(t1, finished.get(0)); assertNotNull(t1.e); assertTrue(t1.e.toString(), t1.e instanceof SchedService.Timeout); assertEmpty(tr.getAcceptedTasks()); }
public void testSizeCalc() throws Exception { SimpleBinarySemaphore sem = new SimpleBinarySemaphore(); mgr.setSem(sem); RepositoryNode node1 = new RepositoryNodeImpl("url1", "testDir", null); RepositoryNode node2 = new RepositoryNodeImpl("url2", "testDir", null); RepositoryNode node3 = new RepositoryNodeImpl("url3", "testDir", null); mgr.queueSizeCalc(node1); assertTrue(sem.take(TIMEOUT_SHOULDNT)); assertEquals(ListUtil.list(node1), mgr.getNodes()); mgr.queueSizeCalc(node2); mgr.queueSizeCalc(node3); assertTrue(sem.take(TIMEOUT_SHOULDNT)); if (mgr.getNodes().size() < 3) { assertTrue(sem.take(TIMEOUT_SHOULDNT)); } assertSameElements(ListUtil.list(node1, node2, node3), mgr.getNodes()); }
// ensure addToSchedule updates structures if (Mock)Scheduler returns true public void testAddToScheduleOk() { StepTask t1 = task(100, 200, 50); StepTask t2 = task(100, 200, 100); Schedule sched = sched(ListUtil.list(t1, t2)); fact.setResult(sched); assertTrue(tr.addToSchedule(t1)); assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks); assertForegroundStat(1, TaskRunner.STAT_ACCEPTED); assertForegroundStat(0, TaskRunner.STAT_REFUSED); fact.setResult(sched); assertTrue(tr.addToSchedule(t2)); assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks)); assertEquals(sched, tr.getCurrentSchedule()); assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks())); assertForegroundStat(2, TaskRunner.STAT_ACCEPTED); assertForegroundStat(2, TaskRunner.STAT_WAITING); assertForegroundStat(0, TaskRunner.STAT_REFUSED); }
public void testCdnStems() { AuState aus = new AuState(mau, historyRepo); assertEquals(Collections.EMPTY_LIST, aus.getCdnStems()); aus.addCdnStem("http://fff.uselesstld"); assertClass(ArrayList.class, aus.getCdnStems()); assertEquals(ListUtil.list("http://fff.uselesstld"), aus.getCdnStems()); aus.addCdnStem("ccc"); assertEquals(ListUtil.list("http://fff.uselesstld", "ccc"), aus.getCdnStems()); aus.setCdnStems(new LinkedList(ListUtil.list("a", "b"))); assertClass(ArrayList.class, aus.getCdnStems()); assertEquals(ListUtil.list("a", "b"), aus.getCdnStems()); aus.setCdnStems(null); assertEmpty(aus.getCdnStems()); aus.addCdnStem("https://a.b/"); aus.addCdnStem("https://b.a/"); assertEquals(ListUtil.list("https://a.b/", "https://b.a/"), aus.getCdnStems()); }
public void testParsesOptionPositive() throws IOException { TypedEntryMap pMap = new TypedEntryMap(); pMap.setMapElement("html-parser-select-attrs", ListUtil.list("value")); mau.setPropertyMap(pMap); singleTagShouldParse( "http://www.example.com/web_link.jpg", "<option value=", "</option>", mau); singleTagShouldParse( "http://www.example.com/web_link.jpg", "<option a=b value=", "</option>", mau); }
public void testCreateSharedPLNKeyStores() throws Exception { List<String> hosts = ListUtil.list("host1", "host2.foo.bar", "host3"); List<String> hosts2 = ListUtil.list("host3", "host4"); File dir = getTempDir(); File pub = new File(dir, "pub.ks"); KeyStoreUtil.createSharedPLNKeyStores( dir, hosts, pub, "pubpass", MiscTestUtil.getSecureRandom()); assertPubKs(pub, "pubpass", hosts); for (String host : hosts) { assertPrivateKs( new File(dir, host + ".jceks"), StringUtil.fromFile(new File(dir, host + ".pass")), host); } KeyStore pubks1 = loadKeyStore("jceks", new File(dir, "pub.ks"), "pubpass"); Certificate host1cert1 = pubks1.getCertificate("host1.crt"); Certificate host3cert1 = pubks1.getCertificate("host3.crt"); String host1priv1 = StringUtil.fromFile(new File(dir, "host1.jceks")); String host3priv1 = StringUtil.fromFile(new File(dir, "host3.jceks")); // Now add host4 and generate a new key for host3 KeyStoreUtil.createSharedPLNKeyStores( dir, hosts2, pub, "pubpass", MiscTestUtil.getSecureRandom()); List<String> both = ListUtils.sum(hosts, hosts2); assertPubKs(pub, "pubpass", both); for (String host : both) { assertPrivateKs( new File(dir, host + ".jceks"), StringUtil.fromFile(new File(dir, host + ".pass")), host); } KeyStore pubks2 = loadKeyStore("jceks", new File(dir, "pub.ks"), "pubpass"); // host1 should have the same cert, host3 not Certificate host1cert2 = pubks2.getCertificate("host1.crt"); Certificate host3cert2 = pubks2.getCertificate("host3.crt"); assertEquals(host1cert1, host1cert2); assertNotEquals(host3cert1, host3cert2); // host1's private key file should be the same, host3's not String host1priv2 = StringUtil.fromFile(new File(dir, "host1.jceks")); String host3priv2 = StringUtil.fromFile(new File(dir, "host3.jceks")); assertEquals(host1priv1, host1priv2); assertNotEquals(host3priv1, host3priv2); }
// test special (non-string) key in row doesn't throw public void testTextNonStringKey() throws Exception { MockStatusAccessor statusAccessor = new MockStatusAccessor(); List cols = ListUtil.list("foo", StatusTable.ROW_SEPARATOR); statusAccessor.setRows(MockStatusAccessor.makeRowsFrom(cols, rowArray1), null); statusAccessor.setColumnDescriptors( MockStatusAccessor.makeColumnDescriptorsFrom(colArray1), null); statSvc.registerStatusAccessor("testtbl", statusAccessor); WebResponse resp = getTable("testtbl", true); log.debug(resp.getText()); }
// one task to drop, two failed schedule tries public void testAddToScheduleFailNoCleanup() { ConfigurationUtil.addFromArgs( TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0"); StepTask t1 = task(100, 200, 50); StepTask t2 = task(100, 200, 100); Schedule sched = sched(ListUtil.list(t1)); fact.setResult(sched); assertTrue(tr.addToSchedule(t1)); assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks); assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks())); assertForegroundStat(1, TaskRunner.STAT_ACCEPTED); assertForegroundStat(0, TaskRunner.STAT_REFUSED); assertForegroundStat(1, TaskRunner.STAT_WAITING); assertForegroundStat(0, TaskRunner.STAT_DROPPED); assertFalse(tr.addToSchedule(t2)); assertEquals( ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)), fact.createArgs); assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks)); assertEquals(sched, tr.getCurrentSchedule()); assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks())); assertForegroundStat(1, TaskRunner.STAT_ACCEPTED); assertForegroundStat(1, TaskRunner.STAT_REFUSED); assertForegroundStat(1, TaskRunner.STAT_WAITING); assertForegroundStat(0, TaskRunner.STAT_DROPPED); }
// one task to drop, succeeds after dropping it public void testAddToScheduleOkAfterDrops() { log.debug("testAddToScheduleOkAfterDrops()"); ConfigurationUtil.addFromArgs( TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0"); StepTask t1 = task(100, 200, 50); StepTask t2 = task(100, 200, 100); Schedule sched1 = sched(ListUtil.list(t1)); Schedule sched2 = sched(ListUtil.list(t2)); fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2)); TimeBase.step(101); assertTrue(tr.addToSchedule(t1)); assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks); assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks())); assertForegroundStat(1, TaskRunner.STAT_ACCEPTED); assertForegroundStat(0, TaskRunner.STAT_REFUSED); assertForegroundStat(1, TaskRunner.STAT_WAITING); assertForegroundStat(0, TaskRunner.STAT_DROPPED); assertTrue(tr.addToSchedule(t2)); assertEquals( ListUtil.list( ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1), Collections.EMPTY_SET, ListUtil.list(t1, t2)), fact.createArgs); assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks)); assertTrue(t1.isDropped()); assertEquals(sched2, tr.getCurrentSchedule()); assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks())); assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getOverrunTasks())); assertForegroundStat(2, TaskRunner.STAT_ACCEPTED); assertForegroundStat(0, TaskRunner.STAT_REFUSED); assertForegroundStat(1, TaskRunner.STAT_WAITING); assertForegroundStat(1, TaskRunner.STAT_DROPPED); t2.cancel(); assertForegroundStat(2, TaskRunner.STAT_ACCEPTED); assertForegroundStat(0, TaskRunner.STAT_REFUSED); assertForegroundStat(0, TaskRunner.STAT_WAITING); assertForegroundStat(1, TaskRunner.STAT_DROPPED); assertForegroundStat(1, TaskRunner.STAT_CANCELLED); }
// test completion & callback public void testDone() throws Exception { HashQueue q = new HashQueue(); final List cookieList = new LinkedList(); final List eList = new LinkedList(); HashService.Callback cb = new HashService.Callback() { public void hashingFinished( CachedUrlSet urlset, long timeUsed, Object cookie, CachedUrlSetHasher hasher, Exception e) { cookieList.add(cookie); eList.add(e); } }; HashQueue.Request r1, r2, r3, r4, r5; r1 = req(2000, 0, 100, cb); r2 = req(10000, 0, 200, cb); r3 = req(20000, 0, 0, cb); r4 = req(50000, 0, 1, cb); assertTrue(q.insert(r1)); assertTrue(q.insert(r2)); assertTrue(q.insert(r4)); assertEquals(0, cookieList.size()); q.removeCompleted(); assertEquals(0, cookieList.size()); // make r1 timeout r1.deadline.expire(); q.removeCompleted(); List exp = ListUtil.list(r1); assertEquals(exp, cookieList); assertEquals(exp, q.getCompletedSnapshot()); // make r2 timeout TimeBase.step(11000); // r3 is finished assertTrue(q.insert(r3)); Exception r4e = new Exception(); // make r4 error r4.e = r4e; q.removeCompleted(); // check that they all finished, and in the right order Object exp2[] = {r1, r2, r3, r4}; assertIsomorphic(exp2, cookieList); assertIsomorphic(exp2, q.getCompletedSnapshot()); // check their exceptions assertTrue(eList.get(0) instanceof HashService.Timeout); assertTrue(eList.get(1) instanceof HashService.Timeout); assertSame(null, eList.get(2)); assertSame(r4e, eList.get(3)); }
public void testSortedRepairCandidates() throws Exception { VersionCounts versionCounts = VersionCounts.make(); VoteBlock vb1 = makeVoteBlock("http://test.com/foo1"); addVersion(vb1, "content 1 for foo1"); VoteBlock vb2 = makeVoteBlock("http://test.com/foo1"); addVersion(vb2, "content 2 for foo1"); VoteBlock vb3 = makeVoteBlock("http://test.com/foo1"); addVersion(vb3, "content 3 for foo1"); addVersion(vb3, "content 2 for foo1"); versionCounts.vote(vb1, participant1); versionCounts.vote(vb2, participant2); versionCounts.vote(vb3, participant3); Map<Integer, Collection<ParticipantUserData>> repairCandidates; repairCandidates = versionCounts.getSortedRepairCandidatesMap(2); assertEquals(SetUtil.set(2), repairCandidates.keySet()); assertSameElements(SetUtil.set(participant2), repairCandidates.get(2)); assertEquals(ListUtil.list(participant2), versionCounts.getSortedRepairCandidates(2)); repairCandidates = versionCounts.getSortedRepairCandidatesMap(1); assertIsomorphic(ListUtil.list(2, 1), repairCandidates.keySet()); assertSameElements(SetUtil.set(participant2), repairCandidates.get(2)); assertSameElements(SetUtil.set(participant1, participant3), repairCandidates.get(1)); List<ParticipantUserData> lst = versionCounts.getSortedRepairCandidates(1); assertTrue( "" + lst, (lst.equals(ListUtil.list(participant2, participant1, participant3)) || lst.equals(ListUtil.list(participant2, participant3, participant1)))); assertEmpty(versionCounts.getSortedRepairCandidatesMap(4)); assertEmpty(versionCounts.getSortedRepairCandidates(4)); }
public void testStoreAuEmptyState() throws Exception { HashSet strCol = new HashSet(); strCol.add("test"); AuState origState = new AuState(mau, repository); repository.storeAuState(origState); AuState loadedState = repository.loadAuState(); assertEquals(-1, loadedState.getLastCrawlTime()); assertEquals(-1, loadedState.getLastCrawlAttempt()); assertEquals(-1, loadedState.getLastCrawlResult()); assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg()); assertEquals(-1, loadedState.getLastTopLevelPollTime()); assertEquals(-1, loadedState.getLastPollStart()); assertEquals(-1, loadedState.getLastPollResult()); assertEquals(null, loadedState.getLastPollResultMsg()); assertEquals(-1, loadedState.getLastPoPPoll()); assertEquals(-1, loadedState.getLastPoPPollResult()); assertEquals(-1, loadedState.getLastLocalHashScan()); assertEquals(0, loadedState.getNumAgreePeersLastPoR()); assertEquals(0, loadedState.getNumWillingRepairers()); assertEquals(0, loadedState.getNumCurrentSuspectVersions()); assertEmpty(loadedState.getCdnStems()); loadedState.addCdnStem("http://this.is.new/"); assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems()); loadedState.addCdnStem("http://this.is.new/"); assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems()); assertEquals(0, loadedState.getPollDuration()); assertEquals(0, loadedState.getClockssSubscriptionStatus()); assertEquals(null, loadedState.getAccessType()); assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState()); assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance)); assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata)); assertEquals(-1, loadedState.getLastMetadataIndex()); assertEquals(0, loadedState.getLastContentChange()); assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId()); }
public void testFromList1() { assertEquals(MapUtil.map(), MapUtil.fromList(ListUtil.list())); assertEquals( MapUtil.map("FOO", "bar", "One", "Two"), MapUtil.fromList(ListUtil.list("FOO", "bar", "One", "Two"))); assertEquals( MapUtil.map("foo", "bar", "one", "two"), MapUtil.fromList(ListUtil.list(ListUtil.list("foo", "bar"), ListUtil.list("one", "two")))); try { MapUtil.fromList(ListUtil.list("FOO", "bar", "One")); fail("Odd length arg list should throw"); } catch (IllegalArgumentException e) { } try { MapUtil.fromList(ListUtil.list(ListUtil.list("foo", "bar"), ListUtil.list("one"))); fail("Short sublist should throw"); } catch (IllegalArgumentException e) { } }
public void testFindOverrunTaskToRun() { assertFalse(tr.findTaskToRun()); StepTask t1 = task(100, 200, 100); Schedule s = sched(ListUtil.list(t1)); fact.setResult(s); assertTrue(tr.addToSchedule(t1)); assertFalse(tr.findTaskToRun()); assertEquals(Deadline.at(100), tr.runningDeadline); StepTask t2 = task(0, 300, 50); tr.addOverrunner(t2); assertTrue(tr.findTaskToRun()); assertEquals(t2, tr.runningTask); assertEquals(Deadline.at(100), tr.runningDeadline); assertNull(tr.runningChunk); }
public void testFindChunkTaskToRun() { assertFalse(tr.findTaskToRun()); StepTask t1 = task(100, 200, 100); StepTask t2 = task(100, 300, 50); Schedule s = sched(ListUtil.list(t1, t2)); fact.setResults(s, s); assertTrue(tr.addToSchedule(t1)); assertTrue(tr.addToSchedule(t2)); assertFalse(tr.findTaskToRun()); assertEquals(Deadline.at(100), tr.runningDeadline); TimeBase.setSimulated(101); assertTrue(tr.findTaskToRun()); assertEquals(t1, tr.runningTask); assertEquals(t1.getLatestFinish(), tr.runningDeadline); assertEquals(s.getEvents().get(0), tr.runningChunk); }
public void testFindTaskToRunRemovesExpiredChunks() { assertFalse(tr.findTaskToRun()); StepTask t1 = task(100, 200, 100); StepTask t2 = task(100, 300, 50); StepTask texp1 = task(0, 0, 50); StepTask texp2 = task(0, 0, 50); Schedule s = sched(ListUtil.list(texp1, texp2, t1, t2)); fact.setResults(s, s); assertTrue(tr.addToSchedule(t1)); assertTrue(tr.addToSchedule(t2)); assertFalse(tr.findTaskToRun()); assertEquals(2, removedChunks.size()); assertEquals( SetUtil.set(texp1, texp2), SetUtil.set( ((Schedule.Chunk) removedChunks.get(0)).getTask(), ((Schedule.Chunk) removedChunks.get(1)).getTask())); }
public void testFindRunnableChunk() { assertFalse(tr.findTaskToRun()); StepTask t1 = task(100, 200, 100); StepTask t2 = task(10, 300, 50); Schedule.Chunk c1 = new Schedule.Chunk(t1, Deadline.at(100), Deadline.at(200), 100); Schedule.Chunk c2 = new Schedule.Chunk(t2, Deadline.at(200), Deadline.at(300), 100); Schedule s = new Schedule(ListUtil.list(c1, c2)); fact.setResults(s, s); assertTrue(tr.addToSchedule(t1)); assertTrue(tr.addToSchedule(t2)); assertFalse(tr.findTaskToRun()); assertEquals(Deadline.at(100), tr.runningDeadline); TimeBase.setSimulated(11); assertTrue(tr.findTaskToRun()); assertEquals(t2, tr.runningTask); assertEquals(c2, tr.runningChunk); assertEquals(Deadline.at(100), tr.runningDeadline); assertEquals(s.getEvents().get(1), tr.runningChunk); }
// one task not ready to start yet, so not droppable public void testAddToScheduleFailNoDroppable() { log.debug("testAddToScheduleOkAfterDrops()"); ConfigurationUtil.addFromArgs( TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0"); StepTask t1 = task(100, 200, 50); StepTask t2 = task(100, 200, 100); Schedule sched1 = sched(ListUtil.list(t1)); Schedule sched2 = sched(ListUtil.list(t2)); fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2)); assertTrue(tr.addToSchedule(t1)); assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks); assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks())); assertFalse(tr.addToSchedule(t2)); assertEquals( ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)), fact.createArgs); assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks)); assertFalse(t1.isDropped()); assertEquals(sched1, tr.getCurrentSchedule()); assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks())); assertEmpty(SetUtil.theSet(tr.getOverrunTasks())); }
public void testNotifyThread() { final List rec = new ArrayList(); final SimpleBinarySemaphore sem = new SimpleBinarySemaphore(); tr.setImmediateNotify(false); TaskCallback cb = new TaskCallback() { public void taskEvent(SchedulableTask task, Schedule.EventType event) { rec.add(new BERec(Deadline.in(0), (BackgroundTask) task, event)); sem.give(); } }; BackgroundTask t1 = btask(100, 200, .1, cb); BackgroundTask t2 = btask(100, 300, .2, cb); tr.notify(t1, Schedule.EventType.START); tr.notify(t1, Schedule.EventType.FINISH); // 2nd finish event should not cause another callback tr.notify(t1, Schedule.EventType.FINISH); tr.notify(t2, Schedule.EventType.START); Interrupter intr = null; try { intr = interruptMeIn(TIMEOUT_SHOULDNT, true); while (rec.size() < 3) { sem.take(); } assertEquals( ListUtil.list( new BERec(0, t1, Schedule.EventType.START), new BERec(0, t1, Schedule.EventType.FINISH), new BERec(0, t2, Schedule.EventType.START)), rec); intr.cancel(); } finally { if (intr.did()) { fail("Notifier didn't run callbacks"); } } }
public void testFindTaskToRunRemovesExpiredOverrunners() { assertFalse(tr.findTaskToRun()); StepTask t1 = task(100, 200, 100); StepTask t2 = task(100, 300, 50); StepTask texp1 = task(0, 0, 50); StepTask texp2 = task(0, 0, 49); Schedule s = sched(ListUtil.list(t1, t2)); fact.setResults(s, s); assertTrue(tr.addToSchedule(t1)); assertTrue(tr.addToSchedule(t2)); tr.addOverrunner(texp1); tr.addOverrunner(texp2); // if this fails, it might be because the sorted list/set is treating // sort-order equivalence as object equality, which we don't want assertEquals(2, tr.getOverrunTasks().size()); assertFalse(tr.findTaskToRun()); assertEquals(0, removedChunks.size()); assertEquals(2, removedTasks.size()); assertEquals( SetUtil.set(texp1, texp2), SetUtil.set((StepTask) removedTasks.get(0), (StepTask) removedTasks.get(1))); }
public class SingleCrawlStatusAccessor implements StatusAccessor { private static final String MIME_TYPE_NAME = "mime_type_name"; private static final String MIME_TYPE_NUM_URLS = "mime_type_num_urls"; private static final String MIMETYPES_URLS_KEY = "mime-type"; private static final String CRAWL_URLS_STATUS_ACCESSOR = CrawlManagerImpl.CRAWL_URLS_STATUS_TABLE; private List colDescsMimeTypes = ListUtil.fromArray( new ColumnDescriptor[] { new ColumnDescriptor(MIME_TYPE_NAME, "Mime Type", ColumnDescriptor.TYPE_STRING), new ColumnDescriptor( MIME_TYPE_NUM_URLS, "URLs Found", ColumnDescriptor.TYPE_INT, "Number of pages of that mime type fetched during this crawl"), }); private static final List statusSortRules = ListUtil.list(new StatusTable.SortRule(MIME_TYPE_NAME, true)); private CrawlManager.StatusSource statusSource; public SingleCrawlStatusAccessor(CrawlManager.StatusSource statusSource) { this.statusSource = statusSource; } public void populateTable(StatusTable table) throws StatusService.NoSuchTableException { if (table == null) { throw new IllegalArgumentException("Called with null table"); } else if (table.getKey() == null) { throw new IllegalArgumentException("SingleCrawlStatusAccessor requires a key"); } String key = table.getKey(); CrawlerStatus status = statusSource.getStatus().getCrawlerStatus(key); if (status == null) { throw new StatusService.NoSuchTableException( "Status info from that crawl is no longer available"); } table.setDefaultSortRules(statusSortRules); table.setColumnDescriptors(colDescsMimeTypes); table.setTitle(getTableTitle(status)); table.setRows(getRows(status, key)); table.setSummaryInfo(getSummaryInfo(status)); } private String getTableTitle(CrawlerStatus status) { return "Status of crawl of " + status.getAuName(); } /** iterate over the mime-types makeRow for each */ private List getRows(CrawlerStatus status, String key) { Collection mimeTypes = status.getMimeTypes(); List rows = new ArrayList(); if (mimeTypes != null) { String mimeType; for (Iterator it = mimeTypes.iterator(); it.hasNext(); ) { mimeType = (String) it.next(); rows.add(makeRow(status, mimeType, key)); } } return rows; } private Map makeRow(CrawlerStatus status, String mimeType, String key) { Map row = new HashMap(); row.put(MIME_TYPE_NAME, mimeType); row.put( MIME_TYPE_NUM_URLS, makeRefIfColl(status.getMimeTypeCtr(mimeType), key, MIMETYPES_URLS_KEY + ":" + mimeType)); return row; } /** Return a reference object to the table, displaying the value */ private Object makeRef(long value, String tableName, String key) { return new StatusTable.Reference(new Long(value), tableName, key); } /** If the UrlCounter has a collection, return a reference to it, else just the count */ Object makeRefIfColl(CrawlerStatus.UrlCount ctr, String crawlKey, String subkey) { if (ctr.hasCollection()) { return makeRef(ctr.getCount(), CRAWL_URLS_STATUS_ACCESSOR, crawlKey + "." + subkey); } return new Long(ctr.getCount()); } public String getDisplayName() { throw new UnsupportedOperationException("No generic name for MimeTypeStatusCrawler"); } public boolean requiresKey() { return true; } public static final String FOOT_NO_SUBSTANCE_CRAWL_STATUS = "Though the crawl finished successfully, no files containing substantial content were collected."; private List getSummaryInfo(CrawlerStatus status) { List res = new ArrayList(); StatusTable.SummaryInfo statusSi = new StatusTable.SummaryInfo( "Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg()); ArchivalUnit au = status.getAu(); if (au != null) { AuState aus = AuUtil.getAuState(au); if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) { statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS); } } res.add(statusSi); String sources = StringUtil.separatedString(status.getSources()); res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources)); String startUrls = StringUtil.separatedString(status.getStartUrls()); res.add( new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls)); return res; } private void addIfNonZero(List res, String head, int val) { if (val != 0) { res.add(new StatusTable.SummaryInfo(head, ColumnDescriptor.TYPE_INT, new Long(val))); } } }
private ARCWriter makeARCWriter() { return new ARCWriter( serialNo, ListUtil.list(dir), prefix, compress, maxSize >= 0 ? maxSize : Long.MAX_VALUE); }