public void testHeadNotPopular() throws Exception {
VersionCounts versionCounts = VersionCounts.make();
VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
byte[] hash1 = addVersion(vb1, "content 1 for foo1");
byte[] hash2 = addVersion(vb1, "content 2 for foo1");
VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
addVersion(vb2, "content 1 for foo1");
addVersion(vb2, "content 2 for foo1");
VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
addVersion(vb3, "content 3 for foo1");
addVersion(vb3, "content 2 for foo1");
versionCounts.vote(vb1, participant1);
versionCounts.vote(vb2, participant2);
versionCounts.vote(vb3, participant3);
Map<ParticipantUserData, HashResult> repairCandidates;
repairCandidates = versionCounts.getRepairCandidates(0);
assertSameElements(
SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());
repairCandidates = versionCounts.getRepairCandidates(1);
assertSameElements(
SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());
repairCandidates = versionCounts.getRepairCandidates(2);
assertSameElements(SetUtil.set(participant1, participant2), repairCandidates.keySet());
repairCandidates = versionCounts.getRepairCandidates(3);
assertEmpty(repairCandidates.keySet());
}
public void testRunStepsWithOverrunDisallowed() {
StepTask t1 = task(100, 300, 100, null, new MyMockStepper(15, -10));
// t1.setOverrunAllowed(true);
StepTask t2 = task(150, 250, 100, null, new MyMockStepper(10, -10));
Schedule s = sched(ListUtil.list(t1, t2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
TimeBase.setSimulated(101);
Interrupter intr = null;
try {
intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
while (tr.findTaskToRun()) {
tr.runSteps(new MutableBoolean(true), null);
}
intr.cancel();
} catch (Exception e) {
log.error("runSteps threw:", e);
} finally {
if (intr.did()) {
fail("runSteps looped");
}
}
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(removedTasks));
assertTrue(t1.e.toString(), t1.e instanceof SchedService.Overrun);
}
public void testDefaults() throws Exception {
Properties p = initProps();
KeyStore ks = KeyStoreUtil.createKeyStore(p);
List aliases = ListUtil.fromIterator(new EnumerationIterator(ks.aliases()));
assertIsomorphic(SetUtil.set("mykey", "mycert"), SetUtil.theSet(aliases));
assertNotNull(ks.getCertificate("mycert"));
assertNull(ks.getCertificate("foocert"));
assertEquals("JCEKS", ks.getType());
}
public void testMultipleIdenticalVersions() throws Exception {
VersionCounts versionCounts = VersionCounts.make();
VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
byte[] hash1 = addVersion(vb1, "content 1 for foo1");
byte[] hash2 = addVersion(vb1, "content 2 for foo1");
VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
addVersion(vb2, "content 1 for foo1");
addVersion(vb2, "content 1 for foo1");
addVersion(vb2, "content 1 for foo1");
addVersion(vb2, "content 1 for foo1");
addVersion(vb2, "content 2 for foo1");
VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
addVersion(vb3, "content 1 for foo1");
addVersion(vb3, "content 2 for foo1");
addVersion(vb3, "content 2 for foo1");
addVersion(vb3, "content 2 for foo1");
addVersion(vb3, "content 2 for foo1");
versionCounts.vote(vb1, participant1);
versionCounts.vote(vb2, participant2);
versionCounts.vote(vb3, participant3);
Map<ParticipantUserData, HashResult> repairCandidates;
repairCandidates = versionCounts.getRepairCandidates(2);
assertSameElements(
SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());
// With only three candidates, no version should reach a threshold
// of 4, unless counting multiples is wrong.
repairCandidates = versionCounts.getRepairCandidates(4);
assertEmpty(repairCandidates.keySet());
}
// ensure that scanning continues after a nested parser throws an error
// XXX Need to cause an IOException while reading CSS from the stream
public void xxxtestDoCrawlStyleError() throws IOException {
String url1 = "http://example.com/blah1.html";
String url2 = "http://example.com/blah2.html";
String url3 = "http://example.com/blah3.html";
String source =
"<html><head>"
+ "<style type=\"text/css\">\n"
+ "<!--\n"
+ "@import url(\'"
+ url1
+ "\');\n"
+ // ensure css parser got invoked
"foo {bgcolor: #FFFF};"
+ // and that this causes an error
"@import url(\'"
+ url2
+ "\');\n"
+ // so that this one isn't found
"-->\n"
+ " </style>\n"
+ "<a href="
+ url3
+ "></a>"
+ // and this one is
"</head></html>";
assertEquals(SetUtil.set(url1, url3), parseSingleSource(source));
}
// remove overrunnable task-ending chunk, before deadline,
public void testRemoveChunkTaskEndOver() {
final List finished = new ArrayList();
StepTask t1 =
task(
100,
200,
100,
new TaskCallback() {
public void taskEvent(SchedulableTask task, Schedule.EventType event) {
if (log.isDebug2()) {
log.debug2("testRemoveChunkTaskEndOver callback");
}
if (event == Schedule.EventType.FINISH) {
finished.add(task);
}
}
});
t1.setOverrunAllowed(true);
Schedule s = sched(ListUtil.list(t1));
fact.setResult(s);
assertTrue(tr.addToSchedule(t1));
Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
chunk.setTaskEnd();
tr.removeChunk(chunk);
assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
assertEmpty(finished);
assertIsomorphic(ListUtil.list(t1), tr.getAcceptedTasks());
assertIsomorphic(SetUtil.set(t1), tr.getOverrunTasks());
}
public void testKeepsSingleQuoteInUrl() throws IOException {
String url = "http://www.example.com/link'with'quotes.html";
String source =
"<html><head><title>Test</title></head><body>" + "<a href=\"" + url + "\">Link</a>";
assertEquals(SetUtil.set(url), parseSingleSource(source));
}
public void testRelativeLinksWithLeadingSlash() throws IOException {
String url1 = "http://www.example.com/blah/branch1/index.html";
String url2 = "http://www.example.com/blah/branch2/index.html";
String url3 = "http://www.example.com/journals/american_imago/toc/aim60.1.html";
String url4 = "http://www.example.com/css/foo.css";
String url5 = "http://www.example.com/javascript/bar.js";
String source =
"<html><head><title>Test</title></head><body>"
+ "<a href= branch1/index.html>link1</a>"
+ "Filler, with <b>bold</b> tags and<i>others</i>"
+ "<a href=\" branch2/index.html\">link2</a>"
+ "<a href =\" /journals/american_imago/toc/aim60.1.html\">"
+ "<link rel=\"stylesheet\" href=\"/css/foo.css\" >"
+ "<script type=\"text/javascript\" src=\"/javascript/bar.js\"></script>"
+ "Number 1, Spring 2003</a>";
MockCachedUrl mcu = new MockCachedUrl("http://www.example.com/blah/");
mcu.setContent(source);
extractor.extractUrls(
mau, new StringInputStream(source), ENC, "http://www.example.com/blah/", cb);
Set expected = SetUtil.set(url1, url2, url3, url4, url5);
assertEquals(expected, cb.getFoundUrls());
}
public void testStore() throws Exception {
File dir = getTempDir();
File file = new File(dir, "test.ks");
Properties p = initProps();
p.put(KeyStoreUtil.PROP_KEYSTORE_FILE, file.toString());
assertFalse(file.exists());
KeyStore ks = KeyStoreUtil.createKeyStore(p);
assertTrue(file.exists());
KeyStore ks2 = loadKeyStore(ks.getType(), file, PASSWD);
List aliases = ListUtil.fromIterator(new EnumerationIterator(ks2.aliases()));
assertIsomorphic(SetUtil.set("mykey", "mycert"), SetUtil.theSet(aliases));
assertNotNull(ks2.getCertificate("mycert"));
assertNull(ks2.getCertificate("foocert"));
assertEquals("JCEKS", ks2.getType());
}
/** Test that a single links is found. */
public void testOneUrl() throws Exception {
try {
Set urls = SetUtil.set("http://www.foo.com/blah.jpg");
assertEquals(urls, extractUrls(constructValidRDF(urls)));
} catch (Throwable ex) {
fail("", ex);
}
}
public void testIgnoresCRInUrl() throws IOException {
String url = "http://www.example.com/linkwithspace.html";
String source =
"<html><head><title>Test</title></head><body>"
+ "<a href=\"http://www.example.com/link\rwith\rspace.html\">Link</a>";
assertEquals(SetUtil.set(url), parseSingleSource(source));
}
public void testResolvesHtmlEntities() throws IOException {
String url1 =
"http://www.example.com/bioone/?" + "request=get-toc&issn=0044-7447&volume=32&issue=1";
String source =
"<html><head><title>Test</title></head><body>"
+ "<a href=http://www.example.com/bioone/?"
+ "request=get-toc&issn=0044-7447&volume=32&issue=1>link1</a>";
assertEquals(SetUtil.set(url1), parseSingleSource(source));
// ensure character entities processed before rel url resolution
source =
"<html><head><title>Test</title></head><body>"
+ "<base href=http://www.example.com/foo/bar>"
+ "<a href=../xxx>link1</a>";
assertEquals(SetUtil.set("http://www.example.com/xxx"), parseSingleSource(source));
}
public void testIgnoresNewLineInField() throws IOException {
String url = "http://www.example.com/link.html";
String source =
"<html><head><title>Test</title></head><body>"
+ "<img\nsrc=\"http://www.example.com/link.html\">Link</a>";
assertEquals(SetUtil.set(url), parseSingleSource(source));
}
public void testParsesFileWithQuotedUrls() throws IOException {
String url = "http://www.example.com/link3.html";
String source =
"<html><head><title>Test</title></head><body>"
+ "<a href=\"http://www.example.com/link3.html\">link3</a>";
assertEquals(SetUtil.set(url), parseSingleSource(source));
}
protected void assertEqualTables(Object[][] a1, List lines) {
assertEquals("numrows", a1.length, lines.size() - NUM_HEADER_LINES);
for (int irow = 0; irow <= a1.length - 1; irow++) {
Object expRow[] = a1[irow];
List row = StringUtil.breakAt((String) lines.get(irow + NUM_HEADER_LINES), ',');
assertEquals("numcols", expRow.length, row.size());
assertEquals(("row " + irow), SetUtil.fromArray(expRow), new HashSet(row));
}
}
public void testHttpEquiv() throws IOException {
String url1 = "http://example.com/blah.html";
String source =
"<html><head>"
+ "<meta http-equiv=\"refresh\" "
+ "content=\"0; url=http://example.com/blah.html\">"
+ "</head></html>";
assertEquals(SetUtil.set(url1), parseSingleSource(source));
source =
"<html><head>"
+ "<meta http-equiv=\"refresh\" "
+ "content=\"0;url=http://example.com/blah.html\">"
+ "</head></html>";
assertEquals(SetUtil.set(url1), parseSingleSource(source));
}
// ensure addToSchedule updates structures if (Mock)Scheduler returns true
public void testAddToScheduleOk() {
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched = sched(ListUtil.list(t1, t2));
fact.setResult(sched);
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
fact.setResult(sched);
assertTrue(tr.addToSchedule(t2));
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks));
assertEquals(sched, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(2, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
}
private void singleTagParse(
String url, String startTag, String endTag, ArchivalUnit au, boolean shouldParse)
throws IOException {
MockCachedUrl mcu = new MockCachedUrl("http://www.example.com");
String content = makeContent(url, startTag, endTag);
mcu.setContent(content);
MyLinkExtractorCallback cb = new MyLinkExtractorCallback();
extractor.extractUrls(mau, new StringInputStream(content), ENC, "http://www.example.com", cb);
if (shouldParse) {
Set expected = SetUtil.set(url);
assertEquals("Misparsed: " + content, expected, cb.getFoundUrls());
} else {
Set expected = SetUtil.set();
assertEquals("Misparsed: " + content, expected, cb.getFoundUrls());
}
}
public void testFindTaskToRunRemovesExpiredChunks() {
assertFalse(tr.findTaskToRun());
StepTask t1 = task(100, 200, 100);
StepTask t2 = task(100, 300, 50);
StepTask texp1 = task(0, 0, 50);
StepTask texp2 = task(0, 0, 50);
Schedule s = sched(ListUtil.list(texp1, texp2, t1, t2));
fact.setResults(s, s);
assertTrue(tr.addToSchedule(t1));
assertTrue(tr.addToSchedule(t2));
assertFalse(tr.findTaskToRun());
assertEquals(2, removedChunks.size());
assertEquals(
SetUtil.set(texp1, texp2),
SetUtil.set(
((Schedule.Chunk) removedChunks.get(0)).getTask(),
((Schedule.Chunk) removedChunks.get(1)).getTask()));
}
public void testSkipsMalformedComments() throws IOException {
String url = "http://www.example.com/link3.html";
String source =
"<html><head><title>Test</title></head><body>"
+ "<!--<a href=http://www.example.com/link1.html>link1</a>"
+ "Filler, with <b>bold</b> tags and<i>others</i>"
+ "<a href=http://www.example.com/link2.html>link2</a>--!>"
+ "<a href=http://www.example.com/link3.html>link3</a>";
assertEquals(SetUtil.set(url), parseSingleSource(source));
}
public void testHeadNotAllowed() throws Exception {
VersionCounts versionCounts = VersionCounts.make();
VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
byte[] hash1 = addVersion(vb1, "content 1 for foo1");
byte[] hash2 = addVersion(vb1, "content 2 for foo1");
versionCounts.vote(vb1, participant1);
Map<ParticipantUserData, HashResult> repairCandidates;
repairCandidates = versionCounts.getRepairCandidates(0);
assertSameElements(SetUtil.set(participant1), repairCandidates.keySet());
// Same, but with an excluded version that doesn't matter.
repairCandidates = versionCounts.getRepairCandidates(0, SetUtil.set(HashResult.make(hash2)));
assertSameElements(SetUtil.set(participant1), repairCandidates.keySet());
// Same, but with an excluded version that does matter
repairCandidates = versionCounts.getRepairCandidates(0, SetUtil.set(HashResult.make(hash1)));
assertEmpty(repairCandidates);
}
// test resched with overrun task doesn't lose task.
public void testRunStepsWithOverrunAllowedPlusResched() {
StepTask t1 = task(100, 500, 30, null, new MyMockStepper(15, -10));
t1.setOverrunAllowed(true);
StepTask t2 = task(150, 250, 100, null, new MyMockStepper(10, -10));
newTr(
new MyMockTaskRunner(
new TaskRunner.SchedulerFactory() {
public Scheduler createScheduler() {
return new SortScheduler();
}
}));
assertTrue(tr.addToSchedule(t1));
assertEmpty(tr.getOverrunTasks());
TimeBase.setSimulated(101);
assertTrue(tr.findTaskToRun());
t1.timeUsed = 1000;
assertTrue(t1.hasOverrun());
assertEmpty(tr.getOverrunTasks());
assertTrue(tr.addToSchedule(t2));
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getOverrunTasks()));
}
public void testDontParseJSByDefault() throws IOException {
String url3 = "http://www.example.com/link1.html";
String source =
"<html><head><title>Test</title></head><body>"
+ "<a href = javascript:newWindow('http://www.example.com/link3.html')</a>"
+ "<a href = javascript:popup('http://www.example.com/link2.html')</a>"
+ "<img src = javascript:popup('"
+ url3
+ "') </img>";
assertEquals(SetUtil.set(), parseSingleSource(source));
}
// one task to drop, two failed schedule tries
public void testAddToScheduleFailNoCleanup() {
ConfigurationUtil.addFromArgs(
TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
StepTask t1 = task(100, 200, 50);
StepTask t2 = task(100, 200, 100);
Schedule sched = sched(ListUtil.list(t1));
fact.setResult(sched);
assertTrue(tr.addToSchedule(t1));
assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(0, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
assertFalse(tr.addToSchedule(t2));
assertEquals(
ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
fact.createArgs);
assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
assertEquals(sched, tr.getCurrentSchedule());
assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
assertForegroundStat(1, TaskRunner.STAT_REFUSED);
assertForegroundStat(1, TaskRunner.STAT_WAITING);
assertForegroundStat(0, TaskRunner.STAT_DROPPED);
}
private void doScriptSkipTest(String openScript, String closeScript, String failMsg)
throws IOException {
String url = "http://www.example.com/link3.html";
String src =
"<html><head><title>Test</title></head><body>"
+ openScript
+ "<a href=http://www.example.com/link1.html>link1</a>"
+ "Filler, with <b>bold</b> tags and<i>others</i>"
+ "<a href=http://www.example.com/link2.html>link2</a>"
+ closeScript
+ "<a href=http://www.example.com/link3.html>link3</a>";
assertEquals(failMsg, SetUtil.set(url), parseSingleSource(src));
}
public void testSortedRepairCandidates() throws Exception {
VersionCounts versionCounts = VersionCounts.make();
VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
addVersion(vb1, "content 1 for foo1");
VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
addVersion(vb2, "content 2 for foo1");
VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
addVersion(vb3, "content 3 for foo1");
addVersion(vb3, "content 2 for foo1");
versionCounts.vote(vb1, participant1);
versionCounts.vote(vb2, participant2);
versionCounts.vote(vb3, participant3);
Map<Integer, Collection<ParticipantUserData>> repairCandidates;
repairCandidates = versionCounts.getSortedRepairCandidatesMap(2);
assertEquals(SetUtil.set(2), repairCandidates.keySet());
assertSameElements(SetUtil.set(participant2), repairCandidates.get(2));
assertEquals(ListUtil.list(participant2), versionCounts.getSortedRepairCandidates(2));
repairCandidates = versionCounts.getSortedRepairCandidatesMap(1);
assertIsomorphic(ListUtil.list(2, 1), repairCandidates.keySet());
assertSameElements(SetUtil.set(participant2), repairCandidates.get(2));
assertSameElements(SetUtil.set(participant1, participant3), repairCandidates.get(1));
List<ParticipantUserData> lst = versionCounts.getSortedRepairCandidates(1);
assertTrue(
"" + lst,
(lst.equals(ListUtil.list(participant2, participant1, participant3))
|| lst.equals(ListUtil.list(participant2, participant3, participant1))));
assertEmpty(versionCounts.getSortedRepairCandidatesMap(4));
assertEmpty(versionCounts.getSortedRepairCandidates(4));
}
public void testDoCrawlImageWithSrcInAltTagAfterSrcProper() throws IOException {
String url = "http://www.example.com/link3.html";
String source =
"<html><head><title>Test</title></head><body>" + "<img src=" + url + " alt=src>link3</a>";
MockCachedUrl mcu = new MockCachedUrl(startUrl);
mcu.setContent(source);
extractor.extractUrls(mau, new StringInputStream(source), ENC, startUrl, cb);
Set expected = SetUtil.set(url);
assertEquals(expected, cb.getFoundUrls());
}
public void testRelativeLinksLocationTagsAndMultipleKeys() throws IOException {
String url1 = "http://www.example.com/link1.html";
String url2 = "http://www.example.com/link2.html#ref";
String url3 = "http://www.example.com/dir/link3.html";
String source =
"<html><head><title>Test</title></head><body>"
+ "<a href=link1.html>link1</a>"
+ "Filler, with <b>bold</b> tags and<i>others</i>"
+ "<a blah1=blah href=link2.html#ref blah2=blah>link2</a>"
+ "<a href=dir/link3.html>link3</a>";
assertEquals(SetUtil.set(url1, url2, url3), parseSingleSource(source));
}
public void testIgnoresEmptyHrefInBaseTag() throws IOException {
String url1 = "http://www.example.com/link1.html";
String url2 = "http://www.example.com/link2.html";
String url3 = "http://www.example.com/link3.html";
String source =
"<html><head><title>Test</title></head><body>"
+ "<a href=link1.html>link1</a>"
+ "Filler, with <b>bold</b> tags and<i>others</i>"
+ "<base href=\"\" blah=blah>"
+ "<a href=link2.html>link2</a>"
+ "<a href=link3.html>link3</a>";
assertEquals(SetUtil.set(url1, url2, url3), parseSingleSource(source));
}
/**
* Crea las combinaciones de k elementos de un conjunto de tamaño n
*
* @param prefix
* @param elements
* @param resultList
* @param r
* @param notAllowedListSuperSets : Cualquier superset de ese conjunto se bloquea
* @param notAllowedCollectionOfSets : Cualquier set generado que este dentro de esta lista se
* bloquea
* @return
*/
public static <E> List<List<E>> doCombinations(
List<E> prefix,
List<E> elements,
List<List<E>> resultList,
int r,
List<List<E>> notAllowedListSuperSets,
List<List<E>> notAllowedCollectionOfSets) {
if (r == 0) {
if ((notAllowedCollectionOfSets == null
|| !SetUtil.<E>verifySetIsSubSetOfCollectionSets(prefix, notAllowedCollectionOfSets))) {
List<E> result = new ArrayList<E>();
result.addAll(prefix);
resultList.add(result);
}
return resultList;
}
for (int i = 0; i < elements.size(); i++) {
List<E> prefix2 = new ArrayList<E>();
prefix2.addAll(prefix);
prefix2.add(elements.get(i));
if ((notAllowedListSuperSets == null
|| !SetUtil.<E>verifySetIsSuperSetOfCollectionSets(prefix2, notAllowedListSuperSets))) {
doCombinations(
prefix2,
elements.subList(i + 1, elements.size()),
resultList,
r - 1,
notAllowedListSuperSets,
notAllowedCollectionOfSets);
}
}
return resultList;
}
