コード例 #1
1
  public void testHeadNotPopular() throws Exception {
    VersionCounts versionCounts = VersionCounts.make();

    VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
    byte[] hash1 = addVersion(vb1, "content 1 for foo1");
    byte[] hash2 = addVersion(vb1, "content 2 for foo1");

    VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 2 for foo1");

    VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb3, "content 3 for foo1");
    addVersion(vb3, "content 2 for foo1");

    versionCounts.vote(vb1, participant1);
    versionCounts.vote(vb2, participant2);
    versionCounts.vote(vb3, participant3);

    Map<ParticipantUserData, HashResult> repairCandidates;
    repairCandidates = versionCounts.getRepairCandidates(0);
    assertSameElements(
        SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());

    repairCandidates = versionCounts.getRepairCandidates(1);
    assertSameElements(
        SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());

    repairCandidates = versionCounts.getRepairCandidates(2);
    assertSameElements(SetUtil.set(participant1, participant2), repairCandidates.keySet());

    repairCandidates = versionCounts.getRepairCandidates(3);
    assertEmpty(repairCandidates.keySet());
  }
コード例 #2
0
  // one task to drop, two failed schedule tries
  public void testAddToScheduleFailNoCleanup() {
    ConfigurationUtil.addFromArgs(
        TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
    StepTask t1 = task(100, 200, 50);
    StepTask t2 = task(100, 200, 100);
    Schedule sched = sched(ListUtil.list(t1));
    fact.setResult(sched);
    assertTrue(tr.addToSchedule(t1));
    assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
    assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(0, TaskRunner.STAT_REFUSED);
    assertForegroundStat(1, TaskRunner.STAT_WAITING);
    assertForegroundStat(0, TaskRunner.STAT_DROPPED);

    assertFalse(tr.addToSchedule(t2));
    assertEquals(
        ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
        fact.createArgs);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
    assertEquals(sched, tr.getCurrentSchedule());
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
    assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(1, TaskRunner.STAT_REFUSED);
    assertForegroundStat(1, TaskRunner.STAT_WAITING);
    assertForegroundStat(0, TaskRunner.STAT_DROPPED);
  }
コード例 #3
0
  public void testMultipleIdenticalVersions() throws Exception {
    VersionCounts versionCounts = VersionCounts.make();

    VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
    byte[] hash1 = addVersion(vb1, "content 1 for foo1");
    byte[] hash2 = addVersion(vb1, "content 2 for foo1");

    VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 1 for foo1");
    addVersion(vb2, "content 2 for foo1");

    VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb3, "content 1 for foo1");
    addVersion(vb3, "content 2 for foo1");
    addVersion(vb3, "content 2 for foo1");
    addVersion(vb3, "content 2 for foo1");
    addVersion(vb3, "content 2 for foo1");

    versionCounts.vote(vb1, participant1);
    versionCounts.vote(vb2, participant2);
    versionCounts.vote(vb3, participant3);

    Map<ParticipantUserData, HashResult> repairCandidates;
    repairCandidates = versionCounts.getRepairCandidates(2);
    assertSameElements(
        SetUtil.set(participant1, participant2, participant3), repairCandidates.keySet());

    // With only three candidates, no version should reach a threshold
    // of 4, unless counting multiples is wrong.
    repairCandidates = versionCounts.getRepairCandidates(4);
    assertEmpty(repairCandidates.keySet());
  }
コード例 #4
0
 public void testRunStepsWithOverrunDisallowed() {
   StepTask t1 = task(100, 300, 100, null, new MyMockStepper(15, -10));
   //    t1.setOverrunAllowed(true);
   StepTask t2 = task(150, 250, 100, null, new MyMockStepper(10, -10));
   Schedule s = sched(ListUtil.list(t1, t2));
   fact.setResults(s, s);
   assertTrue(tr.addToSchedule(t1));
   assertTrue(tr.addToSchedule(t2));
   TimeBase.setSimulated(101);
   Interrupter intr = null;
   try {
     intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
     while (tr.findTaskToRun()) {
       tr.runSteps(new MutableBoolean(true), null);
     }
     intr.cancel();
   } catch (Exception e) {
     log.error("runSteps threw:", e);
   } finally {
     if (intr.did()) {
       fail("runSteps looped");
     }
   }
   assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(removedTasks));
   assertTrue(t1.e.toString(), t1.e instanceof SchedService.Overrun);
 }
コード例 #5
0
  // remove overrunnable task-ending chunk, before deadline,
  public void testRemoveChunkTaskEndOver() {
    final List finished = new ArrayList();
    StepTask t1 =
        task(
            100,
            200,
            100,
            new TaskCallback() {
              public void taskEvent(SchedulableTask task, Schedule.EventType event) {
                if (log.isDebug2()) {
                  log.debug2("testRemoveChunkTaskEndOver callback");
                }
                if (event == Schedule.EventType.FINISH) {
                  finished.add(task);
                }
              }
            });
    t1.setOverrunAllowed(true);
    Schedule s = sched(ListUtil.list(t1));
    fact.setResult(s);
    assertTrue(tr.addToSchedule(t1));

    Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
    assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
    chunk.setTaskEnd();
    tr.removeChunk(chunk);
    assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
    assertEmpty(finished);
    assertIsomorphic(ListUtil.list(t1), tr.getAcceptedTasks());
    assertIsomorphic(SetUtil.set(t1), tr.getOverrunTasks());
  }
コード例 #6
0
  public void testKeepsSingleQuoteInUrl() throws IOException {
    String url = "http://www.example.com/link'with'quotes.html";

    String source =
        "<html><head><title>Test</title></head><body>" + "<a href=\"" + url + "\">Link</a>";
    assertEquals(SetUtil.set(url), parseSingleSource(source));
  }
コード例 #7
0
  public void testRelativeLinksWithLeadingSlash() throws IOException {
    String url1 = "http://www.example.com/blah/branch1/index.html";
    String url2 = "http://www.example.com/blah/branch2/index.html";
    String url3 = "http://www.example.com/journals/american_imago/toc/aim60.1.html";
    String url4 = "http://www.example.com/css/foo.css";
    String url5 = "http://www.example.com/javascript/bar.js";
    String source =
        "<html><head><title>Test</title></head><body>"
            + "<a href= branch1/index.html>link1</a>"
            + "Filler, with <b>bold</b> tags and<i>others</i>"
            + "<a href=\" branch2/index.html\">link2</a>"
            + "<a href =\" /journals/american_imago/toc/aim60.1.html\">"
            + "<link rel=\"stylesheet\" href=\"/css/foo.css\" >"
            + "<script type=\"text/javascript\" src=\"/javascript/bar.js\"></script>"
            + "Number 1, Spring 2003</a>";

    MockCachedUrl mcu = new MockCachedUrl("http://www.example.com/blah/");
    mcu.setContent(source);

    extractor.extractUrls(
        mau, new StringInputStream(source), ENC, "http://www.example.com/blah/", cb);

    Set expected = SetUtil.set(url1, url2, url3, url4, url5);
    assertEquals(expected, cb.getFoundUrls());
  }
コード例 #8
0
 // ensure that scanning continues after a nested parser throws an error
 // XXX Need to cause an IOException while reading CSS from the stream
 public void xxxtestDoCrawlStyleError() throws IOException {
   String url1 = "http://example.com/blah1.html";
   String url2 = "http://example.com/blah2.html";
   String url3 = "http://example.com/blah3.html";
   String source =
       "<html><head>"
           + "<style type=\"text/css\">\n"
           + "<!--\n"
           + "@import url(\'"
           + url1
           + "\');\n"
           + // ensure css parser got invoked
           "foo {bgcolor: #FFFF};"
           + // and that this causes an error
           "@import url(\'"
           + url2
           + "\');\n"
           + // so that this one isn't found
           "-->\n"
           + "  </style>\n"
           + "<a href="
           + url3
           + "></a>"
           + // and this one is
           "</head></html>";
   assertEquals(SetUtil.set(url1, url3), parseSingleSource(source));
 }
コード例 #9
0
  public void testParsesFileWithQuotedUrls() throws IOException {
    String url = "http://www.example.com/link3.html";

    String source =
        "<html><head><title>Test</title></head><body>"
            + "<a href=\"http://www.example.com/link3.html\">link3</a>";
    assertEquals(SetUtil.set(url), parseSingleSource(source));
  }
コード例 #10
0
  public void testResolvesHtmlEntities() throws IOException {
    String url1 =
        "http://www.example.com/bioone/?" + "request=get-toc&issn=0044-7447&volume=32&issue=1";

    String source =
        "<html><head><title>Test</title></head><body>"
            + "<a href=http://www.example.com/bioone/?"
            + "request=get-toc&#38;issn=0044-7447&#38;volume=32&issue=1>link1</a>";
    assertEquals(SetUtil.set(url1), parseSingleSource(source));

    // ensure character entities processed before rel url resolution
    source =
        "<html><head><title>Test</title></head><body>"
            + "<base href=http://www.example.com/foo/bar>"
            + "<a href=&#46&#46/xxx>link1</a>";
    assertEquals(SetUtil.set("http://www.example.com/xxx"), parseSingleSource(source));
  }
コード例 #11
0
  public void testIgnoresNewLineInField() throws IOException {
    String url = "http://www.example.com/link.html";

    String source =
        "<html><head><title>Test</title></head><body>"
            + "<img\nsrc=\"http://www.example.com/link.html\">Link</a>";
    assertEquals(SetUtil.set(url), parseSingleSource(source));
  }
コード例 #12
0
  public void testIgnoresCRInUrl() throws IOException {
    String url = "http://www.example.com/linkwithspace.html";

    String source =
        "<html><head><title>Test</title></head><body>"
            + "<a href=\"http://www.example.com/link\rwith\rspace.html\">Link</a>";
    assertEquals(SetUtil.set(url), parseSingleSource(source));
  }
コード例 #13
0
 /** Test that a single links is found. */
 public void testOneUrl() throws Exception {
   try {
     Set urls = SetUtil.set("http://www.foo.com/blah.jpg");
     assertEquals(urls, extractUrls(constructValidRDF(urls)));
   } catch (Throwable ex) {
     fail("", ex);
   }
 }
コード例 #14
0
 public void testDefaults() throws Exception {
   Properties p = initProps();
   KeyStore ks = KeyStoreUtil.createKeyStore(p);
   List aliases = ListUtil.fromIterator(new EnumerationIterator(ks.aliases()));
   assertIsomorphic(SetUtil.set("mykey", "mycert"), SetUtil.theSet(aliases));
   assertNotNull(ks.getCertificate("mycert"));
   assertNull(ks.getCertificate("foocert"));
   assertEquals("JCEKS", ks.getType());
 }
コード例 #15
0
  private void singleTagParse(
      String url, String startTag, String endTag, ArchivalUnit au, boolean shouldParse)
      throws IOException {
    MockCachedUrl mcu = new MockCachedUrl("http://www.example.com");
    String content = makeContent(url, startTag, endTag);
    mcu.setContent(content);

    MyLinkExtractorCallback cb = new MyLinkExtractorCallback();
    extractor.extractUrls(mau, new StringInputStream(content), ENC, "http://www.example.com", cb);

    if (shouldParse) {
      Set expected = SetUtil.set(url);
      assertEquals("Misparsed: " + content, expected, cb.getFoundUrls());
    } else {
      Set expected = SetUtil.set();
      assertEquals("Misparsed: " + content, expected, cb.getFoundUrls());
    }
  }
コード例 #16
0
 // ensure addToSchedule updates structures if (Mock)Scheduler returns true
 public void testAddToScheduleOk() {
   StepTask t1 = task(100, 200, 50);
   StepTask t2 = task(100, 200, 100);
   Schedule sched = sched(ListUtil.list(t1, t2));
   fact.setResult(sched);
   assertTrue(tr.addToSchedule(t1));
   assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
   assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
   assertForegroundStat(0, TaskRunner.STAT_REFUSED);
   fact.setResult(sched);
   assertTrue(tr.addToSchedule(t2));
   assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks));
   assertEquals(sched, tr.getCurrentSchedule());
   assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks()));
   assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
   assertForegroundStat(2, TaskRunner.STAT_WAITING);
   assertForegroundStat(0, TaskRunner.STAT_REFUSED);
 }
コード例 #17
0
  public void testHttpEquiv() throws IOException {
    String url1 = "http://example.com/blah.html";
    String source =
        "<html><head>"
            + "<meta http-equiv=\"refresh\" "
            + "content=\"0; url=http://example.com/blah.html\">"
            + "</head></html>";

    assertEquals(SetUtil.set(url1), parseSingleSource(source));

    source =
        "<html><head>"
            + "<meta http-equiv=\"refresh\" "
            + "content=\"0;url=http://example.com/blah.html\">"
            + "</head></html>";

    assertEquals(SetUtil.set(url1), parseSingleSource(source));
  }
コード例 #18
0
  // one task to drop, succeeds after dropping it
  public void testAddToScheduleOkAfterDrops() {
    log.debug("testAddToScheduleOkAfterDrops()");
    ConfigurationUtil.addFromArgs(
        TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
    StepTask t1 = task(100, 200, 50);
    StepTask t2 = task(100, 200, 100);
    Schedule sched1 = sched(ListUtil.list(t1));
    Schedule sched2 = sched(ListUtil.list(t2));
    fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2));
    TimeBase.step(101);
    assertTrue(tr.addToSchedule(t1));
    assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
    assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(0, TaskRunner.STAT_REFUSED);
    assertForegroundStat(1, TaskRunner.STAT_WAITING);
    assertForegroundStat(0, TaskRunner.STAT_DROPPED);

    assertTrue(tr.addToSchedule(t2));
    assertEquals(
        ListUtil.list(
            ListUtil.list(t1),
            ListUtil.list(t1, t2),
            ListUtil.list(t1),
            Collections.EMPTY_SET,
            ListUtil.list(t1, t2)),
        fact.createArgs);
    assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks));
    assertTrue(t1.isDropped());
    assertEquals(sched2, tr.getCurrentSchedule());
    assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks()));
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getOverrunTasks()));
    assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(0, TaskRunner.STAT_REFUSED);
    assertForegroundStat(1, TaskRunner.STAT_WAITING);
    assertForegroundStat(1, TaskRunner.STAT_DROPPED);

    t2.cancel();
    assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(0, TaskRunner.STAT_REFUSED);
    assertForegroundStat(0, TaskRunner.STAT_WAITING);
    assertForegroundStat(1, TaskRunner.STAT_DROPPED);
    assertForegroundStat(1, TaskRunner.STAT_CANCELLED);
  }
コード例 #19
0
  public void testFindTaskToRunRemovesExpiredChunks() {
    assertFalse(tr.findTaskToRun());
    StepTask t1 = task(100, 200, 100);
    StepTask t2 = task(100, 300, 50);
    StepTask texp1 = task(0, 0, 50);
    StepTask texp2 = task(0, 0, 50);

    Schedule s = sched(ListUtil.list(texp1, texp2, t1, t2));
    fact.setResults(s, s);
    assertTrue(tr.addToSchedule(t1));
    assertTrue(tr.addToSchedule(t2));
    assertFalse(tr.findTaskToRun());
    assertEquals(2, removedChunks.size());
    assertEquals(
        SetUtil.set(texp1, texp2),
        SetUtil.set(
            ((Schedule.Chunk) removedChunks.get(0)).getTask(),
            ((Schedule.Chunk) removedChunks.get(1)).getTask()));
  }
コード例 #20
0
  public void testSkipsMalformedComments() throws IOException {
    String url = "http://www.example.com/link3.html";

    String source =
        "<html><head><title>Test</title></head><body>"
            + "<!--<a href=http://www.example.com/link1.html>link1</a>"
            + "Filler, with <b>bold</b> tags and<i>others</i>"
            + "<a href=http://www.example.com/link2.html>link2</a>--!>"
            + "<a href=http://www.example.com/link3.html>link3</a>";
    assertEquals(SetUtil.set(url), parseSingleSource(source));
  }
コード例 #21
0
  public void testHeadNotAllowed() throws Exception {
    VersionCounts versionCounts = VersionCounts.make();

    VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
    byte[] hash1 = addVersion(vb1, "content 1 for foo1");
    byte[] hash2 = addVersion(vb1, "content 2 for foo1");

    versionCounts.vote(vb1, participant1);

    Map<ParticipantUserData, HashResult> repairCandidates;
    repairCandidates = versionCounts.getRepairCandidates(0);
    assertSameElements(SetUtil.set(participant1), repairCandidates.keySet());

    // Same, but with an excluded version that doesn't matter.
    repairCandidates = versionCounts.getRepairCandidates(0, SetUtil.set(HashResult.make(hash2)));
    assertSameElements(SetUtil.set(participant1), repairCandidates.keySet());

    // Same, but with an excluded version that does matter
    repairCandidates = versionCounts.getRepairCandidates(0, SetUtil.set(HashResult.make(hash1)));
    assertEmpty(repairCandidates);
  }
コード例 #22
0
  public void testDontParseJSByDefault() throws IOException {
    String url3 = "http://www.example.com/link1.html";

    String source =
        "<html><head><title>Test</title></head><body>"
            + "<a href = javascript:newWindow('http://www.example.com/link3.html')</a>"
            + "<a href = javascript:popup('http://www.example.com/link2.html')</a>"
            + "<img src = javascript:popup('"
            + url3
            + "') </img>";
    assertEquals(SetUtil.set(), parseSingleSource(source));
  }
コード例 #23
0
 private void doScriptSkipTest(String openScript, String closeScript, String failMsg)
     throws IOException {
   String url = "http://www.example.com/link3.html";
   String src =
       "<html><head><title>Test</title></head><body>"
           + openScript
           + "<a href=http://www.example.com/link1.html>link1</a>"
           + "Filler, with <b>bold</b> tags and<i>others</i>"
           + "<a href=http://www.example.com/link2.html>link2</a>"
           + closeScript
           + "<a href=http://www.example.com/link3.html>link3</a>";
   assertEquals(failMsg, SetUtil.set(url), parseSingleSource(src));
 }
コード例 #24
0
  public void testSortedRepairCandidates() throws Exception {
    VersionCounts versionCounts = VersionCounts.make();

    VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb1, "content 1 for foo1");

    VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb2, "content 2 for foo1");

    VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb3, "content 3 for foo1");
    addVersion(vb3, "content 2 for foo1");

    versionCounts.vote(vb1, participant1);
    versionCounts.vote(vb2, participant2);
    versionCounts.vote(vb3, participant3);

    Map<Integer, Collection<ParticipantUserData>> repairCandidates;

    repairCandidates = versionCounts.getSortedRepairCandidatesMap(2);
    assertEquals(SetUtil.set(2), repairCandidates.keySet());
    assertSameElements(SetUtil.set(participant2), repairCandidates.get(2));
    assertEquals(ListUtil.list(participant2), versionCounts.getSortedRepairCandidates(2));

    repairCandidates = versionCounts.getSortedRepairCandidatesMap(1);
    assertIsomorphic(ListUtil.list(2, 1), repairCandidates.keySet());
    assertSameElements(SetUtil.set(participant2), repairCandidates.get(2));
    assertSameElements(SetUtil.set(participant1, participant3), repairCandidates.get(1));

    List<ParticipantUserData> lst = versionCounts.getSortedRepairCandidates(1);
    assertTrue(
        "" + lst,
        (lst.equals(ListUtil.list(participant2, participant1, participant3))
            || lst.equals(ListUtil.list(participant2, participant3, participant1))));

    assertEmpty(versionCounts.getSortedRepairCandidatesMap(4));
    assertEmpty(versionCounts.getSortedRepairCandidates(4));
  }
コード例 #25
0
  // one task not ready to start yet, so not droppable
  public void testAddToScheduleFailNoDroppable() {
    log.debug("testAddToScheduleOkAfterDrops()");
    ConfigurationUtil.addFromArgs(
        TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
    StepTask t1 = task(100, 200, 50);
    StepTask t2 = task(100, 200, 100);
    Schedule sched1 = sched(ListUtil.list(t1));
    Schedule sched2 = sched(ListUtil.list(t2));
    fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2));
    assertTrue(tr.addToSchedule(t1));
    assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));

    assertFalse(tr.addToSchedule(t2));
    assertEquals(
        ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
        fact.createArgs);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
    assertFalse(t1.isDropped());
    assertEquals(sched1, tr.getCurrentSchedule());
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
    assertEmpty(SetUtil.theSet(tr.getOverrunTasks()));
  }
コード例 #26
0
  public void testRelativeLinksLocationTagsAndMultipleKeys() throws IOException {
    String url1 = "http://www.example.com/link1.html";
    String url2 = "http://www.example.com/link2.html#ref";
    String url3 = "http://www.example.com/dir/link3.html";

    String source =
        "<html><head><title>Test</title></head><body>"
            + "<a href=link1.html>link1</a>"
            + "Filler, with <b>bold</b> tags and<i>others</i>"
            + "<a blah1=blah href=link2.html#ref blah2=blah>link2</a>"
            + "<a href=dir/link3.html>link3</a>";

    assertEquals(SetUtil.set(url1, url2, url3), parseSingleSource(source));
  }
コード例 #27
0
  public void testIgnoresEmptyHrefInBaseTag() throws IOException {
    String url1 = "http://www.example.com/link1.html";
    String url2 = "http://www.example.com/link2.html";
    String url3 = "http://www.example.com/link3.html";

    String source =
        "<html><head><title>Test</title></head><body>"
            + "<a href=link1.html>link1</a>"
            + "Filler, with <b>bold</b> tags and<i>others</i>"
            + "<base href=\"\" blah=blah>"
            + "<a href=link2.html>link2</a>"
            + "<a href=link3.html>link3</a>";
    assertEquals(SetUtil.set(url1, url2, url3), parseSingleSource(source));
  }
コード例 #28
0
  public void testDoCrawlImageWithSrcInAltTagAfterSrcProper() throws IOException {
    String url = "http://www.example.com/link3.html";

    String source =
        "<html><head><title>Test</title></head><body>" + "<img src=" + url + " alt=src>link3</a>";

    MockCachedUrl mcu = new MockCachedUrl(startUrl);
    mcu.setContent(source);

    extractor.extractUrls(mau, new StringInputStream(source), ENC, startUrl, cb);

    Set expected = SetUtil.set(url);
    assertEquals(expected, cb.getFoundUrls());
  }
コード例 #29
0
  public void testFindTaskToRunRemovesExpiredOverrunners() {
    assertFalse(tr.findTaskToRun());
    StepTask t1 = task(100, 200, 100);
    StepTask t2 = task(100, 300, 50);
    StepTask texp1 = task(0, 0, 50);
    StepTask texp2 = task(0, 0, 49);

    Schedule s = sched(ListUtil.list(t1, t2));
    fact.setResults(s, s);
    assertTrue(tr.addToSchedule(t1));
    assertTrue(tr.addToSchedule(t2));
    tr.addOverrunner(texp1);
    tr.addOverrunner(texp2);

    // if this fails, it might be because the sorted list/set is treating
    // sort-order equivalence as object equality, which we don't want
    assertEquals(2, tr.getOverrunTasks().size());
    assertFalse(tr.findTaskToRun());
    assertEquals(0, removedChunks.size());
    assertEquals(2, removedTasks.size());
    assertEquals(
        SetUtil.set(texp1, texp2),
        SetUtil.set((StepTask) removedTasks.get(0), (StepTask) removedTasks.get(1)));
  }
コード例 #30
0
  public void testStore() throws Exception {
    File dir = getTempDir();
    File file = new File(dir, "test.ks");
    Properties p = initProps();
    p.put(KeyStoreUtil.PROP_KEYSTORE_FILE, file.toString());
    assertFalse(file.exists());
    KeyStore ks = KeyStoreUtil.createKeyStore(p);
    assertTrue(file.exists());

    KeyStore ks2 = loadKeyStore(ks.getType(), file, PASSWD);
    List aliases = ListUtil.fromIterator(new EnumerationIterator(ks2.aliases()));
    assertIsomorphic(SetUtil.set("mykey", "mycert"), SetUtil.theSet(aliases));
    assertNotNull(ks2.getCertificate("mycert"));
    assertNull(ks2.getCertificate("foocert"));
    assertEquals("JCEKS", ks2.getType());
  }