// remove overrunnable task-ending chunk, before deadline,
  public void testRemoveChunkTaskEndOver() {
    final List finished = new ArrayList();
    StepTask t1 =
        task(
            100,
            200,
            100,
            new TaskCallback() {
              public void taskEvent(SchedulableTask task, Schedule.EventType event) {
                if (log.isDebug2()) {
                  log.debug2("testRemoveChunkTaskEndOver callback");
                }
                if (event == Schedule.EventType.FINISH) {
                  finished.add(task);
                }
              }
            });
    t1.setOverrunAllowed(true);
    Schedule s = sched(ListUtil.list(t1));
    fact.setResult(s);
    assertTrue(tr.addToSchedule(t1));

    Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
    assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
    chunk.setTaskEnd();
    tr.removeChunk(chunk);
    assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
    assertEmpty(finished);
    assertIsomorphic(ListUtil.list(t1), tr.getAcceptedTasks());
    assertIsomorphic(SetUtil.set(t1), tr.getOverrunTasks());
  }
 public void testGetRepositoryList() throws Exception {
   assertEmpty(mgr.getRepositoryList());
   String tempDirPath = setUpDiskSpace();
   assertEquals(ListUtil.list("local:" + tempDirPath), mgr.getRepositoryList());
   String tempdir2 = getTempDir().getAbsolutePath() + File.separator;
   ConfigurationUtil.setFromArgs(
       "org.lockss.platform.diskSpacePaths", tempdir2 + ";" + tempDirPath);
   assertEquals(
       ListUtil.list("local:" + tempdir2, "local:" + tempDirPath), mgr.getRepositoryList());
 }
  public void testRemoveChunk() {
    StepTask t1 = task(100, 200, 100);
    Schedule s = sched(ListUtil.list(t1));
    fact.setResult(s);
    assertTrue(tr.addToSchedule(t1));
    assertIsomorphic(ListUtil.list(t1), tr.getAcceptedTasks());

    Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
    assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
    tr.removeChunk(chunk);
    assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
  }
  public void testStoreDamagedNodeSet() throws Exception {
    DamagedNodeSet damNodes = new DamagedNodeSet(mau, repository);
    damNodes.nodesWithDamage.add("test1");
    damNodes.nodesWithDamage.add("test2");
    damNodes.cusToRepair.put("cus1", ListUtil.list("cus1-1", "cus1-2"));
    damNodes.cusToRepair.put("cus2", ListUtil.list("cus2-1"));
    assertTrue(damNodes.containsWithDamage("test1"));
    assertTrue(damNodes.containsWithDamage("test2"));
    assertFalse(damNodes.containsWithDamage("test3"));

    repository.storeDamagedNodeSet(damNodes);
    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath += HistoryRepositoryImpl.DAMAGED_NODES_FILE_NAME;
    File xmlFile = new File(filePath);
    assertTrue(xmlFile.exists());

    damNodes = null;
    damNodes = repository.loadDamagedNodeSet();
    // check damage
    assertTrue(damNodes.containsWithDamage("test1"));
    assertTrue(damNodes.containsWithDamage("test2"));
    assertFalse(damNodes.containsWithDamage("test3"));

    MockCachedUrlSet mcus1 = new MockCachedUrlSet("cus1");
    MockCachedUrlSet mcus2 = new MockCachedUrlSet("cus2");

    // check repairs
    assertTrue(damNodes.containsToRepair(mcus1, "cus1-1"));
    assertTrue(damNodes.containsToRepair(mcus1, "cus1-2"));
    assertFalse(damNodes.containsToRepair(mcus1, "cus2-1"));
    assertTrue(damNodes.containsToRepair(mcus2, "cus2-1"));
    assertEquals(mau.getAuId(), damNodes.theAu.getAuId());

    // check remove
    damNodes.removeFromRepair(mcus1, "cus1-1");
    assertFalse(damNodes.containsToRepair(mcus1, "cus1-1"));
    assertTrue(damNodes.containsToRepair(mcus1, "cus1-2"));
    damNodes.removeFromRepair(mcus1, "cus1-2");
    assertFalse(damNodes.containsToRepair(mcus1, "cus1-2"));
    assertNull(damNodes.cusToRepair.get(mcus1));

    // check remove from damaged nodes
    damNodes.removeFromDamage("test1");
    damNodes.removeFromDamage("test2");
    repository.storeDamagedNodeSet(damNodes);
    damNodes = repository.loadDamagedNodeSet();
    assertNotNull(damNodes);
    assertFalse(damNodes.containsWithDamage("test1"));
    assertFalse(damNodes.containsWithDamage("test2"));
  }
Beispiel #5
0
 public void getGroup() throws Exception {
   m_testMsg.setGroups(null);
   assertNull(m_testMsg.getGroups());
   assertNull(m_testMsg.getGroupList());
   m_testMsg.setGroups("foo");
   assertEquals("foo", m_testMsg.getGroups());
   assertEquals(ListUtil.list("foo"), m_testMsg.getGroupList());
   m_testMsg.setGroups("foo;bar");
   assertEquals("foo;bar", m_testMsg.getGroups());
   assertEquals(ListUtil.list("foo", "bar"), m_testMsg.getGroupList());
   m_testMsg.setGroups("foo;bar;baz");
   assertEquals("foo;bar;baz", m_testMsg.getGroups());
   assertEquals(ListUtil.list("foo", "bar", "baz"), m_testMsg.getGroupList());
 }
 public void testIsTaskSchedulable() {
   fact.setResult(null);
   StepTask t1 = task(100, 200, 50);
   assertFalse(tr.isTaskSchedulable(t1));
   fact.setResult(sched(ListUtil.list(t1)));
   assertTrue(tr.isTaskSchedulable(t1));
 }
 public void testRunStepsWithOverrunDisallowed() {
   StepTask t1 = task(100, 300, 100, null, new MyMockStepper(15, -10));
   //    t1.setOverrunAllowed(true);
   StepTask t2 = task(150, 250, 100, null, new MyMockStepper(10, -10));
   Schedule s = sched(ListUtil.list(t1, t2));
   fact.setResults(s, s);
   assertTrue(tr.addToSchedule(t1));
   assertTrue(tr.addToSchedule(t2));
   TimeBase.setSimulated(101);
   Interrupter intr = null;
   try {
     intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
     while (tr.findTaskToRun()) {
       tr.runSteps(new MutableBoolean(true), null);
     }
     intr.cancel();
   } catch (Exception e) {
     log.error("runSteps threw:", e);
   } finally {
     if (intr.did()) {
       fail("runSteps looped");
     }
   }
   assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(removedTasks));
   assertTrue(t1.e.toString(), t1.e instanceof SchedService.Overrun);
 }
  public void testStepperThrows() {
    final List finished = new ArrayList();
    TaskCallback cb =
        new TaskCallback() {
          public void taskEvent(SchedulableTask task, Schedule.EventType event) {
            if (event == Schedule.EventType.FINISH) {
              finished.add(task);
            }
          }
        };

    MyMockStepper stepper = new MyMockStepper(10, -10);
    stepper.setWhenToThrow(5);
    StepTask t1 = task(100, 200, 100, cb, stepper);
    Schedule s = sched(ListUtil.list(t1));
    fact.setResult(s);
    assertTrue(tr.addToSchedule(t1));
    TimeBase.setSimulated(101);
    assertTrue(tr.findTaskToRun());
    Interrupter intr = null;
    try {
      intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
      tr.runSteps(new MutableBoolean(true), null);
      intr.cancel();
    } catch (Exception e) {
      log.error("runSteps threw:", e);
    } finally {
      if (intr.did()) {
        fail("runSteps looped");
      }
    }
    assertSame(t1, finished.get(0));
    assertTrue(t1.e instanceof ExpectedRuntimeException);
    assertEquals(5, stepper.nSteps);
  }
  // remove task-ending chunk, past task deadline, s.b. Timeout error.
  public void testRemoveChunkTaskEndTimeout() {
    final List finished = new ArrayList();
    StepTask t1 =
        task(
            100,
            200,
            100,
            new TaskCallback() {
              public void taskEvent(SchedulableTask task, Schedule.EventType event) {
                if (log.isDebug2()) {
                  log.debug2("testRemoveChunkTaskEndTimeout callback");
                }
                if (event == Schedule.EventType.FINISH) {
                  finished.add(task);
                }
              }
            });
    Schedule s = sched(ListUtil.list(t1));
    fact.setResult(s);
    assertTrue(tr.addToSchedule(t1));

    Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
    assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
    chunk.setTaskEnd();
    TimeBase.setSimulated(201);
    tr.removeChunk(chunk);
    assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
    assertSame(t1, finished.get(0));
    assertNotNull(t1.e);
    assertTrue(t1.e.toString(), t1.e instanceof SchedService.Timeout);
    assertEmpty(tr.getAcceptedTasks());
  }
 public void testSizeCalc() throws Exception {
   SimpleBinarySemaphore sem = new SimpleBinarySemaphore();
   mgr.setSem(sem);
   RepositoryNode node1 = new RepositoryNodeImpl("url1", "testDir", null);
   RepositoryNode node2 = new RepositoryNodeImpl("url2", "testDir", null);
   RepositoryNode node3 = new RepositoryNodeImpl("url3", "testDir", null);
   mgr.queueSizeCalc(node1);
   assertTrue(sem.take(TIMEOUT_SHOULDNT));
   assertEquals(ListUtil.list(node1), mgr.getNodes());
   mgr.queueSizeCalc(node2);
   mgr.queueSizeCalc(node3);
   assertTrue(sem.take(TIMEOUT_SHOULDNT));
   if (mgr.getNodes().size() < 3) {
     assertTrue(sem.take(TIMEOUT_SHOULDNT));
   }
   assertSameElements(ListUtil.list(node1, node2, node3), mgr.getNodes());
 }
Beispiel #11
0
 // ensure addToSchedule updates structures if (Mock)Scheduler returns true
 public void testAddToScheduleOk() {
   StepTask t1 = task(100, 200, 50);
   StepTask t2 = task(100, 200, 100);
   Schedule sched = sched(ListUtil.list(t1, t2));
   fact.setResult(sched);
   assertTrue(tr.addToSchedule(t1));
   assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
   assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
   assertForegroundStat(0, TaskRunner.STAT_REFUSED);
   fact.setResult(sched);
   assertTrue(tr.addToSchedule(t2));
   assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks));
   assertEquals(sched, tr.getCurrentSchedule());
   assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks()));
   assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
   assertForegroundStat(2, TaskRunner.STAT_WAITING);
   assertForegroundStat(0, TaskRunner.STAT_REFUSED);
 }
Beispiel #12
0
  public void testCdnStems() {
    AuState aus = new AuState(mau, historyRepo);
    assertEquals(Collections.EMPTY_LIST, aus.getCdnStems());
    aus.addCdnStem("http://fff.uselesstld");
    assertClass(ArrayList.class, aus.getCdnStems());
    assertEquals(ListUtil.list("http://fff.uselesstld"), aus.getCdnStems());
    aus.addCdnStem("ccc");
    assertEquals(ListUtil.list("http://fff.uselesstld", "ccc"), aus.getCdnStems());

    aus.setCdnStems(new LinkedList(ListUtil.list("a", "b")));
    assertClass(ArrayList.class, aus.getCdnStems());
    assertEquals(ListUtil.list("a", "b"), aus.getCdnStems());
    aus.setCdnStems(null);
    assertEmpty(aus.getCdnStems());
    aus.addCdnStem("https://a.b/");
    aus.addCdnStem("https://b.a/");
    assertEquals(ListUtil.list("https://a.b/", "https://b.a/"), aus.getCdnStems());
  }
 public void testParsesOptionPositive() throws IOException {
   TypedEntryMap pMap = new TypedEntryMap();
   pMap.setMapElement("html-parser-select-attrs", ListUtil.list("value"));
   mau.setPropertyMap(pMap);
   singleTagShouldParse(
       "http://www.example.com/web_link.jpg", "<option  value=", "</option>", mau);
   singleTagShouldParse(
       "http://www.example.com/web_link.jpg", "<option a=b value=", "</option>", mau);
 }
  public void testCreateSharedPLNKeyStores() throws Exception {
    List<String> hosts = ListUtil.list("host1", "host2.foo.bar", "host3");
    List<String> hosts2 = ListUtil.list("host3", "host4");
    File dir = getTempDir();
    File pub = new File(dir, "pub.ks");
    KeyStoreUtil.createSharedPLNKeyStores(
        dir, hosts, pub, "pubpass", MiscTestUtil.getSecureRandom());
    assertPubKs(pub, "pubpass", hosts);
    for (String host : hosts) {
      assertPrivateKs(
          new File(dir, host + ".jceks"), StringUtil.fromFile(new File(dir, host + ".pass")), host);
    }
    KeyStore pubks1 = loadKeyStore("jceks", new File(dir, "pub.ks"), "pubpass");

    Certificate host1cert1 = pubks1.getCertificate("host1.crt");
    Certificate host3cert1 = pubks1.getCertificate("host3.crt");

    String host1priv1 = StringUtil.fromFile(new File(dir, "host1.jceks"));
    String host3priv1 = StringUtil.fromFile(new File(dir, "host3.jceks"));

    // Now add host4 and generate a new key for host3
    KeyStoreUtil.createSharedPLNKeyStores(
        dir, hosts2, pub, "pubpass", MiscTestUtil.getSecureRandom());
    List<String> both = ListUtils.sum(hosts, hosts2);
    assertPubKs(pub, "pubpass", both);
    for (String host : both) {
      assertPrivateKs(
          new File(dir, host + ".jceks"), StringUtil.fromFile(new File(dir, host + ".pass")), host);
    }
    KeyStore pubks2 = loadKeyStore("jceks", new File(dir, "pub.ks"), "pubpass");
    // host1 should have the same cert, host3 not
    Certificate host1cert2 = pubks2.getCertificate("host1.crt");
    Certificate host3cert2 = pubks2.getCertificate("host3.crt");
    assertEquals(host1cert1, host1cert2);
    assertNotEquals(host3cert1, host3cert2);

    // host1's private key file should be the same, host3's not
    String host1priv2 = StringUtil.fromFile(new File(dir, "host1.jceks"));
    String host3priv2 = StringUtil.fromFile(new File(dir, "host3.jceks"));
    assertEquals(host1priv1, host1priv2);
    assertNotEquals(host3priv1, host3priv2);
  }
  // test special (non-string) key in row doesn't throw
  public void testTextNonStringKey() throws Exception {
    MockStatusAccessor statusAccessor = new MockStatusAccessor();
    List cols = ListUtil.list("foo", StatusTable.ROW_SEPARATOR);
    statusAccessor.setRows(MockStatusAccessor.makeRowsFrom(cols, rowArray1), null);
    statusAccessor.setColumnDescriptors(
        MockStatusAccessor.makeColumnDescriptorsFrom(colArray1), null);

    statSvc.registerStatusAccessor("testtbl", statusAccessor);

    WebResponse resp = getTable("testtbl", true);
    log.debug(resp.getText());
  }
Beispiel #16
0
  // one task to drop, two failed schedule tries
  public void testAddToScheduleFailNoCleanup() {
    ConfigurationUtil.addFromArgs(
        TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
    StepTask t1 = task(100, 200, 50);
    StepTask t2 = task(100, 200, 100);
    Schedule sched = sched(ListUtil.list(t1));
    fact.setResult(sched);
    assertTrue(tr.addToSchedule(t1));
    assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
    assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(0, TaskRunner.STAT_REFUSED);
    assertForegroundStat(1, TaskRunner.STAT_WAITING);
    assertForegroundStat(0, TaskRunner.STAT_DROPPED);

    assertFalse(tr.addToSchedule(t2));
    assertEquals(
        ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
        fact.createArgs);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
    assertEquals(sched, tr.getCurrentSchedule());
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
    assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(1, TaskRunner.STAT_REFUSED);
    assertForegroundStat(1, TaskRunner.STAT_WAITING);
    assertForegroundStat(0, TaskRunner.STAT_DROPPED);
  }
Beispiel #17
0
  // one task to drop, succeeds after dropping it
  public void testAddToScheduleOkAfterDrops() {
    log.debug("testAddToScheduleOkAfterDrops()");
    ConfigurationUtil.addFromArgs(
        TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
    StepTask t1 = task(100, 200, 50);
    StepTask t2 = task(100, 200, 100);
    Schedule sched1 = sched(ListUtil.list(t1));
    Schedule sched2 = sched(ListUtil.list(t2));
    fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2));
    TimeBase.step(101);
    assertTrue(tr.addToSchedule(t1));
    assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
    assertForegroundStat(1, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(0, TaskRunner.STAT_REFUSED);
    assertForegroundStat(1, TaskRunner.STAT_WAITING);
    assertForegroundStat(0, TaskRunner.STAT_DROPPED);

    assertTrue(tr.addToSchedule(t2));
    assertEquals(
        ListUtil.list(
            ListUtil.list(t1),
            ListUtil.list(t1, t2),
            ListUtil.list(t1),
            Collections.EMPTY_SET,
            ListUtil.list(t1, t2)),
        fact.createArgs);
    assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(fact.scheduler.tasks));
    assertTrue(t1.isDropped());
    assertEquals(sched2, tr.getCurrentSchedule());
    assertEquals(SetUtil.set(t1, t2), SetUtil.theSet(tr.getAcceptedTasks()));
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getOverrunTasks()));
    assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(0, TaskRunner.STAT_REFUSED);
    assertForegroundStat(1, TaskRunner.STAT_WAITING);
    assertForegroundStat(1, TaskRunner.STAT_DROPPED);

    t2.cancel();
    assertForegroundStat(2, TaskRunner.STAT_ACCEPTED);
    assertForegroundStat(0, TaskRunner.STAT_REFUSED);
    assertForegroundStat(0, TaskRunner.STAT_WAITING);
    assertForegroundStat(1, TaskRunner.STAT_DROPPED);
    assertForegroundStat(1, TaskRunner.STAT_CANCELLED);
  }
Beispiel #18
0
 // test completion & callback
 public void testDone() throws Exception {
   HashQueue q = new HashQueue();
   final List cookieList = new LinkedList();
   final List eList = new LinkedList();
   HashService.Callback cb =
       new HashService.Callback() {
         public void hashingFinished(
             CachedUrlSet urlset,
             long timeUsed,
             Object cookie,
             CachedUrlSetHasher hasher,
             Exception e) {
           cookieList.add(cookie);
           eList.add(e);
         }
       };
   HashQueue.Request r1, r2, r3, r4, r5;
   r1 = req(2000, 0, 100, cb);
   r2 = req(10000, 0, 200, cb);
   r3 = req(20000, 0, 0, cb);
   r4 = req(50000, 0, 1, cb);
   assertTrue(q.insert(r1));
   assertTrue(q.insert(r2));
   assertTrue(q.insert(r4));
   assertEquals(0, cookieList.size());
   q.removeCompleted();
   assertEquals(0, cookieList.size());
   // make r1 timeout
   r1.deadline.expire();
   q.removeCompleted();
   List exp = ListUtil.list(r1);
   assertEquals(exp, cookieList);
   assertEquals(exp, q.getCompletedSnapshot());
   // make r2 timeout
   TimeBase.step(11000);
   // r3 is finished
   assertTrue(q.insert(r3));
   Exception r4e = new Exception();
   // make r4 error
   r4.e = r4e;
   q.removeCompleted();
   // check that they all finished, and in the right order
   Object exp2[] = {r1, r2, r3, r4};
   assertIsomorphic(exp2, cookieList);
   assertIsomorphic(exp2, q.getCompletedSnapshot());
   // check their exceptions
   assertTrue(eList.get(0) instanceof HashService.Timeout);
   assertTrue(eList.get(1) instanceof HashService.Timeout);
   assertSame(null, eList.get(2));
   assertSame(r4e, eList.get(3));
 }
  public void testSortedRepairCandidates() throws Exception {
    VersionCounts versionCounts = VersionCounts.make();

    VoteBlock vb1 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb1, "content 1 for foo1");

    VoteBlock vb2 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb2, "content 2 for foo1");

    VoteBlock vb3 = makeVoteBlock("http://test.com/foo1");
    addVersion(vb3, "content 3 for foo1");
    addVersion(vb3, "content 2 for foo1");

    versionCounts.vote(vb1, participant1);
    versionCounts.vote(vb2, participant2);
    versionCounts.vote(vb3, participant3);

    Map<Integer, Collection<ParticipantUserData>> repairCandidates;

    repairCandidates = versionCounts.getSortedRepairCandidatesMap(2);
    assertEquals(SetUtil.set(2), repairCandidates.keySet());
    assertSameElements(SetUtil.set(participant2), repairCandidates.get(2));
    assertEquals(ListUtil.list(participant2), versionCounts.getSortedRepairCandidates(2));

    repairCandidates = versionCounts.getSortedRepairCandidatesMap(1);
    assertIsomorphic(ListUtil.list(2, 1), repairCandidates.keySet());
    assertSameElements(SetUtil.set(participant2), repairCandidates.get(2));
    assertSameElements(SetUtil.set(participant1, participant3), repairCandidates.get(1));

    List<ParticipantUserData> lst = versionCounts.getSortedRepairCandidates(1);
    assertTrue(
        "" + lst,
        (lst.equals(ListUtil.list(participant2, participant1, participant3))
            || lst.equals(ListUtil.list(participant2, participant3, participant1))));

    assertEmpty(versionCounts.getSortedRepairCandidatesMap(4));
    assertEmpty(versionCounts.getSortedRepairCandidates(4));
  }
  public void testStoreAuEmptyState() throws Exception {
    HashSet strCol = new HashSet();
    strCol.add("test");
    AuState origState = new AuState(mau, repository);
    repository.storeAuState(origState);
    AuState loadedState = repository.loadAuState();
    assertEquals(-1, loadedState.getLastCrawlTime());
    assertEquals(-1, loadedState.getLastCrawlAttempt());
    assertEquals(-1, loadedState.getLastCrawlResult());
    assertEquals("Unknown code -1", loadedState.getLastCrawlResultMsg());
    assertEquals(-1, loadedState.getLastTopLevelPollTime());
    assertEquals(-1, loadedState.getLastPollStart());
    assertEquals(-1, loadedState.getLastPollResult());
    assertEquals(null, loadedState.getLastPollResultMsg());

    assertEquals(-1, loadedState.getLastPoPPoll());
    assertEquals(-1, loadedState.getLastPoPPollResult());
    assertEquals(-1, loadedState.getLastLocalHashScan());
    assertEquals(0, loadedState.getNumAgreePeersLastPoR());
    assertEquals(0, loadedState.getNumWillingRepairers());
    assertEquals(0, loadedState.getNumCurrentSuspectVersions());
    assertEmpty(loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());
    loadedState.addCdnStem("http://this.is.new/");
    assertEquals(ListUtil.list("http://this.is.new/"), loadedState.getCdnStems());

    assertEquals(0, loadedState.getPollDuration());
    assertEquals(0, loadedState.getClockssSubscriptionStatus());
    assertEquals(null, loadedState.getAccessType());
    assertEquals(SubstanceChecker.State.Unknown, loadedState.getSubstanceState());
    assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Substance));
    assertEquals(null, loadedState.getFeatureVersion(Plugin.Feature.Metadata));
    assertEquals(-1, loadedState.getLastMetadataIndex());
    assertEquals(0, loadedState.getLastContentChange());
    assertEquals(mau.getAuId(), loadedState.getArchivalUnit().getAuId());
  }
Beispiel #21
0
  public void testFromList1() {
    assertEquals(MapUtil.map(), MapUtil.fromList(ListUtil.list()));
    assertEquals(
        MapUtil.map("FOO", "bar", "One", "Two"),
        MapUtil.fromList(ListUtil.list("FOO", "bar", "One", "Two")));
    assertEquals(
        MapUtil.map("foo", "bar", "one", "two"),
        MapUtil.fromList(ListUtil.list(ListUtil.list("foo", "bar"), ListUtil.list("one", "two"))));

    try {
      MapUtil.fromList(ListUtil.list("FOO", "bar", "One"));
      fail("Odd length arg list should throw");
    } catch (IllegalArgumentException e) {
    }
    try {
      MapUtil.fromList(ListUtil.list(ListUtil.list("foo", "bar"), ListUtil.list("one")));
      fail("Short sublist should throw");
    } catch (IllegalArgumentException e) {
    }
  }
Beispiel #22
0
  public void testFindOverrunTaskToRun() {
    assertFalse(tr.findTaskToRun());
    StepTask t1 = task(100, 200, 100);
    Schedule s = sched(ListUtil.list(t1));
    fact.setResult(s);
    assertTrue(tr.addToSchedule(t1));
    assertFalse(tr.findTaskToRun());
    assertEquals(Deadline.at(100), tr.runningDeadline);

    StepTask t2 = task(0, 300, 50);
    tr.addOverrunner(t2);
    assertTrue(tr.findTaskToRun());
    assertEquals(t2, tr.runningTask);
    assertEquals(Deadline.at(100), tr.runningDeadline);
    assertNull(tr.runningChunk);
  }
Beispiel #23
0
  public void testFindChunkTaskToRun() {
    assertFalse(tr.findTaskToRun());
    StepTask t1 = task(100, 200, 100);
    StepTask t2 = task(100, 300, 50);

    Schedule s = sched(ListUtil.list(t1, t2));
    fact.setResults(s, s);
    assertTrue(tr.addToSchedule(t1));
    assertTrue(tr.addToSchedule(t2));
    assertFalse(tr.findTaskToRun());
    assertEquals(Deadline.at(100), tr.runningDeadline);

    TimeBase.setSimulated(101);
    assertTrue(tr.findTaskToRun());
    assertEquals(t1, tr.runningTask);
    assertEquals(t1.getLatestFinish(), tr.runningDeadline);
    assertEquals(s.getEvents().get(0), tr.runningChunk);
  }
Beispiel #24
0
  public void testFindTaskToRunRemovesExpiredChunks() {
    assertFalse(tr.findTaskToRun());
    StepTask t1 = task(100, 200, 100);
    StepTask t2 = task(100, 300, 50);
    StepTask texp1 = task(0, 0, 50);
    StepTask texp2 = task(0, 0, 50);

    Schedule s = sched(ListUtil.list(texp1, texp2, t1, t2));
    fact.setResults(s, s);
    assertTrue(tr.addToSchedule(t1));
    assertTrue(tr.addToSchedule(t2));
    assertFalse(tr.findTaskToRun());
    assertEquals(2, removedChunks.size());
    assertEquals(
        SetUtil.set(texp1, texp2),
        SetUtil.set(
            ((Schedule.Chunk) removedChunks.get(0)).getTask(),
            ((Schedule.Chunk) removedChunks.get(1)).getTask()));
  }
Beispiel #25
0
 public void testFindRunnableChunk() {
   assertFalse(tr.findTaskToRun());
   StepTask t1 = task(100, 200, 100);
   StepTask t2 = task(10, 300, 50);
   Schedule.Chunk c1 = new Schedule.Chunk(t1, Deadline.at(100), Deadline.at(200), 100);
   Schedule.Chunk c2 = new Schedule.Chunk(t2, Deadline.at(200), Deadline.at(300), 100);
   Schedule s = new Schedule(ListUtil.list(c1, c2));
   fact.setResults(s, s);
   assertTrue(tr.addToSchedule(t1));
   assertTrue(tr.addToSchedule(t2));
   assertFalse(tr.findTaskToRun());
   assertEquals(Deadline.at(100), tr.runningDeadline);
   TimeBase.setSimulated(11);
   assertTrue(tr.findTaskToRun());
   assertEquals(t2, tr.runningTask);
   assertEquals(c2, tr.runningChunk);
   assertEquals(Deadline.at(100), tr.runningDeadline);
   assertEquals(s.getEvents().get(1), tr.runningChunk);
 }
Beispiel #26
0
  // one task not ready to start yet, so not droppable
  public void testAddToScheduleFailNoDroppable() {
    log.debug("testAddToScheduleOkAfterDrops()");
    ConfigurationUtil.addFromArgs(
        TaskRunner.PARAM_DROP_TASK_MAX, "10", TaskRunner.PARAM_MIN_CLEANUP_INTERVAL, "0");
    StepTask t1 = task(100, 200, 50);
    StepTask t2 = task(100, 200, 100);
    Schedule sched1 = sched(ListUtil.list(t1));
    Schedule sched2 = sched(ListUtil.list(t2));
    fact.setResults(ListUtil.list(sched1, null, null, sched2, sched2));
    assertTrue(tr.addToSchedule(t1));
    assertIsomorphic(ListUtil.list(t1), fact.scheduler.tasks);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));

    assertFalse(tr.addToSchedule(t2));
    assertEquals(
        ListUtil.list(ListUtil.list(t1), ListUtil.list(t1, t2), ListUtil.list(t1)),
        fact.createArgs);
    assertEquals(SetUtil.set(t1), SetUtil.theSet(fact.scheduler.tasks));
    assertFalse(t1.isDropped());
    assertEquals(sched1, tr.getCurrentSchedule());
    assertEquals(SetUtil.set(t1), SetUtil.theSet(tr.getAcceptedTasks()));
    assertEmpty(SetUtil.theSet(tr.getOverrunTasks()));
  }
Beispiel #27
0
  public void testNotifyThread() {
    final List rec = new ArrayList();
    final SimpleBinarySemaphore sem = new SimpleBinarySemaphore();
    tr.setImmediateNotify(false);

    TaskCallback cb =
        new TaskCallback() {
          public void taskEvent(SchedulableTask task, Schedule.EventType event) {
            rec.add(new BERec(Deadline.in(0), (BackgroundTask) task, event));
            sem.give();
          }
        };
    BackgroundTask t1 = btask(100, 200, .1, cb);
    BackgroundTask t2 = btask(100, 300, .2, cb);

    tr.notify(t1, Schedule.EventType.START);
    tr.notify(t1, Schedule.EventType.FINISH);
    // 2nd finish event should not cause another callback
    tr.notify(t1, Schedule.EventType.FINISH);
    tr.notify(t2, Schedule.EventType.START);

    Interrupter intr = null;
    try {
      intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
      while (rec.size() < 3) {
        sem.take();
      }
      assertEquals(
          ListUtil.list(
              new BERec(0, t1, Schedule.EventType.START),
              new BERec(0, t1, Schedule.EventType.FINISH),
              new BERec(0, t2, Schedule.EventType.START)),
          rec);
      intr.cancel();
    } finally {
      if (intr.did()) {
        fail("Notifier didn't run callbacks");
      }
    }
  }
Beispiel #28
0
  public void testFindTaskToRunRemovesExpiredOverrunners() {
    assertFalse(tr.findTaskToRun());
    StepTask t1 = task(100, 200, 100);
    StepTask t2 = task(100, 300, 50);
    StepTask texp1 = task(0, 0, 50);
    StepTask texp2 = task(0, 0, 49);

    Schedule s = sched(ListUtil.list(t1, t2));
    fact.setResults(s, s);
    assertTrue(tr.addToSchedule(t1));
    assertTrue(tr.addToSchedule(t2));
    tr.addOverrunner(texp1);
    tr.addOverrunner(texp2);

    // if this fails, it might be because the sorted list/set is treating
    // sort-order equivalence as object equality, which we don't want
    assertEquals(2, tr.getOverrunTasks().size());
    assertFalse(tr.findTaskToRun());
    assertEquals(0, removedChunks.size());
    assertEquals(2, removedTasks.size());
    assertEquals(
        SetUtil.set(texp1, texp2),
        SetUtil.set((StepTask) removedTasks.get(0), (StepTask) removedTasks.get(1)));
  }
public class SingleCrawlStatusAccessor implements StatusAccessor {

  private static final String MIME_TYPE_NAME = "mime_type_name";
  private static final String MIME_TYPE_NUM_URLS = "mime_type_num_urls";
  private static final String MIMETYPES_URLS_KEY = "mime-type";
  private static final String CRAWL_URLS_STATUS_ACCESSOR = CrawlManagerImpl.CRAWL_URLS_STATUS_TABLE;

  private List colDescsMimeTypes =
      ListUtil.fromArray(
          new ColumnDescriptor[] {
            new ColumnDescriptor(MIME_TYPE_NAME, "Mime Type", ColumnDescriptor.TYPE_STRING),
            new ColumnDescriptor(
                MIME_TYPE_NUM_URLS,
                "URLs Found",
                ColumnDescriptor.TYPE_INT,
                "Number of pages of that mime type fetched during this crawl"),
          });

  private static final List statusSortRules =
      ListUtil.list(new StatusTable.SortRule(MIME_TYPE_NAME, true));

  private CrawlManager.StatusSource statusSource;

  public SingleCrawlStatusAccessor(CrawlManager.StatusSource statusSource) {
    this.statusSource = statusSource;
  }

  public void populateTable(StatusTable table) throws StatusService.NoSuchTableException {
    if (table == null) {
      throw new IllegalArgumentException("Called with null table");
    } else if (table.getKey() == null) {
      throw new IllegalArgumentException("SingleCrawlStatusAccessor requires a key");
    }
    String key = table.getKey();
    CrawlerStatus status = statusSource.getStatus().getCrawlerStatus(key);
    if (status == null) {
      throw new StatusService.NoSuchTableException(
          "Status info from that crawl is no longer available");
    }
    table.setDefaultSortRules(statusSortRules);
    table.setColumnDescriptors(colDescsMimeTypes);
    table.setTitle(getTableTitle(status));
    table.setRows(getRows(status, key));
    table.setSummaryInfo(getSummaryInfo(status));
  }

  private String getTableTitle(CrawlerStatus status) {
    return "Status of crawl of " + status.getAuName();
  }

  /** iterate over the mime-types makeRow for each */
  private List getRows(CrawlerStatus status, String key) {
    Collection mimeTypes = status.getMimeTypes();
    List rows = new ArrayList();
    if (mimeTypes != null) {
      String mimeType;
      for (Iterator it = mimeTypes.iterator(); it.hasNext(); ) {
        mimeType = (String) it.next();
        rows.add(makeRow(status, mimeType, key));
      }
    }
    return rows;
  }

  private Map makeRow(CrawlerStatus status, String mimeType, String key) {
    Map row = new HashMap();
    row.put(MIME_TYPE_NAME, mimeType);
    row.put(
        MIME_TYPE_NUM_URLS,
        makeRefIfColl(status.getMimeTypeCtr(mimeType), key, MIMETYPES_URLS_KEY + ":" + mimeType));
    return row;
  }

  /** Return a reference object to the table, displaying the value */
  private Object makeRef(long value, String tableName, String key) {
    return new StatusTable.Reference(new Long(value), tableName, key);
  }

  /** If the UrlCounter has a collection, return a reference to it, else just the count */
  Object makeRefIfColl(CrawlerStatus.UrlCount ctr, String crawlKey, String subkey) {
    if (ctr.hasCollection()) {
      return makeRef(ctr.getCount(), CRAWL_URLS_STATUS_ACCESSOR, crawlKey + "." + subkey);
    }
    return new Long(ctr.getCount());
  }

  public String getDisplayName() {
    throw new UnsupportedOperationException("No generic name for MimeTypeStatusCrawler");
  }

  public boolean requiresKey() {
    return true;
  }

  public static final String FOOT_NO_SUBSTANCE_CRAWL_STATUS =
      "Though the crawl finished successfully, no files containing substantial content were collected.";

  private List getSummaryInfo(CrawlerStatus status) {
    List res = new ArrayList();
    StatusTable.SummaryInfo statusSi =
        new StatusTable.SummaryInfo(
            "Status", ColumnDescriptor.TYPE_STRING, status.getCrawlStatusMsg());
    ArchivalUnit au = status.getAu();
    if (au != null) {
      AuState aus = AuUtil.getAuState(au);
      if (status.getCrawlStatus() == Crawler.STATUS_SUCCESSFUL && aus.hasNoSubstance()) {
        statusSi.setValueFootnote(FOOT_NO_SUBSTANCE_CRAWL_STATUS);
      }
    }
    res.add(statusSi);
    String sources = StringUtil.separatedString(status.getSources());
    res.add(new StatusTable.SummaryInfo("Source", ColumnDescriptor.TYPE_STRING, sources));
    String startUrls = StringUtil.separatedString(status.getStartUrls());
    res.add(
        new StatusTable.SummaryInfo("Starting Url(s)", ColumnDescriptor.TYPE_STRING, startUrls));
    return res;
  }

  private void addIfNonZero(List res, String head, int val) {
    if (val != 0) {
      res.add(new StatusTable.SummaryInfo(head, ColumnDescriptor.TYPE_INT, new Long(val)));
    }
  }
}
Beispiel #30
0
 private ARCWriter makeARCWriter() {
   return new ARCWriter(
       serialNo, ListUtil.list(dir), prefix, compress, maxSize >= 0 ? maxSize : Long.MAX_VALUE);
 }