public void testBaseUrlPath() throws Exception {
    sau1 = setupSimAu(simAuConfig(tempDirPath));
    createContent(sau1);
    crawlContent(sau1);
    CachedUrlSet cus1 = sau1.getAuCachedUrlSet();

    tempDirPath2 = getTempDir().getAbsolutePath() + File.separator;
    Configuration config2 = simAuConfig(tempDirPath2);
    config2.put("base_url", "http://anotherhost.org/some/path/");
    SimulatedArchivalUnit sau2 = setupSimAu(config2);
    createContent(sau2);
    crawlContent(sau2);
    CachedUrlSet cus2 = sau1.getAuCachedUrlSet();
    List urls1 = auUrls(sau1);
    List urls2 = auUrls(sau2);

    Pattern pat1 = Pattern.compile("http://www\\.example\\.com(/.*)$");
    Pattern pat2 = Pattern.compile("http://anotherhost\\.org/some/path(/.*)$");
    List<String> l1 = auUrls(sau1);
    List<String> l2 = auUrls(sau2);
    assertEquals(l1.size(), l2.size());
    for (int ix = 0; ix < l1.size(); ix++) {
      Matcher m1 = pat1.matcher(l1.get(ix));
      assertTrue(m1.matches());
      Matcher m2 = pat2.matcher(l2.get(ix));
      assertTrue(m2.matches());
      assertEquals(m1.group(1), m2.group(1));
    }
  }
Пример #2
0
 // Break the line at commas, return a map of the resulting strings
 // broken at equals sign.  (<i>Ie</i>, name value pairs.)
 Map getRow(String line) {
   Map map = new HashMap();
   for (Iterator iter = StringUtil.breakAt(line, ',').iterator(); iter.hasNext(); ) {
     String item = (String) iter.next();
     List pair = StringUtil.breakAt(item, '=');
     map.put(pair.get(0), pair.get(1));
   }
   return map;
 }
Пример #3
0
 // test completion & callback
 public void testDone() throws Exception {
   HashQueue q = new HashQueue();
   final List cookieList = new LinkedList();
   final List eList = new LinkedList();
   HashService.Callback cb =
       new HashService.Callback() {
         public void hashingFinished(
             CachedUrlSet urlset,
             long timeUsed,
             Object cookie,
             CachedUrlSetHasher hasher,
             Exception e) {
           cookieList.add(cookie);
           eList.add(e);
         }
       };
   HashQueue.Request r1, r2, r3, r4, r5;
   r1 = req(2000, 0, 100, cb);
   r2 = req(10000, 0, 200, cb);
   r3 = req(20000, 0, 0, cb);
   r4 = req(50000, 0, 1, cb);
   assertTrue(q.insert(r1));
   assertTrue(q.insert(r2));
   assertTrue(q.insert(r4));
   assertEquals(0, cookieList.size());
   q.removeCompleted();
   assertEquals(0, cookieList.size());
   // make r1 timeout
   r1.deadline.expire();
   q.removeCompleted();
   List exp = ListUtil.list(r1);
   assertEquals(exp, cookieList);
   assertEquals(exp, q.getCompletedSnapshot());
   // make r2 timeout
   TimeBase.step(11000);
   // r3 is finished
   assertTrue(q.insert(r3));
   Exception r4e = new Exception();
   // make r4 error
   r4.e = r4e;
   q.removeCompleted();
   // check that they all finished, and in the right order
   Object exp2[] = {r1, r2, r3, r4};
   assertIsomorphic(exp2, cookieList);
   assertIsomorphic(exp2, q.getCompletedSnapshot());
   // check their exceptions
   assertTrue(eList.get(0) instanceof HashService.Timeout);
   assertTrue(eList.get(1) instanceof HashService.Timeout);
   assertSame(null, eList.get(2));
   assertSame(r4e, eList.get(3));
 }
  /**
   * Method that creates a simulated Cached URL from the source code provided by the goodContent
   * String. It then asserts that the metadata extracted, by using the
   * MetaPressRisMetadataExtractorFactory, match the metadata in the source code.
   *
   * @throws Exception
   */
  public void testExtractGoodRisContent() throws Exception {
    String goodContent = createGoodRisContent();
    log.debug3(goodContent);

    List<ArticleMetadata> mdlist = setupContentForAU(bau1, RIS_URL, goodContent, false);
    assertNotEmpty(mdlist);
    ArticleMetadata md = mdlist.get(0);
    assertNotNull(md);

    assertEquals(goodVolume, md.get(MetadataField.FIELD_VOLUME));
    assertEquals(goodIssue, md.get(MetadataField.FIELD_ISSUE));
    assertEquals(goodStartPage, md.get(MetadataField.FIELD_START_PAGE));
    assertEquals(goodEndPage, md.get(MetadataField.FIELD_END_PAGE));
    assertEquals(goodIssn, md.get(MetadataField.FIELD_ISSN));
    Iterator<String> actAuthIter = md.getList(MetadataField.FIELD_AUTHOR).iterator();
    for (String expAuth : goodAuthors) {
      assertEquals(expAuth, actAuthIter.next());
    }
    assertEquals(goodTitle, md.get(MetadataField.FIELD_ARTICLE_TITLE));
    assertEquals(goodJournal, md.get(MetadataField.FIELD_PUBLICATION_TITLE));
    assertEquals(goodDate, md.get(MetadataField.FIELD_DATE));

    assertEquals(goodPublisher, md.get(MetadataField.FIELD_PUBLISHER));
    assertEquals(goodDOI, md.get(MetadataField.FIELD_DOI));
    // This shouldn't get set. It will default later to fuill_text_cu
    assertNotEquals(doiURL, md.get(MetadataField.FIELD_ACCESS_URL));
  }
Пример #5
0
  public void testStepperThrows() {
    final List finished = new ArrayList();
    TaskCallback cb =
        new TaskCallback() {
          public void taskEvent(SchedulableTask task, Schedule.EventType event) {
            if (event == Schedule.EventType.FINISH) {
              finished.add(task);
            }
          }
        };

    MyMockStepper stepper = new MyMockStepper(10, -10);
    stepper.setWhenToThrow(5);
    StepTask t1 = task(100, 200, 100, cb, stepper);
    Schedule s = sched(ListUtil.list(t1));
    fact.setResult(s);
    assertTrue(tr.addToSchedule(t1));
    TimeBase.setSimulated(101);
    assertTrue(tr.findTaskToRun());
    Interrupter intr = null;
    try {
      intr = interruptMeIn(TIMEOUT_SHOULDNT, true);
      tr.runSteps(new MutableBoolean(true), null);
      intr.cancel();
    } catch (Exception e) {
      log.error("runSteps threw:", e);
    } finally {
      if (intr.did()) {
        fail("runSteps looped");
      }
    }
    assertSame(t1, finished.get(0));
    assertTrue(t1.e instanceof ExpectedRuntimeException);
    assertEquals(5, stepper.nSteps);
  }
Пример #6
0
  // remove task-ending chunk, past task deadline, s.b. Timeout error.
  public void testRemoveChunkTaskEndTimeout() {
    final List finished = new ArrayList();
    StepTask t1 =
        task(
            100,
            200,
            100,
            new TaskCallback() {
              public void taskEvent(SchedulableTask task, Schedule.EventType event) {
                if (log.isDebug2()) {
                  log.debug2("testRemoveChunkTaskEndTimeout callback");
                }
                if (event == Schedule.EventType.FINISH) {
                  finished.add(task);
                }
              }
            });
    Schedule s = sched(ListUtil.list(t1));
    fact.setResult(s);
    assertTrue(tr.addToSchedule(t1));

    Schedule.Chunk chunk = (Schedule.Chunk) s.getEvents().get(0);
    assertTrue(tr.getCurrentSchedule().getEvents().contains(chunk));
    chunk.setTaskEnd();
    TimeBase.setSimulated(201);
    tr.removeChunk(chunk);
    assertFalse(tr.getCurrentSchedule().getEvents().contains(chunk));
    assertSame(t1, finished.get(0));
    assertNotNull(t1.e);
    assertTrue(t1.e.toString(), t1.e instanceof SchedService.Timeout);
    assertEmpty(tr.getAcceptedTasks());
  }
 public void testSimpleMainXML() throws Exception {
   log.debug3("testSimpleMainXML");
   String xml_url = TAR_A_BASE + SUBDIR + "01420615/v64sC/S0142061514004608/main.xml";
   List<ArticleMetadata> mdList =
       extractFromContent(xml_url, "text/xml", simpleMain, nocheck_mle, null);
   assertEquals(1, mdList.size());
   validateSingleMainMetadataRecord(mdList.get(0), "10.1016/j.jidx.2014.07.028", "article");
 }
Пример #8
0
 protected void assertEqualTables(Object[][] a1, List lines) {
   assertEquals("numrows", a1.length, lines.size() - NUM_HEADER_LINES);
   for (int irow = 0; irow <= a1.length - 1; irow++) {
     Object expRow[] = a1[irow];
     List row = StringUtil.breakAt((String) lines.get(irow + NUM_HEADER_LINES), ',');
     assertEquals("numcols", expRow.length, row.size());
     assertEquals(("row " + irow), SetUtil.fromArray(expRow), new HashSet(row));
   }
 }
Пример #9
0
  public void testText() throws Exception {
    MockStatusAccessor statusAccessor =
        MockStatusAccessor.generateStatusAccessor(colArray1, rowArray1);
    statusAccessor.setTitle("testtbl", null);
    statSvc.registerStatusAccessor("testtbl", statusAccessor);

    WebResponse resp = getTable("testtbl", true);
    assertResponseOk(resp);
    assertEquals("Content type", "text/plain", resp.getContentType());
    log.debug(resp.getText());
    List lines = getLines(resp);
    assertEquals(rowArray1.length + 3, lines.size());
    Map row0 = getRow((String) lines.get(0));
    assertEquals("2.4.6.8", row0.get("host"));

    Map row2 = getRow((String) lines.get(2));
    assertEquals("testtbl", row2.get("table"));

    assertEqualTables(table1, lines);
  }
  public void testDOIExtraction() throws Exception {

    List<ArticleMetadata> mdlist =
        setupContentForAU(bau1, ABS_URL, goodHtmlContentNoDOIorPublisher, true);
    assertNotEmpty(mdlist);
    ArticleMetadata md = mdlist.get(0);
    assertNotNull(md);
    // gets pulled from the URL if not set in the metadata
    assertEquals("10.1175/2010WCAS1063.1", md.get(MetadataField.FIELD_DOI));
    // gets set manually if not in the metadata
    // first it would try the TDB
    assertEquals(goodPublisher, md.get(MetadataField.FIELD_PUBLISHER));
  }
  public void testExtractGoodHtmlContent() throws Exception {

    List<ArticleMetadata> mdlist = setupContentForAU(bau1, ABS_URL, goodHtmlContent, true);
    assertNotEmpty(mdlist);
    ArticleMetadata md = mdlist.get(0);
    assertNotNull(md);
    assertEquals(goodPublisher, md.get(MetadataField.FIELD_PUBLISHER));
    assertEquals(goodTitle, md.get(MetadataField.DC_FIELD_TITLE));
    assertEquals(goodDate, md.get(MetadataField.FIELD_DATE));
    assertEquals(goodFormat, md.get(MetadataField.DC_FIELD_FORMAT));
    assertEquals(goodType, md.get(MetadataField.DC_FIELD_TYPE));
    assertEquals(Arrays.asList(goodAuthors), md.getList(MetadataField.FIELD_AUTHOR));
    assertEquals(goodAuthors[0], md.get(MetadataField.DC_FIELD_CREATOR));
  }
Пример #12
0
  public void testFindTaskToRunRemovesExpiredOverrunners() {
    assertFalse(tr.findTaskToRun());
    StepTask t1 = task(100, 200, 100);
    StepTask t2 = task(100, 300, 50);
    StepTask texp1 = task(0, 0, 50);
    StepTask texp2 = task(0, 0, 49);

    Schedule s = sched(ListUtil.list(t1, t2));
    fact.setResults(s, s);
    assertTrue(tr.addToSchedule(t1));
    assertTrue(tr.addToSchedule(t2));
    tr.addOverrunner(texp1);
    tr.addOverrunner(texp2);

    // if this fails, it might be because the sorted list/set is treating
    // sort-order equivalence as object equality, which we don't want
    assertEquals(2, tr.getOverrunTasks().size());
    assertFalse(tr.findTaskToRun());
    assertEquals(0, removedChunks.size());
    assertEquals(2, removedTasks.size());
    assertEquals(
        SetUtil.set(texp1, texp2),
        SetUtil.set((StepTask) removedTasks.get(0), (StepTask) removedTasks.get(1)));
  }
  /**
   * Method that creates a simulated Cached URL from the source code provided by the goodContent
   * String. It then asserts that the metadata extracted, by using the
   * MetaPressRisMetadataExtractorFactory, match the metadata in the source code.
   *
   * @throws Exception
   */
  public void testExtractAlternateRisContent() throws Exception {
    String goodContent = createAlternateRisContent();
    log.debug3(goodContent);

    List<ArticleMetadata> mdlist = setupContentForAU(bau1, RIS_URL, goodContent, false);
    assertNotEmpty(mdlist);
    ArticleMetadata md = mdlist.get(0);
    assertNotNull(md);

    Iterator<String> actAuthIter = md.getList(MetadataField.FIELD_AUTHOR).iterator();
    for (String expAuth : goodAuthors) {
      assertEquals(expAuth, actAuthIter.next());
    }
    assertEquals(goodTitle, md.get(MetadataField.FIELD_ARTICLE_TITLE));
    assertEquals(goodJournal, md.get(MetadataField.FIELD_PUBLICATION_TITLE));
    assertEquals(goodDate, md.get(MetadataField.FIELD_DATE));
    assertEquals(goodPublisher, md.get(MetadataField.FIELD_PUBLISHER));
  }
Пример #14
0
  public void testFindTaskToRunRemovesExpiredChunks() {
    assertFalse(tr.findTaskToRun());
    StepTask t1 = task(100, 200, 100);
    StepTask t2 = task(100, 300, 50);
    StepTask texp1 = task(0, 0, 50);
    StepTask texp2 = task(0, 0, 50);

    Schedule s = sched(ListUtil.list(texp1, texp2, t1, t2));
    fact.setResults(s, s);
    assertTrue(tr.addToSchedule(t1));
    assertTrue(tr.addToSchedule(t2));
    assertFalse(tr.findTaskToRun());
    assertEquals(2, removedChunks.size());
    assertEquals(
        SetUtil.set(texp1, texp2),
        SetUtil.set(
            ((Schedule.Chunk) removedChunks.get(0)).getTask(),
            ((Schedule.Chunk) removedChunks.get(1)).getTask()));
  }
  public void testStorePollHistories() throws Exception {
    TimeBase.setSimulated(123321);
    MockCachedUrlSetSpec mspec = new MockCachedUrlSetSpec("http://www.example.com", null);
    CachedUrlSet mcus = new MockCachedUrlSet(mau, mspec);
    NodeStateImpl nodeState = new NodeStateImpl(mcus, -1, null, null, repository);
    List histories =
        ListUtil.list(
            createPollHistoryBean(3),
            createPollHistoryBean(3),
            createPollHistoryBean(3),
            createPollHistoryBean(3),
            createPollHistoryBean(3));

    /*
     * CASTOR: [summary] Rewrite test in non-Castor way
     * This is obviously not an appropriate way of writing this test,
     * Right now it creates sample data in Castor format, from legacy
     * code back when Castor was the built-in serialization engine.
     * TODO: Rewrite test in non-Castor way
     */
    // nodeState.setPollHistoryBeanList(histories);
    nodeState.setPollHistoryList(NodeHistoryBean.fromBeanListToList(histories));

    repository.storePollHistories(nodeState);
    String filePath = LockssRepositoryImpl.mapAuToFileLocation(tempDirPath, mau);
    filePath =
        LockssRepositoryImpl.mapUrlToFileLocation(
            filePath, "http://www.example.com/" + HistoryRepositoryImpl.HISTORY_FILE_NAME);
    File xmlFile = new File(filePath);
    assertTrue(xmlFile.exists());

    nodeState.setPollHistoryList(new ArrayList());
    repository.loadPollHistories(nodeState);
    List loadedHistory = nodeState.getPollHistoryList();
    assertEquals(histories.size(), loadedHistory.size());
    // CASTOR: some Castor-tailored stuff here
    // PollHistoryBean expect1 = (PollHistoryBean)histories.get(0);
    // PollHistoryBean elem1 = (PollHistoryBean)loadedHistory.get(0);
    PollHistory expect1 = (PollHistory) histories.get(0);
    PollHistory elem1 = (PollHistory) loadedHistory.get(0);
    assertEquals(expect1.type, elem1.type);
    assertEquals(expect1.lwrBound, elem1.lwrBound);
    assertEquals(expect1.uprBound, elem1.uprBound);
    assertEquals(expect1.status, elem1.status);
    assertEquals(expect1.startTime, elem1.startTime);
    assertEquals(expect1.duration, elem1.duration);
    // CASTOR: some Castor-tailored stuff here
    // List expectBeans = (List)expect1.getVoteBeans();
    // List elemBeans = (List)elem1.getVoteBeans();
    Iterator expectIter = (Iterator) expect1.getVotes();
    Iterator elemIter = (Iterator) elem1.getVotes();
    while (expectIter.hasNext() && elemIter.hasNext()) {
      Vote expectVote = (Vote) expectIter.next();
      Vote elemVote = (Vote) elemIter.next();
      assertEquals(
          expectVote.getVoterIdentity().getIdString(), elemVote.getVoterIdentity().getIdString());
      assertEquals(expectVote.isAgreeVote(), elemVote.isAgreeVote());
      assertEquals(expectVote.getChallengeString(), elemVote.getChallengeString());
      assertEquals(expectVote.getVerifierString(), elemVote.getVerifierString());
      assertEquals(expectVote.getHashString(), elemVote.getHashString());
    }
    assertFalse(expectIter.hasNext());
    assertFalse(expectIter.hasNext());
    TimeBase.setReal();
  }
Пример #16
0
  protected void initResultMap() throws PluginException.InvalidDefinition {
    HttpResultMap hResultMap = new HttpResultMap();
    // XXX Currently this only allows a CacheResultHandler class to
    // initialize the result map.  Instead, don't use a CacheResultMap
    // directly, use either the plugin's CacheResultHandler, if specified,
    // or a default one that wraps the CacheResultMap

    String handler_class = null;
    handler_class = definitionMap.getString(KEY_EXCEPTION_HANDLER, null);
    if (handler_class != null) {
      try {
        resultHandler = (CacheResultHandler) newAuxClass(handler_class, CacheResultHandler.class);
        resultHandler.init(hResultMap);
      } catch (Exception ex) {
        throw new PluginException.InvalidDefinition(
            mapName + " has invalid Exception handler: " + handler_class, ex);
      } catch (LinkageError le) {
        throw new PluginException.InvalidDefinition(
            mapName + " has invalid Exception handler: " + handler_class, le);
      }
    } else {
      // Expect a list of mappings from either result code or exception
      // name to CacheException name
      Collection<String> mappings = definitionMap.getCollection(KEY_EXCEPTION_LIST, null);
      if (mappings != null) {
        // add each entry
        for (String entry : mappings) {
          if (log.isDebug2()) {
            log.debug2("initMap(" + entry + ")");
          }
          String first;
          String ceName;
          try {
            List<String> pair = StringUtil.breakAt(entry, '=', 2, true, true);
            first = pair.get(0);
            ceName = pair.get(1);
          } catch (Exception ex) {
            throw new PluginException.InvalidDefinition(
                "Invalid syntax: " + entry + "in " + mapName);
          }
          Object val;

          // Value should be either a CacheException or CacheResultHandler
          // class name.
          PluginFetchEventResponse resp =
              (PluginFetchEventResponse) newAuxClass(ceName, PluginFetchEventResponse.class, null);
          if (resp instanceof CacheException) {
            val = resp.getClass();
          } else if (resp instanceof CacheResultHandler) {
            val = WrapperUtil.wrap((CacheResultHandler) resp, CacheResultHandler.class);
          } else {
            throw new PluginException.InvalidDefinition(
                "Second arg not a "
                    + "CacheException or "
                    + "CacheResultHandler class: "
                    + entry
                    + ", in "
                    + mapName);
          }
          try {
            int code = Integer.parseInt(first);
            // If parseable as an integer, it's a result code.
            hResultMap.storeMapEntry(code, val);
          } catch (NumberFormatException e) {
            try {
              Class eClass = Class.forName(first);
              // If a class name, it should be an exception class
              if (Exception.class.isAssignableFrom(eClass)) {
                hResultMap.storeMapEntry(eClass, val);
              } else {
                throw new PluginException.InvalidDefinition(
                    "First arg not an " + "Exception class: " + entry + ", in " + mapName);
              }
            } catch (Exception ex) {
              throw new PluginException.InvalidDefinition(
                  "First arg not a " + "number or class: " + entry + ", in " + mapName);
            } catch (LinkageError le) {
              throw new PluginException.InvalidDefinition("Can't load " + first, le);
            }
          }
        }
      }
    }
    resultMap = hResultMap;
  }