Esempio n. 1
0
/** Export to an ARC file */
public class ArcExporter extends Exporter {

  private static Logger log = Logger.getLogger("ArcExporter");

  protected CIProperties arcProps = null;
  String arcFilePrefix = "SimulatedCrawl";
  AtomicInteger serialNo = new AtomicInteger(0);
  ARCWriter aw;
  boolean isResponse;

  public ArcExporter(LockssDaemon daemon, ArchivalUnit au, boolean isResponse) {
    super(daemon, au);
    this.isResponse = isResponse;
  }

  protected void start() {
    aw = makeARCWriter();
  }

  protected void finish() throws IOException {
    aw.close();
  }

  private ARCWriter makeARCWriter() {
    return new ARCWriter(
        serialNo, ListUtil.list(dir), prefix, compress, maxSize >= 0 ? maxSize : Long.MAX_VALUE);
  }

  protected void writeCu(CachedUrl cu) throws IOException {
    String url = cu.getUrl();
    long contentSize = cu.getContentSize();
    CIProperties props = cu.getProperties();
    long fetchTime = Long.parseLong(props.getProperty(CachedUrl.PROPERTY_FETCH_TIME));
    InputStream contentIn = cu.getUnfilteredInputStream();
    try {
      if (isResponse) {
        String hdrString = getHttpResponseString(cu);
        long size = contentSize + hdrString.length();
        InputStream headerIn = new ReaderInputStream(new StringReader(hdrString));
        InputStream concat = new SequenceInputStream(headerIn, contentIn);
        try {
          aw.write(xlateFilename(url), cu.getContentType(), getHostIp(), fetchTime, size, concat);
        } finally {
          IOUtil.safeClose(concat);
        }
      } else {
        aw.write(
            xlateFilename(url),
            cu.getContentType(),
            getHostIp(),
            fetchTime,
            cu.getContentSize(),
            contentIn);
      }
    } finally {
      AuUtil.safeRelease(cu);
    }
  }
}
Esempio n. 2
0
/** JUnitTest case for class: org.lockss.poller.Poll */
public class TestPoll extends LockssTestCase {
  private static Logger log = Logger.getLogger("TestPoll");
  private static String[] rootV1urls = {
    "http://www.test.org", "http://www.test1.org", "http://www.test2.org"
  };
  private static String lwrbnd = "test1.doc";
  private static String uprbnd = "test3.doc";
  private static long testduration = Constants.DAY;

  //   protected ArchivalUnit testau =
  //       PollTestPlugin.PTArchivalUnit.createFromListOfRootUrls(rootV1urls);
  protected MockArchivalUnit testau;
  private IdentityManager idmgr;
  private MockLockssDaemon theDaemon;
  private ArrayList agree_entries = makeEntries(10, 50);
  private ArrayList disagree_entries = makeEntries(15, 57);
  private ArrayList dissenting_entries = makeEntries(7, 50);

  protected PeerIdentity testID;
  protected PeerIdentity testID1;
  protected V1LcapMessage[] testV1msg;
  protected V1Poll[] testV1polls;
  protected PollManager pollmanager;

  protected void setUp() throws Exception {
    super.setUp();
    TimeBase.setSimulated();

    initRequiredServices();

    testau.setPlugin(new MockPlugin());

    initTestPeerIDs();
    initTestMsg();
    initTestPolls();
  }

  /**
   * tearDown method for test case
   *
   * @throws Exception if removePoll failed
   */
  public void tearDown() throws Exception {
    pollmanager.stopService();
    theDaemon.getLockssRepository(testau).stopService();
    theDaemon.getHashService().stopService();
    theDaemon.getDatagramRouterManager().stopService();
    theDaemon.getRouterManager().stopService();
    theDaemon.getSystemMetrics().stopService();
    TimeBase.setReal();
    for (int i = 0; i < testV1msg.length; i++) {
      if (testV1msg[i] != null) pollmanager.removePoll(testV1msg[i].getKey());
    }
    super.tearDown();
  }

  /** test for method scheduleVote(..) */
  public void testScheduleVote() {
    V1Poll p = testV1polls[1];
    assertTrue(p instanceof V1ContentPoll);
    log.debug3("testScheduleVote 1");
    p.scheduleVote();
    log.debug3("testScheduleVote 2");
    assertNotNull(p.m_voteTime);
    assertTrue(p.m_voteTime.getRemainingTime() < p.m_deadline.getRemainingTime());
    log.debug3("at end of testScheduleVote");
  }

  /** test for method checkVote(..) */
  public void testCheckVote() throws Exception {
    V1LcapMessage msg = null;
    log.debug3("starting testCheeckVote");
    msg =
        V1LcapMessage.makeReplyMsg(
            testV1polls[0].getMessage(),
            ByteArray.makeRandomBytes(20),
            ByteArray.makeRandomBytes(20),
            null,
            V1LcapMessage.NAME_POLL_REP,
            testduration,
            testID);
    log.debug3("testCheeckVote 2");
    V1Poll p = null;
    p = createCompletedPoll(theDaemon, testau, msg, 8, 2, pollmanager);
    assertTrue(p instanceof V1NamePoll);
    log.debug3("testCheeckVote 3");
    assertNotNull(p);
    PeerIdentity id = msg.getOriginatorId();
    assertNotNull(id);
    assertNotNull(p.m_tally);
    int rep = p.m_tally.wtAgree + idmgr.getReputation(id);

    // good vote check

    p.checkVote(msg.getHashed(), new Vote(msg, false));
    assertEquals(9, p.m_tally.numAgree);
    assertEquals(2, p.m_tally.numDisagree);
    assertEquals(rep, p.m_tally.wtAgree);

    rep = p.m_tally.wtDisagree + idmgr.getReputation(id);

    // bad vote check
    p.checkVote(ByteArray.makeRandomBytes(20), new Vote(msg, false));
    assertEquals(9, p.m_tally.numAgree);
    assertEquals(3, p.m_tally.numDisagree);
    assertEquals(rep, p.m_tally.wtDisagree);
  }

  /** test for method tally(..) */
  public void testTally() {
    V1Poll p = testV1polls[0];
    LcapMessage msg = p.getMessage();
    PeerIdentity id = msg.getOriginatorId();
    p.m_tally.addVote(new Vote(msg, false), id, false);
    p.m_tally.addVote(new Vote(msg, false), id, false);
    p.m_tally.addVote(new Vote(msg, false), id, false);
    assertEquals(0, p.m_tally.numAgree);
    assertEquals(0, p.m_tally.wtAgree);
    assertEquals(3, p.m_tally.numDisagree);
    assertEquals(1500, p.m_tally.wtDisagree);

    p = testV1polls[1];
    msg = p.getMessage();
    p.m_tally.addVote(new Vote(msg, true), id, false);
    p.m_tally.addVote(new Vote(msg, true), id, false);
    p.m_tally.addVote(new Vote(msg, true), id, false);
    assertEquals(3, p.m_tally.numAgree);
    assertEquals(1500, p.m_tally.wtAgree);
    assertEquals(0, p.m_tally.numDisagree);
    assertEquals(0, p.m_tally.wtDisagree);
  }

  public void testNamePollTally() throws Exception {
    V1NamePoll np;
    // test a name poll we won
    np = makeCompletedNamePoll(4, 1, 0);
    assertEquals(5, np.m_tally.numAgree);
    assertEquals(1, np.m_tally.numDisagree);
    assertEquals(Tallier.RESULT_WON, np.m_tally.getTallyResult());

    // test a name poll we lost with a dissenting vote
    np = makeCompletedNamePoll(1, 8, 1);

    assertEquals(2, np.m_tally.numAgree);
    assertEquals(9, np.m_tally.numDisagree);

    // build a master list
    np.buildPollLists(np.m_tally.pollVotes.iterator());

    // these should be different since we lost the poll
    assertFalse(CollectionUtil.isIsomorphic(np.m_tally.localEntries, np.m_tally.votedEntries));

    // the expected "correct" set is in our disagree msg
    assertTrue(CollectionUtil.isIsomorphic(disagree_entries, np.m_tally.votedEntries));
  }

  /** test for method vote(..) */
  public void testVote() {
    V1Poll p = testV1polls[1];
    p.m_hash = ByteArray.makeRandomBytes(20);
    try {
      p.castOurVote();
    } catch (IllegalStateException e) {
      // the socket isn't inited and should squack
    }
    p.m_pollstate = V1Poll.PS_COMPLETE;
  }

  /** test for method voteInPoll(..) */
  public void testVoteInPoll() {
    V1Poll p = testV1polls[1];
    p.m_tally.quorum = 10;
    p.m_tally.numAgree = 5;
    p.m_tally.numDisagree = 2;
    p.m_tally.wtAgree = 2000;
    p.m_tally.wtDisagree = 200;
    p.m_hash = ByteArray.makeRandomBytes(20);
    try {
      p.voteInPoll();
    } catch (IllegalStateException e) {
      // the socket isn't inited and should squack
    }

    p.m_tally.numAgree = 20;
    try {
      p.voteInPoll();
    } catch (NullPointerException npe) {
      // the socket isn't inited and should squack
    }
    p.m_pollstate = V1Poll.PS_COMPLETE;
  }

  public void testStartPoll() {
    V1Poll p = testV1polls[0];
    p.startPoll();
    assertEquals(V1Poll.PS_WAIT_HASH, p.m_pollstate);
    p.m_pollstate = V1Poll.PS_COMPLETE;
  }

  public void testScheduleOurHash() {
    V1Poll p = testV1polls[0];
    p.m_pollstate = V1Poll.PS_WAIT_HASH;
    // no time has elapsed - so we should be able to schedule our hash
    assertTrue(p.scheduleOurHash());
    // half the time has elapsed so we should be able to schedule our hash
    TimeBase.step(p.m_deadline.getRemainingTime() / 2);
    assertTrue(p.scheduleOurHash());

    // all of the time has elapsed we should not be able to schedule our hash
    TimeBase.step(p.m_deadline.getRemainingTime() - 1000);
    assertFalse(p.scheduleOurHash());
    p.m_pollstate = V1Poll.PS_COMPLETE;
  }

  /** test for method stopPoll(..) */
  public void testStopPoll() {
    V1Poll p = testV1polls[1];
    p.m_tally.quorum = 10;
    p.m_tally.numAgree = 7;
    p.m_tally.numDisagree = 3;
    p.m_pollstate = V1Poll.PS_WAIT_TALLY;
    p.stopPoll();
    assertTrue(p.m_pollstate == V1Poll.PS_COMPLETE);
    p.startPoll();
    assertTrue(p.m_pollstate == V1Poll.PS_COMPLETE);
  }

  /** test for method startVoteCheck(..) */
  public void testStartVote() {
    V1Poll p = testV1polls[0];
    p.m_pendingVotes = 3;
    p.startVoteCheck();
    assertEquals(4, p.m_pendingVotes);
    p.m_pollstate = V1Poll.PS_COMPLETE;
  }

  /** test for method stopVote(..) */
  public void testStopVote() {
    V1Poll p = testV1polls[1];
    p.m_pendingVotes = 3;
    p.stopVoteCheck();
    assertEquals(2, p.m_pendingVotes);
    p.m_pollstate = V1Poll.PS_COMPLETE;
  }

  private V1NamePoll makeCompletedNamePoll(int numAgree, int numDisagree, int numDissenting)
      throws Exception {
    V1NamePoll np = null;
    V1LcapMessage agree_msg = null;
    V1LcapMessage disagree_msg1 = null;
    V1LcapMessage disagree_msg2 = null;

    Plugin plugin = testau.getPlugin();
    PollSpec spec = new MockPollSpec(testau, rootV1urls[0], null, null, Poll.V1_NAME_POLL);
    ((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
    V1LcapMessage poll_msg =
        V1LcapMessage.makeRequestMsg(
            spec,
            null,
            ByteArray.makeRandomBytes(20),
            ByteArray.makeRandomBytes(20),
            V1LcapMessage.NAME_POLL_REQ,
            testduration,
            testID);

    // make our poll
    np =
        (V1NamePoll)
            new V1NamePoll(
                spec,
                pollmanager,
                poll_msg.getOriginatorId(),
                poll_msg.getChallenge(),
                poll_msg.getDuration(),
                poll_msg.getHashAlgorithm());
    np.setMessage(poll_msg);

    // generate agree vote msg
    agree_msg =
        V1LcapMessage.makeReplyMsg(
            poll_msg,
            ByteArray.makeRandomBytes(20),
            poll_msg.getVerifier(),
            agree_entries,
            V1LcapMessage.NAME_POLL_REP,
            testduration,
            testID);

    // generate a disagree vote msg
    disagree_msg1 =
        V1LcapMessage.makeReplyMsg(
            poll_msg,
            ByteArray.makeRandomBytes(20),
            ByteArray.makeRandomBytes(20),
            disagree_entries,
            V1LcapMessage.NAME_POLL_REP,
            testduration,
            testID1);
    // generate a losing disagree vote msg
    disagree_msg2 =
        V1LcapMessage.makeReplyMsg(
            poll_msg,
            ByteArray.makeRandomBytes(20),
            ByteArray.makeRandomBytes(20),
            dissenting_entries,
            V1LcapMessage.NAME_POLL_REP,
            testduration,
            testID1);

    // add our vote
    V1LcapMessage msg = (V1LcapMessage) (np.getMessage());
    PeerIdentity id = msg.getOriginatorId();
    np.m_tally.addVote(np.makeNameVote(msg, true), id, true);

    // add the agree votes
    id = agree_msg.getOriginatorId();
    for (int i = 0; i < numAgree; i++) {
      np.m_tally.addVote(np.makeNameVote(agree_msg, true), id, false);
    }

    // add the disagree votes
    id = disagree_msg1.getOriginatorId();
    for (int i = 0; i < numDisagree; i++) {
      np.m_tally.addVote(np.makeNameVote(disagree_msg1, false), id, false);
    }

    // add dissenting disagree vote
    id = disagree_msg2.getOriginatorId();
    for (int i = 0; i < numDissenting; i++) {
      np.m_tally.addVote(np.makeNameVote(disagree_msg2, false), id, false);
    }
    np.m_pollstate = V1Poll.PS_COMPLETE;
    np.m_tally.tallyVotes();
    return np;
  }

  public static V1Poll createCompletedPoll(
      LockssDaemon daemon,
      ArchivalUnit au,
      V1LcapMessage testmsg,
      int numAgree,
      int numDisagree,
      PollManager pollmanager)
      throws Exception {
    log.debug(
        "createCompletedPoll: au: "
            + au.toString()
            + " peer "
            + testmsg.getOriginatorId()
            + " votes "
            + numAgree
            + "/"
            + numDisagree);
    CachedUrlSetSpec cusSpec = null;
    if ((testmsg.getLwrBound() != null)
        && (testmsg.getLwrBound().equals(PollSpec.SINGLE_NODE_LWRBOUND))) {
      cusSpec = new SingleNodeCachedUrlSetSpec(testmsg.getTargetUrl());
    } else {
      cusSpec =
          new RangeCachedUrlSetSpec(
              testmsg.getTargetUrl(), testmsg.getLwrBound(), testmsg.getUprBound());
    }
    CachedUrlSet cus = au.makeCachedUrlSet(cusSpec);
    PollSpec spec = new PollSpec(cus, Poll.V1_CONTENT_POLL);
    ((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
    V1Poll p = null;
    if (testmsg.isContentPoll()) {
      p =
          new V1ContentPoll(
              spec,
              pollmanager,
              testmsg.getOriginatorId(),
              testmsg.getChallenge(),
              testmsg.getDuration(),
              testmsg.getHashAlgorithm());
    } else if (testmsg.isNamePoll()) {
      p =
          new V1NamePoll(
              spec,
              pollmanager,
              testmsg.getOriginatorId(),
              testmsg.getChallenge(),
              testmsg.getDuration(),
              testmsg.getHashAlgorithm());
    } else if (testmsg.isVerifyPoll()) {
      p =
          new V1VerifyPoll(
              spec,
              pollmanager,
              testmsg.getOriginatorId(),
              testmsg.getChallenge(),
              testmsg.getDuration(),
              testmsg.getHashAlgorithm(),
              testmsg.getVerifier());
    }
    assertNotNull(p);
    p.setMessage(testmsg);
    p.m_tally.quorum = numAgree + numDisagree;
    p.m_tally.numAgree = numAgree;
    p.m_tally.numDisagree = numDisagree;
    p.m_tally.wtAgree = 2000;
    p.m_tally.wtDisagree = 200;
    p.m_tally.localEntries = makeEntries(1, 3);
    p.m_tally.votedEntries = makeEntries(1, 5);
    p.m_tally.votedEntries.remove(1);
    p.m_pollstate = V1Poll.PS_COMPLETE;
    p.m_callerID = testmsg.getOriginatorId();
    log.debug3("poll " + p.toString());
    p.m_tally.tallyVotes();
    return p;
  }

  public static ArrayList makeEntries(int firstEntry, int lastEntry) {
    int numEntries = lastEntry - firstEntry + 1;
    ArrayList ret_arry = new ArrayList(numEntries);

    for (int i = 0; i < numEntries; i++) {
      String name = "/testentry" + (firstEntry + i) + ".html";
      ret_arry.add(new PollTally.NameListEntry(i % 2 == 1, name));
    }

    return ret_arry;
  }

  private void initRequiredServices() {
    theDaemon = getMockLockssDaemon();
    pollmanager = new LocalPollManager();
    pollmanager.initService(theDaemon);
    theDaemon.setPollManager(pollmanager);

    theDaemon.getPluginManager();
    testau = PollTestPlugin.PTArchivalUnit.createFromListOfRootUrls(rootV1urls);
    PluginTestUtil.registerArchivalUnit(testau);

    String tempDirPath = null;
    try {
      tempDirPath = getTempDir().getAbsolutePath() + File.separator;
    } catch (IOException ex) {
      fail("unable to create a temporary directory");
    }

    Properties p = new Properties();
    p.setProperty(IdentityManager.PARAM_IDDB_DIR, tempDirPath + "iddb");
    p.setProperty(LockssRepositoryImpl.PARAM_CACHE_LOCATION, tempDirPath);
    p.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath);
    p.setProperty(IdentityManager.PARAM_LOCAL_IP, "127.0.0.1");
    p.setProperty(ConfigManager.PARAM_NEW_SCHEDULER, "false");
    // XXX we need to disable verification of votes because the
    // voter isn't really there
    p.setProperty(V1Poll.PARAM_AGREE_VERIFY, "0");
    p.setProperty(V1Poll.PARAM_DISAGREE_VERIFY, "0");
    ConfigurationUtil.setCurrentConfigFromProps(p);
    idmgr = theDaemon.getIdentityManager();
    idmgr.startService();
    // theDaemon.getSchedService().startService();
    theDaemon.getHashService().startService();
    theDaemon.getDatagramRouterManager().startService();
    theDaemon.getRouterManager().startService();
    theDaemon.getSystemMetrics().startService();
    theDaemon.getActivityRegulator(testau).startService();
    theDaemon.setNodeManager(new MockNodeManager(), testau);
    pollmanager.startService();
  }

  private void initTestPeerIDs() {
    try {
      testID = idmgr.stringToPeerIdentity("127.0.0.1");
      testID1 = idmgr.stringToPeerIdentity("1.1.1.1");
    } catch (IdentityManager.MalformedIdentityKeyException ex) {
      fail("can't open test host");
    }
  }

  private void initTestMsg() throws Exception {
    testV1msg = new V1LcapMessage[3];
    int[] pollType = {
      Poll.V1_NAME_POLL, Poll.V1_CONTENT_POLL, Poll.V1_VERIFY_POLL,
    };
    PollFactory ppf = pollmanager.getPollFactory(1);
    assertNotNull("PollFactory should not be null", ppf);
    // XXX V1 support mandatory
    assertTrue(ppf instanceof V1PollFactory);
    V1PollFactory pf = (V1PollFactory) ppf;

    for (int i = 0; i < testV1msg.length; i++) {
      PollSpec spec = new MockPollSpec(testau, rootV1urls[i], lwrbnd, uprbnd, pollType[i]);
      log.debug("Created poll spec: " + spec);
      ((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
      int opcode = V1LcapMessage.NAME_POLL_REQ + (i * 2);
      long duration = -1;
      //  NB calcDuration is not applied to Verify polls.
      switch (opcode) {
        case V1LcapMessage.NAME_POLL_REQ:
        case V1LcapMessage.CONTENT_POLL_REQ:
          // this will attempt to schedule and can return -1
          duration = Math.max(pf.calcDuration(spec, pollmanager), 1000);
          break;
        case V1LcapMessage.VERIFY_POLL_REQ:
        case V1LcapMessage.VERIFY_POLL_REP:
          duration = 100000; // Arbitrary
          break;
        default:
          fail("Bad opcode " + opcode);
          break;
      }

      testV1msg[i] =
          V1LcapMessage.makeRequestMsg(
              spec,
              agree_entries,
              pf.makeVerifier(100000),
              pf.makeVerifier(100000),
              opcode,
              duration,
              testID);
      assertNotNull(testV1msg[i]);
    }
  }

  private void initTestPolls() throws Exception {
    testV1polls = new V1Poll[testV1msg.length];
    for (int i = 0; i < testV1polls.length; i++) {
      log.debug3("initTestPolls: V1 " + i);
      BasePoll p = pollmanager.makePoll(testV1msg[i]);
      assertNotNull(p);
      assertNotNull(p.getMessage());
      log.debug("initTestPolls: V1 " + i + " returns " + p);
      assertTrue(p instanceof V1Poll);
      switch (i) {
        case 0:
          assertTrue(p instanceof V1NamePoll);
          break;
        case 1:
          assertTrue(p instanceof V1ContentPoll);
          break;
        case 2:
          assertTrue(p instanceof V1VerifyPoll);
          break;
      }
      testV1polls[i] = (V1Poll) p;
      assertNotNull(testV1polls[i]);
      log.debug3("initTestPolls: " + i + " " + p.toString());
    }
  }

  static class LocalPollManager extends PollManager {
    // ignore message sends
    public void sendMessage(V1LcapMessage msg, ArchivalUnit au) throws IOException {}
  }

  /**
   * Executes the test case
   *
   * @param argv array of Strings containing command line arguments
   */
  public static void main(String[] argv) {
    String[] testCaseList = {TestPoll.class.getName()};
    junit.swingui.TestRunner.main(testCaseList);
  }
}
/** Minimal fully functional plugin capable of serving a little static content. */
public class StaticContentPlugin extends BasePlugin implements PluginTestable {
  static Logger log = Logger.getLogger("StaticContentPlugin");

  Map cuMap = new HashMap();

  public StaticContentPlugin() {}

  public String getVersion() {
    throw new UnsupportedOperationException("Not implemented");
  }

  public String getPluginName() {
    return "Static Content";
  }

  public List getSupportedTitles() {
    throw new UnsupportedOperationException("Not implemented");
  }

  public List getLocalAuConfigDescrs() {
    return Collections.EMPTY_LIST;
    //    throw new UnsupportedOperationException("Not implemented");
  }

  protected ArchivalUnit createAu0(Configuration auConfig)
      throws ArchivalUnit.ConfigurationException {
    return new SAU(this);
  }

  public void registerArchivalUnit(ArchivalUnit au) {
    aus.add(au);
  }

  public void unregisterArchivalUnit(ArchivalUnit au) {
    aus.remove(au);
  }

  public class SAU extends BaseArchivalUnit {

    protected SAU(Plugin myPlugin) {
      super(myPlugin);
    }

    protected String makeName() {
      return "Static Content AU";
    }

    protected String makeStartUrl() {
      throw new UnsupportedOperationException("Not Implemented");
    }

    public CachedUrlSet makeCachedUrlSet(CachedUrlSetSpec cuss) {
      return new SCUS(this, cuss);
    }

    public CachedUrl makeCachedUrl(String url) {
      CachedUrl res = (CachedUrl) cuMap.get(url);
      log.debug("makeCachedUrl(" + url + ") = " + res);
      return (CachedUrl) cuMap.get(url);
    }

    public org.lockss.plugin.UrlCacher makeUrlCacher(String url) {
      throw new UnsupportedOperationException("Not implemented");
    }

    public boolean shouldBeCached(String url) {
      return cuMap.containsKey(url);
    }

    public List getNewContentCrawlUrls() {
      throw new UnsupportedOperationException("Not implemented");
    }

    public Collection getUrlStems() {
      throw new UnsupportedOperationException("Not implemented");
    }

    public CachedUrlSet cachedUrlSetFactory(ArchivalUnit owner, CachedUrlSetSpec cuss) {
      throw new UnsupportedOperationException("Not implemented");
    }

    public CachedUrl cachedUrlFactory(CachedUrlSet owner, String url) {
      throw new UnsupportedOperationException("Not implemented");
    }

    public UrlCacher urlCacherFactory(CachedUrlSet owner, String url) {
      throw new UnsupportedOperationException("Not implemented");
    }

    public String getManifestPage() {
      throw new UnsupportedOperationException("Not Implemented");
    }

    public FilterRule getFilterRule(String mimeType) {
      throw new UnsupportedOperationException("Not implemented");
    }

    /**
     * Create a CU with content and store it in AU
     *
     * @param owner the CUS owner
     * @param url the url
     * @param type the type
     * @param contents the contents
     */
    public void storeCachedUrl(CachedUrlSet owner, String url, String type, String contents) {
      SCU scu = new SCU(owner, url, type, contents);
      cuMap.put(scu.getUrl(), scu);
    }

    public void storeCachedUrl(String url, String type, String contents) {
      storeCachedUrl(null, url, type, contents);
    }

    public String toString() {
      return "[sau: " + cuMap + "]";
    }

    protected CrawlRule makeRules() {
      throw new UnsupportedOperationException("Not implemented");
    }

    /**
     * loadDefiningConfig
     *
     * @param config Configuration
     */
    protected void loadAuConfigDescrs(Configuration config) {}
  }

  public class SCU extends BaseCachedUrl {
    private String contents = null;
    private CIProperties props = new CIProperties();

    public SCU(CachedUrlSet owner, String url) {
      super(null, url);
    }

    /**
     * Create a CachedUrl with content
     *
     * @param owner the CUS owner
     * @param url the url
     * @param type the type
     * @param contents the contents
     */
    public SCU(CachedUrlSet owner, String url, String type, String contents) {
      this(owner, url);
      setContents(contents);
      props.setProperty(CachedUrl.PROPERTY_CONTENT_TYPE, type);
    }

    private void setContents(String s) {
      contents = s;
      props.setProperty("Content-Length", "" + s.length());
    }

    public String getUrl() {
      return url;
    }

    public boolean hasContent() {
      return contents != null;
    }

    public boolean isLeaf() {
      throw new UnsupportedOperationException("Not implemented");
    }

    public InputStream getUnfilteredInputStream() {
      return new StringInputStream(contents);
    }

    public InputStream openForHashing() {
      return getUnfilteredInputStream();
    }

    protected InputStream getFilteredStream() {
      throw new UnsupportedOperationException("Not implemented");
    }

    public Reader openForReading() {
      throw new UnsupportedOperationException("Not implemented");
    }

    public long getContentSize() {
      return contents == null ? 0 : contents.length();
    }

    public CIProperties getProperties() {
      return props;
    }
  }

  class SCUS extends BaseCachedUrlSet {
    public SCUS(ArchivalUnit owner, CachedUrlSetSpec spec) {
      super(owner, spec);
    }

    public void storeActualHashDuration(long elapsed, Exception err) {
      throw new UnsupportedOperationException("Not implemented");
    }

    public Iterator flatSetIterator() {
      throw new UnsupportedOperationException("Not implemented");
    }

    public Iterator contentHashIterator() {
      throw new UnsupportedOperationException("Not implemented");
    }

    public boolean isLeaf() {
      throw new UnsupportedOperationException("Not implemented");
    }

    public CachedUrlSetHasher getContentHasher(MessageDigest digest) {
      throw new UnsupportedOperationException("Not implemented");
    }

    public CachedUrlSetHasher getNameHasher(MessageDigest digest) {
      throw new UnsupportedOperationException("Not implemented");
    }

    public long estimatedHashDuration() {
      return 1000;
    }
  }
}
/** Central repository of loaded keystores */
public class LockssKeyStoreManager extends BaseLockssDaemonManager implements ConfigurableManager {

  protected static Logger log = Logger.getLogger("LockssKeyStoreManager");

  static final String PREFIX = Configuration.PREFIX + "keyMgr.";

  /** Default type for newly created keystores. */
  public static final String PARAM_DEFAULT_KEYSTORE_TYPE = PREFIX + "defaultKeyStoreType";

  public static final String DEFAULT_DEFAULT_KEYSTORE_TYPE = "JCEKS";

  /** Default keystore provider. */
  public static final String PARAM_DEFAULT_KEYSTORE_PROVIDER = PREFIX + "defaultKeyStoreProvider";

  public static final String DEFAULT_DEFAULT_KEYSTORE_PROVIDER = null;

  /**
   * Root of keystore definitions. For each keystore, pick a unique identifier and use it in place
   * of &lt;id&gt; in the following
   */
  public static final String PARAM_KEYSTORE = PREFIX + "keystore";

  /** If true the daemon will exit if a critical keystore is missing. */
  public static final String PARAM_EXIT_IF_MISSING_KEYSTORE = PREFIX + "exitIfMissingKeystore";

  public static final boolean DEFAULT_EXIT_IF_MISSING_KEYSTORE = true;

  /** keystore name, used by clients to refer to it */
  public static final String KEYSTORE_PARAM_NAME = "name";
  /** keystore file. Only one of file, resource or url should be set */
  public static final String KEYSTORE_PARAM_FILE = "file";
  /** keystore resource. Only one of file, resource or url should be set */
  public static final String KEYSTORE_PARAM_RESOURCE = "resource";
  /** keystore url. Only one of file, resource or url should be set */
  public static final String KEYSTORE_PARAM_URL = "url";
  /** keystore type */
  public static final String KEYSTORE_PARAM_TYPE = "type";
  /** keystore provider */
  public static final String KEYSTORE_PARAM_PROVIDER = "provider";
  /** keystore password */
  public static final String KEYSTORE_PARAM_PASSWORD = "******";
  /** private key password */
  public static final String KEYSTORE_PARAM_KEY_PASSWORD = "******";
  /** private key password file */
  public static final String KEYSTORE_PARAM_KEY_PASSWORD_FILE = "keyPasswordFile";
  /**
   * If true, and the keystore doesn't exist, a keystore with a self-signed certificate will be be
   * created.
   */
  public static final String KEYSTORE_PARAM_CREATE = "create";

  protected String defaultKeyStoreType = DEFAULT_DEFAULT_KEYSTORE_TYPE;
  protected String defaultKeyStoreProvider = DEFAULT_DEFAULT_KEYSTORE_PROVIDER;
  protected boolean paramExitIfMissingKeyStore = DEFAULT_EXIT_IF_MISSING_KEYSTORE;

  // Pseudo params for param doc
  public static final String DOC_PREFIX = PARAM_KEYSTORE + ".<id>.";

  /** Name by which daemon component(s) refer to this keystore */
  public static final String PARAM_KEYSTORE_NAME = DOC_PREFIX + KEYSTORE_PARAM_NAME;
  /** Keystore filename. Only one of file, resource or url should be set */
  public static final String PARAM_KEYSTORE_FILE = DOC_PREFIX + KEYSTORE_PARAM_FILE;
  /** Keystore resource. Only one of file, resource or url should be set */
  public static final String PARAM_KEYSTORE_RESOURCE = DOC_PREFIX + KEYSTORE_PARAM_RESOURCE;
  /** Keystore url. Only one of file, resource or url should be set */
  public static final String PARAM_KEYSTORE_URL = DOC_PREFIX + KEYSTORE_PARAM_URL;
  /** Keystore type (JKS, JCEKS, etc.) */
  public static final String PARAM_KEYSTORE_TYPE = DOC_PREFIX + KEYSTORE_PARAM_TYPE;
  /** Keystore provider (SunJCE, etc.) */
  public static final String PARAM_KEYSTORE_PROVIDER = DOC_PREFIX + KEYSTORE_PARAM_PROVIDER;
  /** Keystore password. Default is machine's fqdn */
  public static final String PARAM_KEYSTORE_PASSWORD = DOC_PREFIX + KEYSTORE_PARAM_PASSWORD;
  /** Private key password */
  public static final String PARAM_KEYSTORE_KEY_PASSWORD = DOC_PREFIX + KEYSTORE_PARAM_KEY_PASSWORD;
  /** private key password file */
  public static final String PARAM_KEYSTORE_KEY_PASSWORD_FILE =
      DOC_PREFIX + KEYSTORE_PARAM_KEY_PASSWORD_FILE;
  /**
   * If true, and the keystore doesn't exist, a keystore with a self-signed certificate will be be
   * created.
   */
  public static final String PARAM_KEYSTORE_CREATE = DOC_PREFIX + KEYSTORE_PARAM_CREATE;

  public static boolean DEFAULT_CREATE = false;

  protected Map<String, LockssKeyStore> keystoreMap = new HashMap<String, LockssKeyStore>();

  public void startService() {
    super.startService();
    loadKeyStores();
  }

  public synchronized void setConfig(
      Configuration config, Configuration prevConfig, Configuration.Differences changedKeys) {
    if (changedKeys.contains(PREFIX)) {
      defaultKeyStoreType = config.get(PARAM_DEFAULT_KEYSTORE_TYPE, DEFAULT_DEFAULT_KEYSTORE_TYPE);
      defaultKeyStoreProvider =
          config.get(PARAM_DEFAULT_KEYSTORE_PROVIDER, DEFAULT_DEFAULT_KEYSTORE_PROVIDER);
      paramExitIfMissingKeyStore =
          config.getBoolean(PARAM_EXIT_IF_MISSING_KEYSTORE, DEFAULT_EXIT_IF_MISSING_KEYSTORE);

      if (changedKeys.contains(PARAM_KEYSTORE)) {
        configureKeyStores(config);
        // defer initial set of keystore loading until startService
        if (isInited()) {
          // load any newly added keystores
          loadKeyStores();
        }
      }
    }
  }

  /**
   * Return the named LockssKeyStore or null
   *
   * @param name the keystore name
   */
  public LockssKeyStore getLockssKeyStore(String name) {
    return getLockssKeyStore(name, null);
  }

  /**
   * Return the named LockssKeyStore
   *
   * @param name the keystore name
   * @param criticalServiceName if non-null, this is a criticial keystore whose unavailability
   *     should cause the daemon to exit (if org.lockss.keyMgr.exitIfMissingKeystore is true)
   */
  public LockssKeyStore getLockssKeyStore(String name, String criticalServiceName) {
    LockssKeyStore res = keystoreMap.get(name);
    checkFact(res, name, criticalServiceName, null);
    return res;
  }

  /**
   * Convenience method to return the KeyManagerFactory from the named LockssKeyStore, or null
   *
   * @param name the keystore name
   */
  public KeyManagerFactory getKeyManagerFactory(String name) {
    return getKeyManagerFactory(name, null);
  }

  /**
   * Convenience method to return the KeyManagerFactory from the named LockssKeyStore.
   *
   * @param name the keystore name
   * @param criticalServiceName if non-null, this is a criticial keystore whose unavailability
   *     should cause the daemon to exit (if org.lockss.keyMgr.exitIfMissingKeystore is true)
   */
  public KeyManagerFactory getKeyManagerFactory(String name, String criticalServiceName) {
    LockssKeyStore lk = getLockssKeyStore(name, criticalServiceName);
    if (lk != null) {
      KeyManagerFactory fact = lk.getKeyManagerFactory();
      checkFact(fact, name, criticalServiceName, "found but contains no private keys");

      return fact;
    }
    return null;
  }

  /** Convenience method to return the TrustManagerFactory from the named LockssKeyStore, or null */
  public TrustManagerFactory getTrustManagerFactory(String name) {
    return getTrustManagerFactory(name, null);
  }

  /**
   * Convenience method to return the TrustManagerFactory from the named LockssKeyStore.
   *
   * @param name the keystore name
   * @param criticalServiceName if non-null, this is a criticial keystore whose unavailability
   *     should cause the daemon to exit (if org.lockss.keyMgr.exitIfMissingKeystore is true)
   */
  public TrustManagerFactory getTrustManagerFactory(String name, String criticalServiceName) {
    LockssKeyStore lk = getLockssKeyStore(name, criticalServiceName);
    if (lk != null) {
      TrustManagerFactory fact = lk.getTrustManagerFactory();
      checkFact(fact, name, criticalServiceName, "found but contains no trusted certificates");
      return fact;
    }
    return null;
  }

  private void checkFact(Object fact, String name, String criticalServiceName, String message) {
    if (fact == null && criticalServiceName != null) {
      String msg =
          StringUtil.isNullString(name)
              ? ("No keystore name given for critical keystore"
                  + " needed for service "
                  + criticalServiceName
                  + ", daemon exiting")
              : ("Critical keystore "
                  + name
                  + " "
                  + ((message != null) ? message : "not found or not loadable")
                  + " for service "
                  + criticalServiceName
                  + ", daemon exiting");
      log.critical(msg);

      if (paramExitIfMissingKeyStore) {
        System.exit(Constants.EXIT_CODE_KEYSTORE_MISSING);
      } else {
        throw new IllegalArgumentException(msg);
      }
    }
  }

  /** Create LockssKeystores from config subtree below {@link #PARAM_KEYSTORE} */
  void configureKeyStores(Configuration config) {
    Configuration allKs = config.getConfigTree(PARAM_KEYSTORE);
    for (Iterator iter = allKs.nodeIterator(); iter.hasNext(); ) {
      String id = (String) iter.next();
      Configuration oneKs = allKs.getConfigTree(id);
      try {
        LockssKeyStore lk = createLockssKeyStore(oneKs);
        String name = lk.getName();
        if (name == null) {
          log.error("KeyStore definition missing name: " + oneKs);
          continue;
        }
        LockssKeyStore old = keystoreMap.get(name);
        if (old != null && !lk.equals(old)) {
          log.warning(
              "Keystore "
                  + name
                  + " redefined.  "
                  + "New definition may not take effect until daemon restart");
        }

        log.debug("Adding keystore " + name);
        keystoreMap.put(name, lk);

      } catch (Exception e) {
        log.error("Couldn't create keystore: " + oneKs, e);
      }
    }
  }

  /** Create LockssKeystore from a config subtree */
  LockssKeyStore createLockssKeyStore(Configuration config) {
    log.debug2("Creating LockssKeyStore from config: " + config);
    String name = config.get(KEYSTORE_PARAM_NAME);
    LockssKeyStore lk = new LockssKeyStore(name);

    String file = config.get(KEYSTORE_PARAM_FILE);
    String resource = config.get(KEYSTORE_PARAM_RESOURCE);
    String url = config.get(KEYSTORE_PARAM_URL);

    if (!StringUtil.isNullString(file)) {
      lk.setLocation(file, LocationType.File);
    } else if (!StringUtil.isNullString(resource)) {
      lk.setLocation(resource, LocationType.Resource);
    } else if (!StringUtil.isNullString(url)) {
      lk.setLocation(url, LocationType.Url);
    }

    lk.setType(config.get(KEYSTORE_PARAM_TYPE, defaultKeyStoreType));
    lk.setProvider(config.get(KEYSTORE_PARAM_PROVIDER, defaultKeyStoreProvider));
    lk.setPassword(config.get(KEYSTORE_PARAM_PASSWORD));
    lk.setKeyPassword(config.get(KEYSTORE_PARAM_KEY_PASSWORD));
    lk.setKeyPasswordFile(config.get(KEYSTORE_PARAM_KEY_PASSWORD_FILE));
    lk.setMayCreate(config.getBoolean(KEYSTORE_PARAM_CREATE, DEFAULT_CREATE));
    return lk;
  }

  void loadKeyStores() {
    List<LockssKeyStore> lst = new ArrayList<LockssKeyStore>(keystoreMap.values());
    for (LockssKeyStore lk : lst) {
      try {
        lk.load();
      } catch (Exception e) {
        log.error("Can't load keystore " + lk.getName(), e);
        keystoreMap.remove(lk.getName());
      }
    }
  }
}
Esempio n. 5
0
// SingleThreadModel causes servlet instances to be assigned to only a
// single thread (request) at a time.
public abstract class LockssServlet extends HttpServlet implements SingleThreadModel {
  protected static Logger log = Logger.getLogger("LockssServlet");

  // Constants
  static final String PARAM_LOCAL_IP = Configuration.PREFIX + "localIPAddress";

  static final String PARAM_PLATFORM_VERSION = Configuration.PREFIX + "platform.version";

  /** Inactive HTTP session (cookie) timeout */
  static final String PARAM_UI_SESSION_TIMEOUT = Configuration.PREFIX + "ui.sessionTimeout";

  static final long DEFAULT_UI_SESSION_TIMEOUT = 2 * Constants.DAY;

  /** Maximum size of uploaded file accepted */
  static final String PARAM_MAX_UPLOAD_FILE_SIZE = Configuration.PREFIX + "ui.maxUploadFileSize";

  static final int DEFAULT_MAX_UPLOAD_FILE_SIZE = 500000;

  /** The warning string to display when the UI is disabled. */
  static final String PARAM_UI_WARNING = Configuration.PREFIX + "ui.warning";

  // session keys
  static final String SESSION_KEY_OBJECT_ID = "obj_id";
  static final String SESSION_KEY_OBJ_MAP = "obj_map";
  public static final String SESSION_KEY_RUNNING_SERVLET = "running_servlet";
  public static final String SESSION_KEY_REQUEST_HOST = "request_host";

  // Name given to form element whose value is the action that should be
  // performed when the form is submitted.  (Not always the submit button.)
  public static final String ACTION_TAG = "lockssAction";

  public static final String JAVASCRIPT_RESOURCE = "org/lockss/htdocs/admin.js";

  public static final String ATTR_INCLUDE_SCRIPT = "IncludeScript";
  public static final String ATTR_ALLOW_ROLES = "AllowRoles";

  /** User may configure admin access (add/delete/modify users, set admin access list) */
  public static final String ROLE_USER_ADMIN = "userAdminRole";

  /** User may configure content access (set content access list) */
  public static final String ROLE_CONTENT_ADMIN = "contentAdminRole";

  /** User may change AU configuration (add/delete content) */
  public static final String ROLE_AU_ADMIN = "auAdminRole";

  public static final String ROLE_DEBUG = "debugRole";

  protected ServletContext context;

  private LockssApp theApp = null;
  private ServletManager servletMgr;
  private AccountManager acctMgr;

  // Request-local storage.  Convenient, but requires servlet instances
  // to be single threaded, and must ensure reset them to avoid carrying
  // over state between requests.
  protected HttpServletRequest req;
  protected HttpServletResponse resp;
  protected URL reqURL;
  protected HttpSession session;
  private String adminDir = null;
  protected String clientAddr; // client addr, even if no param
  protected String localAddr;
  protected MultiPartRequest multiReq;

  private Vector footnotes;
  private int footNumber;
  private int tabindex;
  ServletDescr _myServletDescr = null;
  private String myName = null;

  // number submit buttons sequentially so unit tests can find them
  protected int submitButtonNumber = 0;

  /** Run once when servlet loaded. */
  public void init(ServletConfig config) throws ServletException {
    super.init(config);
    context = config.getServletContext();
    theApp = (LockssApp) context.getAttribute(ServletManager.CONTEXT_ATTR_LOCKSS_APP);
    servletMgr = (ServletManager) context.getAttribute(ServletManager.CONTEXT_ATTR_SERVLET_MGR);
    if (theApp instanceof LockssDaemon) {
      acctMgr = getLockssDaemon().getAccountManager();
    }
  }

  public ServletManager getServletManager() {
    return servletMgr;
  }

  protected ServletDescr[] getServletDescrs() {
    return servletMgr.getServletDescrs();
  }

  /** Servlets must implement this method. */
  protected abstract void lockssHandleRequest() throws ServletException, IOException;

  /** Common request handling. */
  public void service(HttpServletRequest req, HttpServletResponse resp)
      throws ServletException, IOException {
    resetState();
    boolean success = false;
    HttpSession session = req.getSession(false);
    try {
      this.req = req;
      this.resp = resp;
      if (log.isDebug()) {
        logParams();
      }
      resp.setContentType("text/html");

      if (!mayPageBeCached()) {
        resp.setHeader("pragma", "no-cache");
        resp.setHeader("Cache-control", "no-cache");
      }

      reqURL = new URL(UrlUtil.getRequestURL(req));
      clientAddr = getLocalIPAddr();

      // check that current user has permission to run this servlet
      if (!isServletAllowed(myServletDescr())) {
        displayWarningInLieuOfPage("You are not authorized to use " + myServletDescr().heading);
        return;
      }

      // check whether servlet is disabled
      String reason = ServletUtil.servletDisabledReason(myServletDescr().getServletName());
      if (reason != null) {
        displayWarningInLieuOfPage("This function is disabled. " + reason);
        return;
      }
      if (session != null) {
        session.setAttribute(SESSION_KEY_RUNNING_SERVLET, getHeading());
        String reqHost = req.getRemoteHost();
        String forw = req.getHeader(HttpFields.__XForwardedFor);
        if (!StringUtil.isNullString(forw)) {
          reqHost += " (proxies for " + forw + ")";
        }
        session.setAttribute(SESSION_KEY_REQUEST_HOST, reqHost);
      }
      lockssHandleRequest();
      success = (errMsg == null);
    } catch (ServletException e) {
      log.error("Servlet threw", e);
      throw e;
    } catch (IOException e) {
      log.error("Servlet threw", e);
      throw e;
    } catch (RuntimeException e) {
      log.error("Servlet threw", e);
      throw e;
    } finally {
      if (session != null) {
        session.setAttribute(SESSION_KEY_RUNNING_SERVLET, null);
        session.setAttribute(LockssFormAuthenticator.__J_AUTH_ACTIVITY, TimeBase.nowMs());
      }
      if ("please".equalsIgnoreCase(req.getHeader("X-Lockss-Result"))) {
        log.debug3("X-Lockss-Result: " + (success ? "Ok" : "Fail"));
        resp.setHeader("X-Lockss-Result", success ? "Ok" : "Fail");
      }
      resetMyLocals();
      resetLocals();
    }
  }

  protected void resetState() {
    multiReq = null;
    footNumber = 0;
    submitButtonNumber = 0;
    tabindex = 1;
    statusMsg = null;
    errMsg = null;
    isFramed = false;
  }

  protected void resetLocals() {}

  protected void resetMyLocals() {
    // Don't hold on to stuff forever
    req = null;
    resp = null;
    session = null;
    reqURL = null;
    adminDir = null;
    localAddr = null;
    footnotes = null;
    _myServletDescr = null;
    myName = null;
    multiReq = null;
  }

  /**
   * Return true if generated page may be cached (e.g., by browser). Default is false as most
   * servlets generate dynamic results
   */
  protected boolean mayPageBeCached() {
    return false;
  }

  /** Set the session timeout to the configured value */
  protected void setSessionTimeout(HttpSession session) {
    Configuration config = CurrentConfig.getCurrentConfig();
    setSessionTimeout(
        session, config.getTimeInterval(PARAM_UI_SESSION_TIMEOUT, DEFAULT_UI_SESSION_TIMEOUT));
  }

  /** Set the session timeout */
  protected void setSessionTimeout(HttpSession session, long time) {
    session.setMaxInactiveInterval((int) (time / Constants.SECOND));
  }

  /** Get the current session, creating it if necessary (and set the timeout if so) */
  protected HttpSession getSession() {
    if (session == null) {
      session = req.getSession(true);
      if (session.isNew()) {
        setSessionTimeout(session);
      }
    }
    return session;
  }

  /** Return true iff a session has already been established */
  protected boolean hasSession() {
    return req.getSession(false) != null;
  }

  /** Get an unused ID string for storing an object in the session */
  protected String getNewSessionObjectId() {
    HttpSession session = getSession();
    synchronized (session) {
      Integer id = (Integer) getSession().getAttribute(SESSION_KEY_OBJECT_ID);
      if (id == null) {
        id = new Integer(1);
      }
      session.setAttribute(SESSION_KEY_OBJECT_ID, new Integer(id.intValue() + 1));
      return id.toString();
    }
  }

  /** Get the object associated with the ID in the session */
  protected Object getSessionIdObject(String id) {
    HttpSession session = getSession();
    synchronized (session) {
      BidiMap map = (BidiMap) session.getAttribute(SESSION_KEY_OBJ_MAP);
      if (map == null) {
        return null;
      }
      return map.getKey(id);
    }
  }

  /** Get the String associated with the ID in the session */
  protected String getSessionIdString(String id) {
    return (String) getSessionIdObject(id);
  }

  /** Get the ID with which the object is associated with the session, if any */
  protected String getSessionObjectId(Object obj) {
    HttpSession session = getSession();
    BidiMap map;
    synchronized (session) {
      map = (BidiMap) session.getAttribute(SESSION_KEY_OBJ_MAP);
      if (map == null) {
        map = new DualHashBidiMap();
        session.setAttribute(SESSION_KEY_OBJ_MAP, map);
      }
    }
    synchronized (map) {
      String id = (String) map.get(obj);
      if (id == null) {
        id = getNewSessionObjectId();
        map.put(obj, id);
      }
      return id;
    }
  }

  // Return descriptor of running servlet
  protected ServletDescr myServletDescr() {
    if (_myServletDescr == null) {
      _myServletDescr = servletMgr.findServletDescr(this);
    }
    return _myServletDescr;
  }

  // By default, servlet heading is in descr.  Override method to
  // compute other heading
  protected String getHeading(ServletDescr d) {
    if (d == null) return "Unknown Servlet";
    return d.heading;
  }

  protected String getHeading() {
    return getHeading(myServletDescr());
  }

  String getLocalIPAddr() {
    if (localAddr == null) {
      try {
        IPAddr localHost = IPAddr.getLocalHost();
        localAddr = localHost.getHostAddress();
      } catch (UnknownHostException e) {
        // shouldn't happen
        log.error("LockssServlet: getLocalHost: " + e.toString());
        return "???";
      }
    }
    return localAddr;
  }

  // Return IP addr used by LCAP.  If specified by (misleadingly named)
  // localIPAddress prop, might not really be our address (if we are
  // behind NAT).
  String getLcapIPAddr() {
    String ip = CurrentConfig.getParam(PARAM_LOCAL_IP);
    if (ip == null || ip.length() <= 0) {
      return getLocalIPAddr();
    }
    return ip;
  }

  String getRequestHost() {
    return reqURL.getHost();
  }

  String getMachineName() {
    return PlatformUtil.getLocalHostname();
  }

  //   String getMachineName0() {
  //     if (myName == null) {
  //       // Return the canonical name of the interface the request was aimed
  //       // at.  (localIPAddress prop isn't necessarily right here, as it
  //       // might be the address of a NAT that we're behind.)
  //       String host = reqURL.getHost();
  //       try {
  // 	IPAddr localHost = IPAddr.getByName(host);
  // 	String ip = localHost.getHostAddress();
  // 	myName = getMachineName(ip);
  //       } catch (UnknownHostException e) {
  // 	// shouldn't happen
  // 	log.error("getMachineName", e);
  // 	return host;
  //       }
  //     }
  //     return myName;
  //   }

  //   String getMachineName(String ip) {
  //     try {
  //       IPAddr inet = IPAddr.getByName(ip);
  //       return inet.getHostName();
  //     } catch (UnknownHostException e) {
  //       log.warning("getMachineName", e);
  //     }
  //     return ip;
  //   }

  // return IP given name or IP
  String getMachineIP(String name) {
    try {
      IPAddr inet = IPAddr.getByName(name);
      return inet.getHostAddress();
    } catch (UnknownHostException e) {
      return null;
    }
  }

  boolean isServletLinkInNav(ServletDescr d) {
    return !isThisServlet(d) || linkMeInNav();
  }

  boolean isThisServlet(ServletDescr d) {
    return d == myServletDescr();
  }

  /** servlets may override this to determine whether they should be a link in nav table */
  protected boolean linkMeInNav() {
    return false;
  }

  boolean isLargeLogo() {
    return myServletDescr().isLargeLogo();
  }

  // user predicates
  String getUsername() {
    Principal user = req.getUserPrincipal();
    return user != null ? user.toString() : null;
  }

  protected UserAccount getUserAccount() {
    if (acctMgr != null) {
      return acctMgr.getUser(getUsername());
    }
    return AccountManager.NOBODY_ACCOUNT;
  }

  protected boolean isDebugUser() {
    return doesUserHaveRole(ROLE_DEBUG);
  }

  protected boolean doesUserHaveRole(String role) {
    if ((req.isUserInRole(role) || req.isUserInRole(ROLE_USER_ADMIN)) && !hasNoRoleParsm(role)) {
      return true;
    }
    return hasTestRole(role);
  }

  static Map<String, String> noRoleParams = new HashMap<String, String>();

  static {
    noRoleParams.put(ROLE_USER_ADMIN, "noadmin");
    noRoleParams.put(ROLE_CONTENT_ADMIN, "nocontent");
    noRoleParams.put(ROLE_AU_ADMIN, "noau");
    noRoleParams.put(ROLE_DEBUG, "nodebug");
  }

  protected boolean hasNoRoleParsm(String roleName) {
    String noRoleParam = noRoleParams.get(roleName);
    return (noRoleParam != null && !StringUtil.isNullString(req.getParameter(noRoleParam)));
  }

  protected boolean hasTestRole(String role) {
    // Servlet test harness puts roles in context
    List roles = (List) context.getAttribute(ATTR_ALLOW_ROLES);
    return roles != null && (roles.contains(role) || roles.contains(ROLE_USER_ADMIN));
  }

  protected boolean isServletAllowed(ServletDescr d) {
    if (d.needsUserAdminRole() && !doesUserHaveRole(ROLE_USER_ADMIN)) return false;
    if (d.needsContentAdminRole() && !doesUserHaveRole(ROLE_CONTENT_ADMIN)) return false;
    if (d.needsAuAdminRole() && !doesUserHaveRole(ROLE_AU_ADMIN)) return false;

    return d.isEnabled(getLockssDaemon());
  }

  protected boolean isServletDisplayed(ServletDescr d) {
    if (!isServletAllowed(d)) return false;
    if (d.needsDebugRole() && !doesUserHaveRole(ROLE_DEBUG)) return false;
    return true;
  }

  protected boolean isServletInNav(ServletDescr d) {
    if (d.cls == ServletDescr.UNAVAILABLE_SERVLET_MARKER) return false;
    return d.isInNav(this) && isServletDisplayed(d);
  }

  // Called when a servlet doesn't get the parameters it expects/needs
  protected void paramError() throws IOException {
    // FIXME: As of 2006-03-15 this method and its only caller checkParam() are not called from
    // anywhere
    PrintWriter wrtr = resp.getWriter();
    Page page = new Page();
    // add referer, params, msg to contact lockss unless from old bookmark
    // or manually entered url
    page.add("Parameter error");
    page.write(wrtr);
  }

  // return true iff error
  protected boolean checkParam(boolean ok, String msg) throws IOException {
    if (ok) return false;
    log.error(myServletDescr().getPath() + ": " + msg);
    paramError();
    return true;
  }

  /** Construct servlet URL */
  String srvURL(ServletDescr d) {
    return srvURL((String) null, d, null);
  }

  /** Construct servlet URL with params */
  String srvURL(ServletDescr d, String params) {
    return srvURL((String) null, d, params);
  }

  /** Construct servlet URL with params */
  String srvURL(ServletDescr d, Properties params) {
    return srvURL(d, concatParams(params));
  }

  /** Construct servlet absolute URL, with params as necessary. */
  String srvAbsURL(ServletDescr d, String params) {
    return srvURL(getRequestHost(), d, params);
  }

  /**
   * Construct servlet URL, with params as necessary. Avoid generating a hostname different from
   * that used in the original request, or browsers will prompt again for login
   */
  String srvURL(String host, ServletDescr d, String params) {
    return srvURLFromStem(srvUrlStem(host), d, params);
  }

  String srvURL(PeerIdentity peer, ServletDescr d, String params) {
    return srvURLFromStem(peer.getUiUrlStem(reqURL.getPort()), d, params);
  }

  /**
   * Construct servlet URL, with params as necessary. Avoid generating a hostname different from
   * that used in the original request, or browsers will prompt again for login
   */
  String srvURLFromStem(String stem, ServletDescr d, String params) {
    if (d.isPathIsUrl()) {
      return d.getPath();
    }
    StringBuilder sb = new StringBuilder(80);
    if (stem != null) {
      sb.append(stem);
      if (stem.charAt(stem.length() - 1) != '/') {
        sb.append('/');
      }
    } else {
      // ensure absolute path even if no scheme/host/port
      sb.append('/');
    }
    sb.append(d.getPath());
    if (params != null) {
      sb.append('?');
      sb.append(params);
    }
    return sb.toString();
  }

  String srvUrlStem(String host) {
    if (host == null) {
      return null;
    }
    StringBuilder sb = new StringBuilder();
    sb.append(reqURL.getProtocol());
    sb.append("://");
    sb.append(host);
    sb.append(':');
    sb.append(reqURL.getPort());
    return sb.toString();
  }

  /** Return a link to a servlet */
  String srvLink(ServletDescr d, String text) {
    return srvLink(d, text, (String) null);
  }

  /** Return a link to a servlet with params */
  String srvLink(ServletDescr d, String text, String params) {
    return new Link(srvURL(d, params), (text != null ? text : d.heading)).toString();
  }

  /** Return a link to a servlet with params */
  String srvLink(ServletDescr d, String text, Properties params) {
    return new Link(srvURL(d, params), text).toString();
  }

  /** Return an absolute link to a servlet with params */
  String srvAbsLink(ServletDescr d, String text, Properties params) {
    return srvAbsLink(d, text, concatParams(params));
  }

  /** Return an absolute link to a servlet with params */
  String srvAbsLink(ServletDescr d, String text, String params) {
    return new Link(srvAbsURL(d, params), (text != null ? text : d.heading)).toString();
  }

  /** Return an absolute link to a servlet with params */
  String srvAbsLink(String host, ServletDescr d, String text, String params) {
    return new Link(srvURL(host, d, params), (text != null ? text : d.heading)).toString();
  }

  /** Return an absolute link to a servlet with params */
  String srvAbsLink(PeerIdentity peer, ServletDescr d, String text, String params) {
    return new Link(srvURL(peer, d, params), (text != null ? text : d.heading)).toString();
  }

  /** Return text as a link iff isLink */
  String conditionalSrvLink(ServletDescr d, String text, String params, boolean isLink) {
    if (isLink) {
      return srvLink(d, text, params);
    } else {
      return text;
    }
  }

  /** Return text as a link iff isLink */
  String conditionalSrvLink(ServletDescr d, String text, boolean isLink) {
    return conditionalSrvLink(d, text, null, isLink);
  }

  /** Concatenate params for URL string */
  static String concatParams(String p1, String p2) {
    if (StringUtil.isNullString(p1)) {
      return p2;
    }
    if (StringUtil.isNullString(p2)) {
      return p1;
    }
    return p1 + "&" + p2;
  }

  /** Concatenate params for URL string */
  String concatParams(Properties props) {
    if (props == null) {
      return null;
    }
    java.util.List list = new ArrayList();
    for (Iterator iter = props.keySet().iterator(); iter.hasNext(); ) {
      String key = (String) iter.next();
      String val = props.getProperty(key);
      if (!StringUtil.isNullString(val)) {
        list.add(key + "=" + urlEncode(val));
      }
    }
    return StringUtil.separatedString(list, "&");
  }

  String modifyParams(String key, String val) {
    Properties props = getParamsAsProps();
    props.setProperty(key, val);
    return concatParams(props);
  }

  /**
   * Return the request parameters as a Properties. Only the first value of multivalued parameters
   * is included.
   */
  Properties getParamsAsProps() {
    Properties props = new Properties();
    for (Enumeration en = req.getParameterNames(); en.hasMoreElements(); ) {
      String name = (String) en.nextElement();
      props.setProperty(name, req.getParameter(name));
    }
    return props;
  }

  /**
   * Return the request parameters as a Map<String,String>. Only the first value of multivalued
   * parameters is included.
   */
  Map<String, String> getParamsAsMap() {
    Map<String, String> map = new HashMap<String, String>();
    for (Enumeration en = req.getParameterNames(); en.hasMoreElements(); ) {
      String name = (String) en.nextElement();
      map.put(name, req.getParameter(name));
    }
    return map;
  }

  protected String urlEncode(String param) {
    return UrlUtil.encodeUrl(param);
  }

  protected String getRequestKey() {
    String key = req.getPathInfo();
    if (key != null && key.startsWith("/")) {
      return key.substring(1);
    }
    return key;
  }

  /** Common page setup. */
  protected Page newPage() {
    // Compute heading
    String heading = getHeading();
    if (heading == null) {
      heading = "Box Administration";
    }

    // Create page and layout header
    Page page = ServletUtil.doNewPage(getPageTitle(), isFramed());
    Iterator inNavIterator;
    if (myServletDescr().hasNoNavTable()) {
      inNavIterator = CollectionUtil.EMPTY_ITERATOR;
    } else {
      inNavIterator =
          new FilterIterator(
              new ObjectArrayIterator(getServletDescrs()),
              new Predicate() {
                public boolean evaluate(Object obj) {
                  return isServletInNav((ServletDescr) obj);
                }
              });
    }
    ServletUtil.layoutHeader(
        this,
        page,
        heading,
        isLargeLogo(),
        getMachineName(),
        getLockssApp().getStartDate(),
        inNavIterator);
    String warnMsg = CurrentConfig.getParam(PARAM_UI_WARNING);
    if (warnMsg != null) {
      Composite warning = new Composite();
      warning.add("<center><font color=red size=+1>");
      warning.add(warnMsg);
      warning.add("</font></center><br>");
      page.add(warning);
    }
    return page;
  }

  protected Page addBarePageHeading(Page page) {
    // FIXME: Move the following fragment elsewhere
    // It causes the doctype statement to appear in the middle,
    // after the <body> tag.
    page.add("<!doctype html public \"-//w3c//dtd html 4.0 transitional//en\">");
    page.addHeader("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">");
    page.addHeader("<meta http-equiv=\"content-type\" content=\"text/html;charset=ISO-8859-1\">");
    page.addHeader("<link rel=\"shortcut icon\" href=\"/favicon.ico\" type=\"image/x-icon\" />");
    return page;
  }

  private boolean isFramed = false;

  protected String errMsg;

  protected String statusMsg;

  protected boolean isFramed() {
    return isFramed;
  }

  protected void setFramed(boolean v) {
    isFramed = v;
  }

  protected String getPageTitle() {
    String heading = getHeading();
    if (heading != null) {
      return "LOCKSS: " + heading;
    } else {
      return "LOCKSS";
    }
  }

  /** Return a button that invokes the javascript submit routine with the specified action */
  protected Element submitButton(String label, String action) {
    return submitButton(label, action, null, null);
  }

  /** Return a button that invokes javascript when clicked. */
  Input jsButton(String label, String js) {
    Input btn = new Input("button", null);
    btn.attribute("value", label);
    setTabOrder(btn);
    btn.attribute("onClick", js);
    return btn;
  }

  /**
   * Return a button that invokes the javascript submit routine with the specified action, first
   * storing the value in the specified form prop.
   */
  protected Element submitButton(String label, String action, String prop, String value) {
    StringBuilder sb = new StringBuilder(40);
    sb.append("lockssButton(this, '");
    sb.append(action);
    sb.append("'");
    if (prop != null && value != null) {
      sb.append(", '");
      sb.append(prop);
      sb.append("', '");
      sb.append(value);
      sb.append("'");
    }
    sb.append(")");
    Input btn = jsButton(label, sb.toString());
    btn.attribute("id", "lsb." + (++submitButtonNumber));
    return btn;
  }

  /**
   * Return a (possibly labelled) checkbox.
   *
   * @param label appears to right of checkbox if non null
   * @param value value included in result set if box checked
   * @param key form key to which result set is assigned
   * @param checked if true, box is initially checked
   * @return a checkbox Element
   */
  Element checkBox(String label, String value, String key, boolean checked) {
    Input in = new Input(Input.Checkbox, key, value);
    if (checked) {
      in.check();
    }
    setTabOrder(in);
    if (StringUtil.isNullString(label)) {
      return in;
    } else {
      Composite c = new Composite();
      c.add(in);
      c.add(" ");
      c.add(label);
      return c;
    }
  }

  /**
   * Return a labelled rasio button
   *
   * @param label label to right of circle, and form value if checked
   * @param key form key to which value is assigned
   * @param checked if true, is initially checked
   * @return a readio button Element
   */
  protected Element radioButton(String label, String key, boolean checked) {
    return radioButton(label, label, key, checked);
  }

  /**
   * Return a labelled rasio button
   *
   * @param label appears to right of circle if non null
   * @param value value assigned to key if box checked
   * @param key form key to which value is assigned
   * @param checked if true, is initially checked
   * @return a readio button Element
   */
  protected Element radioButton(String label, String value, String key, boolean checked) {
    Composite c = new Composite();
    Input in = new Input(Input.Radio, key, value);
    if (checked) {
      in.check();
    }
    setTabOrder(in);
    c.add(in);
    c.add(label);
    return c;
  }

  /** Add html tags to grey the text if isGrey is true */
  protected String greyText(String txt, boolean isGrey) {
    if (!isGrey) {
      return txt;
    }
    return "<font color=gray>" + txt + "</font>";
  }

  /**
   * Set this element next in the tab order. Returns the element for easier nesting in expressions.
   */
  protected Element setTabOrder(Element ele) {
    ele.attribute("tabindex", tabindex++);
    return ele;
  }

  /**
   * Store a footnote, assign it a number, return html for footnote reference. If footnote in null
   * or empty, no footnote is added and an empty string is returned. Footnote numbers get turned
   * into links; <b>Do not put the result of addFootnote inside a link!</b>.
   */
  protected String addFootnote(String s) {
    if (s == null || s.length() == 0) {
      return "";
    }
    if (footNumber == 0) {
      if (footnotes == null) {
        footnotes = new Vector(10, 10);
      } else {
        footnotes.removeAllElements();
      }
    }
    int n = footnotes.indexOf(s);
    if (n < 0) {
      n = footNumber++;
      footnotes.addElement(s);
    }
    return "<sup><font size=-1><a href=#foottag" + (n + 1) + ">" + (n + 1) + "</a></font></sup>";
  }

  /**
   * Add javascript to page. Normally adds a link to the script file, but can be told to include the
   * script directly in the page, to accomodate unit testing of individual servlets, when other
   * fetches won't work.
   */
  protected void addJavaScript(Composite comp) {
    String include = (String) context.getAttribute(ATTR_INCLUDE_SCRIPT);
    if (StringUtil.isNullString(include)) {
      linkToJavaScript(comp);
    } else {
      includeJavaScript0(comp);
    }
  }

  private void includeJavaScript0(Composite comp) {
    Script script = new Script(getJavascript());
    comp.add(script);
  }

  private void linkToJavaScript(Composite comp) {
    Script script = new Script("");
    script.attribute("src", "admin.js");
    comp.add(script);
  }

  private static String jstext = null;

  private static synchronized String getJavascript() {
    if (jstext == null) {
      InputStream istr = null;
      try {
        ClassLoader loader = Thread.currentThread().getContextClassLoader();
        istr = loader.getResourceAsStream(JAVASCRIPT_RESOURCE);
        jstext = StringUtil.fromInputStream(istr);
        istr.close();
      } catch (Exception e) {
        log.error("Can't load javascript", e);
      } finally {
        IOUtil.safeClose(istr);
      }
    }
    return jstext;
  }

  /** Display a message in lieu of the normal page */
  protected void displayMsgInLieuOfPage(String msg) throws IOException {
    // TODO: Look at HTML
    Page page = newPage();
    Composite warning = new Composite();
    warning.add(msg);
    warning.add("<br>");
    page.add(warning);
    layoutFooter(page);
    page.write(resp.getWriter());
  }

  /** Display a warning in red, in lieu of the normal page */
  protected void displayWarningInLieuOfPage(String msg) throws IOException {
    displayMsgInLieuOfPage("<center><font color=red size=+1>" + msg + "</font></center>");
  }

  /** Display "The cache isn't ready yet, come back later" */
  protected void displayNotStarted() throws IOException {
    displayWarningInLieuOfPage(
        "This LOCKSS box is still starting.  Please "
            + srvLink(myServletDescr(), "try again", getParamsAsProps())
            + " in a moment.");
  }

  public MultiPartRequest getMultiPartRequest() throws FormDataTooLongException, IOException {
    int maxUpload =
        CurrentConfig.getIntParam(PARAM_MAX_UPLOAD_FILE_SIZE, DEFAULT_MAX_UPLOAD_FILE_SIZE);
    return getMultiPartRequest(maxUpload);
  }

  public MultiPartRequest getMultiPartRequest(int maxLen)
      throws FormDataTooLongException, IOException {
    if (req.getContentType() == null || !req.getContentType().startsWith("multipart/form-data")) {
      return null;
    }
    if (req.getContentLength() > maxLen) {
      throw new FormDataTooLongException(req.getContentLength() + " bytes, " + maxLen + " allowed");
    }
    MultiPartRequest multi = new MultiPartRequest(req);
    if (log.isDebug2()) {
      String[] parts = multi.getPartNames();
      log.debug3("Multipart request, " + parts.length + " parts");
      if (log.isDebug3()) {
        for (int p = 0; p < parts.length; p++) {
          String name = parts[p];
          String cont = multi.getString(parts[p]);
          log.debug3(name + ": " + cont);
        }
      }
    }
    multiReq = multi;
    return multi;
  }

  public String getParameter(String name) {
    String val = req.getParameter(name);
    if (val == null && multiReq != null) {
      val = multiReq.getString(name);
    }
    if (val == null) {
      return null;
    }
    val = StringUtils.strip(val, " \t");
    //     if (StringUtil.isNullString(val)) {
    if ("".equals(val)) {
      return null;
    }
    return val;
  }

  protected void layoutFooter(Page page) {
    ServletUtil.doLayoutFooter(
        page, (footnotes == null ? null : footnotes.iterator()), getLockssApp().getVersionInfo());
    if (footnotes != null) {
      footnotes.removeAllElements();
    }
  }

  /** Return the app instance. */
  protected LockssApp getLockssApp() {
    return theApp;
  }

  /**
   * Return the daemon instance, assumes that the servlet is running in the daemon.
   *
   * @throws ClassCastException if the servlet is running in an app other than the daemon
   */
  protected LockssDaemon getLockssDaemon() {
    return (LockssDaemon) theApp;
  }

  protected void logParams() {
    Enumeration en = req.getParameterNames();
    while (en.hasMoreElements()) {
      String name = (String) en.nextElement();
      String vals[];
      String dispval;
      if (StringUtil.indexOfIgnoreCase(name, "passw") >= 0) {
        dispval = req.getParameter(name).length() == 0 ? "" : "********";
      } else if (log.isDebug2() && (vals = req.getParameterValues(name)).length > 1) {
        dispval = StringUtil.separatedString(vals, ", ");
      } else {
        dispval = req.getParameter(name);
      }
      log.debug(name + " = " + dispval);
    }
  }

  /** Convenience method */
  protected String encodeText(String s) {
    return HtmlUtil.encode(s, HtmlUtil.ENCODE_TEXT);
  }

  /** Convenience method */
  protected String encodeTextArea(String s) {
    return HtmlUtil.encode(s, HtmlUtil.ENCODE_TEXTAREA);
  }

  /** Convenience method */
  protected String encodeAttr(String s) {
    return HtmlUtil.encode(s, HtmlUtil.ENCODE_ATTR);
  }

  /**
   * Create message and error message block
   *
   * @param composite TODO
   */
  protected void layoutErrorBlock(Composite composite) {
    if (errMsg != null || statusMsg != null) {
      ServletUtil.layoutErrorBlock(composite, errMsg, statusMsg);
    }
  }

  /** Exception thrown if multipart form data is longer than the caller-supplied max */
  public static class FormDataTooLongException extends Exception {
    public FormDataTooLongException(String message) {
      super(message);
    }
  }
}
Esempio n. 6
0
/** Export the contents and metadata from an AU */
public abstract class Exporter {

  private static final Logger log = Logger.getLogger(Exporter.class);

  static final String PREFIX = Configuration.PREFIX + "exporter.";

  /** Abort export after this many errors */
  public static final String PARAM_MAX_ERRORS = PREFIX + "maxErrors";

  public static final int DEFAULT_MAX_ERRORS = 5;

  protected static int maxErrors = DEFAULT_MAX_ERRORS;

  protected LockssDaemon daemon;
  protected ArchivalUnit au;
  protected File dir;
  protected String prefix;
  protected long maxSize = -1;
  protected int maxVersions = 1;
  protected boolean compress = false;
  protected boolean excludeDirNodes = false;
  protected FilenameTranslation xlate = FilenameTranslation.XLATE_NONE;
  protected List errors = new ArrayList();
  protected boolean isDiskFull = false;

  protected abstract void start() throws IOException;

  protected abstract void finish() throws IOException;

  protected abstract void writeCu(CachedUrl cu) throws IOException;

  protected Exporter(LockssDaemon daemon, ArchivalUnit au) {
    this.daemon = daemon;
    this.au = au;
  }

  /** Called by org.lockss.config.MiscConfig */
  public static void setConfig(
      Configuration config, Configuration oldConfig, Configuration.Differences diffs) {
    if (diffs.contains(PREFIX)) {
      maxErrors = config.getInt(PARAM_MAX_ERRORS, DEFAULT_MAX_ERRORS);
    }
  }

  public void setCompress(boolean val) {
    compress = val;
  }

  public boolean getCompress() {
    return compress;
  }

  public void setExcludeDirNodes(boolean val) {
    excludeDirNodes = val;
  }

  public boolean getExcludeDirNodes() {
    return excludeDirNodes;
  }

  public void setFilenameTranslation(FilenameTranslation val) {
    xlate = val;
  }

  public FilenameTranslation getFilenameTranslation() {
    return xlate;
  }

  public void setDir(File val) {
    dir = val;
  }

  public File getDir() {
    return dir;
  }

  public void setPrefix(String val) {
    prefix = val;
  }

  public String getPrefix() {
    return prefix;
  }

  public void setMaxSize(long val) {
    maxSize = val;
  }

  public long getMaxSize() {
    return maxSize;
  }

  public void setMaxVersions(int val) {
    maxVersions = val;
  }

  public int getMaxVersions() {
    return maxVersions;
  }

  public List getErrors() {
    return errors;
  }

  protected void checkArgs() {
    if (getDir() == null) {
      throw new IllegalArgumentException("Must supply output directory");
    }
    if (getPrefix() == null) {
      throw new IllegalArgumentException("Must supply file name/prefix");
    }
  }

  public void export() {
    log.debug("export(" + au.getName() + ")");
    log.debug(
        "dir: "
            + dir
            + ", pref: "
            + prefix
            + ", size: "
            + maxSize
            + ", ver: "
            + maxVersions
            + (compress ? ", (C)" : ""));
    checkArgs();
    try {
      start();
    } catch (IOException e) {
      recordError("Error opening file", e);
      return;
    }
    writeFiles();
    try {
      finish();
    } catch (IOException e) {
      if (!isDiskFull) {
        // If we already knew (and reported) disk full, also reporting it
        // as a close error is misleading.
        recordError("Error closing file", e);
      }
    }
  }

  protected String xlateFilename(String url) {
    return xlate.xlate(url);
  }

  protected void recordError(String msg, Throwable t) {
    log.error(msg, t);
    errors.add(msg + ": " + t.toString());
  }

  protected void recordError(String msg) {
    log.error(msg);
    errors.add(msg);
  }

  protected String getSoftwareVersion() {
    String releaseName = BuildInfo.getBuildProperty(BuildInfo.BUILD_RELEASENAME);
    StringBuilder sb = new StringBuilder();
    sb.append("LOCKSS Daemon ");
    if (releaseName != null) {
      sb.append(releaseName);
    }
    return sb.toString();
  }

  protected String getHostIp() {
    try {
      IPAddr localHost = IPAddr.getLocalHost();
      return localHost.getHostAddress();
    } catch (UnknownHostException e) {
      log.error("getHostIp()", e);
      return "1.1.1.1";
    }
  }

  protected String getHostName() {
    String res = ConfigManager.getPlatformHostname();
    if (res == null) {
      try {
        InetAddress inet = InetAddress.getLocalHost();
        return inet.getHostName();
      } catch (UnknownHostException e) {
        log.warning("Can't get hostname", e);
        return "unknown";
      }
    }
    return res;
  }

  protected Properties filterResponseProps(Properties props) {
    Properties res = new Properties();
    for (Map.Entry ent : props.entrySet()) {
      String key = (String) ent.getKey();
      if (StringUtil.startsWithIgnoreCase(key, "x-lockss")
          || StringUtil.startsWithIgnoreCase(key, "x_lockss")
          || key.equalsIgnoreCase("org.lockss.version.number")) {
        continue;
      }
      // We've lost the original case - capitalize them the way most people
      // expect
      res.put(StringUtil.titleCase(key, '-'), (String) ent.getValue());
    }
    return res;
  }

  protected String getHttpResponseString(CachedUrl cu) {
    Properties cuProps = cu.getProperties();
    Properties filteredProps = filterResponseProps(cuProps);
    String hdrString = PropUtil.toHeaderString(filteredProps);
    StringBuilder sb = new StringBuilder(hdrString.length() + 30);
    String line1 = inferHttpResponseCode(cu, cuProps);
    sb.append(line1);
    sb.append(Constants.CRLF);
    sb.append(hdrString);
    sb.append(Constants.CRLF);
    return sb.toString();
  }

  String inferHttpResponseCode(CachedUrl cu, Properties cuProps) {
    if (cuProps.get("location") == null) {
      return "HTTP/1.1 200 OK";
    } else {
      return "HTTP/1.1 302 Found";
    }
  }

  // return the next CU with content
  private CachedUrl getNextCu(CuIterator iter) {
    return iter.hasNext() ? iter.next() : null;
  }

  /**
   * Return true if, interpreting URLs as filenames, dirCu is a directory containing fileCu. Used to
   * exclude directory content from output files, so they can be unpacked by standard utilities
   * (e.g., unzip). Shouldn't be called with equal URLs, but return false in that case, as we
   * wouldn't want to exclude the URL
   */
  boolean isDirOf(CachedUrl dirCu, CachedUrl fileCu) {
    String dir = dirCu.getUrl();
    String file = fileCu.getUrl();
    if (!dir.endsWith("/")) {
      dir = dir + "/";
    }
    return file.startsWith(dir) && !file.equals(dir);
  }

  private void writeFiles() {
    PlatformUtil platutil = PlatformUtil.getInstance();
    CuIterator iter = AuUtil.getCuIterator(au);
    int errs = 0;
    CachedUrl curCu = null;
    CachedUrl nextCu = getNextCu(iter);
    while (nextCu != null) {
      curCu = nextCu;
      nextCu = getNextCu(iter);
      if (excludeDirNodes && nextCu != null && isDirOf(curCu, nextCu)) {
        continue;
      }
      CachedUrl[] cuVersions =
          curCu.getCuVersions(maxVersions > 0 ? maxVersions : Integer.MAX_VALUE);
      for (CachedUrl cu : cuVersions) {
        try {
          log.debug2("Exporting " + cu.getUrl());
          writeCu(cu);
        } catch (IOException e) {
          if (platutil.isDiskFullError(e)) {
            recordError("Disk full, can't write export file.");
            isDiskFull = true;
            return;
          }
        } catch (Exception e) {
          // XXX Would like to differentiate between errors opening or
          // reading CU, which shouldn't cause abort, and errors writing
          // to export file, which should.
          recordError("Unable to copy " + cu.getUrl(), e);
          if (errs++ >= maxErrors) {
            recordError("Aborting after " + errs + " errors");
            return;
          }
        }
      }
    }
  }

  private Set<File> fileSet = new HashSet<File>();
  private List<File> fileList = new ArrayList<File>();

  protected void recordExportFile(File file) {
    if (fileSet.add(file)) {
      fileList.add(file);
    }
  }

  /**
   * Return the list of export files written
   *
   * @return List of File written
   */
  public List<File> getExportFiles() {
    return fileList;
  }

  public interface Factory {
    public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au);
  }

  static final String WINDOWS_FROM = "?<>:*|\\";
  static final String WINDOWS_TO = "_______";
  static final String MAC_FROM = ":";
  static final String MAC_TO = "_";

  /** Enum of filename translation types */
  public static enum FilenameTranslation {
    XLATE_NONE("None") {
      public String xlate(String s) {
        return s;
      }
    },
    XLATE_WINDOWS("Windows") {
      public String xlate(String s) {
        return StringUtils.replaceChars(s, WINDOWS_FROM, WINDOWS_TO);
      }
    },
    XLATE_MAC("MacOS") {
      public String xlate(String s) {
        return StringUtils.replaceChars(s, MAC_FROM, MAC_TO);
      }
    };

    private final String label;

    FilenameTranslation(String label) {
      this.label = label;
    }

    public String getLabel() {
      return label;
    }

    public abstract String xlate(String s);
  };

  /** Enum of Exporter types, and factories */
  public static enum Type implements Factory {
    ARC_RESOURCE("ARC (content only)") {
      public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
        return new ArcExporter(daemon, au, false);
      }
    },
    ARC_RESPONSE("ARC (response and content)") {
      public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
        return new ArcExporter(daemon, au, true);
      }
    },
    WARC_RESOURCE("WARC (content only)") {
      public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
        return new WarcExporter(daemon, au, false);
      }
    },
    WARC_RESPONSE("WARC (response and content)") {
      public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
        return new WarcExporter(daemon, au, true);
      }
    },
    ZIP("ZIP") {
      public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
        return new ZipExporter(daemon, au);
      }
    };

    private final String label;

    Type(String label) {
      this.label = label;
    }

    public abstract Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au);

    public String getLabel() {
      return label;
    }
  };
}
/**
 * RepositoryManager is the center of the per AU repositories. It manages the repository config
 * parameters.
 */
public class RepositoryManager extends BaseLockssDaemonManager implements ConfigurableManager {

  private static Logger log = Logger.getLogger("RepositoryManager");

  public static final String PREFIX = Configuration.PREFIX + "repository.";

  /** Maximum size of per-AU repository node cache */
  public static final String PARAM_MAX_PER_AU_CACHE_SIZE = PREFIX + "nodeCache.size";

  public static final int DEFAULT_MAX_PER_AU_CACHE_SIZE = 10;

  /*
   * This needs to be a small multiple of the number of simultaneous
   * polls (poller and voter), as there is a cache entry per active AU.
   * Each poll will have one active AU at a time.
   */
  public static final String PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE =
      PREFIX + "suspectVersionsCache.size";
  public static final int DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE = 10;

  static final String GLOBAL_CACHE_PREFIX = PREFIX + "globalNodeCache.";
  public static final String PARAM_MAX_GLOBAL_CACHE_SIZE = GLOBAL_CACHE_PREFIX + "size";
  public static final int DEFAULT_MAX_GLOBAL_CACHE_SIZE = 500;

  public static final String PARAM_GLOBAL_CACHE_ENABLED = GLOBAL_CACHE_PREFIX + "enabled";
  public static final boolean DEFAULT_GLOBAL_CACHE_ENABLED = false;

  /** Max times to loop looking for unused AU directory. */
  public static final String PARAM_MAX_UNUSED_DIR_SEARCH = PREFIX + "maxUnusedDirSearch";

  public static final int DEFAULT_MAX_UNUSED_DIR_SEARCH = 30000;

  /** If true, LocalRepository keeps track of next subdir name to probe. */
  public static final String PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH =
      PREFIX + "enableStatefulUnusedDirSearch";

  public static final boolean DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH = true;

  /** Max percent of time size calculation thread may run. */
  public static final String PARAM_SIZE_CALC_MAX_LOAD = PREFIX + "sizeCalcMaxLoad";

  public static final float DEFAULT_SIZE_CALC_MAX_LOAD = 0.5F;

  /**
   * If true, path components longer than the maximum filesystem path component are encodes are
   * multiple levels of directories
   */
  public static final String PARAM_ENABLE_LONG_COMPONENTS = PREFIX + "enableLongComponents";

  public static final boolean DEFAULT_ENABLE_LONG_COMPONENTS = true;

  /**
   * Prior to 1.61.6, when long component support is enabled, backslahes were normalized to %5c
   * instead of %5C. This is harmless if checkUnnormalized is set to Fix, as they'll be normalized.
   * Otherwise, this can be set true for compatibility with old repositories. Setting
   * checkUnnormalized to Fix is preferred.
   */
  public static final String PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY =
      PREFIX + "enableLongComponentsCompatibility";

  public static final boolean DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY = false;

  /** Maximum length of a filesystem path component. */
  public static final String PARAM_MAX_COMPONENT_LENGTH = PREFIX + "maxComponentLength";

  public static final int DEFAULT_MAX_COMPONENT_LENGTH = 255;

  /** @see #PARAM_CHECK_UNNORMALIZED */
  public enum CheckUnnormalizedMode {
    No,
    Log,
    Fix
  };

  /**
   * Check for existing nodes with unnormalized names (created by very old daemon that didn't
   * normalize): None, Log, Fix
   */
  public static final String PARAM_CHECK_UNNORMALIZED = PREFIX + "checkUnnormalized";

  public static final CheckUnnormalizedMode DEFAULT_CHECK_UNNORMALIZED = CheckUnnormalizedMode.Log;

  static final String WDOG_PARAM_SIZE_CALC = "SizeCalc";
  static final long WDOG_DEFAULT_SIZE_CALC = Constants.DAY;

  static final String PRIORITY_PARAM_SIZE_CALC = "SizeCalc";
  static final int PRIORITY_DEFAULT_SIZE_CALC = Thread.NORM_PRIORITY - 1;

  static final String DISK_PREFIX = PREFIX + "diskSpace.";

  static final String PARAM_DISK_WARN_FRRE_MB = DISK_PREFIX + "warn.freeMB";
  static final int DEFAULT_DISK_WARN_FRRE_MB = 5000;
  static final String PARAM_DISK_FULL_FRRE_MB = DISK_PREFIX + "full.freeMB";
  static final int DEFAULT_DISK_FULL_FRRE_MB = 100;
  static final String PARAM_DISK_WARN_FRRE_PERCENT = DISK_PREFIX + "warn.freePercent";
  static final double DEFAULT_DISK_WARN_FRRE_PERCENT = .02;
  static final String PARAM_DISK_FULL_FRRE_PERCENT = DISK_PREFIX + "full.freePercent";
  static final double DEFAULT_DISK_FULL_FRRE_PERCENT = .01;

  private PlatformUtil platInfo = PlatformUtil.getInstance();
  private List repoList = Collections.EMPTY_LIST;
  int paramNodeCacheSize = DEFAULT_MAX_PER_AU_CACHE_SIZE;
  boolean paramIsGlobalNodeCache = DEFAULT_GLOBAL_CACHE_ENABLED;
  int paramGlobalNodeCacheSize = DEFAULT_MAX_GLOBAL_CACHE_SIZE;
  int paramSuspectVersionsCacheSize = DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE;
  UniqueRefLruCache globalNodeCache = new UniqueRefLruCache(DEFAULT_MAX_GLOBAL_CACHE_SIZE);
  UniqueRefLruCache suspectVersionsCache =
      new UniqueRefLruCache(DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
  Map localRepos = new HashMap();
  private static int maxUnusedDirSearch = DEFAULT_MAX_UNUSED_DIR_SEARCH;
  private static boolean isStatefulUnusedDirSearch = DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH;
  private static boolean enableLongComponents = DEFAULT_ENABLE_LONG_COMPONENTS;
  private static boolean enableLongComponentsCompatibility =
      DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY;
  private static int maxComponentLength = DEFAULT_MAX_COMPONENT_LENGTH;
  private static CheckUnnormalizedMode checkUnnormalized = DEFAULT_CHECK_UNNORMALIZED;

  PlatformUtil.DF paramDFWarn =
      PlatformUtil.DF.makeThreshold(DEFAULT_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_PERCENT);
  PlatformUtil.DF paramDFFull =
      PlatformUtil.DF.makeThreshold(DEFAULT_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_PERCENT);

  private float sizeCalcMaxLoad = DEFAULT_SIZE_CALC_MAX_LOAD;

  public void startService() {
    super.startService();
    localRepos = new HashMap();
  }

  public void setConfig(
      Configuration config, Configuration oldConfig, Configuration.Differences changedKeys) {
    //  Build list of repositories from list of disk (fs) paths).  Needs to
    //  be generalized if ever another repository implementation.
    if (changedKeys.contains(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST)) {
      List lst = new ArrayList();
      String dspace = config.get(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, "");
      List paths = StringUtil.breakAt(dspace, ';');
      if (paths != null) {
        for (Iterator iter = paths.iterator(); iter.hasNext(); ) {
          lst.add("local:" + (String) iter.next());
        }
      }
      repoList = lst;
    }
    if (changedKeys.contains(PARAM_MAX_PER_AU_CACHE_SIZE)) {
      paramNodeCacheSize =
          config.getInt(PARAM_MAX_PER_AU_CACHE_SIZE, DEFAULT_MAX_PER_AU_CACHE_SIZE);
      for (Iterator iter = getDaemon().getAllLockssRepositories().iterator(); iter.hasNext(); ) {
        LockssRepository repo = (LockssRepository) iter.next();
        if (repo instanceof LockssRepositoryImpl) {
          LockssRepositoryImpl repoImpl = (LockssRepositoryImpl) repo;
          repoImpl.setNodeCacheSize(paramNodeCacheSize);
        }
      }
    }
    if (changedKeys.contains(PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE)) {
      paramSuspectVersionsCacheSize =
          config.getInt(
              PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE, DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
      suspectVersionsCache.setMaxSize(paramSuspectVersionsCacheSize);
    }
    if (changedKeys.contains(GLOBAL_CACHE_PREFIX)) {
      paramIsGlobalNodeCache =
          config.getBoolean(PARAM_GLOBAL_CACHE_ENABLED, DEFAULT_GLOBAL_CACHE_ENABLED);
      if (paramIsGlobalNodeCache) {
        paramGlobalNodeCacheSize =
            config.getInt(PARAM_MAX_GLOBAL_CACHE_SIZE, DEFAULT_MAX_GLOBAL_CACHE_SIZE);
        log.debug("global node cache size: " + paramGlobalNodeCacheSize);
        globalNodeCache.setMaxSize(paramGlobalNodeCacheSize);
      }
    }
    if (changedKeys.contains(DISK_PREFIX)) {
      int minMB = config.getInt(PARAM_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_MB);
      double minPer =
          config.getPercentage(PARAM_DISK_WARN_FRRE_PERCENT, DEFAULT_DISK_WARN_FRRE_PERCENT);
      paramDFWarn = PlatformUtil.DF.makeThreshold(minMB, minPer);
      minMB = config.getInt(PARAM_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_MB);
      minPer = config.getPercentage(PARAM_DISK_FULL_FRRE_PERCENT, DEFAULT_DISK_FULL_FRRE_PERCENT);
      paramDFFull = PlatformUtil.DF.makeThreshold(minMB, minPer);
    }
    if (changedKeys.contains(PARAM_SIZE_CALC_MAX_LOAD)) {
      sizeCalcMaxLoad = config.getPercentage(PARAM_SIZE_CALC_MAX_LOAD, DEFAULT_SIZE_CALC_MAX_LOAD);
    }
    if (changedKeys.contains(PREFIX)) {
      maxUnusedDirSearch =
          config.getInt(PARAM_MAX_UNUSED_DIR_SEARCH, DEFAULT_MAX_UNUSED_DIR_SEARCH);
      isStatefulUnusedDirSearch =
          config.getBoolean(
              PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH, DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH);
      enableLongComponents =
          config.getBoolean(PARAM_ENABLE_LONG_COMPONENTS, DEFAULT_ENABLE_LONG_COMPONENTS);
      enableLongComponentsCompatibility =
          config.getBoolean(
              PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY,
              DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY);
      maxComponentLength = config.getInt(PARAM_MAX_COMPONENT_LENGTH, DEFAULT_MAX_COMPONENT_LENGTH);
      checkUnnormalized =
          (CheckUnnormalizedMode)
              config.getEnum(
                  CheckUnnormalizedMode.class,
                  PARAM_CHECK_UNNORMALIZED,
                  DEFAULT_CHECK_UNNORMALIZED);
    }
  }

  public static boolean isEnableLongComponents() {
    return enableLongComponents;
  }

  public static boolean isEnableLongComponentsCompatibility() {
    return enableLongComponentsCompatibility;
  }

  public static int getMaxComponentLength() {
    return maxComponentLength;
  }

  public static CheckUnnormalizedMode getCheckUnnormalizedMode() {
    return checkUnnormalized;
  }

  /**
   * Return list of known repository names. Needs a registration mechanism if ever another
   * repository implementation.
   */
  public List<String> getRepositoryList() {
    return repoList;
  }

  public PlatformUtil.DF getRepositoryDF(String repoName) {
    String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
    log.debug("path: " + path);
    //    try {
    return platInfo.getJavaDF(path);
    //    } catch (PlatformUtil.UnsupportedException e) {
    //      return null;
    //    }
  }

  public Map<String, PlatformUtil.DF> getRepositoryMap() {
    Map<String, PlatformUtil.DF> repoMap = new LinkedMap();
    for (String repo : getRepositoryList()) {
      repoMap.put(repo, getRepositoryDF(repo));
    }
    return repoMap;
  }

  public String findLeastFullRepository() {
    return findLeastFullRepository(getRepositoryMap());
  }

  public String findLeastFullRepository(Map<String, PlatformUtil.DF> repoMap) {
    String mostFree = null;
    for (String repo : repoMap.keySet()) {
      PlatformUtil.DF df = repoMap.get(repo);
      if (df != null) {
        if (mostFree == null || (repoMap.get(mostFree)).getAvail() < df.getAvail()) {
          mostFree = repo;
        }
      }
    }
    return mostFree;
  }

  public PlatformUtil.DF getDiskWarnThreshold() {
    return paramDFWarn;
  }

  public PlatformUtil.DF getDiskFullThreshold() {
    return paramDFFull;
  }

  public static int getMaxUnusedDirSearch() {
    return maxUnusedDirSearch;
  }

  public static boolean isStatefulUnusedDirSearch() {
    return isStatefulUnusedDirSearch;
  }

  public List findExistingRepositoriesFor(String auid) {
    List res = null;
    for (Iterator iter = getRepositoryList().iterator(); iter.hasNext(); ) {
      String repoName = (String) iter.next();
      String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
      if (LockssRepositoryImpl.doesAuDirExist(auid, path)) {
        if (res == null) {
          res = new ArrayList();
        }
        res.add(repoName);
      }
    }
    return res == null ? Collections.EMPTY_LIST : res;
  }

  // hack only local
  public synchronized LockssRepositoryImpl getRepositoryFromPath(String path) {
    LockssRepositoryImpl repo = (LockssRepositoryImpl) localRepos.get(path);
    if (repo == null) {
      repo = new LockssRepositoryImpl(path);
      repo.initService(getDaemon());
      repo.startService();
      localRepos.put(path, repo);
    }
    return repo;
  }

  /**
   * Return the disk space used by the AU, including all overhead, optionally calculating it if
   * necessary.
   *
   * @param repoAuPath the full path to an AU dir in a LockssRepositoryImpl
   * @param calcIfUnknown if true, size will calculated if unknown (time consumeing)
   * @return the AU's disk usage in bytes, or -1 if unknown
   */
  public long getRepoDiskUsage(String repoAuPath, boolean calcIfUnknown) {
    LockssRepository repo = getRepositoryFromPath(repoAuPath);
    if (repo != null) {
      try {
        RepositoryNode repoNode = repo.getNode(AuCachedUrlSetSpec.URL);
        if (repoNode instanceof AuNodeImpl) {
          return ((AuNodeImpl) repoNode).getDiskUsage(calcIfUnknown);
        }
      } catch (MalformedURLException ignore) {
      }
    }
    return -1;
  }

  public synchronized void setRepositoryForPath(String path, LockssRepositoryImpl repo) {
    localRepos.put(path, repo);
  }

  public boolean isGlobalNodeCache() {
    return paramIsGlobalNodeCache;
  }

  public UniqueRefLruCache getGlobalNodeCache() {
    return globalNodeCache;
  }

  public UniqueRefLruCache getSuspectVersionsCache() {
    return suspectVersionsCache;
  }

  // Background thread to (re)calculate AU size and disk usage.

  private Set sizeCalcQueue = new HashSet();
  private BinarySemaphore sizeCalcSem = new BinarySemaphore();
  private SizeCalcThread sizeCalcThread;

  /** engqueue a size calculation for the AU */
  public void queueSizeCalc(ArchivalUnit au) {
    queueSizeCalc(AuUtil.getAuRepoNode(au));
  }

  /** engqueue a size calculation for the node */
  public void queueSizeCalc(RepositoryNode node) {
    synchronized (sizeCalcQueue) {
      if (sizeCalcQueue.add(node)) {
        log.debug2("Queue size calc: " + node);
        startOrKickThread();
      }
    }
  }

  public int sizeCalcQueueLen() {
    synchronized (sizeCalcQueue) {
      return sizeCalcQueue.size();
    }
  }

  void startOrKickThread() {
    if (sizeCalcThread == null) {
      log.debug2("Starting thread");
      sizeCalcThread = new SizeCalcThread();
      sizeCalcThread.start();
      sizeCalcThread.waitRunning();
    }
    sizeCalcSem.give();
  }

  void stopThread() {
    if (sizeCalcThread != null) {
      log.debug2("Stopping thread");
      sizeCalcThread.stopSizeCalc();
      sizeCalcThread = null;
    }
  }

  void doSizeCalc(RepositoryNode node) {
    node.getTreeContentSize(null, true);
    if (node instanceof AuNodeImpl) {
      ((AuNodeImpl) node).getDiskUsage(true);
    }
  }

  long sleepTimeToAchieveLoad(long runDuration, float maxLoad) {
    return Math.round(((double) runDuration / maxLoad) - runDuration);
  }

  private class SizeCalcThread extends LockssThread {
    private volatile boolean goOn = true;

    private SizeCalcThread() {
      super("SizeCalc");
    }

    public void lockssRun() {
      setPriority(PRIORITY_PARAM_SIZE_CALC, PRIORITY_DEFAULT_SIZE_CALC);
      startWDog(WDOG_PARAM_SIZE_CALC, WDOG_DEFAULT_SIZE_CALC);
      triggerWDogOnExit(true);
      nowRunning();

      while (goOn) {
        try {
          pokeWDog();
          if (sizeCalcQueue.isEmpty()) {
            Deadline timeout = Deadline.in(Constants.HOUR);
            sizeCalcSem.take(timeout);
          }
          RepositoryNode node;
          synchronized (sizeCalcQueue) {
            node = (RepositoryNode) CollectionUtil.getAnElement(sizeCalcQueue);
          }
          if (node != null) {
            long start = TimeBase.nowMs();
            log.debug2("CalcSize start: " + node);
            long dur = 0;
            try {
              doSizeCalc(node);
              dur = TimeBase.nowMs() - start;
              log.debug2("CalcSize finish (" + StringUtil.timeIntervalToString(dur) + "): " + node);
            } catch (RuntimeException e) {
              log.warning("doSizeCalc: " + node, e);
            }
            synchronized (sizeCalcQueue) {
              sizeCalcQueue.remove(node);
            }
            pokeWDog();
            long sleep = sleepTimeToAchieveLoad(dur, sizeCalcMaxLoad);
            Deadline.in(sleep).sleep();
          }
        } catch (InterruptedException e) {
          // just wakeup and check for exit
        }
      }
      if (!goOn) {
        triggerWDogOnExit(false);
      }
    }

    private void stopSizeCalc() {
      goOn = false;
      interrupt();
    }
  }
}
Esempio n. 8
0
/** UI to invoke various daemon actions */
@SuppressWarnings("serial")
public class DebugPanel extends LockssServlet {

  public static final String PREFIX = Configuration.PREFIX + "debugPanel.";

  /** Priority for crawls started from the debug panel */
  public static final String PARAM_CRAWL_PRIORITY = PREFIX + "crawlPriority";

  public static final int DEFAULT_CRAWL_PRIORITY = 10;

  /** Priority for crawls started from the debug panel */
  public static final String PARAM_ENABLE_DEEP_CRAWL = PREFIX + "deepCrawlEnabled";

  private static final boolean DEFAULT_ENABLE_DEEP_CRAWL = false;

  static final String KEY_ACTION = "action";
  static final String KEY_MSG = "msg";
  static final String KEY_NAME_SEL = "name_sel";
  static final String KEY_NAME_TYPE = "name_type";
  static final String KEY_AUID = "auid";
  static final String KEY_URL = "url";
  static final String KEY_REFETCH_DEPTH = "depth";
  static final String KEY_TIME = "time";

  static final String ACTION_MAIL_BACKUP = "Mail Backup File";
  static final String ACTION_THROW_IOEXCEPTION = "Throw IOException";
  static final String ACTION_FIND_URL = "Find Preserved URL";

  public static final String ACTION_REINDEX_METADATA = "Reindex Metadata";
  public static final String ACTION_FORCE_REINDEX_METADATA = "Force Reindex Metadata";
  public static final String ACTION_START_V3_POLL = "Start V3 Poll";
  static final String ACTION_FORCE_START_V3_POLL = "Force V3 Poll";
  public static final String ACTION_START_CRAWL = "Start Crawl";
  public static final String ACTION_FORCE_START_CRAWL = "Force Start Crawl";
  public static final String ACTION_START_DEEP_CRAWL = "Deep Crawl";
  public static final String ACTION_FORCE_START_DEEP_CRAWL = "Force Deep Crawl";
  public static final String ACTION_CHECK_SUBSTANCE = "Check Substance";
  static final String ACTION_CRAWL_PLUGINS = "Crawl Plugins";
  static final String ACTION_RELOAD_CONFIG = "Reload Config";
  static final String ACTION_SLEEP = "Sleep";
  public static final String ACTION_DISABLE_METADATA_INDEXING = "Disable Indexing";

  /** Set of actions for which audit alerts shouldn't be generated */
  public static final Set noAuditActions = SetUtil.set(ACTION_FIND_URL);

  static final String COL2 = "colspan=2";
  static final String COL2CENTER = COL2 + " align=center";

  static Logger log = Logger.getLogger("DebugPanel");

  private LockssDaemon daemon;
  private PluginManager pluginMgr;
  private PollManager pollManager;
  private CrawlManager crawlMgr;
  private ConfigManager cfgMgr;
  private DbManager dbMgr;
  private MetadataManager metadataMgr;
  private RemoteApi rmtApi;

  boolean showResult;
  boolean showForcePoll;
  boolean showForceCrawl;
  boolean showForceReindexMetadata;

  String formAuid;
  String formDepth = "100";

  protected void resetLocals() {
    resetVars();
    super.resetLocals();
  }

  void resetVars() {
    formAuid = null;
    errMsg = null;
    statusMsg = null;
    showForcePoll = false;
    showForceCrawl = false;
    showForceReindexMetadata = false;
  }

  public void init(ServletConfig config) throws ServletException {
    super.init(config);
    daemon = getLockssDaemon();
    pluginMgr = daemon.getPluginManager();
    pollManager = daemon.getPollManager();
    crawlMgr = daemon.getCrawlManager();
    cfgMgr = daemon.getConfigManager();
    rmtApi = daemon.getRemoteApi();
    try {
      dbMgr = daemon.getDbManager();
      metadataMgr = daemon.getMetadataManager();
    } catch (IllegalArgumentException ex) {
    }
  }

  public void lockssHandleRequest() throws IOException {
    resetVars();
    boolean showForm = true;
    String action = getParameter(KEY_ACTION);

    if (!StringUtil.isNullString(action)) {

      formAuid = getParameter(KEY_AUID);
      formDepth = getParameter(KEY_REFETCH_DEPTH);

      UserAccount acct = getUserAccount();
      if (acct != null && !noAuditActions.contains(action)) {
        acct.auditableEvent("used debug panel action: " + action + " AU ID: " + formAuid);
      }
    }

    if (ACTION_MAIL_BACKUP.equals(action)) {
      doMailBackup();
    }
    if (ACTION_RELOAD_CONFIG.equals(action)) {
      doReloadConfig();
    }
    if (ACTION_SLEEP.equals(action)) {
      doSleep();
    }
    if (ACTION_THROW_IOEXCEPTION.equals(action)) {
      doThrow();
    }
    if (ACTION_START_V3_POLL.equals(action)) {
      doV3Poll();
    }
    if (ACTION_FORCE_START_V3_POLL.equals(action)) {
      forceV3Poll();
    }
    if (ACTION_START_CRAWL.equals(action)) {
      doCrawl(false, false);
    }
    if (ACTION_FORCE_START_CRAWL.equals(action)) {
      doCrawl(true, false);
    }
    if (ACTION_START_DEEP_CRAWL.equals(action)) {
      doCrawl(false, true);
    }
    if (ACTION_FORCE_START_DEEP_CRAWL.equals(action)) {
      doCrawl(true, true);
    }
    if (ACTION_CHECK_SUBSTANCE.equals(action)) {
      doCheckSubstance();
    }
    if (ACTION_CRAWL_PLUGINS.equals(action)) {
      crawlPluginRegistries();
    }
    if (ACTION_FIND_URL.equals(action)) {
      showForm = doFindUrl();
    }
    if (ACTION_REINDEX_METADATA.equals(action)) {
      doReindexMetadata();
    }
    if (ACTION_FORCE_REINDEX_METADATA.equals(action)) {
      forceReindexMetadata();
    }
    if (ACTION_DISABLE_METADATA_INDEXING.equals(action)) {
      doDisableMetadataIndexing();
    }
    if (showForm) {
      displayPage();
    }
  }

  private void doMailBackup() {
    try {
      rmtApi.createConfigBackupFile(RemoteApi.BackupFileDisposition.Mail);
    } catch (Exception e) {
      errMsg = "Error: " + e.getMessage();
    }
  }

  private void doReloadConfig() {
    cfgMgr.requestReload();
  }

  private void doThrow() throws IOException {
    String msg = getParameter(KEY_MSG);
    throw new IOException(msg != null ? msg : "Test message");
  }

  private void doSleep() throws IOException {
    String timestr = getParameter(KEY_TIME);
    try {
      long time = StringUtil.parseTimeInterval(timestr);
      Deadline.in(time).sleep();
      statusMsg = "Slept for " + StringUtil.timeIntervalToString(time);
    } catch (NumberFormatException e) {
      errMsg = "Illegal duration: " + e;
    } catch (InterruptedException e) {
      errMsg = "Interrupted: " + e;
    }
  }

  private void doReindexMetadata() {
    ArchivalUnit au = getAu();
    if (au == null) return;
    try {
      startReindexingMetadata(au, false);
    } catch (RuntimeException e) {
      log.error("Can't reindex metadata", e);
      errMsg = "Error: " + e.toString();
    }
  }

  private void forceReindexMetadata() {
    ArchivalUnit au = getAu();
    if (au == null) return;
    try {
      startReindexingMetadata(au, true);
    } catch (RuntimeException e) {
      log.error("Can't reindex metadata", e);
      errMsg = "Error: " + e.toString();
    }
  }

  private void doDisableMetadataIndexing() {
    ArchivalUnit au = getAu();
    if (au == null) return;
    try {
      disableMetadataIndexing(au, false);
    } catch (RuntimeException e) {
      log.error("Can't disable metadata indexing", e);
      errMsg = "Error: " + e.toString();
    }
  }

  private void doCrawl(boolean force, boolean deep) {
    ArchivalUnit au = getAu();
    if (au == null) return;
    try {
      startCrawl(au, force, deep);
    } catch (CrawlManagerImpl.NotEligibleException.RateLimiter e) {
      errMsg = "AU has crawled recently (" + e.getMessage() + ").  Click again to override.";
      showForceCrawl = true;
      return;
    } catch (CrawlManagerImpl.NotEligibleException e) {
      errMsg = "Can't enqueue crawl: " + e.getMessage();
    }
  }

  private void crawlPluginRegistries() {
    StringBuilder sb = new StringBuilder();
    for (ArchivalUnit au : pluginMgr.getAllRegistryAus()) {
      sb.append(au.getName());
      sb.append(": ");
      try {
        startCrawl(au, true, false);
        sb.append("Queued.");
      } catch (CrawlManagerImpl.NotEligibleException e) {
        sb.append("Failed: ");
        sb.append(e.getMessage());
      }
      sb.append("\n");
    }
    statusMsg = sb.toString();
  }

  private boolean startCrawl(ArchivalUnit au, boolean force, boolean deep)
      throws CrawlManagerImpl.NotEligibleException {
    CrawlManagerImpl cmi = (CrawlManagerImpl) crawlMgr;
    if (force) {
      RateLimiter limit = cmi.getNewContentRateLimiter(au);
      if (!limit.isEventOk()) {
        limit.unevent();
      }
    }
    cmi.checkEligibleToQueueNewContentCrawl(au);
    String delayMsg = "";
    String deepMsg = "";
    try {
      cmi.checkEligibleForNewContentCrawl(au);
    } catch (CrawlManagerImpl.NotEligibleException e) {
      delayMsg = ", Start delayed due to: " + e.getMessage();
    }
    Configuration config = ConfigManager.getCurrentConfig();
    int pri = config.getInt(PARAM_CRAWL_PRIORITY, DEFAULT_CRAWL_PRIORITY);

    CrawlReq req;
    try {
      req = new CrawlReq(au);
      req.setPriority(pri);
      if (deep) {
        int d = Integer.parseInt(formDepth);
        if (d < 0) {
          errMsg = "Illegal refetch depth: " + d;
          return false;
        }
        req.setRefetchDepth(d);
        deepMsg = "Deep (" + req.getRefetchDepth() + ") ";
      }
    } catch (NumberFormatException e) {
      errMsg = "Illegal refetch depth: " + formDepth;
      return false;
    } catch (RuntimeException e) {
      log.error("Couldn't create CrawlReq: " + au, e);
      errMsg = "Couldn't create CrawlReq: " + e.toString();
      return false;
    }
    cmi.startNewContentCrawl(req, null);
    statusMsg = deepMsg + "Crawl requested for " + au.getName() + delayMsg;
    return true;
  }

  private void doCheckSubstance() {
    ArchivalUnit au = getAu();
    if (au == null) return;
    try {
      checkSubstance(au);
    } catch (RuntimeException e) {
      log.error("Error in SubstanceChecker", e);
      errMsg = "Error in SubstanceChecker; see log.";
    }
  }

  private void checkSubstance(ArchivalUnit au) {
    SubstanceChecker subChecker = new SubstanceChecker(au);
    if (!subChecker.isEnabled()) {
      errMsg = "No substance patterns defined for plugin.";
      return;
    }
    AuState auState = AuUtil.getAuState(au);
    SubstanceChecker.State oldState = auState.getSubstanceState();
    SubstanceChecker.State newState = subChecker.findSubstance();
    String chtxt = (newState == oldState ? "(unchanged)" : "(was " + oldState.toString() + ")");
    switch (newState) {
      case Unknown:
        log.error("Shouldn't happen: SubstanceChecker returned Unknown");
        errMsg = "Error in SubstanceChecker; see log.";
        break;
      case Yes:
        statusMsg = "AU has substance " + chtxt + ": " + au.getName();
        auState.setSubstanceState(SubstanceChecker.State.Yes);
        break;
      case No:
        statusMsg = "AU has no substance " + chtxt + ": " + au.getName();
        auState.setSubstanceState(SubstanceChecker.State.No);
        break;
    }
  }

  private boolean startReindexingMetadata(ArchivalUnit au, boolean force) {
    if (metadataMgr == null) {
      errMsg = "Metadata processing is not enabled.";
      return false;
    }

    if (!force) {
      if (!AuUtil.hasCrawled(au)) {
        errMsg = "Au has never crawled. Click again to reindex metadata";
        showForceReindexMetadata = true;
        return false;
      }

      AuState auState = AuUtil.getAuState(au);
      switch (auState.getSubstanceState()) {
        case No:
          errMsg = "Au has no substance. Click again to reindex metadata";
          showForceReindexMetadata = true;
          return false;
        case Unknown:
          errMsg = "Unknown substance for Au. Click again to reindex metadata.";
          showForceReindexMetadata = true;
          return false;
        case Yes:
          // fall through
      }
    }

    // Fully reindex metadata with the highest priority.
    Connection conn = null;
    PreparedStatement insertPendingAuBatchStatement = null;

    try {
      conn = dbMgr.getConnection();
      insertPendingAuBatchStatement = metadataMgr.getPrioritizedInsertPendingAuBatchStatement(conn);

      if (metadataMgr.enableAndAddAuToReindex(
          au, conn, insertPendingAuBatchStatement, false, true)) {
        statusMsg = "Reindexing metadata for " + au.getName();
        return true;
      }
    } catch (DbException dbe) {
      log.error("Cannot reindex metadata for " + au.getName(), dbe);
    } finally {
      DbManager.safeCloseStatement(insertPendingAuBatchStatement);
      DbManager.safeRollbackAndClose(conn);
    }

    if (force) {
      errMsg = "Still cannot reindex metadata for " + au.getName();
    } else {
      errMsg = "Cannot reindex metadata for " + au.getName();
    }
    return false;
  }

  private boolean disableMetadataIndexing(ArchivalUnit au, boolean force) {
    if (metadataMgr == null) {
      errMsg = "Metadata processing is not enabled.";
      return false;
    }

    try {
      metadataMgr.disableAuIndexing(au);
      statusMsg = "Disabled metadata indexing for " + au.getName();
      return true;
    } catch (Exception e) {
      errMsg = "Cannot reindex metadata for " + au.getName() + ": " + e.getMessage();
      return false;
    }
  }

  private void doV3Poll() {
    ArchivalUnit au = getAu();
    if (au == null) return;
    try {
      callV3ContentPoll(au);
    } catch (PollManager.NotEligibleException e) {
      errMsg = "AU is not eligible for poll: " + e.getMessage();
      //       errMsg = "Ineligible: " + e.getMessage() +
      // 	"<br>Click again to force new poll.";
      //       showForcePoll = true;
      return;
    } catch (Exception e) {
      log.error("Can't start poll", e);
      errMsg = "Error: " + e.toString();
    }
  }

  private void forceV3Poll() {
    ArchivalUnit au = getAu();
    if (au == null) return;
    try {
      callV3ContentPoll(au);
    } catch (Exception e) {
      log.error("Can't start poll", e);
      errMsg = "Error: " + e.toString();
    }
  }

  private void callV3ContentPoll(ArchivalUnit au) throws PollManager.NotEligibleException {
    log.debug("Enqueuing a V3 Content Poll on " + au.getName());
    PollSpec spec = new PollSpec(au.getAuCachedUrlSet(), Poll.V3_POLL);
    pollManager.enqueueHighPriorityPoll(au, spec);
    statusMsg = "Enqueued V3 poll for " + au.getName();
  }

  private boolean doFindUrl() throws IOException {

    String url = getParameter(KEY_URL);

    String redir =
        srvURL(
            AdminServletManager.SERVLET_DAEMON_STATUS,
            PropUtil.fromArgs("table", ArchivalUnitStatus.AUS_WITH_URL_TABLE_NAME, "key", url));

    resp.setContentLength(0);
    //     resp.sendRedirect(resp.encodeRedirectURL(redir));
    resp.sendRedirect(redir);
    return false;
  }

  ArchivalUnit getAu() {
    if (StringUtil.isNullString(formAuid)) {
      errMsg = "Select an AU";
      return null;
    }
    ArchivalUnit au = pluginMgr.getAuFromId(formAuid);
    if (au == null) {
      errMsg = "No such AU.  Select an AU";
      return null;
    }
    return au;
  }

  private void displayPage() throws IOException {
    Page page = newPage();
    layoutErrorBlock(page);
    ServletUtil.layoutExplanationBlock(page, "Debug Actions");
    page.add(makeForm());
    page.add("<br>");
    endPage(page);
  }

  private Element makeForm() {
    Composite comp = new Composite();
    Form frm = new Form(srvURL(myServletDescr()));
    frm.method("POST");

    frm.add("<br><center>");
    Input reload = new Input(Input.Submit, KEY_ACTION, ACTION_RELOAD_CONFIG);
    setTabOrder(reload);
    frm.add(reload);
    frm.add(" ");
    Input backup = new Input(Input.Submit, KEY_ACTION, ACTION_MAIL_BACKUP);
    setTabOrder(backup);
    frm.add(backup);
    frm.add(" ");
    Input crawlplug = new Input(Input.Submit, KEY_ACTION, ACTION_CRAWL_PLUGINS);
    setTabOrder(crawlplug);
    frm.add(crawlplug);
    frm.add("</center>");
    ServletDescr d1 = AdminServletManager.SERVLET_HASH_CUS;
    if (isServletRunnable(d1)) {
      frm.add("<br><center>" + srvLink(d1, d1.heading) + "</center>");
    }
    Input findUrl = new Input(Input.Submit, KEY_ACTION, ACTION_FIND_URL);
    Input findUrlText = new Input(Input.Text, KEY_URL);
    findUrlText.setSize(50);
    setTabOrder(findUrl);
    setTabOrder(findUrlText);
    frm.add("<br><center>" + findUrl + " " + findUrlText + "</center>");

    Input thrw = new Input(Input.Submit, KEY_ACTION, ACTION_THROW_IOEXCEPTION);
    Input thmsg = new Input(Input.Text, KEY_MSG);
    setTabOrder(thrw);
    setTabOrder(thmsg);
    frm.add("<br><center>" + thrw + " " + thmsg + "</center>");

    frm.add("<br><center>AU Actions: select AU</center>");
    Composite ausel = ServletUtil.layoutSelectAu(this, KEY_AUID, formAuid);
    frm.add("<br><center>" + ausel + "</center>");
    setTabOrder(ausel);

    Input v3Poll =
        new Input(
            Input.Submit,
            KEY_ACTION,
            (showForcePoll ? ACTION_FORCE_START_V3_POLL : ACTION_START_V3_POLL));
    Input crawl =
        new Input(
            Input.Submit,
            KEY_ACTION,
            (showForceCrawl ? ACTION_FORCE_START_CRAWL : ACTION_START_CRAWL));

    frm.add("<br><center>");
    frm.add(v3Poll);
    frm.add(" ");
    frm.add(crawl);
    if (CurrentConfig.getBooleanParam(PARAM_ENABLE_DEEP_CRAWL, DEFAULT_ENABLE_DEEP_CRAWL)) {
      Input deepCrawl =
          new Input(
              Input.Submit,
              KEY_ACTION,
              (showForceCrawl ? ACTION_FORCE_START_DEEP_CRAWL : ACTION_START_DEEP_CRAWL));
      Input depthText = new Input(Input.Text, KEY_REFETCH_DEPTH, formDepth);
      depthText.setSize(4);
      setTabOrder(depthText);
      frm.add(" ");
      frm.add(deepCrawl);
      frm.add(depthText);
    }
    Input checkSubstance = new Input(Input.Submit, KEY_ACTION, ACTION_CHECK_SUBSTANCE);
    frm.add("<br>");
    frm.add(checkSubstance);
    if (metadataMgr != null) {
      Input reindex =
          new Input(
              Input.Submit,
              KEY_ACTION,
              (showForceReindexMetadata ? ACTION_FORCE_REINDEX_METADATA : ACTION_REINDEX_METADATA));
      frm.add(" ");
      frm.add(reindex);
      Input disableIndexing = new Input(Input.Submit, KEY_ACTION, ACTION_DISABLE_METADATA_INDEXING);
      frm.add(" ");
      frm.add(disableIndexing);
    }
    frm.add("</center>");

    comp.add(frm);
    return comp;
  }
}
Esempio n. 9
0
/**
 * Class implementing the concept of the set of URLs covered by a poll.
 *
 * @author Claire Griffin
 * @version 1.0
 */
public class PollSpec {
  /**
   * A lower bound value which indicates the poll should use a {@link SingleNodeCachedUrlSetSpec}
   * instead of a {@link RangeCachedUrlSetSpec}.
   */
  public static final String SINGLE_NODE_LWRBOUND = ".";

  public static final String DEFAULT_PLUGIN_VERSION = "1";

  private static Logger theLog = Logger.getLogger("PollSpec");

  private String auId;
  private String pluginVersion;
  private String url;
  private String uprBound = null;
  private String lwrBound = null;
  private CachedUrlSet cus = null;
  private PluginManager pluginMgr = null;
  private int protocolVersion; // poll protocol version
  private int pollType; // One of the types defined by Poll
  private V3Poller.PollVariant variant = V3Poller.PollVariant.PoR;

  /**
   * Construct a PollSpec from a CachedUrlSet and an upper and lower bound
   *
   * @param cus the CachedUrlSet
   * @param lwrBound the lower boundary
   * @param uprBound the upper boundary
   * @param pollType one of the types defined by Poll
   */
  public PollSpec(CachedUrlSet cus, String lwrBound, String uprBound, int pollType) {
    commonSetup(cus, lwrBound, uprBound, pollType);
  }

  /**
   * Construct a PollSpec from a CachedUrlSet.
   *
   * @param cus the CachedUrlSpec which defines the range of interest
   * @param pollType one of the types defined by Poll
   */
  public PollSpec(CachedUrlSet cus, int pollType) {
    CachedUrlSetSpec cuss = cus.getSpec();
    if (cuss instanceof RangeCachedUrlSetSpec) {
      RangeCachedUrlSetSpec rcuss = (RangeCachedUrlSetSpec) cuss;
      commonSetup(cus, rcuss.getLowerBound(), rcuss.getUpperBound(), pollType);
    } else if (cuss.isSingleNode()) {
      commonSetup(cus, SINGLE_NODE_LWRBOUND, null, pollType);
    } else {
      commonSetup(cus, null, null, pollType);
    }
  }

  /**
   * Construct a PollSpec from a V1 LcapMessage
   *
   * @param msg the LcapMessage which defines the range of interest
   */
  public PollSpec(V1LcapMessage msg) {
    auId = msg.getArchivalId();
    pluginVersion = msg.getPluginVersion();
    url = msg.getTargetUrl();
    uprBound = msg.getUprBound();
    lwrBound = msg.getLwrBound();
    protocolVersion = msg.getProtocolVersion();
    if (msg.isContentPoll()) {
      pollType = Poll.V1_CONTENT_POLL;
    } else if (msg.isNamePoll()) {
      pollType = Poll.V1_NAME_POLL;
    } else if (msg.isVerifyPoll()) {
      pollType = Poll.V1_VERIFY_POLL;
    } else {
      pollType = -1;
    }
    cus = getPluginManager().findCachedUrlSet(this);
  }

  public PollSpec(V3LcapMessage msg) {
    this(
        msg.getArchivalId(),
        (msg.getTargetUrl() == null) ? "lockssau:" : msg.getTargetUrl(),
        null,
        null,
        Poll.V3_POLL);
    protocolVersion = msg.getProtocolVersion();
    pluginVersion = msg.getPluginVersion();
  }

  /** Construct a PollSpec from explicit args */
  public PollSpec(String auId, String url, String lower, String upper, int pollType) {
    this.auId = auId;
    this.url = url;
    uprBound = upper;
    lwrBound = lower;
    this.pollType = pollType;
    cus = getPluginManager().findCachedUrlSet(this);
    this.protocolVersion = protocolVersionFromPollType(pollType);
  }

  /**
   * Construct a PollSpec from another PollSpec and a poll type XXX it seems that other constructors
   * are not setting all fields
   */
  public PollSpec(PollSpec ps, int pollType) {
    this.auId = ps.auId;
    this.pluginVersion = ps.pluginVersion;
    this.url = ps.url;
    this.uprBound = ps.uprBound;
    this.lwrBound = ps.lwrBound;
    this.cus = ps.cus;
    this.pluginMgr = ps.pluginMgr;
    this.protocolVersion = ps.protocolVersion;
    this.pollType = pollType;
  }

  /** Setup common to most constructors */
  private void commonSetup(CachedUrlSet cus, String lwrBound, String uprBound, int pollType) {
    CachedUrlSetSpec cuss = cus.getSpec();
    if (cuss instanceof PrunedCachedUrlSetSpec) {
      throw new IllegalArgumentException("Polls do not support PrunedCachedUrlSetSpec");
    }
    this.cus = cus;
    ArchivalUnit au = cus.getArchivalUnit();
    auId = au.getAuId();
    this.pluginVersion = AuUtil.getPollVersion(au);
    url = cuss.getUrl();
    this.lwrBound = lwrBound;
    this.uprBound = uprBound;
    this.protocolVersion = protocolVersionFromPollType(pollType);
    this.pollType = pollType;
  }

  public CachedUrlSet getCachedUrlSet() {
    return cus;
  }

  public String getAuId() {
    return auId;
  }

  public String getPluginVersion() {
    return (pluginVersion != null) ? pluginVersion : DEFAULT_PLUGIN_VERSION;
  }

  public String getUrl() {
    return url;
  }

  public String getLwrBound() {
    return lwrBound;
  }

  public String getUprBound() {
    return uprBound;
  }

  public String getRangeString() {
    if (StringUtil.equalStrings(lwrBound, SINGLE_NODE_LWRBOUND)) {
      return "single node";
    }
    if (lwrBound != null || uprBound != null) {
      String lwrDisplay = lwrBound;
      String uprDisplay = uprBound;
      if (lwrBound != null && lwrBound.startsWith("/")) {
        lwrDisplay = lwrBound.substring(1);
      }
      if (uprBound != null && uprBound.startsWith("/")) {
        uprDisplay = uprBound.substring(1);
      }
      return lwrDisplay + " - " + uprDisplay;
    }
    return null;
  }

  public int getProtocolVersion() {
    return protocolVersion;
  }

  public int getPollType() {
    return pollType;
  }

  public V3Poller.PollVariant getPollVariant() {
    return variant;
  }

  public void setPollVariant(V3Poller.PollVariant v) {
    variant = v;
  }

  private PluginManager getPluginManager() {
    if (pluginMgr == null) {
      pluginMgr = (PluginManager) LockssDaemon.getManager(LockssDaemon.PLUGIN_MANAGER);
    }
    return pluginMgr;
  }

  /**
   * Given a poll type, return the correct version of the protocol to use.
   *
   * @param pollType
   * @return The protocol version to use
   */
  private int protocolVersionFromPollType(int pollType) {
    switch (pollType) {
      case Poll.V1_CONTENT_POLL:
      case Poll.V1_NAME_POLL:
      case Poll.V1_VERIFY_POLL:
        return Poll.V1_PROTOCOL;
      case Poll.V3_POLL:
        return Poll.V3_PROTOCOL;
      default:
        return Poll.UNDEFINED_PROTOCOL;
    }
  }

  public String toString() {
    return "[PS: "
        + Poll.POLL_NAME[pollType]
        + " auid="
        + auId
        + ", url="
        + url
        + ", l="
        + lwrBound
        + ", u="
        + uprBound
        + ", type="
        + pollType
        + ", plugVer="
        + getPluginVersion()
        + ", protocol="
        + protocolVersion
        + "]";
  }
}
Esempio n. 10
0
/**
 * DefinablePlugin: a plugin which uses the data stored in an ExternalizableMap to configure itself.
 *
 * @author Claire Griffin
 * @version 1.0
 */
public class DefinablePlugin extends BasePlugin {
  // configuration map keys
  public static final String KEY_PLUGIN_IDENTIFIER = "plugin_identifier";
  public static final String KEY_PLUGIN_NAME = "plugin_name";
  public static final String KEY_PLUGIN_VERSION = "plugin_version";
  public static final String KEY_PLUGIN_FEATURE_VERSION_MAP = "plugin_feature_version_map";
  public static final String KEY_REQUIRED_DAEMON_VERSION = "required_daemon_version";
  public static final String KEY_PUBLISHING_PLATFORM = "plugin_publishing_platform";
  public static final String KEY_PLUGIN_CONFIG_PROPS = "plugin_config_props";
  public static final String KEY_EXCEPTION_HANDLER = "plugin_cache_result_handler";
  public static final String KEY_EXCEPTION_LIST = "plugin_cache_result_list";
  public static final String KEY_PLUGIN_NOTES = "plugin_notes";
  public static final String KEY_CRAWL_TYPE = "plugin_crawl_type";
  public static final String KEY_FOLLOW_LINKS = "plugin_follow_link";
  /** Message to be displayed when user configures an AU with this plugin */
  public static final String KEY_PLUGIN_AU_CONFIG_USER_MSG = "plugin_au_config_user_msg";

  public static final String KEY_PER_HOST_PERMISSION_PATH = "plugin_per_host_permission_path";
  public static final String KEY_PLUGIN_PARENT = "plugin_parent";
  public static final String KEY_PLUGIN_PARENT_VERSION = "plugin_parent_version";
  public static final String KEY_PLUGIN_CRAWL_URL_COMPARATOR_FACTORY =
      "plugin_crawl_url_comparator_factory";
  public static final String KEY_PLUGIN_FETCH_RATE_LIMITER_SOURCE =
      "plugin_fetch_rate_limiter_source";

  public static final String KEY_PLUGIN_ARTICLE_ITERATOR_FACTORY =
      "plugin_article_iterator_factory";

  public static final String KEY_PLUGIN_ARTICLE_METADATA_EXTRACTOR_FACTORY =
      "plugin_article_metadata_extractor_factory";

  public static final String KEY_DEFAULT_ARTICLE_MIME_TYPE = "plugin_default_article_mime_type";

  public static final String KEY_ARTICLE_ITERATOR_ROOT = "plugin_article_iterator_root";

  public static final String KEY_ARTICLE_ITERATOR_PATTERN = "plugin_article_iterator_pattern";

  public static final String DEFAULT_PLUGIN_VERSION = "1";
  public static final String DEFAULT_REQUIRED_DAEMON_VERSION = "0.0.0";

  public static final String MAP_SUFFIX = ".xml";

  public static final String CRAWL_TYPE_HTML_LINKS = "HTML Links";
  public static final String CRAWL_TYPE_OAI = "OAI";
  public static final String[] CRAWL_TYPES = {
    CRAWL_TYPE_HTML_LINKS, CRAWL_TYPE_OAI,
  };
  public static final String DEFAULT_CRAWL_TYPE = CRAWL_TYPE_HTML_LINKS;

  protected String mapName = null;

  static Logger log = Logger.getLogger("DefinablePlugin");

  protected ExternalizableMap definitionMap = new ExternalizableMap();
  protected CacheResultHandler resultHandler = null;
  protected List<String> loadedFromUrls;
  protected CrawlWindow crawlWindow;
  protected Map<Plugin.Feature, String> featureVersion;

  public void initPlugin(LockssDaemon daemon, String extMapName) throws FileNotFoundException {
    initPlugin(daemon, extMapName, this.getClass().getClassLoader());
  }

  public void initPlugin(LockssDaemon daemon, String extMapName, ClassLoader loader)
      throws FileNotFoundException {
    // convert the plugin class name to an xml file name
    // load the configuration map from jar file
    ExternalizableMap defMap = loadMap(extMapName, loader);
    this.initPlugin(daemon, extMapName, defMap, loader);
  }

  public void initPlugin(
      LockssDaemon daemon, String extMapName, ExternalizableMap defMap, ClassLoader loader) {
    mapName = extMapName;
    this.classLoader = loader;
    this.definitionMap = defMap;
    super.initPlugin(daemon);
    initMimeMap();
    initFeatureVersions();
    initAuFeatureMap();
    checkParamAgreement();
  }

  private ExternalizableMap loadMap(String extMapName, ClassLoader loader)
      throws FileNotFoundException {
    String first = null;
    String next = extMapName;
    List<String> urls = new ArrayList<String>();
    ExternalizableMap res = null;
    while (next != null) {
      // convert the plugin class name to an xml file name
      String mapFile = next.replace('.', '/') + MAP_SUFFIX;
      URL url = loader.getResource(mapFile);
      if (url != null && urls.contains(url.toString())) {
        throw new PluginException.InvalidDefinition("Plugin inheritance loop: " + next);
      }
      // load into map
      ExternalizableMap oneMap = new ExternalizableMap();
      oneMap.loadMapFromResource(mapFile, loader);
      urls.add(url.toString());
      // apply overrides one plugin at a time in inheritance chain
      processOverrides(oneMap);
      if (res == null) {
        res = oneMap;
      } else {
        for (Map.Entry ent : oneMap.entrySet()) {
          String key = (String) ent.getKey();
          Object val = ent.getValue();
          if (!res.containsKey(key)) {
            res.setMapElement(key, val);
          }
        }
      }
      if (oneMap.containsKey(KEY_PLUGIN_PARENT)) {
        next = oneMap.getString(KEY_PLUGIN_PARENT);
      } else {
        next = null;
      }
    }
    loadedFromUrls = urls;
    return res;
  }

  /** If in testing mode FOO, copy values from FOO_override map, if any, to main map */
  void processOverrides(TypedEntryMap map) {
    String testMode = getTestingMode();
    if (StringUtil.isNullString(testMode)) {
      return;
    }
    Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE);
    if (o == null) {
      return;
    }
    if (o instanceof Map) {
      Map overrideMap = (Map) o;
      for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) {
        String key = (String) entry.getKey();
        Object val = entry.getValue();
        log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val);
        map.setMapElement(key, val);
      }
    }
  }

  String getTestingMode() {
    return theDaemon == null ? null : theDaemon.getTestingMode();
  }

  // Used by tests

  public void initPlugin(LockssDaemon daemon, File file) throws PluginException {
    ExternalizableMap oneMap = new ExternalizableMap();
    oneMap.loadMap(file);
    if (oneMap.getErrorString() != null) {
      throw new PluginException(oneMap.getErrorString());
    }
    initPlugin(daemon, file.getPath(), oneMap, null);
  }

  void initPlugin(LockssDaemon daemon, ExternalizableMap defMap) {
    initPlugin(daemon, defMap, this.getClass().getClassLoader());
  }

  void initPlugin(LockssDaemon daemon, ExternalizableMap defMap, ClassLoader loader) {
    initPlugin(daemon, "Internal", defMap, loader);
  }

  enum PrintfContext {
    Regexp,
    URL,
    Display
  };

  void checkParamAgreement() {
    for (Map.Entry<String, PrintfContext> ent :
        DefinableArchivalUnit.printfKeysContext.entrySet()) {
      checkParamAgreement(ent.getKey(), ent.getValue());
    }
  }

  void checkParamAgreement(String key, PrintfContext context) {
    List<String> printfList = getElementList(key);
    if (printfList == null) {
      return;
    }
    for (String printf : printfList) {
      if (StringUtil.isNullString(printf)) {
        log.warning("Null printf string in " + key);
        continue;
      }
      PrintfUtil.PrintfData p_data = PrintfUtil.stringToPrintf(printf);
      Collection<String> p_args = p_data.getArguments();
      for (String arg : p_args) {
        ConfigParamDescr descr = findAuConfigDescr(arg);
        if (descr == null) {
          throw new PluginException.InvalidDefinition(
              "Not a declared parameter: " + arg + " in " + printf + " in " + getPluginName());
        }
        // ensure range and set params used only in legal context
        switch (context) {
          case Regexp:
          case Display:
            // everything is legal in a regexp or a display string
            break;
          case URL:
            // NUM_RANGE and SET legal because can enumerate.  Can't
            // enumerate RANGE
            switch (descr.getType()) {
              case ConfigParamDescr.TYPE_RANGE:
                throw new PluginException.InvalidDefinition(
                    "Range parameter ("
                        + arg
                        + ") used in illegal context in "
                        + getPluginName()
                        + ": "
                        + key
                        + ": "
                        + printf);
              default:
            }
        }
      }
    }
  }

  public List<String> getLoadedFromUrls() {
    return loadedFromUrls;
  }

  public String getPluginName() {
    if (definitionMap.containsKey(KEY_PLUGIN_NAME)) {
      return definitionMap.getString(KEY_PLUGIN_NAME);
    } else {
      return getDefaultPluginName();
    }
  }

  protected String getDefaultPluginName() {
    return StringUtil.shortName(getPluginId());
  }

  public String getVersion() {
    return definitionMap.getString(KEY_PLUGIN_VERSION, DEFAULT_PLUGIN_VERSION);
  }

  public String getFeatureVersion(Plugin.Feature feat) {
    if (featureVersion == null) {
      return null;
    }
    return featureVersion.get(feat);
  }

  public String getRequiredDaemonVersion() {
    return definitionMap.getString(KEY_REQUIRED_DAEMON_VERSION, DEFAULT_REQUIRED_DAEMON_VERSION);
  }

  public String getPublishingPlatform() {
    return definitionMap.getString(KEY_PUBLISHING_PLATFORM, null);
  }

  public String getPluginNotes() {
    return definitionMap.getString(KEY_PLUGIN_NOTES, null);
  }

  public String getDefaultArticleMimeType() {
    String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null);
    log.debug3("DefaultArticleMimeType " + ret);
    if (ret == null) {
      ret = super.getDefaultArticleMimeType();
      log.debug3("DefaultArticleMimeType from super " + ret);
    }
    return ret;
  }

  public List<String> getElementList(String key) {
    Object element = definitionMap.getMapElement(key);
    List<String> lst;

    if (element instanceof String) {
      return Collections.singletonList((String) element);
    } else if (element instanceof List) {
      return (List) element;
    } else {
      return null;
    }
  }

  public List getLocalAuConfigDescrs() throws PluginException.InvalidDefinition {
    List auConfigDescrs = (List) definitionMap.getCollection(KEY_PLUGIN_CONFIG_PROPS, null);
    if (auConfigDescrs == null) {
      throw new PluginException.InvalidDefinition(mapName + " missing ConfigParamDescrs");
    }
    return auConfigDescrs;
  }

  protected ArchivalUnit createAu0(Configuration auConfig)
      throws ArchivalUnit.ConfigurationException {
    DefinableArchivalUnit au = new DefinableArchivalUnit(this, definitionMap);
    au.setConfiguration(auConfig);
    return au;
  }

  public ExternalizableMap getDefinitionMap() {
    return definitionMap;
  }

  CacheResultHandler getCacheResultHandler() {
    return resultHandler;
  }

  String stripSuffix(String str, String suffix) {
    return str.substring(0, str.length() - suffix.length());
  }

  protected void initMimeMap() throws PluginException.InvalidDefinition {
    for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) {
      Map.Entry ent = (Map.Entry) iter.next();
      String key = (String) ent.getKey();
      Object val = ent.getValue();
      if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY);
        if (val instanceof String) {
          String factName = (String) val;
          log.debug(mime + " link extractor: " + factName);
          MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
          LinkExtractorFactory fact =
              (LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class);
          mti.setLinkExtractorFactory(fact);
        }
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) {
        // XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY
        // XXX unless/until that key is changed to not be a terminal substring
        // XXX of this one
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY);
        if (val instanceof String) {
          String factName = (String) val;
          log.debug(mime + " crawl filter: " + factName);
          MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
          FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
          mti.setCrawlFilterFactory(fact);
        }
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY);
        if (val instanceof String) {
          String factName = (String) val;
          log.debug(mime + " filter: " + factName);
          MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
          FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
          mti.setHashFilterFactory(fact);
        }
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT);
        if (val instanceof String) {
          String rate = (String) val;
          log.debug(mime + " fetch rate: " + rate);
          MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
          RateLimiter limit = mti.getFetchRateLimiter();
          if (limit != null) {
            limit.setRate(rate);
          } else {
            mti.setFetchRateLimiter(new RateLimiter(rate));
          }
        }
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY);
        String factName = (String) val;
        log.debug(mime + " link rewriter: " + factName);
        MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
        LinkRewriterFactory fact =
            (LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class);
        mti.setLinkRewriterFactory(fact);
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP);
        Map factNameMap = (Map) val;
        Map factClassMap = new HashMap();
        MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
        for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) {
          String mdTypes = (String) it.next();
          String factName = (String) factNameMap.get(mdTypes);
          log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName);
          for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) {
            setMdTypeFact(factClassMap, mdType, factName);
          }
        }
        mti.setFileMetadataExtractorFactoryMap(factClassMap);
      }
    }
  }

  private void setMdTypeFact(Map factClassMap, String mdType, String factName) {
    log.debug3("Metadata type: " + mdType + " factory " + factName);
    FileMetadataExtractorFactory fact =
        (FileMetadataExtractorFactory) newAuxClass(factName, FileMetadataExtractorFactory.class);
    factClassMap.put(mdType, fact);
  }

  protected void initResultMap() throws PluginException.InvalidDefinition {
    HttpResultMap hResultMap = new HttpResultMap();
    // XXX Currently this only allows a CacheResultHandler class to
    // initialize the result map.  Instead, don't use a CacheResultMap
    // directly, use either the plugin's CacheResultHandler, if specified,
    // or a default one that wraps the CacheResultMap

    String handler_class = null;
    handler_class = definitionMap.getString(KEY_EXCEPTION_HANDLER, null);
    if (handler_class != null) {
      try {
        resultHandler = (CacheResultHandler) newAuxClass(handler_class, CacheResultHandler.class);
        resultHandler.init(hResultMap);
      } catch (Exception ex) {
        throw new PluginException.InvalidDefinition(
            mapName + " has invalid Exception handler: " + handler_class, ex);
      } catch (LinkageError le) {
        throw new PluginException.InvalidDefinition(
            mapName + " has invalid Exception handler: " + handler_class, le);
      }
    } else {
      // Expect a list of mappings from either result code or exception
      // name to CacheException name
      Collection<String> mappings = definitionMap.getCollection(KEY_EXCEPTION_LIST, null);
      if (mappings != null) {
        // add each entry
        for (String entry : mappings) {
          if (log.isDebug2()) {
            log.debug2("initMap(" + entry + ")");
          }
          String first;
          String ceName;
          try {
            List<String> pair = StringUtil.breakAt(entry, '=', 2, true, true);
            first = pair.get(0);
            ceName = pair.get(1);
          } catch (Exception ex) {
            throw new PluginException.InvalidDefinition(
                "Invalid syntax: " + entry + "in " + mapName);
          }
          Object val;

          // Value should be either a CacheException or CacheResultHandler
          // class name.
          PluginFetchEventResponse resp =
              (PluginFetchEventResponse) newAuxClass(ceName, PluginFetchEventResponse.class, null);
          if (resp instanceof CacheException) {
            val = resp.getClass();
          } else if (resp instanceof CacheResultHandler) {
            val = WrapperUtil.wrap((CacheResultHandler) resp, CacheResultHandler.class);
          } else {
            throw new PluginException.InvalidDefinition(
                "Second arg not a "
                    + "CacheException or "
                    + "CacheResultHandler class: "
                    + entry
                    + ", in "
                    + mapName);
          }
          try {
            int code = Integer.parseInt(first);
            // If parseable as an integer, it's a result code.
            hResultMap.storeMapEntry(code, val);
          } catch (NumberFormatException e) {
            try {
              Class eClass = Class.forName(first);
              // If a class name, it should be an exception class
              if (Exception.class.isAssignableFrom(eClass)) {
                hResultMap.storeMapEntry(eClass, val);
              } else {
                throw new PluginException.InvalidDefinition(
                    "First arg not an " + "Exception class: " + entry + ", in " + mapName);
              }
            } catch (Exception ex) {
              throw new PluginException.InvalidDefinition(
                  "First arg not a " + "number or class: " + entry + ", in " + mapName);
            } catch (LinkageError le) {
              throw new PluginException.InvalidDefinition("Can't load " + first, le);
            }
          }
        }
      }
    }
    resultMap = hResultMap;
  }

  protected void initFeatureVersions() throws PluginException.InvalidDefinition {
    if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
      Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
      Map<String, String> spec =
          (Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
      log.debug2("features: " + spec);
      for (Map.Entry<String, String> ent : spec.entrySet()) {
        try {
          // Prefix version string with feature name to create separate
          // namespace for each feature
          String key = ent.getKey();
          map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
        } catch (RuntimeException e) {
          log.warning(
              getPluginName()
                  + " set unknown feature: "
                  + ent.getKey()
                  + " to version "
                  + ent.getValue(),
              e);
          throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
        }
      }
      featureVersion = map;
    } else {
      featureVersion = null;
    }
  }

  protected void initAuFeatureMap() {
    if (definitionMap.containsKey(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP)) {
      Map<String, ?> featMap = definitionMap.getMap(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP);
      for (Map.Entry ent : featMap.entrySet()) {
        Object val = ent.getValue();
        if (val instanceof Map) {
          ent.setValue(MapUtil.expandAlternativeKeyLists((Map) val));
        }
      }
    }
  }

  /** Create a CrawlWindow if necessary and return it. The CrawlWindow must be thread-safe. */
  protected CrawlWindow makeCrawlWindow() {
    if (crawlWindow != null) {
      return crawlWindow;
    }
    CrawlWindow window =
        (CrawlWindow) definitionMap.getMapElement(DefinableArchivalUnit.KEY_AU_CRAWL_WINDOW_SER);
    if (window == null) {
      String window_class =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_CRAWL_WINDOW, null);
      if (window_class != null) {
        ConfigurableCrawlWindow ccw =
            (ConfigurableCrawlWindow) newAuxClass(window_class, ConfigurableCrawlWindow.class);
        try {
          window = ccw.makeCrawlWindow();
        } catch (PluginException e) {
          throw new RuntimeException(e);
        }
      }
    }
    crawlWindow = window;
    return window;
  }

  LoginPageChecker loginChecker;

  protected LoginPageChecker makeLoginPageChecker() {
    if (loginChecker == null) {
      String loginPageCheckerClass =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_LOGIN_PAGE_CHECKER, null);
      if (loginPageCheckerClass != null) {
        loginChecker =
            (LoginPageChecker) newAuxClass(loginPageCheckerClass, LoginPageChecker.class);
      }
    }
    return loginChecker;
  }

  PermissionCheckerFactory permissionCheckerFact;

  protected PermissionCheckerFactory getPermissionCheckerFactory() {
    if (permissionCheckerFact == null) {
      String permissionCheckerFactoryClass =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_PERMISSION_CHECKER_FACTORY, null);
      if (permissionCheckerFactoryClass != null) {
        permissionCheckerFact =
            (PermissionCheckerFactory)
                newAuxClass(permissionCheckerFactoryClass, PermissionCheckerFactory.class);
        log.debug2("Loaded PermissionCheckerFactory: " + permissionCheckerFact);
      }
    }
    return permissionCheckerFact;
  }

  protected UrlNormalizer urlNorm;

  protected UrlNormalizer getUrlNormalizer() {
    if (urlNorm == null) {
      String normalizerClass =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_URL_NORMALIZER, null);
      if (normalizerClass != null) {
        urlNorm = (UrlNormalizer) newAuxClass(normalizerClass, UrlNormalizer.class);
      } else {
        urlNorm = NullUrlNormalizer.INSTANCE;
      }
    }
    return urlNorm;
  }

  protected ExploderHelper exploderHelper = null;

  protected ExploderHelper getExploderHelper() {
    if (exploderHelper == null) {
      String helperClass =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_EXPLODER_HELPER, null);
      if (helperClass != null) {
        exploderHelper = (ExploderHelper) newAuxClass(helperClass, ExploderHelper.class);
      }
    }
    return exploderHelper;
  }

  protected CrawlUrlComparatorFactory crawlUrlComparatorFactory = null;

  protected CrawlUrlComparatorFactory getCrawlUrlComparatorFactory() {
    if (crawlUrlComparatorFactory == null) {
      String factClass =
          definitionMap.getString(DefinablePlugin.KEY_PLUGIN_CRAWL_URL_COMPARATOR_FACTORY, null);
      if (factClass != null) {
        crawlUrlComparatorFactory =
            (CrawlUrlComparatorFactory) newAuxClass(factClass, CrawlUrlComparatorFactory.class);
      }
    }
    return crawlUrlComparatorFactory;
  }

  protected Comparator<CrawlUrl> getCrawlUrlComparator(ArchivalUnit au)
      throws PluginException.LinkageError {
    CrawlUrlComparatorFactory fact = getCrawlUrlComparatorFactory();
    if (fact == null) {
      return null;
    }
    return fact.createCrawlUrlComparator(au);
  }

  protected FilterRule constructFilterRule(String contentType) {
    String mimeType = HeaderUtil.getMimeTypeFromContentType(contentType);

    Object filter_el =
        definitionMap.getMapElement(mimeType + DefinableArchivalUnit.SUFFIX_FILTER_RULE);

    if (filter_el instanceof String) {
      log.debug("Loading filter " + filter_el);
      return (FilterRule) newAuxClass((String) filter_el, FilterRule.class);
    } else if (filter_el instanceof List) {
      if (((List) filter_el).size() > 0) {
        return new DefinableFilterRule((List) filter_el);
      }
    }
    return super.constructFilterRule(mimeType);
  }

  protected ArticleIteratorFactory articleIteratorFact = null;
  protected ArticleMetadataExtractorFactory articleMetadataFact = null;

  /**
   * Returns the plugin's article iterator factory, if any
   *
   * @return the ArticleIteratorFactory
   */
  public ArticleIteratorFactory getArticleIteratorFactory() {
    if (articleIteratorFact == null) {
      String factClass = definitionMap.getString(KEY_PLUGIN_ARTICLE_ITERATOR_FACTORY, null);
      if (factClass != null) {
        articleIteratorFact =
            (ArticleIteratorFactory) newAuxClass(factClass, ArticleIteratorFactory.class);
      }
    }
    return articleIteratorFact;
  }

  /**
   * Returns the article iterator factory for the content type, if any
   *
   * @param contentType the content type
   * @return the ArticleIteratorFactory
   */
  public ArticleMetadataExtractorFactory getArticleMetadataExtractorFactory(MetadataTarget target) {
    if (articleMetadataFact == null) {
      String factClass =
          definitionMap.getString(KEY_PLUGIN_ARTICLE_METADATA_EXTRACTOR_FACTORY, null);
      if (factClass != null) {
        articleMetadataFact =
            (ArticleMetadataExtractorFactory)
                newAuxClass(factClass, ArticleMetadataExtractorFactory.class);
      }
    }
    return articleMetadataFact;
  }

  public String getPluginId() {
    String className;
    if (mapName != null) {
      className = mapName;
    } else {
      // @TODO: eliminate this when we eliminate subclasses
      className = this.getClass().getName();
    }
    return className;
  }
}
/**
 * LockssRepository is used to organize the urls being cached. It keeps a memory cache of the most
 * recently used nodes as a least-recently-used map, and also caches weak references to the
 * instances as they're doled out. This ensures that two instances of the same node are never
 * created, as the weak references only disappear when the object is finalized (they go to null when
 * the last hard reference is gone, then are removed from the cache on finalize()).
 */
public class LockssRepositoryImpl extends BaseLockssDaemonManager implements LockssRepository {
  private static Logger logger = Logger.getLogger("LockssRepository");

  /** Configuration parameter name for Lockss cache location. */
  public static final String PARAM_CACHE_LOCATION = Configuration.PREFIX + "cache.location";

  /** Restores pre repo rescanning bugfix behavior (4050) */
  public static final String PARAM_CLEAR_DIR_MAP = RepositoryManager.PREFIX + "clearDirMapOnAuStop";

  public static final boolean DEFAULT_CLEAR_DIR_MAP = false;

  /** Name of top directory in which the urls are cached. */
  public static final String CACHE_ROOT_NAME = "cache";

  // XXX This is a remnant from the single-disk days, and should go away.
  // It is used only by unit tests, which want it set to the (last)
  // individual repository dir created.
  private static String staticCacheLocation = null;

  // Maps local repository name (disk path) to LocalRepository instance
  static Map localRepositories = new HashMap();

  // starts with a '#' so no possibility of clashing with a URL
  public static final String AU_ID_FILE = "#au_id_file";
  static final String AU_ID_PROP = "au.id";
  static final String PLUGIN_ID_PROP = "plugin.id";
  static final char ESCAPE_CHAR = '#';
  static final String ESCAPE_STR = "#";
  static final char ENCODED_SEPARATOR_CHAR = 's';
  static final String INITIAL_PLUGIN_DIR = String.valueOf((char) ('a' - 1));
  static String lastPluginDir = INITIAL_PLUGIN_DIR;
  // PJG: Windows prohibits use of ':' in file name -- replace with '~' for development
  static final String PORT_SEPARATOR = SystemUtils.IS_OS_WINDOWS ? "%" : ":";

  // this contains a '#' so that it's not defeatable by strings which
  // match the prefix in a url (like '../tmp/')
  private static final String TEST_PREFIX = "/#tmp";

  private RepositoryManager repoMgr;
  private String rootLocation;
  UniqueRefLruCache nodeCache;
  private boolean isGlobalNodeCache = RepositoryManager.DEFAULT_GLOBAL_CACHE_ENABLED;

  LockssRepositoryImpl(String rootPath) {
    if (rootPath.endsWith(File.separator)) {
      rootLocation = rootPath;
    } else {
      // shouldn't happen
      StringBuilder sb = new StringBuilder(rootPath.length() + File.separator.length());
      sb.append(rootPath);
      sb.append(File.separator);
      rootLocation = sb.toString();
    }
    // Test code still needs this.
    nodeCache = new UniqueRefLruCache(RepositoryManager.DEFAULT_MAX_PER_AU_CACHE_SIZE);
    rootLocation =
        rootLocation
            .replace("?", "")
            .replace("COM8", "COMEIGHT")
            .replace("%5c", "/"); // //windows folder structure fix
  }

  public void startService() {
    super.startService();
    repoMgr = getDaemon().getRepositoryManager();
    isGlobalNodeCache = repoMgr.isGlobalNodeCache();
    if (isGlobalNodeCache) {
      nodeCache = repoMgr.getGlobalNodeCache();
    } else {
      //       nodeCache =
      // 	new UniqueRefLruCache(repoMgr.paramNodeCacheSize);
      setNodeCacheSize(repoMgr.paramNodeCacheSize);
    }
  }

  public void stopService() {
    // mainly important in testing to blank this
    lastPluginDir = INITIAL_PLUGIN_DIR;
    if (CurrentConfig.getBooleanParam(PARAM_CLEAR_DIR_MAP, DEFAULT_CLEAR_DIR_MAP)) {
      // This should be in RepositoryManager.stopService()
      localRepositories = new HashMap();
    }
    super.stopService();
  }

  public void setNodeCacheSize(int size) {
    if (nodeCache != null && !isGlobalNodeCache && nodeCache.getMaxSize() != size) {
      nodeCache.setMaxSize(size);
    }
  }

  /**
   * Called between initService() and startService(), then whenever the AU's config changes.
   *
   * @param auConfig the new configuration
   */
  public void setAuConfig(Configuration auConfig) {}

  void queueSizeCalc(RepositoryNode node) {
    repoMgr.queueSizeCalc(node);
  }

  public RepositoryNode getNode(String url) throws MalformedURLException {
    return getNode(url, false);
  }

  public RepositoryNode createNewNode(String url) throws MalformedURLException {
    return getNode(url, true);
  }

  public void deleteNode(String url) throws MalformedURLException {
    RepositoryNode node = getNode(url, false);
    if (node != null) {
      node.markAsDeleted();
    }
  }

  public void deactivateNode(String url) throws MalformedURLException {
    RepositoryNode node = getNode(url, false);
    if (node != null) {
      node.deactivateContent();
    }
  }

  /**
   * This function returns a RepositoryNode with a canonicalized path.
   *
   * @param url the url in String form
   * @param create true iff the node should be created if absent
   * @return RepositoryNode the node
   * @throws MalformedURLException
   */
  private synchronized RepositoryNode getNode(String url, boolean create)
      throws MalformedURLException {
    String canonUrl;
    boolean isAuUrl = false;
    if (AuUrl.isAuUrl(url)) {
      // path information is lost here, but is unimportant if it's an AuUrl
      canonUrl = AuUrl.PROTOCOL;
      isAuUrl = true;
    } else {
      // create a canonical path, handling all illegal path traversal
      canonUrl = canonicalizePath(url);
    }

    // check LRUMap cache for node
    RepositoryNode node = (RepositoryNode) nodeCache.get(nodeCacheKey(canonUrl));
    if (node != null) {
      return node;
    }

    String nodeLocation;
    if (isAuUrl) {
      // base directory of ArchivalUnit
      nodeLocation = rootLocation;
      node = new AuNodeImpl(canonUrl, nodeLocation, this);
    } else {
      // determine proper node location
      nodeLocation =
          LockssRepositoryImpl.mapUrlToFileLocation(rootLocation, canonUrl)
              .replace("?", "")
              .replace("COM8", "COMEIGHT")
              .replace("%5c", "/"); // //windows folder structure fix
      node = new RepositoryNodeImpl(canonUrl, nodeLocation, this);
    }

    if (!create) {
      // if not creating, check for existence
      File nodeDir = new File(nodeLocation);
      if (!nodeDir.exists()) {
        // return null if the node doesn't exist and shouldn't be created
        return null;
      }
      if (!nodeDir.isDirectory()) {
        logger.error("Cache file not a directory: " + nodeLocation);
        throw new LockssRepository.RepositoryStateException("Invalid cache file.");
      }
    }

    // add to node cache
    nodeCache.put(nodeCacheKey(canonUrl), node);
    return node;
  }

  Object nodeCacheKey(String canonUrl) {
    if (isGlobalNodeCache) {
      return new KeyPair(this, canonUrl);
    }
    return canonUrl;
  }

  // functions for testing
  int getCacheHits() {
    return nodeCache.getCacheHits();
  }

  int getCacheMisses() {
    return nodeCache.getCacheMisses();
  }

  int getRefHits() {
    return nodeCache.getRefHits();
  }

  int getRefMisses() {
    return nodeCache.getRefMisses();
  }

  public void nodeConsistencyCheck() {
    // traverse the node tree from the top
    RepositoryNode topNode;
    try {
      topNode = getNode(AuUrl.PROTOCOL_COLON);
      recurseConsistencyCheck((RepositoryNodeImpl) topNode);
    } catch (MalformedURLException ignore) {
    }
  }

  /**
   * Checks the consistency of the node, and continues with its children if it's consistent.
   *
   * @param node RepositoryNodeImpl the node to check
   */
  private void recurseConsistencyCheck(RepositoryNodeImpl node) {
    logger.debug2("Checking node '" + node.getNodeUrl() + "'...");
    // check consistency at each node
    // correct/deactivate as necessary
    // 'checkNodeConsistency()' will repair if possible
    if (node.checkNodeConsistency()) {
      logger.debug3("Node consistent; recursing on children...");
      List children = node.getNodeList(null, false);
      Iterator iter = children.iterator();
      while (iter.hasNext()) {
        RepositoryNodeImpl child = (RepositoryNodeImpl) iter.next();
        recurseConsistencyCheck(child);
      }
    } else {
      logger.debug3("Node inconsistent; deactivating...");
      deactivateInconsistentNode(node);
    }
  }

  /**
   * This is called when a node is in an inconsistent state. It simply creates some necessary
   * directories and deactivates the node. Future polls should restore it properly.
   *
   * @param node the inconsistent node
   */
  void deactivateInconsistentNode(RepositoryNodeImpl node) {
    logger.warning("Deactivating inconsistent node.");
    FileUtil.ensureDirExists(node.contentDir);
    // manually deactivate
    node.deactivateContent();
  }

  /**
   * A method to remove any non-canonical '..' or '.' elements in the path, as well as protecting
   * against illegal path traversal.
   *
   * @param url the raw url
   * @return String the canonicalized url
   * @throws MalformedURLException
   */
  public String canonicalizePath(String url) throws MalformedURLException {
    String canonUrl = UrlUtil.normalizeUrl(url, UrlUtil.PATH_TRAVERSAL_ACTION_THROW);
    // canonicalize "dir" and "dir/"
    // XXX if these are ever two separate nodes, this is wrong
    if (canonUrl.endsWith(UrlUtil.URL_PATH_SEPARATOR)) {
      canonUrl = canonUrl.substring(0, canonUrl.length() - 1);
    }

    return canonUrl;
  }

  // static calls

  /**
   * Factory method to create new LockssRepository instances.
   *
   * @param au the {@link ArchivalUnit}
   * @return the new LockssRepository instance
   */
  public static LockssRepository createNewLockssRepository(ArchivalUnit au) {
    String root = getRepositoryRoot(au);
    if (root == null || root.equals("null")) {
      logger.error("No repository dir set in config");
      throw new LockssRepository.RepositoryStateException("No repository dir set in config");
    }
    String auDir = LockssRepositoryImpl.mapAuToFileLocation(root, au);
    if (logger.isDebug2()) {
      logger.debug2("repo: " + auDir + ", au: " + au.getName());
    }
    staticCacheLocation = extendCacheLocation(root);
    LockssRepositoryImpl repo = new LockssRepositoryImpl(auDir);
    Plugin plugin = au.getPlugin();
    if (plugin != null) {
      LockssDaemon daemon = plugin.getDaemon();
      if (daemon != null) {
        RepositoryManager mgr = daemon.getRepositoryManager();
        if (mgr != null) {
          mgr.setRepositoryForPath(auDir, repo);
        }
      }
    }
    return repo;
  }

  public static String getRepositorySpec(ArchivalUnit au) {
    Configuration auConfig = au.getConfiguration();
    if (auConfig != null) { // can be null in unit tests
      String repoSpec = auConfig.get(PluginManager.AU_PARAM_REPOSITORY);
      if (repoSpec != null && repoSpec.startsWith("local:")) {
        return repoSpec;
      }
    }
    return "local:" + CurrentConfig.getParam(PARAM_CACHE_LOCATION);
  }

  public static String getRepositoryRoot(ArchivalUnit au) {
    return getLocalRepositoryPath(getRepositorySpec(au));
  }

  public static String getLocalRepositoryPath(String repoSpec) {
    if (repoSpec != null) {
      if (repoSpec.startsWith("local:")) {
        return repoSpec.substring(6);
      }
    }
    return null;
  }

  // The OpenBSD platform has renamed the first disk from /cache to
  // /cache.wd0, leaving behind a symbolic link in /cache .  This is
  // transparent everywhere except the repository status table, which needs
  // to match AU configs with AUs it finds when enumerating the repository.
  // Existing AU configs have repository=local:/cache, so the relative link
  // needs to be resolved to detect that that's the same as
  // local:/cache.wd0

  private static Map canonicalRoots = new HashMap();

  public static boolean isDirInRepository(String dir, String repoRoot) {
    if (dir.startsWith(repoRoot)) {
      return true;
    }
    return canonRoot(dir).startsWith(canonRoot(repoRoot));
  }

  static String canonRoot(String root) {
    synchronized (canonicalRoots) {
      String canon = (String) canonicalRoots.get(root);
      if (canon == null) {
        try {
          canon = new File(root).getCanonicalPath();
          canonicalRoots.put(root, canon);
        } catch (IOException e) {
          logger.warning("Can't canonicalize: " + root, e);
          return root;
        }
      }
      return canon;
    }
  }

  static String getCacheLocation() {
    return staticCacheLocation;
  }

  /**
   * Adds the 'cache' directory to the HD location.
   *
   * @param cacheDir the root location.
   * @return String the extended location
   */
  static String extendCacheLocation(String cacheDir) {
    StringBuilder buffer = new StringBuilder(cacheDir);
    if (!cacheDir.endsWith(File.separator)) {
      buffer.append(File.separator);
    }
    buffer.append(CACHE_ROOT_NAME);
    buffer.append(File.separator);
    return buffer.toString();
  }

  /**
   * mapAuToFileLocation() is the method used to resolve {@link ArchivalUnit}s into directory names.
   * This maps a given au to directories, using the cache root as the base. Given an au with
   * PluginId of 'plugin' and AuId of 'au', it would return the string '<rootLocation>/plugin/au/'.
   *
   * @param repoRoot the root of a LOCKSS repository
   * @param au the ArchivalUnit to resolve
   * @return the directory location
   */
  public static String mapAuToFileLocation(String repoRoot, ArchivalUnit au) {
    return getAuDir(au, repoRoot, true);
  }

  /**
   * mapUrlToFileLocation() is the method used to resolve urls into file names. This maps a given
   * url to a file location, using the au top directory as the base. It creates directories which
   * mirror the html string, so 'http://www.journal.org/issue1/index.html' would be cached in the
   * file: <rootLocation>/www.journal.org/http/issue1/index.html
   *
   * @param rootLocation the top directory for ArchivalUnit this URL is in
   * @param urlStr the url to translate
   * @return the url file location
   * @throws java.net.MalformedURLException
   */
  public static String mapUrlToFileLocation(String rootLocation, String urlStr)
      throws MalformedURLException {
    int totalLength = rootLocation.length() + urlStr.length();
    URL url = new URL(urlStr);
    StringBuilder buffer = new StringBuilder(totalLength);
    buffer.append(rootLocation);
    if (!rootLocation.endsWith(File.separator)) {
      buffer.append(File.separator);
    }
    buffer.append(url.getHost().toLowerCase());
    int port = url.getPort();
    if (port != -1) {
      buffer.append(PORT_SEPARATOR);
      buffer.append(port);
    }
    buffer.append(File.separator);
    buffer.append(url.getProtocol());
    if (RepositoryManager.isEnableLongComponents()) {
      String escapedPath =
          escapePath(
              StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator));
      String query = url.getQuery();
      if (query != null) {
        escapedPath = escapedPath + "?" + escapeQuery(query);
      }
      String encodedPath = RepositoryNodeImpl.encodeUrl(escapedPath);
      // encodeUrl strips leading / from path
      buffer.append(File.separator);
      buffer.append(encodedPath);
    } else {
      buffer.append(
          escapePath(
              StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator)));
      String query = url.getQuery();
      if (query != null) {
        buffer.append("?");
        buffer.append(escapeQuery(query));
      }
    }
    return buffer.toString();
  }

  // name mapping functions

  /**
   * Return true iff a repository for the auid exists under the root
   *
   * @param auid
   * @param repoRoot the repository root
   * @return true iff a repository for the auid exists
   */
  static boolean doesAuDirExist(String auid, String repoRoot) {
    return null != getAuDir(auid, repoRoot, false);
  }

  /**
   * Finds the directory for this AU. If none found in the map, designates a new dir for it.
   *
   * @param au the AU
   * @param repoRoot root of the repository
   * @return the dir {@link String}
   */
  static String getAuDir(ArchivalUnit au, String repoRoot, boolean create) {
    return getAuDir(au.getAuId(), repoRoot, create);
  }

  /**
   * Finds the directory for this AU. If none found in the map, designates a new dir for it.
   *
   * @param auid AU id representing the au
   * @param repoRoot path to the root of the repository
   * @return the dir String
   */
  static String getAuDir(String auid, String repoRoot, boolean create) {
    String repoCachePath = extendCacheLocation(repoRoot);
    LocalRepository localRepo = getLocalRepository(repoRoot);
    synchronized (localRepo) {
      Map aumap = localRepo.getAuMap();
      String auPathSlash = (String) aumap.get(auid);
      if (auPathSlash != null) {
        return auPathSlash;
      }
      if (!create) {
        return null;
      }
      logger.debug3("Creating new au directory for '" + auid + "'.");
      String auDir = localRepo.getPrevAuDir();
      for (int cnt = RepositoryManager.getMaxUnusedDirSearch(); cnt > 0; cnt--) {
        // loop through looking for an available dir
        auDir = getNextDirName(auDir);
        File testDir = new File(repoCachePath, auDir);
        if (logger.isDebug3()) logger.debug3("Probe for unused: " + testDir);
        if (!testDir.exists()) {
          if (RepositoryManager.isStatefulUnusedDirSearch()) {
            localRepo.setPrevAuDir(auDir);
          }
          String auPath = testDir.toString();
          logger.debug3("New au directory: " + auPath);
          auPathSlash = auPath + File.separator;
          // write the new au property file to the new dir
          // XXX this data should be backed up elsewhere to avoid single-point
          // corruption
          Properties idProps = new Properties();
          idProps.setProperty(AU_ID_PROP, auid);
          saveAuIdProperties(auPath, idProps);
          aumap.put(auid, auPathSlash);
          return auPathSlash;
        } else {
          if (logger.isDebug3()) {
            logger.debug3("Existing directory found at '" + auDir + "'.  Checking next...");
          }
        }
      }
    }
    throw new RuntimeException(
        "Can't find unused repository dir after "
            + RepositoryManager.getMaxUnusedDirSearch()
            + " tries in "
            + repoCachePath);
  }

  static LocalRepository getLocalRepository(ArchivalUnit au) {
    return getLocalRepository(getRepositoryRoot(au));
  }

  static LocalRepository getLocalRepository(String repoRoot) {
    synchronized (localRepositories) {
      LocalRepository localRepo = (LocalRepository) localRepositories.get(repoRoot);
      if (localRepo == null) {
        logger.debug2("Creating LocalRepository(" + repoRoot + ")");
        localRepo = new LocalRepository(repoRoot);
        localRepositories.put(repoRoot, localRepo);
      }
      return localRepo;
    }
  }

  /** Return next string in the sequence "a", "b", ... "z", "aa", "ab", ... */
  static String getNextDirName(String old) {
    StringBuilder sb = new StringBuilder(old);
    // go through and increment the first non-'z' char
    // counts back from the last char, so 'aa'->'ab', not 'ba'
    for (int ii = sb.length() - 1; ii >= 0; ii--) {
      char curChar = sb.charAt(ii);
      if (curChar < 'z') {
        sb.setCharAt(ii, (char) (curChar + 1));
        return sb.toString();
      }
      sb.setCharAt(ii, 'a');
    }
    sb.insert(0, 'a');
    return sb.toString();
  }

  public static File getAuIdFile(String location) {
    return new File(location + File.separator + AU_ID_FILE);
  }

  static Properties getAuIdProperties(String location) {
    File propFile = new File(location + File.separator + AU_ID_FILE);
    try {
      InputStream is = new BufferedInputStream(new FileInputStream(propFile));
      Properties idProps = new Properties();
      idProps.load(is);
      is.close();
      return idProps;
    } catch (Exception e) {
      logger.warning("Error loading au id from " + propFile.getPath() + ".");
      return null;
    }
  }

  static void saveAuIdProperties(String location, Properties props) {
    // XXX these AU_ID_FILE entries need to be backed up elsewhere to avoid
    // single-point corruption
    File propDir = new File(location);
    if (!propDir.exists()) {
      logger.debug("Creating directory '" + propDir.getAbsolutePath() + "'");
      propDir.mkdirs();
    }
    File propFile = new File(propDir, AU_ID_FILE);
    try {
      logger.debug3("Saving au id properties at '" + location + "'.");
      OutputStream os = new BufferedOutputStream(new FileOutputStream(propFile));
      props.store(os, "ArchivalUnit id info");
      os.close();
      propFile.setReadOnly();
    } catch (IOException ioe) {
      logger.error("Couldn't write properties for " + propFile.getPath() + ".", ioe);
      throw new LockssRepository.RepositoryStateException("Couldn't write au id properties file.");
    }
  }

  // lockss filename-specific encoding methods

  /**
   * Escapes instances of the ESCAPE_CHAR from the path. This avoids name conflicts with the
   * repository files, such as '#nodestate.xml'.
   *
   * @param path the path
   * @return the escaped path
   */
  static String escapePath(String path) {
    // XXX escaping disabled because of URL encoding
    if (false && path.indexOf(ESCAPE_CHAR) >= 0) {
      return StringUtil.replaceString(path, ESCAPE_STR, ESCAPE_STR + ESCAPE_STR);
    } else {
      return path;
    }
  }

  /**
   * Escapes instances of File.separator from the query. These are safe from filename overlap, but
   * can't convert into extended paths and directories.
   *
   * @param query the query
   * @return the escaped query
   */
  static String escapeQuery(String query) {
    if (query.indexOf(File.separator) >= 0) {
      return StringUtil.replaceString(query, File.separator, ESCAPE_STR + ENCODED_SEPARATOR_CHAR);
    } else {
      return query;
    }
  }

  /**
   * Extracts '#x' encoding and converts back to 'x'.
   *
   * @param orig the original
   * @return the unescaped version.
   */
  static String unescape(String orig) {
    if (orig.indexOf(ESCAPE_CHAR) < 0) {
      // fast treatment of non-escaped strings
      return orig;
    }
    int index = -1;
    StringBuilder buffer = new StringBuilder(orig.length());
    String oldStr = orig;
    while ((index = oldStr.indexOf(ESCAPE_CHAR)) >= 0) {
      buffer.append(oldStr.substring(0, index));
      buffer.append(convertCode(oldStr.substring(index, index + 2)));
      if (oldStr.length() > 2) {
        oldStr = oldStr.substring(index + 2);
      } else {
        oldStr = "";
      }
    }
    buffer.append(oldStr);
    return buffer.toString();
  }

  /**
   * Returns the second char in the escaped segment, unless it is 's', which is a stand-in for the
   * File.separatorChar.
   *
   * @param code the code segment (length 2)
   * @return the encoded char
   */
  static char convertCode(String code) {
    char encodedChar = code.charAt(1);
    if (encodedChar == ENCODED_SEPARATOR_CHAR) {
      return File.separatorChar;
    } else {
      return encodedChar;
    }
  }

  public static class Factory implements LockssAuManager.Factory {
    public LockssAuManager createAuManager(ArchivalUnit au) {
      return createNewLockssRepository(au);
    }
  }

  /** Maintains state for a local repository root dir (<i>eg</i>, auid of each au subdir). */
  static class LocalRepository {
    String repoPath;
    File repoCacheFile;
    Map auMap;
    String prevAuDir;

    LocalRepository(String repoPath) {
      this.repoPath = repoPath;
      repoCacheFile = new File(repoPath, CACHE_ROOT_NAME);
    }

    public String getRepositoryPath() {
      return repoPath;
    }

    public String getPrevAuDir() {
      if (prevAuDir == null) {
        prevAuDir = lastPluginDir;
      }
      return prevAuDir;
    }

    public void setPrevAuDir(String dir) {
      prevAuDir = dir;
    }

    /**
     * Return the auid -> au-subdir-path mapping. Enumerating the directories if necessary to
     * initialize the map
     */
    Map getAuMap() {
      if (auMap == null) {
        logger.debug3("Loading name map for '" + repoCacheFile + "'.");
        auMap = new HashMap();
        if (!repoCacheFile.exists()) {
          logger.debug3("Creating cache dir:" + repoCacheFile + "'.");
          if (!repoCacheFile.mkdirs()) {
            logger.critical("Couldn't create directory, check owner/permissions: " + repoCacheFile);
            // return empty map
            return auMap;
          }
        } else {
          // read each dir's property file and store mapping auid -> dir
          File[] auDirs = repoCacheFile.listFiles();
          for (int ii = 0; ii < auDirs.length; ii++) {
            String dirName = auDirs[ii].getName();
            //       if (dirName.compareTo(lastPluginDir) == 1) {
            //         // adjust the 'lastPluginDir' upwards if necessary
            //         lastPluginDir = dirName;
            //       }

            String path = auDirs[ii].getAbsolutePath();
            Properties idProps = getAuIdProperties(path);
            if (idProps != null) {
              String auid = idProps.getProperty(AU_ID_PROP);
              StringBuilder sb = new StringBuilder(path.length() + File.separator.length());
              sb.append(path);
              sb.append(File.separator);
              auMap.put(auid, sb.toString());
              logger.debug3("Mapping to: " + auMap.get(auid) + ": " + auid);
            } else {
              logger.debug3("Not mapping " + path + ", no auid file.");
            }
          }
        }
      }
      return auMap;
    }

    public String toString() {
      return "[LR: " + repoPath + "]";
    }
  }

  String getRootLocation() {
    return rootLocation;
  }
}