/** Export to an ARC file */
public class ArcExporter extends Exporter {
private static Logger log = Logger.getLogger("ArcExporter");
protected CIProperties arcProps = null;
String arcFilePrefix = "SimulatedCrawl";
AtomicInteger serialNo = new AtomicInteger(0);
ARCWriter aw;
boolean isResponse;
public ArcExporter(LockssDaemon daemon, ArchivalUnit au, boolean isResponse) {
super(daemon, au);
this.isResponse = isResponse;
}
protected void start() {
aw = makeARCWriter();
}
protected void finish() throws IOException {
aw.close();
}
private ARCWriter makeARCWriter() {
return new ARCWriter(
serialNo, ListUtil.list(dir), prefix, compress, maxSize >= 0 ? maxSize : Long.MAX_VALUE);
}
protected void writeCu(CachedUrl cu) throws IOException {
String url = cu.getUrl();
long contentSize = cu.getContentSize();
CIProperties props = cu.getProperties();
long fetchTime = Long.parseLong(props.getProperty(CachedUrl.PROPERTY_FETCH_TIME));
InputStream contentIn = cu.getUnfilteredInputStream();
try {
if (isResponse) {
String hdrString = getHttpResponseString(cu);
long size = contentSize + hdrString.length();
InputStream headerIn = new ReaderInputStream(new StringReader(hdrString));
InputStream concat = new SequenceInputStream(headerIn, contentIn);
try {
aw.write(xlateFilename(url), cu.getContentType(), getHostIp(), fetchTime, size, concat);
} finally {
IOUtil.safeClose(concat);
}
} else {
aw.write(
xlateFilename(url),
cu.getContentType(),
getHostIp(),
fetchTime,
cu.getContentSize(),
contentIn);
}
} finally {
AuUtil.safeRelease(cu);
}
}
}
/** JUnitTest case for class: org.lockss.poller.Poll */
public class TestPoll extends LockssTestCase {
private static Logger log = Logger.getLogger("TestPoll");
private static String[] rootV1urls = {
"http://www.test.org", "http://www.test1.org", "http://www.test2.org"
};
private static String lwrbnd = "test1.doc";
private static String uprbnd = "test3.doc";
private static long testduration = Constants.DAY;
// protected ArchivalUnit testau =
// PollTestPlugin.PTArchivalUnit.createFromListOfRootUrls(rootV1urls);
protected MockArchivalUnit testau;
private IdentityManager idmgr;
private MockLockssDaemon theDaemon;
private ArrayList agree_entries = makeEntries(10, 50);
private ArrayList disagree_entries = makeEntries(15, 57);
private ArrayList dissenting_entries = makeEntries(7, 50);
protected PeerIdentity testID;
protected PeerIdentity testID1;
protected V1LcapMessage[] testV1msg;
protected V1Poll[] testV1polls;
protected PollManager pollmanager;
protected void setUp() throws Exception {
super.setUp();
TimeBase.setSimulated();
initRequiredServices();
testau.setPlugin(new MockPlugin());
initTestPeerIDs();
initTestMsg();
initTestPolls();
}
/**
* tearDown method for test case
*
* @throws Exception if removePoll failed
*/
public void tearDown() throws Exception {
pollmanager.stopService();
theDaemon.getLockssRepository(testau).stopService();
theDaemon.getHashService().stopService();
theDaemon.getDatagramRouterManager().stopService();
theDaemon.getRouterManager().stopService();
theDaemon.getSystemMetrics().stopService();
TimeBase.setReal();
for (int i = 0; i < testV1msg.length; i++) {
if (testV1msg[i] != null) pollmanager.removePoll(testV1msg[i].getKey());
}
super.tearDown();
}
/** test for method scheduleVote(..) */
public void testScheduleVote() {
V1Poll p = testV1polls[1];
assertTrue(p instanceof V1ContentPoll);
log.debug3("testScheduleVote 1");
p.scheduleVote();
log.debug3("testScheduleVote 2");
assertNotNull(p.m_voteTime);
assertTrue(p.m_voteTime.getRemainingTime() < p.m_deadline.getRemainingTime());
log.debug3("at end of testScheduleVote");
}
/** test for method checkVote(..) */
public void testCheckVote() throws Exception {
V1LcapMessage msg = null;
log.debug3("starting testCheeckVote");
msg =
V1LcapMessage.makeReplyMsg(
testV1polls[0].getMessage(),
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
null,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID);
log.debug3("testCheeckVote 2");
V1Poll p = null;
p = createCompletedPoll(theDaemon, testau, msg, 8, 2, pollmanager);
assertTrue(p instanceof V1NamePoll);
log.debug3("testCheeckVote 3");
assertNotNull(p);
PeerIdentity id = msg.getOriginatorId();
assertNotNull(id);
assertNotNull(p.m_tally);
int rep = p.m_tally.wtAgree + idmgr.getReputation(id);
// good vote check
p.checkVote(msg.getHashed(), new Vote(msg, false));
assertEquals(9, p.m_tally.numAgree);
assertEquals(2, p.m_tally.numDisagree);
assertEquals(rep, p.m_tally.wtAgree);
rep = p.m_tally.wtDisagree + idmgr.getReputation(id);
// bad vote check
p.checkVote(ByteArray.makeRandomBytes(20), new Vote(msg, false));
assertEquals(9, p.m_tally.numAgree);
assertEquals(3, p.m_tally.numDisagree);
assertEquals(rep, p.m_tally.wtDisagree);
}
/** test for method tally(..) */
public void testTally() {
V1Poll p = testV1polls[0];
LcapMessage msg = p.getMessage();
PeerIdentity id = msg.getOriginatorId();
p.m_tally.addVote(new Vote(msg, false), id, false);
p.m_tally.addVote(new Vote(msg, false), id, false);
p.m_tally.addVote(new Vote(msg, false), id, false);
assertEquals(0, p.m_tally.numAgree);
assertEquals(0, p.m_tally.wtAgree);
assertEquals(3, p.m_tally.numDisagree);
assertEquals(1500, p.m_tally.wtDisagree);
p = testV1polls[1];
msg = p.getMessage();
p.m_tally.addVote(new Vote(msg, true), id, false);
p.m_tally.addVote(new Vote(msg, true), id, false);
p.m_tally.addVote(new Vote(msg, true), id, false);
assertEquals(3, p.m_tally.numAgree);
assertEquals(1500, p.m_tally.wtAgree);
assertEquals(0, p.m_tally.numDisagree);
assertEquals(0, p.m_tally.wtDisagree);
}
public void testNamePollTally() throws Exception {
V1NamePoll np;
// test a name poll we won
np = makeCompletedNamePoll(4, 1, 0);
assertEquals(5, np.m_tally.numAgree);
assertEquals(1, np.m_tally.numDisagree);
assertEquals(Tallier.RESULT_WON, np.m_tally.getTallyResult());
// test a name poll we lost with a dissenting vote
np = makeCompletedNamePoll(1, 8, 1);
assertEquals(2, np.m_tally.numAgree);
assertEquals(9, np.m_tally.numDisagree);
// build a master list
np.buildPollLists(np.m_tally.pollVotes.iterator());
// these should be different since we lost the poll
assertFalse(CollectionUtil.isIsomorphic(np.m_tally.localEntries, np.m_tally.votedEntries));
// the expected "correct" set is in our disagree msg
assertTrue(CollectionUtil.isIsomorphic(disagree_entries, np.m_tally.votedEntries));
}
/** test for method vote(..) */
public void testVote() {
V1Poll p = testV1polls[1];
p.m_hash = ByteArray.makeRandomBytes(20);
try {
p.castOurVote();
} catch (IllegalStateException e) {
// the socket isn't inited and should squack
}
p.m_pollstate = V1Poll.PS_COMPLETE;
}
/** test for method voteInPoll(..) */
public void testVoteInPoll() {
V1Poll p = testV1polls[1];
p.m_tally.quorum = 10;
p.m_tally.numAgree = 5;
p.m_tally.numDisagree = 2;
p.m_tally.wtAgree = 2000;
p.m_tally.wtDisagree = 200;
p.m_hash = ByteArray.makeRandomBytes(20);
try {
p.voteInPoll();
} catch (IllegalStateException e) {
// the socket isn't inited and should squack
}
p.m_tally.numAgree = 20;
try {
p.voteInPoll();
} catch (NullPointerException npe) {
// the socket isn't inited and should squack
}
p.m_pollstate = V1Poll.PS_COMPLETE;
}
public void testStartPoll() {
V1Poll p = testV1polls[0];
p.startPoll();
assertEquals(V1Poll.PS_WAIT_HASH, p.m_pollstate);
p.m_pollstate = V1Poll.PS_COMPLETE;
}
public void testScheduleOurHash() {
V1Poll p = testV1polls[0];
p.m_pollstate = V1Poll.PS_WAIT_HASH;
// no time has elapsed - so we should be able to schedule our hash
assertTrue(p.scheduleOurHash());
// half the time has elapsed so we should be able to schedule our hash
TimeBase.step(p.m_deadline.getRemainingTime() / 2);
assertTrue(p.scheduleOurHash());
// all of the time has elapsed we should not be able to schedule our hash
TimeBase.step(p.m_deadline.getRemainingTime() - 1000);
assertFalse(p.scheduleOurHash());
p.m_pollstate = V1Poll.PS_COMPLETE;
}
/** test for method stopPoll(..) */
public void testStopPoll() {
V1Poll p = testV1polls[1];
p.m_tally.quorum = 10;
p.m_tally.numAgree = 7;
p.m_tally.numDisagree = 3;
p.m_pollstate = V1Poll.PS_WAIT_TALLY;
p.stopPoll();
assertTrue(p.m_pollstate == V1Poll.PS_COMPLETE);
p.startPoll();
assertTrue(p.m_pollstate == V1Poll.PS_COMPLETE);
}
/** test for method startVoteCheck(..) */
public void testStartVote() {
V1Poll p = testV1polls[0];
p.m_pendingVotes = 3;
p.startVoteCheck();
assertEquals(4, p.m_pendingVotes);
p.m_pollstate = V1Poll.PS_COMPLETE;
}
/** test for method stopVote(..) */
public void testStopVote() {
V1Poll p = testV1polls[1];
p.m_pendingVotes = 3;
p.stopVoteCheck();
assertEquals(2, p.m_pendingVotes);
p.m_pollstate = V1Poll.PS_COMPLETE;
}
private V1NamePoll makeCompletedNamePoll(int numAgree, int numDisagree, int numDissenting)
throws Exception {
V1NamePoll np = null;
V1LcapMessage agree_msg = null;
V1LcapMessage disagree_msg1 = null;
V1LcapMessage disagree_msg2 = null;
Plugin plugin = testau.getPlugin();
PollSpec spec = new MockPollSpec(testau, rootV1urls[0], null, null, Poll.V1_NAME_POLL);
((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
V1LcapMessage poll_msg =
V1LcapMessage.makeRequestMsg(
spec,
null,
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
V1LcapMessage.NAME_POLL_REQ,
testduration,
testID);
// make our poll
np =
(V1NamePoll)
new V1NamePoll(
spec,
pollmanager,
poll_msg.getOriginatorId(),
poll_msg.getChallenge(),
poll_msg.getDuration(),
poll_msg.getHashAlgorithm());
np.setMessage(poll_msg);
// generate agree vote msg
agree_msg =
V1LcapMessage.makeReplyMsg(
poll_msg,
ByteArray.makeRandomBytes(20),
poll_msg.getVerifier(),
agree_entries,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID);
// generate a disagree vote msg
disagree_msg1 =
V1LcapMessage.makeReplyMsg(
poll_msg,
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
disagree_entries,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID1);
// generate a losing disagree vote msg
disagree_msg2 =
V1LcapMessage.makeReplyMsg(
poll_msg,
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
dissenting_entries,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID1);
// add our vote
V1LcapMessage msg = (V1LcapMessage) (np.getMessage());
PeerIdentity id = msg.getOriginatorId();
np.m_tally.addVote(np.makeNameVote(msg, true), id, true);
// add the agree votes
id = agree_msg.getOriginatorId();
for (int i = 0; i < numAgree; i++) {
np.m_tally.addVote(np.makeNameVote(agree_msg, true), id, false);
}
// add the disagree votes
id = disagree_msg1.getOriginatorId();
for (int i = 0; i < numDisagree; i++) {
np.m_tally.addVote(np.makeNameVote(disagree_msg1, false), id, false);
}
// add dissenting disagree vote
id = disagree_msg2.getOriginatorId();
for (int i = 0; i < numDissenting; i++) {
np.m_tally.addVote(np.makeNameVote(disagree_msg2, false), id, false);
}
np.m_pollstate = V1Poll.PS_COMPLETE;
np.m_tally.tallyVotes();
return np;
}
public static V1Poll createCompletedPoll(
LockssDaemon daemon,
ArchivalUnit au,
V1LcapMessage testmsg,
int numAgree,
int numDisagree,
PollManager pollmanager)
throws Exception {
log.debug(
"createCompletedPoll: au: "
+ au.toString()
+ " peer "
+ testmsg.getOriginatorId()
+ " votes "
+ numAgree
+ "/"
+ numDisagree);
CachedUrlSetSpec cusSpec = null;
if ((testmsg.getLwrBound() != null)
&& (testmsg.getLwrBound().equals(PollSpec.SINGLE_NODE_LWRBOUND))) {
cusSpec = new SingleNodeCachedUrlSetSpec(testmsg.getTargetUrl());
} else {
cusSpec =
new RangeCachedUrlSetSpec(
testmsg.getTargetUrl(), testmsg.getLwrBound(), testmsg.getUprBound());
}
CachedUrlSet cus = au.makeCachedUrlSet(cusSpec);
PollSpec spec = new PollSpec(cus, Poll.V1_CONTENT_POLL);
((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
V1Poll p = null;
if (testmsg.isContentPoll()) {
p =
new V1ContentPoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm());
} else if (testmsg.isNamePoll()) {
p =
new V1NamePoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm());
} else if (testmsg.isVerifyPoll()) {
p =
new V1VerifyPoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm(),
testmsg.getVerifier());
}
assertNotNull(p);
p.setMessage(testmsg);
p.m_tally.quorum = numAgree + numDisagree;
p.m_tally.numAgree = numAgree;
p.m_tally.numDisagree = numDisagree;
p.m_tally.wtAgree = 2000;
p.m_tally.wtDisagree = 200;
p.m_tally.localEntries = makeEntries(1, 3);
p.m_tally.votedEntries = makeEntries(1, 5);
p.m_tally.votedEntries.remove(1);
p.m_pollstate = V1Poll.PS_COMPLETE;
p.m_callerID = testmsg.getOriginatorId();
log.debug3("poll " + p.toString());
p.m_tally.tallyVotes();
return p;
}
public static ArrayList makeEntries(int firstEntry, int lastEntry) {
int numEntries = lastEntry - firstEntry + 1;
ArrayList ret_arry = new ArrayList(numEntries);
for (int i = 0; i < numEntries; i++) {
String name = "/testentry" + (firstEntry + i) + ".html";
ret_arry.add(new PollTally.NameListEntry(i % 2 == 1, name));
}
return ret_arry;
}
private void initRequiredServices() {
theDaemon = getMockLockssDaemon();
pollmanager = new LocalPollManager();
pollmanager.initService(theDaemon);
theDaemon.setPollManager(pollmanager);
theDaemon.getPluginManager();
testau = PollTestPlugin.PTArchivalUnit.createFromListOfRootUrls(rootV1urls);
PluginTestUtil.registerArchivalUnit(testau);
String tempDirPath = null;
try {
tempDirPath = getTempDir().getAbsolutePath() + File.separator;
} catch (IOException ex) {
fail("unable to create a temporary directory");
}
Properties p = new Properties();
p.setProperty(IdentityManager.PARAM_IDDB_DIR, tempDirPath + "iddb");
p.setProperty(LockssRepositoryImpl.PARAM_CACHE_LOCATION, tempDirPath);
p.setProperty(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, tempDirPath);
p.setProperty(IdentityManager.PARAM_LOCAL_IP, "127.0.0.1");
p.setProperty(ConfigManager.PARAM_NEW_SCHEDULER, "false");
// XXX we need to disable verification of votes because the
// voter isn't really there
p.setProperty(V1Poll.PARAM_AGREE_VERIFY, "0");
p.setProperty(V1Poll.PARAM_DISAGREE_VERIFY, "0");
ConfigurationUtil.setCurrentConfigFromProps(p);
idmgr = theDaemon.getIdentityManager();
idmgr.startService();
// theDaemon.getSchedService().startService();
theDaemon.getHashService().startService();
theDaemon.getDatagramRouterManager().startService();
theDaemon.getRouterManager().startService();
theDaemon.getSystemMetrics().startService();
theDaemon.getActivityRegulator(testau).startService();
theDaemon.setNodeManager(new MockNodeManager(), testau);
pollmanager.startService();
}
private void initTestPeerIDs() {
try {
testID = idmgr.stringToPeerIdentity("127.0.0.1");
testID1 = idmgr.stringToPeerIdentity("1.1.1.1");
} catch (IdentityManager.MalformedIdentityKeyException ex) {
fail("can't open test host");
}
}
private void initTestMsg() throws Exception {
testV1msg = new V1LcapMessage[3];
int[] pollType = {
Poll.V1_NAME_POLL, Poll.V1_CONTENT_POLL, Poll.V1_VERIFY_POLL,
};
PollFactory ppf = pollmanager.getPollFactory(1);
assertNotNull("PollFactory should not be null", ppf);
// XXX V1 support mandatory
assertTrue(ppf instanceof V1PollFactory);
V1PollFactory pf = (V1PollFactory) ppf;
for (int i = 0; i < testV1msg.length; i++) {
PollSpec spec = new MockPollSpec(testau, rootV1urls[i], lwrbnd, uprbnd, pollType[i]);
log.debug("Created poll spec: " + spec);
((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
int opcode = V1LcapMessage.NAME_POLL_REQ + (i * 2);
long duration = -1;
// NB calcDuration is not applied to Verify polls.
switch (opcode) {
case V1LcapMessage.NAME_POLL_REQ:
case V1LcapMessage.CONTENT_POLL_REQ:
// this will attempt to schedule and can return -1
duration = Math.max(pf.calcDuration(spec, pollmanager), 1000);
break;
case V1LcapMessage.VERIFY_POLL_REQ:
case V1LcapMessage.VERIFY_POLL_REP:
duration = 100000; // Arbitrary
break;
default:
fail("Bad opcode " + opcode);
break;
}
testV1msg[i] =
V1LcapMessage.makeRequestMsg(
spec,
agree_entries,
pf.makeVerifier(100000),
pf.makeVerifier(100000),
opcode,
duration,
testID);
assertNotNull(testV1msg[i]);
}
}
private void initTestPolls() throws Exception {
testV1polls = new V1Poll[testV1msg.length];
for (int i = 0; i < testV1polls.length; i++) {
log.debug3("initTestPolls: V1 " + i);
BasePoll p = pollmanager.makePoll(testV1msg[i]);
assertNotNull(p);
assertNotNull(p.getMessage());
log.debug("initTestPolls: V1 " + i + " returns " + p);
assertTrue(p instanceof V1Poll);
switch (i) {
case 0:
assertTrue(p instanceof V1NamePoll);
break;
case 1:
assertTrue(p instanceof V1ContentPoll);
break;
case 2:
assertTrue(p instanceof V1VerifyPoll);
break;
}
testV1polls[i] = (V1Poll) p;
assertNotNull(testV1polls[i]);
log.debug3("initTestPolls: " + i + " " + p.toString());
}
}
static class LocalPollManager extends PollManager {
// ignore message sends
public void sendMessage(V1LcapMessage msg, ArchivalUnit au) throws IOException {}
}
/**
* Executes the test case
*
* @param argv array of Strings containing command line arguments
*/
public static void main(String[] argv) {
String[] testCaseList = {TestPoll.class.getName()};
junit.swingui.TestRunner.main(testCaseList);
}
}
/** Minimal fully functional plugin capable of serving a little static content. */
public class StaticContentPlugin extends BasePlugin implements PluginTestable {
static Logger log = Logger.getLogger("StaticContentPlugin");
Map cuMap = new HashMap();
public StaticContentPlugin() {}
public String getVersion() {
throw new UnsupportedOperationException("Not implemented");
}
public String getPluginName() {
return "Static Content";
}
public List getSupportedTitles() {
throw new UnsupportedOperationException("Not implemented");
}
public List getLocalAuConfigDescrs() {
return Collections.EMPTY_LIST;
// throw new UnsupportedOperationException("Not implemented");
}
protected ArchivalUnit createAu0(Configuration auConfig)
throws ArchivalUnit.ConfigurationException {
return new SAU(this);
}
public void registerArchivalUnit(ArchivalUnit au) {
aus.add(au);
}
public void unregisterArchivalUnit(ArchivalUnit au) {
aus.remove(au);
}
public class SAU extends BaseArchivalUnit {
protected SAU(Plugin myPlugin) {
super(myPlugin);
}
protected String makeName() {
return "Static Content AU";
}
protected String makeStartUrl() {
throw new UnsupportedOperationException("Not Implemented");
}
public CachedUrlSet makeCachedUrlSet(CachedUrlSetSpec cuss) {
return new SCUS(this, cuss);
}
public CachedUrl makeCachedUrl(String url) {
CachedUrl res = (CachedUrl) cuMap.get(url);
log.debug("makeCachedUrl(" + url + ") = " + res);
return (CachedUrl) cuMap.get(url);
}
public org.lockss.plugin.UrlCacher makeUrlCacher(String url) {
throw new UnsupportedOperationException("Not implemented");
}
public boolean shouldBeCached(String url) {
return cuMap.containsKey(url);
}
public List getNewContentCrawlUrls() {
throw new UnsupportedOperationException("Not implemented");
}
public Collection getUrlStems() {
throw new UnsupportedOperationException("Not implemented");
}
public CachedUrlSet cachedUrlSetFactory(ArchivalUnit owner, CachedUrlSetSpec cuss) {
throw new UnsupportedOperationException("Not implemented");
}
public CachedUrl cachedUrlFactory(CachedUrlSet owner, String url) {
throw new UnsupportedOperationException("Not implemented");
}
public UrlCacher urlCacherFactory(CachedUrlSet owner, String url) {
throw new UnsupportedOperationException("Not implemented");
}
public String getManifestPage() {
throw new UnsupportedOperationException("Not Implemented");
}
public FilterRule getFilterRule(String mimeType) {
throw new UnsupportedOperationException("Not implemented");
}
/**
* Create a CU with content and store it in AU
*
* @param owner the CUS owner
* @param url the url
* @param type the type
* @param contents the contents
*/
public void storeCachedUrl(CachedUrlSet owner, String url, String type, String contents) {
SCU scu = new SCU(owner, url, type, contents);
cuMap.put(scu.getUrl(), scu);
}
public void storeCachedUrl(String url, String type, String contents) {
storeCachedUrl(null, url, type, contents);
}
public String toString() {
return "[sau: " + cuMap + "]";
}
protected CrawlRule makeRules() {
throw new UnsupportedOperationException("Not implemented");
}
/**
* loadDefiningConfig
*
* @param config Configuration
*/
protected void loadAuConfigDescrs(Configuration config) {}
}
public class SCU extends BaseCachedUrl {
private String contents = null;
private CIProperties props = new CIProperties();
public SCU(CachedUrlSet owner, String url) {
super(null, url);
}
/**
* Create a CachedUrl with content
*
* @param owner the CUS owner
* @param url the url
* @param type the type
* @param contents the contents
*/
public SCU(CachedUrlSet owner, String url, String type, String contents) {
this(owner, url);
setContents(contents);
props.setProperty(CachedUrl.PROPERTY_CONTENT_TYPE, type);
}
private void setContents(String s) {
contents = s;
props.setProperty("Content-Length", "" + s.length());
}
public String getUrl() {
return url;
}
public boolean hasContent() {
return contents != null;
}
public boolean isLeaf() {
throw new UnsupportedOperationException("Not implemented");
}
public InputStream getUnfilteredInputStream() {
return new StringInputStream(contents);
}
public InputStream openForHashing() {
return getUnfilteredInputStream();
}
protected InputStream getFilteredStream() {
throw new UnsupportedOperationException("Not implemented");
}
public Reader openForReading() {
throw new UnsupportedOperationException("Not implemented");
}
public long getContentSize() {
return contents == null ? 0 : contents.length();
}
public CIProperties getProperties() {
return props;
}
}
class SCUS extends BaseCachedUrlSet {
public SCUS(ArchivalUnit owner, CachedUrlSetSpec spec) {
super(owner, spec);
}
public void storeActualHashDuration(long elapsed, Exception err) {
throw new UnsupportedOperationException("Not implemented");
}
public Iterator flatSetIterator() {
throw new UnsupportedOperationException("Not implemented");
}
public Iterator contentHashIterator() {
throw new UnsupportedOperationException("Not implemented");
}
public boolean isLeaf() {
throw new UnsupportedOperationException("Not implemented");
}
public CachedUrlSetHasher getContentHasher(MessageDigest digest) {
throw new UnsupportedOperationException("Not implemented");
}
public CachedUrlSetHasher getNameHasher(MessageDigest digest) {
throw new UnsupportedOperationException("Not implemented");
}
public long estimatedHashDuration() {
return 1000;
}
}
}
/** Central repository of loaded keystores */
public class LockssKeyStoreManager extends BaseLockssDaemonManager implements ConfigurableManager {
protected static Logger log = Logger.getLogger("LockssKeyStoreManager");
static final String PREFIX = Configuration.PREFIX + "keyMgr.";
/** Default type for newly created keystores. */
public static final String PARAM_DEFAULT_KEYSTORE_TYPE = PREFIX + "defaultKeyStoreType";
public static final String DEFAULT_DEFAULT_KEYSTORE_TYPE = "JCEKS";
/** Default keystore provider. */
public static final String PARAM_DEFAULT_KEYSTORE_PROVIDER = PREFIX + "defaultKeyStoreProvider";
public static final String DEFAULT_DEFAULT_KEYSTORE_PROVIDER = null;
/**
* Root of keystore definitions. For each keystore, pick a unique identifier and use it in place
* of <id> in the following
*/
public static final String PARAM_KEYSTORE = PREFIX + "keystore";
/** If true the daemon will exit if a critical keystore is missing. */
public static final String PARAM_EXIT_IF_MISSING_KEYSTORE = PREFIX + "exitIfMissingKeystore";
public static final boolean DEFAULT_EXIT_IF_MISSING_KEYSTORE = true;
/** keystore name, used by clients to refer to it */
public static final String KEYSTORE_PARAM_NAME = "name";
/** keystore file. Only one of file, resource or url should be set */
public static final String KEYSTORE_PARAM_FILE = "file";
/** keystore resource. Only one of file, resource or url should be set */
public static final String KEYSTORE_PARAM_RESOURCE = "resource";
/** keystore url. Only one of file, resource or url should be set */
public static final String KEYSTORE_PARAM_URL = "url";
/** keystore type */
public static final String KEYSTORE_PARAM_TYPE = "type";
/** keystore provider */
public static final String KEYSTORE_PARAM_PROVIDER = "provider";
/** keystore password */
public static final String KEYSTORE_PARAM_PASSWORD = "******";
/** private key password */
public static final String KEYSTORE_PARAM_KEY_PASSWORD = "******";
/** private key password file */
public static final String KEYSTORE_PARAM_KEY_PASSWORD_FILE = "keyPasswordFile";
/**
* If true, and the keystore doesn't exist, a keystore with a self-signed certificate will be be
* created.
*/
public static final String KEYSTORE_PARAM_CREATE = "create";
protected String defaultKeyStoreType = DEFAULT_DEFAULT_KEYSTORE_TYPE;
protected String defaultKeyStoreProvider = DEFAULT_DEFAULT_KEYSTORE_PROVIDER;
protected boolean paramExitIfMissingKeyStore = DEFAULT_EXIT_IF_MISSING_KEYSTORE;
// Pseudo params for param doc
public static final String DOC_PREFIX = PARAM_KEYSTORE + ".<id>.";
/** Name by which daemon component(s) refer to this keystore */
public static final String PARAM_KEYSTORE_NAME = DOC_PREFIX + KEYSTORE_PARAM_NAME;
/** Keystore filename. Only one of file, resource or url should be set */
public static final String PARAM_KEYSTORE_FILE = DOC_PREFIX + KEYSTORE_PARAM_FILE;
/** Keystore resource. Only one of file, resource or url should be set */
public static final String PARAM_KEYSTORE_RESOURCE = DOC_PREFIX + KEYSTORE_PARAM_RESOURCE;
/** Keystore url. Only one of file, resource or url should be set */
public static final String PARAM_KEYSTORE_URL = DOC_PREFIX + KEYSTORE_PARAM_URL;
/** Keystore type (JKS, JCEKS, etc.) */
public static final String PARAM_KEYSTORE_TYPE = DOC_PREFIX + KEYSTORE_PARAM_TYPE;
/** Keystore provider (SunJCE, etc.) */
public static final String PARAM_KEYSTORE_PROVIDER = DOC_PREFIX + KEYSTORE_PARAM_PROVIDER;
/** Keystore password. Default is machine's fqdn */
public static final String PARAM_KEYSTORE_PASSWORD = DOC_PREFIX + KEYSTORE_PARAM_PASSWORD;
/** Private key password */
public static final String PARAM_KEYSTORE_KEY_PASSWORD = DOC_PREFIX + KEYSTORE_PARAM_KEY_PASSWORD;
/** private key password file */
public static final String PARAM_KEYSTORE_KEY_PASSWORD_FILE =
DOC_PREFIX + KEYSTORE_PARAM_KEY_PASSWORD_FILE;
/**
* If true, and the keystore doesn't exist, a keystore with a self-signed certificate will be be
* created.
*/
public static final String PARAM_KEYSTORE_CREATE = DOC_PREFIX + KEYSTORE_PARAM_CREATE;
public static boolean DEFAULT_CREATE = false;
protected Map<String, LockssKeyStore> keystoreMap = new HashMap<String, LockssKeyStore>();
public void startService() {
super.startService();
loadKeyStores();
}
public synchronized void setConfig(
Configuration config, Configuration prevConfig, Configuration.Differences changedKeys) {
if (changedKeys.contains(PREFIX)) {
defaultKeyStoreType = config.get(PARAM_DEFAULT_KEYSTORE_TYPE, DEFAULT_DEFAULT_KEYSTORE_TYPE);
defaultKeyStoreProvider =
config.get(PARAM_DEFAULT_KEYSTORE_PROVIDER, DEFAULT_DEFAULT_KEYSTORE_PROVIDER);
paramExitIfMissingKeyStore =
config.getBoolean(PARAM_EXIT_IF_MISSING_KEYSTORE, DEFAULT_EXIT_IF_MISSING_KEYSTORE);
if (changedKeys.contains(PARAM_KEYSTORE)) {
configureKeyStores(config);
// defer initial set of keystore loading until startService
if (isInited()) {
// load any newly added keystores
loadKeyStores();
}
}
}
}
/**
* Return the named LockssKeyStore or null
*
* @param name the keystore name
*/
public LockssKeyStore getLockssKeyStore(String name) {
return getLockssKeyStore(name, null);
}
/**
* Return the named LockssKeyStore
*
* @param name the keystore name
* @param criticalServiceName if non-null, this is a criticial keystore whose unavailability
* should cause the daemon to exit (if org.lockss.keyMgr.exitIfMissingKeystore is true)
*/
public LockssKeyStore getLockssKeyStore(String name, String criticalServiceName) {
LockssKeyStore res = keystoreMap.get(name);
checkFact(res, name, criticalServiceName, null);
return res;
}
/**
* Convenience method to return the KeyManagerFactory from the named LockssKeyStore, or null
*
* @param name the keystore name
*/
public KeyManagerFactory getKeyManagerFactory(String name) {
return getKeyManagerFactory(name, null);
}
/**
* Convenience method to return the KeyManagerFactory from the named LockssKeyStore.
*
* @param name the keystore name
* @param criticalServiceName if non-null, this is a criticial keystore whose unavailability
* should cause the daemon to exit (if org.lockss.keyMgr.exitIfMissingKeystore is true)
*/
public KeyManagerFactory getKeyManagerFactory(String name, String criticalServiceName) {
LockssKeyStore lk = getLockssKeyStore(name, criticalServiceName);
if (lk != null) {
KeyManagerFactory fact = lk.getKeyManagerFactory();
checkFact(fact, name, criticalServiceName, "found but contains no private keys");
return fact;
}
return null;
}
/** Convenience method to return the TrustManagerFactory from the named LockssKeyStore, or null */
public TrustManagerFactory getTrustManagerFactory(String name) {
return getTrustManagerFactory(name, null);
}
/**
* Convenience method to return the TrustManagerFactory from the named LockssKeyStore.
*
* @param name the keystore name
* @param criticalServiceName if non-null, this is a criticial keystore whose unavailability
* should cause the daemon to exit (if org.lockss.keyMgr.exitIfMissingKeystore is true)
*/
public TrustManagerFactory getTrustManagerFactory(String name, String criticalServiceName) {
LockssKeyStore lk = getLockssKeyStore(name, criticalServiceName);
if (lk != null) {
TrustManagerFactory fact = lk.getTrustManagerFactory();
checkFact(fact, name, criticalServiceName, "found but contains no trusted certificates");
return fact;
}
return null;
}
private void checkFact(Object fact, String name, String criticalServiceName, String message) {
if (fact == null && criticalServiceName != null) {
String msg =
StringUtil.isNullString(name)
? ("No keystore name given for critical keystore"
+ " needed for service "
+ criticalServiceName
+ ", daemon exiting")
: ("Critical keystore "
+ name
+ " "
+ ((message != null) ? message : "not found or not loadable")
+ " for service "
+ criticalServiceName
+ ", daemon exiting");
log.critical(msg);
if (paramExitIfMissingKeyStore) {
System.exit(Constants.EXIT_CODE_KEYSTORE_MISSING);
} else {
throw new IllegalArgumentException(msg);
}
}
}
/** Create LockssKeystores from config subtree below {@link #PARAM_KEYSTORE} */
void configureKeyStores(Configuration config) {
Configuration allKs = config.getConfigTree(PARAM_KEYSTORE);
for (Iterator iter = allKs.nodeIterator(); iter.hasNext(); ) {
String id = (String) iter.next();
Configuration oneKs = allKs.getConfigTree(id);
try {
LockssKeyStore lk = createLockssKeyStore(oneKs);
String name = lk.getName();
if (name == null) {
log.error("KeyStore definition missing name: " + oneKs);
continue;
}
LockssKeyStore old = keystoreMap.get(name);
if (old != null && !lk.equals(old)) {
log.warning(
"Keystore "
+ name
+ " redefined. "
+ "New definition may not take effect until daemon restart");
}
log.debug("Adding keystore " + name);
keystoreMap.put(name, lk);
} catch (Exception e) {
log.error("Couldn't create keystore: " + oneKs, e);
}
}
}
/** Create LockssKeystore from a config subtree */
LockssKeyStore createLockssKeyStore(Configuration config) {
log.debug2("Creating LockssKeyStore from config: " + config);
String name = config.get(KEYSTORE_PARAM_NAME);
LockssKeyStore lk = new LockssKeyStore(name);
String file = config.get(KEYSTORE_PARAM_FILE);
String resource = config.get(KEYSTORE_PARAM_RESOURCE);
String url = config.get(KEYSTORE_PARAM_URL);
if (!StringUtil.isNullString(file)) {
lk.setLocation(file, LocationType.File);
} else if (!StringUtil.isNullString(resource)) {
lk.setLocation(resource, LocationType.Resource);
} else if (!StringUtil.isNullString(url)) {
lk.setLocation(url, LocationType.Url);
}
lk.setType(config.get(KEYSTORE_PARAM_TYPE, defaultKeyStoreType));
lk.setProvider(config.get(KEYSTORE_PARAM_PROVIDER, defaultKeyStoreProvider));
lk.setPassword(config.get(KEYSTORE_PARAM_PASSWORD));
lk.setKeyPassword(config.get(KEYSTORE_PARAM_KEY_PASSWORD));
lk.setKeyPasswordFile(config.get(KEYSTORE_PARAM_KEY_PASSWORD_FILE));
lk.setMayCreate(config.getBoolean(KEYSTORE_PARAM_CREATE, DEFAULT_CREATE));
return lk;
}
void loadKeyStores() {
List<LockssKeyStore> lst = new ArrayList<LockssKeyStore>(keystoreMap.values());
for (LockssKeyStore lk : lst) {
try {
lk.load();
} catch (Exception e) {
log.error("Can't load keystore " + lk.getName(), e);
keystoreMap.remove(lk.getName());
}
}
}
}
// SingleThreadModel causes servlet instances to be assigned to only a
// single thread (request) at a time.
public abstract class LockssServlet extends HttpServlet implements SingleThreadModel {
protected static Logger log = Logger.getLogger("LockssServlet");
// Constants
static final String PARAM_LOCAL_IP = Configuration.PREFIX + "localIPAddress";
static final String PARAM_PLATFORM_VERSION = Configuration.PREFIX + "platform.version";
/** Inactive HTTP session (cookie) timeout */
static final String PARAM_UI_SESSION_TIMEOUT = Configuration.PREFIX + "ui.sessionTimeout";
static final long DEFAULT_UI_SESSION_TIMEOUT = 2 * Constants.DAY;
/** Maximum size of uploaded file accepted */
static final String PARAM_MAX_UPLOAD_FILE_SIZE = Configuration.PREFIX + "ui.maxUploadFileSize";
static final int DEFAULT_MAX_UPLOAD_FILE_SIZE = 500000;
/** The warning string to display when the UI is disabled. */
static final String PARAM_UI_WARNING = Configuration.PREFIX + "ui.warning";
// session keys
static final String SESSION_KEY_OBJECT_ID = "obj_id";
static final String SESSION_KEY_OBJ_MAP = "obj_map";
public static final String SESSION_KEY_RUNNING_SERVLET = "running_servlet";
public static final String SESSION_KEY_REQUEST_HOST = "request_host";
// Name given to form element whose value is the action that should be
// performed when the form is submitted. (Not always the submit button.)
public static final String ACTION_TAG = "lockssAction";
public static final String JAVASCRIPT_RESOURCE = "org/lockss/htdocs/admin.js";
public static final String ATTR_INCLUDE_SCRIPT = "IncludeScript";
public static final String ATTR_ALLOW_ROLES = "AllowRoles";
/** User may configure admin access (add/delete/modify users, set admin access list) */
public static final String ROLE_USER_ADMIN = "userAdminRole";
/** User may configure content access (set content access list) */
public static final String ROLE_CONTENT_ADMIN = "contentAdminRole";
/** User may change AU configuration (add/delete content) */
public static final String ROLE_AU_ADMIN = "auAdminRole";
public static final String ROLE_DEBUG = "debugRole";
protected ServletContext context;
private LockssApp theApp = null;
private ServletManager servletMgr;
private AccountManager acctMgr;
// Request-local storage. Convenient, but requires servlet instances
// to be single threaded, and must ensure reset them to avoid carrying
// over state between requests.
protected HttpServletRequest req;
protected HttpServletResponse resp;
protected URL reqURL;
protected HttpSession session;
private String adminDir = null;
protected String clientAddr; // client addr, even if no param
protected String localAddr;
protected MultiPartRequest multiReq;
private Vector footnotes;
private int footNumber;
private int tabindex;
ServletDescr _myServletDescr = null;
private String myName = null;
// number submit buttons sequentially so unit tests can find them
protected int submitButtonNumber = 0;
/** Run once when servlet loaded. */
public void init(ServletConfig config) throws ServletException {
super.init(config);
context = config.getServletContext();
theApp = (LockssApp) context.getAttribute(ServletManager.CONTEXT_ATTR_LOCKSS_APP);
servletMgr = (ServletManager) context.getAttribute(ServletManager.CONTEXT_ATTR_SERVLET_MGR);
if (theApp instanceof LockssDaemon) {
acctMgr = getLockssDaemon().getAccountManager();
}
}
public ServletManager getServletManager() {
return servletMgr;
}
protected ServletDescr[] getServletDescrs() {
return servletMgr.getServletDescrs();
}
/** Servlets must implement this method. */
protected abstract void lockssHandleRequest() throws ServletException, IOException;
/** Common request handling. */
public void service(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
resetState();
boolean success = false;
HttpSession session = req.getSession(false);
try {
this.req = req;
this.resp = resp;
if (log.isDebug()) {
logParams();
}
resp.setContentType("text/html");
if (!mayPageBeCached()) {
resp.setHeader("pragma", "no-cache");
resp.setHeader("Cache-control", "no-cache");
}
reqURL = new URL(UrlUtil.getRequestURL(req));
clientAddr = getLocalIPAddr();
// check that current user has permission to run this servlet
if (!isServletAllowed(myServletDescr())) {
displayWarningInLieuOfPage("You are not authorized to use " + myServletDescr().heading);
return;
}
// check whether servlet is disabled
String reason = ServletUtil.servletDisabledReason(myServletDescr().getServletName());
if (reason != null) {
displayWarningInLieuOfPage("This function is disabled. " + reason);
return;
}
if (session != null) {
session.setAttribute(SESSION_KEY_RUNNING_SERVLET, getHeading());
String reqHost = req.getRemoteHost();
String forw = req.getHeader(HttpFields.__XForwardedFor);
if (!StringUtil.isNullString(forw)) {
reqHost += " (proxies for " + forw + ")";
}
session.setAttribute(SESSION_KEY_REQUEST_HOST, reqHost);
}
lockssHandleRequest();
success = (errMsg == null);
} catch (ServletException e) {
log.error("Servlet threw", e);
throw e;
} catch (IOException e) {
log.error("Servlet threw", e);
throw e;
} catch (RuntimeException e) {
log.error("Servlet threw", e);
throw e;
} finally {
if (session != null) {
session.setAttribute(SESSION_KEY_RUNNING_SERVLET, null);
session.setAttribute(LockssFormAuthenticator.__J_AUTH_ACTIVITY, TimeBase.nowMs());
}
if ("please".equalsIgnoreCase(req.getHeader("X-Lockss-Result"))) {
log.debug3("X-Lockss-Result: " + (success ? "Ok" : "Fail"));
resp.setHeader("X-Lockss-Result", success ? "Ok" : "Fail");
}
resetMyLocals();
resetLocals();
}
}
protected void resetState() {
multiReq = null;
footNumber = 0;
submitButtonNumber = 0;
tabindex = 1;
statusMsg = null;
errMsg = null;
isFramed = false;
}
protected void resetLocals() {}
protected void resetMyLocals() {
// Don't hold on to stuff forever
req = null;
resp = null;
session = null;
reqURL = null;
adminDir = null;
localAddr = null;
footnotes = null;
_myServletDescr = null;
myName = null;
multiReq = null;
}
/**
* Return true if generated page may be cached (e.g., by browser). Default is false as most
* servlets generate dynamic results
*/
protected boolean mayPageBeCached() {
return false;
}
/** Set the session timeout to the configured value */
protected void setSessionTimeout(HttpSession session) {
Configuration config = CurrentConfig.getCurrentConfig();
setSessionTimeout(
session, config.getTimeInterval(PARAM_UI_SESSION_TIMEOUT, DEFAULT_UI_SESSION_TIMEOUT));
}
/** Set the session timeout */
protected void setSessionTimeout(HttpSession session, long time) {
session.setMaxInactiveInterval((int) (time / Constants.SECOND));
}
/** Get the current session, creating it if necessary (and set the timeout if so) */
protected HttpSession getSession() {
if (session == null) {
session = req.getSession(true);
if (session.isNew()) {
setSessionTimeout(session);
}
}
return session;
}
/** Return true iff a session has already been established */
protected boolean hasSession() {
return req.getSession(false) != null;
}
/** Get an unused ID string for storing an object in the session */
protected String getNewSessionObjectId() {
HttpSession session = getSession();
synchronized (session) {
Integer id = (Integer) getSession().getAttribute(SESSION_KEY_OBJECT_ID);
if (id == null) {
id = new Integer(1);
}
session.setAttribute(SESSION_KEY_OBJECT_ID, new Integer(id.intValue() + 1));
return id.toString();
}
}
/** Get the object associated with the ID in the session */
protected Object getSessionIdObject(String id) {
HttpSession session = getSession();
synchronized (session) {
BidiMap map = (BidiMap) session.getAttribute(SESSION_KEY_OBJ_MAP);
if (map == null) {
return null;
}
return map.getKey(id);
}
}
/** Get the String associated with the ID in the session */
protected String getSessionIdString(String id) {
return (String) getSessionIdObject(id);
}
/** Get the ID with which the object is associated with the session, if any */
protected String getSessionObjectId(Object obj) {
HttpSession session = getSession();
BidiMap map;
synchronized (session) {
map = (BidiMap) session.getAttribute(SESSION_KEY_OBJ_MAP);
if (map == null) {
map = new DualHashBidiMap();
session.setAttribute(SESSION_KEY_OBJ_MAP, map);
}
}
synchronized (map) {
String id = (String) map.get(obj);
if (id == null) {
id = getNewSessionObjectId();
map.put(obj, id);
}
return id;
}
}
// Return descriptor of running servlet
protected ServletDescr myServletDescr() {
if (_myServletDescr == null) {
_myServletDescr = servletMgr.findServletDescr(this);
}
return _myServletDescr;
}
// By default, servlet heading is in descr. Override method to
// compute other heading
protected String getHeading(ServletDescr d) {
if (d == null) return "Unknown Servlet";
return d.heading;
}
protected String getHeading() {
return getHeading(myServletDescr());
}
String getLocalIPAddr() {
if (localAddr == null) {
try {
IPAddr localHost = IPAddr.getLocalHost();
localAddr = localHost.getHostAddress();
} catch (UnknownHostException e) {
// shouldn't happen
log.error("LockssServlet: getLocalHost: " + e.toString());
return "???";
}
}
return localAddr;
}
// Return IP addr used by LCAP. If specified by (misleadingly named)
// localIPAddress prop, might not really be our address (if we are
// behind NAT).
String getLcapIPAddr() {
String ip = CurrentConfig.getParam(PARAM_LOCAL_IP);
if (ip == null || ip.length() <= 0) {
return getLocalIPAddr();
}
return ip;
}
String getRequestHost() {
return reqURL.getHost();
}
String getMachineName() {
return PlatformUtil.getLocalHostname();
}
// String getMachineName0() {
// if (myName == null) {
// // Return the canonical name of the interface the request was aimed
// // at. (localIPAddress prop isn't necessarily right here, as it
// // might be the address of a NAT that we're behind.)
// String host = reqURL.getHost();
// try {
// IPAddr localHost = IPAddr.getByName(host);
// String ip = localHost.getHostAddress();
// myName = getMachineName(ip);
// } catch (UnknownHostException e) {
// // shouldn't happen
// log.error("getMachineName", e);
// return host;
// }
// }
// return myName;
// }
// String getMachineName(String ip) {
// try {
// IPAddr inet = IPAddr.getByName(ip);
// return inet.getHostName();
// } catch (UnknownHostException e) {
// log.warning("getMachineName", e);
// }
// return ip;
// }
// return IP given name or IP
String getMachineIP(String name) {
try {
IPAddr inet = IPAddr.getByName(name);
return inet.getHostAddress();
} catch (UnknownHostException e) {
return null;
}
}
boolean isServletLinkInNav(ServletDescr d) {
return !isThisServlet(d) || linkMeInNav();
}
boolean isThisServlet(ServletDescr d) {
return d == myServletDescr();
}
/** servlets may override this to determine whether they should be a link in nav table */
protected boolean linkMeInNav() {
return false;
}
boolean isLargeLogo() {
return myServletDescr().isLargeLogo();
}
// user predicates
String getUsername() {
Principal user = req.getUserPrincipal();
return user != null ? user.toString() : null;
}
protected UserAccount getUserAccount() {
if (acctMgr != null) {
return acctMgr.getUser(getUsername());
}
return AccountManager.NOBODY_ACCOUNT;
}
protected boolean isDebugUser() {
return doesUserHaveRole(ROLE_DEBUG);
}
protected boolean doesUserHaveRole(String role) {
if ((req.isUserInRole(role) || req.isUserInRole(ROLE_USER_ADMIN)) && !hasNoRoleParsm(role)) {
return true;
}
return hasTestRole(role);
}
static Map<String, String> noRoleParams = new HashMap<String, String>();
static {
noRoleParams.put(ROLE_USER_ADMIN, "noadmin");
noRoleParams.put(ROLE_CONTENT_ADMIN, "nocontent");
noRoleParams.put(ROLE_AU_ADMIN, "noau");
noRoleParams.put(ROLE_DEBUG, "nodebug");
}
protected boolean hasNoRoleParsm(String roleName) {
String noRoleParam = noRoleParams.get(roleName);
return (noRoleParam != null && !StringUtil.isNullString(req.getParameter(noRoleParam)));
}
protected boolean hasTestRole(String role) {
// Servlet test harness puts roles in context
List roles = (List) context.getAttribute(ATTR_ALLOW_ROLES);
return roles != null && (roles.contains(role) || roles.contains(ROLE_USER_ADMIN));
}
protected boolean isServletAllowed(ServletDescr d) {
if (d.needsUserAdminRole() && !doesUserHaveRole(ROLE_USER_ADMIN)) return false;
if (d.needsContentAdminRole() && !doesUserHaveRole(ROLE_CONTENT_ADMIN)) return false;
if (d.needsAuAdminRole() && !doesUserHaveRole(ROLE_AU_ADMIN)) return false;
return d.isEnabled(getLockssDaemon());
}
protected boolean isServletDisplayed(ServletDescr d) {
if (!isServletAllowed(d)) return false;
if (d.needsDebugRole() && !doesUserHaveRole(ROLE_DEBUG)) return false;
return true;
}
protected boolean isServletInNav(ServletDescr d) {
if (d.cls == ServletDescr.UNAVAILABLE_SERVLET_MARKER) return false;
return d.isInNav(this) && isServletDisplayed(d);
}
// Called when a servlet doesn't get the parameters it expects/needs
protected void paramError() throws IOException {
// FIXME: As of 2006-03-15 this method and its only caller checkParam() are not called from
// anywhere
PrintWriter wrtr = resp.getWriter();
Page page = new Page();
// add referer, params, msg to contact lockss unless from old bookmark
// or manually entered url
page.add("Parameter error");
page.write(wrtr);
}
// return true iff error
protected boolean checkParam(boolean ok, String msg) throws IOException {
if (ok) return false;
log.error(myServletDescr().getPath() + ": " + msg);
paramError();
return true;
}
/** Construct servlet URL */
String srvURL(ServletDescr d) {
return srvURL((String) null, d, null);
}
/** Construct servlet URL with params */
String srvURL(ServletDescr d, String params) {
return srvURL((String) null, d, params);
}
/** Construct servlet URL with params */
String srvURL(ServletDescr d, Properties params) {
return srvURL(d, concatParams(params));
}
/** Construct servlet absolute URL, with params as necessary. */
String srvAbsURL(ServletDescr d, String params) {
return srvURL(getRequestHost(), d, params);
}
/**
* Construct servlet URL, with params as necessary. Avoid generating a hostname different from
* that used in the original request, or browsers will prompt again for login
*/
String srvURL(String host, ServletDescr d, String params) {
return srvURLFromStem(srvUrlStem(host), d, params);
}
String srvURL(PeerIdentity peer, ServletDescr d, String params) {
return srvURLFromStem(peer.getUiUrlStem(reqURL.getPort()), d, params);
}
/**
* Construct servlet URL, with params as necessary. Avoid generating a hostname different from
* that used in the original request, or browsers will prompt again for login
*/
String srvURLFromStem(String stem, ServletDescr d, String params) {
if (d.isPathIsUrl()) {
return d.getPath();
}
StringBuilder sb = new StringBuilder(80);
if (stem != null) {
sb.append(stem);
if (stem.charAt(stem.length() - 1) != '/') {
sb.append('/');
}
} else {
// ensure absolute path even if no scheme/host/port
sb.append('/');
}
sb.append(d.getPath());
if (params != null) {
sb.append('?');
sb.append(params);
}
return sb.toString();
}
String srvUrlStem(String host) {
if (host == null) {
return null;
}
StringBuilder sb = new StringBuilder();
sb.append(reqURL.getProtocol());
sb.append("://");
sb.append(host);
sb.append(':');
sb.append(reqURL.getPort());
return sb.toString();
}
/** Return a link to a servlet */
String srvLink(ServletDescr d, String text) {
return srvLink(d, text, (String) null);
}
/** Return a link to a servlet with params */
String srvLink(ServletDescr d, String text, String params) {
return new Link(srvURL(d, params), (text != null ? text : d.heading)).toString();
}
/** Return a link to a servlet with params */
String srvLink(ServletDescr d, String text, Properties params) {
return new Link(srvURL(d, params), text).toString();
}
/** Return an absolute link to a servlet with params */
String srvAbsLink(ServletDescr d, String text, Properties params) {
return srvAbsLink(d, text, concatParams(params));
}
/** Return an absolute link to a servlet with params */
String srvAbsLink(ServletDescr d, String text, String params) {
return new Link(srvAbsURL(d, params), (text != null ? text : d.heading)).toString();
}
/** Return an absolute link to a servlet with params */
String srvAbsLink(String host, ServletDescr d, String text, String params) {
return new Link(srvURL(host, d, params), (text != null ? text : d.heading)).toString();
}
/** Return an absolute link to a servlet with params */
String srvAbsLink(PeerIdentity peer, ServletDescr d, String text, String params) {
return new Link(srvURL(peer, d, params), (text != null ? text : d.heading)).toString();
}
/** Return text as a link iff isLink */
String conditionalSrvLink(ServletDescr d, String text, String params, boolean isLink) {
if (isLink) {
return srvLink(d, text, params);
} else {
return text;
}
}
/** Return text as a link iff isLink */
String conditionalSrvLink(ServletDescr d, String text, boolean isLink) {
return conditionalSrvLink(d, text, null, isLink);
}
/** Concatenate params for URL string */
static String concatParams(String p1, String p2) {
if (StringUtil.isNullString(p1)) {
return p2;
}
if (StringUtil.isNullString(p2)) {
return p1;
}
return p1 + "&" + p2;
}
/** Concatenate params for URL string */
String concatParams(Properties props) {
if (props == null) {
return null;
}
java.util.List list = new ArrayList();
for (Iterator iter = props.keySet().iterator(); iter.hasNext(); ) {
String key = (String) iter.next();
String val = props.getProperty(key);
if (!StringUtil.isNullString(val)) {
list.add(key + "=" + urlEncode(val));
}
}
return StringUtil.separatedString(list, "&");
}
String modifyParams(String key, String val) {
Properties props = getParamsAsProps();
props.setProperty(key, val);
return concatParams(props);
}
/**
* Return the request parameters as a Properties. Only the first value of multivalued parameters
* is included.
*/
Properties getParamsAsProps() {
Properties props = new Properties();
for (Enumeration en = req.getParameterNames(); en.hasMoreElements(); ) {
String name = (String) en.nextElement();
props.setProperty(name, req.getParameter(name));
}
return props;
}
/**
* Return the request parameters as a Map<String,String>. Only the first value of multivalued
* parameters is included.
*/
Map<String, String> getParamsAsMap() {
Map<String, String> map = new HashMap<String, String>();
for (Enumeration en = req.getParameterNames(); en.hasMoreElements(); ) {
String name = (String) en.nextElement();
map.put(name, req.getParameter(name));
}
return map;
}
protected String urlEncode(String param) {
return UrlUtil.encodeUrl(param);
}
protected String getRequestKey() {
String key = req.getPathInfo();
if (key != null && key.startsWith("/")) {
return key.substring(1);
}
return key;
}
/** Common page setup. */
protected Page newPage() {
// Compute heading
String heading = getHeading();
if (heading == null) {
heading = "Box Administration";
}
// Create page and layout header
Page page = ServletUtil.doNewPage(getPageTitle(), isFramed());
Iterator inNavIterator;
if (myServletDescr().hasNoNavTable()) {
inNavIterator = CollectionUtil.EMPTY_ITERATOR;
} else {
inNavIterator =
new FilterIterator(
new ObjectArrayIterator(getServletDescrs()),
new Predicate() {
public boolean evaluate(Object obj) {
return isServletInNav((ServletDescr) obj);
}
});
}
ServletUtil.layoutHeader(
this,
page,
heading,
isLargeLogo(),
getMachineName(),
getLockssApp().getStartDate(),
inNavIterator);
String warnMsg = CurrentConfig.getParam(PARAM_UI_WARNING);
if (warnMsg != null) {
Composite warning = new Composite();
warning.add("<center><font color=red size=+1>");
warning.add(warnMsg);
warning.add("</font></center><br>");
page.add(warning);
}
return page;
}
protected Page addBarePageHeading(Page page) {
// FIXME: Move the following fragment elsewhere
// It causes the doctype statement to appear in the middle,
// after the <body> tag.
page.add("<!doctype html public \"-//w3c//dtd html 4.0 transitional//en\">");
page.addHeader("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=iso-8859-1\">");
page.addHeader("<meta http-equiv=\"content-type\" content=\"text/html;charset=ISO-8859-1\">");
page.addHeader("<link rel=\"shortcut icon\" href=\"/favicon.ico\" type=\"image/x-icon\" />");
return page;
}
private boolean isFramed = false;
protected String errMsg;
protected String statusMsg;
protected boolean isFramed() {
return isFramed;
}
protected void setFramed(boolean v) {
isFramed = v;
}
protected String getPageTitle() {
String heading = getHeading();
if (heading != null) {
return "LOCKSS: " + heading;
} else {
return "LOCKSS";
}
}
/** Return a button that invokes the javascript submit routine with the specified action */
protected Element submitButton(String label, String action) {
return submitButton(label, action, null, null);
}
/** Return a button that invokes javascript when clicked. */
Input jsButton(String label, String js) {
Input btn = new Input("button", null);
btn.attribute("value", label);
setTabOrder(btn);
btn.attribute("onClick", js);
return btn;
}
/**
* Return a button that invokes the javascript submit routine with the specified action, first
* storing the value in the specified form prop.
*/
protected Element submitButton(String label, String action, String prop, String value) {
StringBuilder sb = new StringBuilder(40);
sb.append("lockssButton(this, '");
sb.append(action);
sb.append("'");
if (prop != null && value != null) {
sb.append(", '");
sb.append(prop);
sb.append("', '");
sb.append(value);
sb.append("'");
}
sb.append(")");
Input btn = jsButton(label, sb.toString());
btn.attribute("id", "lsb." + (++submitButtonNumber));
return btn;
}
/**
* Return a (possibly labelled) checkbox.
*
* @param label appears to right of checkbox if non null
* @param value value included in result set if box checked
* @param key form key to which result set is assigned
* @param checked if true, box is initially checked
* @return a checkbox Element
*/
Element checkBox(String label, String value, String key, boolean checked) {
Input in = new Input(Input.Checkbox, key, value);
if (checked) {
in.check();
}
setTabOrder(in);
if (StringUtil.isNullString(label)) {
return in;
} else {
Composite c = new Composite();
c.add(in);
c.add(" ");
c.add(label);
return c;
}
}
/**
* Return a labelled rasio button
*
* @param label label to right of circle, and form value if checked
* @param key form key to which value is assigned
* @param checked if true, is initially checked
* @return a readio button Element
*/
protected Element radioButton(String label, String key, boolean checked) {
return radioButton(label, label, key, checked);
}
/**
* Return a labelled rasio button
*
* @param label appears to right of circle if non null
* @param value value assigned to key if box checked
* @param key form key to which value is assigned
* @param checked if true, is initially checked
* @return a readio button Element
*/
protected Element radioButton(String label, String value, String key, boolean checked) {
Composite c = new Composite();
Input in = new Input(Input.Radio, key, value);
if (checked) {
in.check();
}
setTabOrder(in);
c.add(in);
c.add(label);
return c;
}
/** Add html tags to grey the text if isGrey is true */
protected String greyText(String txt, boolean isGrey) {
if (!isGrey) {
return txt;
}
return "<font color=gray>" + txt + "</font>";
}
/**
* Set this element next in the tab order. Returns the element for easier nesting in expressions.
*/
protected Element setTabOrder(Element ele) {
ele.attribute("tabindex", tabindex++);
return ele;
}
/**
* Store a footnote, assign it a number, return html for footnote reference. If footnote in null
* or empty, no footnote is added and an empty string is returned. Footnote numbers get turned
* into links; <b>Do not put the result of addFootnote inside a link!</b>.
*/
protected String addFootnote(String s) {
if (s == null || s.length() == 0) {
return "";
}
if (footNumber == 0) {
if (footnotes == null) {
footnotes = new Vector(10, 10);
} else {
footnotes.removeAllElements();
}
}
int n = footnotes.indexOf(s);
if (n < 0) {
n = footNumber++;
footnotes.addElement(s);
}
return "<sup><font size=-1><a href=#foottag" + (n + 1) + ">" + (n + 1) + "</a></font></sup>";
}
/**
* Add javascript to page. Normally adds a link to the script file, but can be told to include the
* script directly in the page, to accomodate unit testing of individual servlets, when other
* fetches won't work.
*/
protected void addJavaScript(Composite comp) {
String include = (String) context.getAttribute(ATTR_INCLUDE_SCRIPT);
if (StringUtil.isNullString(include)) {
linkToJavaScript(comp);
} else {
includeJavaScript0(comp);
}
}
private void includeJavaScript0(Composite comp) {
Script script = new Script(getJavascript());
comp.add(script);
}
private void linkToJavaScript(Composite comp) {
Script script = new Script("");
script.attribute("src", "admin.js");
comp.add(script);
}
private static String jstext = null;
private static synchronized String getJavascript() {
if (jstext == null) {
InputStream istr = null;
try {
ClassLoader loader = Thread.currentThread().getContextClassLoader();
istr = loader.getResourceAsStream(JAVASCRIPT_RESOURCE);
jstext = StringUtil.fromInputStream(istr);
istr.close();
} catch (Exception e) {
log.error("Can't load javascript", e);
} finally {
IOUtil.safeClose(istr);
}
}
return jstext;
}
/** Display a message in lieu of the normal page */
protected void displayMsgInLieuOfPage(String msg) throws IOException {
// TODO: Look at HTML
Page page = newPage();
Composite warning = new Composite();
warning.add(msg);
warning.add("<br>");
page.add(warning);
layoutFooter(page);
page.write(resp.getWriter());
}
/** Display a warning in red, in lieu of the normal page */
protected void displayWarningInLieuOfPage(String msg) throws IOException {
displayMsgInLieuOfPage("<center><font color=red size=+1>" + msg + "</font></center>");
}
/** Display "The cache isn't ready yet, come back later" */
protected void displayNotStarted() throws IOException {
displayWarningInLieuOfPage(
"This LOCKSS box is still starting. Please "
+ srvLink(myServletDescr(), "try again", getParamsAsProps())
+ " in a moment.");
}
public MultiPartRequest getMultiPartRequest() throws FormDataTooLongException, IOException {
int maxUpload =
CurrentConfig.getIntParam(PARAM_MAX_UPLOAD_FILE_SIZE, DEFAULT_MAX_UPLOAD_FILE_SIZE);
return getMultiPartRequest(maxUpload);
}
public MultiPartRequest getMultiPartRequest(int maxLen)
throws FormDataTooLongException, IOException {
if (req.getContentType() == null || !req.getContentType().startsWith("multipart/form-data")) {
return null;
}
if (req.getContentLength() > maxLen) {
throw new FormDataTooLongException(req.getContentLength() + " bytes, " + maxLen + " allowed");
}
MultiPartRequest multi = new MultiPartRequest(req);
if (log.isDebug2()) {
String[] parts = multi.getPartNames();
log.debug3("Multipart request, " + parts.length + " parts");
if (log.isDebug3()) {
for (int p = 0; p < parts.length; p++) {
String name = parts[p];
String cont = multi.getString(parts[p]);
log.debug3(name + ": " + cont);
}
}
}
multiReq = multi;
return multi;
}
public String getParameter(String name) {
String val = req.getParameter(name);
if (val == null && multiReq != null) {
val = multiReq.getString(name);
}
if (val == null) {
return null;
}
val = StringUtils.strip(val, " \t");
// if (StringUtil.isNullString(val)) {
if ("".equals(val)) {
return null;
}
return val;
}
protected void layoutFooter(Page page) {
ServletUtil.doLayoutFooter(
page, (footnotes == null ? null : footnotes.iterator()), getLockssApp().getVersionInfo());
if (footnotes != null) {
footnotes.removeAllElements();
}
}
/** Return the app instance. */
protected LockssApp getLockssApp() {
return theApp;
}
/**
* Return the daemon instance, assumes that the servlet is running in the daemon.
*
* @throws ClassCastException if the servlet is running in an app other than the daemon
*/
protected LockssDaemon getLockssDaemon() {
return (LockssDaemon) theApp;
}
protected void logParams() {
Enumeration en = req.getParameterNames();
while (en.hasMoreElements()) {
String name = (String) en.nextElement();
String vals[];
String dispval;
if (StringUtil.indexOfIgnoreCase(name, "passw") >= 0) {
dispval = req.getParameter(name).length() == 0 ? "" : "********";
} else if (log.isDebug2() && (vals = req.getParameterValues(name)).length > 1) {
dispval = StringUtil.separatedString(vals, ", ");
} else {
dispval = req.getParameter(name);
}
log.debug(name + " = " + dispval);
}
}
/** Convenience method */
protected String encodeText(String s) {
return HtmlUtil.encode(s, HtmlUtil.ENCODE_TEXT);
}
/** Convenience method */
protected String encodeTextArea(String s) {
return HtmlUtil.encode(s, HtmlUtil.ENCODE_TEXTAREA);
}
/** Convenience method */
protected String encodeAttr(String s) {
return HtmlUtil.encode(s, HtmlUtil.ENCODE_ATTR);
}
/**
* Create message and error message block
*
* @param composite TODO
*/
protected void layoutErrorBlock(Composite composite) {
if (errMsg != null || statusMsg != null) {
ServletUtil.layoutErrorBlock(composite, errMsg, statusMsg);
}
}
/** Exception thrown if multipart form data is longer than the caller-supplied max */
public static class FormDataTooLongException extends Exception {
public FormDataTooLongException(String message) {
super(message);
}
}
}
/** Export the contents and metadata from an AU */
public abstract class Exporter {
private static final Logger log = Logger.getLogger(Exporter.class);
static final String PREFIX = Configuration.PREFIX + "exporter.";
/** Abort export after this many errors */
public static final String PARAM_MAX_ERRORS = PREFIX + "maxErrors";
public static final int DEFAULT_MAX_ERRORS = 5;
protected static int maxErrors = DEFAULT_MAX_ERRORS;
protected LockssDaemon daemon;
protected ArchivalUnit au;
protected File dir;
protected String prefix;
protected long maxSize = -1;
protected int maxVersions = 1;
protected boolean compress = false;
protected boolean excludeDirNodes = false;
protected FilenameTranslation xlate = FilenameTranslation.XLATE_NONE;
protected List errors = new ArrayList();
protected boolean isDiskFull = false;
protected abstract void start() throws IOException;
protected abstract void finish() throws IOException;
protected abstract void writeCu(CachedUrl cu) throws IOException;
protected Exporter(LockssDaemon daemon, ArchivalUnit au) {
this.daemon = daemon;
this.au = au;
}
/** Called by org.lockss.config.MiscConfig */
public static void setConfig(
Configuration config, Configuration oldConfig, Configuration.Differences diffs) {
if (diffs.contains(PREFIX)) {
maxErrors = config.getInt(PARAM_MAX_ERRORS, DEFAULT_MAX_ERRORS);
}
}
public void setCompress(boolean val) {
compress = val;
}
public boolean getCompress() {
return compress;
}
public void setExcludeDirNodes(boolean val) {
excludeDirNodes = val;
}
public boolean getExcludeDirNodes() {
return excludeDirNodes;
}
public void setFilenameTranslation(FilenameTranslation val) {
xlate = val;
}
public FilenameTranslation getFilenameTranslation() {
return xlate;
}
public void setDir(File val) {
dir = val;
}
public File getDir() {
return dir;
}
public void setPrefix(String val) {
prefix = val;
}
public String getPrefix() {
return prefix;
}
public void setMaxSize(long val) {
maxSize = val;
}
public long getMaxSize() {
return maxSize;
}
public void setMaxVersions(int val) {
maxVersions = val;
}
public int getMaxVersions() {
return maxVersions;
}
public List getErrors() {
return errors;
}
protected void checkArgs() {
if (getDir() == null) {
throw new IllegalArgumentException("Must supply output directory");
}
if (getPrefix() == null) {
throw new IllegalArgumentException("Must supply file name/prefix");
}
}
public void export() {
log.debug("export(" + au.getName() + ")");
log.debug(
"dir: "
+ dir
+ ", pref: "
+ prefix
+ ", size: "
+ maxSize
+ ", ver: "
+ maxVersions
+ (compress ? ", (C)" : ""));
checkArgs();
try {
start();
} catch (IOException e) {
recordError("Error opening file", e);
return;
}
writeFiles();
try {
finish();
} catch (IOException e) {
if (!isDiskFull) {
// If we already knew (and reported) disk full, also reporting it
// as a close error is misleading.
recordError("Error closing file", e);
}
}
}
protected String xlateFilename(String url) {
return xlate.xlate(url);
}
protected void recordError(String msg, Throwable t) {
log.error(msg, t);
errors.add(msg + ": " + t.toString());
}
protected void recordError(String msg) {
log.error(msg);
errors.add(msg);
}
protected String getSoftwareVersion() {
String releaseName = BuildInfo.getBuildProperty(BuildInfo.BUILD_RELEASENAME);
StringBuilder sb = new StringBuilder();
sb.append("LOCKSS Daemon ");
if (releaseName != null) {
sb.append(releaseName);
}
return sb.toString();
}
protected String getHostIp() {
try {
IPAddr localHost = IPAddr.getLocalHost();
return localHost.getHostAddress();
} catch (UnknownHostException e) {
log.error("getHostIp()", e);
return "1.1.1.1";
}
}
protected String getHostName() {
String res = ConfigManager.getPlatformHostname();
if (res == null) {
try {
InetAddress inet = InetAddress.getLocalHost();
return inet.getHostName();
} catch (UnknownHostException e) {
log.warning("Can't get hostname", e);
return "unknown";
}
}
return res;
}
protected Properties filterResponseProps(Properties props) {
Properties res = new Properties();
for (Map.Entry ent : props.entrySet()) {
String key = (String) ent.getKey();
if (StringUtil.startsWithIgnoreCase(key, "x-lockss")
|| StringUtil.startsWithIgnoreCase(key, "x_lockss")
|| key.equalsIgnoreCase("org.lockss.version.number")) {
continue;
}
// We've lost the original case - capitalize them the way most people
// expect
res.put(StringUtil.titleCase(key, '-'), (String) ent.getValue());
}
return res;
}
protected String getHttpResponseString(CachedUrl cu) {
Properties cuProps = cu.getProperties();
Properties filteredProps = filterResponseProps(cuProps);
String hdrString = PropUtil.toHeaderString(filteredProps);
StringBuilder sb = new StringBuilder(hdrString.length() + 30);
String line1 = inferHttpResponseCode(cu, cuProps);
sb.append(line1);
sb.append(Constants.CRLF);
sb.append(hdrString);
sb.append(Constants.CRLF);
return sb.toString();
}
String inferHttpResponseCode(CachedUrl cu, Properties cuProps) {
if (cuProps.get("location") == null) {
return "HTTP/1.1 200 OK";
} else {
return "HTTP/1.1 302 Found";
}
}
// return the next CU with content
private CachedUrl getNextCu(CuIterator iter) {
return iter.hasNext() ? iter.next() : null;
}
/**
* Return true if, interpreting URLs as filenames, dirCu is a directory containing fileCu. Used to
* exclude directory content from output files, so they can be unpacked by standard utilities
* (e.g., unzip). Shouldn't be called with equal URLs, but return false in that case, as we
* wouldn't want to exclude the URL
*/
boolean isDirOf(CachedUrl dirCu, CachedUrl fileCu) {
String dir = dirCu.getUrl();
String file = fileCu.getUrl();
if (!dir.endsWith("/")) {
dir = dir + "/";
}
return file.startsWith(dir) && !file.equals(dir);
}
private void writeFiles() {
PlatformUtil platutil = PlatformUtil.getInstance();
CuIterator iter = AuUtil.getCuIterator(au);
int errs = 0;
CachedUrl curCu = null;
CachedUrl nextCu = getNextCu(iter);
while (nextCu != null) {
curCu = nextCu;
nextCu = getNextCu(iter);
if (excludeDirNodes && nextCu != null && isDirOf(curCu, nextCu)) {
continue;
}
CachedUrl[] cuVersions =
curCu.getCuVersions(maxVersions > 0 ? maxVersions : Integer.MAX_VALUE);
for (CachedUrl cu : cuVersions) {
try {
log.debug2("Exporting " + cu.getUrl());
writeCu(cu);
} catch (IOException e) {
if (platutil.isDiskFullError(e)) {
recordError("Disk full, can't write export file.");
isDiskFull = true;
return;
}
} catch (Exception e) {
// XXX Would like to differentiate between errors opening or
// reading CU, which shouldn't cause abort, and errors writing
// to export file, which should.
recordError("Unable to copy " + cu.getUrl(), e);
if (errs++ >= maxErrors) {
recordError("Aborting after " + errs + " errors");
return;
}
}
}
}
}
private Set<File> fileSet = new HashSet<File>();
private List<File> fileList = new ArrayList<File>();
protected void recordExportFile(File file) {
if (fileSet.add(file)) {
fileList.add(file);
}
}
/**
* Return the list of export files written
*
* @return List of File written
*/
public List<File> getExportFiles() {
return fileList;
}
public interface Factory {
public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au);
}
static final String WINDOWS_FROM = "?<>:*|\\";
static final String WINDOWS_TO = "_______";
static final String MAC_FROM = ":";
static final String MAC_TO = "_";
/** Enum of filename translation types */
public static enum FilenameTranslation {
XLATE_NONE("None") {
public String xlate(String s) {
return s;
}
},
XLATE_WINDOWS("Windows") {
public String xlate(String s) {
return StringUtils.replaceChars(s, WINDOWS_FROM, WINDOWS_TO);
}
},
XLATE_MAC("MacOS") {
public String xlate(String s) {
return StringUtils.replaceChars(s, MAC_FROM, MAC_TO);
}
};
private final String label;
FilenameTranslation(String label) {
this.label = label;
}
public String getLabel() {
return label;
}
public abstract String xlate(String s);
};
/** Enum of Exporter types, and factories */
public static enum Type implements Factory {
ARC_RESOURCE("ARC (content only)") {
public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
return new ArcExporter(daemon, au, false);
}
},
ARC_RESPONSE("ARC (response and content)") {
public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
return new ArcExporter(daemon, au, true);
}
},
WARC_RESOURCE("WARC (content only)") {
public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
return new WarcExporter(daemon, au, false);
}
},
WARC_RESPONSE("WARC (response and content)") {
public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
return new WarcExporter(daemon, au, true);
}
},
ZIP("ZIP") {
public Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au) {
return new ZipExporter(daemon, au);
}
};
private final String label;
Type(String label) {
this.label = label;
}
public abstract Exporter makeExporter(LockssDaemon daemon, ArchivalUnit au);
public String getLabel() {
return label;
}
};
}
/**
* RepositoryManager is the center of the per AU repositories. It manages the repository config
* parameters.
*/
public class RepositoryManager extends BaseLockssDaemonManager implements ConfigurableManager {
private static Logger log = Logger.getLogger("RepositoryManager");
public static final String PREFIX = Configuration.PREFIX + "repository.";
/** Maximum size of per-AU repository node cache */
public static final String PARAM_MAX_PER_AU_CACHE_SIZE = PREFIX + "nodeCache.size";
public static final int DEFAULT_MAX_PER_AU_CACHE_SIZE = 10;
/*
* This needs to be a small multiple of the number of simultaneous
* polls (poller and voter), as there is a cache entry per active AU.
* Each poll will have one active AU at a time.
*/
public static final String PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE =
PREFIX + "suspectVersionsCache.size";
public static final int DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE = 10;
static final String GLOBAL_CACHE_PREFIX = PREFIX + "globalNodeCache.";
public static final String PARAM_MAX_GLOBAL_CACHE_SIZE = GLOBAL_CACHE_PREFIX + "size";
public static final int DEFAULT_MAX_GLOBAL_CACHE_SIZE = 500;
public static final String PARAM_GLOBAL_CACHE_ENABLED = GLOBAL_CACHE_PREFIX + "enabled";
public static final boolean DEFAULT_GLOBAL_CACHE_ENABLED = false;
/** Max times to loop looking for unused AU directory. */
public static final String PARAM_MAX_UNUSED_DIR_SEARCH = PREFIX + "maxUnusedDirSearch";
public static final int DEFAULT_MAX_UNUSED_DIR_SEARCH = 30000;
/** If true, LocalRepository keeps track of next subdir name to probe. */
public static final String PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH =
PREFIX + "enableStatefulUnusedDirSearch";
public static final boolean DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH = true;
/** Max percent of time size calculation thread may run. */
public static final String PARAM_SIZE_CALC_MAX_LOAD = PREFIX + "sizeCalcMaxLoad";
public static final float DEFAULT_SIZE_CALC_MAX_LOAD = 0.5F;
/**
* If true, path components longer than the maximum filesystem path component are encodes are
* multiple levels of directories
*/
public static final String PARAM_ENABLE_LONG_COMPONENTS = PREFIX + "enableLongComponents";
public static final boolean DEFAULT_ENABLE_LONG_COMPONENTS = true;
/**
* Prior to 1.61.6, when long component support is enabled, backslahes were normalized to %5c
* instead of %5C. This is harmless if checkUnnormalized is set to Fix, as they'll be normalized.
* Otherwise, this can be set true for compatibility with old repositories. Setting
* checkUnnormalized to Fix is preferred.
*/
public static final String PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY =
PREFIX + "enableLongComponentsCompatibility";
public static final boolean DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY = false;
/** Maximum length of a filesystem path component. */
public static final String PARAM_MAX_COMPONENT_LENGTH = PREFIX + "maxComponentLength";
public static final int DEFAULT_MAX_COMPONENT_LENGTH = 255;
/** @see #PARAM_CHECK_UNNORMALIZED */
public enum CheckUnnormalizedMode {
No,
Log,
Fix
};
/**
* Check for existing nodes with unnormalized names (created by very old daemon that didn't
* normalize): None, Log, Fix
*/
public static final String PARAM_CHECK_UNNORMALIZED = PREFIX + "checkUnnormalized";
public static final CheckUnnormalizedMode DEFAULT_CHECK_UNNORMALIZED = CheckUnnormalizedMode.Log;
static final String WDOG_PARAM_SIZE_CALC = "SizeCalc";
static final long WDOG_DEFAULT_SIZE_CALC = Constants.DAY;
static final String PRIORITY_PARAM_SIZE_CALC = "SizeCalc";
static final int PRIORITY_DEFAULT_SIZE_CALC = Thread.NORM_PRIORITY - 1;
static final String DISK_PREFIX = PREFIX + "diskSpace.";
static final String PARAM_DISK_WARN_FRRE_MB = DISK_PREFIX + "warn.freeMB";
static final int DEFAULT_DISK_WARN_FRRE_MB = 5000;
static final String PARAM_DISK_FULL_FRRE_MB = DISK_PREFIX + "full.freeMB";
static final int DEFAULT_DISK_FULL_FRRE_MB = 100;
static final String PARAM_DISK_WARN_FRRE_PERCENT = DISK_PREFIX + "warn.freePercent";
static final double DEFAULT_DISK_WARN_FRRE_PERCENT = .02;
static final String PARAM_DISK_FULL_FRRE_PERCENT = DISK_PREFIX + "full.freePercent";
static final double DEFAULT_DISK_FULL_FRRE_PERCENT = .01;
private PlatformUtil platInfo = PlatformUtil.getInstance();
private List repoList = Collections.EMPTY_LIST;
int paramNodeCacheSize = DEFAULT_MAX_PER_AU_CACHE_SIZE;
boolean paramIsGlobalNodeCache = DEFAULT_GLOBAL_CACHE_ENABLED;
int paramGlobalNodeCacheSize = DEFAULT_MAX_GLOBAL_CACHE_SIZE;
int paramSuspectVersionsCacheSize = DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE;
UniqueRefLruCache globalNodeCache = new UniqueRefLruCache(DEFAULT_MAX_GLOBAL_CACHE_SIZE);
UniqueRefLruCache suspectVersionsCache =
new UniqueRefLruCache(DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
Map localRepos = new HashMap();
private static int maxUnusedDirSearch = DEFAULT_MAX_UNUSED_DIR_SEARCH;
private static boolean isStatefulUnusedDirSearch = DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH;
private static boolean enableLongComponents = DEFAULT_ENABLE_LONG_COMPONENTS;
private static boolean enableLongComponentsCompatibility =
DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY;
private static int maxComponentLength = DEFAULT_MAX_COMPONENT_LENGTH;
private static CheckUnnormalizedMode checkUnnormalized = DEFAULT_CHECK_UNNORMALIZED;
PlatformUtil.DF paramDFWarn =
PlatformUtil.DF.makeThreshold(DEFAULT_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_PERCENT);
PlatformUtil.DF paramDFFull =
PlatformUtil.DF.makeThreshold(DEFAULT_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_PERCENT);
private float sizeCalcMaxLoad = DEFAULT_SIZE_CALC_MAX_LOAD;
public void startService() {
super.startService();
localRepos = new HashMap();
}
public void setConfig(
Configuration config, Configuration oldConfig, Configuration.Differences changedKeys) {
// Build list of repositories from list of disk (fs) paths). Needs to
// be generalized if ever another repository implementation.
if (changedKeys.contains(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST)) {
List lst = new ArrayList();
String dspace = config.get(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, "");
List paths = StringUtil.breakAt(dspace, ';');
if (paths != null) {
for (Iterator iter = paths.iterator(); iter.hasNext(); ) {
lst.add("local:" + (String) iter.next());
}
}
repoList = lst;
}
if (changedKeys.contains(PARAM_MAX_PER_AU_CACHE_SIZE)) {
paramNodeCacheSize =
config.getInt(PARAM_MAX_PER_AU_CACHE_SIZE, DEFAULT_MAX_PER_AU_CACHE_SIZE);
for (Iterator iter = getDaemon().getAllLockssRepositories().iterator(); iter.hasNext(); ) {
LockssRepository repo = (LockssRepository) iter.next();
if (repo instanceof LockssRepositoryImpl) {
LockssRepositoryImpl repoImpl = (LockssRepositoryImpl) repo;
repoImpl.setNodeCacheSize(paramNodeCacheSize);
}
}
}
if (changedKeys.contains(PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE)) {
paramSuspectVersionsCacheSize =
config.getInt(
PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE, DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
suspectVersionsCache.setMaxSize(paramSuspectVersionsCacheSize);
}
if (changedKeys.contains(GLOBAL_CACHE_PREFIX)) {
paramIsGlobalNodeCache =
config.getBoolean(PARAM_GLOBAL_CACHE_ENABLED, DEFAULT_GLOBAL_CACHE_ENABLED);
if (paramIsGlobalNodeCache) {
paramGlobalNodeCacheSize =
config.getInt(PARAM_MAX_GLOBAL_CACHE_SIZE, DEFAULT_MAX_GLOBAL_CACHE_SIZE);
log.debug("global node cache size: " + paramGlobalNodeCacheSize);
globalNodeCache.setMaxSize(paramGlobalNodeCacheSize);
}
}
if (changedKeys.contains(DISK_PREFIX)) {
int minMB = config.getInt(PARAM_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_MB);
double minPer =
config.getPercentage(PARAM_DISK_WARN_FRRE_PERCENT, DEFAULT_DISK_WARN_FRRE_PERCENT);
paramDFWarn = PlatformUtil.DF.makeThreshold(minMB, minPer);
minMB = config.getInt(PARAM_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_MB);
minPer = config.getPercentage(PARAM_DISK_FULL_FRRE_PERCENT, DEFAULT_DISK_FULL_FRRE_PERCENT);
paramDFFull = PlatformUtil.DF.makeThreshold(minMB, minPer);
}
if (changedKeys.contains(PARAM_SIZE_CALC_MAX_LOAD)) {
sizeCalcMaxLoad = config.getPercentage(PARAM_SIZE_CALC_MAX_LOAD, DEFAULT_SIZE_CALC_MAX_LOAD);
}
if (changedKeys.contains(PREFIX)) {
maxUnusedDirSearch =
config.getInt(PARAM_MAX_UNUSED_DIR_SEARCH, DEFAULT_MAX_UNUSED_DIR_SEARCH);
isStatefulUnusedDirSearch =
config.getBoolean(
PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH, DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH);
enableLongComponents =
config.getBoolean(PARAM_ENABLE_LONG_COMPONENTS, DEFAULT_ENABLE_LONG_COMPONENTS);
enableLongComponentsCompatibility =
config.getBoolean(
PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY,
DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY);
maxComponentLength = config.getInt(PARAM_MAX_COMPONENT_LENGTH, DEFAULT_MAX_COMPONENT_LENGTH);
checkUnnormalized =
(CheckUnnormalizedMode)
config.getEnum(
CheckUnnormalizedMode.class,
PARAM_CHECK_UNNORMALIZED,
DEFAULT_CHECK_UNNORMALIZED);
}
}
public static boolean isEnableLongComponents() {
return enableLongComponents;
}
public static boolean isEnableLongComponentsCompatibility() {
return enableLongComponentsCompatibility;
}
public static int getMaxComponentLength() {
return maxComponentLength;
}
public static CheckUnnormalizedMode getCheckUnnormalizedMode() {
return checkUnnormalized;
}
/**
* Return list of known repository names. Needs a registration mechanism if ever another
* repository implementation.
*/
public List<String> getRepositoryList() {
return repoList;
}
public PlatformUtil.DF getRepositoryDF(String repoName) {
String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
log.debug("path: " + path);
// try {
return platInfo.getJavaDF(path);
// } catch (PlatformUtil.UnsupportedException e) {
// return null;
// }
}
public Map<String, PlatformUtil.DF> getRepositoryMap() {
Map<String, PlatformUtil.DF> repoMap = new LinkedMap();
for (String repo : getRepositoryList()) {
repoMap.put(repo, getRepositoryDF(repo));
}
return repoMap;
}
public String findLeastFullRepository() {
return findLeastFullRepository(getRepositoryMap());
}
public String findLeastFullRepository(Map<String, PlatformUtil.DF> repoMap) {
String mostFree = null;
for (String repo : repoMap.keySet()) {
PlatformUtil.DF df = repoMap.get(repo);
if (df != null) {
if (mostFree == null || (repoMap.get(mostFree)).getAvail() < df.getAvail()) {
mostFree = repo;
}
}
}
return mostFree;
}
public PlatformUtil.DF getDiskWarnThreshold() {
return paramDFWarn;
}
public PlatformUtil.DF getDiskFullThreshold() {
return paramDFFull;
}
public static int getMaxUnusedDirSearch() {
return maxUnusedDirSearch;
}
public static boolean isStatefulUnusedDirSearch() {
return isStatefulUnusedDirSearch;
}
public List findExistingRepositoriesFor(String auid) {
List res = null;
for (Iterator iter = getRepositoryList().iterator(); iter.hasNext(); ) {
String repoName = (String) iter.next();
String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
if (LockssRepositoryImpl.doesAuDirExist(auid, path)) {
if (res == null) {
res = new ArrayList();
}
res.add(repoName);
}
}
return res == null ? Collections.EMPTY_LIST : res;
}
// hack only local
public synchronized LockssRepositoryImpl getRepositoryFromPath(String path) {
LockssRepositoryImpl repo = (LockssRepositoryImpl) localRepos.get(path);
if (repo == null) {
repo = new LockssRepositoryImpl(path);
repo.initService(getDaemon());
repo.startService();
localRepos.put(path, repo);
}
return repo;
}
/**
* Return the disk space used by the AU, including all overhead, optionally calculating it if
* necessary.
*
* @param repoAuPath the full path to an AU dir in a LockssRepositoryImpl
* @param calcIfUnknown if true, size will calculated if unknown (time consumeing)
* @return the AU's disk usage in bytes, or -1 if unknown
*/
public long getRepoDiskUsage(String repoAuPath, boolean calcIfUnknown) {
LockssRepository repo = getRepositoryFromPath(repoAuPath);
if (repo != null) {
try {
RepositoryNode repoNode = repo.getNode(AuCachedUrlSetSpec.URL);
if (repoNode instanceof AuNodeImpl) {
return ((AuNodeImpl) repoNode).getDiskUsage(calcIfUnknown);
}
} catch (MalformedURLException ignore) {
}
}
return -1;
}
public synchronized void setRepositoryForPath(String path, LockssRepositoryImpl repo) {
localRepos.put(path, repo);
}
public boolean isGlobalNodeCache() {
return paramIsGlobalNodeCache;
}
public UniqueRefLruCache getGlobalNodeCache() {
return globalNodeCache;
}
public UniqueRefLruCache getSuspectVersionsCache() {
return suspectVersionsCache;
}
// Background thread to (re)calculate AU size and disk usage.
private Set sizeCalcQueue = new HashSet();
private BinarySemaphore sizeCalcSem = new BinarySemaphore();
private SizeCalcThread sizeCalcThread;
/** engqueue a size calculation for the AU */
public void queueSizeCalc(ArchivalUnit au) {
queueSizeCalc(AuUtil.getAuRepoNode(au));
}
/** engqueue a size calculation for the node */
public void queueSizeCalc(RepositoryNode node) {
synchronized (sizeCalcQueue) {
if (sizeCalcQueue.add(node)) {
log.debug2("Queue size calc: " + node);
startOrKickThread();
}
}
}
public int sizeCalcQueueLen() {
synchronized (sizeCalcQueue) {
return sizeCalcQueue.size();
}
}
void startOrKickThread() {
if (sizeCalcThread == null) {
log.debug2("Starting thread");
sizeCalcThread = new SizeCalcThread();
sizeCalcThread.start();
sizeCalcThread.waitRunning();
}
sizeCalcSem.give();
}
void stopThread() {
if (sizeCalcThread != null) {
log.debug2("Stopping thread");
sizeCalcThread.stopSizeCalc();
sizeCalcThread = null;
}
}
void doSizeCalc(RepositoryNode node) {
node.getTreeContentSize(null, true);
if (node instanceof AuNodeImpl) {
((AuNodeImpl) node).getDiskUsage(true);
}
}
long sleepTimeToAchieveLoad(long runDuration, float maxLoad) {
return Math.round(((double) runDuration / maxLoad) - runDuration);
}
private class SizeCalcThread extends LockssThread {
private volatile boolean goOn = true;
private SizeCalcThread() {
super("SizeCalc");
}
public void lockssRun() {
setPriority(PRIORITY_PARAM_SIZE_CALC, PRIORITY_DEFAULT_SIZE_CALC);
startWDog(WDOG_PARAM_SIZE_CALC, WDOG_DEFAULT_SIZE_CALC);
triggerWDogOnExit(true);
nowRunning();
while (goOn) {
try {
pokeWDog();
if (sizeCalcQueue.isEmpty()) {
Deadline timeout = Deadline.in(Constants.HOUR);
sizeCalcSem.take(timeout);
}
RepositoryNode node;
synchronized (sizeCalcQueue) {
node = (RepositoryNode) CollectionUtil.getAnElement(sizeCalcQueue);
}
if (node != null) {
long start = TimeBase.nowMs();
log.debug2("CalcSize start: " + node);
long dur = 0;
try {
doSizeCalc(node);
dur = TimeBase.nowMs() - start;
log.debug2("CalcSize finish (" + StringUtil.timeIntervalToString(dur) + "): " + node);
} catch (RuntimeException e) {
log.warning("doSizeCalc: " + node, e);
}
synchronized (sizeCalcQueue) {
sizeCalcQueue.remove(node);
}
pokeWDog();
long sleep = sleepTimeToAchieveLoad(dur, sizeCalcMaxLoad);
Deadline.in(sleep).sleep();
}
} catch (InterruptedException e) {
// just wakeup and check for exit
}
}
if (!goOn) {
triggerWDogOnExit(false);
}
}
private void stopSizeCalc() {
goOn = false;
interrupt();
}
}
}
/** UI to invoke various daemon actions */
@SuppressWarnings("serial")
public class DebugPanel extends LockssServlet {
public static final String PREFIX = Configuration.PREFIX + "debugPanel.";
/** Priority for crawls started from the debug panel */
public static final String PARAM_CRAWL_PRIORITY = PREFIX + "crawlPriority";
public static final int DEFAULT_CRAWL_PRIORITY = 10;
/** Priority for crawls started from the debug panel */
public static final String PARAM_ENABLE_DEEP_CRAWL = PREFIX + "deepCrawlEnabled";
private static final boolean DEFAULT_ENABLE_DEEP_CRAWL = false;
static final String KEY_ACTION = "action";
static final String KEY_MSG = "msg";
static final String KEY_NAME_SEL = "name_sel";
static final String KEY_NAME_TYPE = "name_type";
static final String KEY_AUID = "auid";
static final String KEY_URL = "url";
static final String KEY_REFETCH_DEPTH = "depth";
static final String KEY_TIME = "time";
static final String ACTION_MAIL_BACKUP = "Mail Backup File";
static final String ACTION_THROW_IOEXCEPTION = "Throw IOException";
static final String ACTION_FIND_URL = "Find Preserved URL";
public static final String ACTION_REINDEX_METADATA = "Reindex Metadata";
public static final String ACTION_FORCE_REINDEX_METADATA = "Force Reindex Metadata";
public static final String ACTION_START_V3_POLL = "Start V3 Poll";
static final String ACTION_FORCE_START_V3_POLL = "Force V3 Poll";
public static final String ACTION_START_CRAWL = "Start Crawl";
public static final String ACTION_FORCE_START_CRAWL = "Force Start Crawl";
public static final String ACTION_START_DEEP_CRAWL = "Deep Crawl";
public static final String ACTION_FORCE_START_DEEP_CRAWL = "Force Deep Crawl";
public static final String ACTION_CHECK_SUBSTANCE = "Check Substance";
static final String ACTION_CRAWL_PLUGINS = "Crawl Plugins";
static final String ACTION_RELOAD_CONFIG = "Reload Config";
static final String ACTION_SLEEP = "Sleep";
public static final String ACTION_DISABLE_METADATA_INDEXING = "Disable Indexing";
/** Set of actions for which audit alerts shouldn't be generated */
public static final Set noAuditActions = SetUtil.set(ACTION_FIND_URL);
static final String COL2 = "colspan=2";
static final String COL2CENTER = COL2 + " align=center";
static Logger log = Logger.getLogger("DebugPanel");
private LockssDaemon daemon;
private PluginManager pluginMgr;
private PollManager pollManager;
private CrawlManager crawlMgr;
private ConfigManager cfgMgr;
private DbManager dbMgr;
private MetadataManager metadataMgr;
private RemoteApi rmtApi;
boolean showResult;
boolean showForcePoll;
boolean showForceCrawl;
boolean showForceReindexMetadata;
String formAuid;
String formDepth = "100";
protected void resetLocals() {
resetVars();
super.resetLocals();
}
void resetVars() {
formAuid = null;
errMsg = null;
statusMsg = null;
showForcePoll = false;
showForceCrawl = false;
showForceReindexMetadata = false;
}
public void init(ServletConfig config) throws ServletException {
super.init(config);
daemon = getLockssDaemon();
pluginMgr = daemon.getPluginManager();
pollManager = daemon.getPollManager();
crawlMgr = daemon.getCrawlManager();
cfgMgr = daemon.getConfigManager();
rmtApi = daemon.getRemoteApi();
try {
dbMgr = daemon.getDbManager();
metadataMgr = daemon.getMetadataManager();
} catch (IllegalArgumentException ex) {
}
}
public void lockssHandleRequest() throws IOException {
resetVars();
boolean showForm = true;
String action = getParameter(KEY_ACTION);
if (!StringUtil.isNullString(action)) {
formAuid = getParameter(KEY_AUID);
formDepth = getParameter(KEY_REFETCH_DEPTH);
UserAccount acct = getUserAccount();
if (acct != null && !noAuditActions.contains(action)) {
acct.auditableEvent("used debug panel action: " + action + " AU ID: " + formAuid);
}
}
if (ACTION_MAIL_BACKUP.equals(action)) {
doMailBackup();
}
if (ACTION_RELOAD_CONFIG.equals(action)) {
doReloadConfig();
}
if (ACTION_SLEEP.equals(action)) {
doSleep();
}
if (ACTION_THROW_IOEXCEPTION.equals(action)) {
doThrow();
}
if (ACTION_START_V3_POLL.equals(action)) {
doV3Poll();
}
if (ACTION_FORCE_START_V3_POLL.equals(action)) {
forceV3Poll();
}
if (ACTION_START_CRAWL.equals(action)) {
doCrawl(false, false);
}
if (ACTION_FORCE_START_CRAWL.equals(action)) {
doCrawl(true, false);
}
if (ACTION_START_DEEP_CRAWL.equals(action)) {
doCrawl(false, true);
}
if (ACTION_FORCE_START_DEEP_CRAWL.equals(action)) {
doCrawl(true, true);
}
if (ACTION_CHECK_SUBSTANCE.equals(action)) {
doCheckSubstance();
}
if (ACTION_CRAWL_PLUGINS.equals(action)) {
crawlPluginRegistries();
}
if (ACTION_FIND_URL.equals(action)) {
showForm = doFindUrl();
}
if (ACTION_REINDEX_METADATA.equals(action)) {
doReindexMetadata();
}
if (ACTION_FORCE_REINDEX_METADATA.equals(action)) {
forceReindexMetadata();
}
if (ACTION_DISABLE_METADATA_INDEXING.equals(action)) {
doDisableMetadataIndexing();
}
if (showForm) {
displayPage();
}
}
private void doMailBackup() {
try {
rmtApi.createConfigBackupFile(RemoteApi.BackupFileDisposition.Mail);
} catch (Exception e) {
errMsg = "Error: " + e.getMessage();
}
}
private void doReloadConfig() {
cfgMgr.requestReload();
}
private void doThrow() throws IOException {
String msg = getParameter(KEY_MSG);
throw new IOException(msg != null ? msg : "Test message");
}
private void doSleep() throws IOException {
String timestr = getParameter(KEY_TIME);
try {
long time = StringUtil.parseTimeInterval(timestr);
Deadline.in(time).sleep();
statusMsg = "Slept for " + StringUtil.timeIntervalToString(time);
} catch (NumberFormatException e) {
errMsg = "Illegal duration: " + e;
} catch (InterruptedException e) {
errMsg = "Interrupted: " + e;
}
}
private void doReindexMetadata() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
startReindexingMetadata(au, false);
} catch (RuntimeException e) {
log.error("Can't reindex metadata", e);
errMsg = "Error: " + e.toString();
}
}
private void forceReindexMetadata() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
startReindexingMetadata(au, true);
} catch (RuntimeException e) {
log.error("Can't reindex metadata", e);
errMsg = "Error: " + e.toString();
}
}
private void doDisableMetadataIndexing() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
disableMetadataIndexing(au, false);
} catch (RuntimeException e) {
log.error("Can't disable metadata indexing", e);
errMsg = "Error: " + e.toString();
}
}
private void doCrawl(boolean force, boolean deep) {
ArchivalUnit au = getAu();
if (au == null) return;
try {
startCrawl(au, force, deep);
} catch (CrawlManagerImpl.NotEligibleException.RateLimiter e) {
errMsg = "AU has crawled recently (" + e.getMessage() + "). Click again to override.";
showForceCrawl = true;
return;
} catch (CrawlManagerImpl.NotEligibleException e) {
errMsg = "Can't enqueue crawl: " + e.getMessage();
}
}
private void crawlPluginRegistries() {
StringBuilder sb = new StringBuilder();
for (ArchivalUnit au : pluginMgr.getAllRegistryAus()) {
sb.append(au.getName());
sb.append(": ");
try {
startCrawl(au, true, false);
sb.append("Queued.");
} catch (CrawlManagerImpl.NotEligibleException e) {
sb.append("Failed: ");
sb.append(e.getMessage());
}
sb.append("\n");
}
statusMsg = sb.toString();
}
private boolean startCrawl(ArchivalUnit au, boolean force, boolean deep)
throws CrawlManagerImpl.NotEligibleException {
CrawlManagerImpl cmi = (CrawlManagerImpl) crawlMgr;
if (force) {
RateLimiter limit = cmi.getNewContentRateLimiter(au);
if (!limit.isEventOk()) {
limit.unevent();
}
}
cmi.checkEligibleToQueueNewContentCrawl(au);
String delayMsg = "";
String deepMsg = "";
try {
cmi.checkEligibleForNewContentCrawl(au);
} catch (CrawlManagerImpl.NotEligibleException e) {
delayMsg = ", Start delayed due to: " + e.getMessage();
}
Configuration config = ConfigManager.getCurrentConfig();
int pri = config.getInt(PARAM_CRAWL_PRIORITY, DEFAULT_CRAWL_PRIORITY);
CrawlReq req;
try {
req = new CrawlReq(au);
req.setPriority(pri);
if (deep) {
int d = Integer.parseInt(formDepth);
if (d < 0) {
errMsg = "Illegal refetch depth: " + d;
return false;
}
req.setRefetchDepth(d);
deepMsg = "Deep (" + req.getRefetchDepth() + ") ";
}
} catch (NumberFormatException e) {
errMsg = "Illegal refetch depth: " + formDepth;
return false;
} catch (RuntimeException e) {
log.error("Couldn't create CrawlReq: " + au, e);
errMsg = "Couldn't create CrawlReq: " + e.toString();
return false;
}
cmi.startNewContentCrawl(req, null);
statusMsg = deepMsg + "Crawl requested for " + au.getName() + delayMsg;
return true;
}
private void doCheckSubstance() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
checkSubstance(au);
} catch (RuntimeException e) {
log.error("Error in SubstanceChecker", e);
errMsg = "Error in SubstanceChecker; see log.";
}
}
private void checkSubstance(ArchivalUnit au) {
SubstanceChecker subChecker = new SubstanceChecker(au);
if (!subChecker.isEnabled()) {
errMsg = "No substance patterns defined for plugin.";
return;
}
AuState auState = AuUtil.getAuState(au);
SubstanceChecker.State oldState = auState.getSubstanceState();
SubstanceChecker.State newState = subChecker.findSubstance();
String chtxt = (newState == oldState ? "(unchanged)" : "(was " + oldState.toString() + ")");
switch (newState) {
case Unknown:
log.error("Shouldn't happen: SubstanceChecker returned Unknown");
errMsg = "Error in SubstanceChecker; see log.";
break;
case Yes:
statusMsg = "AU has substance " + chtxt + ": " + au.getName();
auState.setSubstanceState(SubstanceChecker.State.Yes);
break;
case No:
statusMsg = "AU has no substance " + chtxt + ": " + au.getName();
auState.setSubstanceState(SubstanceChecker.State.No);
break;
}
}
private boolean startReindexingMetadata(ArchivalUnit au, boolean force) {
if (metadataMgr == null) {
errMsg = "Metadata processing is not enabled.";
return false;
}
if (!force) {
if (!AuUtil.hasCrawled(au)) {
errMsg = "Au has never crawled. Click again to reindex metadata";
showForceReindexMetadata = true;
return false;
}
AuState auState = AuUtil.getAuState(au);
switch (auState.getSubstanceState()) {
case No:
errMsg = "Au has no substance. Click again to reindex metadata";
showForceReindexMetadata = true;
return false;
case Unknown:
errMsg = "Unknown substance for Au. Click again to reindex metadata.";
showForceReindexMetadata = true;
return false;
case Yes:
// fall through
}
}
// Fully reindex metadata with the highest priority.
Connection conn = null;
PreparedStatement insertPendingAuBatchStatement = null;
try {
conn = dbMgr.getConnection();
insertPendingAuBatchStatement = metadataMgr.getPrioritizedInsertPendingAuBatchStatement(conn);
if (metadataMgr.enableAndAddAuToReindex(
au, conn, insertPendingAuBatchStatement, false, true)) {
statusMsg = "Reindexing metadata for " + au.getName();
return true;
}
} catch (DbException dbe) {
log.error("Cannot reindex metadata for " + au.getName(), dbe);
} finally {
DbManager.safeCloseStatement(insertPendingAuBatchStatement);
DbManager.safeRollbackAndClose(conn);
}
if (force) {
errMsg = "Still cannot reindex metadata for " + au.getName();
} else {
errMsg = "Cannot reindex metadata for " + au.getName();
}
return false;
}
private boolean disableMetadataIndexing(ArchivalUnit au, boolean force) {
if (metadataMgr == null) {
errMsg = "Metadata processing is not enabled.";
return false;
}
try {
metadataMgr.disableAuIndexing(au);
statusMsg = "Disabled metadata indexing for " + au.getName();
return true;
} catch (Exception e) {
errMsg = "Cannot reindex metadata for " + au.getName() + ": " + e.getMessage();
return false;
}
}
private void doV3Poll() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
callV3ContentPoll(au);
} catch (PollManager.NotEligibleException e) {
errMsg = "AU is not eligible for poll: " + e.getMessage();
// errMsg = "Ineligible: " + e.getMessage() +
// "<br>Click again to force new poll.";
// showForcePoll = true;
return;
} catch (Exception e) {
log.error("Can't start poll", e);
errMsg = "Error: " + e.toString();
}
}
private void forceV3Poll() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
callV3ContentPoll(au);
} catch (Exception e) {
log.error("Can't start poll", e);
errMsg = "Error: " + e.toString();
}
}
private void callV3ContentPoll(ArchivalUnit au) throws PollManager.NotEligibleException {
log.debug("Enqueuing a V3 Content Poll on " + au.getName());
PollSpec spec = new PollSpec(au.getAuCachedUrlSet(), Poll.V3_POLL);
pollManager.enqueueHighPriorityPoll(au, spec);
statusMsg = "Enqueued V3 poll for " + au.getName();
}
private boolean doFindUrl() throws IOException {
String url = getParameter(KEY_URL);
String redir =
srvURL(
AdminServletManager.SERVLET_DAEMON_STATUS,
PropUtil.fromArgs("table", ArchivalUnitStatus.AUS_WITH_URL_TABLE_NAME, "key", url));
resp.setContentLength(0);
// resp.sendRedirect(resp.encodeRedirectURL(redir));
resp.sendRedirect(redir);
return false;
}
ArchivalUnit getAu() {
if (StringUtil.isNullString(formAuid)) {
errMsg = "Select an AU";
return null;
}
ArchivalUnit au = pluginMgr.getAuFromId(formAuid);
if (au == null) {
errMsg = "No such AU. Select an AU";
return null;
}
return au;
}
private void displayPage() throws IOException {
Page page = newPage();
layoutErrorBlock(page);
ServletUtil.layoutExplanationBlock(page, "Debug Actions");
page.add(makeForm());
page.add("<br>");
endPage(page);
}
private Element makeForm() {
Composite comp = new Composite();
Form frm = new Form(srvURL(myServletDescr()));
frm.method("POST");
frm.add("<br><center>");
Input reload = new Input(Input.Submit, KEY_ACTION, ACTION_RELOAD_CONFIG);
setTabOrder(reload);
frm.add(reload);
frm.add(" ");
Input backup = new Input(Input.Submit, KEY_ACTION, ACTION_MAIL_BACKUP);
setTabOrder(backup);
frm.add(backup);
frm.add(" ");
Input crawlplug = new Input(Input.Submit, KEY_ACTION, ACTION_CRAWL_PLUGINS);
setTabOrder(crawlplug);
frm.add(crawlplug);
frm.add("</center>");
ServletDescr d1 = AdminServletManager.SERVLET_HASH_CUS;
if (isServletRunnable(d1)) {
frm.add("<br><center>" + srvLink(d1, d1.heading) + "</center>");
}
Input findUrl = new Input(Input.Submit, KEY_ACTION, ACTION_FIND_URL);
Input findUrlText = new Input(Input.Text, KEY_URL);
findUrlText.setSize(50);
setTabOrder(findUrl);
setTabOrder(findUrlText);
frm.add("<br><center>" + findUrl + " " + findUrlText + "</center>");
Input thrw = new Input(Input.Submit, KEY_ACTION, ACTION_THROW_IOEXCEPTION);
Input thmsg = new Input(Input.Text, KEY_MSG);
setTabOrder(thrw);
setTabOrder(thmsg);
frm.add("<br><center>" + thrw + " " + thmsg + "</center>");
frm.add("<br><center>AU Actions: select AU</center>");
Composite ausel = ServletUtil.layoutSelectAu(this, KEY_AUID, formAuid);
frm.add("<br><center>" + ausel + "</center>");
setTabOrder(ausel);
Input v3Poll =
new Input(
Input.Submit,
KEY_ACTION,
(showForcePoll ? ACTION_FORCE_START_V3_POLL : ACTION_START_V3_POLL));
Input crawl =
new Input(
Input.Submit,
KEY_ACTION,
(showForceCrawl ? ACTION_FORCE_START_CRAWL : ACTION_START_CRAWL));
frm.add("<br><center>");
frm.add(v3Poll);
frm.add(" ");
frm.add(crawl);
if (CurrentConfig.getBooleanParam(PARAM_ENABLE_DEEP_CRAWL, DEFAULT_ENABLE_DEEP_CRAWL)) {
Input deepCrawl =
new Input(
Input.Submit,
KEY_ACTION,
(showForceCrawl ? ACTION_FORCE_START_DEEP_CRAWL : ACTION_START_DEEP_CRAWL));
Input depthText = new Input(Input.Text, KEY_REFETCH_DEPTH, formDepth);
depthText.setSize(4);
setTabOrder(depthText);
frm.add(" ");
frm.add(deepCrawl);
frm.add(depthText);
}
Input checkSubstance = new Input(Input.Submit, KEY_ACTION, ACTION_CHECK_SUBSTANCE);
frm.add("<br>");
frm.add(checkSubstance);
if (metadataMgr != null) {
Input reindex =
new Input(
Input.Submit,
KEY_ACTION,
(showForceReindexMetadata ? ACTION_FORCE_REINDEX_METADATA : ACTION_REINDEX_METADATA));
frm.add(" ");
frm.add(reindex);
Input disableIndexing = new Input(Input.Submit, KEY_ACTION, ACTION_DISABLE_METADATA_INDEXING);
frm.add(" ");
frm.add(disableIndexing);
}
frm.add("</center>");
comp.add(frm);
return comp;
}
}
/**
* Class implementing the concept of the set of URLs covered by a poll.
*
* @author Claire Griffin
* @version 1.0
*/
public class PollSpec {
/**
* A lower bound value which indicates the poll should use a {@link SingleNodeCachedUrlSetSpec}
* instead of a {@link RangeCachedUrlSetSpec}.
*/
public static final String SINGLE_NODE_LWRBOUND = ".";
public static final String DEFAULT_PLUGIN_VERSION = "1";
private static Logger theLog = Logger.getLogger("PollSpec");
private String auId;
private String pluginVersion;
private String url;
private String uprBound = null;
private String lwrBound = null;
private CachedUrlSet cus = null;
private PluginManager pluginMgr = null;
private int protocolVersion; // poll protocol version
private int pollType; // One of the types defined by Poll
private V3Poller.PollVariant variant = V3Poller.PollVariant.PoR;
/**
* Construct a PollSpec from a CachedUrlSet and an upper and lower bound
*
* @param cus the CachedUrlSet
* @param lwrBound the lower boundary
* @param uprBound the upper boundary
* @param pollType one of the types defined by Poll
*/
public PollSpec(CachedUrlSet cus, String lwrBound, String uprBound, int pollType) {
commonSetup(cus, lwrBound, uprBound, pollType);
}
/**
* Construct a PollSpec from a CachedUrlSet.
*
* @param cus the CachedUrlSpec which defines the range of interest
* @param pollType one of the types defined by Poll
*/
public PollSpec(CachedUrlSet cus, int pollType) {
CachedUrlSetSpec cuss = cus.getSpec();
if (cuss instanceof RangeCachedUrlSetSpec) {
RangeCachedUrlSetSpec rcuss = (RangeCachedUrlSetSpec) cuss;
commonSetup(cus, rcuss.getLowerBound(), rcuss.getUpperBound(), pollType);
} else if (cuss.isSingleNode()) {
commonSetup(cus, SINGLE_NODE_LWRBOUND, null, pollType);
} else {
commonSetup(cus, null, null, pollType);
}
}
/**
* Construct a PollSpec from a V1 LcapMessage
*
* @param msg the LcapMessage which defines the range of interest
*/
public PollSpec(V1LcapMessage msg) {
auId = msg.getArchivalId();
pluginVersion = msg.getPluginVersion();
url = msg.getTargetUrl();
uprBound = msg.getUprBound();
lwrBound = msg.getLwrBound();
protocolVersion = msg.getProtocolVersion();
if (msg.isContentPoll()) {
pollType = Poll.V1_CONTENT_POLL;
} else if (msg.isNamePoll()) {
pollType = Poll.V1_NAME_POLL;
} else if (msg.isVerifyPoll()) {
pollType = Poll.V1_VERIFY_POLL;
} else {
pollType = -1;
}
cus = getPluginManager().findCachedUrlSet(this);
}
public PollSpec(V3LcapMessage msg) {
this(
msg.getArchivalId(),
(msg.getTargetUrl() == null) ? "lockssau:" : msg.getTargetUrl(),
null,
null,
Poll.V3_POLL);
protocolVersion = msg.getProtocolVersion();
pluginVersion = msg.getPluginVersion();
}
/** Construct a PollSpec from explicit args */
public PollSpec(String auId, String url, String lower, String upper, int pollType) {
this.auId = auId;
this.url = url;
uprBound = upper;
lwrBound = lower;
this.pollType = pollType;
cus = getPluginManager().findCachedUrlSet(this);
this.protocolVersion = protocolVersionFromPollType(pollType);
}
/**
* Construct a PollSpec from another PollSpec and a poll type XXX it seems that other constructors
* are not setting all fields
*/
public PollSpec(PollSpec ps, int pollType) {
this.auId = ps.auId;
this.pluginVersion = ps.pluginVersion;
this.url = ps.url;
this.uprBound = ps.uprBound;
this.lwrBound = ps.lwrBound;
this.cus = ps.cus;
this.pluginMgr = ps.pluginMgr;
this.protocolVersion = ps.protocolVersion;
this.pollType = pollType;
}
/** Setup common to most constructors */
private void commonSetup(CachedUrlSet cus, String lwrBound, String uprBound, int pollType) {
CachedUrlSetSpec cuss = cus.getSpec();
if (cuss instanceof PrunedCachedUrlSetSpec) {
throw new IllegalArgumentException("Polls do not support PrunedCachedUrlSetSpec");
}
this.cus = cus;
ArchivalUnit au = cus.getArchivalUnit();
auId = au.getAuId();
this.pluginVersion = AuUtil.getPollVersion(au);
url = cuss.getUrl();
this.lwrBound = lwrBound;
this.uprBound = uprBound;
this.protocolVersion = protocolVersionFromPollType(pollType);
this.pollType = pollType;
}
public CachedUrlSet getCachedUrlSet() {
return cus;
}
public String getAuId() {
return auId;
}
public String getPluginVersion() {
return (pluginVersion != null) ? pluginVersion : DEFAULT_PLUGIN_VERSION;
}
public String getUrl() {
return url;
}
public String getLwrBound() {
return lwrBound;
}
public String getUprBound() {
return uprBound;
}
public String getRangeString() {
if (StringUtil.equalStrings(lwrBound, SINGLE_NODE_LWRBOUND)) {
return "single node";
}
if (lwrBound != null || uprBound != null) {
String lwrDisplay = lwrBound;
String uprDisplay = uprBound;
if (lwrBound != null && lwrBound.startsWith("/")) {
lwrDisplay = lwrBound.substring(1);
}
if (uprBound != null && uprBound.startsWith("/")) {
uprDisplay = uprBound.substring(1);
}
return lwrDisplay + " - " + uprDisplay;
}
return null;
}
public int getProtocolVersion() {
return protocolVersion;
}
public int getPollType() {
return pollType;
}
public V3Poller.PollVariant getPollVariant() {
return variant;
}
public void setPollVariant(V3Poller.PollVariant v) {
variant = v;
}
private PluginManager getPluginManager() {
if (pluginMgr == null) {
pluginMgr = (PluginManager) LockssDaemon.getManager(LockssDaemon.PLUGIN_MANAGER);
}
return pluginMgr;
}
/**
* Given a poll type, return the correct version of the protocol to use.
*
* @param pollType
* @return The protocol version to use
*/
private int protocolVersionFromPollType(int pollType) {
switch (pollType) {
case Poll.V1_CONTENT_POLL:
case Poll.V1_NAME_POLL:
case Poll.V1_VERIFY_POLL:
return Poll.V1_PROTOCOL;
case Poll.V3_POLL:
return Poll.V3_PROTOCOL;
default:
return Poll.UNDEFINED_PROTOCOL;
}
}
public String toString() {
return "[PS: "
+ Poll.POLL_NAME[pollType]
+ " auid="
+ auId
+ ", url="
+ url
+ ", l="
+ lwrBound
+ ", u="
+ uprBound
+ ", type="
+ pollType
+ ", plugVer="
+ getPluginVersion()
+ ", protocol="
+ protocolVersion
+ "]";
}
}
/**
* DefinablePlugin: a plugin which uses the data stored in an ExternalizableMap to configure itself.
*
* @author Claire Griffin
* @version 1.0
*/
public class DefinablePlugin extends BasePlugin {
// configuration map keys
public static final String KEY_PLUGIN_IDENTIFIER = "plugin_identifier";
public static final String KEY_PLUGIN_NAME = "plugin_name";
public static final String KEY_PLUGIN_VERSION = "plugin_version";
public static final String KEY_PLUGIN_FEATURE_VERSION_MAP = "plugin_feature_version_map";
public static final String KEY_REQUIRED_DAEMON_VERSION = "required_daemon_version";
public static final String KEY_PUBLISHING_PLATFORM = "plugin_publishing_platform";
public static final String KEY_PLUGIN_CONFIG_PROPS = "plugin_config_props";
public static final String KEY_EXCEPTION_HANDLER = "plugin_cache_result_handler";
public static final String KEY_EXCEPTION_LIST = "plugin_cache_result_list";
public static final String KEY_PLUGIN_NOTES = "plugin_notes";
public static final String KEY_CRAWL_TYPE = "plugin_crawl_type";
public static final String KEY_FOLLOW_LINKS = "plugin_follow_link";
/** Message to be displayed when user configures an AU with this plugin */
public static final String KEY_PLUGIN_AU_CONFIG_USER_MSG = "plugin_au_config_user_msg";
public static final String KEY_PER_HOST_PERMISSION_PATH = "plugin_per_host_permission_path";
public static final String KEY_PLUGIN_PARENT = "plugin_parent";
public static final String KEY_PLUGIN_PARENT_VERSION = "plugin_parent_version";
public static final String KEY_PLUGIN_CRAWL_URL_COMPARATOR_FACTORY =
"plugin_crawl_url_comparator_factory";
public static final String KEY_PLUGIN_FETCH_RATE_LIMITER_SOURCE =
"plugin_fetch_rate_limiter_source";
public static final String KEY_PLUGIN_ARTICLE_ITERATOR_FACTORY =
"plugin_article_iterator_factory";
public static final String KEY_PLUGIN_ARTICLE_METADATA_EXTRACTOR_FACTORY =
"plugin_article_metadata_extractor_factory";
public static final String KEY_DEFAULT_ARTICLE_MIME_TYPE = "plugin_default_article_mime_type";
public static final String KEY_ARTICLE_ITERATOR_ROOT = "plugin_article_iterator_root";
public static final String KEY_ARTICLE_ITERATOR_PATTERN = "plugin_article_iterator_pattern";
public static final String DEFAULT_PLUGIN_VERSION = "1";
public static final String DEFAULT_REQUIRED_DAEMON_VERSION = "0.0.0";
public static final String MAP_SUFFIX = ".xml";
public static final String CRAWL_TYPE_HTML_LINKS = "HTML Links";
public static final String CRAWL_TYPE_OAI = "OAI";
public static final String[] CRAWL_TYPES = {
CRAWL_TYPE_HTML_LINKS, CRAWL_TYPE_OAI,
};
public static final String DEFAULT_CRAWL_TYPE = CRAWL_TYPE_HTML_LINKS;
protected String mapName = null;
static Logger log = Logger.getLogger("DefinablePlugin");
protected ExternalizableMap definitionMap = new ExternalizableMap();
protected CacheResultHandler resultHandler = null;
protected List<String> loadedFromUrls;
protected CrawlWindow crawlWindow;
protected Map<Plugin.Feature, String> featureVersion;
public void initPlugin(LockssDaemon daemon, String extMapName) throws FileNotFoundException {
initPlugin(daemon, extMapName, this.getClass().getClassLoader());
}
public void initPlugin(LockssDaemon daemon, String extMapName, ClassLoader loader)
throws FileNotFoundException {
// convert the plugin class name to an xml file name
// load the configuration map from jar file
ExternalizableMap defMap = loadMap(extMapName, loader);
this.initPlugin(daemon, extMapName, defMap, loader);
}
public void initPlugin(
LockssDaemon daemon, String extMapName, ExternalizableMap defMap, ClassLoader loader) {
mapName = extMapName;
this.classLoader = loader;
this.definitionMap = defMap;
super.initPlugin(daemon);
initMimeMap();
initFeatureVersions();
initAuFeatureMap();
checkParamAgreement();
}
private ExternalizableMap loadMap(String extMapName, ClassLoader loader)
throws FileNotFoundException {
String first = null;
String next = extMapName;
List<String> urls = new ArrayList<String>();
ExternalizableMap res = null;
while (next != null) {
// convert the plugin class name to an xml file name
String mapFile = next.replace('.', '/') + MAP_SUFFIX;
URL url = loader.getResource(mapFile);
if (url != null && urls.contains(url.toString())) {
throw new PluginException.InvalidDefinition("Plugin inheritance loop: " + next);
}
// load into map
ExternalizableMap oneMap = new ExternalizableMap();
oneMap.loadMapFromResource(mapFile, loader);
urls.add(url.toString());
// apply overrides one plugin at a time in inheritance chain
processOverrides(oneMap);
if (res == null) {
res = oneMap;
} else {
for (Map.Entry ent : oneMap.entrySet()) {
String key = (String) ent.getKey();
Object val = ent.getValue();
if (!res.containsKey(key)) {
res.setMapElement(key, val);
}
}
}
if (oneMap.containsKey(KEY_PLUGIN_PARENT)) {
next = oneMap.getString(KEY_PLUGIN_PARENT);
} else {
next = null;
}
}
loadedFromUrls = urls;
return res;
}
/** If in testing mode FOO, copy values from FOO_override map, if any, to main map */
void processOverrides(TypedEntryMap map) {
String testMode = getTestingMode();
if (StringUtil.isNullString(testMode)) {
return;
}
Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE);
if (o == null) {
return;
}
if (o instanceof Map) {
Map overrideMap = (Map) o;
for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) {
String key = (String) entry.getKey();
Object val = entry.getValue();
log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val);
map.setMapElement(key, val);
}
}
}
String getTestingMode() {
return theDaemon == null ? null : theDaemon.getTestingMode();
}
// Used by tests
public void initPlugin(LockssDaemon daemon, File file) throws PluginException {
ExternalizableMap oneMap = new ExternalizableMap();
oneMap.loadMap(file);
if (oneMap.getErrorString() != null) {
throw new PluginException(oneMap.getErrorString());
}
initPlugin(daemon, file.getPath(), oneMap, null);
}
void initPlugin(LockssDaemon daemon, ExternalizableMap defMap) {
initPlugin(daemon, defMap, this.getClass().getClassLoader());
}
void initPlugin(LockssDaemon daemon, ExternalizableMap defMap, ClassLoader loader) {
initPlugin(daemon, "Internal", defMap, loader);
}
enum PrintfContext {
Regexp,
URL,
Display
};
void checkParamAgreement() {
for (Map.Entry<String, PrintfContext> ent :
DefinableArchivalUnit.printfKeysContext.entrySet()) {
checkParamAgreement(ent.getKey(), ent.getValue());
}
}
void checkParamAgreement(String key, PrintfContext context) {
List<String> printfList = getElementList(key);
if (printfList == null) {
return;
}
for (String printf : printfList) {
if (StringUtil.isNullString(printf)) {
log.warning("Null printf string in " + key);
continue;
}
PrintfUtil.PrintfData p_data = PrintfUtil.stringToPrintf(printf);
Collection<String> p_args = p_data.getArguments();
for (String arg : p_args) {
ConfigParamDescr descr = findAuConfigDescr(arg);
if (descr == null) {
throw new PluginException.InvalidDefinition(
"Not a declared parameter: " + arg + " in " + printf + " in " + getPluginName());
}
// ensure range and set params used only in legal context
switch (context) {
case Regexp:
case Display:
// everything is legal in a regexp or a display string
break;
case URL:
// NUM_RANGE and SET legal because can enumerate. Can't
// enumerate RANGE
switch (descr.getType()) {
case ConfigParamDescr.TYPE_RANGE:
throw new PluginException.InvalidDefinition(
"Range parameter ("
+ arg
+ ") used in illegal context in "
+ getPluginName()
+ ": "
+ key
+ ": "
+ printf);
default:
}
}
}
}
}
public List<String> getLoadedFromUrls() {
return loadedFromUrls;
}
public String getPluginName() {
if (definitionMap.containsKey(KEY_PLUGIN_NAME)) {
return definitionMap.getString(KEY_PLUGIN_NAME);
} else {
return getDefaultPluginName();
}
}
protected String getDefaultPluginName() {
return StringUtil.shortName(getPluginId());
}
public String getVersion() {
return definitionMap.getString(KEY_PLUGIN_VERSION, DEFAULT_PLUGIN_VERSION);
}
public String getFeatureVersion(Plugin.Feature feat) {
if (featureVersion == null) {
return null;
}
return featureVersion.get(feat);
}
public String getRequiredDaemonVersion() {
return definitionMap.getString(KEY_REQUIRED_DAEMON_VERSION, DEFAULT_REQUIRED_DAEMON_VERSION);
}
public String getPublishingPlatform() {
return definitionMap.getString(KEY_PUBLISHING_PLATFORM, null);
}
public String getPluginNotes() {
return definitionMap.getString(KEY_PLUGIN_NOTES, null);
}
public String getDefaultArticleMimeType() {
String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null);
log.debug3("DefaultArticleMimeType " + ret);
if (ret == null) {
ret = super.getDefaultArticleMimeType();
log.debug3("DefaultArticleMimeType from super " + ret);
}
return ret;
}
public List<String> getElementList(String key) {
Object element = definitionMap.getMapElement(key);
List<String> lst;
if (element instanceof String) {
return Collections.singletonList((String) element);
} else if (element instanceof List) {
return (List) element;
} else {
return null;
}
}
public List getLocalAuConfigDescrs() throws PluginException.InvalidDefinition {
List auConfigDescrs = (List) definitionMap.getCollection(KEY_PLUGIN_CONFIG_PROPS, null);
if (auConfigDescrs == null) {
throw new PluginException.InvalidDefinition(mapName + " missing ConfigParamDescrs");
}
return auConfigDescrs;
}
protected ArchivalUnit createAu0(Configuration auConfig)
throws ArchivalUnit.ConfigurationException {
DefinableArchivalUnit au = new DefinableArchivalUnit(this, definitionMap);
au.setConfiguration(auConfig);
return au;
}
public ExternalizableMap getDefinitionMap() {
return definitionMap;
}
CacheResultHandler getCacheResultHandler() {
return resultHandler;
}
String stripSuffix(String str, String suffix) {
return str.substring(0, str.length() - suffix.length());
}
protected void initMimeMap() throws PluginException.InvalidDefinition {
for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry ent = (Map.Entry) iter.next();
String key = (String) ent.getKey();
Object val = ent.getValue();
if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " link extractor: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
LinkExtractorFactory fact =
(LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class);
mti.setLinkExtractorFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) {
// XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY
// XXX unless/until that key is changed to not be a terminal substring
// XXX of this one
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " crawl filter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
mti.setCrawlFilterFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " filter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
mti.setHashFilterFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT);
if (val instanceof String) {
String rate = (String) val;
log.debug(mime + " fetch rate: " + rate);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
RateLimiter limit = mti.getFetchRateLimiter();
if (limit != null) {
limit.setRate(rate);
} else {
mti.setFetchRateLimiter(new RateLimiter(rate));
}
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY);
String factName = (String) val;
log.debug(mime + " link rewriter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
LinkRewriterFactory fact =
(LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class);
mti.setLinkRewriterFactory(fact);
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP);
Map factNameMap = (Map) val;
Map factClassMap = new HashMap();
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) {
String mdTypes = (String) it.next();
String factName = (String) factNameMap.get(mdTypes);
log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName);
for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) {
setMdTypeFact(factClassMap, mdType, factName);
}
}
mti.setFileMetadataExtractorFactoryMap(factClassMap);
}
}
}
private void setMdTypeFact(Map factClassMap, String mdType, String factName) {
log.debug3("Metadata type: " + mdType + " factory " + factName);
FileMetadataExtractorFactory fact =
(FileMetadataExtractorFactory) newAuxClass(factName, FileMetadataExtractorFactory.class);
factClassMap.put(mdType, fact);
}
protected void initResultMap() throws PluginException.InvalidDefinition {
HttpResultMap hResultMap = new HttpResultMap();
// XXX Currently this only allows a CacheResultHandler class to
// initialize the result map. Instead, don't use a CacheResultMap
// directly, use either the plugin's CacheResultHandler, if specified,
// or a default one that wraps the CacheResultMap
String handler_class = null;
handler_class = definitionMap.getString(KEY_EXCEPTION_HANDLER, null);
if (handler_class != null) {
try {
resultHandler = (CacheResultHandler) newAuxClass(handler_class, CacheResultHandler.class);
resultHandler.init(hResultMap);
} catch (Exception ex) {
throw new PluginException.InvalidDefinition(
mapName + " has invalid Exception handler: " + handler_class, ex);
} catch (LinkageError le) {
throw new PluginException.InvalidDefinition(
mapName + " has invalid Exception handler: " + handler_class, le);
}
} else {
// Expect a list of mappings from either result code or exception
// name to CacheException name
Collection<String> mappings = definitionMap.getCollection(KEY_EXCEPTION_LIST, null);
if (mappings != null) {
// add each entry
for (String entry : mappings) {
if (log.isDebug2()) {
log.debug2("initMap(" + entry + ")");
}
String first;
String ceName;
try {
List<String> pair = StringUtil.breakAt(entry, '=', 2, true, true);
first = pair.get(0);
ceName = pair.get(1);
} catch (Exception ex) {
throw new PluginException.InvalidDefinition(
"Invalid syntax: " + entry + "in " + mapName);
}
Object val;
// Value should be either a CacheException or CacheResultHandler
// class name.
PluginFetchEventResponse resp =
(PluginFetchEventResponse) newAuxClass(ceName, PluginFetchEventResponse.class, null);
if (resp instanceof CacheException) {
val = resp.getClass();
} else if (resp instanceof CacheResultHandler) {
val = WrapperUtil.wrap((CacheResultHandler) resp, CacheResultHandler.class);
} else {
throw new PluginException.InvalidDefinition(
"Second arg not a "
+ "CacheException or "
+ "CacheResultHandler class: "
+ entry
+ ", in "
+ mapName);
}
try {
int code = Integer.parseInt(first);
// If parseable as an integer, it's a result code.
hResultMap.storeMapEntry(code, val);
} catch (NumberFormatException e) {
try {
Class eClass = Class.forName(first);
// If a class name, it should be an exception class
if (Exception.class.isAssignableFrom(eClass)) {
hResultMap.storeMapEntry(eClass, val);
} else {
throw new PluginException.InvalidDefinition(
"First arg not an " + "Exception class: " + entry + ", in " + mapName);
}
} catch (Exception ex) {
throw new PluginException.InvalidDefinition(
"First arg not a " + "number or class: " + entry + ", in " + mapName);
} catch (LinkageError le) {
throw new PluginException.InvalidDefinition("Can't load " + first, le);
}
}
}
}
}
resultMap = hResultMap;
}
protected void initFeatureVersions() throws PluginException.InvalidDefinition {
if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
Map<String, String> spec =
(Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
log.debug2("features: " + spec);
for (Map.Entry<String, String> ent : spec.entrySet()) {
try {
// Prefix version string with feature name to create separate
// namespace for each feature
String key = ent.getKey();
map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
} catch (RuntimeException e) {
log.warning(
getPluginName()
+ " set unknown feature: "
+ ent.getKey()
+ " to version "
+ ent.getValue(),
e);
throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
}
}
featureVersion = map;
} else {
featureVersion = null;
}
}
protected void initAuFeatureMap() {
if (definitionMap.containsKey(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP)) {
Map<String, ?> featMap = definitionMap.getMap(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP);
for (Map.Entry ent : featMap.entrySet()) {
Object val = ent.getValue();
if (val instanceof Map) {
ent.setValue(MapUtil.expandAlternativeKeyLists((Map) val));
}
}
}
}
/** Create a CrawlWindow if necessary and return it. The CrawlWindow must be thread-safe. */
protected CrawlWindow makeCrawlWindow() {
if (crawlWindow != null) {
return crawlWindow;
}
CrawlWindow window =
(CrawlWindow) definitionMap.getMapElement(DefinableArchivalUnit.KEY_AU_CRAWL_WINDOW_SER);
if (window == null) {
String window_class =
definitionMap.getString(DefinableArchivalUnit.KEY_AU_CRAWL_WINDOW, null);
if (window_class != null) {
ConfigurableCrawlWindow ccw =
(ConfigurableCrawlWindow) newAuxClass(window_class, ConfigurableCrawlWindow.class);
try {
window = ccw.makeCrawlWindow();
} catch (PluginException e) {
throw new RuntimeException(e);
}
}
}
crawlWindow = window;
return window;
}
LoginPageChecker loginChecker;
protected LoginPageChecker makeLoginPageChecker() {
if (loginChecker == null) {
String loginPageCheckerClass =
definitionMap.getString(DefinableArchivalUnit.KEY_AU_LOGIN_PAGE_CHECKER, null);
if (loginPageCheckerClass != null) {
loginChecker =
(LoginPageChecker) newAuxClass(loginPageCheckerClass, LoginPageChecker.class);
}
}
return loginChecker;
}
PermissionCheckerFactory permissionCheckerFact;
protected PermissionCheckerFactory getPermissionCheckerFactory() {
if (permissionCheckerFact == null) {
String permissionCheckerFactoryClass =
definitionMap.getString(DefinableArchivalUnit.KEY_AU_PERMISSION_CHECKER_FACTORY, null);
if (permissionCheckerFactoryClass != null) {
permissionCheckerFact =
(PermissionCheckerFactory)
newAuxClass(permissionCheckerFactoryClass, PermissionCheckerFactory.class);
log.debug2("Loaded PermissionCheckerFactory: " + permissionCheckerFact);
}
}
return permissionCheckerFact;
}
protected UrlNormalizer urlNorm;
protected UrlNormalizer getUrlNormalizer() {
if (urlNorm == null) {
String normalizerClass =
definitionMap.getString(DefinableArchivalUnit.KEY_AU_URL_NORMALIZER, null);
if (normalizerClass != null) {
urlNorm = (UrlNormalizer) newAuxClass(normalizerClass, UrlNormalizer.class);
} else {
urlNorm = NullUrlNormalizer.INSTANCE;
}
}
return urlNorm;
}
protected ExploderHelper exploderHelper = null;
protected ExploderHelper getExploderHelper() {
if (exploderHelper == null) {
String helperClass =
definitionMap.getString(DefinableArchivalUnit.KEY_AU_EXPLODER_HELPER, null);
if (helperClass != null) {
exploderHelper = (ExploderHelper) newAuxClass(helperClass, ExploderHelper.class);
}
}
return exploderHelper;
}
protected CrawlUrlComparatorFactory crawlUrlComparatorFactory = null;
protected CrawlUrlComparatorFactory getCrawlUrlComparatorFactory() {
if (crawlUrlComparatorFactory == null) {
String factClass =
definitionMap.getString(DefinablePlugin.KEY_PLUGIN_CRAWL_URL_COMPARATOR_FACTORY, null);
if (factClass != null) {
crawlUrlComparatorFactory =
(CrawlUrlComparatorFactory) newAuxClass(factClass, CrawlUrlComparatorFactory.class);
}
}
return crawlUrlComparatorFactory;
}
protected Comparator<CrawlUrl> getCrawlUrlComparator(ArchivalUnit au)
throws PluginException.LinkageError {
CrawlUrlComparatorFactory fact = getCrawlUrlComparatorFactory();
if (fact == null) {
return null;
}
return fact.createCrawlUrlComparator(au);
}
protected FilterRule constructFilterRule(String contentType) {
String mimeType = HeaderUtil.getMimeTypeFromContentType(contentType);
Object filter_el =
definitionMap.getMapElement(mimeType + DefinableArchivalUnit.SUFFIX_FILTER_RULE);
if (filter_el instanceof String) {
log.debug("Loading filter " + filter_el);
return (FilterRule) newAuxClass((String) filter_el, FilterRule.class);
} else if (filter_el instanceof List) {
if (((List) filter_el).size() > 0) {
return new DefinableFilterRule((List) filter_el);
}
}
return super.constructFilterRule(mimeType);
}
protected ArticleIteratorFactory articleIteratorFact = null;
protected ArticleMetadataExtractorFactory articleMetadataFact = null;
/**
* Returns the plugin's article iterator factory, if any
*
* @return the ArticleIteratorFactory
*/
public ArticleIteratorFactory getArticleIteratorFactory() {
if (articleIteratorFact == null) {
String factClass = definitionMap.getString(KEY_PLUGIN_ARTICLE_ITERATOR_FACTORY, null);
if (factClass != null) {
articleIteratorFact =
(ArticleIteratorFactory) newAuxClass(factClass, ArticleIteratorFactory.class);
}
}
return articleIteratorFact;
}
/**
* Returns the article iterator factory for the content type, if any
*
* @param contentType the content type
* @return the ArticleIteratorFactory
*/
public ArticleMetadataExtractorFactory getArticleMetadataExtractorFactory(MetadataTarget target) {
if (articleMetadataFact == null) {
String factClass =
definitionMap.getString(KEY_PLUGIN_ARTICLE_METADATA_EXTRACTOR_FACTORY, null);
if (factClass != null) {
articleMetadataFact =
(ArticleMetadataExtractorFactory)
newAuxClass(factClass, ArticleMetadataExtractorFactory.class);
}
}
return articleMetadataFact;
}
public String getPluginId() {
String className;
if (mapName != null) {
className = mapName;
} else {
// @TODO: eliminate this when we eliminate subclasses
className = this.getClass().getName();
}
return className;
}
}
/**
* LockssRepository is used to organize the urls being cached. It keeps a memory cache of the most
* recently used nodes as a least-recently-used map, and also caches weak references to the
* instances as they're doled out. This ensures that two instances of the same node are never
* created, as the weak references only disappear when the object is finalized (they go to null when
* the last hard reference is gone, then are removed from the cache on finalize()).
*/
public class LockssRepositoryImpl extends BaseLockssDaemonManager implements LockssRepository {
private static Logger logger = Logger.getLogger("LockssRepository");
/** Configuration parameter name for Lockss cache location. */
public static final String PARAM_CACHE_LOCATION = Configuration.PREFIX + "cache.location";
/** Restores pre repo rescanning bugfix behavior (4050) */
public static final String PARAM_CLEAR_DIR_MAP = RepositoryManager.PREFIX + "clearDirMapOnAuStop";
public static final boolean DEFAULT_CLEAR_DIR_MAP = false;
/** Name of top directory in which the urls are cached. */
public static final String CACHE_ROOT_NAME = "cache";
// XXX This is a remnant from the single-disk days, and should go away.
// It is used only by unit tests, which want it set to the (last)
// individual repository dir created.
private static String staticCacheLocation = null;
// Maps local repository name (disk path) to LocalRepository instance
static Map localRepositories = new HashMap();
// starts with a '#' so no possibility of clashing with a URL
public static final String AU_ID_FILE = "#au_id_file";
static final String AU_ID_PROP = "au.id";
static final String PLUGIN_ID_PROP = "plugin.id";
static final char ESCAPE_CHAR = '#';
static final String ESCAPE_STR = "#";
static final char ENCODED_SEPARATOR_CHAR = 's';
static final String INITIAL_PLUGIN_DIR = String.valueOf((char) ('a' - 1));
static String lastPluginDir = INITIAL_PLUGIN_DIR;
// PJG: Windows prohibits use of ':' in file name -- replace with '~' for development
static final String PORT_SEPARATOR = SystemUtils.IS_OS_WINDOWS ? "%" : ":";
// this contains a '#' so that it's not defeatable by strings which
// match the prefix in a url (like '../tmp/')
private static final String TEST_PREFIX = "/#tmp";
private RepositoryManager repoMgr;
private String rootLocation;
UniqueRefLruCache nodeCache;
private boolean isGlobalNodeCache = RepositoryManager.DEFAULT_GLOBAL_CACHE_ENABLED;
LockssRepositoryImpl(String rootPath) {
if (rootPath.endsWith(File.separator)) {
rootLocation = rootPath;
} else {
// shouldn't happen
StringBuilder sb = new StringBuilder(rootPath.length() + File.separator.length());
sb.append(rootPath);
sb.append(File.separator);
rootLocation = sb.toString();
}
// Test code still needs this.
nodeCache = new UniqueRefLruCache(RepositoryManager.DEFAULT_MAX_PER_AU_CACHE_SIZE);
rootLocation =
rootLocation
.replace("?", "")
.replace("COM8", "COMEIGHT")
.replace("%5c", "/"); // //windows folder structure fix
}
public void startService() {
super.startService();
repoMgr = getDaemon().getRepositoryManager();
isGlobalNodeCache = repoMgr.isGlobalNodeCache();
if (isGlobalNodeCache) {
nodeCache = repoMgr.getGlobalNodeCache();
} else {
// nodeCache =
// new UniqueRefLruCache(repoMgr.paramNodeCacheSize);
setNodeCacheSize(repoMgr.paramNodeCacheSize);
}
}
public void stopService() {
// mainly important in testing to blank this
lastPluginDir = INITIAL_PLUGIN_DIR;
if (CurrentConfig.getBooleanParam(PARAM_CLEAR_DIR_MAP, DEFAULT_CLEAR_DIR_MAP)) {
// This should be in RepositoryManager.stopService()
localRepositories = new HashMap();
}
super.stopService();
}
public void setNodeCacheSize(int size) {
if (nodeCache != null && !isGlobalNodeCache && nodeCache.getMaxSize() != size) {
nodeCache.setMaxSize(size);
}
}
/**
* Called between initService() and startService(), then whenever the AU's config changes.
*
* @param auConfig the new configuration
*/
public void setAuConfig(Configuration auConfig) {}
void queueSizeCalc(RepositoryNode node) {
repoMgr.queueSizeCalc(node);
}
public RepositoryNode getNode(String url) throws MalformedURLException {
return getNode(url, false);
}
public RepositoryNode createNewNode(String url) throws MalformedURLException {
return getNode(url, true);
}
public void deleteNode(String url) throws MalformedURLException {
RepositoryNode node = getNode(url, false);
if (node != null) {
node.markAsDeleted();
}
}
public void deactivateNode(String url) throws MalformedURLException {
RepositoryNode node = getNode(url, false);
if (node != null) {
node.deactivateContent();
}
}
/**
* This function returns a RepositoryNode with a canonicalized path.
*
* @param url the url in String form
* @param create true iff the node should be created if absent
* @return RepositoryNode the node
* @throws MalformedURLException
*/
private synchronized RepositoryNode getNode(String url, boolean create)
throws MalformedURLException {
String canonUrl;
boolean isAuUrl = false;
if (AuUrl.isAuUrl(url)) {
// path information is lost here, but is unimportant if it's an AuUrl
canonUrl = AuUrl.PROTOCOL;
isAuUrl = true;
} else {
// create a canonical path, handling all illegal path traversal
canonUrl = canonicalizePath(url);
}
// check LRUMap cache for node
RepositoryNode node = (RepositoryNode) nodeCache.get(nodeCacheKey(canonUrl));
if (node != null) {
return node;
}
String nodeLocation;
if (isAuUrl) {
// base directory of ArchivalUnit
nodeLocation = rootLocation;
node = new AuNodeImpl(canonUrl, nodeLocation, this);
} else {
// determine proper node location
nodeLocation =
LockssRepositoryImpl.mapUrlToFileLocation(rootLocation, canonUrl)
.replace("?", "")
.replace("COM8", "COMEIGHT")
.replace("%5c", "/"); // //windows folder structure fix
node = new RepositoryNodeImpl(canonUrl, nodeLocation, this);
}
if (!create) {
// if not creating, check for existence
File nodeDir = new File(nodeLocation);
if (!nodeDir.exists()) {
// return null if the node doesn't exist and shouldn't be created
return null;
}
if (!nodeDir.isDirectory()) {
logger.error("Cache file not a directory: " + nodeLocation);
throw new LockssRepository.RepositoryStateException("Invalid cache file.");
}
}
// add to node cache
nodeCache.put(nodeCacheKey(canonUrl), node);
return node;
}
Object nodeCacheKey(String canonUrl) {
if (isGlobalNodeCache) {
return new KeyPair(this, canonUrl);
}
return canonUrl;
}
// functions for testing
int getCacheHits() {
return nodeCache.getCacheHits();
}
int getCacheMisses() {
return nodeCache.getCacheMisses();
}
int getRefHits() {
return nodeCache.getRefHits();
}
int getRefMisses() {
return nodeCache.getRefMisses();
}
public void nodeConsistencyCheck() {
// traverse the node tree from the top
RepositoryNode topNode;
try {
topNode = getNode(AuUrl.PROTOCOL_COLON);
recurseConsistencyCheck((RepositoryNodeImpl) topNode);
} catch (MalformedURLException ignore) {
}
}
/**
* Checks the consistency of the node, and continues with its children if it's consistent.
*
* @param node RepositoryNodeImpl the node to check
*/
private void recurseConsistencyCheck(RepositoryNodeImpl node) {
logger.debug2("Checking node '" + node.getNodeUrl() + "'...");
// check consistency at each node
// correct/deactivate as necessary
// 'checkNodeConsistency()' will repair if possible
if (node.checkNodeConsistency()) {
logger.debug3("Node consistent; recursing on children...");
List children = node.getNodeList(null, false);
Iterator iter = children.iterator();
while (iter.hasNext()) {
RepositoryNodeImpl child = (RepositoryNodeImpl) iter.next();
recurseConsistencyCheck(child);
}
} else {
logger.debug3("Node inconsistent; deactivating...");
deactivateInconsistentNode(node);
}
}
/**
* This is called when a node is in an inconsistent state. It simply creates some necessary
* directories and deactivates the node. Future polls should restore it properly.
*
* @param node the inconsistent node
*/
void deactivateInconsistentNode(RepositoryNodeImpl node) {
logger.warning("Deactivating inconsistent node.");
FileUtil.ensureDirExists(node.contentDir);
// manually deactivate
node.deactivateContent();
}
/**
* A method to remove any non-canonical '..' or '.' elements in the path, as well as protecting
* against illegal path traversal.
*
* @param url the raw url
* @return String the canonicalized url
* @throws MalformedURLException
*/
public String canonicalizePath(String url) throws MalformedURLException {
String canonUrl = UrlUtil.normalizeUrl(url, UrlUtil.PATH_TRAVERSAL_ACTION_THROW);
// canonicalize "dir" and "dir/"
// XXX if these are ever two separate nodes, this is wrong
if (canonUrl.endsWith(UrlUtil.URL_PATH_SEPARATOR)) {
canonUrl = canonUrl.substring(0, canonUrl.length() - 1);
}
return canonUrl;
}
// static calls
/**
* Factory method to create new LockssRepository instances.
*
* @param au the {@link ArchivalUnit}
* @return the new LockssRepository instance
*/
public static LockssRepository createNewLockssRepository(ArchivalUnit au) {
String root = getRepositoryRoot(au);
if (root == null || root.equals("null")) {
logger.error("No repository dir set in config");
throw new LockssRepository.RepositoryStateException("No repository dir set in config");
}
String auDir = LockssRepositoryImpl.mapAuToFileLocation(root, au);
if (logger.isDebug2()) {
logger.debug2("repo: " + auDir + ", au: " + au.getName());
}
staticCacheLocation = extendCacheLocation(root);
LockssRepositoryImpl repo = new LockssRepositoryImpl(auDir);
Plugin plugin = au.getPlugin();
if (plugin != null) {
LockssDaemon daemon = plugin.getDaemon();
if (daemon != null) {
RepositoryManager mgr = daemon.getRepositoryManager();
if (mgr != null) {
mgr.setRepositoryForPath(auDir, repo);
}
}
}
return repo;
}
public static String getRepositorySpec(ArchivalUnit au) {
Configuration auConfig = au.getConfiguration();
if (auConfig != null) { // can be null in unit tests
String repoSpec = auConfig.get(PluginManager.AU_PARAM_REPOSITORY);
if (repoSpec != null && repoSpec.startsWith("local:")) {
return repoSpec;
}
}
return "local:" + CurrentConfig.getParam(PARAM_CACHE_LOCATION);
}
public static String getRepositoryRoot(ArchivalUnit au) {
return getLocalRepositoryPath(getRepositorySpec(au));
}
public static String getLocalRepositoryPath(String repoSpec) {
if (repoSpec != null) {
if (repoSpec.startsWith("local:")) {
return repoSpec.substring(6);
}
}
return null;
}
// The OpenBSD platform has renamed the first disk from /cache to
// /cache.wd0, leaving behind a symbolic link in /cache . This is
// transparent everywhere except the repository status table, which needs
// to match AU configs with AUs it finds when enumerating the repository.
// Existing AU configs have repository=local:/cache, so the relative link
// needs to be resolved to detect that that's the same as
// local:/cache.wd0
private static Map canonicalRoots = new HashMap();
public static boolean isDirInRepository(String dir, String repoRoot) {
if (dir.startsWith(repoRoot)) {
return true;
}
return canonRoot(dir).startsWith(canonRoot(repoRoot));
}
static String canonRoot(String root) {
synchronized (canonicalRoots) {
String canon = (String) canonicalRoots.get(root);
if (canon == null) {
try {
canon = new File(root).getCanonicalPath();
canonicalRoots.put(root, canon);
} catch (IOException e) {
logger.warning("Can't canonicalize: " + root, e);
return root;
}
}
return canon;
}
}
static String getCacheLocation() {
return staticCacheLocation;
}
/**
* Adds the 'cache' directory to the HD location.
*
* @param cacheDir the root location.
* @return String the extended location
*/
static String extendCacheLocation(String cacheDir) {
StringBuilder buffer = new StringBuilder(cacheDir);
if (!cacheDir.endsWith(File.separator)) {
buffer.append(File.separator);
}
buffer.append(CACHE_ROOT_NAME);
buffer.append(File.separator);
return buffer.toString();
}
/**
* mapAuToFileLocation() is the method used to resolve {@link ArchivalUnit}s into directory names.
* This maps a given au to directories, using the cache root as the base. Given an au with
* PluginId of 'plugin' and AuId of 'au', it would return the string '<rootLocation>/plugin/au/'.
*
* @param repoRoot the root of a LOCKSS repository
* @param au the ArchivalUnit to resolve
* @return the directory location
*/
public static String mapAuToFileLocation(String repoRoot, ArchivalUnit au) {
return getAuDir(au, repoRoot, true);
}
/**
* mapUrlToFileLocation() is the method used to resolve urls into file names. This maps a given
* url to a file location, using the au top directory as the base. It creates directories which
* mirror the html string, so 'http://www.journal.org/issue1/index.html' would be cached in the
* file: <rootLocation>/www.journal.org/http/issue1/index.html
*
* @param rootLocation the top directory for ArchivalUnit this URL is in
* @param urlStr the url to translate
* @return the url file location
* @throws java.net.MalformedURLException
*/
public static String mapUrlToFileLocation(String rootLocation, String urlStr)
throws MalformedURLException {
int totalLength = rootLocation.length() + urlStr.length();
URL url = new URL(urlStr);
StringBuilder buffer = new StringBuilder(totalLength);
buffer.append(rootLocation);
if (!rootLocation.endsWith(File.separator)) {
buffer.append(File.separator);
}
buffer.append(url.getHost().toLowerCase());
int port = url.getPort();
if (port != -1) {
buffer.append(PORT_SEPARATOR);
buffer.append(port);
}
buffer.append(File.separator);
buffer.append(url.getProtocol());
if (RepositoryManager.isEnableLongComponents()) {
String escapedPath =
escapePath(
StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator));
String query = url.getQuery();
if (query != null) {
escapedPath = escapedPath + "?" + escapeQuery(query);
}
String encodedPath = RepositoryNodeImpl.encodeUrl(escapedPath);
// encodeUrl strips leading / from path
buffer.append(File.separator);
buffer.append(encodedPath);
} else {
buffer.append(
escapePath(
StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator)));
String query = url.getQuery();
if (query != null) {
buffer.append("?");
buffer.append(escapeQuery(query));
}
}
return buffer.toString();
}
// name mapping functions
/**
* Return true iff a repository for the auid exists under the root
*
* @param auid
* @param repoRoot the repository root
* @return true iff a repository for the auid exists
*/
static boolean doesAuDirExist(String auid, String repoRoot) {
return null != getAuDir(auid, repoRoot, false);
}
/**
* Finds the directory for this AU. If none found in the map, designates a new dir for it.
*
* @param au the AU
* @param repoRoot root of the repository
* @return the dir {@link String}
*/
static String getAuDir(ArchivalUnit au, String repoRoot, boolean create) {
return getAuDir(au.getAuId(), repoRoot, create);
}
/**
* Finds the directory for this AU. If none found in the map, designates a new dir for it.
*
* @param auid AU id representing the au
* @param repoRoot path to the root of the repository
* @return the dir String
*/
static String getAuDir(String auid, String repoRoot, boolean create) {
String repoCachePath = extendCacheLocation(repoRoot);
LocalRepository localRepo = getLocalRepository(repoRoot);
synchronized (localRepo) {
Map aumap = localRepo.getAuMap();
String auPathSlash = (String) aumap.get(auid);
if (auPathSlash != null) {
return auPathSlash;
}
if (!create) {
return null;
}
logger.debug3("Creating new au directory for '" + auid + "'.");
String auDir = localRepo.getPrevAuDir();
for (int cnt = RepositoryManager.getMaxUnusedDirSearch(); cnt > 0; cnt--) {
// loop through looking for an available dir
auDir = getNextDirName(auDir);
File testDir = new File(repoCachePath, auDir);
if (logger.isDebug3()) logger.debug3("Probe for unused: " + testDir);
if (!testDir.exists()) {
if (RepositoryManager.isStatefulUnusedDirSearch()) {
localRepo.setPrevAuDir(auDir);
}
String auPath = testDir.toString();
logger.debug3("New au directory: " + auPath);
auPathSlash = auPath + File.separator;
// write the new au property file to the new dir
// XXX this data should be backed up elsewhere to avoid single-point
// corruption
Properties idProps = new Properties();
idProps.setProperty(AU_ID_PROP, auid);
saveAuIdProperties(auPath, idProps);
aumap.put(auid, auPathSlash);
return auPathSlash;
} else {
if (logger.isDebug3()) {
logger.debug3("Existing directory found at '" + auDir + "'. Checking next...");
}
}
}
}
throw new RuntimeException(
"Can't find unused repository dir after "
+ RepositoryManager.getMaxUnusedDirSearch()
+ " tries in "
+ repoCachePath);
}
static LocalRepository getLocalRepository(ArchivalUnit au) {
return getLocalRepository(getRepositoryRoot(au));
}
static LocalRepository getLocalRepository(String repoRoot) {
synchronized (localRepositories) {
LocalRepository localRepo = (LocalRepository) localRepositories.get(repoRoot);
if (localRepo == null) {
logger.debug2("Creating LocalRepository(" + repoRoot + ")");
localRepo = new LocalRepository(repoRoot);
localRepositories.put(repoRoot, localRepo);
}
return localRepo;
}
}
/** Return next string in the sequence "a", "b", ... "z", "aa", "ab", ... */
static String getNextDirName(String old) {
StringBuilder sb = new StringBuilder(old);
// go through and increment the first non-'z' char
// counts back from the last char, so 'aa'->'ab', not 'ba'
for (int ii = sb.length() - 1; ii >= 0; ii--) {
char curChar = sb.charAt(ii);
if (curChar < 'z') {
sb.setCharAt(ii, (char) (curChar + 1));
return sb.toString();
}
sb.setCharAt(ii, 'a');
}
sb.insert(0, 'a');
return sb.toString();
}
public static File getAuIdFile(String location) {
return new File(location + File.separator + AU_ID_FILE);
}
static Properties getAuIdProperties(String location) {
File propFile = new File(location + File.separator + AU_ID_FILE);
try {
InputStream is = new BufferedInputStream(new FileInputStream(propFile));
Properties idProps = new Properties();
idProps.load(is);
is.close();
return idProps;
} catch (Exception e) {
logger.warning("Error loading au id from " + propFile.getPath() + ".");
return null;
}
}
static void saveAuIdProperties(String location, Properties props) {
// XXX these AU_ID_FILE entries need to be backed up elsewhere to avoid
// single-point corruption
File propDir = new File(location);
if (!propDir.exists()) {
logger.debug("Creating directory '" + propDir.getAbsolutePath() + "'");
propDir.mkdirs();
}
File propFile = new File(propDir, AU_ID_FILE);
try {
logger.debug3("Saving au id properties at '" + location + "'.");
OutputStream os = new BufferedOutputStream(new FileOutputStream(propFile));
props.store(os, "ArchivalUnit id info");
os.close();
propFile.setReadOnly();
} catch (IOException ioe) {
logger.error("Couldn't write properties for " + propFile.getPath() + ".", ioe);
throw new LockssRepository.RepositoryStateException("Couldn't write au id properties file.");
}
}
// lockss filename-specific encoding methods
/**
* Escapes instances of the ESCAPE_CHAR from the path. This avoids name conflicts with the
* repository files, such as '#nodestate.xml'.
*
* @param path the path
* @return the escaped path
*/
static String escapePath(String path) {
// XXX escaping disabled because of URL encoding
if (false && path.indexOf(ESCAPE_CHAR) >= 0) {
return StringUtil.replaceString(path, ESCAPE_STR, ESCAPE_STR + ESCAPE_STR);
} else {
return path;
}
}
/**
* Escapes instances of File.separator from the query. These are safe from filename overlap, but
* can't convert into extended paths and directories.
*
* @param query the query
* @return the escaped query
*/
static String escapeQuery(String query) {
if (query.indexOf(File.separator) >= 0) {
return StringUtil.replaceString(query, File.separator, ESCAPE_STR + ENCODED_SEPARATOR_CHAR);
} else {
return query;
}
}
/**
* Extracts '#x' encoding and converts back to 'x'.
*
* @param orig the original
* @return the unescaped version.
*/
static String unescape(String orig) {
if (orig.indexOf(ESCAPE_CHAR) < 0) {
// fast treatment of non-escaped strings
return orig;
}
int index = -1;
StringBuilder buffer = new StringBuilder(orig.length());
String oldStr = orig;
while ((index = oldStr.indexOf(ESCAPE_CHAR)) >= 0) {
buffer.append(oldStr.substring(0, index));
buffer.append(convertCode(oldStr.substring(index, index + 2)));
if (oldStr.length() > 2) {
oldStr = oldStr.substring(index + 2);
} else {
oldStr = "";
}
}
buffer.append(oldStr);
return buffer.toString();
}
/**
* Returns the second char in the escaped segment, unless it is 's', which is a stand-in for the
* File.separatorChar.
*
* @param code the code segment (length 2)
* @return the encoded char
*/
static char convertCode(String code) {
char encodedChar = code.charAt(1);
if (encodedChar == ENCODED_SEPARATOR_CHAR) {
return File.separatorChar;
} else {
return encodedChar;
}
}
public static class Factory implements LockssAuManager.Factory {
public LockssAuManager createAuManager(ArchivalUnit au) {
return createNewLockssRepository(au);
}
}
/** Maintains state for a local repository root dir (<i>eg</i>, auid of each au subdir). */
static class LocalRepository {
String repoPath;
File repoCacheFile;
Map auMap;
String prevAuDir;
LocalRepository(String repoPath) {
this.repoPath = repoPath;
repoCacheFile = new File(repoPath, CACHE_ROOT_NAME);
}
public String getRepositoryPath() {
return repoPath;
}
public String getPrevAuDir() {
if (prevAuDir == null) {
prevAuDir = lastPluginDir;
}
return prevAuDir;
}
public void setPrevAuDir(String dir) {
prevAuDir = dir;
}
/**
* Return the auid -> au-subdir-path mapping. Enumerating the directories if necessary to
* initialize the map
*/
Map getAuMap() {
if (auMap == null) {
logger.debug3("Loading name map for '" + repoCacheFile + "'.");
auMap = new HashMap();
if (!repoCacheFile.exists()) {
logger.debug3("Creating cache dir:" + repoCacheFile + "'.");
if (!repoCacheFile.mkdirs()) {
logger.critical("Couldn't create directory, check owner/permissions: " + repoCacheFile);
// return empty map
return auMap;
}
} else {
// read each dir's property file and store mapping auid -> dir
File[] auDirs = repoCacheFile.listFiles();
for (int ii = 0; ii < auDirs.length; ii++) {
String dirName = auDirs[ii].getName();
// if (dirName.compareTo(lastPluginDir) == 1) {
// // adjust the 'lastPluginDir' upwards if necessary
// lastPluginDir = dirName;
// }
String path = auDirs[ii].getAbsolutePath();
Properties idProps = getAuIdProperties(path);
if (idProps != null) {
String auid = idProps.getProperty(AU_ID_PROP);
StringBuilder sb = new StringBuilder(path.length() + File.separator.length());
sb.append(path);
sb.append(File.separator);
auMap.put(auid, sb.toString());
logger.debug3("Mapping to: " + auMap.get(auid) + ": " + auid);
} else {
logger.debug3("Not mapping " + path + ", no auid file.");
}
}
}
}
return auMap;
}
public String toString() {
return "[LR: " + repoPath + "]";
}
}
String getRootLocation() {
return rootLocation;
}
}
