public void export() {
log.debug("export(" + au.getName() + ")");
log.debug(
"dir: "
+ dir
+ ", pref: "
+ prefix
+ ", size: "
+ maxSize
+ ", ver: "
+ maxVersions
+ (compress ? ", (C)" : ""));
checkArgs();
try {
start();
} catch (IOException e) {
recordError("Error opening file", e);
return;
}
writeFiles();
try {
finish();
} catch (IOException e) {
if (!isDiskFull) {
// If we already knew (and reported) disk full, also reporting it
// as a close error is misleading.
recordError("Error closing file", e);
}
}
}
private void initTestPolls() throws Exception {
testV1polls = new V1Poll[testV1msg.length];
for (int i = 0; i < testV1polls.length; i++) {
log.debug3("initTestPolls: V1 " + i);
BasePoll p = pollmanager.makePoll(testV1msg[i]);
assertNotNull(p);
assertNotNull(p.getMessage());
log.debug("initTestPolls: V1 " + i + " returns " + p);
assertTrue(p instanceof V1Poll);
switch (i) {
case 0:
assertTrue(p instanceof V1NamePoll);
break;
case 1:
assertTrue(p instanceof V1ContentPoll);
break;
case 2:
assertTrue(p instanceof V1VerifyPoll);
break;
}
testV1polls[i] = (V1Poll) p;
assertNotNull(testV1polls[i]);
log.debug3("initTestPolls: " + i + " " + p.toString());
}
}
/** Create LockssKeystores from config subtree below {@link #PARAM_KEYSTORE} */
void configureKeyStores(Configuration config) {
Configuration allKs = config.getConfigTree(PARAM_KEYSTORE);
for (Iterator iter = allKs.nodeIterator(); iter.hasNext(); ) {
String id = (String) iter.next();
Configuration oneKs = allKs.getConfigTree(id);
try {
LockssKeyStore lk = createLockssKeyStore(oneKs);
String name = lk.getName();
if (name == null) {
log.error("KeyStore definition missing name: " + oneKs);
continue;
}
LockssKeyStore old = keystoreMap.get(name);
if (old != null && !lk.equals(old)) {
log.warning(
"Keystore "
+ name
+ " redefined. "
+ "New definition may not take effect until daemon restart");
}
log.debug("Adding keystore " + name);
keystoreMap.put(name, lk);
} catch (Exception e) {
log.error("Couldn't create keystore: " + oneKs, e);
}
}
}
public PlatformUtil.DF getRepositoryDF(String repoName) {
String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
log.debug("path: " + path);
// try {
return platInfo.getJavaDF(path);
// } catch (PlatformUtil.UnsupportedException e) {
// return null;
// }
}
protected void logParams() {
Enumeration en = req.getParameterNames();
while (en.hasMoreElements()) {
String name = (String) en.nextElement();
String vals[];
String dispval;
if (StringUtil.indexOfIgnoreCase(name, "passw") >= 0) {
dispval = req.getParameter(name).length() == 0 ? "" : "********";
} else if (log.isDebug2() && (vals = req.getParameterValues(name)).length > 1) {
dispval = StringUtil.separatedString(vals, ", ");
} else {
dispval = req.getParameter(name);
}
log.debug(name + " = " + dispval);
}
}
protected FilterRule constructFilterRule(String contentType) {
String mimeType = HeaderUtil.getMimeTypeFromContentType(contentType);
Object filter_el =
definitionMap.getMapElement(mimeType + DefinableArchivalUnit.SUFFIX_FILTER_RULE);
if (filter_el instanceof String) {
log.debug("Loading filter " + filter_el);
return (FilterRule) newAuxClass((String) filter_el, FilterRule.class);
} else if (filter_el instanceof List) {
if (((List) filter_el).size() > 0) {
return new DefinableFilterRule((List) filter_el);
}
}
return super.constructFilterRule(mimeType);
}
private void initTestMsg() throws Exception {
testV1msg = new V1LcapMessage[3];
int[] pollType = {
Poll.V1_NAME_POLL, Poll.V1_CONTENT_POLL, Poll.V1_VERIFY_POLL,
};
PollFactory ppf = pollmanager.getPollFactory(1);
assertNotNull("PollFactory should not be null", ppf);
// XXX V1 support mandatory
assertTrue(ppf instanceof V1PollFactory);
V1PollFactory pf = (V1PollFactory) ppf;
for (int i = 0; i < testV1msg.length; i++) {
PollSpec spec = new MockPollSpec(testau, rootV1urls[i], lwrbnd, uprbnd, pollType[i]);
log.debug("Created poll spec: " + spec);
((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
int opcode = V1LcapMessage.NAME_POLL_REQ + (i * 2);
long duration = -1;
// NB calcDuration is not applied to Verify polls.
switch (opcode) {
case V1LcapMessage.NAME_POLL_REQ:
case V1LcapMessage.CONTENT_POLL_REQ:
// this will attempt to schedule and can return -1
duration = Math.max(pf.calcDuration(spec, pollmanager), 1000);
break;
case V1LcapMessage.VERIFY_POLL_REQ:
case V1LcapMessage.VERIFY_POLL_REP:
duration = 100000; // Arbitrary
break;
default:
fail("Bad opcode " + opcode);
break;
}
testV1msg[i] =
V1LcapMessage.makeRequestMsg(
spec,
agree_entries,
pf.makeVerifier(100000),
pf.makeVerifier(100000),
opcode,
duration,
testID);
assertNotNull(testV1msg[i]);
}
}
/** If in testing mode FOO, copy values from FOO_override map, if any, to main map */
void processOverrides(TypedEntryMap map) {
String testMode = getTestingMode();
if (StringUtil.isNullString(testMode)) {
return;
}
Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE);
if (o == null) {
return;
}
if (o instanceof Map) {
Map overrideMap = (Map) o;
for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) {
String key = (String) entry.getKey();
Object val = entry.getValue();
log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val);
map.setMapElement(key, val);
}
}
}
static void saveAuIdProperties(String location, Properties props) {
// XXX these AU_ID_FILE entries need to be backed up elsewhere to avoid
// single-point corruption
File propDir = new File(location);
if (!propDir.exists()) {
logger.debug("Creating directory '" + propDir.getAbsolutePath() + "'");
propDir.mkdirs();
}
File propFile = new File(propDir, AU_ID_FILE);
try {
logger.debug3("Saving au id properties at '" + location + "'.");
OutputStream os = new BufferedOutputStream(new FileOutputStream(propFile));
props.store(os, "ArchivalUnit id info");
os.close();
propFile.setReadOnly();
} catch (IOException ioe) {
logger.error("Couldn't write properties for " + propFile.getPath() + ".", ioe);
throw new LockssRepository.RepositoryStateException("Couldn't write au id properties file.");
}
}
public static V1Poll createCompletedPoll(
LockssDaemon daemon,
ArchivalUnit au,
V1LcapMessage testmsg,
int numAgree,
int numDisagree,
PollManager pollmanager)
throws Exception {
log.debug(
"createCompletedPoll: au: "
+ au.toString()
+ " peer "
+ testmsg.getOriginatorId()
+ " votes "
+ numAgree
+ "/"
+ numDisagree);
CachedUrlSetSpec cusSpec = null;
if ((testmsg.getLwrBound() != null)
&& (testmsg.getLwrBound().equals(PollSpec.SINGLE_NODE_LWRBOUND))) {
cusSpec = new SingleNodeCachedUrlSetSpec(testmsg.getTargetUrl());
} else {
cusSpec =
new RangeCachedUrlSetSpec(
testmsg.getTargetUrl(), testmsg.getLwrBound(), testmsg.getUprBound());
}
CachedUrlSet cus = au.makeCachedUrlSet(cusSpec);
PollSpec spec = new PollSpec(cus, Poll.V1_CONTENT_POLL);
((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false);
V1Poll p = null;
if (testmsg.isContentPoll()) {
p =
new V1ContentPoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm());
} else if (testmsg.isNamePoll()) {
p =
new V1NamePoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm());
} else if (testmsg.isVerifyPoll()) {
p =
new V1VerifyPoll(
spec,
pollmanager,
testmsg.getOriginatorId(),
testmsg.getChallenge(),
testmsg.getDuration(),
testmsg.getHashAlgorithm(),
testmsg.getVerifier());
}
assertNotNull(p);
p.setMessage(testmsg);
p.m_tally.quorum = numAgree + numDisagree;
p.m_tally.numAgree = numAgree;
p.m_tally.numDisagree = numDisagree;
p.m_tally.wtAgree = 2000;
p.m_tally.wtDisagree = 200;
p.m_tally.localEntries = makeEntries(1, 3);
p.m_tally.votedEntries = makeEntries(1, 5);
p.m_tally.votedEntries.remove(1);
p.m_pollstate = V1Poll.PS_COMPLETE;
p.m_callerID = testmsg.getOriginatorId();
log.debug3("poll " + p.toString());
p.m_tally.tallyVotes();
return p;
}
public void setConfig(
Configuration config, Configuration oldConfig, Configuration.Differences changedKeys) {
// Build list of repositories from list of disk (fs) paths). Needs to
// be generalized if ever another repository implementation.
if (changedKeys.contains(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST)) {
List lst = new ArrayList();
String dspace = config.get(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, "");
List paths = StringUtil.breakAt(dspace, ';');
if (paths != null) {
for (Iterator iter = paths.iterator(); iter.hasNext(); ) {
lst.add("local:" + (String) iter.next());
}
}
repoList = lst;
}
if (changedKeys.contains(PARAM_MAX_PER_AU_CACHE_SIZE)) {
paramNodeCacheSize =
config.getInt(PARAM_MAX_PER_AU_CACHE_SIZE, DEFAULT_MAX_PER_AU_CACHE_SIZE);
for (Iterator iter = getDaemon().getAllLockssRepositories().iterator(); iter.hasNext(); ) {
LockssRepository repo = (LockssRepository) iter.next();
if (repo instanceof LockssRepositoryImpl) {
LockssRepositoryImpl repoImpl = (LockssRepositoryImpl) repo;
repoImpl.setNodeCacheSize(paramNodeCacheSize);
}
}
}
if (changedKeys.contains(PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE)) {
paramSuspectVersionsCacheSize =
config.getInt(
PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE, DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
suspectVersionsCache.setMaxSize(paramSuspectVersionsCacheSize);
}
if (changedKeys.contains(GLOBAL_CACHE_PREFIX)) {
paramIsGlobalNodeCache =
config.getBoolean(PARAM_GLOBAL_CACHE_ENABLED, DEFAULT_GLOBAL_CACHE_ENABLED);
if (paramIsGlobalNodeCache) {
paramGlobalNodeCacheSize =
config.getInt(PARAM_MAX_GLOBAL_CACHE_SIZE, DEFAULT_MAX_GLOBAL_CACHE_SIZE);
log.debug("global node cache size: " + paramGlobalNodeCacheSize);
globalNodeCache.setMaxSize(paramGlobalNodeCacheSize);
}
}
if (changedKeys.contains(DISK_PREFIX)) {
int minMB = config.getInt(PARAM_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_MB);
double minPer =
config.getPercentage(PARAM_DISK_WARN_FRRE_PERCENT, DEFAULT_DISK_WARN_FRRE_PERCENT);
paramDFWarn = PlatformUtil.DF.makeThreshold(minMB, minPer);
minMB = config.getInt(PARAM_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_MB);
minPer = config.getPercentage(PARAM_DISK_FULL_FRRE_PERCENT, DEFAULT_DISK_FULL_FRRE_PERCENT);
paramDFFull = PlatformUtil.DF.makeThreshold(minMB, minPer);
}
if (changedKeys.contains(PARAM_SIZE_CALC_MAX_LOAD)) {
sizeCalcMaxLoad = config.getPercentage(PARAM_SIZE_CALC_MAX_LOAD, DEFAULT_SIZE_CALC_MAX_LOAD);
}
if (changedKeys.contains(PREFIX)) {
maxUnusedDirSearch =
config.getInt(PARAM_MAX_UNUSED_DIR_SEARCH, DEFAULT_MAX_UNUSED_DIR_SEARCH);
isStatefulUnusedDirSearch =
config.getBoolean(
PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH, DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH);
enableLongComponents =
config.getBoolean(PARAM_ENABLE_LONG_COMPONENTS, DEFAULT_ENABLE_LONG_COMPONENTS);
enableLongComponentsCompatibility =
config.getBoolean(
PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY,
DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY);
maxComponentLength = config.getInt(PARAM_MAX_COMPONENT_LENGTH, DEFAULT_MAX_COMPONENT_LENGTH);
checkUnnormalized =
(CheckUnnormalizedMode)
config.getEnum(
CheckUnnormalizedMode.class,
PARAM_CHECK_UNNORMALIZED,
DEFAULT_CHECK_UNNORMALIZED);
}
}
private void callV3ContentPoll(ArchivalUnit au) throws PollManager.NotEligibleException {
log.debug("Enqueuing a V3 Content Poll on " + au.getName());
PollSpec spec = new PollSpec(au.getAuCachedUrlSet(), Poll.V3_POLL);
pollManager.enqueueHighPriorityPoll(au, spec);
statusMsg = "Enqueued V3 poll for " + au.getName();
}
protected void initMimeMap() throws PluginException.InvalidDefinition {
for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry ent = (Map.Entry) iter.next();
String key = (String) ent.getKey();
Object val = ent.getValue();
if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " link extractor: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
LinkExtractorFactory fact =
(LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class);
mti.setLinkExtractorFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) {
// XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY
// XXX unless/until that key is changed to not be a terminal substring
// XXX of this one
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " crawl filter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
mti.setCrawlFilterFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " filter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
mti.setHashFilterFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT);
if (val instanceof String) {
String rate = (String) val;
log.debug(mime + " fetch rate: " + rate);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
RateLimiter limit = mti.getFetchRateLimiter();
if (limit != null) {
limit.setRate(rate);
} else {
mti.setFetchRateLimiter(new RateLimiter(rate));
}
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY);
String factName = (String) val;
log.debug(mime + " link rewriter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
LinkRewriterFactory fact =
(LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class);
mti.setLinkRewriterFactory(fact);
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP);
Map factNameMap = (Map) val;
Map factClassMap = new HashMap();
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) {
String mdTypes = (String) it.next();
String factName = (String) factNameMap.get(mdTypes);
log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName);
for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) {
setMdTypeFact(factClassMap, mdType, factName);
}
}
mti.setFileMetadataExtractorFactoryMap(factClassMap);
}
}
}