public void export() { log.debug("export(" + au.getName() + ")"); log.debug( "dir: " + dir + ", pref: " + prefix + ", size: " + maxSize + ", ver: " + maxVersions + (compress ? ", (C)" : "")); checkArgs(); try { start(); } catch (IOException e) { recordError("Error opening file", e); return; } writeFiles(); try { finish(); } catch (IOException e) { if (!isDiskFull) { // If we already knew (and reported) disk full, also reporting it // as a close error is misleading. recordError("Error closing file", e); } } }
private void initTestPolls() throws Exception { testV1polls = new V1Poll[testV1msg.length]; for (int i = 0; i < testV1polls.length; i++) { log.debug3("initTestPolls: V1 " + i); BasePoll p = pollmanager.makePoll(testV1msg[i]); assertNotNull(p); assertNotNull(p.getMessage()); log.debug("initTestPolls: V1 " + i + " returns " + p); assertTrue(p instanceof V1Poll); switch (i) { case 0: assertTrue(p instanceof V1NamePoll); break; case 1: assertTrue(p instanceof V1ContentPoll); break; case 2: assertTrue(p instanceof V1VerifyPoll); break; } testV1polls[i] = (V1Poll) p; assertNotNull(testV1polls[i]); log.debug3("initTestPolls: " + i + " " + p.toString()); } }
/** Create LockssKeystores from config subtree below {@link #PARAM_KEYSTORE} */ void configureKeyStores(Configuration config) { Configuration allKs = config.getConfigTree(PARAM_KEYSTORE); for (Iterator iter = allKs.nodeIterator(); iter.hasNext(); ) { String id = (String) iter.next(); Configuration oneKs = allKs.getConfigTree(id); try { LockssKeyStore lk = createLockssKeyStore(oneKs); String name = lk.getName(); if (name == null) { log.error("KeyStore definition missing name: " + oneKs); continue; } LockssKeyStore old = keystoreMap.get(name); if (old != null && !lk.equals(old)) { log.warning( "Keystore " + name + " redefined. " + "New definition may not take effect until daemon restart"); } log.debug("Adding keystore " + name); keystoreMap.put(name, lk); } catch (Exception e) { log.error("Couldn't create keystore: " + oneKs, e); } } }
public PlatformUtil.DF getRepositoryDF(String repoName) { String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName); log.debug("path: " + path); // try { return platInfo.getJavaDF(path); // } catch (PlatformUtil.UnsupportedException e) { // return null; // } }
protected void logParams() { Enumeration en = req.getParameterNames(); while (en.hasMoreElements()) { String name = (String) en.nextElement(); String vals[]; String dispval; if (StringUtil.indexOfIgnoreCase(name, "passw") >= 0) { dispval = req.getParameter(name).length() == 0 ? "" : "********"; } else if (log.isDebug2() && (vals = req.getParameterValues(name)).length > 1) { dispval = StringUtil.separatedString(vals, ", "); } else { dispval = req.getParameter(name); } log.debug(name + " = " + dispval); } }
protected FilterRule constructFilterRule(String contentType) { String mimeType = HeaderUtil.getMimeTypeFromContentType(contentType); Object filter_el = definitionMap.getMapElement(mimeType + DefinableArchivalUnit.SUFFIX_FILTER_RULE); if (filter_el instanceof String) { log.debug("Loading filter " + filter_el); return (FilterRule) newAuxClass((String) filter_el, FilterRule.class); } else if (filter_el instanceof List) { if (((List) filter_el).size() > 0) { return new DefinableFilterRule((List) filter_el); } } return super.constructFilterRule(mimeType); }
private void initTestMsg() throws Exception { testV1msg = new V1LcapMessage[3]; int[] pollType = { Poll.V1_NAME_POLL, Poll.V1_CONTENT_POLL, Poll.V1_VERIFY_POLL, }; PollFactory ppf = pollmanager.getPollFactory(1); assertNotNull("PollFactory should not be null", ppf); // XXX V1 support mandatory assertTrue(ppf instanceof V1PollFactory); V1PollFactory pf = (V1PollFactory) ppf; for (int i = 0; i < testV1msg.length; i++) { PollSpec spec = new MockPollSpec(testau, rootV1urls[i], lwrbnd, uprbnd, pollType[i]); log.debug("Created poll spec: " + spec); ((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false); int opcode = V1LcapMessage.NAME_POLL_REQ + (i * 2); long duration = -1; // NB calcDuration is not applied to Verify polls. switch (opcode) { case V1LcapMessage.NAME_POLL_REQ: case V1LcapMessage.CONTENT_POLL_REQ: // this will attempt to schedule and can return -1 duration = Math.max(pf.calcDuration(spec, pollmanager), 1000); break; case V1LcapMessage.VERIFY_POLL_REQ: case V1LcapMessage.VERIFY_POLL_REP: duration = 100000; // Arbitrary break; default: fail("Bad opcode " + opcode); break; } testV1msg[i] = V1LcapMessage.makeRequestMsg( spec, agree_entries, pf.makeVerifier(100000), pf.makeVerifier(100000), opcode, duration, testID); assertNotNull(testV1msg[i]); } }
/** If in testing mode FOO, copy values from FOO_override map, if any, to main map */ void processOverrides(TypedEntryMap map) { String testMode = getTestingMode(); if (StringUtil.isNullString(testMode)) { return; } Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE); if (o == null) { return; } if (o instanceof Map) { Map overrideMap = (Map) o; for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) { String key = (String) entry.getKey(); Object val = entry.getValue(); log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val); map.setMapElement(key, val); } } }
static void saveAuIdProperties(String location, Properties props) { // XXX these AU_ID_FILE entries need to be backed up elsewhere to avoid // single-point corruption File propDir = new File(location); if (!propDir.exists()) { logger.debug("Creating directory '" + propDir.getAbsolutePath() + "'"); propDir.mkdirs(); } File propFile = new File(propDir, AU_ID_FILE); try { logger.debug3("Saving au id properties at '" + location + "'."); OutputStream os = new BufferedOutputStream(new FileOutputStream(propFile)); props.store(os, "ArchivalUnit id info"); os.close(); propFile.setReadOnly(); } catch (IOException ioe) { logger.error("Couldn't write properties for " + propFile.getPath() + ".", ioe); throw new LockssRepository.RepositoryStateException("Couldn't write au id properties file."); } }
public static V1Poll createCompletedPoll( LockssDaemon daemon, ArchivalUnit au, V1LcapMessage testmsg, int numAgree, int numDisagree, PollManager pollmanager) throws Exception { log.debug( "createCompletedPoll: au: " + au.toString() + " peer " + testmsg.getOriginatorId() + " votes " + numAgree + "/" + numDisagree); CachedUrlSetSpec cusSpec = null; if ((testmsg.getLwrBound() != null) && (testmsg.getLwrBound().equals(PollSpec.SINGLE_NODE_LWRBOUND))) { cusSpec = new SingleNodeCachedUrlSetSpec(testmsg.getTargetUrl()); } else { cusSpec = new RangeCachedUrlSetSpec( testmsg.getTargetUrl(), testmsg.getLwrBound(), testmsg.getUprBound()); } CachedUrlSet cus = au.makeCachedUrlSet(cusSpec); PollSpec spec = new PollSpec(cus, Poll.V1_CONTENT_POLL); ((MockCachedUrlSet) spec.getCachedUrlSet()).setHasContent(false); V1Poll p = null; if (testmsg.isContentPoll()) { p = new V1ContentPoll( spec, pollmanager, testmsg.getOriginatorId(), testmsg.getChallenge(), testmsg.getDuration(), testmsg.getHashAlgorithm()); } else if (testmsg.isNamePoll()) { p = new V1NamePoll( spec, pollmanager, testmsg.getOriginatorId(), testmsg.getChallenge(), testmsg.getDuration(), testmsg.getHashAlgorithm()); } else if (testmsg.isVerifyPoll()) { p = new V1VerifyPoll( spec, pollmanager, testmsg.getOriginatorId(), testmsg.getChallenge(), testmsg.getDuration(), testmsg.getHashAlgorithm(), testmsg.getVerifier()); } assertNotNull(p); p.setMessage(testmsg); p.m_tally.quorum = numAgree + numDisagree; p.m_tally.numAgree = numAgree; p.m_tally.numDisagree = numDisagree; p.m_tally.wtAgree = 2000; p.m_tally.wtDisagree = 200; p.m_tally.localEntries = makeEntries(1, 3); p.m_tally.votedEntries = makeEntries(1, 5); p.m_tally.votedEntries.remove(1); p.m_pollstate = V1Poll.PS_COMPLETE; p.m_callerID = testmsg.getOriginatorId(); log.debug3("poll " + p.toString()); p.m_tally.tallyVotes(); return p; }
public void setConfig( Configuration config, Configuration oldConfig, Configuration.Differences changedKeys) { // Build list of repositories from list of disk (fs) paths). Needs to // be generalized if ever another repository implementation. if (changedKeys.contains(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST)) { List lst = new ArrayList(); String dspace = config.get(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, ""); List paths = StringUtil.breakAt(dspace, ';'); if (paths != null) { for (Iterator iter = paths.iterator(); iter.hasNext(); ) { lst.add("local:" + (String) iter.next()); } } repoList = lst; } if (changedKeys.contains(PARAM_MAX_PER_AU_CACHE_SIZE)) { paramNodeCacheSize = config.getInt(PARAM_MAX_PER_AU_CACHE_SIZE, DEFAULT_MAX_PER_AU_CACHE_SIZE); for (Iterator iter = getDaemon().getAllLockssRepositories().iterator(); iter.hasNext(); ) { LockssRepository repo = (LockssRepository) iter.next(); if (repo instanceof LockssRepositoryImpl) { LockssRepositoryImpl repoImpl = (LockssRepositoryImpl) repo; repoImpl.setNodeCacheSize(paramNodeCacheSize); } } } if (changedKeys.contains(PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE)) { paramSuspectVersionsCacheSize = config.getInt( PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE, DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE); suspectVersionsCache.setMaxSize(paramSuspectVersionsCacheSize); } if (changedKeys.contains(GLOBAL_CACHE_PREFIX)) { paramIsGlobalNodeCache = config.getBoolean(PARAM_GLOBAL_CACHE_ENABLED, DEFAULT_GLOBAL_CACHE_ENABLED); if (paramIsGlobalNodeCache) { paramGlobalNodeCacheSize = config.getInt(PARAM_MAX_GLOBAL_CACHE_SIZE, DEFAULT_MAX_GLOBAL_CACHE_SIZE); log.debug("global node cache size: " + paramGlobalNodeCacheSize); globalNodeCache.setMaxSize(paramGlobalNodeCacheSize); } } if (changedKeys.contains(DISK_PREFIX)) { int minMB = config.getInt(PARAM_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_MB); double minPer = config.getPercentage(PARAM_DISK_WARN_FRRE_PERCENT, DEFAULT_DISK_WARN_FRRE_PERCENT); paramDFWarn = PlatformUtil.DF.makeThreshold(minMB, minPer); minMB = config.getInt(PARAM_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_MB); minPer = config.getPercentage(PARAM_DISK_FULL_FRRE_PERCENT, DEFAULT_DISK_FULL_FRRE_PERCENT); paramDFFull = PlatformUtil.DF.makeThreshold(minMB, minPer); } if (changedKeys.contains(PARAM_SIZE_CALC_MAX_LOAD)) { sizeCalcMaxLoad = config.getPercentage(PARAM_SIZE_CALC_MAX_LOAD, DEFAULT_SIZE_CALC_MAX_LOAD); } if (changedKeys.contains(PREFIX)) { maxUnusedDirSearch = config.getInt(PARAM_MAX_UNUSED_DIR_SEARCH, DEFAULT_MAX_UNUSED_DIR_SEARCH); isStatefulUnusedDirSearch = config.getBoolean( PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH, DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH); enableLongComponents = config.getBoolean(PARAM_ENABLE_LONG_COMPONENTS, DEFAULT_ENABLE_LONG_COMPONENTS); enableLongComponentsCompatibility = config.getBoolean( PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY, DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY); maxComponentLength = config.getInt(PARAM_MAX_COMPONENT_LENGTH, DEFAULT_MAX_COMPONENT_LENGTH); checkUnnormalized = (CheckUnnormalizedMode) config.getEnum( CheckUnnormalizedMode.class, PARAM_CHECK_UNNORMALIZED, DEFAULT_CHECK_UNNORMALIZED); } }
private void callV3ContentPoll(ArchivalUnit au) throws PollManager.NotEligibleException { log.debug("Enqueuing a V3 Content Poll on " + au.getName()); PollSpec spec = new PollSpec(au.getAuCachedUrlSet(), Poll.V3_POLL); pollManager.enqueueHighPriorityPoll(au, spec); statusMsg = "Enqueued V3 poll for " + au.getName(); }
protected void initMimeMap() throws PluginException.InvalidDefinition { for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) { Map.Entry ent = (Map.Entry) iter.next(); String key = (String) ent.getKey(); Object val = ent.getValue(); if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " link extractor: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); LinkExtractorFactory fact = (LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class); mti.setLinkExtractorFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) { // XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY // XXX unless/until that key is changed to not be a terminal substring // XXX of this one String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " crawl filter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class); mti.setCrawlFilterFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " filter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class); mti.setHashFilterFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT); if (val instanceof String) { String rate = (String) val; log.debug(mime + " fetch rate: " + rate); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); RateLimiter limit = mti.getFetchRateLimiter(); if (limit != null) { limit.setRate(rate); } else { mti.setFetchRateLimiter(new RateLimiter(rate)); } } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY); String factName = (String) val; log.debug(mime + " link rewriter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); LinkRewriterFactory fact = (LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class); mti.setLinkRewriterFactory(fact); } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP); Map factNameMap = (Map) val; Map factClassMap = new HashMap(); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) { String mdTypes = (String) it.next(); String factName = (String) factNameMap.get(mdTypes); log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName); for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) { setMdTypeFact(factClassMap, mdType, factName); } } mti.setFileMetadataExtractorFactoryMap(factClassMap); } } }