public PlatformUtil.DF getRepositoryDF(String repoName) {
String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
log.debug("path: " + path);
// try {
return platInfo.getJavaDF(path);
// } catch (PlatformUtil.UnsupportedException e) {
// return null;
// }
}
private void writeFiles() {
PlatformUtil platutil = PlatformUtil.getInstance();
CuIterator iter = AuUtil.getCuIterator(au);
int errs = 0;
CachedUrl curCu = null;
CachedUrl nextCu = getNextCu(iter);
while (nextCu != null) {
curCu = nextCu;
nextCu = getNextCu(iter);
if (excludeDirNodes && nextCu != null && isDirOf(curCu, nextCu)) {
continue;
}
CachedUrl[] cuVersions =
curCu.getCuVersions(maxVersions > 0 ? maxVersions : Integer.MAX_VALUE);
for (CachedUrl cu : cuVersions) {
try {
log.debug2("Exporting " + cu.getUrl());
writeCu(cu);
} catch (IOException e) {
if (platutil.isDiskFullError(e)) {
recordError("Disk full, can't write export file.");
isDiskFull = true;
return;
}
} catch (Exception e) {
// XXX Would like to differentiate between errors opening or
// reading CU, which shouldn't cause abort, and errors writing
// to export file, which should.
recordError("Unable to copy " + cu.getUrl(), e);
if (errs++ >= maxErrors) {
recordError("Aborting after " + errs + " errors");
return;
}
}
}
}
}
String getMachineName() {
return PlatformUtil.getLocalHostname();
}
/**
* RepositoryManager is the center of the per AU repositories. It manages the repository config
* parameters.
*/
public class RepositoryManager extends BaseLockssDaemonManager implements ConfigurableManager {
private static Logger log = Logger.getLogger("RepositoryManager");
public static final String PREFIX = Configuration.PREFIX + "repository.";
/** Maximum size of per-AU repository node cache */
public static final String PARAM_MAX_PER_AU_CACHE_SIZE = PREFIX + "nodeCache.size";
public static final int DEFAULT_MAX_PER_AU_CACHE_SIZE = 10;
/*
* This needs to be a small multiple of the number of simultaneous
* polls (poller and voter), as there is a cache entry per active AU.
* Each poll will have one active AU at a time.
*/
public static final String PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE =
PREFIX + "suspectVersionsCache.size";
public static final int DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE = 10;
static final String GLOBAL_CACHE_PREFIX = PREFIX + "globalNodeCache.";
public static final String PARAM_MAX_GLOBAL_CACHE_SIZE = GLOBAL_CACHE_PREFIX + "size";
public static final int DEFAULT_MAX_GLOBAL_CACHE_SIZE = 500;
public static final String PARAM_GLOBAL_CACHE_ENABLED = GLOBAL_CACHE_PREFIX + "enabled";
public static final boolean DEFAULT_GLOBAL_CACHE_ENABLED = false;
/** Max times to loop looking for unused AU directory. */
public static final String PARAM_MAX_UNUSED_DIR_SEARCH = PREFIX + "maxUnusedDirSearch";
public static final int DEFAULT_MAX_UNUSED_DIR_SEARCH = 30000;
/** If true, LocalRepository keeps track of next subdir name to probe. */
public static final String PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH =
PREFIX + "enableStatefulUnusedDirSearch";
public static final boolean DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH = true;
/** Max percent of time size calculation thread may run. */
public static final String PARAM_SIZE_CALC_MAX_LOAD = PREFIX + "sizeCalcMaxLoad";
public static final float DEFAULT_SIZE_CALC_MAX_LOAD = 0.5F;
/**
* If true, path components longer than the maximum filesystem path component are encodes are
* multiple levels of directories
*/
public static final String PARAM_ENABLE_LONG_COMPONENTS = PREFIX + "enableLongComponents";
public static final boolean DEFAULT_ENABLE_LONG_COMPONENTS = true;
/**
* Prior to 1.61.6, when long component support is enabled, backslahes were normalized to %5c
* instead of %5C. This is harmless if checkUnnormalized is set to Fix, as they'll be normalized.
* Otherwise, this can be set true for compatibility with old repositories. Setting
* checkUnnormalized to Fix is preferred.
*/
public static final String PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY =
PREFIX + "enableLongComponentsCompatibility";
public static final boolean DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY = false;
/** Maximum length of a filesystem path component. */
public static final String PARAM_MAX_COMPONENT_LENGTH = PREFIX + "maxComponentLength";
public static final int DEFAULT_MAX_COMPONENT_LENGTH = 255;
/** @see #PARAM_CHECK_UNNORMALIZED */
public enum CheckUnnormalizedMode {
No,
Log,
Fix
};
/**
* Check for existing nodes with unnormalized names (created by very old daemon that didn't
* normalize): None, Log, Fix
*/
public static final String PARAM_CHECK_UNNORMALIZED = PREFIX + "checkUnnormalized";
public static final CheckUnnormalizedMode DEFAULT_CHECK_UNNORMALIZED = CheckUnnormalizedMode.Log;
static final String WDOG_PARAM_SIZE_CALC = "SizeCalc";
static final long WDOG_DEFAULT_SIZE_CALC = Constants.DAY;
static final String PRIORITY_PARAM_SIZE_CALC = "SizeCalc";
static final int PRIORITY_DEFAULT_SIZE_CALC = Thread.NORM_PRIORITY - 1;
static final String DISK_PREFIX = PREFIX + "diskSpace.";
static final String PARAM_DISK_WARN_FRRE_MB = DISK_PREFIX + "warn.freeMB";
static final int DEFAULT_DISK_WARN_FRRE_MB = 5000;
static final String PARAM_DISK_FULL_FRRE_MB = DISK_PREFIX + "full.freeMB";
static final int DEFAULT_DISK_FULL_FRRE_MB = 100;
static final String PARAM_DISK_WARN_FRRE_PERCENT = DISK_PREFIX + "warn.freePercent";
static final double DEFAULT_DISK_WARN_FRRE_PERCENT = .02;
static final String PARAM_DISK_FULL_FRRE_PERCENT = DISK_PREFIX + "full.freePercent";
static final double DEFAULT_DISK_FULL_FRRE_PERCENT = .01;
private PlatformUtil platInfo = PlatformUtil.getInstance();
private List repoList = Collections.EMPTY_LIST;
int paramNodeCacheSize = DEFAULT_MAX_PER_AU_CACHE_SIZE;
boolean paramIsGlobalNodeCache = DEFAULT_GLOBAL_CACHE_ENABLED;
int paramGlobalNodeCacheSize = DEFAULT_MAX_GLOBAL_CACHE_SIZE;
int paramSuspectVersionsCacheSize = DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE;
UniqueRefLruCache globalNodeCache = new UniqueRefLruCache(DEFAULT_MAX_GLOBAL_CACHE_SIZE);
UniqueRefLruCache suspectVersionsCache =
new UniqueRefLruCache(DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
Map localRepos = new HashMap();
private static int maxUnusedDirSearch = DEFAULT_MAX_UNUSED_DIR_SEARCH;
private static boolean isStatefulUnusedDirSearch = DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH;
private static boolean enableLongComponents = DEFAULT_ENABLE_LONG_COMPONENTS;
private static boolean enableLongComponentsCompatibility =
DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY;
private static int maxComponentLength = DEFAULT_MAX_COMPONENT_LENGTH;
private static CheckUnnormalizedMode checkUnnormalized = DEFAULT_CHECK_UNNORMALIZED;
PlatformUtil.DF paramDFWarn =
PlatformUtil.DF.makeThreshold(DEFAULT_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_PERCENT);
PlatformUtil.DF paramDFFull =
PlatformUtil.DF.makeThreshold(DEFAULT_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_PERCENT);
private float sizeCalcMaxLoad = DEFAULT_SIZE_CALC_MAX_LOAD;
public void startService() {
super.startService();
localRepos = new HashMap();
}
public void setConfig(
Configuration config, Configuration oldConfig, Configuration.Differences changedKeys) {
// Build list of repositories from list of disk (fs) paths). Needs to
// be generalized if ever another repository implementation.
if (changedKeys.contains(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST)) {
List lst = new ArrayList();
String dspace = config.get(ConfigManager.PARAM_PLATFORM_DISK_SPACE_LIST, "");
List paths = StringUtil.breakAt(dspace, ';');
if (paths != null) {
for (Iterator iter = paths.iterator(); iter.hasNext(); ) {
lst.add("local:" + (String) iter.next());
}
}
repoList = lst;
}
if (changedKeys.contains(PARAM_MAX_PER_AU_CACHE_SIZE)) {
paramNodeCacheSize =
config.getInt(PARAM_MAX_PER_AU_CACHE_SIZE, DEFAULT_MAX_PER_AU_CACHE_SIZE);
for (Iterator iter = getDaemon().getAllLockssRepositories().iterator(); iter.hasNext(); ) {
LockssRepository repo = (LockssRepository) iter.next();
if (repo instanceof LockssRepositoryImpl) {
LockssRepositoryImpl repoImpl = (LockssRepositoryImpl) repo;
repoImpl.setNodeCacheSize(paramNodeCacheSize);
}
}
}
if (changedKeys.contains(PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE)) {
paramSuspectVersionsCacheSize =
config.getInt(
PARAM_MAX_SUSPECT_VERSIONS_CACHE_SIZE, DEFAULT_MAX_SUSPECT_VERSIONS_CACHE_SIZE);
suspectVersionsCache.setMaxSize(paramSuspectVersionsCacheSize);
}
if (changedKeys.contains(GLOBAL_CACHE_PREFIX)) {
paramIsGlobalNodeCache =
config.getBoolean(PARAM_GLOBAL_CACHE_ENABLED, DEFAULT_GLOBAL_CACHE_ENABLED);
if (paramIsGlobalNodeCache) {
paramGlobalNodeCacheSize =
config.getInt(PARAM_MAX_GLOBAL_CACHE_SIZE, DEFAULT_MAX_GLOBAL_CACHE_SIZE);
log.debug("global node cache size: " + paramGlobalNodeCacheSize);
globalNodeCache.setMaxSize(paramGlobalNodeCacheSize);
}
}
if (changedKeys.contains(DISK_PREFIX)) {
int minMB = config.getInt(PARAM_DISK_WARN_FRRE_MB, DEFAULT_DISK_WARN_FRRE_MB);
double minPer =
config.getPercentage(PARAM_DISK_WARN_FRRE_PERCENT, DEFAULT_DISK_WARN_FRRE_PERCENT);
paramDFWarn = PlatformUtil.DF.makeThreshold(minMB, minPer);
minMB = config.getInt(PARAM_DISK_FULL_FRRE_MB, DEFAULT_DISK_FULL_FRRE_MB);
minPer = config.getPercentage(PARAM_DISK_FULL_FRRE_PERCENT, DEFAULT_DISK_FULL_FRRE_PERCENT);
paramDFFull = PlatformUtil.DF.makeThreshold(minMB, minPer);
}
if (changedKeys.contains(PARAM_SIZE_CALC_MAX_LOAD)) {
sizeCalcMaxLoad = config.getPercentage(PARAM_SIZE_CALC_MAX_LOAD, DEFAULT_SIZE_CALC_MAX_LOAD);
}
if (changedKeys.contains(PREFIX)) {
maxUnusedDirSearch =
config.getInt(PARAM_MAX_UNUSED_DIR_SEARCH, DEFAULT_MAX_UNUSED_DIR_SEARCH);
isStatefulUnusedDirSearch =
config.getBoolean(
PARAM_IS_STATEFUL_UNUSED_DIR_SEARCH, DEFAULT_IS_STATEFUL_UNUSED_DIR_SEARCH);
enableLongComponents =
config.getBoolean(PARAM_ENABLE_LONG_COMPONENTS, DEFAULT_ENABLE_LONG_COMPONENTS);
enableLongComponentsCompatibility =
config.getBoolean(
PARAM_ENABLE_LONG_COMPONENTS_COMPATIBILITY,
DEFAULT_ENABLE_LONG_COMPONENTS_COMPATIBILITY);
maxComponentLength = config.getInt(PARAM_MAX_COMPONENT_LENGTH, DEFAULT_MAX_COMPONENT_LENGTH);
checkUnnormalized =
(CheckUnnormalizedMode)
config.getEnum(
CheckUnnormalizedMode.class,
PARAM_CHECK_UNNORMALIZED,
DEFAULT_CHECK_UNNORMALIZED);
}
}
public static boolean isEnableLongComponents() {
return enableLongComponents;
}
public static boolean isEnableLongComponentsCompatibility() {
return enableLongComponentsCompatibility;
}
public static int getMaxComponentLength() {
return maxComponentLength;
}
public static CheckUnnormalizedMode getCheckUnnormalizedMode() {
return checkUnnormalized;
}
/**
* Return list of known repository names. Needs a registration mechanism if ever another
* repository implementation.
*/
public List<String> getRepositoryList() {
return repoList;
}
public PlatformUtil.DF getRepositoryDF(String repoName) {
String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
log.debug("path: " + path);
// try {
return platInfo.getJavaDF(path);
// } catch (PlatformUtil.UnsupportedException e) {
// return null;
// }
}
public Map<String, PlatformUtil.DF> getRepositoryMap() {
Map<String, PlatformUtil.DF> repoMap = new LinkedMap();
for (String repo : getRepositoryList()) {
repoMap.put(repo, getRepositoryDF(repo));
}
return repoMap;
}
public String findLeastFullRepository() {
return findLeastFullRepository(getRepositoryMap());
}
public String findLeastFullRepository(Map<String, PlatformUtil.DF> repoMap) {
String mostFree = null;
for (String repo : repoMap.keySet()) {
PlatformUtil.DF df = repoMap.get(repo);
if (df != null) {
if (mostFree == null || (repoMap.get(mostFree)).getAvail() < df.getAvail()) {
mostFree = repo;
}
}
}
return mostFree;
}
public PlatformUtil.DF getDiskWarnThreshold() {
return paramDFWarn;
}
public PlatformUtil.DF getDiskFullThreshold() {
return paramDFFull;
}
public static int getMaxUnusedDirSearch() {
return maxUnusedDirSearch;
}
public static boolean isStatefulUnusedDirSearch() {
return isStatefulUnusedDirSearch;
}
public List findExistingRepositoriesFor(String auid) {
List res = null;
for (Iterator iter = getRepositoryList().iterator(); iter.hasNext(); ) {
String repoName = (String) iter.next();
String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
if (LockssRepositoryImpl.doesAuDirExist(auid, path)) {
if (res == null) {
res = new ArrayList();
}
res.add(repoName);
}
}
return res == null ? Collections.EMPTY_LIST : res;
}
// hack only local
public synchronized LockssRepositoryImpl getRepositoryFromPath(String path) {
LockssRepositoryImpl repo = (LockssRepositoryImpl) localRepos.get(path);
if (repo == null) {
repo = new LockssRepositoryImpl(path);
repo.initService(getDaemon());
repo.startService();
localRepos.put(path, repo);
}
return repo;
}
/**
* Return the disk space used by the AU, including all overhead, optionally calculating it if
* necessary.
*
* @param repoAuPath the full path to an AU dir in a LockssRepositoryImpl
* @param calcIfUnknown if true, size will calculated if unknown (time consumeing)
* @return the AU's disk usage in bytes, or -1 if unknown
*/
public long getRepoDiskUsage(String repoAuPath, boolean calcIfUnknown) {
LockssRepository repo = getRepositoryFromPath(repoAuPath);
if (repo != null) {
try {
RepositoryNode repoNode = repo.getNode(AuCachedUrlSetSpec.URL);
if (repoNode instanceof AuNodeImpl) {
return ((AuNodeImpl) repoNode).getDiskUsage(calcIfUnknown);
}
} catch (MalformedURLException ignore) {
}
}
return -1;
}
public synchronized void setRepositoryForPath(String path, LockssRepositoryImpl repo) {
localRepos.put(path, repo);
}
public boolean isGlobalNodeCache() {
return paramIsGlobalNodeCache;
}
public UniqueRefLruCache getGlobalNodeCache() {
return globalNodeCache;
}
public UniqueRefLruCache getSuspectVersionsCache() {
return suspectVersionsCache;
}
// Background thread to (re)calculate AU size and disk usage.
private Set sizeCalcQueue = new HashSet();
private BinarySemaphore sizeCalcSem = new BinarySemaphore();
private SizeCalcThread sizeCalcThread;
/** engqueue a size calculation for the AU */
public void queueSizeCalc(ArchivalUnit au) {
queueSizeCalc(AuUtil.getAuRepoNode(au));
}
/** engqueue a size calculation for the node */
public void queueSizeCalc(RepositoryNode node) {
synchronized (sizeCalcQueue) {
if (sizeCalcQueue.add(node)) {
log.debug2("Queue size calc: " + node);
startOrKickThread();
}
}
}
public int sizeCalcQueueLen() {
synchronized (sizeCalcQueue) {
return sizeCalcQueue.size();
}
}
void startOrKickThread() {
if (sizeCalcThread == null) {
log.debug2("Starting thread");
sizeCalcThread = new SizeCalcThread();
sizeCalcThread.start();
sizeCalcThread.waitRunning();
}
sizeCalcSem.give();
}
void stopThread() {
if (sizeCalcThread != null) {
log.debug2("Stopping thread");
sizeCalcThread.stopSizeCalc();
sizeCalcThread = null;
}
}
void doSizeCalc(RepositoryNode node) {
node.getTreeContentSize(null, true);
if (node instanceof AuNodeImpl) {
((AuNodeImpl) node).getDiskUsage(true);
}
}
long sleepTimeToAchieveLoad(long runDuration, float maxLoad) {
return Math.round(((double) runDuration / maxLoad) - runDuration);
}
private class SizeCalcThread extends LockssThread {
private volatile boolean goOn = true;
private SizeCalcThread() {
super("SizeCalc");
}
public void lockssRun() {
setPriority(PRIORITY_PARAM_SIZE_CALC, PRIORITY_DEFAULT_SIZE_CALC);
startWDog(WDOG_PARAM_SIZE_CALC, WDOG_DEFAULT_SIZE_CALC);
triggerWDogOnExit(true);
nowRunning();
while (goOn) {
try {
pokeWDog();
if (sizeCalcQueue.isEmpty()) {
Deadline timeout = Deadline.in(Constants.HOUR);
sizeCalcSem.take(timeout);
}
RepositoryNode node;
synchronized (sizeCalcQueue) {
node = (RepositoryNode) CollectionUtil.getAnElement(sizeCalcQueue);
}
if (node != null) {
long start = TimeBase.nowMs();
log.debug2("CalcSize start: " + node);
long dur = 0;
try {
doSizeCalc(node);
dur = TimeBase.nowMs() - start;
log.debug2("CalcSize finish (" + StringUtil.timeIntervalToString(dur) + "): " + node);
} catch (RuntimeException e) {
log.warning("doSizeCalc: " + node, e);
}
synchronized (sizeCalcQueue) {
sizeCalcQueue.remove(node);
}
pokeWDog();
long sleep = sleepTimeToAchieveLoad(dur, sizeCalcMaxLoad);
Deadline.in(sleep).sleep();
}
} catch (InterruptedException e) {
// just wakeup and check for exit
}
}
if (!goOn) {
triggerWDogOnExit(false);
}
}
private void stopSizeCalc() {
goOn = false;
interrupt();
}
}
}
