/** Create LockssKeystores from config subtree below {@link #PARAM_KEYSTORE} */
void configureKeyStores(Configuration config) {
Configuration allKs = config.getConfigTree(PARAM_KEYSTORE);
for (Iterator iter = allKs.nodeIterator(); iter.hasNext(); ) {
String id = (String) iter.next();
Configuration oneKs = allKs.getConfigTree(id);
try {
LockssKeyStore lk = createLockssKeyStore(oneKs);
String name = lk.getName();
if (name == null) {
log.error("KeyStore definition missing name: " + oneKs);
continue;
}
LockssKeyStore old = keystoreMap.get(name);
if (old != null && !lk.equals(old)) {
log.warning(
"Keystore "
+ name
+ " redefined. "
+ "New definition may not take effect until daemon restart");
}
log.debug("Adding keystore " + name);
keystoreMap.put(name, lk);
} catch (Exception e) {
log.error("Couldn't create keystore: " + oneKs, e);
}
}
}
public void export() {
log.debug("export(" + au.getName() + ")");
log.debug(
"dir: "
+ dir
+ ", pref: "
+ prefix
+ ", size: "
+ maxSize
+ ", ver: "
+ maxVersions
+ (compress ? ", (C)" : ""));
checkArgs();
try {
start();
} catch (IOException e) {
recordError("Error opening file", e);
return;
}
writeFiles();
try {
finish();
} catch (IOException e) {
if (!isDiskFull) {
// If we already knew (and reported) disk full, also reporting it
// as a close error is misleading.
recordError("Error closing file", e);
}
}
}
protected void initFeatureVersions() throws PluginException.InvalidDefinition {
if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
Map<String, String> spec =
(Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
log.debug2("features: " + spec);
for (Map.Entry<String, String> ent : spec.entrySet()) {
try {
// Prefix version string with feature name to create separate
// namespace for each feature
String key = ent.getKey();
map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
} catch (RuntimeException e) {
log.warning(
getPluginName()
+ " set unknown feature: "
+ ent.getKey()
+ " to version "
+ ent.getValue(),
e);
throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
}
}
featureVersion = map;
} else {
featureVersion = null;
}
}
private void initTestPolls() throws Exception {
testV1polls = new V1Poll[testV1msg.length];
for (int i = 0; i < testV1polls.length; i++) {
log.debug3("initTestPolls: V1 " + i);
BasePoll p = pollmanager.makePoll(testV1msg[i]);
assertNotNull(p);
assertNotNull(p.getMessage());
log.debug("initTestPolls: V1 " + i + " returns " + p);
assertTrue(p instanceof V1Poll);
switch (i) {
case 0:
assertTrue(p instanceof V1NamePoll);
break;
case 1:
assertTrue(p instanceof V1ContentPoll);
break;
case 2:
assertTrue(p instanceof V1VerifyPoll);
break;
}
testV1polls[i] = (V1Poll) p;
assertNotNull(testV1polls[i]);
log.debug3("initTestPolls: " + i + " " + p.toString());
}
}
public String getDefaultArticleMimeType() {
String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null);
log.debug3("DefaultArticleMimeType " + ret);
if (ret == null) {
ret = super.getDefaultArticleMimeType();
log.debug3("DefaultArticleMimeType from super " + ret);
}
return ret;
}
/** test for method scheduleVote(..) */
public void testScheduleVote() {
V1Poll p = testV1polls[1];
assertTrue(p instanceof V1ContentPoll);
log.debug3("testScheduleVote 1");
p.scheduleVote();
log.debug3("testScheduleVote 2");
assertNotNull(p.m_voteTime);
assertTrue(p.m_voteTime.getRemainingTime() < p.m_deadline.getRemainingTime());
log.debug3("at end of testScheduleVote");
}
void stopThread() {
if (sizeCalcThread != null) {
log.debug2("Stopping thread");
sizeCalcThread.stopSizeCalc();
sizeCalcThread = null;
}
}
protected void logParams() {
Enumeration en = req.getParameterNames();
while (en.hasMoreElements()) {
String name = (String) en.nextElement();
String vals[];
String dispval;
if (StringUtil.indexOfIgnoreCase(name, "passw") >= 0) {
dispval = req.getParameter(name).length() == 0 ? "" : "********";
} else if (log.isDebug2() && (vals = req.getParameterValues(name)).length > 1) {
dispval = StringUtil.separatedString(vals, ", ");
} else {
dispval = req.getParameter(name);
}
log.debug(name + " = " + dispval);
}
}
/** Create LockssKeystore from a config subtree */
LockssKeyStore createLockssKeyStore(Configuration config) {
log.debug2("Creating LockssKeyStore from config: " + config);
String name = config.get(KEYSTORE_PARAM_NAME);
LockssKeyStore lk = new LockssKeyStore(name);
String file = config.get(KEYSTORE_PARAM_FILE);
String resource = config.get(KEYSTORE_PARAM_RESOURCE);
String url = config.get(KEYSTORE_PARAM_URL);
if (!StringUtil.isNullString(file)) {
lk.setLocation(file, LocationType.File);
} else if (!StringUtil.isNullString(resource)) {
lk.setLocation(resource, LocationType.Resource);
} else if (!StringUtil.isNullString(url)) {
lk.setLocation(url, LocationType.Url);
}
lk.setType(config.get(KEYSTORE_PARAM_TYPE, defaultKeyStoreType));
lk.setProvider(config.get(KEYSTORE_PARAM_PROVIDER, defaultKeyStoreProvider));
lk.setPassword(config.get(KEYSTORE_PARAM_PASSWORD));
lk.setKeyPassword(config.get(KEYSTORE_PARAM_KEY_PASSWORD));
lk.setKeyPasswordFile(config.get(KEYSTORE_PARAM_KEY_PASSWORD_FILE));
lk.setMayCreate(config.getBoolean(KEYSTORE_PARAM_CREATE, DEFAULT_CREATE));
return lk;
}
private void checkSubstance(ArchivalUnit au) {
SubstanceChecker subChecker = new SubstanceChecker(au);
if (!subChecker.isEnabled()) {
errMsg = "No substance patterns defined for plugin.";
return;
}
AuState auState = AuUtil.getAuState(au);
SubstanceChecker.State oldState = auState.getSubstanceState();
SubstanceChecker.State newState = subChecker.findSubstance();
String chtxt = (newState == oldState ? "(unchanged)" : "(was " + oldState.toString() + ")");
switch (newState) {
case Unknown:
log.error("Shouldn't happen: SubstanceChecker returned Unknown");
errMsg = "Error in SubstanceChecker; see log.";
break;
case Yes:
statusMsg = "AU has substance " + chtxt + ": " + au.getName();
auState.setSubstanceState(SubstanceChecker.State.Yes);
break;
case No:
statusMsg = "AU has no substance " + chtxt + ": " + au.getName();
auState.setSubstanceState(SubstanceChecker.State.No);
break;
}
}
/** Export to an ARC file */
public class ArcExporter extends Exporter {
private static Logger log = Logger.getLogger("ArcExporter");
protected CIProperties arcProps = null;
String arcFilePrefix = "SimulatedCrawl";
AtomicInteger serialNo = new AtomicInteger(0);
ARCWriter aw;
boolean isResponse;
public ArcExporter(LockssDaemon daemon, ArchivalUnit au, boolean isResponse) {
super(daemon, au);
this.isResponse = isResponse;
}
protected void start() {
aw = makeARCWriter();
}
protected void finish() throws IOException {
aw.close();
}
private ARCWriter makeARCWriter() {
return new ARCWriter(
serialNo, ListUtil.list(dir), prefix, compress, maxSize >= 0 ? maxSize : Long.MAX_VALUE);
}
protected void writeCu(CachedUrl cu) throws IOException {
String url = cu.getUrl();
long contentSize = cu.getContentSize();
CIProperties props = cu.getProperties();
long fetchTime = Long.parseLong(props.getProperty(CachedUrl.PROPERTY_FETCH_TIME));
InputStream contentIn = cu.getUnfilteredInputStream();
try {
if (isResponse) {
String hdrString = getHttpResponseString(cu);
long size = contentSize + hdrString.length();
InputStream headerIn = new ReaderInputStream(new StringReader(hdrString));
InputStream concat = new SequenceInputStream(headerIn, contentIn);
try {
aw.write(xlateFilename(url), cu.getContentType(), getHostIp(), fetchTime, size, concat);
} finally {
IOUtil.safeClose(concat);
}
} else {
aw.write(
xlateFilename(url),
cu.getContentType(),
getHostIp(),
fetchTime,
cu.getContentSize(),
contentIn);
}
} finally {
AuUtil.safeRelease(cu);
}
}
}
/** engqueue a size calculation for the node */
public void queueSizeCalc(RepositoryNode node) {
synchronized (sizeCalcQueue) {
if (sizeCalcQueue.add(node)) {
log.debug2("Queue size calc: " + node);
startOrKickThread();
}
}
}
void startOrKickThread() {
if (sizeCalcThread == null) {
log.debug2("Starting thread");
sizeCalcThread = new SizeCalcThread();
sizeCalcThread.start();
sizeCalcThread.waitRunning();
}
sizeCalcSem.give();
}
protected String getHostIp() {
try {
IPAddr localHost = IPAddr.getLocalHost();
return localHost.getHostAddress();
} catch (UnknownHostException e) {
log.error("getHostIp()", e);
return "1.1.1.1";
}
}
/**
* Checks the consistency of the node, and continues with its children if it's consistent.
*
* @param node RepositoryNodeImpl the node to check
*/
private void recurseConsistencyCheck(RepositoryNodeImpl node) {
logger.debug2("Checking node '" + node.getNodeUrl() + "'...");
// check consistency at each node
// correct/deactivate as necessary
// 'checkNodeConsistency()' will repair if possible
if (node.checkNodeConsistency()) {
logger.debug3("Node consistent; recursing on children...");
List children = node.getNodeList(null, false);
Iterator iter = children.iterator();
while (iter.hasNext()) {
RepositoryNodeImpl child = (RepositoryNodeImpl) iter.next();
recurseConsistencyCheck(child);
}
} else {
logger.debug3("Node inconsistent; deactivating...");
deactivateInconsistentNode(node);
}
}
public PlatformUtil.DF getRepositoryDF(String repoName) {
String path = LockssRepositoryImpl.getLocalRepositoryPath(repoName);
log.debug("path: " + path);
// try {
return platInfo.getJavaDF(path);
// } catch (PlatformUtil.UnsupportedException e) {
// return null;
// }
}
private void forceV3Poll() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
callV3ContentPoll(au);
} catch (Exception e) {
log.error("Can't start poll", e);
errMsg = "Error: " + e.toString();
}
}
private void forceReindexMetadata() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
startReindexingMetadata(au, true);
} catch (RuntimeException e) {
log.error("Can't reindex metadata", e);
errMsg = "Error: " + e.toString();
}
}
private void doDisableMetadataIndexing() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
disableMetadataIndexing(au, false);
} catch (RuntimeException e) {
log.error("Can't disable metadata indexing", e);
errMsg = "Error: " + e.toString();
}
}
private void doCheckSubstance() {
ArchivalUnit au = getAu();
if (au == null) return;
try {
checkSubstance(au);
} catch (RuntimeException e) {
log.error("Error in SubstanceChecker", e);
errMsg = "Error in SubstanceChecker; see log.";
}
}
private boolean startReindexingMetadata(ArchivalUnit au, boolean force) {
if (metadataMgr == null) {
errMsg = "Metadata processing is not enabled.";
return false;
}
if (!force) {
if (!AuUtil.hasCrawled(au)) {
errMsg = "Au has never crawled. Click again to reindex metadata";
showForceReindexMetadata = true;
return false;
}
AuState auState = AuUtil.getAuState(au);
switch (auState.getSubstanceState()) {
case No:
errMsg = "Au has no substance. Click again to reindex metadata";
showForceReindexMetadata = true;
return false;
case Unknown:
errMsg = "Unknown substance for Au. Click again to reindex metadata.";
showForceReindexMetadata = true;
return false;
case Yes:
// fall through
}
}
// Fully reindex metadata with the highest priority.
Connection conn = null;
PreparedStatement insertPendingAuBatchStatement = null;
try {
conn = dbMgr.getConnection();
insertPendingAuBatchStatement = metadataMgr.getPrioritizedInsertPendingAuBatchStatement(conn);
if (metadataMgr.enableAndAddAuToReindex(
au, conn, insertPendingAuBatchStatement, false, true)) {
statusMsg = "Reindexing metadata for " + au.getName();
return true;
}
} catch (DbException dbe) {
log.error("Cannot reindex metadata for " + au.getName(), dbe);
} finally {
DbManager.safeCloseStatement(insertPendingAuBatchStatement);
DbManager.safeRollbackAndClose(conn);
}
if (force) {
errMsg = "Still cannot reindex metadata for " + au.getName();
} else {
errMsg = "Cannot reindex metadata for " + au.getName();
}
return false;
}
void loadKeyStores() {
List<LockssKeyStore> lst = new ArrayList<LockssKeyStore>(keystoreMap.values());
for (LockssKeyStore lk : lst) {
try {
lk.load();
} catch (Exception e) {
log.error("Can't load keystore " + lk.getName(), e);
keystoreMap.remove(lk.getName());
}
}
}
static void saveAuIdProperties(String location, Properties props) {
// XXX these AU_ID_FILE entries need to be backed up elsewhere to avoid
// single-point corruption
File propDir = new File(location);
if (!propDir.exists()) {
logger.debug("Creating directory '" + propDir.getAbsolutePath() + "'");
propDir.mkdirs();
}
File propFile = new File(propDir, AU_ID_FILE);
try {
logger.debug3("Saving au id properties at '" + location + "'.");
OutputStream os = new BufferedOutputStream(new FileOutputStream(propFile));
props.store(os, "ArchivalUnit id info");
os.close();
propFile.setReadOnly();
} catch (IOException ioe) {
logger.error("Couldn't write properties for " + propFile.getPath() + ".", ioe);
throw new LockssRepository.RepositoryStateException("Couldn't write au id properties file.");
}
}
static LocalRepository getLocalRepository(String repoRoot) {
synchronized (localRepositories) {
LocalRepository localRepo = (LocalRepository) localRepositories.get(repoRoot);
if (localRepo == null) {
logger.debug2("Creating LocalRepository(" + repoRoot + ")");
localRepo = new LocalRepository(repoRoot);
localRepositories.put(repoRoot, localRepo);
}
return localRepo;
}
}
static Properties getAuIdProperties(String location) {
File propFile = new File(location + File.separator + AU_ID_FILE);
try {
InputStream is = new BufferedInputStream(new FileInputStream(propFile));
Properties idProps = new Properties();
idProps.load(is);
is.close();
return idProps;
} catch (Exception e) {
logger.warning("Error loading au id from " + propFile.getPath() + ".");
return null;
}
}
String getLocalIPAddr() {
if (localAddr == null) {
try {
IPAddr localHost = IPAddr.getLocalHost();
localAddr = localHost.getHostAddress();
} catch (UnknownHostException e) {
// shouldn't happen
log.error("LockssServlet: getLocalHost: " + e.toString());
return "???";
}
}
return localAddr;
}
protected PermissionCheckerFactory getPermissionCheckerFactory() {
if (permissionCheckerFact == null) {
String permissionCheckerFactoryClass =
definitionMap.getString(DefinableArchivalUnit.KEY_AU_PERMISSION_CHECKER_FACTORY, null);
if (permissionCheckerFactoryClass != null) {
permissionCheckerFact =
(PermissionCheckerFactory)
newAuxClass(permissionCheckerFactoryClass, PermissionCheckerFactory.class);
log.debug2("Loaded PermissionCheckerFactory: " + permissionCheckerFact);
}
}
return permissionCheckerFact;
}
protected String getHostName() {
String res = ConfigManager.getPlatformHostname();
if (res == null) {
try {
InetAddress inet = InetAddress.getLocalHost();
return inet.getHostName();
} catch (UnknownHostException e) {
log.warning("Can't get hostname", e);
return "unknown";
}
}
return res;
}
/**
* This function returns a RepositoryNode with a canonicalized path.
*
* @param url the url in String form
* @param create true iff the node should be created if absent
* @return RepositoryNode the node
* @throws MalformedURLException
*/
private synchronized RepositoryNode getNode(String url, boolean create)
throws MalformedURLException {
String canonUrl;
boolean isAuUrl = false;
if (AuUrl.isAuUrl(url)) {
// path information is lost here, but is unimportant if it's an AuUrl
canonUrl = AuUrl.PROTOCOL;
isAuUrl = true;
} else {
// create a canonical path, handling all illegal path traversal
canonUrl = canonicalizePath(url);
}
// check LRUMap cache for node
RepositoryNode node = (RepositoryNode) nodeCache.get(nodeCacheKey(canonUrl));
if (node != null) {
return node;
}
String nodeLocation;
if (isAuUrl) {
// base directory of ArchivalUnit
nodeLocation = rootLocation;
node = new AuNodeImpl(canonUrl, nodeLocation, this);
} else {
// determine proper node location
nodeLocation =
LockssRepositoryImpl.mapUrlToFileLocation(rootLocation, canonUrl)
.replace("?", "")
.replace("COM8", "COMEIGHT")
.replace("%5c", "/"); // //windows folder structure fix
node = new RepositoryNodeImpl(canonUrl, nodeLocation, this);
}
if (!create) {
// if not creating, check for existence
File nodeDir = new File(nodeLocation);
if (!nodeDir.exists()) {
// return null if the node doesn't exist and shouldn't be created
return null;
}
if (!nodeDir.isDirectory()) {
logger.error("Cache file not a directory: " + nodeLocation);
throw new LockssRepository.RepositoryStateException("Invalid cache file.");
}
}
// add to node cache
nodeCache.put(nodeCacheKey(canonUrl), node);
return node;
}
/** test for method checkVote(..) */
public void testCheckVote() throws Exception {
V1LcapMessage msg = null;
log.debug3("starting testCheeckVote");
msg =
V1LcapMessage.makeReplyMsg(
testV1polls[0].getMessage(),
ByteArray.makeRandomBytes(20),
ByteArray.makeRandomBytes(20),
null,
V1LcapMessage.NAME_POLL_REP,
testduration,
testID);
log.debug3("testCheeckVote 2");
V1Poll p = null;
p = createCompletedPoll(theDaemon, testau, msg, 8, 2, pollmanager);
assertTrue(p instanceof V1NamePoll);
log.debug3("testCheeckVote 3");
assertNotNull(p);
PeerIdentity id = msg.getOriginatorId();
assertNotNull(id);
assertNotNull(p.m_tally);
int rep = p.m_tally.wtAgree + idmgr.getReputation(id);
// good vote check
p.checkVote(msg.getHashed(), new Vote(msg, false));
assertEquals(9, p.m_tally.numAgree);
assertEquals(2, p.m_tally.numDisagree);
assertEquals(rep, p.m_tally.wtAgree);
rep = p.m_tally.wtDisagree + idmgr.getReputation(id);
// bad vote check
p.checkVote(ByteArray.makeRandomBytes(20), new Vote(msg, false));
assertEquals(9, p.m_tally.numAgree);
assertEquals(3, p.m_tally.numDisagree);
assertEquals(rep, p.m_tally.wtDisagree);
}
