protected void initAuFeatureMap() {
if (definitionMap.containsKey(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP)) {
Map<String, ?> featMap = definitionMap.getMap(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP);
for (Map.Entry ent : featMap.entrySet()) {
Object val = ent.getValue();
if (val instanceof Map) {
ent.setValue(MapUtil.expandAlternativeKeyLists((Map) val));
}
}
}
}
protected Properties filterResponseProps(Properties props) {
Properties res = new Properties();
for (Map.Entry ent : props.entrySet()) {
String key = (String) ent.getKey();
if (StringUtil.startsWithIgnoreCase(key, "x-lockss")
|| StringUtil.startsWithIgnoreCase(key, "x_lockss")
|| key.equalsIgnoreCase("org.lockss.version.number")) {
continue;
}
// We've lost the original case - capitalize them the way most people
// expect
res.put(StringUtil.titleCase(key, '-'), (String) ent.getValue());
}
return res;
}
/** If in testing mode FOO, copy values from FOO_override map, if any, to main map */
void processOverrides(TypedEntryMap map) {
String testMode = getTestingMode();
if (StringUtil.isNullString(testMode)) {
return;
}
Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE);
if (o == null) {
return;
}
if (o instanceof Map) {
Map overrideMap = (Map) o;
for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) {
String key = (String) entry.getKey();
Object val = entry.getValue();
log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val);
map.setMapElement(key, val);
}
}
}
protected void initFeatureVersions() throws PluginException.InvalidDefinition {
if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
Map<String, String> spec =
(Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
log.debug2("features: " + spec);
for (Map.Entry<String, String> ent : spec.entrySet()) {
try {
// Prefix version string with feature name to create separate
// namespace for each feature
String key = ent.getKey();
map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
} catch (RuntimeException e) {
log.warning(
getPluginName()
+ " set unknown feature: "
+ ent.getKey()
+ " to version "
+ ent.getValue(),
e);
throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
}
}
featureVersion = map;
} else {
featureVersion = null;
}
}
private ExternalizableMap loadMap(String extMapName, ClassLoader loader)
throws FileNotFoundException {
String first = null;
String next = extMapName;
List<String> urls = new ArrayList<String>();
ExternalizableMap res = null;
while (next != null) {
// convert the plugin class name to an xml file name
String mapFile = next.replace('.', '/') + MAP_SUFFIX;
URL url = loader.getResource(mapFile);
if (url != null && urls.contains(url.toString())) {
throw new PluginException.InvalidDefinition("Plugin inheritance loop: " + next);
}
// load into map
ExternalizableMap oneMap = new ExternalizableMap();
oneMap.loadMapFromResource(mapFile, loader);
urls.add(url.toString());
// apply overrides one plugin at a time in inheritance chain
processOverrides(oneMap);
if (res == null) {
res = oneMap;
} else {
for (Map.Entry ent : oneMap.entrySet()) {
String key = (String) ent.getKey();
Object val = ent.getValue();
if (!res.containsKey(key)) {
res.setMapElement(key, val);
}
}
}
if (oneMap.containsKey(KEY_PLUGIN_PARENT)) {
next = oneMap.getString(KEY_PLUGIN_PARENT);
} else {
next = null;
}
}
loadedFromUrls = urls;
return res;
}
/**
* Add a TdbAu to a TdbTitle and TdbPubisher, and add links to the TdbTitle specified by the
* properties.
*
* @param props the properties
* @param au the TdbAu to add
* @throws TdbException if the AU already exists in this Tdb
*/
private void addTdbAu(Properties props, TdbAu au) throws TdbException {
// add au for plugin assuming it is not a duplicate
if (!addTdbAuForPlugin(au)) {
// au already registered -- report existing au
TdbAu existingAu = findExistingTdbAu(au);
String titleName = getTdbTitleName(props, au);
if (!titleName.equals(existingAu.getTdbTitle().getName())) {
throw new TdbException(
"Cannot add duplicate au entry: \""
+ au.getName()
+ "\" for title \""
+ titleName
+ "\" with same definition as existing au entry: \""
+ existingAu.getName()
+ "\" for title \""
+ existingAu.getTdbTitle().getName()
+ "\" to title database");
} else if (!existingAu.getName().equals(au.getName())) {
// error because it could lead to a missing AU -- one probably has a typo
throw new TdbException(
"Cannot add duplicate au entry: \""
+ au.getName()
+ "\" with the same definition as \""
+ existingAu.getName()
+ "\" for title \""
+ titleName
+ "\" to title database");
} else {
throw new TdbException(
"Cannot add duplicate au entry: \""
+ au.getName()
+ "\" for title \""
+ titleName
+ "\" to title database");
}
}
// get or create the TdbTitle for this
TdbTitle title = getTdbTitle(props, au);
try {
// add AU to title
title.addTdbAu(au);
} catch (TdbException ex) {
// if we can't add au to title, remove for plugin and re-throw exception
removeTdbAuForPlugin(au);
throw ex;
}
// process title links
Map<String, Map<String, String>> linkMap = new HashMap<String, Map<String, String>>();
for (Map.Entry<Object, Object> entry : props.entrySet()) {
String key = "" + entry.getKey();
String value = "" + entry.getValue();
if (key.startsWith("journal.link.")) {
// skip to link name
String param = key.substring("link.".length());
int i;
if (((i = param.indexOf(".type")) < 0) && ((i = param.indexOf(".journalId")) < 0)) {
logger.warning(
"Ignoring nexpected link key for au \"" + au.getName() + "\" key: \"" + key + "\"");
} else {
// get link map for linkName
String lname = param.substring(0, i);
Map<String, String> lmap = linkMap.get(lname);
if (lmap == null) {
lmap = new HashMap<String, String>();
linkMap.put(lname, lmap);
}
// add name and value to link map for link
String name = param.substring(i + 1);
lmap.put(name, value);
}
}
}
// add links to title from accumulated "type", "journalId" entries
for (Map<String, String> lmap : linkMap.values()) {
String name = lmap.get("type");
String value = lmap.get("journalId");
if ((name != null) && (value != null)) {
try {
TdbTitle.LinkType linkType = TdbTitle.LinkType.valueOf(name);
title.addLinkToTdbTitleId(linkType, value);
} catch (IllegalArgumentException ex) {
logger.warning(
"Ignoring unknown link type for au \"" + au.getName() + "\" name: \"" + name + "\"");
}
}
}
}
/**
* Create a new TdbAu instance from the properties.
*
* @param props the properties
* @return a TdbAu instance set built from the properties
*/
private TdbAu newTdbAu(Properties props) {
String pluginId = (String) props.get("plugin");
if (pluginId == null) {
throw new IllegalArgumentException("TdbAu plugin ID not specified");
}
String auName = props.getProperty("title");
if (auName == null) {
throw new IllegalArgumentException("TdbAu title not specified");
}
// create a new TdbAu and set its elements
TdbAu au = new TdbAu(auName, pluginId);
// process attrs, and params
Map<String, Map<String, String>> paramMap = new HashMap<String, Map<String, String>>();
for (Map.Entry<Object, Object> entry : props.entrySet()) {
String key = String.valueOf(entry.getKey());
String value = String.valueOf(entry.getValue());
if (key.startsWith("attributes.")) {
// set attributes directly
String name = key.substring("attributes.".length());
try {
au.setAttr(name, value);
} catch (TdbException ex) {
logger.warning(
"Cannot set attribute \"" + name + "\" with value \"" + value + "\" -- ignoring");
}
} else if (key.startsWith("param.")) {
// skip to param name
String param = key.substring("param.".length());
int i;
if (((i = param.indexOf(".key")) < 0) && ((i = param.indexOf(".value")) < 0)) {
logger.warning(
"Ignoring unexpected param key for au \""
+ auName
+ "\" key: \""
+ key
+ "\" -- ignoring");
} else {
// get param map for pname
String pname = param.substring(0, i);
Map<String, String> pmap = paramMap.get(pname);
if (pmap == null) {
pmap = new HashMap<String, String>();
paramMap.put(pname, pmap);
}
// add name and value to param map for pname
String name = param.substring(i + 1);
pmap.put(name, value);
}
} else if (!key.equals("title") // TdbAu has "name" property
&& !key.equals("plugin") // TdbAu has "pluginId" property
&& !key.equals("journalTitle") // TdbAu has "title" TdbTitle property
&& !key.startsWith("journal.")) { // TdbAu has "title" TdbTitle property
// translate all other properties into AU properties
try {
au.setPropertyByName(key, value);
} catch (TdbException ex) {
logger.warning(
"Cannot set property \"" + key + "\" with value \"" + value + "\" -- ignoring");
}
}
}
// set param from accumulated "key", and "value" entries
for (Map<String, String> pmap : paramMap.values()) {
String name = pmap.get("key");
String value = pmap.get("value");
if (name == null) {
logger.warning("Ignoring property with null name");
} else if (value == null) {
logger.warning("Ignoring property \"" + name + "\" with null value");
} else {
try {
au.setParam(name, value);
} catch (TdbException ex) {
logger.warning(
"Cannot set param \"" + name + "\" with value \"" + value + "\" -- ignoring");
}
}
}
return au;
}
protected void initMimeMap() throws PluginException.InvalidDefinition {
for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry ent = (Map.Entry) iter.next();
String key = (String) ent.getKey();
Object val = ent.getValue();
if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " link extractor: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
LinkExtractorFactory fact =
(LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class);
mti.setLinkExtractorFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) {
// XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY
// XXX unless/until that key is changed to not be a terminal substring
// XXX of this one
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " crawl filter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
mti.setCrawlFilterFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " filter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
mti.setHashFilterFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT);
if (val instanceof String) {
String rate = (String) val;
log.debug(mime + " fetch rate: " + rate);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
RateLimiter limit = mti.getFetchRateLimiter();
if (limit != null) {
limit.setRate(rate);
} else {
mti.setFetchRateLimiter(new RateLimiter(rate));
}
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY);
String factName = (String) val;
log.debug(mime + " link rewriter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
LinkRewriterFactory fact =
(LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class);
mti.setLinkRewriterFactory(fact);
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP);
Map factNameMap = (Map) val;
Map factClassMap = new HashMap();
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) {
String mdTypes = (String) it.next();
String factName = (String) factNameMap.get(mdTypes);
log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName);
for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) {
setMdTypeFact(factClassMap, mdType, factName);
}
}
mti.setFileMetadataExtractorFactoryMap(factClassMap);
}
}
}
void checkParamAgreement() {
for (Map.Entry<String, PrintfContext> ent :
DefinableArchivalUnit.printfKeysContext.entrySet()) {
checkParamAgreement(ent.getKey(), ent.getValue());
}
}
