protected void initFeatureVersions() throws PluginException.InvalidDefinition { if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) { Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>(); Map<String, String> spec = (Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP); log.debug2("features: " + spec); for (Map.Entry<String, String> ent : spec.entrySet()) { try { // Prefix version string with feature name to create separate // namespace for each feature String key = ent.getKey(); map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue()); } catch (RuntimeException e) { log.warning( getPluginName() + " set unknown feature: " + ent.getKey() + " to version " + ent.getValue(), e); throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e); } } featureVersion = map; } else { featureVersion = null; } }
public String getDefaultArticleMimeType() { String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null); log.debug3("DefaultArticleMimeType " + ret); if (ret == null) { ret = super.getDefaultArticleMimeType(); log.debug3("DefaultArticleMimeType from super " + ret); } return ret; }
protected PermissionCheckerFactory getPermissionCheckerFactory() { if (permissionCheckerFact == null) { String permissionCheckerFactoryClass = definitionMap.getString(DefinableArchivalUnit.KEY_AU_PERMISSION_CHECKER_FACTORY, null); if (permissionCheckerFactoryClass != null) { permissionCheckerFact = (PermissionCheckerFactory) newAuxClass(permissionCheckerFactoryClass, PermissionCheckerFactory.class); log.debug2("Loaded PermissionCheckerFactory: " + permissionCheckerFact); } } return permissionCheckerFact; }
protected FilterRule constructFilterRule(String contentType) { String mimeType = HeaderUtil.getMimeTypeFromContentType(contentType); Object filter_el = definitionMap.getMapElement(mimeType + DefinableArchivalUnit.SUFFIX_FILTER_RULE); if (filter_el instanceof String) { log.debug("Loading filter " + filter_el); return (FilterRule) newAuxClass((String) filter_el, FilterRule.class); } else if (filter_el instanceof List) { if (((List) filter_el).size() > 0) { return new DefinableFilterRule((List) filter_el); } } return super.constructFilterRule(mimeType); }
void checkParamAgreement(String key, PrintfContext context) { List<String> printfList = getElementList(key); if (printfList == null) { return; } for (String printf : printfList) { if (StringUtil.isNullString(printf)) { log.warning("Null printf string in " + key); continue; } PrintfUtil.PrintfData p_data = PrintfUtil.stringToPrintf(printf); Collection<String> p_args = p_data.getArguments(); for (String arg : p_args) { ConfigParamDescr descr = findAuConfigDescr(arg); if (descr == null) { throw new PluginException.InvalidDefinition( "Not a declared parameter: " + arg + " in " + printf + " in " + getPluginName()); } // ensure range and set params used only in legal context switch (context) { case Regexp: case Display: // everything is legal in a regexp or a display string break; case URL: // NUM_RANGE and SET legal because can enumerate. Can't // enumerate RANGE switch (descr.getType()) { case ConfigParamDescr.TYPE_RANGE: throw new PluginException.InvalidDefinition( "Range parameter (" + arg + ") used in illegal context in " + getPluginName() + ": " + key + ": " + printf); default: } } } } }
/** If in testing mode FOO, copy values from FOO_override map, if any, to main map */ void processOverrides(TypedEntryMap map) { String testMode = getTestingMode(); if (StringUtil.isNullString(testMode)) { return; } Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE); if (o == null) { return; } if (o instanceof Map) { Map overrideMap = (Map) o; for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) { String key = (String) entry.getKey(); Object val = entry.getValue(); log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val); map.setMapElement(key, val); } } }
/** * DefinablePlugin: a plugin which uses the data stored in an ExternalizableMap to configure itself. * * @author Claire Griffin * @version 1.0 */ public class DefinablePlugin extends BasePlugin { // configuration map keys public static final String KEY_PLUGIN_IDENTIFIER = "plugin_identifier"; public static final String KEY_PLUGIN_NAME = "plugin_name"; public static final String KEY_PLUGIN_VERSION = "plugin_version"; public static final String KEY_PLUGIN_FEATURE_VERSION_MAP = "plugin_feature_version_map"; public static final String KEY_REQUIRED_DAEMON_VERSION = "required_daemon_version"; public static final String KEY_PUBLISHING_PLATFORM = "plugin_publishing_platform"; public static final String KEY_PLUGIN_CONFIG_PROPS = "plugin_config_props"; public static final String KEY_EXCEPTION_HANDLER = "plugin_cache_result_handler"; public static final String KEY_EXCEPTION_LIST = "plugin_cache_result_list"; public static final String KEY_PLUGIN_NOTES = "plugin_notes"; public static final String KEY_CRAWL_TYPE = "plugin_crawl_type"; public static final String KEY_FOLLOW_LINKS = "plugin_follow_link"; /** Message to be displayed when user configures an AU with this plugin */ public static final String KEY_PLUGIN_AU_CONFIG_USER_MSG = "plugin_au_config_user_msg"; public static final String KEY_PER_HOST_PERMISSION_PATH = "plugin_per_host_permission_path"; public static final String KEY_PLUGIN_PARENT = "plugin_parent"; public static final String KEY_PLUGIN_PARENT_VERSION = "plugin_parent_version"; public static final String KEY_PLUGIN_CRAWL_URL_COMPARATOR_FACTORY = "plugin_crawl_url_comparator_factory"; public static final String KEY_PLUGIN_FETCH_RATE_LIMITER_SOURCE = "plugin_fetch_rate_limiter_source"; public static final String KEY_PLUGIN_ARTICLE_ITERATOR_FACTORY = "plugin_article_iterator_factory"; public static final String KEY_PLUGIN_ARTICLE_METADATA_EXTRACTOR_FACTORY = "plugin_article_metadata_extractor_factory"; public static final String KEY_DEFAULT_ARTICLE_MIME_TYPE = "plugin_default_article_mime_type"; public static final String KEY_ARTICLE_ITERATOR_ROOT = "plugin_article_iterator_root"; public static final String KEY_ARTICLE_ITERATOR_PATTERN = "plugin_article_iterator_pattern"; public static final String DEFAULT_PLUGIN_VERSION = "1"; public static final String DEFAULT_REQUIRED_DAEMON_VERSION = "0.0.0"; public static final String MAP_SUFFIX = ".xml"; public static final String CRAWL_TYPE_HTML_LINKS = "HTML Links"; public static final String CRAWL_TYPE_OAI = "OAI"; public static final String[] CRAWL_TYPES = { CRAWL_TYPE_HTML_LINKS, CRAWL_TYPE_OAI, }; public static final String DEFAULT_CRAWL_TYPE = CRAWL_TYPE_HTML_LINKS; protected String mapName = null; static Logger log = Logger.getLogger("DefinablePlugin"); protected ExternalizableMap definitionMap = new ExternalizableMap(); protected CacheResultHandler resultHandler = null; protected List<String> loadedFromUrls; protected CrawlWindow crawlWindow; protected Map<Plugin.Feature, String> featureVersion; public void initPlugin(LockssDaemon daemon, String extMapName) throws FileNotFoundException { initPlugin(daemon, extMapName, this.getClass().getClassLoader()); } public void initPlugin(LockssDaemon daemon, String extMapName, ClassLoader loader) throws FileNotFoundException { // convert the plugin class name to an xml file name // load the configuration map from jar file ExternalizableMap defMap = loadMap(extMapName, loader); this.initPlugin(daemon, extMapName, defMap, loader); } public void initPlugin( LockssDaemon daemon, String extMapName, ExternalizableMap defMap, ClassLoader loader) { mapName = extMapName; this.classLoader = loader; this.definitionMap = defMap; super.initPlugin(daemon); initMimeMap(); initFeatureVersions(); initAuFeatureMap(); checkParamAgreement(); } private ExternalizableMap loadMap(String extMapName, ClassLoader loader) throws FileNotFoundException { String first = null; String next = extMapName; List<String> urls = new ArrayList<String>(); ExternalizableMap res = null; while (next != null) { // convert the plugin class name to an xml file name String mapFile = next.replace('.', '/') + MAP_SUFFIX; URL url = loader.getResource(mapFile); if (url != null && urls.contains(url.toString())) { throw new PluginException.InvalidDefinition("Plugin inheritance loop: " + next); } // load into map ExternalizableMap oneMap = new ExternalizableMap(); oneMap.loadMapFromResource(mapFile, loader); urls.add(url.toString()); // apply overrides one plugin at a time in inheritance chain processOverrides(oneMap); if (res == null) { res = oneMap; } else { for (Map.Entry ent : oneMap.entrySet()) { String key = (String) ent.getKey(); Object val = ent.getValue(); if (!res.containsKey(key)) { res.setMapElement(key, val); } } } if (oneMap.containsKey(KEY_PLUGIN_PARENT)) { next = oneMap.getString(KEY_PLUGIN_PARENT); } else { next = null; } } loadedFromUrls = urls; return res; } /** If in testing mode FOO, copy values from FOO_override map, if any, to main map */ void processOverrides(TypedEntryMap map) { String testMode = getTestingMode(); if (StringUtil.isNullString(testMode)) { return; } Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE); if (o == null) { return; } if (o instanceof Map) { Map overrideMap = (Map) o; for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) { String key = (String) entry.getKey(); Object val = entry.getValue(); log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val); map.setMapElement(key, val); } } } String getTestingMode() { return theDaemon == null ? null : theDaemon.getTestingMode(); } // Used by tests public void initPlugin(LockssDaemon daemon, File file) throws PluginException { ExternalizableMap oneMap = new ExternalizableMap(); oneMap.loadMap(file); if (oneMap.getErrorString() != null) { throw new PluginException(oneMap.getErrorString()); } initPlugin(daemon, file.getPath(), oneMap, null); } void initPlugin(LockssDaemon daemon, ExternalizableMap defMap) { initPlugin(daemon, defMap, this.getClass().getClassLoader()); } void initPlugin(LockssDaemon daemon, ExternalizableMap defMap, ClassLoader loader) { initPlugin(daemon, "Internal", defMap, loader); } enum PrintfContext { Regexp, URL, Display }; void checkParamAgreement() { for (Map.Entry<String, PrintfContext> ent : DefinableArchivalUnit.printfKeysContext.entrySet()) { checkParamAgreement(ent.getKey(), ent.getValue()); } } void checkParamAgreement(String key, PrintfContext context) { List<String> printfList = getElementList(key); if (printfList == null) { return; } for (String printf : printfList) { if (StringUtil.isNullString(printf)) { log.warning("Null printf string in " + key); continue; } PrintfUtil.PrintfData p_data = PrintfUtil.stringToPrintf(printf); Collection<String> p_args = p_data.getArguments(); for (String arg : p_args) { ConfigParamDescr descr = findAuConfigDescr(arg); if (descr == null) { throw new PluginException.InvalidDefinition( "Not a declared parameter: " + arg + " in " + printf + " in " + getPluginName()); } // ensure range and set params used only in legal context switch (context) { case Regexp: case Display: // everything is legal in a regexp or a display string break; case URL: // NUM_RANGE and SET legal because can enumerate. Can't // enumerate RANGE switch (descr.getType()) { case ConfigParamDescr.TYPE_RANGE: throw new PluginException.InvalidDefinition( "Range parameter (" + arg + ") used in illegal context in " + getPluginName() + ": " + key + ": " + printf); default: } } } } } public List<String> getLoadedFromUrls() { return loadedFromUrls; } public String getPluginName() { if (definitionMap.containsKey(KEY_PLUGIN_NAME)) { return definitionMap.getString(KEY_PLUGIN_NAME); } else { return getDefaultPluginName(); } } protected String getDefaultPluginName() { return StringUtil.shortName(getPluginId()); } public String getVersion() { return definitionMap.getString(KEY_PLUGIN_VERSION, DEFAULT_PLUGIN_VERSION); } public String getFeatureVersion(Plugin.Feature feat) { if (featureVersion == null) { return null; } return featureVersion.get(feat); } public String getRequiredDaemonVersion() { return definitionMap.getString(KEY_REQUIRED_DAEMON_VERSION, DEFAULT_REQUIRED_DAEMON_VERSION); } public String getPublishingPlatform() { return definitionMap.getString(KEY_PUBLISHING_PLATFORM, null); } public String getPluginNotes() { return definitionMap.getString(KEY_PLUGIN_NOTES, null); } public String getDefaultArticleMimeType() { String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null); log.debug3("DefaultArticleMimeType " + ret); if (ret == null) { ret = super.getDefaultArticleMimeType(); log.debug3("DefaultArticleMimeType from super " + ret); } return ret; } public List<String> getElementList(String key) { Object element = definitionMap.getMapElement(key); List<String> lst; if (element instanceof String) { return Collections.singletonList((String) element); } else if (element instanceof List) { return (List) element; } else { return null; } } public List getLocalAuConfigDescrs() throws PluginException.InvalidDefinition { List auConfigDescrs = (List) definitionMap.getCollection(KEY_PLUGIN_CONFIG_PROPS, null); if (auConfigDescrs == null) { throw new PluginException.InvalidDefinition(mapName + " missing ConfigParamDescrs"); } return auConfigDescrs; } protected ArchivalUnit createAu0(Configuration auConfig) throws ArchivalUnit.ConfigurationException { DefinableArchivalUnit au = new DefinableArchivalUnit(this, definitionMap); au.setConfiguration(auConfig); return au; } public ExternalizableMap getDefinitionMap() { return definitionMap; } CacheResultHandler getCacheResultHandler() { return resultHandler; } String stripSuffix(String str, String suffix) { return str.substring(0, str.length() - suffix.length()); } protected void initMimeMap() throws PluginException.InvalidDefinition { for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) { Map.Entry ent = (Map.Entry) iter.next(); String key = (String) ent.getKey(); Object val = ent.getValue(); if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " link extractor: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); LinkExtractorFactory fact = (LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class); mti.setLinkExtractorFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) { // XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY // XXX unless/until that key is changed to not be a terminal substring // XXX of this one String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " crawl filter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class); mti.setCrawlFilterFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " filter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class); mti.setHashFilterFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT); if (val instanceof String) { String rate = (String) val; log.debug(mime + " fetch rate: " + rate); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); RateLimiter limit = mti.getFetchRateLimiter(); if (limit != null) { limit.setRate(rate); } else { mti.setFetchRateLimiter(new RateLimiter(rate)); } } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY); String factName = (String) val; log.debug(mime + " link rewriter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); LinkRewriterFactory fact = (LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class); mti.setLinkRewriterFactory(fact); } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP); Map factNameMap = (Map) val; Map factClassMap = new HashMap(); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) { String mdTypes = (String) it.next(); String factName = (String) factNameMap.get(mdTypes); log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName); for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) { setMdTypeFact(factClassMap, mdType, factName); } } mti.setFileMetadataExtractorFactoryMap(factClassMap); } } } private void setMdTypeFact(Map factClassMap, String mdType, String factName) { log.debug3("Metadata type: " + mdType + " factory " + factName); FileMetadataExtractorFactory fact = (FileMetadataExtractorFactory) newAuxClass(factName, FileMetadataExtractorFactory.class); factClassMap.put(mdType, fact); } protected void initResultMap() throws PluginException.InvalidDefinition { HttpResultMap hResultMap = new HttpResultMap(); // XXX Currently this only allows a CacheResultHandler class to // initialize the result map. Instead, don't use a CacheResultMap // directly, use either the plugin's CacheResultHandler, if specified, // or a default one that wraps the CacheResultMap String handler_class = null; handler_class = definitionMap.getString(KEY_EXCEPTION_HANDLER, null); if (handler_class != null) { try { resultHandler = (CacheResultHandler) newAuxClass(handler_class, CacheResultHandler.class); resultHandler.init(hResultMap); } catch (Exception ex) { throw new PluginException.InvalidDefinition( mapName + " has invalid Exception handler: " + handler_class, ex); } catch (LinkageError le) { throw new PluginException.InvalidDefinition( mapName + " has invalid Exception handler: " + handler_class, le); } } else { // Expect a list of mappings from either result code or exception // name to CacheException name Collection<String> mappings = definitionMap.getCollection(KEY_EXCEPTION_LIST, null); if (mappings != null) { // add each entry for (String entry : mappings) { if (log.isDebug2()) { log.debug2("initMap(" + entry + ")"); } String first; String ceName; try { List<String> pair = StringUtil.breakAt(entry, '=', 2, true, true); first = pair.get(0); ceName = pair.get(1); } catch (Exception ex) { throw new PluginException.InvalidDefinition( "Invalid syntax: " + entry + "in " + mapName); } Object val; // Value should be either a CacheException or CacheResultHandler // class name. PluginFetchEventResponse resp = (PluginFetchEventResponse) newAuxClass(ceName, PluginFetchEventResponse.class, null); if (resp instanceof CacheException) { val = resp.getClass(); } else if (resp instanceof CacheResultHandler) { val = WrapperUtil.wrap((CacheResultHandler) resp, CacheResultHandler.class); } else { throw new PluginException.InvalidDefinition( "Second arg not a " + "CacheException or " + "CacheResultHandler class: " + entry + ", in " + mapName); } try { int code = Integer.parseInt(first); // If parseable as an integer, it's a result code. hResultMap.storeMapEntry(code, val); } catch (NumberFormatException e) { try { Class eClass = Class.forName(first); // If a class name, it should be an exception class if (Exception.class.isAssignableFrom(eClass)) { hResultMap.storeMapEntry(eClass, val); } else { throw new PluginException.InvalidDefinition( "First arg not an " + "Exception class: " + entry + ", in " + mapName); } } catch (Exception ex) { throw new PluginException.InvalidDefinition( "First arg not a " + "number or class: " + entry + ", in " + mapName); } catch (LinkageError le) { throw new PluginException.InvalidDefinition("Can't load " + first, le); } } } } } resultMap = hResultMap; } protected void initFeatureVersions() throws PluginException.InvalidDefinition { if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) { Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>(); Map<String, String> spec = (Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP); log.debug2("features: " + spec); for (Map.Entry<String, String> ent : spec.entrySet()) { try { // Prefix version string with feature name to create separate // namespace for each feature String key = ent.getKey(); map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue()); } catch (RuntimeException e) { log.warning( getPluginName() + " set unknown feature: " + ent.getKey() + " to version " + ent.getValue(), e); throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e); } } featureVersion = map; } else { featureVersion = null; } } protected void initAuFeatureMap() { if (definitionMap.containsKey(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP)) { Map<String, ?> featMap = definitionMap.getMap(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP); for (Map.Entry ent : featMap.entrySet()) { Object val = ent.getValue(); if (val instanceof Map) { ent.setValue(MapUtil.expandAlternativeKeyLists((Map) val)); } } } } /** Create a CrawlWindow if necessary and return it. The CrawlWindow must be thread-safe. */ protected CrawlWindow makeCrawlWindow() { if (crawlWindow != null) { return crawlWindow; } CrawlWindow window = (CrawlWindow) definitionMap.getMapElement(DefinableArchivalUnit.KEY_AU_CRAWL_WINDOW_SER); if (window == null) { String window_class = definitionMap.getString(DefinableArchivalUnit.KEY_AU_CRAWL_WINDOW, null); if (window_class != null) { ConfigurableCrawlWindow ccw = (ConfigurableCrawlWindow) newAuxClass(window_class, ConfigurableCrawlWindow.class); try { window = ccw.makeCrawlWindow(); } catch (PluginException e) { throw new RuntimeException(e); } } } crawlWindow = window; return window; } LoginPageChecker loginChecker; protected LoginPageChecker makeLoginPageChecker() { if (loginChecker == null) { String loginPageCheckerClass = definitionMap.getString(DefinableArchivalUnit.KEY_AU_LOGIN_PAGE_CHECKER, null); if (loginPageCheckerClass != null) { loginChecker = (LoginPageChecker) newAuxClass(loginPageCheckerClass, LoginPageChecker.class); } } return loginChecker; } PermissionCheckerFactory permissionCheckerFact; protected PermissionCheckerFactory getPermissionCheckerFactory() { if (permissionCheckerFact == null) { String permissionCheckerFactoryClass = definitionMap.getString(DefinableArchivalUnit.KEY_AU_PERMISSION_CHECKER_FACTORY, null); if (permissionCheckerFactoryClass != null) { permissionCheckerFact = (PermissionCheckerFactory) newAuxClass(permissionCheckerFactoryClass, PermissionCheckerFactory.class); log.debug2("Loaded PermissionCheckerFactory: " + permissionCheckerFact); } } return permissionCheckerFact; } protected UrlNormalizer urlNorm; protected UrlNormalizer getUrlNormalizer() { if (urlNorm == null) { String normalizerClass = definitionMap.getString(DefinableArchivalUnit.KEY_AU_URL_NORMALIZER, null); if (normalizerClass != null) { urlNorm = (UrlNormalizer) newAuxClass(normalizerClass, UrlNormalizer.class); } else { urlNorm = NullUrlNormalizer.INSTANCE; } } return urlNorm; } protected ExploderHelper exploderHelper = null; protected ExploderHelper getExploderHelper() { if (exploderHelper == null) { String helperClass = definitionMap.getString(DefinableArchivalUnit.KEY_AU_EXPLODER_HELPER, null); if (helperClass != null) { exploderHelper = (ExploderHelper) newAuxClass(helperClass, ExploderHelper.class); } } return exploderHelper; } protected CrawlUrlComparatorFactory crawlUrlComparatorFactory = null; protected CrawlUrlComparatorFactory getCrawlUrlComparatorFactory() { if (crawlUrlComparatorFactory == null) { String factClass = definitionMap.getString(DefinablePlugin.KEY_PLUGIN_CRAWL_URL_COMPARATOR_FACTORY, null); if (factClass != null) { crawlUrlComparatorFactory = (CrawlUrlComparatorFactory) newAuxClass(factClass, CrawlUrlComparatorFactory.class); } } return crawlUrlComparatorFactory; } protected Comparator<CrawlUrl> getCrawlUrlComparator(ArchivalUnit au) throws PluginException.LinkageError { CrawlUrlComparatorFactory fact = getCrawlUrlComparatorFactory(); if (fact == null) { return null; } return fact.createCrawlUrlComparator(au); } protected FilterRule constructFilterRule(String contentType) { String mimeType = HeaderUtil.getMimeTypeFromContentType(contentType); Object filter_el = definitionMap.getMapElement(mimeType + DefinableArchivalUnit.SUFFIX_FILTER_RULE); if (filter_el instanceof String) { log.debug("Loading filter " + filter_el); return (FilterRule) newAuxClass((String) filter_el, FilterRule.class); } else if (filter_el instanceof List) { if (((List) filter_el).size() > 0) { return new DefinableFilterRule((List) filter_el); } } return super.constructFilterRule(mimeType); } protected ArticleIteratorFactory articleIteratorFact = null; protected ArticleMetadataExtractorFactory articleMetadataFact = null; /** * Returns the plugin's article iterator factory, if any * * @return the ArticleIteratorFactory */ public ArticleIteratorFactory getArticleIteratorFactory() { if (articleIteratorFact == null) { String factClass = definitionMap.getString(KEY_PLUGIN_ARTICLE_ITERATOR_FACTORY, null); if (factClass != null) { articleIteratorFact = (ArticleIteratorFactory) newAuxClass(factClass, ArticleIteratorFactory.class); } } return articleIteratorFact; } /** * Returns the article iterator factory for the content type, if any * * @param contentType the content type * @return the ArticleIteratorFactory */ public ArticleMetadataExtractorFactory getArticleMetadataExtractorFactory(MetadataTarget target) { if (articleMetadataFact == null) { String factClass = definitionMap.getString(KEY_PLUGIN_ARTICLE_METADATA_EXTRACTOR_FACTORY, null); if (factClass != null) { articleMetadataFact = (ArticleMetadataExtractorFactory) newAuxClass(factClass, ArticleMetadataExtractorFactory.class); } } return articleMetadataFact; } public String getPluginId() { String className; if (mapName != null) { className = mapName; } else { // @TODO: eliminate this when we eliminate subclasses className = this.getClass().getName(); } return className; } }
protected void initResultMap() throws PluginException.InvalidDefinition { HttpResultMap hResultMap = new HttpResultMap(); // XXX Currently this only allows a CacheResultHandler class to // initialize the result map. Instead, don't use a CacheResultMap // directly, use either the plugin's CacheResultHandler, if specified, // or a default one that wraps the CacheResultMap String handler_class = null; handler_class = definitionMap.getString(KEY_EXCEPTION_HANDLER, null); if (handler_class != null) { try { resultHandler = (CacheResultHandler) newAuxClass(handler_class, CacheResultHandler.class); resultHandler.init(hResultMap); } catch (Exception ex) { throw new PluginException.InvalidDefinition( mapName + " has invalid Exception handler: " + handler_class, ex); } catch (LinkageError le) { throw new PluginException.InvalidDefinition( mapName + " has invalid Exception handler: " + handler_class, le); } } else { // Expect a list of mappings from either result code or exception // name to CacheException name Collection<String> mappings = definitionMap.getCollection(KEY_EXCEPTION_LIST, null); if (mappings != null) { // add each entry for (String entry : mappings) { if (log.isDebug2()) { log.debug2("initMap(" + entry + ")"); } String first; String ceName; try { List<String> pair = StringUtil.breakAt(entry, '=', 2, true, true); first = pair.get(0); ceName = pair.get(1); } catch (Exception ex) { throw new PluginException.InvalidDefinition( "Invalid syntax: " + entry + "in " + mapName); } Object val; // Value should be either a CacheException or CacheResultHandler // class name. PluginFetchEventResponse resp = (PluginFetchEventResponse) newAuxClass(ceName, PluginFetchEventResponse.class, null); if (resp instanceof CacheException) { val = resp.getClass(); } else if (resp instanceof CacheResultHandler) { val = WrapperUtil.wrap((CacheResultHandler) resp, CacheResultHandler.class); } else { throw new PluginException.InvalidDefinition( "Second arg not a " + "CacheException or " + "CacheResultHandler class: " + entry + ", in " + mapName); } try { int code = Integer.parseInt(first); // If parseable as an integer, it's a result code. hResultMap.storeMapEntry(code, val); } catch (NumberFormatException e) { try { Class eClass = Class.forName(first); // If a class name, it should be an exception class if (Exception.class.isAssignableFrom(eClass)) { hResultMap.storeMapEntry(eClass, val); } else { throw new PluginException.InvalidDefinition( "First arg not an " + "Exception class: " + entry + ", in " + mapName); } } catch (Exception ex) { throw new PluginException.InvalidDefinition( "First arg not a " + "number or class: " + entry + ", in " + mapName); } catch (LinkageError le) { throw new PluginException.InvalidDefinition("Can't load " + first, le); } } } } } resultMap = hResultMap; }
private void setMdTypeFact(Map factClassMap, String mdType, String factName) { log.debug3("Metadata type: " + mdType + " factory " + factName); FileMetadataExtractorFactory fact = (FileMetadataExtractorFactory) newAuxClass(factName, FileMetadataExtractorFactory.class); factClassMap.put(mdType, fact); }
protected void initMimeMap() throws PluginException.InvalidDefinition { for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) { Map.Entry ent = (Map.Entry) iter.next(); String key = (String) ent.getKey(); Object val = ent.getValue(); if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " link extractor: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); LinkExtractorFactory fact = (LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class); mti.setLinkExtractorFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) { // XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY // XXX unless/until that key is changed to not be a terminal substring // XXX of this one String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " crawl filter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class); mti.setCrawlFilterFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY); if (val instanceof String) { String factName = (String) val; log.debug(mime + " filter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class); mti.setHashFilterFactory(fact); } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT); if (val instanceof String) { String rate = (String) val; log.debug(mime + " fetch rate: " + rate); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); RateLimiter limit = mti.getFetchRateLimiter(); if (limit != null) { limit.setRate(rate); } else { mti.setFetchRateLimiter(new RateLimiter(rate)); } } } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY); String factName = (String) val; log.debug(mime + " link rewriter: " + factName); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); LinkRewriterFactory fact = (LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class); mti.setLinkRewriterFactory(fact); } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) { String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP); Map factNameMap = (Map) val; Map factClassMap = new HashMap(); MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime); for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) { String mdTypes = (String) it.next(); String factName = (String) factNameMap.get(mdTypes); log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName); for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) { setMdTypeFact(factClassMap, mdType, factName); } } mti.setFileMetadataExtractorFactoryMap(factClassMap); } } }