예제 #1
0
 protected void initFeatureVersions() throws PluginException.InvalidDefinition {
   if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
     Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
     Map<String, String> spec =
         (Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
     log.debug2("features: " + spec);
     for (Map.Entry<String, String> ent : spec.entrySet()) {
       try {
         // Prefix version string with feature name to create separate
         // namespace for each feature
         String key = ent.getKey();
         map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
       } catch (RuntimeException e) {
         log.warning(
             getPluginName()
                 + " set unknown feature: "
                 + ent.getKey()
                 + " to version "
                 + ent.getValue(),
             e);
         throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
       }
     }
     featureVersion = map;
   } else {
     featureVersion = null;
   }
 }
예제 #2
0
 public String getDefaultArticleMimeType() {
   String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null);
   log.debug3("DefaultArticleMimeType " + ret);
   if (ret == null) {
     ret = super.getDefaultArticleMimeType();
     log.debug3("DefaultArticleMimeType from super " + ret);
   }
   return ret;
 }
예제 #3
0
 protected PermissionCheckerFactory getPermissionCheckerFactory() {
   if (permissionCheckerFact == null) {
     String permissionCheckerFactoryClass =
         definitionMap.getString(DefinableArchivalUnit.KEY_AU_PERMISSION_CHECKER_FACTORY, null);
     if (permissionCheckerFactoryClass != null) {
       permissionCheckerFact =
           (PermissionCheckerFactory)
               newAuxClass(permissionCheckerFactoryClass, PermissionCheckerFactory.class);
       log.debug2("Loaded PermissionCheckerFactory: " + permissionCheckerFact);
     }
   }
   return permissionCheckerFact;
 }
예제 #4
0
  protected FilterRule constructFilterRule(String contentType) {
    String mimeType = HeaderUtil.getMimeTypeFromContentType(contentType);

    Object filter_el =
        definitionMap.getMapElement(mimeType + DefinableArchivalUnit.SUFFIX_FILTER_RULE);

    if (filter_el instanceof String) {
      log.debug("Loading filter " + filter_el);
      return (FilterRule) newAuxClass((String) filter_el, FilterRule.class);
    } else if (filter_el instanceof List) {
      if (((List) filter_el).size() > 0) {
        return new DefinableFilterRule((List) filter_el);
      }
    }
    return super.constructFilterRule(mimeType);
  }
예제 #5
0
 void checkParamAgreement(String key, PrintfContext context) {
   List<String> printfList = getElementList(key);
   if (printfList == null) {
     return;
   }
   for (String printf : printfList) {
     if (StringUtil.isNullString(printf)) {
       log.warning("Null printf string in " + key);
       continue;
     }
     PrintfUtil.PrintfData p_data = PrintfUtil.stringToPrintf(printf);
     Collection<String> p_args = p_data.getArguments();
     for (String arg : p_args) {
       ConfigParamDescr descr = findAuConfigDescr(arg);
       if (descr == null) {
         throw new PluginException.InvalidDefinition(
             "Not a declared parameter: " + arg + " in " + printf + " in " + getPluginName());
       }
       // ensure range and set params used only in legal context
       switch (context) {
         case Regexp:
         case Display:
           // everything is legal in a regexp or a display string
           break;
         case URL:
           // NUM_RANGE and SET legal because can enumerate.  Can't
           // enumerate RANGE
           switch (descr.getType()) {
             case ConfigParamDescr.TYPE_RANGE:
               throw new PluginException.InvalidDefinition(
                   "Range parameter ("
                       + arg
                       + ") used in illegal context in "
                       + getPluginName()
                       + ": "
                       + key
                       + ": "
                       + printf);
             default:
           }
       }
     }
   }
 }
예제 #6
0
 /** If in testing mode FOO, copy values from FOO_override map, if any, to main map */
 void processOverrides(TypedEntryMap map) {
   String testMode = getTestingMode();
   if (StringUtil.isNullString(testMode)) {
     return;
   }
   Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE);
   if (o == null) {
     return;
   }
   if (o instanceof Map) {
     Map overrideMap = (Map) o;
     for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) {
       String key = (String) entry.getKey();
       Object val = entry.getValue();
       log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val);
       map.setMapElement(key, val);
     }
   }
 }
예제 #7
0
/**
 * DefinablePlugin: a plugin which uses the data stored in an ExternalizableMap to configure itself.
 *
 * @author Claire Griffin
 * @version 1.0
 */
public class DefinablePlugin extends BasePlugin {
  // configuration map keys
  public static final String KEY_PLUGIN_IDENTIFIER = "plugin_identifier";
  public static final String KEY_PLUGIN_NAME = "plugin_name";
  public static final String KEY_PLUGIN_VERSION = "plugin_version";
  public static final String KEY_PLUGIN_FEATURE_VERSION_MAP = "plugin_feature_version_map";
  public static final String KEY_REQUIRED_DAEMON_VERSION = "required_daemon_version";
  public static final String KEY_PUBLISHING_PLATFORM = "plugin_publishing_platform";
  public static final String KEY_PLUGIN_CONFIG_PROPS = "plugin_config_props";
  public static final String KEY_EXCEPTION_HANDLER = "plugin_cache_result_handler";
  public static final String KEY_EXCEPTION_LIST = "plugin_cache_result_list";
  public static final String KEY_PLUGIN_NOTES = "plugin_notes";
  public static final String KEY_CRAWL_TYPE = "plugin_crawl_type";
  public static final String KEY_FOLLOW_LINKS = "plugin_follow_link";
  /** Message to be displayed when user configures an AU with this plugin */
  public static final String KEY_PLUGIN_AU_CONFIG_USER_MSG = "plugin_au_config_user_msg";

  public static final String KEY_PER_HOST_PERMISSION_PATH = "plugin_per_host_permission_path";
  public static final String KEY_PLUGIN_PARENT = "plugin_parent";
  public static final String KEY_PLUGIN_PARENT_VERSION = "plugin_parent_version";
  public static final String KEY_PLUGIN_CRAWL_URL_COMPARATOR_FACTORY =
      "plugin_crawl_url_comparator_factory";
  public static final String KEY_PLUGIN_FETCH_RATE_LIMITER_SOURCE =
      "plugin_fetch_rate_limiter_source";

  public static final String KEY_PLUGIN_ARTICLE_ITERATOR_FACTORY =
      "plugin_article_iterator_factory";

  public static final String KEY_PLUGIN_ARTICLE_METADATA_EXTRACTOR_FACTORY =
      "plugin_article_metadata_extractor_factory";

  public static final String KEY_DEFAULT_ARTICLE_MIME_TYPE = "plugin_default_article_mime_type";

  public static final String KEY_ARTICLE_ITERATOR_ROOT = "plugin_article_iterator_root";

  public static final String KEY_ARTICLE_ITERATOR_PATTERN = "plugin_article_iterator_pattern";

  public static final String DEFAULT_PLUGIN_VERSION = "1";
  public static final String DEFAULT_REQUIRED_DAEMON_VERSION = "0.0.0";

  public static final String MAP_SUFFIX = ".xml";

  public static final String CRAWL_TYPE_HTML_LINKS = "HTML Links";
  public static final String CRAWL_TYPE_OAI = "OAI";
  public static final String[] CRAWL_TYPES = {
    CRAWL_TYPE_HTML_LINKS, CRAWL_TYPE_OAI,
  };
  public static final String DEFAULT_CRAWL_TYPE = CRAWL_TYPE_HTML_LINKS;

  protected String mapName = null;

  static Logger log = Logger.getLogger("DefinablePlugin");

  protected ExternalizableMap definitionMap = new ExternalizableMap();
  protected CacheResultHandler resultHandler = null;
  protected List<String> loadedFromUrls;
  protected CrawlWindow crawlWindow;
  protected Map<Plugin.Feature, String> featureVersion;

  public void initPlugin(LockssDaemon daemon, String extMapName) throws FileNotFoundException {
    initPlugin(daemon, extMapName, this.getClass().getClassLoader());
  }

  public void initPlugin(LockssDaemon daemon, String extMapName, ClassLoader loader)
      throws FileNotFoundException {
    // convert the plugin class name to an xml file name
    // load the configuration map from jar file
    ExternalizableMap defMap = loadMap(extMapName, loader);
    this.initPlugin(daemon, extMapName, defMap, loader);
  }

  public void initPlugin(
      LockssDaemon daemon, String extMapName, ExternalizableMap defMap, ClassLoader loader) {
    mapName = extMapName;
    this.classLoader = loader;
    this.definitionMap = defMap;
    super.initPlugin(daemon);
    initMimeMap();
    initFeatureVersions();
    initAuFeatureMap();
    checkParamAgreement();
  }

  private ExternalizableMap loadMap(String extMapName, ClassLoader loader)
      throws FileNotFoundException {
    String first = null;
    String next = extMapName;
    List<String> urls = new ArrayList<String>();
    ExternalizableMap res = null;
    while (next != null) {
      // convert the plugin class name to an xml file name
      String mapFile = next.replace('.', '/') + MAP_SUFFIX;
      URL url = loader.getResource(mapFile);
      if (url != null && urls.contains(url.toString())) {
        throw new PluginException.InvalidDefinition("Plugin inheritance loop: " + next);
      }
      // load into map
      ExternalizableMap oneMap = new ExternalizableMap();
      oneMap.loadMapFromResource(mapFile, loader);
      urls.add(url.toString());
      // apply overrides one plugin at a time in inheritance chain
      processOverrides(oneMap);
      if (res == null) {
        res = oneMap;
      } else {
        for (Map.Entry ent : oneMap.entrySet()) {
          String key = (String) ent.getKey();
          Object val = ent.getValue();
          if (!res.containsKey(key)) {
            res.setMapElement(key, val);
          }
        }
      }
      if (oneMap.containsKey(KEY_PLUGIN_PARENT)) {
        next = oneMap.getString(KEY_PLUGIN_PARENT);
      } else {
        next = null;
      }
    }
    loadedFromUrls = urls;
    return res;
  }

  /** If in testing mode FOO, copy values from FOO_override map, if any, to main map */
  void processOverrides(TypedEntryMap map) {
    String testMode = getTestingMode();
    if (StringUtil.isNullString(testMode)) {
      return;
    }
    Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE);
    if (o == null) {
      return;
    }
    if (o instanceof Map) {
      Map overrideMap = (Map) o;
      for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) {
        String key = (String) entry.getKey();
        Object val = entry.getValue();
        log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val);
        map.setMapElement(key, val);
      }
    }
  }

  String getTestingMode() {
    return theDaemon == null ? null : theDaemon.getTestingMode();
  }

  // Used by tests

  public void initPlugin(LockssDaemon daemon, File file) throws PluginException {
    ExternalizableMap oneMap = new ExternalizableMap();
    oneMap.loadMap(file);
    if (oneMap.getErrorString() != null) {
      throw new PluginException(oneMap.getErrorString());
    }
    initPlugin(daemon, file.getPath(), oneMap, null);
  }

  void initPlugin(LockssDaemon daemon, ExternalizableMap defMap) {
    initPlugin(daemon, defMap, this.getClass().getClassLoader());
  }

  void initPlugin(LockssDaemon daemon, ExternalizableMap defMap, ClassLoader loader) {
    initPlugin(daemon, "Internal", defMap, loader);
  }

  enum PrintfContext {
    Regexp,
    URL,
    Display
  };

  void checkParamAgreement() {
    for (Map.Entry<String, PrintfContext> ent :
        DefinableArchivalUnit.printfKeysContext.entrySet()) {
      checkParamAgreement(ent.getKey(), ent.getValue());
    }
  }

  void checkParamAgreement(String key, PrintfContext context) {
    List<String> printfList = getElementList(key);
    if (printfList == null) {
      return;
    }
    for (String printf : printfList) {
      if (StringUtil.isNullString(printf)) {
        log.warning("Null printf string in " + key);
        continue;
      }
      PrintfUtil.PrintfData p_data = PrintfUtil.stringToPrintf(printf);
      Collection<String> p_args = p_data.getArguments();
      for (String arg : p_args) {
        ConfigParamDescr descr = findAuConfigDescr(arg);
        if (descr == null) {
          throw new PluginException.InvalidDefinition(
              "Not a declared parameter: " + arg + " in " + printf + " in " + getPluginName());
        }
        // ensure range and set params used only in legal context
        switch (context) {
          case Regexp:
          case Display:
            // everything is legal in a regexp or a display string
            break;
          case URL:
            // NUM_RANGE and SET legal because can enumerate.  Can't
            // enumerate RANGE
            switch (descr.getType()) {
              case ConfigParamDescr.TYPE_RANGE:
                throw new PluginException.InvalidDefinition(
                    "Range parameter ("
                        + arg
                        + ") used in illegal context in "
                        + getPluginName()
                        + ": "
                        + key
                        + ": "
                        + printf);
              default:
            }
        }
      }
    }
  }

  public List<String> getLoadedFromUrls() {
    return loadedFromUrls;
  }

  public String getPluginName() {
    if (definitionMap.containsKey(KEY_PLUGIN_NAME)) {
      return definitionMap.getString(KEY_PLUGIN_NAME);
    } else {
      return getDefaultPluginName();
    }
  }

  protected String getDefaultPluginName() {
    return StringUtil.shortName(getPluginId());
  }

  public String getVersion() {
    return definitionMap.getString(KEY_PLUGIN_VERSION, DEFAULT_PLUGIN_VERSION);
  }

  public String getFeatureVersion(Plugin.Feature feat) {
    if (featureVersion == null) {
      return null;
    }
    return featureVersion.get(feat);
  }

  public String getRequiredDaemonVersion() {
    return definitionMap.getString(KEY_REQUIRED_DAEMON_VERSION, DEFAULT_REQUIRED_DAEMON_VERSION);
  }

  public String getPublishingPlatform() {
    return definitionMap.getString(KEY_PUBLISHING_PLATFORM, null);
  }

  public String getPluginNotes() {
    return definitionMap.getString(KEY_PLUGIN_NOTES, null);
  }

  public String getDefaultArticleMimeType() {
    String ret = definitionMap.getString(KEY_DEFAULT_ARTICLE_MIME_TYPE, null);
    log.debug3("DefaultArticleMimeType " + ret);
    if (ret == null) {
      ret = super.getDefaultArticleMimeType();
      log.debug3("DefaultArticleMimeType from super " + ret);
    }
    return ret;
  }

  public List<String> getElementList(String key) {
    Object element = definitionMap.getMapElement(key);
    List<String> lst;

    if (element instanceof String) {
      return Collections.singletonList((String) element);
    } else if (element instanceof List) {
      return (List) element;
    } else {
      return null;
    }
  }

  public List getLocalAuConfigDescrs() throws PluginException.InvalidDefinition {
    List auConfigDescrs = (List) definitionMap.getCollection(KEY_PLUGIN_CONFIG_PROPS, null);
    if (auConfigDescrs == null) {
      throw new PluginException.InvalidDefinition(mapName + " missing ConfigParamDescrs");
    }
    return auConfigDescrs;
  }

  protected ArchivalUnit createAu0(Configuration auConfig)
      throws ArchivalUnit.ConfigurationException {
    DefinableArchivalUnit au = new DefinableArchivalUnit(this, definitionMap);
    au.setConfiguration(auConfig);
    return au;
  }

  public ExternalizableMap getDefinitionMap() {
    return definitionMap;
  }

  CacheResultHandler getCacheResultHandler() {
    return resultHandler;
  }

  String stripSuffix(String str, String suffix) {
    return str.substring(0, str.length() - suffix.length());
  }

  protected void initMimeMap() throws PluginException.InvalidDefinition {
    for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) {
      Map.Entry ent = (Map.Entry) iter.next();
      String key = (String) ent.getKey();
      Object val = ent.getValue();
      if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY);
        if (val instanceof String) {
          String factName = (String) val;
          log.debug(mime + " link extractor: " + factName);
          MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
          LinkExtractorFactory fact =
              (LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class);
          mti.setLinkExtractorFactory(fact);
        }
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) {
        // XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY
        // XXX unless/until that key is changed to not be a terminal substring
        // XXX of this one
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY);
        if (val instanceof String) {
          String factName = (String) val;
          log.debug(mime + " crawl filter: " + factName);
          MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
          FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
          mti.setCrawlFilterFactory(fact);
        }
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY);
        if (val instanceof String) {
          String factName = (String) val;
          log.debug(mime + " filter: " + factName);
          MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
          FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
          mti.setHashFilterFactory(fact);
        }
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT);
        if (val instanceof String) {
          String rate = (String) val;
          log.debug(mime + " fetch rate: " + rate);
          MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
          RateLimiter limit = mti.getFetchRateLimiter();
          if (limit != null) {
            limit.setRate(rate);
          } else {
            mti.setFetchRateLimiter(new RateLimiter(rate));
          }
        }
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY);
        String factName = (String) val;
        log.debug(mime + " link rewriter: " + factName);
        MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
        LinkRewriterFactory fact =
            (LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class);
        mti.setLinkRewriterFactory(fact);
      } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) {
        String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP);
        Map factNameMap = (Map) val;
        Map factClassMap = new HashMap();
        MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
        for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) {
          String mdTypes = (String) it.next();
          String factName = (String) factNameMap.get(mdTypes);
          log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName);
          for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) {
            setMdTypeFact(factClassMap, mdType, factName);
          }
        }
        mti.setFileMetadataExtractorFactoryMap(factClassMap);
      }
    }
  }

  private void setMdTypeFact(Map factClassMap, String mdType, String factName) {
    log.debug3("Metadata type: " + mdType + " factory " + factName);
    FileMetadataExtractorFactory fact =
        (FileMetadataExtractorFactory) newAuxClass(factName, FileMetadataExtractorFactory.class);
    factClassMap.put(mdType, fact);
  }

  protected void initResultMap() throws PluginException.InvalidDefinition {
    HttpResultMap hResultMap = new HttpResultMap();
    // XXX Currently this only allows a CacheResultHandler class to
    // initialize the result map.  Instead, don't use a CacheResultMap
    // directly, use either the plugin's CacheResultHandler, if specified,
    // or a default one that wraps the CacheResultMap

    String handler_class = null;
    handler_class = definitionMap.getString(KEY_EXCEPTION_HANDLER, null);
    if (handler_class != null) {
      try {
        resultHandler = (CacheResultHandler) newAuxClass(handler_class, CacheResultHandler.class);
        resultHandler.init(hResultMap);
      } catch (Exception ex) {
        throw new PluginException.InvalidDefinition(
            mapName + " has invalid Exception handler: " + handler_class, ex);
      } catch (LinkageError le) {
        throw new PluginException.InvalidDefinition(
            mapName + " has invalid Exception handler: " + handler_class, le);
      }
    } else {
      // Expect a list of mappings from either result code or exception
      // name to CacheException name
      Collection<String> mappings = definitionMap.getCollection(KEY_EXCEPTION_LIST, null);
      if (mappings != null) {
        // add each entry
        for (String entry : mappings) {
          if (log.isDebug2()) {
            log.debug2("initMap(" + entry + ")");
          }
          String first;
          String ceName;
          try {
            List<String> pair = StringUtil.breakAt(entry, '=', 2, true, true);
            first = pair.get(0);
            ceName = pair.get(1);
          } catch (Exception ex) {
            throw new PluginException.InvalidDefinition(
                "Invalid syntax: " + entry + "in " + mapName);
          }
          Object val;

          // Value should be either a CacheException or CacheResultHandler
          // class name.
          PluginFetchEventResponse resp =
              (PluginFetchEventResponse) newAuxClass(ceName, PluginFetchEventResponse.class, null);
          if (resp instanceof CacheException) {
            val = resp.getClass();
          } else if (resp instanceof CacheResultHandler) {
            val = WrapperUtil.wrap((CacheResultHandler) resp, CacheResultHandler.class);
          } else {
            throw new PluginException.InvalidDefinition(
                "Second arg not a "
                    + "CacheException or "
                    + "CacheResultHandler class: "
                    + entry
                    + ", in "
                    + mapName);
          }
          try {
            int code = Integer.parseInt(first);
            // If parseable as an integer, it's a result code.
            hResultMap.storeMapEntry(code, val);
          } catch (NumberFormatException e) {
            try {
              Class eClass = Class.forName(first);
              // If a class name, it should be an exception class
              if (Exception.class.isAssignableFrom(eClass)) {
                hResultMap.storeMapEntry(eClass, val);
              } else {
                throw new PluginException.InvalidDefinition(
                    "First arg not an " + "Exception class: " + entry + ", in " + mapName);
              }
            } catch (Exception ex) {
              throw new PluginException.InvalidDefinition(
                  "First arg not a " + "number or class: " + entry + ", in " + mapName);
            } catch (LinkageError le) {
              throw new PluginException.InvalidDefinition("Can't load " + first, le);
            }
          }
        }
      }
    }
    resultMap = hResultMap;
  }

  protected void initFeatureVersions() throws PluginException.InvalidDefinition {
    if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
      Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
      Map<String, String> spec =
          (Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
      log.debug2("features: " + spec);
      for (Map.Entry<String, String> ent : spec.entrySet()) {
        try {
          // Prefix version string with feature name to create separate
          // namespace for each feature
          String key = ent.getKey();
          map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
        } catch (RuntimeException e) {
          log.warning(
              getPluginName()
                  + " set unknown feature: "
                  + ent.getKey()
                  + " to version "
                  + ent.getValue(),
              e);
          throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
        }
      }
      featureVersion = map;
    } else {
      featureVersion = null;
    }
  }

  protected void initAuFeatureMap() {
    if (definitionMap.containsKey(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP)) {
      Map<String, ?> featMap = definitionMap.getMap(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP);
      for (Map.Entry ent : featMap.entrySet()) {
        Object val = ent.getValue();
        if (val instanceof Map) {
          ent.setValue(MapUtil.expandAlternativeKeyLists((Map) val));
        }
      }
    }
  }

  /** Create a CrawlWindow if necessary and return it. The CrawlWindow must be thread-safe. */
  protected CrawlWindow makeCrawlWindow() {
    if (crawlWindow != null) {
      return crawlWindow;
    }
    CrawlWindow window =
        (CrawlWindow) definitionMap.getMapElement(DefinableArchivalUnit.KEY_AU_CRAWL_WINDOW_SER);
    if (window == null) {
      String window_class =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_CRAWL_WINDOW, null);
      if (window_class != null) {
        ConfigurableCrawlWindow ccw =
            (ConfigurableCrawlWindow) newAuxClass(window_class, ConfigurableCrawlWindow.class);
        try {
          window = ccw.makeCrawlWindow();
        } catch (PluginException e) {
          throw new RuntimeException(e);
        }
      }
    }
    crawlWindow = window;
    return window;
  }

  LoginPageChecker loginChecker;

  protected LoginPageChecker makeLoginPageChecker() {
    if (loginChecker == null) {
      String loginPageCheckerClass =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_LOGIN_PAGE_CHECKER, null);
      if (loginPageCheckerClass != null) {
        loginChecker =
            (LoginPageChecker) newAuxClass(loginPageCheckerClass, LoginPageChecker.class);
      }
    }
    return loginChecker;
  }

  PermissionCheckerFactory permissionCheckerFact;

  protected PermissionCheckerFactory getPermissionCheckerFactory() {
    if (permissionCheckerFact == null) {
      String permissionCheckerFactoryClass =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_PERMISSION_CHECKER_FACTORY, null);
      if (permissionCheckerFactoryClass != null) {
        permissionCheckerFact =
            (PermissionCheckerFactory)
                newAuxClass(permissionCheckerFactoryClass, PermissionCheckerFactory.class);
        log.debug2("Loaded PermissionCheckerFactory: " + permissionCheckerFact);
      }
    }
    return permissionCheckerFact;
  }

  protected UrlNormalizer urlNorm;

  protected UrlNormalizer getUrlNormalizer() {
    if (urlNorm == null) {
      String normalizerClass =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_URL_NORMALIZER, null);
      if (normalizerClass != null) {
        urlNorm = (UrlNormalizer) newAuxClass(normalizerClass, UrlNormalizer.class);
      } else {
        urlNorm = NullUrlNormalizer.INSTANCE;
      }
    }
    return urlNorm;
  }

  protected ExploderHelper exploderHelper = null;

  protected ExploderHelper getExploderHelper() {
    if (exploderHelper == null) {
      String helperClass =
          definitionMap.getString(DefinableArchivalUnit.KEY_AU_EXPLODER_HELPER, null);
      if (helperClass != null) {
        exploderHelper = (ExploderHelper) newAuxClass(helperClass, ExploderHelper.class);
      }
    }
    return exploderHelper;
  }

  protected CrawlUrlComparatorFactory crawlUrlComparatorFactory = null;

  protected CrawlUrlComparatorFactory getCrawlUrlComparatorFactory() {
    if (crawlUrlComparatorFactory == null) {
      String factClass =
          definitionMap.getString(DefinablePlugin.KEY_PLUGIN_CRAWL_URL_COMPARATOR_FACTORY, null);
      if (factClass != null) {
        crawlUrlComparatorFactory =
            (CrawlUrlComparatorFactory) newAuxClass(factClass, CrawlUrlComparatorFactory.class);
      }
    }
    return crawlUrlComparatorFactory;
  }

  protected Comparator<CrawlUrl> getCrawlUrlComparator(ArchivalUnit au)
      throws PluginException.LinkageError {
    CrawlUrlComparatorFactory fact = getCrawlUrlComparatorFactory();
    if (fact == null) {
      return null;
    }
    return fact.createCrawlUrlComparator(au);
  }

  protected FilterRule constructFilterRule(String contentType) {
    String mimeType = HeaderUtil.getMimeTypeFromContentType(contentType);

    Object filter_el =
        definitionMap.getMapElement(mimeType + DefinableArchivalUnit.SUFFIX_FILTER_RULE);

    if (filter_el instanceof String) {
      log.debug("Loading filter " + filter_el);
      return (FilterRule) newAuxClass((String) filter_el, FilterRule.class);
    } else if (filter_el instanceof List) {
      if (((List) filter_el).size() > 0) {
        return new DefinableFilterRule((List) filter_el);
      }
    }
    return super.constructFilterRule(mimeType);
  }

  protected ArticleIteratorFactory articleIteratorFact = null;
  protected ArticleMetadataExtractorFactory articleMetadataFact = null;

  /**
   * Returns the plugin's article iterator factory, if any
   *
   * @return the ArticleIteratorFactory
   */
  public ArticleIteratorFactory getArticleIteratorFactory() {
    if (articleIteratorFact == null) {
      String factClass = definitionMap.getString(KEY_PLUGIN_ARTICLE_ITERATOR_FACTORY, null);
      if (factClass != null) {
        articleIteratorFact =
            (ArticleIteratorFactory) newAuxClass(factClass, ArticleIteratorFactory.class);
      }
    }
    return articleIteratorFact;
  }

  /**
   * Returns the article iterator factory for the content type, if any
   *
   * @param contentType the content type
   * @return the ArticleIteratorFactory
   */
  public ArticleMetadataExtractorFactory getArticleMetadataExtractorFactory(MetadataTarget target) {
    if (articleMetadataFact == null) {
      String factClass =
          definitionMap.getString(KEY_PLUGIN_ARTICLE_METADATA_EXTRACTOR_FACTORY, null);
      if (factClass != null) {
        articleMetadataFact =
            (ArticleMetadataExtractorFactory)
                newAuxClass(factClass, ArticleMetadataExtractorFactory.class);
      }
    }
    return articleMetadataFact;
  }

  public String getPluginId() {
    String className;
    if (mapName != null) {
      className = mapName;
    } else {
      // @TODO: eliminate this when we eliminate subclasses
      className = this.getClass().getName();
    }
    return className;
  }
}
예제 #8
0
  protected void initResultMap() throws PluginException.InvalidDefinition {
    HttpResultMap hResultMap = new HttpResultMap();
    // XXX Currently this only allows a CacheResultHandler class to
    // initialize the result map.  Instead, don't use a CacheResultMap
    // directly, use either the plugin's CacheResultHandler, if specified,
    // or a default one that wraps the CacheResultMap

    String handler_class = null;
    handler_class = definitionMap.getString(KEY_EXCEPTION_HANDLER, null);
    if (handler_class != null) {
      try {
        resultHandler = (CacheResultHandler) newAuxClass(handler_class, CacheResultHandler.class);
        resultHandler.init(hResultMap);
      } catch (Exception ex) {
        throw new PluginException.InvalidDefinition(
            mapName + " has invalid Exception handler: " + handler_class, ex);
      } catch (LinkageError le) {
        throw new PluginException.InvalidDefinition(
            mapName + " has invalid Exception handler: " + handler_class, le);
      }
    } else {
      // Expect a list of mappings from either result code or exception
      // name to CacheException name
      Collection<String> mappings = definitionMap.getCollection(KEY_EXCEPTION_LIST, null);
      if (mappings != null) {
        // add each entry
        for (String entry : mappings) {
          if (log.isDebug2()) {
            log.debug2("initMap(" + entry + ")");
          }
          String first;
          String ceName;
          try {
            List<String> pair = StringUtil.breakAt(entry, '=', 2, true, true);
            first = pair.get(0);
            ceName = pair.get(1);
          } catch (Exception ex) {
            throw new PluginException.InvalidDefinition(
                "Invalid syntax: " + entry + "in " + mapName);
          }
          Object val;

          // Value should be either a CacheException or CacheResultHandler
          // class name.
          PluginFetchEventResponse resp =
              (PluginFetchEventResponse) newAuxClass(ceName, PluginFetchEventResponse.class, null);
          if (resp instanceof CacheException) {
            val = resp.getClass();
          } else if (resp instanceof CacheResultHandler) {
            val = WrapperUtil.wrap((CacheResultHandler) resp, CacheResultHandler.class);
          } else {
            throw new PluginException.InvalidDefinition(
                "Second arg not a "
                    + "CacheException or "
                    + "CacheResultHandler class: "
                    + entry
                    + ", in "
                    + mapName);
          }
          try {
            int code = Integer.parseInt(first);
            // If parseable as an integer, it's a result code.
            hResultMap.storeMapEntry(code, val);
          } catch (NumberFormatException e) {
            try {
              Class eClass = Class.forName(first);
              // If a class name, it should be an exception class
              if (Exception.class.isAssignableFrom(eClass)) {
                hResultMap.storeMapEntry(eClass, val);
              } else {
                throw new PluginException.InvalidDefinition(
                    "First arg not an " + "Exception class: " + entry + ", in " + mapName);
              }
            } catch (Exception ex) {
              throw new PluginException.InvalidDefinition(
                  "First arg not a " + "number or class: " + entry + ", in " + mapName);
            } catch (LinkageError le) {
              throw new PluginException.InvalidDefinition("Can't load " + first, le);
            }
          }
        }
      }
    }
    resultMap = hResultMap;
  }
예제 #9
0
 private void setMdTypeFact(Map factClassMap, String mdType, String factName) {
   log.debug3("Metadata type: " + mdType + " factory " + factName);
   FileMetadataExtractorFactory fact =
       (FileMetadataExtractorFactory) newAuxClass(factName, FileMetadataExtractorFactory.class);
   factClassMap.put(mdType, fact);
 }
예제 #10
0
 protected void initMimeMap() throws PluginException.InvalidDefinition {
   for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) {
     Map.Entry ent = (Map.Entry) iter.next();
     String key = (String) ent.getKey();
     Object val = ent.getValue();
     if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY);
       if (val instanceof String) {
         String factName = (String) val;
         log.debug(mime + " link extractor: " + factName);
         MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
         LinkExtractorFactory fact =
             (LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class);
         mti.setLinkExtractorFactory(fact);
       }
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) {
       // XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY
       // XXX unless/until that key is changed to not be a terminal substring
       // XXX of this one
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY);
       if (val instanceof String) {
         String factName = (String) val;
         log.debug(mime + " crawl filter: " + factName);
         MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
         FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
         mti.setCrawlFilterFactory(fact);
       }
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY);
       if (val instanceof String) {
         String factName = (String) val;
         log.debug(mime + " filter: " + factName);
         MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
         FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
         mti.setHashFilterFactory(fact);
       }
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT);
       if (val instanceof String) {
         String rate = (String) val;
         log.debug(mime + " fetch rate: " + rate);
         MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
         RateLimiter limit = mti.getFetchRateLimiter();
         if (limit != null) {
           limit.setRate(rate);
         } else {
           mti.setFetchRateLimiter(new RateLimiter(rate));
         }
       }
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY);
       String factName = (String) val;
       log.debug(mime + " link rewriter: " + factName);
       MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
       LinkRewriterFactory fact =
           (LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class);
       mti.setLinkRewriterFactory(fact);
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP);
       Map factNameMap = (Map) val;
       Map factClassMap = new HashMap();
       MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
       for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) {
         String mdTypes = (String) it.next();
         String factName = (String) factNameMap.get(mdTypes);
         log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName);
         for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) {
           setMdTypeFact(factClassMap, mdType, factName);
         }
       }
       mti.setFileMetadataExtractorFactoryMap(factClassMap);
     }
   }
 }