protected void initAuFeatureMap() {
   if (definitionMap.containsKey(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP)) {
     Map<String, ?> featMap = definitionMap.getMap(DefinableArchivalUnit.KEY_AU_FEATURE_URL_MAP);
     for (Map.Entry ent : featMap.entrySet()) {
       Object val = ent.getValue();
       if (val instanceof Map) {
         ent.setValue(MapUtil.expandAlternativeKeyLists((Map) val));
       }
     }
   }
 }
Esempio n. 2
0
 protected Properties filterResponseProps(Properties props) {
   Properties res = new Properties();
   for (Map.Entry ent : props.entrySet()) {
     String key = (String) ent.getKey();
     if (StringUtil.startsWithIgnoreCase(key, "x-lockss")
         || StringUtil.startsWithIgnoreCase(key, "x_lockss")
         || key.equalsIgnoreCase("org.lockss.version.number")) {
       continue;
     }
     // We've lost the original case - capitalize them the way most people
     // expect
     res.put(StringUtil.titleCase(key, '-'), (String) ent.getValue());
   }
   return res;
 }
 /** If in testing mode FOO, copy values from FOO_override map, if any, to main map */
 void processOverrides(TypedEntryMap map) {
   String testMode = getTestingMode();
   if (StringUtil.isNullString(testMode)) {
     return;
   }
   Object o = map.getMapElement(testMode + DefinableArchivalUnit.SUFFIX_OVERRIDE);
   if (o == null) {
     return;
   }
   if (o instanceof Map) {
     Map overrideMap = (Map) o;
     for (Map.Entry entry : (Set<Map.Entry>) overrideMap.entrySet()) {
       String key = (String) entry.getKey();
       Object val = entry.getValue();
       log.debug(getDefaultPluginName() + ": Overriding " + key + " with " + val);
       map.setMapElement(key, val);
     }
   }
 }
 protected void initFeatureVersions() throws PluginException.InvalidDefinition {
   if (definitionMap.containsKey(KEY_PLUGIN_FEATURE_VERSION_MAP)) {
     Map<Plugin.Feature, String> map = new HashMap<Plugin.Feature, String>();
     Map<String, String> spec =
         (Map<String, String>) definitionMap.getMap(KEY_PLUGIN_FEATURE_VERSION_MAP);
     log.debug2("features: " + spec);
     for (Map.Entry<String, String> ent : spec.entrySet()) {
       try {
         // Prefix version string with feature name to create separate
         // namespace for each feature
         String key = ent.getKey();
         map.put(Plugin.Feature.valueOf(key), key + "_" + ent.getValue());
       } catch (RuntimeException e) {
         log.warning(
             getPluginName()
                 + " set unknown feature: "
                 + ent.getKey()
                 + " to version "
                 + ent.getValue(),
             e);
         throw new PluginException.InvalidDefinition("Unknown feature: " + ent.getKey(), e);
       }
     }
     featureVersion = map;
   } else {
     featureVersion = null;
   }
 }
 private ExternalizableMap loadMap(String extMapName, ClassLoader loader)
     throws FileNotFoundException {
   String first = null;
   String next = extMapName;
   List<String> urls = new ArrayList<String>();
   ExternalizableMap res = null;
   while (next != null) {
     // convert the plugin class name to an xml file name
     String mapFile = next.replace('.', '/') + MAP_SUFFIX;
     URL url = loader.getResource(mapFile);
     if (url != null && urls.contains(url.toString())) {
       throw new PluginException.InvalidDefinition("Plugin inheritance loop: " + next);
     }
     // load into map
     ExternalizableMap oneMap = new ExternalizableMap();
     oneMap.loadMapFromResource(mapFile, loader);
     urls.add(url.toString());
     // apply overrides one plugin at a time in inheritance chain
     processOverrides(oneMap);
     if (res == null) {
       res = oneMap;
     } else {
       for (Map.Entry ent : oneMap.entrySet()) {
         String key = (String) ent.getKey();
         Object val = ent.getValue();
         if (!res.containsKey(key)) {
           res.setMapElement(key, val);
         }
       }
     }
     if (oneMap.containsKey(KEY_PLUGIN_PARENT)) {
       next = oneMap.getString(KEY_PLUGIN_PARENT);
     } else {
       next = null;
     }
   }
   loadedFromUrls = urls;
   return res;
 }
Esempio n. 6
0
  /**
   * Add a TdbAu to a TdbTitle and TdbPubisher, and add links to the TdbTitle specified by the
   * properties.
   *
   * @param props the properties
   * @param au the TdbAu to add
   * @throws TdbException if the AU already exists in this Tdb
   */
  private void addTdbAu(Properties props, TdbAu au) throws TdbException {
    // add au for plugin assuming it is not a duplicate
    if (!addTdbAuForPlugin(au)) {
      // au already registered -- report existing au
      TdbAu existingAu = findExistingTdbAu(au);
      String titleName = getTdbTitleName(props, au);
      if (!titleName.equals(existingAu.getTdbTitle().getName())) {
        throw new TdbException(
            "Cannot add duplicate au entry: \""
                + au.getName()
                + "\" for title \""
                + titleName
                + "\" with same definition as existing au entry: \""
                + existingAu.getName()
                + "\" for title \""
                + existingAu.getTdbTitle().getName()
                + "\" to title database");
      } else if (!existingAu.getName().equals(au.getName())) {
        // error because it could lead to a missing AU -- one probably has a typo
        throw new TdbException(
            "Cannot add duplicate au entry: \""
                + au.getName()
                + "\" with the same definition as \""
                + existingAu.getName()
                + "\" for title \""
                + titleName
                + "\" to title database");
      } else {
        throw new TdbException(
            "Cannot add duplicate au entry: \""
                + au.getName()
                + "\" for title \""
                + titleName
                + "\" to title database");
      }
    }

    // get or create the TdbTitle for this
    TdbTitle title = getTdbTitle(props, au);
    try {
      // add AU to title
      title.addTdbAu(au);
    } catch (TdbException ex) {
      // if we can't add au to title, remove for plugin and re-throw exception
      removeTdbAuForPlugin(au);
      throw ex;
    }

    // process title links
    Map<String, Map<String, String>> linkMap = new HashMap<String, Map<String, String>>();
    for (Map.Entry<Object, Object> entry : props.entrySet()) {
      String key = "" + entry.getKey();
      String value = "" + entry.getValue();
      if (key.startsWith("journal.link.")) {
        // skip to link name
        String param = key.substring("link.".length());
        int i;
        if (((i = param.indexOf(".type")) < 0) && ((i = param.indexOf(".journalId")) < 0)) {
          logger.warning(
              "Ignoring nexpected link key for au \"" + au.getName() + "\" key: \"" + key + "\"");
        } else {
          // get link map for linkName
          String lname = param.substring(0, i);
          Map<String, String> lmap = linkMap.get(lname);
          if (lmap == null) {
            lmap = new HashMap<String, String>();
            linkMap.put(lname, lmap);
          }
          // add name and value to link map for link
          String name = param.substring(i + 1);
          lmap.put(name, value);
        }
      }
    }

    // add links to title from accumulated "type", "journalId" entries
    for (Map<String, String> lmap : linkMap.values()) {
      String name = lmap.get("type");
      String value = lmap.get("journalId");
      if ((name != null) && (value != null)) {
        try {
          TdbTitle.LinkType linkType = TdbTitle.LinkType.valueOf(name);
          title.addLinkToTdbTitleId(linkType, value);
        } catch (IllegalArgumentException ex) {
          logger.warning(
              "Ignoring unknown link type for au \"" + au.getName() + "\" name: \"" + name + "\"");
        }
      }
    }
  }
Esempio n. 7
0
  /**
   * Create a new TdbAu instance from the properties.
   *
   * @param props the properties
   * @return a TdbAu instance set built from the properties
   */
  private TdbAu newTdbAu(Properties props) {
    String pluginId = (String) props.get("plugin");
    if (pluginId == null) {
      throw new IllegalArgumentException("TdbAu plugin ID not specified");
    }

    String auName = props.getProperty("title");
    if (auName == null) {
      throw new IllegalArgumentException("TdbAu title not specified");
    }

    // create a new TdbAu and set its elements
    TdbAu au = new TdbAu(auName, pluginId);

    // process attrs, and params
    Map<String, Map<String, String>> paramMap = new HashMap<String, Map<String, String>>();
    for (Map.Entry<Object, Object> entry : props.entrySet()) {
      String key = String.valueOf(entry.getKey());
      String value = String.valueOf(entry.getValue());
      if (key.startsWith("attributes.")) {
        // set attributes directly
        String name = key.substring("attributes.".length());
        try {
          au.setAttr(name, value);
        } catch (TdbException ex) {
          logger.warning(
              "Cannot set attribute \"" + name + "\" with value \"" + value + "\" -- ignoring");
        }

      } else if (key.startsWith("param.")) {
        // skip to param name
        String param = key.substring("param.".length());
        int i;
        if (((i = param.indexOf(".key")) < 0) && ((i = param.indexOf(".value")) < 0)) {
          logger.warning(
              "Ignoring unexpected param key for au \""
                  + auName
                  + "\" key: \""
                  + key
                  + "\" -- ignoring");
        } else {
          // get param map for pname
          String pname = param.substring(0, i);
          Map<String, String> pmap = paramMap.get(pname);
          if (pmap == null) {
            pmap = new HashMap<String, String>();
            paramMap.put(pname, pmap);
          }
          // add name and value to param map for pname
          String name = param.substring(i + 1);
          pmap.put(name, value);
        }

      } else if (!key.equals("title") // TdbAu has "name" property
          && !key.equals("plugin") // TdbAu has "pluginId" property
          && !key.equals("journalTitle") // TdbAu has "title" TdbTitle property
          && !key.startsWith("journal.")) { // TdbAu has "title" TdbTitle property
        // translate all other properties into AU properties
        try {
          au.setPropertyByName(key, value);
        } catch (TdbException ex) {
          logger.warning(
              "Cannot set property \"" + key + "\" with value \"" + value + "\" -- ignoring");
        }
      }
    }

    // set param from accumulated "key", and "value" entries
    for (Map<String, String> pmap : paramMap.values()) {
      String name = pmap.get("key");
      String value = pmap.get("value");
      if (name == null) {
        logger.warning("Ignoring property with null name");
      } else if (value == null) {
        logger.warning("Ignoring property \"" + name + "\" with null value");
      } else {
        try {
          au.setParam(name, value);
        } catch (TdbException ex) {
          logger.warning(
              "Cannot set param \"" + name + "\" with value \"" + value + "\" -- ignoring");
        }
      }
    }

    return au;
  }
 protected void initMimeMap() throws PluginException.InvalidDefinition {
   for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) {
     Map.Entry ent = (Map.Entry) iter.next();
     String key = (String) ent.getKey();
     Object val = ent.getValue();
     if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY);
       if (val instanceof String) {
         String factName = (String) val;
         log.debug(mime + " link extractor: " + factName);
         MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
         LinkExtractorFactory fact =
             (LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class);
         mti.setLinkExtractorFactory(fact);
       }
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) {
       // XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY
       // XXX unless/until that key is changed to not be a terminal substring
       // XXX of this one
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY);
       if (val instanceof String) {
         String factName = (String) val;
         log.debug(mime + " crawl filter: " + factName);
         MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
         FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
         mti.setCrawlFilterFactory(fact);
       }
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY);
       if (val instanceof String) {
         String factName = (String) val;
         log.debug(mime + " filter: " + factName);
         MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
         FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
         mti.setHashFilterFactory(fact);
       }
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT);
       if (val instanceof String) {
         String rate = (String) val;
         log.debug(mime + " fetch rate: " + rate);
         MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
         RateLimiter limit = mti.getFetchRateLimiter();
         if (limit != null) {
           limit.setRate(rate);
         } else {
           mti.setFetchRateLimiter(new RateLimiter(rate));
         }
       }
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY);
       String factName = (String) val;
       log.debug(mime + " link rewriter: " + factName);
       MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
       LinkRewriterFactory fact =
           (LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class);
       mti.setLinkRewriterFactory(fact);
     } else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) {
       String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP);
       Map factNameMap = (Map) val;
       Map factClassMap = new HashMap();
       MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
       for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) {
         String mdTypes = (String) it.next();
         String factName = (String) factNameMap.get(mdTypes);
         log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName);
         for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) {
           setMdTypeFact(factClassMap, mdType, factName);
         }
       }
       mti.setFileMetadataExtractorFactoryMap(factClassMap);
     }
   }
 }
 void checkParamAgreement() {
   for (Map.Entry<String, PrintfContext> ent :
       DefinableArchivalUnit.printfKeysContext.entrySet()) {
     checkParamAgreement(ent.getKey(), ent.getValue());
   }
 }