protected RateLimiter recomputeFetchRateLimiter(RateLimiter oldLimiter) {
String rate = CurrentConfig.getParam(PARAM_REGISTRY_FETCH_RATE, DEFAULT_REGISTRY_FETCH_RATE);
Object limiterKey = getFetchRateLimiterKey();
if (limiterKey == null) {
return RateLimiter.getRateLimiter(oldLimiter, rate, DEFAULT_REGISTRY_FETCH_RATE);
} else {
RateLimiter.Pool pool = RateLimiter.getPool();
return pool.findNamedRateLimiter(limiterKey, rate, DEFAULT_REGISTRY_FETCH_RATE);
}
}
protected void initMimeMap() throws PluginException.InvalidDefinition {
for (Iterator iter = definitionMap.entrySet().iterator(); iter.hasNext(); ) {
Map.Entry ent = (Map.Entry) iter.next();
String key = (String) ent.getKey();
Object val = ent.getValue();
if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_EXTRACTOR_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " link extractor: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
LinkExtractorFactory fact =
(LinkExtractorFactory) newAuxClass(factName, LinkExtractorFactory.class);
mti.setLinkExtractorFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY)) {
// XXX This clause must precede the one for SUFFIX_HASH_FILTER_FACTORY
// XXX unless/until that key is changed to not be a terminal substring
// XXX of this one
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_CRAWL_FILTER_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " crawl filter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
mti.setCrawlFilterFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_HASH_FILTER_FACTORY);
if (val instanceof String) {
String factName = (String) val;
log.debug(mime + " filter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
FilterFactory fact = (FilterFactory) newAuxClass(factName, FilterFactory.class);
mti.setHashFilterFactory(fact);
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_FETCH_RATE_LIMIT);
if (val instanceof String) {
String rate = (String) val;
log.debug(mime + " fetch rate: " + rate);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
RateLimiter limit = mti.getFetchRateLimiter();
if (limit != null) {
limit.setRate(rate);
} else {
mti.setFetchRateLimiter(new RateLimiter(rate));
}
}
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_LINK_REWRITER_FACTORY);
String factName = (String) val;
log.debug(mime + " link rewriter: " + factName);
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
LinkRewriterFactory fact =
(LinkRewriterFactory) newAuxClass(factName, LinkRewriterFactory.class);
mti.setLinkRewriterFactory(fact);
} else if (key.endsWith(DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP)) {
String mime = stripSuffix(key, DefinableArchivalUnit.SUFFIX_METADATA_EXTRACTOR_FACTORY_MAP);
Map factNameMap = (Map) val;
Map factClassMap = new HashMap();
MimeTypeInfo.Mutable mti = mimeMap.modifyMimeTypeInfo(mime);
for (Iterator it = factNameMap.keySet().iterator(); it.hasNext(); ) {
String mdTypes = (String) it.next();
String factName = (String) factNameMap.get(mdTypes);
log.debug(mime + " (" + mdTypes + ") metadata extractor: " + factName);
for (String mdType : (List<String>) StringUtil.breakAt(mdTypes, ";")) {
setMdTypeFact(factClassMap, mdType, factName);
}
}
mti.setFileMetadataExtractorFactoryMap(factClassMap);
}
}
}
