private Optional<EvictionStrategy> findEvictionStrategyForAccessLevel(
RetentionLevel accessLevel) {
if (evictionStrategyMap.containsKey(accessLevel.name())) {
return Optional.of(evictionStrategyMap.get(accessLevel.name()));
}
return Optional.empty();
}
public void checkAllLevelCaches() throws InvalidConfigurationException {
Collection<RetentionLevel> allLevels = environment.retentions().getAllLevels();
List<RetentionLevel> list = new ArrayList<>();
for (RetentionLevel level : allLevels) {
if (level.next() == null) {
list.add(level);
}
}
log.trace("start updating config with top-levels: " + list);
for (RetentionLevel level : list) {
getLevelsAbove(level);
}
}
private void getLevelsAbove(RetentionLevel level) throws InvalidConfigurationException {
Collection<RetentionLevel> list = environment.retentions().getAllLevels();
List<RetentionLevel> pointingToCurrentLevel = new LinkedList<>();
for (RetentionLevel rlevel : list) {
if (rlevel.next() != null) {
if (rlevel.next().equals(level.name())) {
pointingToCurrentLevel.add(rlevel);
}
}
}
if (!pointingToCurrentLevel.isEmpty()) {
for (RetentionLevel nextLevel : pointingToCurrentLevel) {
getLevelsAbove(nextLevel);
}
}
updateCachingLevel(level);
}
public void updateEvictionStrategies() {
Iterator<String> iterator = evictionStrategyMap.keySet().iterator();
while (iterator.hasNext()) {
String levelname = iterator.next();
Optional<RetentionLevel> level = environment.retentions().getLevelForName(levelname);
if (!level.isPresent() || !levelIsAccessLevel(level.get())) {
iterator.remove();
}
}
Collection<RetentionLevel> accessLevels = environment.retentions().getAllAccessLevels();
for (RetentionLevel accessLevel : accessLevels) {
if (caches.containsKey(accessLevel.name())
&& !evictionStrategyMap.containsKey(accessLevel.name())) {
Optional<CachingLevel> cachingLevel =
environment.cachingConfiguration().findLevelForLevelName(accessLevel.name());
if (cachingLevel.isPresent()) {
evictionStrategyMap.put(
accessLevel.name(),
new NaiveLRUStrategy(
accessLevel.name(),
metricStorage,
findAllFollowingLevelCaches(accessLevel),
cachingLevel.get().visibleCacheSize()));
}
}
}
}
public void loadSavedMetrics() {
List<RetentionLevel> accessLevels = environment.retentions().getAllAccessLevels();
int numberOfAccessLevels = accessLevels.size();
log.info("trying to load Metrics for " + numberOfAccessLevels + " accessLevels");
List<Future<?>> futures = new ArrayList<>();
try {
for (RetentionLevel rlevel : accessLevels) {
futures.addAll(loadSavedMetricsToCaches(rlevel.name()));
}
} catch (Exception e) {
loadMetricsThreadPool.shutdownNow();
try {
loadMetricsThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
} catch (InterruptedException e1) {
log.error("Waiting for loadMetrics thread pool shutdown was interrupted", e1);
}
for (LevelCache cache : caches.values()) {
cache.shutdown();
}
throw e;
}
log.info("Submitted a total of {} futures to pre-load metrics", futures.size());
int logEvery = Math.max(1, futures.size() / 10);
int futuresDone = 0;
for (Future<?> f : futures) {
try {
f.get();
} catch (InterruptedException | ExecutionException e) {
log.warn("Exception while waiting for load metrics threads", e);
}
futuresDone++;
if (futuresDone % logEvery == 0) {
log.info("Preloaded {} of {} metrics", futuresDone, futures.size());
}
}
log.info("all metrics preloaded!");
}
@Override
public void run() {
Optional<RetentionLevel> accessLevel =
environment.retentions().findAccessLevelForMetric(metricName);
if (accessLevel.isPresent()) {
Optional<EvictionStrategy> evictionStrategy =
findEvictionStrategyForAccessLevel(accessLevel.get());
if (!evictionStrategy.isPresent()) {
log.warn("metric {} with undefined evictionStrategy!!", metricName);
return;
}
caches
.get(retentionLevel.name())
.putDatabaseMetrics(metricName, evictionStrategy.get(), databaseMetrics, interval);
LoadMetricEvent.fire(metricName);
}
}
public MetricCache(BifroestEnvironment environment) {
this.caches = new HashMap<>();
this.evictionStrategyMap = new HashMap<>();
this.environment = environment;
this.metricStorage = environment.cachingConfiguration().getMetricStorage();
this.cachingConfig = environment.cachingConfiguration();
ThreadFactory threads =
new BasicThreadFactory.Builder().namingPattern("loadMetricsThread").build();
loadMetricsThreadPool =
new ThreadPoolExecutor(
environment.cachingConfiguration().getMetricCacheThreads(),
environment
.cachingConfiguration()
.getMetricCacheThreads(), // thread count is set to the real initial value on the
// first run()
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(),
threads);
ThreadFactory databaseThreads =
new BasicThreadFactory.Builder().namingPattern("databaseQueryThread").build();
databaseQueryPool =
new ThreadPoolExecutor(
environment.cachingConfiguration().getDatabaseThreads(),
environment
.cachingConfiguration()
.getDatabaseThreads(), // thread count is set to the real initial value on the first
// run()
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(),
databaseThreads);
for (RetentionLevel retentionLevel : environment.retentions().getAllLevels()) {
Optional<CachingLevel> cachingLevel =
environment.cachingConfiguration().findLevelForLevelName(retentionLevel.name());
if (cachingLevel.isPresent()) {
caches.put(retentionLevel.name(), new LevelCache(environment, cachingLevel.get()));
log.info("Created cache for {}", retentionLevel.name());
} else {
log.warn("RetentionLevel with unconfigured Cache {}", retentionLevel.name());
}
}
for (RetentionLevel level : environment.retentions().getAllAccessLevels()) {
Optional<CachingLevel> cachingLevel =
environment.cachingConfiguration().findLevelForLevelName(level.name());
if (cachingLevel.isPresent()) {
int evictionStrategySize = cachingLevel.get().visibleCacheSize();
evictionStrategyMap.put(
level.name(),
new NaiveLRUStrategy(
level.name(),
metricStorage,
findAllFollowingLevelCaches(level),
evictionStrategySize));
log.trace("EvictionStrategyMap.put({})", level.name());
}
}
MBeanManager.registerStandardMBean(
this,
MetricCache.class.getPackage().getName() + ":type=" + MetricCache.class.getSimpleName(),
MetricCacheMBean.class);
}
private void updateCachingLevel(RetentionLevel level) {
Optional<CachingLevel> cachingLevel =
environment.cachingConfiguration().findLevelForLevelName(level.name());
Optional<CachingLevel> oldCachingLevel = cachingConfig.findLevelForLevelName(level.name());
if (!cachingLevel.isPresent()) {
log.warn("RetentionLevel with unconfigured Cache " + level.name());
} else if (!caches.containsKey(level.name()) || !oldCachingLevel.isPresent()) {
caches.put(level.name(), new LevelCache(environment, cachingLevel.get()));
if (levelIsAccessLevel(level) && !evictionStrategyMap.containsKey(level.name())) {
evictionStrategyMap.put(
level.name(),
new NaiveLRUStrategy(
level.name(),
metricStorage,
findAllFollowingLevelCaches(level),
cachingLevel.get().visibleCacheSize()));
}
log.debug("created new Cache for " + level.name());
} else if (oldCachingLevel.get().equals(cachingLevel.get())) {
log.debug("no changes in cachingLevel " + level.name());
} else if (oldCachingLevel.get().cacheLineWidth() != cachingLevel.get().cacheLineWidth()) {
caches.remove(oldCachingLevel.get().name());
caches.put(level.name(), new LevelCache(environment, cachingLevel.get()));
log.debug("dropped and created new cache for " + level.name());
} else {
log.debug("resizing level cache " + level.name());
if (levelIsAccessLevel(level)) {
caches
.get(oldCachingLevel.get().name())
.resizeAccessLevel(cachingLevel.get(), evictionStrategyMap.get(level.name()));
evictionStrategyMap.get(level.name()).resize(cachingLevel.get().visibleCacheSize());
} else {
caches.get(oldCachingLevel.get().name()).resize(cachingLevel.get());
}
}
}