public void addInPlace(SentenceKey key, SentenceStatistics sentenceStatistics) {
for (Map<SentenceKey, EnsembleStatistics> impl : this.impl) {
EnsembleStatistics stats = impl.get(key);
if (stats == null) {
stats = new EnsembleStatistics(new LinkedList<SentenceStatistics>());
impl.put(key, stats);
}
stats.addInPlace(sentenceStatistics);
}
}
/** Run some sanity checks on the training statistics, to make sure they look valid. */
public void validate() {
for (Map<SentenceKey, EnsembleStatistics> map : impl) {
for (EnsembleStatistics stats : map.values()) {
for (SentenceStatistics component : stats.statisticsForClassifiers) {
assert !Counters.isUniformDistribution(component.relationDistribution, 1e-5);
Counters.normalize(
component.relationDistribution); // TODO(gabor) this shouldn't be necessary
assert (Math.abs(component.relationDistribution.totalCount() - 1.0)) < 1e-5;
}
assert (Math.abs(stats.mean().relationDistribution.totalCount() - 1.0)) < 1e-5;
assert !Counters.isUniformDistribution(stats.mean().relationDistribution, 1e-5);
}
}
}
public TrainingStatistics merge(TrainingStatistics other) {
Map<SentenceKey, EnsembleStatistics> newStats = new HashMap<>();
// Add elements from this statistics
for (Map<SentenceKey, EnsembleStatistics> map : this.impl) {
for (SentenceKey key : map.keySet()) {
newStats.put(key, new EnsembleStatistics(map.get(key)));
}
}
// Add elements from other statistics
for (Map<SentenceKey, EnsembleStatistics> map : other.impl) {
for (SentenceKey key : map.keySet()) {
EnsembleStatistics existing = newStats.get(key);
if (existing == null) {
existing = new EnsembleStatistics(new LinkedList<SentenceStatistics>());
newStats.put(key, existing);
}
existing.addInPlace(map.get(key));
}
}
// Return
return new TrainingStatistics(Maybe.Just(newStats));
}