/**
* Load string map from a URL.
*
* @param mapURL URL for map file.
* @param separator Field separator.
* @param qualifier Quote character.
* @param encoding Character encoding for the file.
* @throws FileNotFoundException If input file does not exist.
* @throws IOException If input file cannot be opened.
* @return Map with values read from file.
*/
public static Map<String, String> loadMap(
URL mapURL, String separator, String qualifier, String encoding)
throws IOException, FileNotFoundException {
Map<String, String> map = MapFactory.createNewMap();
if (mapURL != null) {
BufferedReader bufferedReader =
new BufferedReader(new UnicodeReader(mapURL.openStream(), encoding));
String inputLine = bufferedReader.readLine();
String[] tokens;
while (inputLine != null) {
tokens = inputLine.split(separator);
if (tokens.length > 1) {
map.put(tokens[0], tokens[1]);
}
inputLine = bufferedReader.readLine();
}
bufferedReader.close();
}
return map;
}
private Counter<String> uniformRandom() {
Counter<String> uniformRandom =
new ClassicCounter<>(MapFactory.<String, MutableDouble>linkedHashMapFactory());
for (Map<SentenceKey, EnsembleStatistics> impl : this.impl) {
for (Map.Entry<SentenceKey, EnsembleStatistics> entry : impl.entrySet()) {
uniformRandom.setCount(entry.getKey().sentenceHash, 1.0);
}
}
return uniformRandom;
}
private Counter<String> highKLFromMean() {
// Get confidences
Counter<String> kl =
new ClassicCounter<>(MapFactory.<String, MutableDouble>linkedHashMapFactory());
for (Map<SentenceKey, EnsembleStatistics> impl : this.impl) {
for (Map.Entry<SentenceKey, EnsembleStatistics> entry : impl.entrySet()) {
kl.setCount(entry.getKey().sentenceHash, entry.getValue().averageKLFromMean());
}
}
return kl;
}
private Counter<String> lowAverageConfidence() {
// Get confidences
Counter<String> lowConfidence =
new ClassicCounter<>(MapFactory.<String, MutableDouble>linkedHashMapFactory());
for (Map<SentenceKey, EnsembleStatistics> impl : this.impl) {
for (Map.Entry<SentenceKey, EnsembleStatistics> entry : impl.entrySet()) {
SentenceStatistics average = entry.getValue().mean();
for (double confidence : average.confidence) {
lowConfidence.setCount(entry.getKey().sentenceHash, 1 - confidence);
}
}
}
return lowConfidence;
}
public static Map<String, Object> readMap(XContentParser parser, MapFactory mapFactory)
throws IOException {
Map<String, Object> map = mapFactory.newMap();
XContentParser.Token t = parser.currentToken();
if (t == null) {
t = parser.nextToken();
}
if (t == XContentParser.Token.START_OBJECT) {
t = parser.nextToken();
}
for (; t == XContentParser.Token.FIELD_NAME; t = parser.nextToken()) {
// Must point to field name
String fieldName = parser.currentName();
// And then the value...
t = parser.nextToken();
Object value = readValue(parser, mapFactory, t);
map.put(fieldName, value);
}
return map;
}
public SentenceStatistics mean() {
double sumConfidence = 0;
int countWithConfidence = 0;
Counter<String> avePredictions =
new ClassicCounter<>(MapFactory.<String, MutableDouble>linkedHashMapFactory());
// Sum
for (SentenceStatistics stat : this.statisticsForClassifiers) {
for (Double confidence : stat.confidence) {
sumConfidence += confidence;
countWithConfidence += 1;
}
assert Math.abs(stat.relationDistribution.totalCount() - 1.0) < 1e-5;
for (Map.Entry<String, Double> entry : stat.relationDistribution.entrySet()) {
assert entry.getValue() >= 0.0;
assert entry.getValue() == stat.relationDistribution.getCount(entry.getKey());
avePredictions.incrementCount(entry.getKey(), entry.getValue());
assert stat.relationDistribution.getCount(entry.getKey())
== stat.relationDistribution.getCount(entry.getKey());
}
}
// Normalize
double aveConfidence = sumConfidence / ((double) countWithConfidence);
// Return
if (this.statisticsForClassifiers.size() > 1) {
Counters.divideInPlace(avePredictions, (double) this.statisticsForClassifiers.size());
}
if (Math.abs(avePredictions.totalCount() - 1.0) > 1e-5) {
throw new IllegalStateException("Mean relation distribution is not a distribution!");
}
assert this.statisticsForClassifiers.size() > 1
|| this.statisticsForClassifiers.size() == 0
|| Counters.equals(
avePredictions,
statisticsForClassifiers.iterator().next().relationDistribution,
1e-5);
return countWithConfidence > 0
? new SentenceStatistics(avePredictions, aveConfidence)
: new SentenceStatistics(avePredictions);
}