/** * {@inheritDoc} * * <p>Computes KL as follows, where Γ(x) is the gamma function and ψ(x) is the digamma * function. * * <p>(α<sub>P</sub>-α<sub>Q</sub>)ψ(α<sub>P</sub>) - * ln(Γ(α<sub>P</sub>)) + ln(Γ(α<sub>Q</sub>)) + * α<sub>Q</sub>(ln(β<sub>P</sub>/β<sub>Q</sub>)) + * α<sub>P</sub>(β<sub>Q</sub>-β<sub>P</sub>)/β<sub>P</sub> * * @see <a * href="http://en.wikipedia.org/wiki/Gamma_distribution#Kullback.E2.80.93Leibler_divergence" * >Gamma distribution (Wikipedia)</a> */ @Override public double computeKLDivergence(IParameterizedMessage that) { if (that instanceof GammaParameters) { // KL(P|Q) == (ap-aq)*digamma(ap) - log(gamma(ap)) + log(gamma(aq)) + aq*(log(bp)-log(bq)) + // ap*(bq-bp)/bp final GammaParameters P = this, Q = (GammaParameters) that; final double ap = P.getAlpha(), aq = Q.getAlpha(); final double bp = P.getBeta(), bq = Q.getBeta(); double divergence = 0.0; if (ap != aq) { divergence += (ap - aq) * digamma(ap); divergence -= logGamma(ap); divergence += logGamma(aq); } if (bp != bq) { divergence += aq * (Math.log(bp) - Math.log(bq)) + ap * ((bq - bp) / bp); } return divergence; } throw new IllegalArgumentException( String.format("Expected '%s' but got '%s'", getClass(), that.getClass())); }
/** * {@inheritDoc} * * <p>Discrete messages compute KL using: * * <blockquote> * * <big>Σ</big> ln(P<sub>i</sub> / Q<sub>i</sub>) P<sub>i</sub> * * </blockquote> */ @Override public double computeKLDivergence(IParameterizedMessage that) { if (that instanceof DiscreteMessage) { // KL(P|Q) == sum(log(Pi/Qi) * Pi) // // To normalize you need to divide Pi by sum(Pi) and Qi by sum(Qi), denote these // by Ps and Qs: // // ==> sum(log((Pi/Ps)/(Qi/Qs)) * Pi/Ps) // // ==> 1/Ps * sum(log(Pi/Qi) * Pi + log(Qs/Ps) * Pi) // // ==> sum(Pi*(log(Pi) - log(Qi)))/Ps + log(Qs/Ps) // // This formulation allows you to perform the computation using a single loop. final DiscreteMessage P = this; final DiscreteMessage Q = (DiscreteMessage) that; final int size = P.size(); if (size != Q.size()) { throw new IllegalArgumentException( String.format("Mismatched domain sizes '%d' and '%d'", P.size(), Q.size())); } double Ps = 0.0, Qs = 0.0, unnormalizedKL = 0.0; for (int i = 0; i < size; ++i) { final double pw = P.getWeight(i); if (pw == 0.0) continue; final double qw = Q.getWeight(i); Ps += pw; Qs += qw; final double pe = P.getEnergy(i); final double qe = Q.getEnergy(i); unnormalizedKL += pw * (qe - pe); } return unnormalizedKL / Ps + Math.log(Qs / Ps); } throw new IllegalArgumentException( String.format("Expected '%s' but got '%s'", getClass(), that.getClass())); }