示例#1
0
  /**
   * Make sure that empirical distribution of random Poisson(4)'s has P(X <= 5) close to actual
   * cumulative Poisson probability and that nextPoisson fails when mean is non-positive.
   */
  @Test
  public void testNextPoisson() {
    try {
      randomData.nextPoisson(0);
      Assert.fail("zero mean -- expecting MathIllegalArgumentException");
    } catch (MathIllegalArgumentException ex) {
      // ignored
    }
    try {
      randomData.nextPoisson(-1);
      Assert.fail("negative mean supplied -- MathIllegalArgumentException expected");
    } catch (MathIllegalArgumentException ex) {
      // ignored
    }
    try {
      randomData.nextPoisson(0);
      Assert.fail("0 mean supplied -- MathIllegalArgumentException expected");
    } catch (MathIllegalArgumentException ex) {
      // ignored
    }

    final double mean = 4.0d;
    final int len = 5;
    PoissonDistribution poissonDistribution = new PoissonDistribution(mean);
    Frequency f = new Frequency();
    randomData.reSeed(1000);
    for (int i = 0; i < largeSampleSize; i++) {
      f.addValue(randomData.nextPoisson(mean));
    }
    final long[] observed = new long[len];
    for (int i = 0; i < len; i++) {
      observed[i] = f.getCount(i + 1);
    }
    final double[] expected = new double[len];
    for (int i = 0; i < len; i++) {
      expected[i] = poissonDistribution.probability(i + 1) * largeSampleSize;
    }

    TestUtils.assertChiSquareAccept(expected, observed, 0.0001);
  }
示例#2
0
 @Test
 /** MATH-720 */
 public void testReseed() {
   PoissonDistribution x = new PoissonDistribution(3.0);
   x.reseedRandomGenerator(0);
   final double u = x.sample();
   PoissonDistribution y = new PoissonDistribution(3.0);
   y.reseedRandomGenerator(0);
   Assert.assertEquals(u, y.sample(), 0);
 }
示例#3
0
  /**
   * Verifies that nextPoisson(mean) generates an empirical distribution of values consistent with
   * PoissonDistributionImpl by generating 1000 values, computing a grouped frequency distribution
   * of the observed values and comparing this distribution to the corresponding expected
   * distribution computed using PoissonDistributionImpl. Uses ChiSquare test of goodness of fit to
   * evaluate the null hypothesis that the distributions are the same. If the null hypothesis can be
   * rejected with confidence 1 - alpha, the check fails.
   */
  public void checkNextPoissonConsistency(double mean) {
    // Generate sample values
    final int sampleSize = 1000; // Number of deviates to generate
    final int minExpectedCount = 7; // Minimum size of expected bin count
    long maxObservedValue = 0;
    final double alpha = 0.001; // Probability of false failure
    Frequency frequency = new Frequency();
    for (int i = 0; i < sampleSize; i++) {
      long value = randomData.nextPoisson(mean);
      if (value > maxObservedValue) {
        maxObservedValue = value;
      }
      frequency.addValue(value);
    }

    /*
     *  Set up bins for chi-square test.
     *  Ensure expected counts are all at least minExpectedCount.
     *  Start with upper and lower tail bins.
     *  Lower bin = [0, lower); Upper bin = [upper, +inf).
     */
    PoissonDistribution poissonDistribution = new PoissonDistribution(mean);
    int lower = 1;
    while (poissonDistribution.cumulativeProbability(lower - 1) * sampleSize < minExpectedCount) {
      lower++;
    }
    int upper = (int) (5 * mean); // Even for mean = 1, not much mass beyond 5
    while ((1 - poissonDistribution.cumulativeProbability(upper - 1)) * sampleSize
        < minExpectedCount) {
      upper--;
    }

    // Set bin width for interior bins.  For poisson, only need to look at end bins.
    int binWidth = 0;
    boolean widthSufficient = false;
    double lowerBinMass = 0;
    double upperBinMass = 0;
    while (!widthSufficient) {
      binWidth++;
      lowerBinMass = poissonDistribution.cumulativeProbability(lower - 1, lower + binWidth - 1);
      upperBinMass = poissonDistribution.cumulativeProbability(upper - binWidth - 1, upper - 1);
      widthSufficient = FastMath.min(lowerBinMass, upperBinMass) * sampleSize >= minExpectedCount;
    }

    /*
     *  Determine interior bin bounds.  Bins are
     *  [1, lower = binBounds[0]), [lower, binBounds[1]), [binBounds[1], binBounds[2]), ... ,
     *    [binBounds[binCount - 2], upper = binBounds[binCount - 1]), [upper, +inf)
     *
     */
    List<Integer> binBounds = new ArrayList<Integer>();
    binBounds.add(lower);
    int bound = lower + binWidth;
    while (bound < upper - binWidth) {
      binBounds.add(bound);
      bound += binWidth;
    }
    binBounds.add(
        upper); // The size of bin [binBounds[binCount - 2], upper) satisfies binWidth <= size <
                // 2*binWidth.

    // Compute observed and expected bin counts
    final int binCount = binBounds.size() + 1;
    long[] observed = new long[binCount];
    double[] expected = new double[binCount];

    // Bottom bin
    observed[0] = 0;
    for (int i = 0; i < lower; i++) {
      observed[0] += frequency.getCount(i);
    }
    expected[0] = poissonDistribution.cumulativeProbability(lower - 1) * sampleSize;

    // Top bin
    observed[binCount - 1] = 0;
    for (int i = upper; i <= maxObservedValue; i++) {
      observed[binCount - 1] += frequency.getCount(i);
    }
    expected[binCount - 1] =
        (1 - poissonDistribution.cumulativeProbability(upper - 1)) * sampleSize;

    // Interior bins
    for (int i = 1; i < binCount - 1; i++) {
      observed[i] = 0;
      for (int j = binBounds.get(i - 1); j < binBounds.get(i); j++) {
        observed[i] += frequency.getCount(j);
      } // Expected count is (mass in [binBounds[i-1], binBounds[i])) * sampleSize
      expected[i] =
          (poissonDistribution.cumulativeProbability(binBounds.get(i) - 1)
                  - poissonDistribution.cumulativeProbability(binBounds.get(i - 1) - 1))
              * sampleSize;
    }

    // Use chisquare test to verify that generated values are poisson(mean)-distributed
    ChiSquareTest chiSquareTest = new ChiSquareTest();
    // Fail if we can reject null hypothesis that distributions are the same
    if (chiSquareTest.chiSquareTest(expected, observed, alpha)) {
      StringBuilder msgBuffer = new StringBuilder();
      DecimalFormat df = new DecimalFormat("#.##");
      msgBuffer.append("Chisquare test failed for mean = ");
      msgBuffer.append(mean);
      msgBuffer.append(" p-value = ");
      msgBuffer.append(chiSquareTest.chiSquareTest(expected, observed));
      msgBuffer.append(" chisquare statistic = ");
      msgBuffer.append(chiSquareTest.chiSquare(expected, observed));
      msgBuffer.append(". \n");
      msgBuffer.append("bin\t\texpected\tobserved\n");
      for (int i = 0; i < expected.length; i++) {
        msgBuffer.append("[");
        msgBuffer.append(i == 0 ? 1 : binBounds.get(i - 1));
        msgBuffer.append(",");
        msgBuffer.append(i == binBounds.size() ? "inf" : binBounds.get(i));
        msgBuffer.append(")");
        msgBuffer.append("\t\t");
        msgBuffer.append(df.format(expected[i]));
        msgBuffer.append("\t\t");
        msgBuffer.append(observed[i]);
        msgBuffer.append("\n");
      }
      msgBuffer.append("This test can fail randomly due to sampling error with probability ");
      msgBuffer.append(alpha);
      msgBuffer.append(".");
      Assert.fail(msgBuffer.toString());
    }
  }
示例#4
0
  @Test
  public void testLSHEffect() {
    RandomGenerator random = RandomManager.getRandom();
    PoissonDistribution itemPerUserDist =
        new PoissonDistribution(
            random,
            20,
            PoissonDistribution.DEFAULT_EPSILON,
            PoissonDistribution.DEFAULT_MAX_ITERATIONS);
    int features = 20;
    ALSServingModel mainModel = new ALSServingModel(features, true, 1.0, null);
    ALSServingModel lshModel = new ALSServingModel(features, true, 0.5, null);

    int userItemCount = 20000;
    for (int user = 0; user < userItemCount; user++) {
      String userID = "U" + user;
      float[] vec = VectorMath.randomVectorF(features, random);
      mainModel.setUserVector(userID, vec);
      lshModel.setUserVector(userID, vec);
      int itemsPerUser = itemPerUserDist.sample();
      Collection<String> knownIDs = new ArrayList<>(itemsPerUser);
      for (int i = 0; i < itemsPerUser; i++) {
        knownIDs.add("I" + random.nextInt(userItemCount));
      }
      mainModel.addKnownItems(userID, knownIDs);
      lshModel.addKnownItems(userID, knownIDs);
    }

    for (int item = 0; item < userItemCount; item++) {
      String itemID = "I" + item;
      float[] vec = VectorMath.randomVectorF(features, random);
      mainModel.setItemVector(itemID, vec);
      lshModel.setItemVector(itemID, vec);
    }

    int numRecs = 10;
    Mean meanMatchLength = new Mean();
    for (int user = 0; user < userItemCount; user++) {
      String userID = "U" + user;
      List<Pair<String, Double>> mainRecs =
          mainModel.topN(new DotsFunction(mainModel.getUserVector(userID)), null, numRecs, null);
      List<Pair<String, Double>> lshRecs =
          lshModel.topN(new DotsFunction(lshModel.getUserVector(userID)), null, numRecs, null);
      int i = 0;
      while (i < lshRecs.size() && i < mainRecs.size() && lshRecs.get(i).equals(mainRecs.get(i))) {
        i++;
      }
      meanMatchLength.increment(i);
    }
    log.info("Mean matching prefix: {}", meanMatchLength.getResult());
    assertTrue(meanMatchLength.getResult() >= 4.0);

    meanMatchLength.clear();
    for (int item = 0; item < userItemCount; item++) {
      String itemID = "I" + item;
      List<Pair<String, Double>> mainRecs =
          mainModel.topN(
              new CosineAverageFunction(mainModel.getItemVector(itemID)), null, numRecs, null);
      List<Pair<String, Double>> lshRecs =
          lshModel.topN(
              new CosineAverageFunction(lshModel.getItemVector(itemID)), null, numRecs, null);
      int i = 0;
      while (i < lshRecs.size() && i < mainRecs.size() && lshRecs.get(i).equals(mainRecs.get(i))) {
        i++;
      }
      meanMatchLength.increment(i);
    }
    log.info("Mean matching prefix: {}", meanMatchLength.getResult());
    assertTrue(meanMatchLength.getResult() >= 5.0);
  }