public void operateOnPartition( PartitionDefinition definition, RowIterator inputIterator, RowEmitter outputEmitter) { errorHandler.enterOperateOnPartition(definition, inputIterator, outputEmitter); try { // Collect input rows for observed and expected values ArrayList<Double> expectedList = new ArrayList<Double>(); ArrayList<Long> observedList = new ArrayList<Long>(); while (inputIterator.advanceToNextRow()) { errorHandler.enterOperateOnRow(inputIterator, outputEmitter); if (inputIterator.isNullAt(observedArgumentIdx) || inputIterator.isNullAt(expectedArgumentIdx)) throw new IllegalArgumentException("observed and expected values cannot be null"); expectedList.add(inputIterator.getDoubleAt(expectedArgumentIdx)); observedList.add(inputIterator.getLongAt(observedArgumentIdx)); errorHandler.exitOperateOnRow(); } double[] expected = new double[expectedList.size()]; for (int i = 0; i < expected.length; i++) expected[i] = expectedList.get(i); long[] observed = new long[observedList.size()]; for (int i = 0; i < observed.length; i++) observed[i] = observedList.get(i); // Run test double pValue = chiSquareTest.chiSquareTest(expected, observed); // Emit result accumulator.emit(inputIterator, outputEmitter); outputEmitter.addDouble(pValue); outputEmitter.emitRow(); } catch (IllegalArgumentException e) { errorHandler.catchException(e); return; // End this partition and go to next if stopOnError is set to false (otherwise // exception is thrown) } errorHandler.exitOperateOnPartition(); }
/** * Verifies that nextPoisson(mean) generates an empirical distribution of values consistent with * PoissonDistributionImpl by generating 1000 values, computing a grouped frequency distribution * of the observed values and comparing this distribution to the corresponding expected * distribution computed using PoissonDistributionImpl. Uses ChiSquare test of goodness of fit to * evaluate the null hypothesis that the distributions are the same. If the null hypothesis can be * rejected with confidence 1 - alpha, the check fails. */ public void checkNextPoissonConsistency(double mean) { // Generate sample values final int sampleSize = 1000; // Number of deviates to generate final int minExpectedCount = 7; // Minimum size of expected bin count long maxObservedValue = 0; final double alpha = 0.001; // Probability of false failure Frequency frequency = new Frequency(); for (int i = 0; i < sampleSize; i++) { long value = randomData.nextPoisson(mean); if (value > maxObservedValue) { maxObservedValue = value; } frequency.addValue(value); } /* * Set up bins for chi-square test. * Ensure expected counts are all at least minExpectedCount. * Start with upper and lower tail bins. * Lower bin = [0, lower); Upper bin = [upper, +inf). */ PoissonDistribution poissonDistribution = new PoissonDistribution(mean); int lower = 1; while (poissonDistribution.cumulativeProbability(lower - 1) * sampleSize < minExpectedCount) { lower++; } int upper = (int) (5 * mean); // Even for mean = 1, not much mass beyond 5 while ((1 - poissonDistribution.cumulativeProbability(upper - 1)) * sampleSize < minExpectedCount) { upper--; } // Set bin width for interior bins. For poisson, only need to look at end bins. int binWidth = 0; boolean widthSufficient = false; double lowerBinMass = 0; double upperBinMass = 0; while (!widthSufficient) { binWidth++; lowerBinMass = poissonDistribution.cumulativeProbability(lower - 1, lower + binWidth - 1); upperBinMass = poissonDistribution.cumulativeProbability(upper - binWidth - 1, upper - 1); widthSufficient = FastMath.min(lowerBinMass, upperBinMass) * sampleSize >= minExpectedCount; } /* * Determine interior bin bounds. Bins are * [1, lower = binBounds[0]), [lower, binBounds[1]), [binBounds[1], binBounds[2]), ... , * [binBounds[binCount - 2], upper = binBounds[binCount - 1]), [upper, +inf) * */ List<Integer> binBounds = new ArrayList<Integer>(); binBounds.add(lower); int bound = lower + binWidth; while (bound < upper - binWidth) { binBounds.add(bound); bound += binWidth; } binBounds.add( upper); // The size of bin [binBounds[binCount - 2], upper) satisfies binWidth <= size < // 2*binWidth. // Compute observed and expected bin counts final int binCount = binBounds.size() + 1; long[] observed = new long[binCount]; double[] expected = new double[binCount]; // Bottom bin observed[0] = 0; for (int i = 0; i < lower; i++) { observed[0] += frequency.getCount(i); } expected[0] = poissonDistribution.cumulativeProbability(lower - 1) * sampleSize; // Top bin observed[binCount - 1] = 0; for (int i = upper; i <= maxObservedValue; i++) { observed[binCount - 1] += frequency.getCount(i); } expected[binCount - 1] = (1 - poissonDistribution.cumulativeProbability(upper - 1)) * sampleSize; // Interior bins for (int i = 1; i < binCount - 1; i++) { observed[i] = 0; for (int j = binBounds.get(i - 1); j < binBounds.get(i); j++) { observed[i] += frequency.getCount(j); } // Expected count is (mass in [binBounds[i-1], binBounds[i])) * sampleSize expected[i] = (poissonDistribution.cumulativeProbability(binBounds.get(i) - 1) - poissonDistribution.cumulativeProbability(binBounds.get(i - 1) - 1)) * sampleSize; } // Use chisquare test to verify that generated values are poisson(mean)-distributed ChiSquareTest chiSquareTest = new ChiSquareTest(); // Fail if we can reject null hypothesis that distributions are the same if (chiSquareTest.chiSquareTest(expected, observed, alpha)) { StringBuilder msgBuffer = new StringBuilder(); DecimalFormat df = new DecimalFormat("#.##"); msgBuffer.append("Chisquare test failed for mean = "); msgBuffer.append(mean); msgBuffer.append(" p-value = "); msgBuffer.append(chiSquareTest.chiSquareTest(expected, observed)); msgBuffer.append(" chisquare statistic = "); msgBuffer.append(chiSquareTest.chiSquare(expected, observed)); msgBuffer.append(". \n"); msgBuffer.append("bin\t\texpected\tobserved\n"); for (int i = 0; i < expected.length; i++) { msgBuffer.append("["); msgBuffer.append(i == 0 ? 1 : binBounds.get(i - 1)); msgBuffer.append(","); msgBuffer.append(i == binBounds.size() ? "inf" : binBounds.get(i)); msgBuffer.append(")"); msgBuffer.append("\t\t"); msgBuffer.append(df.format(expected[i])); msgBuffer.append("\t\t"); msgBuffer.append(observed[i]); msgBuffer.append("\n"); } msgBuffer.append("This test can fail randomly due to sampling error with probability "); msgBuffer.append(alpha); msgBuffer.append("."); Assert.fail(msgBuffer.toString()); } }