@Test
public void testNextZipf() {
ZipfDistributionTest testInstance = new ZipfDistributionTest();
int[] densityPoints = testInstance.makeDensityTestPoints();
double[] densityValues = testInstance.makeDensityTestValues();
int sampleSize = 1000;
int length = TestUtils.eliminateZeroMassPoints(densityPoints, densityValues);
ZipfDistribution distribution = (ZipfDistribution) testInstance.makeDistribution();
double[] expectedCounts = new double[length];
long[] observedCounts = new long[length];
for (int i = 0; i < length; i++) {
expectedCounts[i] = sampleSize * densityValues[i];
}
randomData.reSeed(1000);
for (int i = 0; i < sampleSize; i++) {
int value =
randomData.nextZipf(distribution.getNumberOfElements(), distribution.getExponent());
for (int j = 0; j < length; j++) {
if (value == densityPoints[j]) {
observedCounts[j]++;
}
}
}
TestUtils.assertChiSquareAccept(densityPoints, expectedCounts, observedCounts, .001);
}
@Test
public void testWithFunctions() {
final double tol = 1E-12;
final double[] data =
new double[] {
-0.012086732064244697,
-0.24975668704012527,
0.5706168483164684,
-0.322111769955327,
0.24166759508327315,
// Double.NaN,
// Double.POSITIVE_INFINITY,
0.16698443218942854,
-0.10427763937565114,
-0.15595963093172435,
-0.028075857595882995,
-0.24137994506058857,
0.47543170476574426,
-0.07495595384947631,
0.37445697625436497,
-0.09944199541668033
};
final DescriptiveStatisticalSummary<StorelessUnivariateStatistic> ds =
createDescriptiveStatistics();
final DescriptiveStatisticalSummary<StorelessUnivariateStatistic> dsMean =
createDescriptiveStatistics().withMean(new Mean());
final DescriptiveStatisticalSummary<StorelessUnivariateStatistic> dsKurt =
createDescriptiveStatistics().withKurtosis(new Kurtosis());
final DescriptiveStatisticalSummary<StorelessUnivariateStatistic> dsSkew =
createDescriptiveStatistics().withSkewness(new Skewness());
final DescriptiveStatisticalSummary<StorelessUnivariateStatistic> ds50 =
createDescriptiveStatistics().withPercentile(new PSquarePercentile(50d));
final DescriptiveStatisticalSummary<StorelessUnivariateStatistic> dsVar =
createDescriptiveStatistics().withVariance(new Variance());
for (double d : data) {
ds.addValue(d);
dsMean.addValue(d);
ds50.addValue(d);
dsKurt.addValue(d);
dsSkew.addValue(d);
dsVar.addValue(d);
}
TestUtils.assertEquals(ds.getMean(), dsMean.getMean(), tol);
TestUtils.assertEquals(ds.getMean(), dsKurt.getMean(), tol);
TestUtils.assertEquals(ds.getMean(), dsSkew.getMean(), tol);
TestUtils.assertEquals(ds.getMean(), ds50.getMean(), tol);
TestUtils.assertEquals(ds.getMean(), dsVar.getMean(), tol);
TestUtils.assertEquals(ds.getMean(), dsKurt.getMean(), tol);
TestUtils.assertEquals(ds.getKurtosis(), dsKurt.getKurtosis(), tol);
TestUtils.assertEquals(ds.getSkewness(), dsSkew.getSkewness(), tol);
TestUtils.assertEquals(ds.getPercentile(), ds50.getPercentile(), tol);
TestUtils.assertEquals(ds.getVariance(), dsVar.getVariance(), tol);
}
@Test
public void testNextWeibull() {
double[] quartiles = TestUtils.getDistributionQuartiles(new WeibullDistribution(1.2, 2.1));
long[] counts = new long[4];
randomData.reSeed(1000);
for (int i = 0; i < 1000; i++) {
double value = randomData.nextWeibull(1.2, 2.1);
TestUtils.updateCounts(value, counts, quartiles);
}
TestUtils.assertChiSquareAccept(expected, counts, 0.001);
}
private void checkNextUniformUniform(double min, double max) {
// Set up bin bounds - min, binBound[0], ..., binBound[binCount-2], max
final int binCount = 5;
final double binSize =
max / binCount - min / binCount; // Prevent overflow in extreme value case
final double[] binBounds = new double[binCount - 1];
binBounds[0] = min + binSize;
for (int i = 1; i < binCount - 1; i++) {
binBounds[i] = binBounds[i - 1] + binSize; // + instead of * to avoid overflow in extreme case
}
final Frequency freq = new Frequency();
for (int i = 0; i < smallSampleSize; i++) {
final double value = randomData.nextUniform(min, max);
Assert.assertTrue("nextUniform range", (value > min) && (value < max));
// Find bin
int j = 0;
while (j < binCount - 1 && value > binBounds[j]) {
j++;
}
freq.addValue(j);
}
final long[] observed = new long[binCount];
for (int i = 0; i < binCount; i++) {
observed[i] = freq.getCount(i);
}
final double[] expected = new double[binCount];
for (int i = 0; i < binCount; i++) {
expected[i] = 1d / binCount;
}
TestUtils.assertChiSquareAccept(expected, observed, 0.01);
}
/** test failure modes and distribution of nextGaussian() */
@Test
public void testNextGaussian() {
try {
randomData.nextGaussian(0, 0);
Assert.fail("zero sigma -- MathIllegalArgumentException expected");
} catch (MathIllegalArgumentException ex) {
// ignored
}
double[] quartiles = TestUtils.getDistributionQuartiles(new NormalDistribution(0, 1));
long[] counts = new long[4];
randomData.reSeed(1000);
for (int i = 0; i < 1000; i++) {
double value = randomData.nextGaussian(0, 1);
TestUtils.updateCounts(value, counts, quartiles);
}
TestUtils.assertChiSquareAccept(expected, counts, 0.001);
}
@Test
public void testNextGamma() {
double[] quartiles;
long[] counts;
// Tests shape > 1, one case in the rejection sampling
quartiles = TestUtils.getDistributionQuartiles(new GammaDistribution(4, 2));
counts = new long[4];
randomData.reSeed(1000);
for (int i = 0; i < 1000; i++) {
double value = randomData.nextGamma(4, 2);
TestUtils.updateCounts(value, counts, quartiles);
}
TestUtils.assertChiSquareAccept(expected, counts, 0.001);
// Tests shape <= 1, another case in the rejection sampling
quartiles = TestUtils.getDistributionQuartiles(new GammaDistribution(0.3, 3));
counts = new long[4];
randomData.reSeed(1000);
for (int i = 0; i < 1000; i++) {
double value = randomData.nextGamma(0.3, 3);
TestUtils.updateCounts(value, counts, quartiles);
}
TestUtils.assertChiSquareAccept(expected, counts, 0.001);
}
@Test
public void testSamplerHelper2() {
final double tol = 1e-12;
final double[] testValues = {
-1e0, -1e-1, -1e-2, -1e-3, -1e-4, -1e-5, -1e-6, -1e-7, -1e-8, -1e-9, -1e-10, -1e-11, 0.,
1e-11, 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0
};
for (double testValue : testValues) {
final double expected = FastMath.expm1(testValue);
TestUtils.assertRelativelyEquals(
expected, ZipfRejectionInversionSampler.helper2(testValue) * testValue, tol);
}
}
/** test dispersion and failure modes for nextHex() */
@Test
@Retry(3)
public void testNextSecureHex() {
try {
randomData.nextSecureHexString(-1);
Assert.fail("negative length -- MathIllegalArgumentException expected");
} catch (MathIllegalArgumentException ex) {
// ignored
}
try {
randomData.nextSecureHexString(0);
Assert.fail("zero length -- MathIllegalArgumentException expected");
} catch (MathIllegalArgumentException ex) {
// ignored
}
String hexString = randomData.nextSecureHexString(3);
if (hexString.length() != 3) {
Assert.fail("incorrect length for generated string");
}
hexString = randomData.nextSecureHexString(1);
if (hexString.length() != 1) {
Assert.fail("incorrect length for generated string");
}
try {
hexString = randomData.nextSecureHexString(0);
Assert.fail("zero length requested -- expecting MathIllegalArgumentException");
} catch (MathIllegalArgumentException ex) {
// ignored
}
Frequency f = new Frequency();
for (int i = 0; i < smallSampleSize; i++) {
hexString = randomData.nextSecureHexString(100);
if (hexString.length() != 100) {
Assert.fail("incorrect length for generated string");
}
for (int j = 0; j < hexString.length(); j++) {
f.addValue(hexString.substring(j, j + 1));
}
}
double[] expected = new double[16];
long[] observed = new long[16];
for (int i = 0; i < 16; i++) {
expected[i] = (double) smallSampleSize * 100 / 16;
observed[i] = f.getCount(hex[i]);
}
TestUtils.assertChiSquareAccept(expected, observed, 0.001);
}
/** tests for nextPermutation */
@Test
public void testNextPermutation() {
int[][] p = {{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}};
long[] observed = {0, 0, 0, 0, 0, 0};
double[] expected = {100, 100, 100, 100, 100, 100};
for (int i = 0; i < 600; i++) {
int[] perm = randomData.nextPermutation(3, 3);
observed[findPerm(p, perm)]++;
}
String[] labels = {
"{0, 1, 2}", "{ 0, 2, 1 }", "{ 1, 0, 2 }", "{ 1, 2, 0 }", "{ 2, 0, 1 }", "{ 2, 1, 0 }"
};
TestUtils.assertChiSquareAccept(labels, expected, observed, 0.001);
// Check size = 1 boundary case
int[] perm = randomData.nextPermutation(1, 1);
if ((perm.length != 1) || (perm[0] != 0)) {
Assert.fail("bad permutation for n = 1, sample k = 1");
// Make sure we fail for k size > n
try {
perm = randomData.nextPermutation(2, 3);
Assert.fail("permutation k > n, expecting MathIllegalArgumentException");
} catch (MathIllegalArgumentException ex) {
// ignored
}
// Make sure we fail for n = 0
try {
perm = randomData.nextPermutation(0, 0);
Assert.fail("permutation k = n = 0, expecting MathIllegalArgumentException");
} catch (MathIllegalArgumentException ex) {
// ignored
}
// Make sure we fail for k < n < 0
try {
perm = randomData.nextPermutation(-1, -3);
Assert.fail("permutation k < n < 0, expecting MathIllegalArgumentException");
} catch (MathIllegalArgumentException ex) {
// ignored
}
}
}
private void checkNextSecureIntUniform(int min, int max) {
final Frequency freq = new Frequency();
for (int i = 0; i < smallSampleSize; i++) {
final int value = randomData.nextSecureInt(min, max);
Assert.assertTrue("nextInt range", (value >= min) && (value <= max));
freq.addValue(value);
}
final int len = max - min + 1;
final long[] observed = new long[len];
for (int i = 0; i < len; i++) {
observed[i] = freq.getCount(min + i);
}
final double[] expected = new double[len];
for (int i = 0; i < len; i++) {
expected[i] = 1d / len;
}
TestUtils.assertChiSquareAccept(expected, observed, 0.0001);
}
private void checkNextLongUniform(long min, long max) {
final Frequency freq = new Frequency();
for (int i = 0; i < smallSampleSize; i++) {
final long value = randomData.nextLong(min, max);
Assert.assertTrue(
"nextLong range: " + value + " " + min + " " + max, (value >= min) && (value <= max));
freq.addValue(value);
}
final int len = ((int) (max - min)) + 1;
final long[] observed = new long[len];
for (int i = 0; i < len; i++) {
observed[i] = freq.getCount(min + i);
}
final double[] expected = new double[len];
for (int i = 0; i < len; i++) {
expected[i] = 1d / len;
}
TestUtils.assertChiSquareAccept(expected, observed, 0.01);
}
/** Test sampling for various number of points and exponents. */
@Test
public void testSamplingExtended() {
int sampleSize = 1000;
int[] numPointsValues = {
2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 35, 40, 45, 50, 60, 70, 80, 90, 100
};
double[] exponentValues = {
1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 2e-1, 5e-1, 1. - 1e-9, 1.0,
1. + 1e-9, 1.1, 1.2, 1.3, 1.5, 1.6, 1.7, 1.8, 2.0, 2.5, 3.0, 4., 5., 6., 7., 8., 9., 10., 20.,
30., 100., 150.
};
for (int numPoints : numPointsValues) {
for (double exponent : exponentValues) {
double weightSum = 0.;
double[] weights = new double[numPoints];
for (int i = numPoints; i >= 1; i -= 1) {
weights[i - 1] = Math.pow(i, -exponent);
weightSum += weights[i - 1];
}
ZipfDistribution distribution = new ZipfDistribution(numPoints, exponent);
distribution.reseedRandomGenerator(
6); // use fixed seed, the test is expected to fail for more than 50% of all seeds
// because each test case can fail with probability 0.001, the chance that all test
// cases do not fail is 0.999^(32*22) = 0.49442874426
double[] expectedCounts = new double[numPoints];
long[] observedCounts = new long[numPoints];
for (int i = 0; i < numPoints; i++) {
expectedCounts[i] = sampleSize * (weights[i] / weightSum);
}
int[] sample = distribution.sample(sampleSize);
for (int s : sample) {
observedCounts[s - 1]++;
}
TestUtils.assertChiSquareAccept(expectedCounts, observedCounts, 0.001);
}
}
}
/**
* Make sure that empirical distribution of random Poisson(4)'s has P(X <= 5) close to actual
* cumulative Poisson probability and that nextPoisson fails when mean is non-positive.
*/
@Test
public void testNextPoisson() {
try {
randomData.nextPoisson(0);
Assert.fail("zero mean -- expecting MathIllegalArgumentException");
} catch (MathIllegalArgumentException ex) {
// ignored
}
try {
randomData.nextPoisson(-1);
Assert.fail("negative mean supplied -- MathIllegalArgumentException expected");
} catch (MathIllegalArgumentException ex) {
// ignored
}
try {
randomData.nextPoisson(0);
Assert.fail("0 mean supplied -- MathIllegalArgumentException expected");
} catch (MathIllegalArgumentException ex) {
// ignored
}
final double mean = 4.0d;
final int len = 5;
PoissonDistribution poissonDistribution = new PoissonDistribution(mean);
Frequency f = new Frequency();
randomData.reSeed(1000);
for (int i = 0; i < largeSampleSize; i++) {
f.addValue(randomData.nextPoisson(mean));
}
final long[] observed = new long[len];
for (int i = 0; i < len; i++) {
observed[i] = f.getCount(i + 1);
}
final double[] expected = new double[len];
for (int i = 0; i < len; i++) {
expected[i] = poissonDistribution.probability(i + 1) * largeSampleSize;
}
TestUtils.assertChiSquareAccept(expected, observed, 0.0001);
}
/** test failure modes and distribution of nextExponential() */
@Test
public void testNextExponential() {
try {
randomData.nextExponential(-1);
Assert.fail("negative mean -- expecting MathIllegalArgumentException");
} catch (MathIllegalArgumentException ex) {
// ignored
}
try {
randomData.nextExponential(0);
Assert.fail("zero mean -- expecting MathIllegalArgumentException");
} catch (MathIllegalArgumentException ex) {
// ignored
}
double[] quartiles;
long[] counts;
// Mean 1
quartiles = TestUtils.getDistributionQuartiles(new ExponentialDistribution(1));
counts = new long[4];
randomData.reSeed(1000);
for (int i = 0; i < 1000; i++) {
double value = randomData.nextExponential(1);
TestUtils.updateCounts(value, counts, quartiles);
}
TestUtils.assertChiSquareAccept(expected, counts, 0.001);
// Mean 5
quartiles = TestUtils.getDistributionQuartiles(new ExponentialDistribution(5));
counts = new long[4];
randomData.reSeed(1000);
for (int i = 0; i < 1000; i++) {
double value = randomData.nextExponential(5);
TestUtils.updateCounts(value, counts, quartiles);
}
TestUtils.assertChiSquareAccept(expected, counts, 0.001);
}
@Test
public void testSerial() {
ArrayFieldVector<Fraction> v = new ArrayFieldVector<Fraction>(vec1);
Assert.assertEquals(v, TestUtils.serializeAndRecover(v));
}
@Test
public void testSummaryConsistency() {
final DescriptiveStatisticalSummary<UnivariateStatistic> dstats = new DescriptiveStatistics(5);
final DescriptiveStatisticalSummary<StorelessUnivariateStatistic> sstats =
createDescriptiveStatistics();
final double tol = 1E-12;
for (int i = 0; i < 20; i++) {
dstats.addValue(i);
sstats.clear();
double[] values = ((DescriptiveStatistics) dstats).getValues();
for (int j = 0; j < values.length; j++) {
sstats.addValue(values[j]);
}
TestUtils.assertEquals(dstats.getMean(), sstats.getMean(), tol);
TestUtils.assertEquals(new Mean().evaluate(values), dstats.getMean(), tol);
TestUtils.assertEquals(dstats.getMax(), sstats.getMax(), tol);
TestUtils.assertEquals(new Max().evaluate(values), dstats.getMax(), tol);
TestUtils.assertEquals(dstats.getGeometricMean(), sstats.getGeometricMean(), tol);
TestUtils.assertEquals(new GeometricMean().evaluate(values), dstats.getGeometricMean(), tol);
TestUtils.assertEquals(dstats.getMin(), sstats.getMin(), tol);
TestUtils.assertEquals(new Min().evaluate(values), dstats.getMin(), tol);
TestUtils.assertEquals(dstats.getStandardDeviation(), sstats.getStandardDeviation(), tol);
TestUtils.assertEquals(dstats.getVariance(), sstats.getVariance(), tol);
TestUtils.assertEquals(new Variance().evaluate(values), dstats.getVariance(), tol);
TestUtils.assertEquals(dstats.getSum(), sstats.getSum(), tol);
TestUtils.assertEquals(new Sum().evaluate(values), dstats.getSum(), tol);
TestUtils.assertEquals(dstats.getSumsq(), sstats.getSumsq(), tol);
TestUtils.assertEquals(new SumOfSquares().evaluate(values), dstats.getSumsq(), tol);
TestUtils.assertEquals(
new Variance(false).evaluate(values),
((DescriptiveStatistics) dstats).getPopulationVariance(),
tol);
}
}
