@Test
public void testSanityWithSerde() throws Exception {
setupSimpleWithSerde(4);
Assert.assertEquals(4, supplier.getBaseBuffers().size());
assertIndexMatchesVals();
setupSimpleWithSerde(2);
Assert.assertEquals(8, supplier.getBaseBuffers().size());
assertIndexMatchesVals();
}
@Test
public void testmaxIntsInBuffer() throws Exception {
Assert.assertEquals(
CompressedPools.BUFFER_SIZE, CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForBytes(1));
Assert.assertEquals(
CompressedPools.BUFFER_SIZE / 2,
CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForBytes(2));
Assert.assertEquals(
CompressedPools.BUFFER_SIZE / 4,
CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForBytes(4));
Assert.assertEquals(CompressedPools.BUFFER_SIZE, 0x10000); // nearest power of 2 is 2^14
Assert.assertEquals(1 << 14, CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForBytes(3));
}
private void setupSimple(final int chunkSize) {
CloseQuietly.close(indexed);
vals = new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16};
supplier =
CompressedVSizeIntsIndexedSupplier.fromList(
Ints.asList(vals),
Ints.max(vals),
chunkSize,
ByteOrder.nativeOrder(),
compressionStrategy);
indexed = supplier.get();
}
@Test
public void testChunkTooBig() throws Exception {
for (int maxValue : MAX_VALUES) {
final int maxChunkSize = CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForValue(maxValue);
try {
setupLargeChunks(maxChunkSize + 1, 10 * (maxChunkSize + 1), maxValue);
Assert.fail();
} catch (IllegalArgumentException e) {
Assert.assertTrue("chunk too big for maxValue " + maxValue, true);
}
}
}
@Test
public void testLargeChunks() throws Exception {
for (int maxValue : MAX_VALUES) {
final int maxChunkSize = CompressedVSizeIntsIndexedSupplier.maxIntsInBufferForValue(maxValue);
setupLargeChunks(maxChunkSize, 10 * maxChunkSize, maxValue);
Assert.assertEquals(10, supplier.getBaseBuffers().size());
assertIndexMatchesVals();
setupLargeChunks(maxChunkSize, 10 * maxChunkSize + 1, maxValue);
Assert.assertEquals(11, supplier.getBaseBuffers().size());
assertIndexMatchesVals();
setupLargeChunks(1, 0xFFFF, maxValue);
Assert.assertEquals(0xFFFF, supplier.getBaseBuffers().size());
assertIndexMatchesVals();
setupLargeChunks(maxChunkSize / 2, 10 * (maxChunkSize / 2) + 1, maxValue);
Assert.assertEquals(11, supplier.getBaseBuffers().size());
assertIndexMatchesVals();
}
}
private void makeWithSerde(final int chunkSize) throws IOException {
CloseQuietly.close(indexed);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
final CompressedVSizeIntsIndexedSupplier theSupplier =
CompressedVSizeIntsIndexedSupplier.fromList(
Ints.asList(vals), Ints.max(vals), chunkSize, byteOrder, compressionStrategy);
theSupplier.writeToChannel(Channels.newChannel(baos));
final byte[] bytes = baos.toByteArray();
Assert.assertEquals(theSupplier.getSerializedSize(), bytes.length);
supplier = CompressedVSizeIntsIndexedSupplier.fromByteBuffer(ByteBuffer.wrap(bytes), byteOrder);
indexed = supplier.get();
}
// This test attempts to cause a race condition with the DirectByteBuffers, it's non-deterministic
// in causing it,
// which sucks but I can't think of a way to deterministically cause it...
@Test
public void testConcurrentThreadReads() throws Exception {
setupSimple(4);
final AtomicReference<String> reason = new AtomicReference<>("none");
final int numRuns = 1000;
final CountDownLatch startLatch = new CountDownLatch(1);
final CountDownLatch stopLatch = new CountDownLatch(2);
final AtomicBoolean failureHappened = new AtomicBoolean(false);
new Thread(
new Runnable() {
@Override
public void run() {
try {
startLatch.await();
} catch (InterruptedException e) {
failureHappened.set(true);
reason.set("interrupt.");
stopLatch.countDown();
return;
}
try {
for (int i = 0; i < numRuns; ++i) {
for (int j = 0; j < indexed.size(); ++j) {
final long val = vals[j];
final long indexedVal = indexed.get(j);
if (Longs.compare(val, indexedVal) != 0) {
failureHappened.set(true);
reason.set(String.format("Thread1[%d]: %d != %d", j, val, indexedVal));
stopLatch.countDown();
return;
}
}
}
} catch (Exception e) {
e.printStackTrace();
failureHappened.set(true);
reason.set(e.getMessage());
}
stopLatch.countDown();
}
})
.start();
final IndexedInts indexed2 = supplier.get();
try {
new Thread(
new Runnable() {
@Override
public void run() {
try {
startLatch.await();
} catch (InterruptedException e) {
stopLatch.countDown();
return;
}
try {
for (int i = 0; i < numRuns; ++i) {
for (int j = indexed2.size() - 1; j >= 0; --j) {
final long val = vals[j];
final long indexedVal = indexed2.get(j);
if (Longs.compare(val, indexedVal) != 0) {
failureHappened.set(true);
reason.set(String.format("Thread2[%d]: %d != %d", j, val, indexedVal));
stopLatch.countDown();
return;
}
}
}
} catch (Exception e) {
e.printStackTrace();
reason.set(e.getMessage());
failureHappened.set(true);
}
stopLatch.countDown();
}
})
.start();
startLatch.countDown();
stopLatch.await();
} finally {
CloseQuietly.close(indexed2);
}
if (failureHappened.get()) {
Assert.fail("Failure happened. Reason: " + reason.get());
}
}