private void logPerFile(final BlockCacheUtil.CachedBlocksByFile cbsbf)
throws JsonGenerationException, JsonMappingException, IOException {
for (Map.Entry<String, NavigableSet<CachedBlock>> e :
cbsbf.getCachedBlockStatsByFile().entrySet()) {
int count = 0;
long size = 0;
int countData = 0;
long sizeData = 0;
for (CachedBlock cb : e.getValue()) {
count++;
size += cb.getSize();
BlockType bt = cb.getBlockType();
if (bt != null && bt.isData()) {
countData++;
sizeData += cb.getSize();
}
}
LOG.info(
"filename="
+ e.getKey()
+ ", count="
+ count
+ ", countData="
+ countData
+ ", size="
+ size
+ ", sizeData="
+ sizeData);
LOG.info(BlockCacheUtil.toJSON(e.getKey(), e.getValue()));
}
}
private BlockCacheUtil.CachedBlocksByFile logPerBlock(final BlockCache bc)
throws JsonGenerationException, JsonMappingException, IOException {
BlockCacheUtil.CachedBlocksByFile cbsbf = new BlockCacheUtil.CachedBlocksByFile();
for (CachedBlock cb : bc) {
LOG.info(cb.toString());
LOG.info(BlockCacheUtil.toJSON(bc));
cbsbf.update(cb);
}
return cbsbf;
}
public void testQueueSmallBlockEdgeCase() throws Exception {
CachedBlock cb1 = new CachedBlock(1000, "cb1", 1);
CachedBlock cb2 = new CachedBlock(1500, "cb2", 2);
CachedBlock cb3 = new CachedBlock(1000, "cb3", 3);
CachedBlock cb4 = new CachedBlock(1500, "cb4", 4);
CachedBlock cb5 = new CachedBlock(1000, "cb5", 5);
CachedBlock cb6 = new CachedBlock(1750, "cb6", 6);
CachedBlock cb7 = new CachedBlock(1000, "cb7", 7);
CachedBlock cb8 = new CachedBlock(1500, "cb8", 8);
CachedBlock cb9 = new CachedBlock(1000, "cb9", 9);
CachedBlock cb10 = new CachedBlock(1500, "cb10", 10);
CachedBlockQueue queue = new CachedBlockQueue(10000, 1000);
queue.add(cb1);
queue.add(cb2);
queue.add(cb3);
queue.add(cb4);
queue.add(cb5);
queue.add(cb6);
queue.add(cb7);
queue.add(cb8);
queue.add(cb9);
queue.add(cb10);
CachedBlock cb0 = new CachedBlock(10 + CachedBlock.PER_BLOCK_OVERHEAD, "cb0", 0);
queue.add(cb0);
// This is older so we must include it, but it will not end up kicking
// anything out because (heapSize - cb8.heapSize + cb0.heapSize < maxSize)
// and we must always maintain heapSize >= maxSize once we achieve it.
// We expect cb0 through cb8 to be in the queue
long expectedSize =
cb1.heapSize()
+ cb2.heapSize()
+ cb3.heapSize()
+ cb4.heapSize()
+ cb5.heapSize()
+ cb6.heapSize()
+ cb7.heapSize()
+ cb8.heapSize()
+ cb0.heapSize();
assertEquals(queue.heapSize(), expectedSize);
for (int i = 0; i <= 8; i++) {
assertEquals(queue.pollLast().getCacheKey().getHfileName(), "cb" + i);
}
}
public void testQueue() throws Exception {
CachedBlock cb1 = new CachedBlock(1000, "cb1", 1);
CachedBlock cb2 = new CachedBlock(1500, "cb2", 2);
CachedBlock cb3 = new CachedBlock(1000, "cb3", 3);
CachedBlock cb4 = new CachedBlock(1500, "cb4", 4);
CachedBlock cb5 = new CachedBlock(1000, "cb5", 5);
CachedBlock cb6 = new CachedBlock(1750, "cb6", 6);
CachedBlock cb7 = new CachedBlock(1000, "cb7", 7);
CachedBlock cb8 = new CachedBlock(1500, "cb8", 8);
CachedBlock cb9 = new CachedBlock(1000, "cb9", 9);
CachedBlock cb10 = new CachedBlock(1500, "cb10", 10);
CachedBlockQueue queue = new CachedBlockQueue(10000, 1000);
queue.add(cb1);
queue.add(cb2);
queue.add(cb3);
queue.add(cb4);
queue.add(cb5);
queue.add(cb6);
queue.add(cb7);
queue.add(cb8);
queue.add(cb9);
queue.add(cb10);
// We expect cb1 through cb8 to be in the queue
long expectedSize =
cb1.heapSize()
+ cb2.heapSize()
+ cb3.heapSize()
+ cb4.heapSize()
+ cb5.heapSize()
+ cb6.heapSize()
+ cb7.heapSize()
+ cb8.heapSize();
assertEquals(queue.heapSize(), expectedSize);
for (int i = 1; i <= 8; i++) {
assertEquals(queue.pollLast().getCacheKey().getHfileName(), "cb" + i);
}
}