public void prepare(ColumnFamilyStore cfs) {
if (tree.partitioner() instanceof RandomPartitioner) {
// You can't beat an even tree distribution for md5
tree.init();
} else {
List<DecoratedKey> keys = new ArrayList<DecoratedKey>();
for (DecoratedKey sample : cfs.keySamples(request.range)) {
assert request.range.contains(sample.token)
: "Token " + sample.token + " is not within range " + request.range;
keys.add(sample);
}
if (keys.isEmpty()) {
// use an even tree distribution
tree.init();
} else {
int numkeys = keys.size();
Random random = new Random();
// sample the column family using random keys from the index
while (true) {
DecoratedKey dk = keys.get(random.nextInt(numkeys));
if (!tree.split(dk.token)) break;
}
}
}
logger.debug("Prepared AEService tree of size " + tree.size() + " for " + request);
ranges = tree.invalids();
}
@Test
public void testInvalids() {
Iterator<TreeRange> ranges;
// (zero, zero]
ranges = mt.invalids(new Range(tok(-1), tok(-1)));
assertEquals(new Range(tok(-1), tok(-1)), ranges.next());
assertFalse(ranges.hasNext());
// all invalid
mt.split(tok(4));
mt.split(tok(2));
mt.split(tok(6));
mt.split(tok(3));
mt.split(tok(5));
ranges = mt.invalids(new Range(tok(-1), tok(-1)));
assertEquals(new Range(tok(-1), tok(2)), ranges.next());
assertEquals(new Range(tok(2), tok(3)), ranges.next());
assertEquals(new Range(tok(3), tok(4)), ranges.next());
assertEquals(new Range(tok(4), tok(5)), ranges.next());
assertEquals(new Range(tok(5), tok(6)), ranges.next());
assertEquals(new Range(tok(6), tok(-1)), ranges.next());
assertFalse(ranges.hasNext());
// some invalid
mt.get(tok(2)).hash("non-null!".getBytes());
mt.get(tok(4)).hash("non-null!".getBytes());
mt.get(tok(5)).hash("non-null!".getBytes());
mt.get(tok(-1)).hash("non-null!".getBytes());
ranges = mt.invalids(new Range(tok(-1), tok(-1)));
assertEquals(new Range(tok(2), tok(3)), ranges.next());
assertEquals(new Range(tok(5), tok(6)), ranges.next());
assertFalse(ranges.hasNext());
// some invalid in left subrange
ranges = mt.invalids(new Range(tok(-1), tok(6)));
assertEquals(new Range(tok(2), tok(3)), ranges.next());
assertEquals(new Range(tok(5), tok(6)), ranges.next());
assertFalse(ranges.hasNext());
// some invalid in right subrange
ranges = mt.invalids(new Range(tok(2), tok(-1)));
assertEquals(new Range(tok(2), tok(3)), ranges.next());
assertEquals(new Range(tok(5), tok(6)), ranges.next());
assertFalse(ranges.hasNext());
}
/**
* Generate two trees with different splits, but containing the same keys, and check that they
* compare equally.
*
* <p>The set of keys used in this test is: #{2,4,6,8,12,14,0}
*/
@Test
public void testValidateTree() {
TOKEN_SCALE = new BigInteger("16"); // this test needs slightly more resolution
Range full = new Range(tok(-1), tok(-1));
Iterator<TreeRange> ranges;
MerkleTree mt2 = new MerkleTree(partitioner, RECOMMENDED_DEPTH, Integer.MAX_VALUE);
mt.split(tok(8));
mt.split(tok(4));
mt.split(tok(12));
mt.split(tok(6));
mt.split(tok(10));
ranges = mt.invalids(full);
ranges.next().addAll(new HIterator(2, 4)); // (-1,4]: depth 2
ranges.next().addAll(new HIterator(6)); // (4,6]
ranges.next().addAll(new HIterator(8)); // (6,8]
ranges.next().addAll(new HIterator(/*empty*/ new int[0])); // (8,10]
ranges.next().addAll(new HIterator(12)); // (10,12]
ranges.next().addAll(new HIterator(14, -1)); // (12,-1]: depth 2
mt2.split(tok(8));
mt2.split(tok(4));
mt2.split(tok(12));
mt2.split(tok(2));
mt2.split(tok(10));
mt2.split(tok(9));
mt2.split(tok(11));
ranges = mt2.invalids(full);
ranges.next().addAll(new HIterator(2)); // (-1,2]
ranges.next().addAll(new HIterator(4)); // (2,4]
ranges.next().addAll(new HIterator(6, 8)); // (4,8]: depth 2
ranges.next().addAll(new HIterator(/*empty*/ new int[0])); // (8,9]
ranges.next().addAll(new HIterator(/*empty*/ new int[0])); // (9,10]
ranges.next().addAll(new HIterator(/*empty*/ new int[0])); // (10,11]: depth 4
ranges.next().addAll(new HIterator(12)); // (11,12]: depth 4
ranges.next().addAll(new HIterator(14, -1)); // (12,-1]: depth 2
byte[] mthash = mt.hash(full);
byte[] mt2hash = mt2.hash(full);
assertHashEquals("Tree hashes did not match: " + mt + " && " + mt2, mthash, mt2hash);
}
@Test
public void testDifference() {
Range full = new Range(tok(-1), tok(-1));
int maxsize = 16;
mt.maxsize(maxsize);
MerkleTree mt2 = new MerkleTree(partitioner, RECOMMENDED_DEPTH, maxsize);
mt.init();
mt2.init();
TreeRange leftmost = null;
TreeRange middle = null;
TreeRange rightmost = null;
// compact the leftmost, and split the rightmost
Iterator<TreeRange> ranges = mt.invalids(full);
leftmost = ranges.next();
rightmost = null;
while (ranges.hasNext()) rightmost = ranges.next();
mt.compact(leftmost.right);
leftmost = mt.get(leftmost.right); // leftmost is now a larger range
mt.split(rightmost.right);
// set the hash for the left neighbor of rightmost
middle = mt.get(rightmost.left);
middle.hash("arbitrary!".getBytes());
byte depth = middle.depth;
// add dummy hashes to the rest of both trees
for (TreeRange range : mt.invalids(full)) range.addAll(new HIterator(range.right));
for (TreeRange range : mt2.invalids(full)) range.addAll(new HIterator(range.right));
// trees should disagree for leftmost, (middle.left, rightmost.right]
List<TreeRange> diffs = MerkleTree.difference(mt, mt2);
assertEquals(diffs + " contains wrong number of differences:", 2, diffs.size());
assertTrue(diffs.contains(leftmost));
assertTrue(diffs.contains(new Range(middle.left, rightmost.right)));
}
@Test
public void testHashRandom() {
int max = 1000000;
TOKEN_SCALE = new BigInteger("" + max);
mt = new MerkleTree(partitioner, RECOMMENDED_DEPTH, 32);
Random random = new Random();
while (true) {
if (!mt.split(tok(random.nextInt(max)))) break;
}
// validate the tree
TreeRangeIterator ranges = mt.invalids(new Range(tok(-1), tok(-1)));
for (TreeRange range : ranges) range.addHash(new RowHash(range.right, new byte[0]));
assert null != mt.hash(new Range(tok(-1), tok(-1))) : "Could not hash tree " + mt;
}
@Test
public void testSerialization() throws Exception {
Range full = new Range(tok(-1), tok(-1));
ByteArrayOutputStream bout = new ByteArrayOutputStream();
ObjectOutputStream oout = new ObjectOutputStream(bout);
// populate and validate the tree
mt.maxsize(256);
mt.init();
for (TreeRange range : mt.invalids(full)) range.addAll(new HIterator(range.right));
byte[] initialhash = mt.hash(full);
oout.writeObject(mt);
oout.close();
ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
ObjectInputStream oin = new ObjectInputStream(bin);
MerkleTree restored = (MerkleTree) oin.readObject();
// restore partitioner after serialization
restored.partitioner(partitioner);
assertHashEquals(initialhash, restored.hash(full));
}