// And then one more test just for Bytes-based symbol table
public void testByteBasedSymbolTable() throws Exception {
// combination of short, medium1/2, long names...
final String JSON =
aposToQuotes(
"{'abc':1, 'abc\\u0000':2, '\\u0000abc':3, "
// then some medium
+ "'abc123':4,'abcd1234':5,"
+ "'abcd1234a':6,'abcd1234abcd':7,"
+ "'abcd1234abcd1':8"
+ "}");
JsonFactory f = new JsonFactory();
JsonParser p = f.createParser(JSON.getBytes("UTF-8"));
ByteQuadsCanonicalizer symbols = _findSymbols(p);
assertEquals(0, symbols.size());
_streamThrough(p);
assertEquals(8, symbols.size());
p.close();
// and, for fun, try again
p = f.createParser(JSON.getBytes("UTF-8"));
_streamThrough(p);
symbols = _findSymbols(p);
assertEquals(8, symbols.size());
p.close();
p = f.createParser(JSON.getBytes("UTF-8"));
_streamThrough(p);
symbols = _findSymbols(p);
assertEquals(8, symbols.size());
p.close();
}
public void testSyntheticWithBytesNew() throws IOException {
// pass seed, to keep results consistent:
final int SEED = 33333;
ByteQuadsCanonicalizer symbols =
ByteQuadsCanonicalizer.createRoot(SEED).makeChild(JsonFactory.Feature.collectDefaults());
final int COUNT = 12000;
for (int i = 0; i < COUNT; ++i) {
String id = fieldNameFor(i);
int[] quads = calcQuads(id.getBytes("UTF-8"));
symbols.addName(id, quads, quads.length);
}
assertEquals(COUNT, symbols.size());
assertEquals(16384, symbols.bucketCount());
// fragile, but essential to verify low collision counts;
// anywhere between 70-80% primary matches
assertEquals(8534, symbols.primaryCount());
// secondary between 10-20%
assertEquals(2534, symbols.secondaryCount());
// and most of remaining in tertiary
assertEquals(932, symbols.tertiaryCount());
// so that spill-over is empty or close to
assertEquals(0, symbols.spilloverCount());
}
public void testShortQuotedDirectBytes() throws IOException {
final int COUNT = 400;
ByteQuadsCanonicalizer symbols =
ByteQuadsCanonicalizer.createRoot(123).makeChild(JsonFactory.Feature.collectDefaults());
for (int i = 0; i < COUNT; ++i) {
String id = String.format("\\u%04x", i);
int[] quads = calcQuads(id.getBytes("UTF-8"));
symbols.addName(id, quads, quads.length);
}
assertEquals(COUNT, symbols.size());
assertEquals(512, symbols.bucketCount());
assertEquals(285, symbols.primaryCount());
assertEquals(90, symbols.secondaryCount());
assertEquals(25, symbols.tertiaryCount());
assertEquals(0, symbols.spilloverCount());
}
// Another variant, but with 1-quad names
public void testCollisionsWithBytesNew187b() throws IOException {
ByteQuadsCanonicalizer symbols =
ByteQuadsCanonicalizer.createRoot(1).makeChild(JsonFactory.Feature.collectDefaults());
final int COUNT = 10000;
for (int i = 0; i < COUNT; ++i) {
String id = String.valueOf(i);
int[] quads = calcQuads(id.getBytes("UTF-8"));
symbols.addName(id, quads, quads.length);
}
assertEquals(COUNT, symbols.size());
assertEquals(16384, symbols.bucketCount());
// fragile, but essential to verify low collision counts;
// here bit low primary, 55%
assertEquals(5402, symbols.primaryCount());
// secondary higher than usual, above 25%
assertEquals(2744, symbols.secondaryCount());
// and most of remaining in tertiary
assertEquals(1834, symbols.tertiaryCount());
// with a bit of spillover
assertEquals(20, symbols.spilloverCount());
}
// [core#187]: unexpectedly high number of collisions for straight numbers
public void testCollisionsWithBytesNew187a() throws IOException {
ByteQuadsCanonicalizer symbols =
ByteQuadsCanonicalizer.createRoot(1).makeChild(JsonFactory.Feature.collectDefaults());
final int COUNT = 43000;
for (int i = 0; i < COUNT; ++i) {
String id = String.valueOf(10000 + i);
int[] quads = calcQuads(id.getBytes("UTF-8"));
symbols.addName(id, quads, quads.length);
}
assertEquals(COUNT, symbols.size());
assertEquals(65536, symbols.bucketCount());
/* 29-Mar-2015, tatu: To get collision counts down for this
* test took quite a bit of tweaking...
*/
assertEquals(32342, symbols.primaryCount());
assertEquals(8863, symbols.secondaryCount());
assertEquals(1795, symbols.tertiaryCount());
// finally managed to get this to 0; other variants produced thousands
assertEquals(0, symbols.spilloverCount());
}
public void testShortNameCollisionsDirectNew() throws IOException {
final int COUNT = 700;
{
ByteQuadsCanonicalizer symbols =
ByteQuadsCanonicalizer.createRoot(333).makeChild(JsonFactory.Feature.collectDefaults());
for (int i = 0; i < COUNT; ++i) {
String id = String.valueOf((char) i);
int[] quads = calcQuads(id.getBytes("UTF-8"));
symbols.addName(id, quads, quads.length);
}
assertEquals(COUNT, symbols.size());
assertEquals(1024, symbols.bucketCount());
// Primary is good, but secondary spills cluster in nasty way...
assertEquals(564, symbols.primaryCount());
assertEquals(122, symbols.secondaryCount());
assertEquals(14, symbols.tertiaryCount());
assertEquals(0, symbols.spilloverCount());
assertEquals(
COUNT,
symbols.primaryCount()
+ symbols.secondaryCount()
+ symbols.tertiaryCount()
+ symbols.spilloverCount());
}
}
// Since 2.6
public void testThousandsOfSymbolsWithNew() throws IOException {
final int SEED = 33333;
ByteQuadsCanonicalizer symbolsBRoot = ByteQuadsCanonicalizer.createRoot(SEED);
final Charset utf8 = Charset.forName("UTF-8");
int exp = 0;
ByteQuadsCanonicalizer symbolsB = null;
// loop to get
for (int doc = 0; doc < 100; ++doc) {
symbolsB = symbolsBRoot.makeChild(JsonFactory.Feature.collectDefaults());
for (int i = 0; i < 250; ++i) {
String name = "f_" + doc + "_" + i;
int[] quads = calcQuads(name.getBytes(utf8));
symbolsB.addName(name, quads, quads.length);
String n = symbolsB.findName(quads, quads.length);
assertEquals(name, n);
}
symbolsB.release();
exp += 250;
if (exp > ByteQuadsCanonicalizer.MAX_ENTRIES_FOR_REUSE) {
exp = 0;
}
assertEquals(exp, symbolsBRoot.size());
}
/* 05-Feb-2015, tatu: Fragile, but it is important to ensure that collision
* rates are not accidentally increased...
*/
assertEquals(6250, symbolsB.size());
assertEquals(4761, symbolsB.primaryCount()); // 80% primary hit rate
assertEquals(1190, symbolsB.secondaryCount()); // 13% secondary
assertEquals(299, symbolsB.tertiaryCount()); // 7% tertiary
assertEquals(0, symbolsB.spilloverCount()); // and couple of leftovers
}
