@Test public void testOpenIfChangedResult() throws Exception { Directory dir = null; DirectoryTaxonomyWriter ltw = null; DirectoryTaxonomyReader ltr = null; try { dir = newDirectory(); ltw = new DirectoryTaxonomyWriter(dir); ltw.addCategory(new FacetLabel("a")); ltw.commit(); ltr = new DirectoryTaxonomyReader(dir); assertNull("Nothing has changed", TaxonomyReader.openIfChanged(ltr)); ltw.addCategory(new FacetLabel("b")); ltw.commit(); DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(ltr); assertNotNull("changes were committed", newtr); assertNull("Nothing has changed", TaxonomyReader.openIfChanged(newtr)); newtr.close(); } finally { IOUtils.close(ltw, ltr, dir); } }
@Test public void testCloseTwice() throws Exception { Directory dir = newDirectory(); DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir); ltw.addCategory(new FacetLabel("a")); ltw.close(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.close(); ltr.close(); // no exception should be thrown dir.close(); }
@Test public void testCloseAfterIncRef() throws Exception { Directory dir = newDirectory(); DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir); ltw.addCategory(new FacetLabel("a")); ltw.close(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.incRef(); ltr.close(); // should not fail as we incRef() before close ltr.getSize(); ltr.decRef(); dir.close(); }
@Test public void testAlreadyClosed() throws Exception { Directory dir = newDirectory(); DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir); ltw.addCategory(new FacetLabel("a")); ltw.close(); DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir); ltr.close(); expectThrows( AlreadyClosedException.class, () -> { ltr.getSize(); }); dir.close(); }
@Test public void testBackwardsCompatibility() throws Exception { // tests that if the taxonomy index doesn't have the INDEX_EPOCH // property (supports pre-3.6 indexes), all still works. Directory dir = newDirectory(); // create an empty index first, so that DirTaxoWriter initializes indexEpoch to 1. new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE); taxoWriter.close(); DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir); assertEquals( 1, Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH))); assertNull(TaxonomyReader.openIfChanged(taxoReader)); taxoReader.close(); dir.close(); }
@Test public void testOpenIfChangedReuse() throws Exception { // test the reuse of data from the old DTR instance for (boolean nrt : new boolean[] {false, true}) { Directory dir = newDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.addCategory(cp_a); if (!nrt) writer.commit(); DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir); // fill r1's caches assertEquals(1, r1.getOrdinal(cp_a)); assertEquals(cp_a, r1.getPath(1)); FacetLabel cp_b = new FacetLabel("b"); writer.addCategory(cp_b); if (!nrt) writer.commit(); DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1); assertNotNull(r2); // add r2's categories to the caches assertEquals(2, r2.getOrdinal(cp_b)); assertEquals(cp_b, r2.getPath(2)); // check that r1 doesn't see cp_b assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b)); assertNull(r1.getPath(2)); r1.close(); r2.close(); writer.close(); dir.close(); } }
public void testConcurrency() throws Exception { final int ncats = atLeast(100000); // add many categories final int range = ncats * 3; // affects the categories selection final AtomicInteger numCats = new AtomicInteger(ncats); final Directory dir = newDirectory(); final ConcurrentHashMap<String, String> values = new ConcurrentHashMap<String, String>(); final double d = random().nextDouble(); final TaxonomyWriterCache cache; if (d < 0.7) { // this is the fastest, yet most memory consuming cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3); } else if (TEST_NIGHTLY && d > 0.98) { // this is the slowest, but tests the writer concurrency when no caching is done. // only pick it during NIGHTLY tests, and even then, with very low chances. cache = NO_OP_CACHE; } else { // this is slower than CL2O, but less memory consuming, and exercises finding categories on // disk too. cache = new LruTaxonomyWriterCache(ncats / 10); } final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache); Thread[] addThreads = new Thread[atLeast(4)]; for (int z = 0; z < addThreads.length; z++) { addThreads[z] = new Thread() { @Override public void run() { Random random = random(); while (numCats.decrementAndGet() > 0) { try { int value = random.nextInt(range); CategoryPath cp = new CategoryPath( Integer.toString(value / 1000), Integer.toString(value / 10000), Integer.toString(value / 100000), Integer.toString(value)); int ord = tw.addCategory(cp); assertTrue( "invalid parent for ordinal " + ord + ", category " + cp, tw.getParent(ord) != -1); String l1 = cp.subpath(1).toString('/'); String l2 = cp.subpath(2).toString('/'); String l3 = cp.subpath(3).toString('/'); String l4 = cp.subpath(4).toString('/'); values.put(l1, l1); values.put(l2, l2); values.put(l3, l3); values.put(l4, l4); } catch (IOException e) { throw new RuntimeException(e); } } } }; } for (Thread t : addThreads) t.start(); for (Thread t : addThreads) t.join(); tw.close(); DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir); assertEquals( "mismatch number of categories", values.size() + 1, dtr.getSize()); // +1 for root category int[] parents = dtr.getParallelTaxonomyArrays().parents(); for (String cat : values.keySet()) { CategoryPath cp = new CategoryPath(cat, '/'); assertTrue("category not found " + cp, dtr.getOrdinal(cp) > 0); int level = cp.length; int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0) CategoryPath path = CategoryPath.EMPTY; for (int i = 0; i < level; i++) { path = cp.subpath(i + 1); int ord = dtr.getOrdinal(path); assertEquals("invalid parent for cp=" + path, parentOrd, parents[ord]); parentOrd = ord; // next level should have this parent } } dtr.close(); dir.close(); }
@Test public void testGetChildren() throws Exception { Directory dir = newDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir); int numCategories = atLeast(10); int numA = 0, numB = 0; Random random = random(); // add the two categories for which we'll also add children (so asserts are simpler) taxoWriter.addCategory(new FacetLabel("a")); taxoWriter.addCategory(new FacetLabel("b")); for (int i = 0; i < numCategories; i++) { if (random.nextBoolean()) { taxoWriter.addCategory(new FacetLabel("a", Integer.toString(i))); ++numA; } else { taxoWriter.addCategory(new FacetLabel("b", Integer.toString(i))); ++numB; } } // add category with no children taxoWriter.addCategory(new FacetLabel("c")); taxoWriter.close(); DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir); // non existing category ChildrenIterator it = taxoReader.getChildren(taxoReader.getOrdinal(new FacetLabel("invalid"))); assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next()); // a category with no children it = taxoReader.getChildren(taxoReader.getOrdinal(new FacetLabel("c"))); assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next()); // arbitrary negative ordinal it = taxoReader.getChildren(-2); assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next()); // root's children Set<String> roots = new HashSet<>(Arrays.asList("a", "b", "c")); it = taxoReader.getChildren(TaxonomyReader.ROOT_ORDINAL); while (!roots.isEmpty()) { FacetLabel root = taxoReader.getPath(it.next()); assertEquals(1, root.length); assertTrue(roots.remove(root.components[0])); } assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next()); for (int i = 0; i < 2; i++) { FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b"); int ordinal = taxoReader.getOrdinal(cp); it = taxoReader.getChildren(ordinal); int numChildren = 0; int child; while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) { FacetLabel path = taxoReader.getPath(child); assertEquals(2, path.length); assertEquals(path.components[0], i == 0 ? "a" : "b"); ++numChildren; } int expected = i == 0 ? numA : numB; assertEquals("invalid num children", expected, numChildren); } taxoReader.close(); dir.close(); }
@Test public void testOpenIfChangedReplaceTaxonomy() throws Exception { // test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate // only can work with NRT as well Directory src = newDirectory(); DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src); FacetLabel cp_b = new FacetLabel("b"); w.addCategory(cp_b); w.close(); for (boolean nrt : new boolean[] {false, true}) { Directory dir = newDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.addCategory(cp_a); if (!nrt) writer.commit(); DirectoryTaxonomyReader r1 = nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir); // fill r1's caches assertEquals(1, r1.getOrdinal(cp_a)); assertEquals(cp_a, r1.getPath(1)); // now replace taxonomy writer.replaceTaxonomy(src); if (!nrt) writer.commit(); DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1); assertNotNull(r2); // fill r2's caches assertEquals(1, r2.getOrdinal(cp_b)); assertEquals(cp_b, r2.getPath(1)); // check that r1 doesn't see cp_b assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b)); assertEquals(cp_a, r1.getPath(1)); // check that r2 doesn't see cp_a assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a)); assertEquals(cp_b, r2.getPath(1)); r2.close(); r1.close(); writer.close(); dir.close(); } src.close(); }
@Test public void testOpenIfChangedReuseAfterRecreate() throws Exception { // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy Directory dir = newDirectory(); DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir); FacetLabel cp_a = new FacetLabel("a"); writer.addCategory(cp_a); writer.close(); DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir); // fill r1's caches assertEquals(1, r1.getOrdinal(cp_a)); assertEquals(cp_a, r1.getPath(1)); // now recreate, add a different category writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE); FacetLabel cp_b = new FacetLabel("b"); writer.addCategory(cp_b); writer.close(); DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1); assertNotNull(r2); // fill r2's caches assertEquals(1, r2.getOrdinal(cp_b)); assertEquals(cp_b, r2.getPath(1)); // check that r1 doesn't see cp_b assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b)); assertEquals(cp_a, r1.getPath(1)); // check that r2 doesn't see cp_a assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a)); assertEquals(cp_b, r2.getPath(1)); r2.close(); r1.close(); dir.close(); }