@Test
  public void testOpenIfChangedResult() throws Exception {
    Directory dir = null;
    DirectoryTaxonomyWriter ltw = null;
    DirectoryTaxonomyReader ltr = null;

    try {
      dir = newDirectory();
      ltw = new DirectoryTaxonomyWriter(dir);

      ltw.addCategory(new FacetLabel("a"));
      ltw.commit();

      ltr = new DirectoryTaxonomyReader(dir);
      assertNull("Nothing has changed", TaxonomyReader.openIfChanged(ltr));

      ltw.addCategory(new FacetLabel("b"));
      ltw.commit();

      DirectoryTaxonomyReader newtr = TaxonomyReader.openIfChanged(ltr);
      assertNotNull("changes were committed", newtr);
      assertNull("Nothing has changed", TaxonomyReader.openIfChanged(newtr));
      newtr.close();
    } finally {
      IOUtils.close(ltw, ltr, dir);
    }
  }
  @Test
  public void testCloseTwice() throws Exception {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir);
    ltw.addCategory(new FacetLabel("a"));
    ltw.close();

    DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
    ltr.close();
    ltr.close(); // no exception should be thrown

    dir.close();
  }
  @Test
  public void testCloseAfterIncRef() throws Exception {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir);
    ltw.addCategory(new FacetLabel("a"));
    ltw.close();

    DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
    ltr.incRef();
    ltr.close();

    // should not fail as we incRef() before close
    ltr.getSize();
    ltr.decRef();

    dir.close();
  }
  @Test
  public void testAlreadyClosed() throws Exception {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter ltw = new DirectoryTaxonomyWriter(dir);
    ltw.addCategory(new FacetLabel("a"));
    ltw.close();

    DirectoryTaxonomyReader ltr = new DirectoryTaxonomyReader(dir);
    ltr.close();
    expectThrows(
        AlreadyClosedException.class,
        () -> {
          ltr.getSize();
        });

    dir.close();
  }
  @Test
  public void testBackwardsCompatibility() throws Exception {
    // tests that if the taxonomy index doesn't have the INDEX_EPOCH
    // property (supports pre-3.6 indexes), all still works.
    Directory dir = newDirectory();

    // create an empty index first, so that DirTaxoWriter initializes indexEpoch to 1.
    new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null)).close();

    DirectoryTaxonomyWriter taxoWriter =
        new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
    taxoWriter.close();

    DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);
    assertEquals(
        1,
        Integer.parseInt(taxoReader.getCommitUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH)));
    assertNull(TaxonomyReader.openIfChanged(taxoReader));
    taxoReader.close();

    dir.close();
  }
  @Test
  public void testOpenIfChangedReuse() throws Exception {
    // test the reuse of data from the old DTR instance
    for (boolean nrt : new boolean[] {false, true}) {
      Directory dir = newDirectory();
      DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);

      FacetLabel cp_a = new FacetLabel("a");
      writer.addCategory(cp_a);
      if (!nrt) writer.commit();

      DirectoryTaxonomyReader r1 =
          nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
      // fill r1's caches
      assertEquals(1, r1.getOrdinal(cp_a));
      assertEquals(cp_a, r1.getPath(1));

      FacetLabel cp_b = new FacetLabel("b");
      writer.addCategory(cp_b);
      if (!nrt) writer.commit();

      DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
      assertNotNull(r2);

      // add r2's categories to the caches
      assertEquals(2, r2.getOrdinal(cp_b));
      assertEquals(cp_b, r2.getPath(2));

      // check that r1 doesn't see cp_b
      assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
      assertNull(r1.getPath(2));

      r1.close();
      r2.close();
      writer.close();
      dir.close();
    }
  }
  public void testConcurrency() throws Exception {
    final int ncats = atLeast(100000); // add many categories
    final int range = ncats * 3; // affects the categories selection
    final AtomicInteger numCats = new AtomicInteger(ncats);
    final Directory dir = newDirectory();
    final ConcurrentHashMap<String, String> values = new ConcurrentHashMap<String, String>();
    final double d = random().nextDouble();
    final TaxonomyWriterCache cache;
    if (d < 0.7) {
      // this is the fastest, yet most memory consuming
      cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3);
    } else if (TEST_NIGHTLY && d > 0.98) {
      // this is the slowest, but tests the writer concurrency when no caching is done.
      // only pick it during NIGHTLY tests, and even then, with very low chances.
      cache = NO_OP_CACHE;
    } else {
      // this is slower than CL2O, but less memory consuming, and exercises finding categories on
      // disk too.
      cache = new LruTaxonomyWriterCache(ncats / 10);
    }
    final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache);
    Thread[] addThreads = new Thread[atLeast(4)];
    for (int z = 0; z < addThreads.length; z++) {
      addThreads[z] =
          new Thread() {
            @Override
            public void run() {
              Random random = random();
              while (numCats.decrementAndGet() > 0) {
                try {
                  int value = random.nextInt(range);
                  CategoryPath cp =
                      new CategoryPath(
                          Integer.toString(value / 1000),
                          Integer.toString(value / 10000),
                          Integer.toString(value / 100000),
                          Integer.toString(value));
                  int ord = tw.addCategory(cp);
                  assertTrue(
                      "invalid parent for ordinal " + ord + ", category " + cp,
                      tw.getParent(ord) != -1);
                  String l1 = cp.subpath(1).toString('/');
                  String l2 = cp.subpath(2).toString('/');
                  String l3 = cp.subpath(3).toString('/');
                  String l4 = cp.subpath(4).toString('/');
                  values.put(l1, l1);
                  values.put(l2, l2);
                  values.put(l3, l3);
                  values.put(l4, l4);
                } catch (IOException e) {
                  throw new RuntimeException(e);
                }
              }
            }
          };
    }

    for (Thread t : addThreads) t.start();
    for (Thread t : addThreads) t.join();
    tw.close();

    DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir);
    assertEquals(
        "mismatch number of categories", values.size() + 1, dtr.getSize()); // +1 for root category
    int[] parents = dtr.getParallelTaxonomyArrays().parents();
    for (String cat : values.keySet()) {
      CategoryPath cp = new CategoryPath(cat, '/');
      assertTrue("category not found " + cp, dtr.getOrdinal(cp) > 0);
      int level = cp.length;
      int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
      CategoryPath path = CategoryPath.EMPTY;
      for (int i = 0; i < level; i++) {
        path = cp.subpath(i + 1);
        int ord = dtr.getOrdinal(path);
        assertEquals("invalid parent for cp=" + path, parentOrd, parents[ord]);
        parentOrd = ord; // next level should have this parent
      }
    }
    dtr.close();

    dir.close();
  }
  @Test
  public void testGetChildren() throws Exception {
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
    int numCategories = atLeast(10);
    int numA = 0, numB = 0;
    Random random = random();
    // add the two categories for which we'll also add children (so asserts are simpler)
    taxoWriter.addCategory(new FacetLabel("a"));
    taxoWriter.addCategory(new FacetLabel("b"));
    for (int i = 0; i < numCategories; i++) {
      if (random.nextBoolean()) {
        taxoWriter.addCategory(new FacetLabel("a", Integer.toString(i)));
        ++numA;
      } else {
        taxoWriter.addCategory(new FacetLabel("b", Integer.toString(i)));
        ++numB;
      }
    }
    // add category with no children
    taxoWriter.addCategory(new FacetLabel("c"));
    taxoWriter.close();

    DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(dir);

    // non existing category
    ChildrenIterator it = taxoReader.getChildren(taxoReader.getOrdinal(new FacetLabel("invalid")));
    assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next());

    // a category with no children
    it = taxoReader.getChildren(taxoReader.getOrdinal(new FacetLabel("c")));
    assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next());

    // arbitrary negative ordinal
    it = taxoReader.getChildren(-2);
    assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next());

    // root's children
    Set<String> roots = new HashSet<>(Arrays.asList("a", "b", "c"));
    it = taxoReader.getChildren(TaxonomyReader.ROOT_ORDINAL);
    while (!roots.isEmpty()) {
      FacetLabel root = taxoReader.getPath(it.next());
      assertEquals(1, root.length);
      assertTrue(roots.remove(root.components[0]));
    }
    assertEquals(TaxonomyReader.INVALID_ORDINAL, it.next());

    for (int i = 0; i < 2; i++) {
      FacetLabel cp = i == 0 ? new FacetLabel("a") : new FacetLabel("b");
      int ordinal = taxoReader.getOrdinal(cp);
      it = taxoReader.getChildren(ordinal);
      int numChildren = 0;
      int child;
      while ((child = it.next()) != TaxonomyReader.INVALID_ORDINAL) {
        FacetLabel path = taxoReader.getPath(child);
        assertEquals(2, path.length);
        assertEquals(path.components[0], i == 0 ? "a" : "b");
        ++numChildren;
      }
      int expected = i == 0 ? numA : numB;
      assertEquals("invalid num children", expected, numChildren);
    }
    taxoReader.close();

    dir.close();
  }
  @Test
  public void testOpenIfChangedReplaceTaxonomy() throws Exception {
    // test openIfChanged when replaceTaxonomy is called, which is equivalent to recreate
    // only can work with NRT as well
    Directory src = newDirectory();
    DirectoryTaxonomyWriter w = new DirectoryTaxonomyWriter(src);
    FacetLabel cp_b = new FacetLabel("b");
    w.addCategory(cp_b);
    w.close();

    for (boolean nrt : new boolean[] {false, true}) {
      Directory dir = newDirectory();
      DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);

      FacetLabel cp_a = new FacetLabel("a");
      writer.addCategory(cp_a);
      if (!nrt) writer.commit();

      DirectoryTaxonomyReader r1 =
          nrt ? new DirectoryTaxonomyReader(writer) : new DirectoryTaxonomyReader(dir);
      // fill r1's caches
      assertEquals(1, r1.getOrdinal(cp_a));
      assertEquals(cp_a, r1.getPath(1));

      // now replace taxonomy
      writer.replaceTaxonomy(src);
      if (!nrt) writer.commit();

      DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
      assertNotNull(r2);

      // fill r2's caches
      assertEquals(1, r2.getOrdinal(cp_b));
      assertEquals(cp_b, r2.getPath(1));

      // check that r1 doesn't see cp_b
      assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
      assertEquals(cp_a, r1.getPath(1));

      // check that r2 doesn't see cp_a
      assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a));
      assertEquals(cp_b, r2.getPath(1));

      r2.close();
      r1.close();
      writer.close();
      dir.close();
    }

    src.close();
  }
  @Test
  public void testOpenIfChangedReuseAfterRecreate() throws Exception {
    // tests that if the taxonomy is recreated, no data is reused from the previous taxonomy
    Directory dir = newDirectory();
    DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir);
    FacetLabel cp_a = new FacetLabel("a");
    writer.addCategory(cp_a);
    writer.close();

    DirectoryTaxonomyReader r1 = new DirectoryTaxonomyReader(dir);
    // fill r1's caches
    assertEquals(1, r1.getOrdinal(cp_a));
    assertEquals(cp_a, r1.getPath(1));

    // now recreate, add a different category
    writer = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE);
    FacetLabel cp_b = new FacetLabel("b");
    writer.addCategory(cp_b);
    writer.close();

    DirectoryTaxonomyReader r2 = TaxonomyReader.openIfChanged(r1);
    assertNotNull(r2);

    // fill r2's caches
    assertEquals(1, r2.getOrdinal(cp_b));
    assertEquals(cp_b, r2.getPath(1));

    // check that r1 doesn't see cp_b
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r1.getOrdinal(cp_b));
    assertEquals(cp_a, r1.getPath(1));

    // check that r2 doesn't see cp_a
    assertEquals(TaxonomyReader.INVALID_ORDINAL, r2.getOrdinal(cp_a));
    assertEquals(cp_b, r2.getPath(1));

    r2.close();
    r1.close();
    dir.close();
  }