@Test
  public void testPayloadIntDecodingIterator() throws Exception {
    Directory dir = newDirectory();
    DataTokenStream dts =
        new DataTokenStream(
            "1",
            new SortingIntEncoder(
                new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
    RandomIndexWriter writer =
        new RandomIndexWriter(
            random,
            dir,
            newIndexWriterConfig(
                    TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.KEYWORD, false))
                .setMergePolicy(newLogMergePolicy()));
    for (int i = 0; i < data.length; i++) {
      dts.setIdx(i);
      Document doc = new Document();
      doc.add(new Field("f", dts));
      writer.addDocument(doc);
    }
    IndexReader reader = writer.getReader();
    writer.close();

    CategoryListIterator cli =
        new PayloadIntDecodingIterator(
            reader, new Term("f", "1"), dts.encoder.createMatchingDecoder());
    cli.init();
    int totalCategories = 0;
    for (int i = 0; i < data.length; i++) {
      Set<Integer> values = new HashSet<Integer>();
      for (int j = 0; j < data[i].length; j++) {
        values.add(data[i][j]);
      }
      cli.skipTo(i);
      long cat;
      while ((cat = cli.nextCategory()) < Integer.MAX_VALUE) {
        assertTrue("expected category not found: " + cat, values.contains((int) cat));
        totalCategories++;
      }
    }
    assertEquals("Missing categories!", 10, totalCategories);
    reader.close();
    dir.close();
  }
  /** Test that a document with no payloads does not confuse the payload decoder. */
  @Test
  public void testPayloadIteratorWithInvalidDoc() throws Exception {
    Directory dir = newDirectory();
    DataTokenStream dts =
        new DataTokenStream(
            "1",
            new SortingIntEncoder(
                new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
    DataTokenStream dts2 =
        new DataTokenStream(
            "2",
            new SortingIntEncoder(
                new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder()))));
    // this test requires that no payloads ever be randomly present!
    final Analyzer noPayloadsAnalyzer =
        new Analyzer() {
          @Override
          public TokenStream tokenStream(String fieldName, Reader reader) {
            return new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
          }
        };
    // NOTE: test is wired to LogMP... because test relies on certain docids having payloads
    RandomIndexWriter writer =
        new RandomIndexWriter(
            random,
            dir,
            newIndexWriterConfig(TEST_VERSION_CURRENT, noPayloadsAnalyzer)
                .setMergePolicy(newLogMergePolicy()));
    for (int i = 0; i < data.length; i++) {
      dts.setIdx(i);
      Document doc = new Document();
      if (i == 0 || i == 2) {
        doc.add(new Field("f", dts)); // only docs 0 & 2 have payloads!
      }
      dts2.setIdx(i);
      doc.add(new Field("f", dts2));
      writer.addDocument(doc);
      writer.commit();
    }

    // add more documents to expose the bug.
    // for some reason, this bug is not exposed unless these additional documents are added.
    for (int i = 0; i < 10; ++i) {
      Document d = new Document();
      dts.setIdx(2);
      d.add(new Field("f", dts2));
      writer.addDocument(d);
      if (i % 10 == 0) {
        writer.commit();
      }
    }

    IndexReader reader = writer.getReader();
    writer.close();

    CategoryListIterator cli =
        new PayloadIntDecodingIterator(
            reader, new Term("f", "1"), dts.encoder.createMatchingDecoder());
    cli.init();
    int totalCats = 0;
    for (int i = 0; i < data.length; i++) {
      // doc no. i
      Set<Integer> values = new HashSet<Integer>();
      for (int j = 0; j < data[i].length; j++) {
        values.add(data[i][j]);
      }
      boolean hasDoc = cli.skipTo(i);
      if (hasDoc) {
        assertTrue("Document " + i + " must not have a payload!", i == 0 || i == 2);
        long cat;
        while ((cat = cli.nextCategory()) < Integer.MAX_VALUE) {
          assertTrue("expected category not found: " + cat, values.contains((int) cat));
          ++totalCats;
        }
      } else {
        assertFalse("Document " + i + " must have a payload!", i == 0 || i == 2);
      }
    }
    assertEquals("Wrong number of total categories!", 4, totalCats);

    // Ok.. went through the first 4 docs, now lets try the 6th doc (docid 5)
    assertFalse("Doc #6 (docid=5) should not have a payload!", cli.skipTo(5));
    reader.close();
    dir.close();
  }