// LUCENE-1130: make sure immeidate disk full on creating
  // an IndexWriter (hit during DW.ThreadState.init()) is
  // OK:
  public void testImmediateDiskFull() throws IOException {
    MockDirectoryWrapper dir = newMockDirectory();
    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                .setMaxBufferedDocs(2)
                .setMergeScheduler(new ConcurrentMergeScheduler()));
    dir.setMaxSizeInBytes(Math.max(1, dir.getRecomputedActualSizeInBytes()));
    final Document doc = new Document();
    FieldType customType = new FieldType(TextField.TYPE_STORED);
    doc.add(newField("field", "aaa bbb ccc ddd eee fff ggg hhh iii jjj", customType));
    try {
      writer.addDocument(doc);
      fail("did not hit disk full");
    } catch (IOException ioe) {
    }
    // Without fix for LUCENE-1130: this call will hang:
    try {
      writer.addDocument(doc);
      fail("did not hit disk full");
    } catch (IOException ioe) {
    }
    try {
      writer.close(false);
      fail("did not hit disk full");
    } catch (IOException ioe) {
    }

    // Make sure once disk space is avail again, we can
    // cleanly close:
    dir.setMaxSizeInBytes(0);
    writer.close(false);
    dir.close();
  }
Ejemplo n.º 2
0
 private void crash(final IndexWriter writer) throws IOException {
   final MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
   ConcurrentMergeScheduler cms =
       (ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler();
   cms.sync();
   dir.crash();
   cms.sync();
   dir.clearCrash();
 }
  // Make sure running BG merges still work fine even when
  // we are hitting exceptions during flushing.
  public void testFlushExceptions() throws IOException {
    MockDirectoryWrapper directory = newMockDirectory();
    FailOnlyOnFlush failure = new FailOnlyOnFlush();
    directory.failOn(failure);

    IndexWriter writer =
        new IndexWriter(
            directory,
            newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                .setMaxBufferedDocs(2));
    Document doc = new Document();
    Field idField = newStringField("id", "", Field.Store.YES);
    doc.add(idField);
    int extraCount = 0;

    for (int i = 0; i < 10; i++) {
      if (VERBOSE) {
        System.out.println("TEST: iter=" + i);
      }

      for (int j = 0; j < 20; j++) {
        idField.setStringValue(Integer.toString(i * 20 + j));
        writer.addDocument(doc);
      }

      // must cycle here because sometimes the merge flushes
      // the doc we just added and so there's nothing to
      // flush, and we don't hit the exception
      while (true) {
        writer.addDocument(doc);
        failure.setDoFail();
        try {
          writer.flush(true, true);
          if (failure.hitExc) {
            fail("failed to hit IOException");
          }
          extraCount++;
        } catch (IOException ioe) {
          if (VERBOSE) {
            ioe.printStackTrace(System.out);
          }
          failure.clearDoFail();
          break;
        }
      }
      assertEquals(20 * (i + 1) + extraCount, writer.numDocs());
    }

    writer.close();
    IndexReader reader = DirectoryReader.open(directory);
    assertEquals(200 + extraCount, reader.numDocs());
    reader.close();
    directory.close();
  }
Ejemplo n.º 4
0
  // LUCENE-1274: test writer.prepareCommit()
  public void testPrepareCommitRollback() throws IOException {
    MockDirectoryWrapper dir = newDirectory();
    dir.setPreventDoubleWrite(false);

    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                .setMaxBufferedDocs(2)
                .setMergePolicy(newLogMergePolicy(5)));
    writer.commit();

    for (int i = 0; i < 23; i++) TestIndexWriter.addDoc(writer);

    DirectoryReader reader = DirectoryReader.open(dir);
    assertEquals(0, reader.numDocs());

    writer.prepareCommit();

    IndexReader reader2 = DirectoryReader.open(dir);
    assertEquals(0, reader2.numDocs());

    writer.rollback();

    IndexReader reader3 = DirectoryReader.openIfChanged(reader);
    assertNull(reader3);
    assertEquals(0, reader.numDocs());
    assertEquals(0, reader2.numDocs());
    reader.close();
    reader2.close();

    writer =
        new IndexWriter(
            dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
    for (int i = 0; i < 17; i++) TestIndexWriter.addDoc(writer);

    reader = DirectoryReader.open(dir);
    assertEquals(0, reader.numDocs());
    reader.close();

    writer.prepareCommit();

    reader = DirectoryReader.open(dir);
    assertEquals(0, reader.numDocs());
    reader.close();

    writer.commit();
    reader = DirectoryReader.open(dir);
    assertEquals(17, reader.numDocs());
    reader.close();
    writer.close();
    dir.close();
  }
  public void testStallControl() throws InterruptedException, IOException {

    int[] numThreads = new int[] {4 + random().nextInt(8), 1};
    final int numDocumentsToIndex = 50 + random().nextInt(50);
    for (int i = 0; i < numThreads.length; i++) {
      AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
      MockDirectoryWrapper dir = newMockDirectory();
      // mock a very slow harddisk sometimes here so that flushing is very slow
      dir.setThrottling(MockDirectoryWrapper.Throttling.SOMETIMES);
      IndexWriterConfig iwc =
          newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
      iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
      iwc.setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
      FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
      iwc.setFlushPolicy(flushPolicy);

      DocumentsWriterPerThreadPool threadPool =
          new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2);
      iwc.setIndexerThreadPool(threadPool);
      // with such a small ram buffer we should be stalled quiet quickly
      iwc.setRAMBufferSizeMB(0.25);
      IndexWriter writer = new IndexWriter(dir, iwc);
      IndexThread[] threads = new IndexThread[numThreads[i]];
      for (int x = 0; x < threads.length; x++) {
        threads[x] = new IndexThread(numDocs, numThreads[i], writer, lineDocFile, false);
        threads[x].start();
      }

      for (int x = 0; x < threads.length; x++) {
        threads[x].join();
      }
      DocumentsWriter docsWriter = writer.getDocsWriter();
      assertNotNull(docsWriter);
      DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
      assertEquals(" all flushes must be due", 0, flushControl.flushBytes());
      assertEquals(numDocumentsToIndex, writer.numDocs());
      assertEquals(numDocumentsToIndex, writer.maxDoc());
      if (numThreads[i] == 1) {
        assertFalse(
            "single thread must not block numThreads: " + numThreads[i],
            docsWriter.flushControl.stallControl.hasBlocked());
      }
      if (docsWriter.flushControl.peakNetBytes
          > (2.d * iwc.getRAMBufferSizeMB() * 1024.d * 1024.d)) {
        assertTrue(docsWriter.flushControl.stallControl.wasStalled());
      }
      assertActiveBytesAfter(flushControl);
      writer.close(true);
      dir.close();
    }
  }
Ejemplo n.º 6
0
  public void testWriterAfterCrash() throws IOException {
    // This test relies on being able to open a reader before any commit
    // happened, so we must create an initial commit just to allow that, but
    // before any documents were added.
    IndexWriter writer = initIndex(random, true);
    MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
    dir.setPreventDoubleWrite(false);
    crash(writer);
    writer = initIndex(random, dir, false);
    writer.close();

    IndexReader reader = IndexReader.open(dir, false);
    assertTrue(reader.numDocs() < 314);
    reader.close();
    dir.close();
  }
  public void testTotalBytesSize() throws Exception {
    Directory d = newDirectory();
    if (d instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) d).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    iwc.setMaxBufferedDocs(5);
    iwc.setMergeScheduler(new TrackingCMS());
    if (TestUtil.getPostingsFormat("id").equals("SimpleText")) {
      // no
      iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene41PostingsFormat()));
    }
    IndexWriter w = new IndexWriter(d, iwc);
    for (int i = 0; i < 1000; i++) {
      Document doc = new Document();
      doc.add(new StringField("id", "" + i, Field.Store.NO));
      w.addDocument(doc);

      if (random().nextBoolean()) {
        w.deleteDocuments(new Term("id", "" + random().nextInt(i + 1)));
      }
    }
    assertTrue(((TrackingCMS) w.getConfig().getMergeScheduler()).totMergedBytes != 0);
    w.close();
    d.close();
  }
Ejemplo n.º 8
0
  /*
   * Test a deletion policy that keeps last N commits.
   */
  public void testKeepLastNDeletionPolicy() throws IOException {
    final int N = 5;

    for (int pass = 0; pass < 2; pass++) {

      boolean useCompoundFile = (pass % 2) != 0;

      Directory dir = newDirectory();
      if (dir instanceof MockDirectoryWrapper) {
        // test manually deletes files
        ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
      }

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
      for (int j = 0; j < N + 1; j++) {
        IndexWriterConfig conf =
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.CREATE)
                .setIndexDeletionPolicy(policy)
                .setMaxBufferedDocs(10);
        MergePolicy mp = conf.getMergePolicy();
        mp.setNoCFSRatio(useCompoundFile ? 1.0 : 0.0);
        IndexWriter writer = new IndexWriter(dir, conf);
        policy = (KeepLastNDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
        for (int i = 0; i < 17; i++) {
          addDoc(writer);
        }
        writer.forceMerge(1);
        writer.close();
      }

      assertTrue(policy.numDelete > 0);
      assertEquals(N + 1, policy.numOnInit);
      assertEquals(N + 1, policy.numOnCommit);

      // Simplistic check: just verify only the past N segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getLastCommitGeneration(dir);
      for (int i = 0; i < N + 1; i++) {
        try {
          IndexReader reader = DirectoryReader.open(dir);
          reader.close();
          if (i == N) {
            fail("should have failed on commits prior to last " + N);
          }
        } catch (IOException e) {
          if (i != N) {
            throw e;
          }
        }
        if (i < N) {
          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        }
        gen--;
      }

      dir.close();
    }
  }
Ejemplo n.º 9
0
  @BeforeClass
  public static void beforeClassAddIndexesTaskTest() throws Exception {
    testDir = _TestUtil.getTempDir("addIndexesTask");

    // create a dummy index under inputDir
    inputDir = new File(testDir, "input");
    MockDirectoryWrapper tmpDir = newFSDirectory(inputDir);
    try {
      IndexWriter writer =
          new IndexWriter(tmpDir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
      for (int i = 0; i < 10; i++) {
        writer.addDocument(new Document());
      }
      writer.close();
    } finally {
      tmpDir.close();
    }
  }
Ejemplo n.º 10
0
  // indexes Integer.MAX_VALUE docs with a fixed binary field
  public void testFixedSorted() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BFixedSorted"));
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }

    IndexWriter w =
        new IndexWriter(
            dir,
            new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
                .setRAMBufferSizeMB(256.0)
                .setMergeScheduler(new ConcurrentMergeScheduler())
                .setMergePolicy(newLogMergePolicy(false, 10))
                .setOpenMode(IndexWriterConfig.OpenMode.CREATE));

    Document doc = new Document();
    byte bytes[] = new byte[2];
    BytesRef data = new BytesRef(bytes);
    SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
    doc.add(dvField);

    for (int i = 0; i < Integer.MAX_VALUE; i++) {
      bytes[0] = (byte) (i >> 8);
      bytes[1] = (byte) i;
      w.addDocument(doc);
      if (i % 100000 == 0) {
        System.out.println("indexed: " + i);
        System.out.flush();
      }
    }

    w.forceMerge(1);
    w.close();

    System.out.println("verifying...");
    System.out.flush();

    DirectoryReader r = DirectoryReader.open(dir);
    int expectedValue = 0;
    for (AtomicReaderContext context : r.leaves()) {
      AtomicReader reader = context.reader();
      BytesRef scratch = new BytesRef();
      BinaryDocValues dv = reader.getSortedDocValues("dv");
      for (int i = 0; i < reader.maxDoc(); i++) {
        bytes[0] = (byte) (expectedValue >> 8);
        bytes[1] = (byte) expectedValue;
        dv.get(i, scratch);
        assertEquals(data, scratch);
        expectedValue++;
      }
    }

    r.close();
    dir.close();
  }
  // LUCENE-2593
  public void testCorruptionAfterDiskFullDuringMerge() throws IOException {
    MockDirectoryWrapper dir = newDirectory();
    // IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new
    // MockAnalyzer(random)).setReaderPooling(true));
    IndexWriter w =
        new IndexWriter(
            dir,
            newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
                .setMergeScheduler(new SerialMergeScheduler())
                .setReaderPooling(true)
                .setMergePolicy(newLogMergePolicy(2)));

    _TestUtil.keepFullyDeletedSegments(w);

    ((LogMergePolicy) w.getMergePolicy()).setMergeFactor(2);

    Document doc = new Document();
    doc.add(newField("f", "doctor who", Field.Store.YES, Field.Index.ANALYZED));
    w.addDocument(doc);
    w.commit();

    w.deleteDocuments(new Term("f", "who"));
    w.addDocument(doc);

    // disk fills up!
    FailTwiceDuringMerge ftdm = new FailTwiceDuringMerge();
    ftdm.setDoFail();
    dir.failOn(ftdm);

    try {
      w.commit();
      fail("fake disk full IOExceptions not hit");
    } catch (IOException ioe) {
      // expected
      assertTrue(ftdm.didFail1 || ftdm.didFail2);
    }
    _TestUtil.checkIndex(dir);
    ftdm.clearDoFail();
    w.addDocument(doc);
    w.close();

    dir.close();
  }
  /**
   * Test segment infos read that hits exception on close make sure we get our exception back, no
   * file handle leaks, etc.
   */
  public void testExceptionOnCloseInput() throws Exception {
    Failure fail =
        new Failure() {
          @Override
          public void eval(MockDirectoryWrapper dir) throws IOException {
            for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
              if (doFail && "close".equals(e.getMethodName())) {
                throw new FakeIOException();
              }
            }
          }
        };

    MockDirectoryWrapper dir = newMockDirectory();
    dir.failOn(fail);
    Codec codec = getCodec();
    byte id[] = StringHelper.randomId();
    SegmentInfo info =
        new SegmentInfo(
            dir,
            getVersions()[0],
            "_123",
            1,
            false,
            codec,
            Collections.<String, String>emptyMap(),
            id,
            new HashMap<>(),
            null);
    info.setFiles(Collections.<String>emptySet());
    codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);

    fail.setDoFail();
    expectThrows(
        FakeIOException.class,
        () -> {
          codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
        });
    fail.clearDoFail();

    dir.close();
  }
Ejemplo n.º 13
0
  public void testCrashAfterCloseNoWait() throws IOException {

    IndexWriter writer = initIndex(random, false);
    MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();

    writer.close(false);

    dir.crash();

    /*
    String[] l = dir.list();
    Arrays.sort(l);
    for(int i=0;i<l.length;i++)
      System.out.println("file " + i + " = " + l[i] + " " + dir.fileLength(l[i]) + " bytes");
    */
    IndexReader reader = IndexReader.open(dir, false);
    assertEquals(157, reader.numDocs());
    reader.close();
    dir.close();
  }
Ejemplo n.º 14
0
  public void testCrashAfterReopen() throws IOException {
    IndexWriter writer = initIndex(random, false);
    MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
    writer.close();
    writer = initIndex(random, dir, false);
    assertEquals(314, writer.maxDoc());
    crash(writer);

    /*
    System.out.println("\n\nTEST: open reader");
    String[] l = dir.list();
    Arrays.sort(l);
    for(int i=0;i<l.length;i++)
      System.out.println("file " + i + " = " + l[i] + " " +
    dir.fileLength(l[i]) + " bytes");
    */

    IndexReader reader = IndexReader.open(dir, false);
    assertTrue(reader.numDocs() >= 157);
    reader.close();
    dir.close();
  }
Ejemplo n.º 15
0
  // Verifies no *.prx exists when all fields omit term positions:
  public void testNoPrxFile() throws Throwable {
    Directory ram = newDirectory();
    if (ram instanceof MockDirectoryWrapper) {
      // we verify some files get deleted
      ((MockDirectoryWrapper) ram).setEnableVirusScanner(false);
    }
    Analyzer analyzer = new MockAnalyzer(random());
    IndexWriter writer =
        new IndexWriter(
            ram,
            newIndexWriterConfig(analyzer)
                .setMaxBufferedDocs(3)
                .setMergePolicy(newLogMergePolicy()));
    LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
    lmp.setMergeFactor(2);
    lmp.setNoCFSRatio(0.0);
    Document d = new Document();

    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
    Field f1 = newField("f1", "This field has term freqs", ft);
    d.add(f1);

    for (int i = 0; i < 30; i++) writer.addDocument(d);

    writer.commit();

    assertNoPrx(ram);

    // now add some documents with positions, and check there is no prox after optimization
    d = new Document();
    f1 = newTextField("f1", "This field has positions", Field.Store.NO);
    d.add(f1);

    for (int i = 0; i < 30; i++) writer.addDocument(d);

    // force merge
    writer.forceMerge(1);
    // flush
    writer.close();

    assertNoPrx(ram);
    ram.close();
  }
  @Override
  public Directory create(String path, LockFactory lockFactory, DirContext dirContext)
      throws IOException {
    // we pass NoLockFactory, because the real lock factory is set later by injectLockFactory:
    Directory dir = LuceneTestCase.newFSDirectory(new File(path).toPath(), lockFactory);
    // we can't currently do this check because of how
    // Solr has to reboot a new Directory sometimes when replicating
    // or rolling back - the old directory is closed and the following
    // test assumes it can open an IndexWriter when that happens - we
    // have a new Directory for the same dir and still an open IW at
    // this point

    Directory cdir = reduce(dir);
    cdir = reduce(cdir);
    cdir = reduce(cdir);

    if (cdir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) cdir).setAssertNoUnrefencedFilesOnClose(false);
    }
    return dir;
  }
Ejemplo n.º 17
0
  private IndexWriter initIndex(Random random, MockDirectoryWrapper dir, boolean initialCommit)
      throws IOException {
    dir.setLockFactory(NoLockFactory.getNoLockFactory());

    IndexWriter writer =
        new IndexWriter(
            dir,
            newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))
                .setMaxBufferedDocs(10)
                .setMergeScheduler(new ConcurrentMergeScheduler()));
    ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
    if (initialCommit) {
      writer.commit();
    }

    Document doc = new Document();
    doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED));
    doc.add(newField("id", "0", Field.Store.YES, Field.Index.ANALYZED));
    for (int i = 0; i < 157; i++) writer.addDocument(doc);

    return writer;
  }
  public void test() throws Exception {
    IndexWriterConfig defaultConfig = new IndexWriterConfig(null);
    Codec defaultCodec = defaultConfig.getCodec();
    if ((new IndexWriterConfig(null)).getCodec() instanceof CompressingCodec) {
      Pattern regex = Pattern.compile("maxDocsPerChunk=(\\d+), blockSize=(\\d+)");
      Matcher matcher = regex.matcher(defaultCodec.toString());
      assertTrue(
          "Unexpected CompressingCodec toString() output: " + defaultCodec.toString(),
          matcher.find());
      int maxDocsPerChunk = Integer.parseInt(matcher.group(1));
      int blockSize = Integer.parseInt(matcher.group(2));
      int product = maxDocsPerChunk * blockSize;
      assumeTrue(
          defaultCodec.getName()
              + " maxDocsPerChunk ("
              + maxDocsPerChunk
              + ") * blockSize ("
              + blockSize
              + ") < 16 - this can trigger OOM with -Dtests.heapsize=30g",
          product >= 16);
    }

    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BPostingsBytes1"));
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }

    IndexWriter w =
        new IndexWriter(
            dir,
            new IndexWriterConfig(new MockAnalyzer(random()))
                .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
                .setRAMBufferSizeMB(256.0)
                .setMergeScheduler(new ConcurrentMergeScheduler())
                .setMergePolicy(newLogMergePolicy(false, 10))
                .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
                .setCodec(TestUtil.getDefaultCodec()));

    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogByteSizeMergePolicy) {
      // 1 petabyte:
      ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
    }

    Document doc = new Document();
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
    ft.setOmitNorms(true);
    MyTokenStream tokenStream = new MyTokenStream();
    Field field = new Field("field", tokenStream, ft);
    doc.add(field);

    final int numDocs = 1000;
    for (int i = 0; i < numDocs; i++) {
      if (i % 2 == 1) { // trick blockPF's little optimization
        tokenStream.n = 65536;
      } else {
        tokenStream.n = 65537;
      }
      w.addDocument(doc);
    }
    w.forceMerge(1);
    w.close();

    DirectoryReader oneThousand = DirectoryReader.open(dir);
    DirectoryReader subReaders[] = new DirectoryReader[1000];
    Arrays.fill(subReaders, oneThousand);
    BaseDirectoryWrapper dir2 = newFSDirectory(createTempDir("2BPostingsBytes2"));
    if (dir2 instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir2).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w2 = new IndexWriter(dir2, new IndexWriterConfig(null));
    TestUtil.addIndexesSlowly(w2, subReaders);
    w2.forceMerge(1);
    w2.close();
    oneThousand.close();

    DirectoryReader oneMillion = DirectoryReader.open(dir2);
    subReaders = new DirectoryReader[2000];
    Arrays.fill(subReaders, oneMillion);
    BaseDirectoryWrapper dir3 = newFSDirectory(createTempDir("2BPostingsBytes3"));
    if (dir3 instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir3).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w3 = new IndexWriter(dir3, new IndexWriterConfig(null));
    TestUtil.addIndexesSlowly(w3, subReaders);
    w3.forceMerge(1);
    w3.close();
    oneMillion.close();

    dir.close();
    dir2.close();
    dir3.close();
  }
Ejemplo n.º 19
0
  @Slow
  public void testNoWaitClose() throws Throwable {
    Directory directory = newDirectory();

    if (directory instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) directory).setPreventDoubleWrite(false);
    }

    final Document doc = new Document();
    FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
    customType.setTokenized(false);

    Field idField = newField("id", "", customType);
    doc.add(idField);

    for (int pass = 0; pass < 2; pass++) {
      if (VERBOSE) {
        System.out.println("TEST: pass="******"TEST: iter=" + iter);
        }
        for (int j = 0; j < 199; j++) {
          idField.setStringValue(Integer.toString(iter * 201 + j));
          writer.addDocument(doc);
        }

        int delID = iter * 199;
        for (int j = 0; j < 20; j++) {
          writer.deleteDocuments(new Term("id", Integer.toString(delID)));
          delID += 5;
        }

        writer.commit();

        // Force a bunch of merge threads to kick off so we
        // stress out aborting them on close:
        ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);

        final IndexWriter finalWriter = writer;
        final AtomicReference<Throwable> failure = new AtomicReference<>();
        Thread t1 =
            new Thread() {
              @Override
              public void run() {
                boolean done = false;
                while (!done) {
                  for (int i = 0; i < 100; i++) {
                    try {
                      finalWriter.addDocument(doc);
                    } catch (AlreadyClosedException e) {
                      done = true;
                      break;
                    } catch (NullPointerException e) {
                      done = true;
                      break;
                    } catch (Throwable e) {
                      e.printStackTrace(System.out);
                      failure.set(e);
                      done = true;
                      break;
                    }
                  }
                  Thread.yield();
                }
              }
            };

        t1.start();

        writer.close();
        t1.join();

        if (failure.get() != null) {
          throw failure.get();
        }

        // Make sure reader can read
        IndexReader reader = DirectoryReader.open(directory);
        reader.close();

        // Reopen
        writer =
            new IndexWriter(
                directory,
                newIndexWriterConfig(new MockAnalyzer(random()))
                    .setOpenMode(OpenMode.APPEND)
                    .setMergePolicy(newLogMergePolicy())
                    .setCommitOnClose(false));
      }
      writer.close();
    }

    directory.close();
  }
Ejemplo n.º 20
0
  /*
   * Test a deletion policy that keeps last N commits
   * around, through creates.
   */
  public void testKeepLastNDeletionPolicyWithCreates() throws IOException {

    final int N = 10;

    for (int pass = 0; pass < 2; pass++) {

      boolean useCompoundFile = (pass % 2) != 0;

      Directory dir = newDirectory();
      if (dir instanceof MockDirectoryWrapper) {
        // test manually deletes files
        ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
      }
      IndexWriterConfig conf =
          newIndexWriterConfig(new MockAnalyzer(random()))
              .setOpenMode(OpenMode.CREATE)
              .setIndexDeletionPolicy(new KeepLastNDeletionPolicy(N))
              .setMaxBufferedDocs(10);
      MergePolicy mp = conf.getMergePolicy();
      mp.setNoCFSRatio(useCompoundFile ? 1.0 : 0.0);
      IndexWriter writer = new IndexWriter(dir, conf);
      KeepLastNDeletionPolicy policy =
          (KeepLastNDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
      writer.close();
      Term searchTerm = new Term("content", "aaa");
      Query query = new TermQuery(searchTerm);

      for (int i = 0; i < N + 1; i++) {

        conf =
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.APPEND)
                .setIndexDeletionPolicy(policy)
                .setMaxBufferedDocs(10);
        mp = conf.getMergePolicy();
        mp.setNoCFSRatio(useCompoundFile ? 1.0 : 0.0);
        writer = new IndexWriter(dir, conf);
        policy = (KeepLastNDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
        for (int j = 0; j < 17; j++) {
          addDocWithID(writer, i * (N + 1) + j);
        }
        // this is a commit
        writer.close();
        conf =
            new IndexWriterConfig(new MockAnalyzer(random()))
                .setIndexDeletionPolicy(policy)
                .setMergePolicy(NoMergePolicy.INSTANCE);
        writer = new IndexWriter(dir, conf);
        policy = (KeepLastNDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
        writer.deleteDocuments(new Term("id", "" + (i * (N + 1) + 3)));
        // this is a commit
        writer.close();
        IndexReader reader = DirectoryReader.open(dir);
        IndexSearcher searcher = newSearcher(reader);
        ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
        assertEquals(16, hits.length);
        reader.close();

        writer =
            new IndexWriter(
                dir,
                newIndexWriterConfig(new MockAnalyzer(random()))
                    .setOpenMode(OpenMode.CREATE)
                    .setIndexDeletionPolicy(policy));
        policy = (KeepLastNDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
        // This will not commit: there are no changes
        // pending because we opened for "create":
        writer.close();
      }

      assertEquals(3 * (N + 1) + 1, policy.numOnInit);
      assertEquals(3 * (N + 1) + 1, policy.numOnCommit);

      IndexReader rwReader = DirectoryReader.open(dir);
      IndexSearcher searcher = newSearcher(rwReader);
      ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
      assertEquals(0, hits.length);

      // Simplistic check: just verify only the past N segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getLastCommitGeneration(dir);

      int expectedCount = 0;

      rwReader.close();

      for (int i = 0; i < N + 1; i++) {
        try {
          IndexReader reader = DirectoryReader.open(dir);

          // Work backwards in commits on what the expected
          // count should be.
          searcher = newSearcher(reader);
          hits = searcher.search(query, 1000).scoreDocs;
          assertEquals(expectedCount, hits.length);
          if (expectedCount == 0) {
            expectedCount = 16;
          } else if (expectedCount == 16) {
            expectedCount = 17;
          } else if (expectedCount == 17) {
            expectedCount = 0;
          }
          reader.close();
          if (i == N) {
            fail("should have failed on commits before last " + N);
          }
        } catch (IOException e) {
          if (i != N) {
            throw e;
          }
        }
        if (i < N) {
          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        }
        gen--;
      }

      dir.close();
    }
  }
Ejemplo n.º 21
0
  // @Absurd @Ignore takes ~20GB-30GB of space and 10 minutes.
  // with some codecs needs more heap space as well.
  @Ignore("Very slow. Enable manually by removing @Ignore.")
  public void test() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BPostingsBytes1"));
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }

    IndexWriter w =
        new IndexWriter(
            dir,
            new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
                .setRAMBufferSizeMB(256.0)
                .setMergeScheduler(new ConcurrentMergeScheduler())
                .setMergePolicy(newLogMergePolicy(false, 10))
                .setOpenMode(IndexWriterConfig.OpenMode.CREATE));

    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogByteSizeMergePolicy) {
      // 1 petabyte:
      ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
    }

    Document doc = new Document();
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
    ft.setOmitNorms(true);
    MyTokenStream tokenStream = new MyTokenStream();
    Field field = new Field("field", tokenStream, ft);
    doc.add(field);

    final int numDocs = 1000;
    for (int i = 0; i < numDocs; i++) {
      if (i % 2 == 1) { // trick blockPF's little optimization
        tokenStream.n = 65536;
      } else {
        tokenStream.n = 65537;
      }
      w.addDocument(doc);
    }
    w.forceMerge(1);
    w.close();

    DirectoryReader oneThousand = DirectoryReader.open(dir);
    IndexReader subReaders[] = new IndexReader[1000];
    Arrays.fill(subReaders, oneThousand);
    MultiReader mr = new MultiReader(subReaders);
    BaseDirectoryWrapper dir2 = newFSDirectory(_TestUtil.getTempDir("2BPostingsBytes2"));
    if (dir2 instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir2).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w2 = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
    w2.addIndexes(mr);
    w2.forceMerge(1);
    w2.close();
    oneThousand.close();

    DirectoryReader oneMillion = DirectoryReader.open(dir2);
    subReaders = new IndexReader[2000];
    Arrays.fill(subReaders, oneMillion);
    mr = new MultiReader(subReaders);
    BaseDirectoryWrapper dir3 = newFSDirectory(_TestUtil.getTempDir("2BPostingsBytes3"));
    if (dir3 instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir3).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w3 = new IndexWriter(dir3, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
    w3.addIndexes(mr);
    w3.forceMerge(1);
    w3.close();
    oneMillion.close();

    dir.close();
    dir2.close();
    dir3.close();
  }
Ejemplo n.º 22
0
  /*
   * Test a silly deletion policy that keeps all commits around.
   */
  public void testKeepAllDeletionPolicy() throws IOException {
    for (int pass = 0; pass < 2; pass++) {

      if (VERBOSE) {
        System.out.println("TEST: cycle pass="******"TEST: open writer for forceMerge");
        }
        writer = new IndexWriter(dir, conf);
        policy = (KeepAllDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
        writer.forceMerge(1);
        writer.close();
      }

      assertEquals(needsMerging ? 2 : 1, policy.numOnInit);

      // If we are not auto committing then there should
      // be exactly 2 commits (one per close above):
      assertEquals(1 + (needsMerging ? 1 : 0), policy.numOnCommit);

      // Test listCommits
      Collection<IndexCommit> commits = DirectoryReader.listCommits(dir);
      // 2 from closing writer
      assertEquals(1 + (needsMerging ? 1 : 0), commits.size());

      // Make sure we can open a reader on each commit:
      for (final IndexCommit commit : commits) {
        IndexReader r = DirectoryReader.open(commit);
        r.close();
      }

      // Simplistic check: just verify all segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getLastCommitGeneration(dir);
      while (gen > 0) {
        IndexReader reader = DirectoryReader.open(dir);
        reader.close();
        dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        gen--;

        if (gen > 0) {
          // Now that we've removed a commit point, which
          // should have orphan'd at least one index file.
          // Open & close a writer and assert that it
          // actually removed something:
          int preCount = dir.listAll().length;
          writer =
              new IndexWriter(
                  dir,
                  newIndexWriterConfig(new MockAnalyzer(random()))
                      .setOpenMode(OpenMode.APPEND)
                      .setIndexDeletionPolicy(policy));
          writer.close();
          int postCount = dir.listAll().length;
          assertTrue(postCount < preCount);
        }
      }

      dir.close();
    }
  }
Ejemplo n.º 23
0
  public void testTransactions() throws Throwable {
    // we cant use non-ramdir on windows, because this test needs to double-write.
    MockDirectoryWrapper dir1 = new MockDirectoryWrapper(random(), new RAMDirectory());
    MockDirectoryWrapper dir2 = new MockDirectoryWrapper(random(), new RAMDirectory());
    dir1.setPreventDoubleWrite(false);
    dir2.setPreventDoubleWrite(false);
    dir1.failOn(new RandomFailure());
    dir2.failOn(new RandomFailure());
    dir1.setFailOnOpenInput(false);
    dir2.setFailOnOpenInput(false);

    // We throw exceptions in deleteFile, which creates
    // leftover files:
    dir1.setAssertNoUnrefencedFilesOnClose(false);
    dir2.setAssertNoUnrefencedFilesOnClose(false);

    initIndex(dir1);
    initIndex(dir2);

    TimedThread[] threads = new TimedThread[3];
    int numThread = 0;

    IndexerThread indexerThread = new IndexerThread(this, dir1, dir2, threads);
    threads[numThread++] = indexerThread;
    indexerThread.start();

    SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads);
    threads[numThread++] = searcherThread1;
    searcherThread1.start();

    SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads);
    threads[numThread++] = searcherThread2;
    searcherThread2.start();

    for (int i = 0; i < numThread; i++) threads[i].join();

    for (int i = 0; i < numThread; i++) assertTrue(!threads[i].failed);
    dir1.close();
    dir2.close();
  }
Ejemplo n.º 24
0
  @Test
  public void testRollingUpdates() throws Exception {
    Random random = new Random(random().nextLong());
    final BaseDirectoryWrapper dir = newDirectory();
    // test checks for no unref'ed files with the IW helper method, which isn't aware of "tried to
    // delete files"
    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
    }

    final LineFileDocs docs = new LineFileDocs(random, true);

    // provider.register(new MemoryCodec());
    if (random().nextBoolean()) {
      Codec.setDefault(
          TestUtil.alwaysPostingsFormat(
              new MemoryPostingsFormat(random().nextBoolean(), random.nextFloat())));
    }

    MockAnalyzer analyzer = new MockAnalyzer(random());
    analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));

    final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(analyzer));
    final int SIZE = atLeast(20);
    int id = 0;
    IndexReader r = null;
    IndexSearcher s = null;
    final int numUpdates =
        (int)
            (SIZE * (2 + (TEST_NIGHTLY ? 200 * random().nextDouble() : 5 * random().nextDouble())));
    if (VERBOSE) {
      System.out.println("TEST: numUpdates=" + numUpdates);
    }
    int updateCount = 0;
    // TODO: sometimes update ids not in order...
    for (int docIter = 0; docIter < numUpdates; docIter++) {
      final Document doc = docs.nextDoc();
      final String myID = Integer.toString(id);
      if (id == SIZE - 1) {
        id = 0;
      } else {
        id++;
      }
      if (VERBOSE) {
        System.out.println("  docIter=" + docIter + " id=" + id);
      }
      ((Field) doc.getField("docid")).setStringValue(myID);

      Term idTerm = new Term("docid", myID);

      final boolean doUpdate;
      if (s != null && updateCount < SIZE) {
        TopDocs hits = s.search(new TermQuery(idTerm), 1);
        assertEquals(1, hits.totalHits);
        doUpdate = !w.tryDeleteDocument(r, hits.scoreDocs[0].doc);
        if (VERBOSE) {
          if (doUpdate) {
            System.out.println("  tryDeleteDocument failed");
          } else {
            System.out.println("  tryDeleteDocument succeeded");
          }
        }
      } else {
        doUpdate = true;
        if (VERBOSE) {
          System.out.println("  no searcher: doUpdate=true");
        }
      }

      updateCount++;

      if (doUpdate) {
        if (random().nextBoolean()) {
          w.updateDocument(idTerm, doc);
        } else {
          // It's OK to not be atomic for this test (no separate thread reopening readers):
          w.deleteDocuments(new TermQuery(idTerm));
          w.addDocument(doc);
        }
      } else {
        w.addDocument(doc);
      }

      if (docIter >= SIZE && random().nextInt(50) == 17) {
        if (r != null) {
          r.close();
        }

        final boolean applyDeletions = random().nextBoolean();

        if (VERBOSE) {
          System.out.println("TEST: reopen applyDeletions=" + applyDeletions);
        }

        r = w.getReader(applyDeletions);
        if (applyDeletions) {
          s = newSearcher(r);
        } else {
          s = null;
        }
        assertTrue(
            "applyDeletions=" + applyDeletions + " r.numDocs()=" + r.numDocs() + " vs SIZE=" + SIZE,
            !applyDeletions || r.numDocs() == SIZE);
        updateCount = 0;
      }
    }

    if (r != null) {
      r.close();
    }

    w.commit();
    assertEquals(SIZE, w.numDocs());

    w.close();

    TestIndexWriter.assertNoUnreferencedFiles(dir, "leftover files after rolling updates");

    docs.close();

    // LUCENE-4455:
    SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
    long totalBytes = 0;
    for (SegmentCommitInfo sipc : infos) {
      totalBytes += sipc.sizeInBytes();
    }
    long totalBytes2 = 0;

    for (String fileName : dir.listAll()) {
      if (IndexFileNames.CODEC_FILE_PATTERN.matcher(fileName).matches()) {
        totalBytes2 += dir.fileLength(fileName);
      }
    }
    assertEquals(totalBytes2, totalBytes);
    dir.close();
  }
Ejemplo n.º 25
0
  /*
   * Test "by time expiration" deletion policy:
   */
  public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException {

    final double SECONDS = 2.0;

    Directory dir = newDirectory();
    if (dir instanceof MockDirectoryWrapper) {
      // test manually deletes files
      ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
    }
    IndexWriterConfig conf =
        newIndexWriterConfig(new MockAnalyzer(random()))
            .setIndexDeletionPolicy(new ExpirationTimeDeletionPolicy(dir, SECONDS));
    MergePolicy mp = conf.getMergePolicy();
    mp.setNoCFSRatio(1.0);
    IndexWriter writer = new IndexWriter(dir, conf);
    ExpirationTimeDeletionPolicy policy =
        (ExpirationTimeDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
    Map<String, String> commitData = new HashMap<>();
    commitData.put("commitTime", String.valueOf(System.currentTimeMillis()));
    writer.setCommitData(commitData);
    writer.commit();
    writer.close();

    long lastDeleteTime = 0;
    final int targetNumDelete = TestUtil.nextInt(random(), 1, 5);
    while (policy.numDelete < targetNumDelete) {
      // Record last time when writer performed deletes of
      // past commits
      lastDeleteTime = System.currentTimeMillis();
      conf =
          newIndexWriterConfig(new MockAnalyzer(random()))
              .setOpenMode(OpenMode.APPEND)
              .setIndexDeletionPolicy(policy);
      mp = conf.getMergePolicy();
      mp.setNoCFSRatio(1.0);
      writer = new IndexWriter(dir, conf);
      policy = (ExpirationTimeDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
      for (int j = 0; j < 17; j++) {
        addDoc(writer);
      }
      commitData = new HashMap<>();
      commitData.put("commitTime", String.valueOf(System.currentTimeMillis()));
      writer.setCommitData(commitData);
      writer.commit();
      writer.close();

      Thread.sleep((int) (1000.0 * (SECONDS / 5.0)));
    }

    // Then simplistic check: just verify that the
    // segments_N's that still exist are in fact within SECONDS
    // seconds of the last one's mod time, and, that I can
    // open a reader on each:
    long gen = SegmentInfos.getLastCommitGeneration(dir);

    String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
    boolean oneSecondResolution = true;

    while (gen > 0) {
      try {
        IndexReader reader = DirectoryReader.open(dir);
        reader.close();
        fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);

        // if we are on a filesystem that seems to have only
        // 1 second resolution, allow +1 second in commit
        // age tolerance:
        SegmentInfos sis = SegmentInfos.readCommit(dir, fileName);
        long modTime = Long.parseLong(sis.getUserData().get("commitTime"));
        oneSecondResolution &= (modTime % 1000) == 0;
        final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0 : 0.0)) * 1000);

        assertTrue(
            "commit point was older than "
                + SECONDS
                + " seconds ("
                + (lastDeleteTime - modTime)
                + " msec) but did not get deleted ",
            lastDeleteTime - modTime <= leeway);
      } catch (IOException e) {
        // OK
        break;
      }

      dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
      gen--;
    }

    dir.close();
  }
  @Nightly
  public void testBigDocuments() throws IOException {
    // "big" as "much bigger than the chunk size"
    // for this test we force a FS dir
    // we can't just use newFSDirectory, because this test doesn't really index anything.
    // so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
    Directory dir =
        new MockDirectoryWrapper(
            random(), new MMapDirectory(_TestUtil.getTempDir("testBigDocuments")));
    IndexWriterConfig iwConf =
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);

    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir).setThrottling(Throttling.NEVER);
    }

    final Document emptyDoc = new Document(); // emptyDoc
    final Document bigDoc1 = new Document(); // lot of small fields
    final Document bigDoc2 = new Document(); // 1 very big field

    final Field idField = new StringField("id", "", Store.NO);
    emptyDoc.add(idField);
    bigDoc1.add(idField);
    bigDoc2.add(idField);

    final FieldType onlyStored = new FieldType(StringField.TYPE_STORED);
    onlyStored.setIndexed(false);

    final Field smallField =
        new Field("fld", randomByteArray(random().nextInt(10), 256), onlyStored);
    final int numFields = RandomInts.randomIntBetween(random(), 500000, 1000000);
    for (int i = 0; i < numFields; ++i) {
      bigDoc1.add(smallField);
    }

    final Field bigField =
        new Field(
            "fld",
            randomByteArray(RandomInts.randomIntBetween(random(), 1000000, 5000000), 2),
            onlyStored);
    bigDoc2.add(bigField);

    final int numDocs = atLeast(5);
    final Document[] docs = new Document[numDocs];
    for (int i = 0; i < numDocs; ++i) {
      docs[i] = RandomPicks.randomFrom(random(), Arrays.asList(emptyDoc, bigDoc1, bigDoc2));
    }
    for (int i = 0; i < numDocs; ++i) {
      idField.setStringValue("" + i);
      iw.addDocument(docs[i]);
      if (random().nextInt(numDocs) == 0) {
        iw.commit();
      }
    }
    iw.commit();
    iw.forceMerge(1); // look at what happens when big docs are merged
    final DirectoryReader rd = DirectoryReader.open(dir);
    final IndexSearcher searcher = new IndexSearcher(rd);
    for (int i = 0; i < numDocs; ++i) {
      final Query query = new TermQuery(new Term("id", "" + i));
      final TopDocs topDocs = searcher.search(query, 1);
      assertEquals("" + i, 1, topDocs.totalHits);
      final Document doc = rd.document(topDocs.scoreDocs[0].doc);
      assertNotNull(doc);
      final IndexableField[] fieldValues = doc.getFields("fld");
      assertEquals(docs[i].getFields("fld").length, fieldValues.length);
      if (fieldValues.length > 0) {
        assertEquals(docs[i].getFields("fld")[0].binaryValue(), fieldValues[0].binaryValue());
      }
    }
    rd.close();
    iw.close();
    dir.close();
  }
  /*
   * Make sure IndexWriter cleans up on hitting a disk
   * full exception in addDocument.
   * TODO: how to do this on windows with FSDirectory?
   */
  public void testAddDocumentOnDiskFull() throws IOException {

    for (int pass = 0; pass < 2; pass++) {
      if (VERBOSE) {
        System.out.println("TEST: pass="******"TEST: cycle: diskFree=" + diskFree);
        }
        MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory());
        dir.setMaxSizeInBytes(diskFree);
        IndexWriter writer =
            new IndexWriter(
                dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
        writer.setInfoStream(VERBOSE ? System.out : null);
        MergeScheduler ms = writer.getConfig().getMergeScheduler();
        if (ms instanceof ConcurrentMergeScheduler) {
          // This test intentionally produces exceptions
          // in the threads that CMS launches; we don't
          // want to pollute test output with these.
          ((ConcurrentMergeScheduler) ms).setSuppressExceptions();
        }

        boolean hitError = false;
        try {
          for (int i = 0; i < 200; i++) {
            addDoc(writer);
          }
          if (VERBOSE) {
            System.out.println("TEST: done adding docs; now commit");
          }
          writer.commit();
        } catch (IOException e) {
          if (VERBOSE) {
            System.out.println("TEST: exception on addDoc");
            e.printStackTrace(System.out);
          }
          hitError = true;
        }

        if (hitError) {
          if (doAbort) {
            if (VERBOSE) {
              System.out.println("TEST: now rollback");
            }
            writer.rollback();
          } else {
            try {
              if (VERBOSE) {
                System.out.println("TEST: now close");
              }
              writer.close();
            } catch (IOException e) {
              if (VERBOSE) {
                System.out.println("TEST: exception on close; retry w/ no disk space limit");
                e.printStackTrace(System.out);
              }
              dir.setMaxSizeInBytes(0);
              writer.close();
            }
          }

          // _TestUtil.syncConcurrentMerges(ms);

          if (_TestUtil.anyFilesExceptWriteLock(dir)) {
            assertNoUnreferencedFiles(dir, "after disk full during addDocument");

            // Make sure reader can open the index:
            IndexReader.open(dir, true).close();
          }

          dir.close();
          // Now try again w/ more space:

          diskFree +=
              TEST_NIGHTLY
                  ? _TestUtil.nextInt(random, 400, 600)
                  : _TestUtil.nextInt(random, 3000, 5000);
        } else {
          // _TestUtil.syncConcurrentMerges(writer);
          dir.setMaxSizeInBytes(0);
          writer.close();
          dir.close();
          break;
        }
      }
    }
  }
  /*
  Test: make sure when we run out of disk space or hit
  random IOExceptions in any of the addIndexes(*) calls
  that 1) index is not corrupt (searcher can open/search
  it) and 2) transactional semantics are followed:
  either all or none of the incoming documents were in
  fact added.
   */
  public void testAddIndexOnDiskFull() throws IOException {
    int START_COUNT = 57;
    int NUM_DIR = 50;
    int END_COUNT = START_COUNT + NUM_DIR * 25;

    // Build up a bunch of dirs that have indexes which we
    // will then merge together by calling addIndexes(*):
    Directory[] dirs = new Directory[NUM_DIR];
    long inputDiskUsage = 0;
    for (int i = 0; i < NUM_DIR; i++) {
      dirs[i] = newDirectory();
      IndexWriter writer =
          new IndexWriter(
              dirs[i], newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
      for (int j = 0; j < 25; j++) {
        addDocWithIndex(writer, 25 * i + j);
      }
      writer.close();
      String[] files = dirs[i].listAll();
      for (int j = 0; j < files.length; j++) {
        inputDiskUsage += dirs[i].fileLength(files[j]);
      }
    }

    // Now, build a starting index that has START_COUNT docs.  We
    // will then try to addIndexesNoOptimize into a copy of this:
    MockDirectoryWrapper startDir = newDirectory();
    IndexWriter writer =
        new IndexWriter(
            startDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
    for (int j = 0; j < START_COUNT; j++) {
      addDocWithIndex(writer, j);
    }
    writer.close();

    // Make sure starting index seems to be working properly:
    Term searchTerm = new Term("content", "aaa");
    IndexReader reader = IndexReader.open(startDir, true);
    assertEquals("first docFreq", 57, reader.docFreq(searchTerm));

    IndexSearcher searcher = newSearcher(reader);
    ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
    assertEquals("first number of hits", 57, hits.length);
    searcher.close();
    reader.close();

    // Iterate with larger and larger amounts of free
    // disk space.  With little free disk space,
    // addIndexes will certainly run out of space &
    // fail.  Verify that when this happens, index is
    // not corrupt and index in fact has added no
    // documents.  Then, we increase disk space by 2000
    // bytes each iteration.  At some point there is
    // enough free disk space and addIndexes should
    // succeed and index should show all documents were
    // added.

    // String[] files = startDir.listAll();
    long diskUsage = startDir.sizeInBytes();

    long startDiskUsage = 0;
    String[] files = startDir.listAll();
    for (int i = 0; i < files.length; i++) {
      startDiskUsage += startDir.fileLength(files[i]);
    }

    for (int iter = 0; iter < 3; iter++) {

      if (VERBOSE) System.out.println("TEST: iter=" + iter);

      // Start with 100 bytes more than we are currently using:
      long diskFree = diskUsage + _TestUtil.nextInt(random, 50, 200);

      int method = iter;

      boolean success = false;
      boolean done = false;

      String methodName;
      if (0 == method) {
        methodName = "addIndexes(Directory[]) + optimize()";
      } else if (1 == method) {
        methodName = "addIndexes(IndexReader[])";
      } else {
        methodName = "addIndexes(Directory[])";
      }

      while (!done) {
        if (VERBOSE) {
          System.out.println("TEST: cycle...");
        }

        // Make a new dir that will enforce disk usage:
        MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
        writer =
            new IndexWriter(
                dir,
                newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
                    .setOpenMode(OpenMode.APPEND)
                    .setMergePolicy(newLogMergePolicy()));
        IOException err = null;
        writer.setInfoStream(VERBOSE ? System.out : null);

        MergeScheduler ms = writer.getConfig().getMergeScheduler();
        for (int x = 0; x < 2; x++) {
          if (ms instanceof ConcurrentMergeScheduler)
            // This test intentionally produces exceptions
            // in the threads that CMS launches; we don't
            // want to pollute test output with these.
            if (0 == x) ((ConcurrentMergeScheduler) ms).setSuppressExceptions();
            else ((ConcurrentMergeScheduler) ms).clearSuppressExceptions();

          // Two loops: first time, limit disk space &
          // throw random IOExceptions; second time, no
          // disk space limit:

          double rate = 0.05;
          double diskRatio = ((double) diskFree) / diskUsage;
          long thisDiskFree;

          String testName = null;

          if (0 == x) {
            thisDiskFree = diskFree;
            if (diskRatio >= 2.0) {
              rate /= 2;
            }
            if (diskRatio >= 4.0) {
              rate /= 2;
            }
            if (diskRatio >= 6.0) {
              rate = 0.0;
            }
            if (VERBOSE)
              testName =
                  "disk full test " + methodName + " with disk full at " + diskFree + " bytes";
          } else {
            thisDiskFree = 0;
            rate = 0.0;
            if (VERBOSE) testName = "disk full test " + methodName + " with unlimited disk space";
          }

          if (VERBOSE) System.out.println("\ncycle: " + testName);

          dir.setTrackDiskUsage(true);
          dir.setMaxSizeInBytes(thisDiskFree);
          dir.setRandomIOExceptionRate(rate);

          try {

            if (0 == method) {
              writer.addIndexes(dirs);
              writer.optimize();
            } else if (1 == method) {
              IndexReader readers[] = new IndexReader[dirs.length];
              for (int i = 0; i < dirs.length; i++) {
                readers[i] = IndexReader.open(dirs[i], true);
              }
              try {
                writer.addIndexes(readers);
              } finally {
                for (int i = 0; i < dirs.length; i++) {
                  readers[i].close();
                }
              }
            } else {
              writer.addIndexes(dirs);
            }

            success = true;
            if (VERBOSE) {
              System.out.println("  success!");
            }

            if (0 == x) {
              done = true;
            }

          } catch (IOException e) {
            success = false;
            err = e;
            if (VERBOSE) {
              System.out.println("  hit IOException: " + e);
              e.printStackTrace(System.out);
            }

            if (1 == x) {
              e.printStackTrace(System.out);
              fail(methodName + " hit IOException after disk space was freed up");
            }
          }

          // Make sure all threads from
          // ConcurrentMergeScheduler are done
          _TestUtil.syncConcurrentMerges(writer);

          if (VERBOSE) {
            System.out.println("  now test readers");
          }

          // Finally, verify index is not corrupt, and, if
          // we succeeded, we see all docs added, and if we
          // failed, we see either all docs or no docs added
          // (transactional semantics):
          try {
            reader = IndexReader.open(dir, true);
          } catch (IOException e) {
            e.printStackTrace(System.out);
            fail(testName + ": exception when creating IndexReader: " + e);
          }
          int result = reader.docFreq(searchTerm);
          if (success) {
            if (result != START_COUNT) {
              fail(
                  testName
                      + ": method did not throw exception but docFreq('aaa') is "
                      + result
                      + " instead of expected "
                      + START_COUNT);
            }
          } else {
            // On hitting exception we still may have added
            // all docs:
            if (result != START_COUNT && result != END_COUNT) {
              err.printStackTrace(System.out);
              fail(
                  testName
                      + ": method did throw exception but docFreq('aaa') is "
                      + result
                      + " instead of expected "
                      + START_COUNT
                      + " or "
                      + END_COUNT);
            }
          }

          searcher = newSearcher(reader);
          try {
            hits = searcher.search(new TermQuery(searchTerm), null, END_COUNT).scoreDocs;
          } catch (IOException e) {
            e.printStackTrace(System.out);
            fail(testName + ": exception when searching: " + e);
          }
          int result2 = hits.length;
          if (success) {
            if (result2 != result) {
              fail(
                  testName
                      + ": method did not throw exception but hits.length for search on term 'aaa' is "
                      + result2
                      + " instead of expected "
                      + result);
            }
          } else {
            // On hitting exception we still may have added
            // all docs:
            if (result2 != result) {
              err.printStackTrace(System.out);
              fail(
                  testName
                      + ": method did throw exception but hits.length for search on term 'aaa' is "
                      + result2
                      + " instead of expected "
                      + result);
            }
          }

          searcher.close();
          reader.close();
          if (VERBOSE) {
            System.out.println("  count is " + result);
          }

          if (done || result == END_COUNT) {
            break;
          }
        }

        if (VERBOSE) {
          System.out.println(
              "  start disk = "
                  + startDiskUsage
                  + "; input disk = "
                  + inputDiskUsage
                  + "; max used = "
                  + dir.getMaxUsedSizeInBytes());
        }

        if (done) {
          // Javadocs state that temp free Directory space
          // required is at most 2X total input size of
          // indices so let's make sure:
          assertTrue(
              "max free Directory space required exceeded 1X the total input index sizes during "
                  + methodName
                  + ": max temp usage = "
                  + (dir.getMaxUsedSizeInBytes() - startDiskUsage)
                  + " bytes vs limit="
                  + (2 * (startDiskUsage + inputDiskUsage))
                  + "; starting disk usage = "
                  + startDiskUsage
                  + " bytes; "
                  + "input index disk usage = "
                  + inputDiskUsage
                  + " bytes",
              (dir.getMaxUsedSizeInBytes() - startDiskUsage)
                  < 2 * (startDiskUsage + inputDiskUsage));
        }

        // Make sure we don't hit disk full during close below:
        dir.setMaxSizeInBytes(0);
        dir.setRandomIOExceptionRate(0.0);

        writer.close();

        // Wait for all BG threads to finish else
        // dir.close() will throw IOException because
        // there are still open files
        _TestUtil.syncConcurrentMerges(ms);

        dir.close();

        // Try again with more free space:
        diskFree +=
            TEST_NIGHTLY
                ? _TestUtil.nextInt(random, 4000, 8000)
                : _TestUtil.nextInt(random, 40000, 80000);
      }
    }

    startDir.close();
    for (Directory dir : dirs) dir.close();
  }
Ejemplo n.º 29
0
  @Nightly
  public void test() throws Exception {
    MockDirectoryWrapper dir =
        new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("4GBStoredFields")));
    dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);

    IndexWriter w =
        new IndexWriter(
            dir,
            new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
                .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
                .setRAMBufferSizeMB(256.0)
                .setMergeScheduler(new ConcurrentMergeScheduler())
                .setMergePolicy(newLogMergePolicy(false, 10))
                .setOpenMode(IndexWriterConfig.OpenMode.CREATE));

    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogByteSizeMergePolicy) {
      // 1 petabyte:
      ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
    }

    final Document doc = new Document();
    final FieldType ft = new FieldType();
    ft.setIndexed(false);
    ft.setStored(true);
    ft.freeze();
    final int valueLength = RandomInts.randomIntBetween(random(), 1 << 13, 1 << 20);
    final byte[] value = new byte[valueLength];
    for (int i = 0; i < valueLength; ++i) {
      // random so that even compressing codecs can't compress it
      value[i] = (byte) random().nextInt(256);
    }
    final Field f = new Field("fld", value, ft);
    doc.add(f);

    final int numDocs = (int) ((1L << 32) / valueLength + 100);
    for (int i = 0; i < numDocs; ++i) {
      w.addDocument(doc);
      if (VERBOSE && i % (numDocs / 10) == 0) {
        System.out.println(i + " of " + numDocs + "...");
      }
    }
    w.forceMerge(1);
    w.close();
    if (VERBOSE) {
      boolean found = false;
      for (String file : dir.listAll()) {
        if (file.endsWith(".fdt")) {
          final long fileLength = dir.fileLength(file);
          if (fileLength >= 1L << 32) {
            found = true;
          }
          System.out.println("File length of " + file + " : " + fileLength);
        }
      }
      if (!found) {
        System.out.println("No .fdt file larger than 4GB, test bug?");
      }
    }

    DirectoryReader rd = DirectoryReader.open(dir);
    Document sd = rd.document(numDocs - 1);
    assertNotNull(sd);
    assertEquals(1, sd.getFields().size());
    BytesRef valueRef = sd.getBinaryValue("fld");
    assertNotNull(valueRef);
    assertEquals(new BytesRef(value), valueRef);
    rd.close();

    dir.close();
  }
Ejemplo n.º 30
0
  /**
   * This test executes a number of merges and compares the contents of the segments created when
   * using compound file or not using one.
   *
   * <p>TODO: the original test used to print the segment contents to System.out for visual
   * validation. To have the same effect, a new method checkSegment(String name, ...) should be
   * created that would assert various things about the segment.
   */
  public void testIndexAndMerge() throws Exception {
    StringWriter sw = new StringWriter();
    PrintWriter out = new PrintWriter(sw, true);

    Directory directory = newFSDirectory(indexDir, null);

    if (directory instanceof MockDirectoryWrapper) {
      // We create unreferenced files (we don't even write
      // a segments file):
      ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
    }

    IndexWriter writer =
        new IndexWriter(
            directory,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.CREATE)
                .setMaxBufferedDocs(-1)
                .setMergePolicy(newLogMergePolicy(10)));

    SegmentCommitInfo si1 = indexDoc(writer, "test.txt");
    printSegment(out, si1);

    SegmentCommitInfo si2 = indexDoc(writer, "test2.txt");
    printSegment(out, si2);
    writer.close();

    SegmentCommitInfo siMerge = merge(directory, si1, si2, "_merge", false);
    printSegment(out, siMerge);

    SegmentCommitInfo siMerge2 = merge(directory, si1, si2, "_merge2", false);
    printSegment(out, siMerge2);

    SegmentCommitInfo siMerge3 = merge(directory, siMerge, siMerge2, "_merge3", false);
    printSegment(out, siMerge3);

    directory.close();
    out.close();
    sw.close();

    String multiFileOutput = sw.toString();
    // System.out.println(multiFileOutput);

    sw = new StringWriter();
    out = new PrintWriter(sw, true);

    directory = newFSDirectory(indexDir, null);

    if (directory instanceof MockDirectoryWrapper) {
      // We create unreferenced files (we don't even write
      // a segments file):
      ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
    }

    writer =
        new IndexWriter(
            directory,
            newIndexWriterConfig(new MockAnalyzer(random()))
                .setOpenMode(OpenMode.CREATE)
                .setMaxBufferedDocs(-1)
                .setMergePolicy(newLogMergePolicy(10)));

    si1 = indexDoc(writer, "test.txt");
    printSegment(out, si1);

    si2 = indexDoc(writer, "test2.txt");
    printSegment(out, si2);
    writer.close();

    siMerge = merge(directory, si1, si2, "_merge", true);
    printSegment(out, siMerge);

    siMerge2 = merge(directory, si1, si2, "_merge2", true);
    printSegment(out, siMerge2);

    siMerge3 = merge(directory, siMerge, siMerge2, "_merge3", true);
    printSegment(out, siMerge3);

    directory.close();
    out.close();
    sw.close();
    String singleFileOutput = sw.toString();

    assertEquals(multiFileOutput, singleFileOutput);
  }