@Test
  public void testVersioningIndexConflictWithFlush() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(1l));

    index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(2l));

    engine.flush(new Engine.Flush());

    index = new Engine.Index(null, newUid("1"), doc).version(1l);
    try {
      engine.index(index);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    // future versions should not work as well
    index = new Engine.Index(null, newUid("1"), doc).version(3l);
    try {
      engine.index(index);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }
  }
  @Test
  public void testVersioningDeleteConflictWithFlush() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(1l));

    index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(2l));

    engine.flush(new Engine.Flush());

    Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")).version(1l);
    try {
      engine.delete(delete);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    // future versions should not work as well
    delete = new Engine.Delete("test", "1", newUid("1")).version(3l);
    try {
      engine.delete(delete);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    engine.flush(new Engine.Flush());

    // now actually delete
    delete = new Engine.Delete("test", "1", newUid("1")).version(2l);
    engine.delete(delete);
    assertThat(delete.version(), equalTo(3l));

    engine.flush(new Engine.Flush());

    // now check if we can index to a delete doc with version
    index = new Engine.Index(null, newUid("1"), doc).version(2l);
    try {
      engine.index(index);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    // we shouldn't be able to create as well
    Engine.Create create = new Engine.Create(null, newUid("1"), doc).version(2l);
    try {
      engine.create(create);
    } catch (VersionConflictEngineException e) {
      // all is well
    }
  }
  @Test
  public void testSimpleRecover() throws Exception {
    ParsedDocument doc =
        testParsedDocument(
            "1",
            "1",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_1,
            false);
    engine.create(new Engine.Create(null, newUid("1"), doc));
    engine.flush(new Engine.Flush());

    engine.recover(
        new Engine.RecoveryHandler() {
          @Override
          public void phase1(SnapshotIndexCommit snapshot) throws EngineException {
            try {
              engine.flush(new Engine.Flush());
              assertThat("flush is not allowed in phase 3", false, equalTo(true));
            } catch (FlushNotAllowedEngineException e) {
              // all is well
            }
          }

          @Override
          public void phase2(Translog.Snapshot snapshot) throws EngineException {
            MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
            try {
              engine.flush(new Engine.Flush());
              assertThat("flush is not allowed in phase 3", false, equalTo(true));
            } catch (FlushNotAllowedEngineException e) {
              // all is well
            }
          }

          @Override
          public void phase3(Translog.Snapshot snapshot) throws EngineException {
            MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
            try {
              // we can do this here since we are on the same thread
              engine.flush(new Engine.Flush());
              assertThat("flush is not allowed in phase 3", false, equalTo(true));
            } catch (FlushNotAllowedEngineException e) {
              // all is well
            }
          }
        });

    engine.flush(new Engine.Flush());
    engine.close();
  }
  public void run() throws Exception {
    for (Thread t : searcherThreads) {
      t.start();
    }
    for (Thread t : writerThreads) {
      t.start();
    }
    barrier1.await();

    Refresher refresher = new Refresher();
    scheduledExecutorService.scheduleWithFixedDelay(
        refresher, refreshSchedule.millis(), refreshSchedule.millis(), TimeUnit.MILLISECONDS);
    Flusher flusher = new Flusher();
    scheduledExecutorService.scheduleWithFixedDelay(
        flusher, flushSchedule.millis(), flushSchedule.millis(), TimeUnit.MILLISECONDS);

    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    barrier2.await();

    latch.await();
    stopWatch.stop();

    System.out.println("Summary");
    System.out.println(
        "   -- Readers ["
            + searcherThreads.length
            + "] with ["
            + searcherIterations
            + "] iterations");
    System.out.println(
        "   -- Writers [" + writerThreads.length + "] with [" + writerIterations + "] iterations");
    System.out.println("   -- Took: " + stopWatch.totalTime());
    System.out.println(
        "   -- Refresh [" + refresher.id + "] took: " + refresher.stopWatch.totalTime());
    System.out.println("   -- Flush [" + flusher.id + "] took: " + flusher.stopWatch.totalTime());
    System.out.println("   -- Store size " + store.estimateSize());

    scheduledExecutorService.shutdown();

    engine.refresh(new Engine.Refresh(true));
    stopWatch = new StopWatch();
    stopWatch.start();
    Engine.Searcher searcher = engine.searcher();
    TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), idGenerator.get() + 1);
    stopWatch.stop();
    System.out.println(
        "   -- Indexed ["
            + idGenerator.get()
            + "] docs, found ["
            + topDocs.totalHits
            + "] hits, took "
            + stopWatch.totalTime());
    searcher.release();
  }
 @BeforeMethod
 public void setUp() throws Exception {
   threadPool = new ThreadPool();
   store = createStore();
   store.deleteContent();
   storeReplica = createStoreReplica();
   storeReplica.deleteContent();
   engine = createEngine(store, createTranslog());
   engine.start();
   replicaEngine = createEngine(storeReplica, createTranslogReplica());
   replicaEngine.start();
 }
  @AfterMethod
  public void tearDown() throws Exception {
    replicaEngine.close();
    storeReplica.close();

    engine.close();
    store.close();

    if (threadPool != null) {
      threadPool.shutdownNow();
    }
  }
  @Test
  public void testVersioningNewCreate() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Create create = new Engine.Create(null, newUid("1"), doc);
    engine.create(create);
    assertThat(create.version(), equalTo(1l));

    create = new Engine.Create(null, newUid("1"), doc).version(create.version()).origin(REPLICA);
    replicaEngine.create(create);
    assertThat(create.version(), equalTo(1l));
  }
  @Test
  public void testVersioningNewIndex() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(1l));

    index = new Engine.Index(null, newUid("1"), doc).version(index.version()).origin(REPLICA);
    replicaEngine.index(index);
    assertThat(index.version(), equalTo(1l));
  }
  @Test
  public void testVersioningCreateExistsException() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Create create = new Engine.Create(null, newUid("1"), doc);
    engine.create(create);
    assertThat(create.version(), equalTo(1l));

    create = new Engine.Create(null, newUid("1"), doc);
    try {
      engine.create(create);
      assert false;
    } catch (DocumentAlreadyExistsException e) {
      // all is well
    }
  }
Exemplo n.º 10
0
  @Test
  public void testCreatedFlagAfterFlush() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertTrue(index.created());

    engine.delete(new Engine.Delete(null, "1", newUid("1")));

    engine.flush(new Engine.Flush());

    index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertTrue(index.created());
  }
  public static void main(String[] args) throws Exception {
    ShardId shardId = new ShardId(new Index("index"), 1);
    Settings settings = EMPTY_SETTINGS;

    //        Store store = new RamStore(shardId, settings);
    Store store = new ByteBufferStore(shardId, settings, null, new ByteBufferCache(settings));
    //        Store store = new NioFsStore(shardId, settings);

    store.deleteContent();

    ThreadPool threadPool = new ScalingThreadPool();
    SnapshotDeletionPolicy deletionPolicy =
        new SnapshotDeletionPolicy(new KeepOnlyLastDeletionPolicy(shardId, settings));
    Engine engine =
        new RobinEngine(
            shardId,
            settings,
            store,
            deletionPolicy,
            new MemoryTranslog(shardId, settings),
            new LogByteSizeMergePolicyProvider(store),
            new ConcurrentMergeSchedulerProvider(shardId, settings),
            new AnalysisService(shardId.index()),
            new SimilarityService(shardId.index()));
    engine.start();

    SimpleEngineBenchmark benchmark =
        new SimpleEngineBenchmark(store, engine)
            .numberOfContentItems(1000)
            .searcherThreads(50)
            .searcherIterations(10000)
            .writerThreads(10)
            .writerIterations(10000)
            .refreshSchedule(new TimeValue(1, TimeUnit.SECONDS))
            .flushSchedule(new TimeValue(1, TimeUnit.MINUTES))
            .create(false)
            .build();

    benchmark.run();

    engine.close();
    store.close();
    threadPool.shutdown();
  }
Exemplo n.º 12
0
 @Before
 public void setUp() throws Exception {
   super.setUp();
   defaultSettings =
       ImmutableSettings.builder()
           .put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, getRandom().nextBoolean())
           .build(); // TODO randomize more settings
   threadPool = new ThreadPool();
   store = createStore();
   store.deleteContent();
   storeReplica = createStoreReplica();
   storeReplica.deleteContent();
   engineSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
   engine = createEngine(engineSettingsService, store, createTranslog());
   engine.start();
   replicaSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
   replicaEngine = createEngine(replicaSettingsService, storeReplica, createTranslogReplica());
   replicaEngine.start();
 }
  public SimpleEngineBenchmark build() {
    for (int i = 0; i < searcherThreads.length; i++) {
      searcherThreads[i] = new Thread(new SearcherThread(), "Searcher[" + i + "]");
    }
    for (int i = 0; i < writerThreads.length; i++) {
      writerThreads[i] = new Thread(new WriterThread(), "Writer[" + i + "]");
    }

    latch = new CountDownLatch(searcherThreads.length + writerThreads.length);
    barrier1 = new CyclicBarrier(searcherThreads.length + writerThreads.length + 1);
    barrier2 = new CyclicBarrier(searcherThreads.length + writerThreads.length + 1);

    // warmup by indexing all content items
    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    for (String contentItem : contentItems) {
      int id = idGenerator.incrementAndGet();
      String sId = Integer.toString(id);
      Document doc = doc().add(field("_id", sId)).add(field("content", contentItem)).build();
      if (create) {
        engine.create(
            new Engine.Create(doc, Lucene.STANDARD_ANALYZER, "type", sId, TRANSLOG_PAYLOAD));
      } else {
        engine.index(
            new Engine.Index(
                new Term("_id", sId),
                doc,
                Lucene.STANDARD_ANALYZER,
                "type",
                sId,
                TRANSLOG_PAYLOAD));
      }
    }
    engine.refresh(new Engine.Refresh(true));
    stopWatch.stop();
    System.out.println(
        "Warmup of [" + contentItems.length + "] content items, took " + stopWatch.totalTime());

    return this;
  }
Exemplo n.º 14
0
  @Test
  public void testExternalVersioningIndexConflict() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index =
        new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(12);
    engine.index(index);
    assertThat(index.version(), equalTo(12l));

    index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(14);
    engine.index(index);
    assertThat(index.version(), equalTo(14l));

    index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(13l);
    try {
      engine.index(index);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }
  }
Exemplo n.º 15
0
  @Test
  public void testVersioningReplicaConflict1() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(1l));

    index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(2l));

    // apply the second index to the replica, should work fine
    index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
    replicaEngine.index(index);
    assertThat(index.version(), equalTo(2l));

    // now, the old one should not work
    index = new Engine.Index(null, newUid("1"), doc).version(1l).origin(REPLICA);
    try {
      replicaEngine.index(index);
      assert false;
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    // second version on replica should fail as well
    try {
      index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
      replicaEngine.index(index);
      assertThat(index.version(), equalTo(2l));
    } catch (VersionConflictEngineException e) {
      // all is well
    }
  }
Exemplo n.º 16
0
  @Test
  public void testRecoverWithOperationsBetweenPhase1AndPhase2() throws Exception {
    ParsedDocument doc1 =
        testParsedDocument(
            "1",
            "1",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_1,
            false);
    engine.create(new Engine.Create(null, newUid("1"), doc1));
    engine.flush(new Engine.Flush());
    ParsedDocument doc2 =
        testParsedDocument(
            "2",
            "2",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_2,
            false);
    engine.create(new Engine.Create(null, newUid("2"), doc2));

    engine.recover(
        new Engine.RecoveryHandler() {
          @Override
          public void phase1(SnapshotIndexCommit snapshot) throws EngineException {}

          @Override
          public void phase2(Translog.Snapshot snapshot) throws EngineException {
            assertThat(snapshot.hasNext(), equalTo(true));
            Translog.Create create = (Translog.Create) snapshot.next();
            assertThat(create.source().toBytesArray(), equalTo(B_2));
            assertThat(snapshot.hasNext(), equalTo(false));
          }

          @Override
          public void phase3(Translog.Snapshot snapshot) throws EngineException {
            MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
          }
        });

    engine.flush(new Engine.Flush());
    engine.close();
  }
Exemplo n.º 17
0
  @Test
  public void testVersioningReplicaConflict2() {
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
    Engine.Index index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(1l));

    // apply the first index to the replica, should work fine
    index = new Engine.Index(null, newUid("1"), doc).version(1l).origin(REPLICA);
    replicaEngine.index(index);
    assertThat(index.version(), equalTo(1l));

    // index it again
    index = new Engine.Index(null, newUid("1"), doc);
    engine.index(index);
    assertThat(index.version(), equalTo(2l));

    // now delete it
    Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"));
    engine.delete(delete);
    assertThat(delete.version(), equalTo(3l));

    // apply the delete on the replica (skipping the second index)
    delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA);
    replicaEngine.delete(delete);
    assertThat(delete.version(), equalTo(3l));

    // second time delete with same version should fail
    try {
      delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA);
      replicaEngine.delete(delete);
      assertThat(delete.version(), equalTo(3l));
    } catch (VersionConflictEngineException e) {
      // all is well
    }

    // now do the second index on the replica, it should fail
    try {
      index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
      replicaEngine.index(index);
      assertThat(index.version(), equalTo(2l));
    } catch (VersionConflictEngineException e) {
      // all is well
    }
  }
Exemplo n.º 18
0
  @Test
  public void testSimpleSnapshot() throws Exception {
    // create a document
    ParsedDocument doc1 =
        testParsedDocument(
            "1",
            "1",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_1,
            false);
    engine.create(new Engine.Create(null, newUid("1"), doc1));

    final ExecutorService executorService = Executors.newCachedThreadPool();

    engine.snapshot(
        new Engine.SnapshotHandler<Void>() {
          @Override
          public Void snapshot(
              final SnapshotIndexCommit snapshotIndexCommit1,
              final Translog.Snapshot translogSnapshot1) {
            MatcherAssert.assertThat(
                snapshotIndexCommit1, SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
            assertThat(translogSnapshot1.hasNext(), equalTo(true));
            Translog.Create create1 = (Translog.Create) translogSnapshot1.next();
            assertThat(create1.source().toBytesArray(), equalTo(B_1.toBytesArray()));
            assertThat(translogSnapshot1.hasNext(), equalTo(false));

            Future<Object> future =
                executorService.submit(
                    new Callable<Object>() {
                      @Override
                      public Object call() throws Exception {
                        engine.flush(new Engine.Flush());
                        ParsedDocument doc2 =
                            testParsedDocument(
                                "2",
                                "2",
                                "test",
                                null,
                                -1,
                                -1,
                                testDocumentWithTextField(),
                                Lucene.STANDARD_ANALYZER,
                                B_2,
                                false);
                        engine.create(new Engine.Create(null, newUid("2"), doc2));
                        engine.flush(new Engine.Flush());
                        ParsedDocument doc3 =
                            testParsedDocument(
                                "3",
                                "3",
                                "test",
                                null,
                                -1,
                                -1,
                                testDocumentWithTextField(),
                                Lucene.STANDARD_ANALYZER,
                                B_3,
                                false);
                        engine.create(new Engine.Create(null, newUid("3"), doc3));
                        return null;
                      }
                    });

            try {
              future.get();
            } catch (Exception e) {
              e.printStackTrace();
              assertThat(e.getMessage(), false, equalTo(true));
            }

            MatcherAssert.assertThat(
                snapshotIndexCommit1, SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());

            engine.snapshot(
                new Engine.SnapshotHandler<Void>() {
                  @Override
                  public Void snapshot(
                      SnapshotIndexCommit snapshotIndexCommit2, Translog.Snapshot translogSnapshot2)
                      throws EngineException {
                    MatcherAssert.assertThat(
                        snapshotIndexCommit1,
                        SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
                    MatcherAssert.assertThat(
                        snapshotIndexCommit2,
                        SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
                    assertThat(
                        snapshotIndexCommit2.getSegmentsFileName(),
                        not(equalTo(snapshotIndexCommit1.getSegmentsFileName())));
                    assertThat(translogSnapshot2.hasNext(), equalTo(true));
                    Translog.Create create3 = (Translog.Create) translogSnapshot2.next();
                    assertThat(create3.source().toBytesArray(), equalTo(B_3.toBytesArray()));
                    assertThat(translogSnapshot2.hasNext(), equalTo(false));
                    return null;
                  }
                });
            return null;
          }
        });

    engine.close();
  }
Exemplo n.º 19
0
  @Test
  public void testSearchResultRelease() throws Exception {
    Engine.Searcher searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    searchResult.release();

    // create a document
    ParsedDocument doc =
        testParsedDocument(
            "1",
            "1",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_1,
            false);
    engine.create(new Engine.Create(null, newUid("1"), doc));

    // its not there...
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 0));
    searchResult.release();

    // refresh and it should be there
    engine.refresh(new Engine.Refresh(true));

    // now its there...
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 1));
    // don't release the search result yet...

    // delete, refresh and do a new search, it should not be there
    engine.delete(new Engine.Delete("test", "1", newUid("1")));
    engine.refresh(new Engine.Refresh(true));
    Engine.Searcher updateSearchResult = engine.searcher();
    MatcherAssert.assertThat(
        updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    updateSearchResult.release();

    // the non release search result should not see the deleted yet...
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 1));
    searchResult.release();
  }
Exemplo n.º 20
0
  @Test
  public void testSimpleOperations() throws Exception {
    Engine.Searcher searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    searchResult.release();

    // create a document
    Document document = testDocumentWithTextField();
    document.add(
        new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
    ParsedDocument doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
    engine.create(new Engine.Create(null, newUid("1"), doc));

    // its not there...
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 0));
    searchResult.release();

    // but, we can still get it (in realtime)
    Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1")));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray()));
    assertThat(getResult.docIdAndVersion(), nullValue());

    // but, not there non realtime
    getResult = engine.get(new Engine.Get(false, newUid("1")));
    assertThat(getResult.exists(), equalTo(false));

    // refresh and it should be there
    engine.refresh(new Engine.Refresh(true));

    // now its there...
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 1));
    searchResult.release();

    // also in non realtime
    getResult = engine.get(new Engine.Get(false, newUid("1")));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.docIdAndVersion(), notNullValue());

    // now do an update
    document = testDocument();
    document.add(new TextField("value", "test1", Field.Store.YES));
    document.add(
        new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
    doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_2, false);
    engine.index(new Engine.Index(null, newUid("1"), doc));

    // its not updated yet...
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test1")), 0));
    searchResult.release();

    // but, we can still get it (in realtime)
    getResult = engine.get(new Engine.Get(true, newUid("1")));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray()));
    assertThat(getResult.docIdAndVersion(), nullValue());

    // refresh and it should be updated
    engine.refresh(new Engine.Refresh(true));

    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test1")), 1));
    searchResult.release();

    // now delete
    engine.delete(new Engine.Delete("test", "1", newUid("1")));

    // its not deleted yet
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test1")), 1));
    searchResult.release();

    // but, get should not see it (in realtime)
    getResult = engine.get(new Engine.Get(true, newUid("1")));
    assertThat(getResult.exists(), equalTo(false));

    // refresh and it should be deleted
    engine.refresh(new Engine.Refresh(true));

    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test1")), 0));
    searchResult.release();

    // add it back
    document = testDocumentWithTextField();
    document.add(
        new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
    doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
    engine.create(new Engine.Create(null, newUid("1"), doc));

    // its not there...
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test1")), 0));
    searchResult.release();

    // refresh and it should be there
    engine.refresh(new Engine.Refresh(true));

    // now its there...
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test1")), 0));
    searchResult.release();

    // now flush
    engine.flush(new Engine.Flush());

    // and, verify get (in real time)
    getResult = engine.get(new Engine.Get(true, newUid("1")));
    assertThat(getResult.exists(), equalTo(true));
    assertThat(getResult.source(), nullValue());
    assertThat(getResult.docIdAndVersion(), notNullValue());

    // make sure we can still work with the engine
    // now do an update
    document = testDocument();
    document.add(new TextField("value", "test1", Field.Store.YES));
    doc =
        testParsedDocument(
            "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
    engine.index(new Engine.Index(null, newUid("1"), doc));

    // its not updated yet...
    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test1")), 0));
    searchResult.release();

    // refresh and it should be updated
    engine.refresh(new Engine.Refresh(true));

    searchResult = engine.searcher();
    MatcherAssert.assertThat(
        searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test")), 0));
    MatcherAssert.assertThat(
        searchResult,
        EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
            new TermQuery(new Term("value", "test1")), 1));
    searchResult.release();

    engine.close();
  }
Exemplo n.º 21
0
  @Test
  public void testSegments() throws Exception {
    List<Segment> segments = engine.segments();
    assertThat(segments.isEmpty(), equalTo(true));
    final boolean defaultCompound =
        defaultSettings.getAsBoolean(RobinEngine.INDEX_COMPOUND_ON_FLUSH, true);

    // create a doc and refresh
    ParsedDocument doc =
        testParsedDocument(
            "1",
            "1",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_1,
            false);
    engine.create(new Engine.Create(null, newUid("1"), doc));

    ParsedDocument doc2 =
        testParsedDocument(
            "2",
            "2",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_2,
            false);
    engine.create(new Engine.Create(null, newUid("2"), doc2));
    engine.refresh(new Engine.Refresh(true));

    segments = engine.segments();
    assertThat(segments.size(), equalTo(1));
    assertThat(segments.get(0).isCommitted(), equalTo(false));
    assertThat(segments.get(0).isSearch(), equalTo(true));
    assertThat(segments.get(0).getNumDocs(), equalTo(2));
    assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
    assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));

    engine.flush(new Engine.Flush());

    segments = engine.segments();
    assertThat(segments.size(), equalTo(1));
    assertThat(segments.get(0).isCommitted(), equalTo(true));
    assertThat(segments.get(0).isSearch(), equalTo(true));
    assertThat(segments.get(0).getNumDocs(), equalTo(2));
    assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
    assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));

    engineSettingsService.refreshSettings(
        ImmutableSettings.builder().put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, false).build());

    ParsedDocument doc3 =
        testParsedDocument(
            "3",
            "3",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_3,
            false);
    engine.create(new Engine.Create(null, newUid("3"), doc3));
    engine.refresh(new Engine.Refresh(true));

    segments = engine.segments();
    assertThat(segments.size(), equalTo(2));
    assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
    assertThat(segments.get(0).isCommitted(), equalTo(true));
    assertThat(segments.get(0).isSearch(), equalTo(true));
    assertThat(segments.get(0).getNumDocs(), equalTo(2));
    assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
    assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));

    assertThat(segments.get(1).isCommitted(), equalTo(false));
    assertThat(segments.get(1).isSearch(), equalTo(true));
    assertThat(segments.get(1).getNumDocs(), equalTo(1));
    assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
    assertThat(segments.get(1).isCompound(), equalTo(false));

    engine.delete(new Engine.Delete("test", "1", newUid("1")));
    engine.refresh(new Engine.Refresh(true));

    segments = engine.segments();
    assertThat(segments.size(), equalTo(2));
    assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
    assertThat(segments.get(0).isCommitted(), equalTo(true));
    assertThat(segments.get(0).isSearch(), equalTo(true));
    assertThat(segments.get(0).getNumDocs(), equalTo(1));
    assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
    assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));

    assertThat(segments.get(1).isCommitted(), equalTo(false));
    assertThat(segments.get(1).isSearch(), equalTo(true));
    assertThat(segments.get(1).getNumDocs(), equalTo(1));
    assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
    assertThat(segments.get(1).isCompound(), equalTo(false));

    engineSettingsService.refreshSettings(
        ImmutableSettings.builder().put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, true).build());
    ParsedDocument doc4 =
        testParsedDocument(
            "4",
            "4",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_3,
            false);
    engine.create(new Engine.Create(null, newUid("4"), doc4));
    engine.refresh(new Engine.Refresh(true));

    segments = engine.segments();
    assertThat(segments.size(), equalTo(3));
    assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
    assertThat(segments.get(0).isCommitted(), equalTo(true));
    assertThat(segments.get(0).isSearch(), equalTo(true));
    assertThat(segments.get(0).getNumDocs(), equalTo(1));
    assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
    assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));

    assertThat(segments.get(1).isCommitted(), equalTo(false));
    assertThat(segments.get(1).isSearch(), equalTo(true));
    assertThat(segments.get(1).getNumDocs(), equalTo(1));
    assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
    assertThat(segments.get(1).isCompound(), equalTo(false));

    assertThat(segments.get(2).isCommitted(), equalTo(false));
    assertThat(segments.get(2).isSearch(), equalTo(true));
    assertThat(segments.get(2).getNumDocs(), equalTo(1));
    assertThat(segments.get(2).getDeletedDocs(), equalTo(0));
    assertThat(segments.get(2).isCompound(), equalTo(true));
  }
  @Test
  public void testSegments() throws Exception {
    List<Segment> segments = engine.segments();
    assertThat(segments.isEmpty(), equalTo(true));

    // create a doc and refresh
    ParsedDocument doc =
        testParsedDocument(
            "1",
            "1",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_1,
            false);
    engine.create(new Engine.Create(null, newUid("1"), doc));

    ParsedDocument doc2 =
        testParsedDocument(
            "2",
            "2",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_2,
            false);
    engine.create(new Engine.Create(null, newUid("2"), doc2));
    engine.refresh(new Engine.Refresh(true));

    segments = engine.segments();
    assertThat(segments.size(), equalTo(1));
    assertThat(segments.get(0).committed(), equalTo(false));
    assertThat(segments.get(0).search(), equalTo(true));
    assertThat(segments.get(0).numDocs(), equalTo(2));
    assertThat(segments.get(0).deletedDocs(), equalTo(0));

    engine.flush(new Engine.Flush());

    segments = engine.segments();
    assertThat(segments.size(), equalTo(1));
    assertThat(segments.get(0).committed(), equalTo(true));
    assertThat(segments.get(0).search(), equalTo(true));
    assertThat(segments.get(0).numDocs(), equalTo(2));
    assertThat(segments.get(0).deletedDocs(), equalTo(0));

    ParsedDocument doc3 =
        testParsedDocument(
            "3",
            "3",
            "test",
            null,
            -1,
            -1,
            testDocumentWithTextField(),
            Lucene.STANDARD_ANALYZER,
            B_3,
            false);
    engine.create(new Engine.Create(null, newUid("3"), doc3));
    engine.refresh(new Engine.Refresh(true));

    segments = engine.segments();
    assertThat(segments.size(), equalTo(2));
    assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true));
    assertThat(segments.get(0).committed(), equalTo(true));
    assertThat(segments.get(0).search(), equalTo(true));
    assertThat(segments.get(0).numDocs(), equalTo(2));
    assertThat(segments.get(0).deletedDocs(), equalTo(0));

    assertThat(segments.get(1).committed(), equalTo(false));
    assertThat(segments.get(1).search(), equalTo(true));
    assertThat(segments.get(1).numDocs(), equalTo(1));
    assertThat(segments.get(1).deletedDocs(), equalTo(0));

    engine.delete(new Engine.Delete("test", "1", newUid("1")));
    engine.refresh(new Engine.Refresh(true));

    segments = engine.segments();
    assertThat(segments.size(), equalTo(2));
    assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true));
    assertThat(segments.get(0).committed(), equalTo(true));
    assertThat(segments.get(0).search(), equalTo(true));
    assertThat(segments.get(0).numDocs(), equalTo(1));
    assertThat(segments.get(0).deletedDocs(), equalTo(1));

    assertThat(segments.get(1).committed(), equalTo(false));
    assertThat(segments.get(1).search(), equalTo(true));
    assertThat(segments.get(1).numDocs(), equalTo(1));
    assertThat(segments.get(1).deletedDocs(), equalTo(0));
  }