@Test public void testVersioningDeleteConflictWithFlush() { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false); Engine.Index index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); engine.flush(new Engine.Flush()); Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")).version(1l); try { engine.delete(delete); assert false; } catch (VersionConflictEngineException e) { // all is well } // future versions should not work as well delete = new Engine.Delete("test", "1", newUid("1")).version(3l); try { engine.delete(delete); assert false; } catch (VersionConflictEngineException e) { // all is well } engine.flush(new Engine.Flush()); // now actually delete delete = new Engine.Delete("test", "1", newUid("1")).version(2l); engine.delete(delete); assertThat(delete.version(), equalTo(3l)); engine.flush(new Engine.Flush()); // now check if we can index to a delete doc with version index = new Engine.Index(null, newUid("1"), doc).version(2l); try { engine.index(index); assert false; } catch (VersionConflictEngineException e) { // all is well } // we shouldn't be able to create as well Engine.Create create = new Engine.Create(null, newUid("1"), doc).version(2l); try { engine.create(create); } catch (VersionConflictEngineException e) { // all is well } }
@Test public void testSimpleRecover() throws Exception { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); engine.flush(new Engine.Flush()); engine.recover( new Engine.RecoveryHandler() { @Override public void phase1(SnapshotIndexCommit snapshot) throws EngineException { try { engine.flush(new Engine.Flush()); assertThat("flush is not allowed in phase 3", false, equalTo(true)); } catch (FlushNotAllowedEngineException e) { // all is well } } @Override public void phase2(Translog.Snapshot snapshot) throws EngineException { MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0)); try { engine.flush(new Engine.Flush()); assertThat("flush is not allowed in phase 3", false, equalTo(true)); } catch (FlushNotAllowedEngineException e) { // all is well } } @Override public void phase3(Translog.Snapshot snapshot) throws EngineException { MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0)); try { // we can do this here since we are on the same thread engine.flush(new Engine.Flush()); assertThat("flush is not allowed in phase 3", false, equalTo(true)); } catch (FlushNotAllowedEngineException e) { // all is well } } }); engine.flush(new Engine.Flush()); engine.close(); }
@Test public void testRecoverWithOperationsBetweenPhase1AndPhase2() throws Exception { ParsedDocument doc1 = testParsedDocument( "1", "1", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc1)); engine.flush(new Engine.Flush()); ParsedDocument doc2 = testParsedDocument( "2", "2", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_2, false); engine.create(new Engine.Create(null, newUid("2"), doc2)); engine.recover( new Engine.RecoveryHandler() { @Override public void phase1(SnapshotIndexCommit snapshot) throws EngineException {} @Override public void phase2(Translog.Snapshot snapshot) throws EngineException { assertThat(snapshot.hasNext(), equalTo(true)); Translog.Create create = (Translog.Create) snapshot.next(); assertThat(create.source().toBytesArray(), equalTo(B_2)); assertThat(snapshot.hasNext(), equalTo(false)); } @Override public void phase3(Translog.Snapshot snapshot) throws EngineException { MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0)); } }); engine.flush(new Engine.Flush()); engine.close(); }
@Test public void testVersioningIndexConflictWithFlush() { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false); Engine.Index index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); engine.flush(new Engine.Flush()); index = new Engine.Index(null, newUid("1"), doc).version(1l); try { engine.index(index); assert false; } catch (VersionConflictEngineException e) { // all is well } // future versions should not work as well index = new Engine.Index(null, newUid("1"), doc).version(3l); try { engine.index(index); assert false; } catch (VersionConflictEngineException e) { // all is well } }
@Test public void testCreatedFlagAfterFlush() { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false); Engine.Index index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertTrue(index.created()); engine.delete(new Engine.Delete(null, "1", newUid("1"))); engine.flush(new Engine.Flush()); index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertTrue(index.created()); }
@Test public void testVersioningCreateExistsExceptionWithFlush() { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false); Engine.Create create = new Engine.Create(null, newUid("1"), doc); engine.create(create); assertThat(create.version(), equalTo(1l)); engine.flush(new Engine.Flush()); create = new Engine.Create(null, newUid("1"), doc); try { engine.create(create); assert false; } catch (DocumentAlreadyExistsException e) { // all is well } }
@Test public void testExternalVersioningIndexConflictWithFlush() { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false); Engine.Index index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(12); engine.index(index); assertThat(index.version(), equalTo(12l)); index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(14); engine.index(index); assertThat(index.version(), equalTo(14l)); engine.flush(new Engine.Flush()); index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(13); try { engine.index(index); assert false; } catch (VersionConflictEngineException e) { // all is well } }
@Test public void testSimpleOperations() throws Exception { Engine.Searcher searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); searchResult.release(); // create a document Document document = testDocumentWithTextField(); document.add( new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); // its not there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); searchResult.release(); // but, we can still get it (in realtime) Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray())); assertThat(getResult.docIdAndVersion(), nullValue()); // but, not there non realtime getResult = engine.get(new Engine.Get(false, newUid("1"))); assertThat(getResult.exists(), equalTo(false)); // refresh and it should be there engine.refresh(new Engine.Refresh(true)); // now its there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); searchResult.release(); // also in non realtime getResult = engine.get(new Engine.Get(false, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); document.add( new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument( "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_2, false); engine.index(new Engine.Index(null, newUid("1"), doc)); // its not updated yet... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // but, we can still get it (in realtime) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray())); assertThat(getResult.docIdAndVersion(), nullValue()); // refresh and it should be updated engine.refresh(new Engine.Refresh(true)); searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 1)); searchResult.release(); // now delete engine.delete(new Engine.Delete("test", "1", newUid("1"))); // its not deleted yet searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 1)); searchResult.release(); // but, get should not see it (in realtime) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(false)); // refresh and it should be deleted engine.refresh(new Engine.Refresh(true)); searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // add it back document = testDocumentWithTextField(); document.add( new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument( "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); // its not there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // refresh and it should be there engine.refresh(new Engine.Refresh(true)); // now its there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // now flush engine.flush(new Engine.Flush()); // and, verify get (in real time) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source(), nullValue()); assertThat(getResult.docIdAndVersion(), notNullValue()); // make sure we can still work with the engine // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); doc = testParsedDocument( "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false); engine.index(new Engine.Index(null, newUid("1"), doc)); // its not updated yet... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // refresh and it should be updated engine.refresh(new Engine.Refresh(true)); searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 1)); searchResult.release(); engine.close(); }
@Test public void testSegments() throws Exception { List<Segment> segments = engine.segments(); assertThat(segments.isEmpty(), equalTo(true)); final boolean defaultCompound = defaultSettings.getAsBoolean(RobinEngine.INDEX_COMPOUND_ON_FLUSH, true); // create a doc and refresh ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); ParsedDocument doc2 = testParsedDocument( "2", "2", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_2, false); engine.create(new Engine.Create(null, newUid("2"), doc2)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).isCommitted(), equalTo(false)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); engine.flush(new Engine.Flush()); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); engineSettingsService.refreshSettings( ImmutableSettings.builder().put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, false).build()); ParsedDocument doc3 = testParsedDocument( "3", "3", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_3, false); engine.create(new Engine.Create(null, newUid("3"), doc3)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); engine.delete(new Engine.Delete("test", "1", newUid("1"))); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); engineSettingsService.refreshSettings( ImmutableSettings.builder().put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, true).build()); ParsedDocument doc4 = testParsedDocument( "4", "4", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_3, false); engine.create(new Engine.Create(null, newUid("4"), doc4)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(3)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); assertThat(segments.get(2).isCommitted(), equalTo(false)); assertThat(segments.get(2).isSearch(), equalTo(true)); assertThat(segments.get(2).getNumDocs(), equalTo(1)); assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); assertThat(segments.get(2).isCompound(), equalTo(true)); }
@Test public void testSegments() throws Exception { List<Segment> segments = engine.segments(); assertThat(segments.isEmpty(), equalTo(true)); // create a doc and refresh ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); ParsedDocument doc2 = testParsedDocument( "2", "2", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_2, false); engine.create(new Engine.Create(null, newUid("2"), doc2)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).committed(), equalTo(false)); assertThat(segments.get(0).search(), equalTo(true)); assertThat(segments.get(0).numDocs(), equalTo(2)); assertThat(segments.get(0).deletedDocs(), equalTo(0)); engine.flush(new Engine.Flush()); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).committed(), equalTo(true)); assertThat(segments.get(0).search(), equalTo(true)); assertThat(segments.get(0).numDocs(), equalTo(2)); assertThat(segments.get(0).deletedDocs(), equalTo(0)); ParsedDocument doc3 = testParsedDocument( "3", "3", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_3, false); engine.create(new Engine.Create(null, newUid("3"), doc3)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true)); assertThat(segments.get(0).committed(), equalTo(true)); assertThat(segments.get(0).search(), equalTo(true)); assertThat(segments.get(0).numDocs(), equalTo(2)); assertThat(segments.get(0).deletedDocs(), equalTo(0)); assertThat(segments.get(1).committed(), equalTo(false)); assertThat(segments.get(1).search(), equalTo(true)); assertThat(segments.get(1).numDocs(), equalTo(1)); assertThat(segments.get(1).deletedDocs(), equalTo(0)); engine.delete(new Engine.Delete("test", "1", newUid("1"))); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true)); assertThat(segments.get(0).committed(), equalTo(true)); assertThat(segments.get(0).search(), equalTo(true)); assertThat(segments.get(0).numDocs(), equalTo(1)); assertThat(segments.get(0).deletedDocs(), equalTo(1)); assertThat(segments.get(1).committed(), equalTo(false)); assertThat(segments.get(1).search(), equalTo(true)); assertThat(segments.get(1).numDocs(), equalTo(1)); assertThat(segments.get(1).deletedDocs(), equalTo(0)); }