@Test public void testVersioningDeleteConflictWithFlush() { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false); Engine.Index index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); engine.flush(new Engine.Flush()); Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")).version(1l); try { engine.delete(delete); assert false; } catch (VersionConflictEngineException e) { // all is well } // future versions should not work as well delete = new Engine.Delete("test", "1", newUid("1")).version(3l); try { engine.delete(delete); assert false; } catch (VersionConflictEngineException e) { // all is well } engine.flush(new Engine.Flush()); // now actually delete delete = new Engine.Delete("test", "1", newUid("1")).version(2l); engine.delete(delete); assertThat(delete.version(), equalTo(3l)); engine.flush(new Engine.Flush()); // now check if we can index to a delete doc with version index = new Engine.Index(null, newUid("1"), doc).version(2l); try { engine.index(index); assert false; } catch (VersionConflictEngineException e) { // all is well } // we shouldn't be able to create as well Engine.Create create = new Engine.Create(null, newUid("1"), doc).version(2l); try { engine.create(create); } catch (VersionConflictEngineException e) { // all is well } }
@Test public void testVersioningReplicaConflict2() { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false); Engine.Index index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(1l)); // apply the first index to the replica, should work fine index = new Engine.Index(null, newUid("1"), doc).version(1l).origin(REPLICA); replicaEngine.index(index); assertThat(index.version(), equalTo(1l)); // index it again index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertThat(index.version(), equalTo(2l)); // now delete it Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")); engine.delete(delete); assertThat(delete.version(), equalTo(3l)); // apply the delete on the replica (skipping the second index) delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA); replicaEngine.delete(delete); assertThat(delete.version(), equalTo(3l)); // second time delete with same version should fail try { delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA); replicaEngine.delete(delete); assertThat(delete.version(), equalTo(3l)); } catch (VersionConflictEngineException e) { // all is well } // now do the second index on the replica, it should fail try { index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA); replicaEngine.index(index); assertThat(index.version(), equalTo(2l)); } catch (VersionConflictEngineException e) { // all is well } }
@Test public void testCreatedFlagAfterFlush() { ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false); Engine.Index index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertTrue(index.created()); engine.delete(new Engine.Delete(null, "1", newUid("1"))); engine.flush(new Engine.Flush()); index = new Engine.Index(null, newUid("1"), doc); engine.index(index); assertTrue(index.created()); }
/** delete record/collection with given name */ public synchronized void delete(String name) { Object r = get(name); if (r instanceof Atomic.Boolean) { engine.delete(((Atomic.Boolean) r).recid, Serializer.BOOLEAN); } else if (r instanceof Atomic.Integer) { engine.delete(((Atomic.Integer) r).recid, Serializer.INTEGER); } else if (r instanceof Atomic.Long) { engine.delete(((Atomic.Long) r).recid, Serializer.LONG); } else if (r instanceof Atomic.String) { engine.delete(((Atomic.String) r).recid, Serializer.STRING_NOSIZE); } else if (r instanceof Atomic.Var) { engine.delete(((Atomic.Var) r).recid, ((Atomic.Var) r).serializer); } else if (r instanceof Queue) { // drain queue Queue q = (Queue) r; while (q.poll() != null) { // do nothing } } else if (r instanceof HTreeMap || r instanceof HTreeMap.KeySet) { HTreeMap m = (r instanceof HTreeMap) ? (HTreeMap) r : ((HTreeMap.KeySet) r).parent(); m.clear(); // delete segments for (long segmentRecid : m.segmentRecids) { engine.delete(segmentRecid, HTreeMap.DIR_SERIALIZER); } } else if (r instanceof BTreeMap || r instanceof BTreeMap.KeySet) { BTreeMap m = (r instanceof BTreeMap) ? (BTreeMap) r : (BTreeMap) ((BTreeMap.KeySet) r).m; // TODO on BTreeMap recursively delete all nodes m.clear(); if (m.counter != null) engine.delete(m.counter.recid, Serializer.LONG); } for (String n : catalog.keySet()) { if (!n.startsWith(name)) continue; catalog.remove(n); } namesInstanciated.remove(name); namesLookup.remove(r); }
@Test public void testSearchResultRelease() throws Exception { Engine.Searcher searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); searchResult.release(); // create a document ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); // its not there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); searchResult.release(); // refresh and it should be there engine.refresh(new Engine.Refresh(true)); // now its there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); // don't release the search result yet... // delete, refresh and do a new search, it should not be there engine.delete(new Engine.Delete("test", "1", newUid("1"))); engine.refresh(new Engine.Refresh(true)); Engine.Searcher updateSearchResult = engine.searcher(); MatcherAssert.assertThat( updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); updateSearchResult.release(); // the non release search result should not see the deleted yet... MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); searchResult.release(); }
@Test public void testSimpleOperations() throws Exception { Engine.Searcher searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); searchResult.release(); // create a document Document document = testDocumentWithTextField(); document.add( new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); // its not there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); searchResult.release(); // but, we can still get it (in realtime) Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray())); assertThat(getResult.docIdAndVersion(), nullValue()); // but, not there non realtime getResult = engine.get(new Engine.Get(false, newUid("1"))); assertThat(getResult.exists(), equalTo(false)); // refresh and it should be there engine.refresh(new Engine.Refresh(true)); // now its there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); searchResult.release(); // also in non realtime getResult = engine.get(new Engine.Get(false, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.docIdAndVersion(), notNullValue()); // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); document.add( new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument( "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_2, false); engine.index(new Engine.Index(null, newUid("1"), doc)); // its not updated yet... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // but, we can still get it (in realtime) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray())); assertThat(getResult.docIdAndVersion(), nullValue()); // refresh and it should be updated engine.refresh(new Engine.Refresh(true)); searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 1)); searchResult.release(); // now delete engine.delete(new Engine.Delete("test", "1", newUid("1"))); // its not deleted yet searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 1)); searchResult.release(); // but, get should not see it (in realtime) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(false)); // refresh and it should be deleted engine.refresh(new Engine.Refresh(true)); searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // add it back document = testDocumentWithTextField(); document.add( new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument( "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); // its not there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // refresh and it should be there engine.refresh(new Engine.Refresh(true)); // now its there... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // now flush engine.flush(new Engine.Flush()); // and, verify get (in real time) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); assertThat(getResult.source(), nullValue()); assertThat(getResult.docIdAndVersion(), notNullValue()); // make sure we can still work with the engine // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); doc = testParsedDocument( "1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false); engine.index(new Engine.Index(null, newUid("1"), doc)); // its not updated yet... searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 0)); searchResult.release(); // refresh and it should be updated engine.refresh(new Engine.Refresh(true)); searchResult = engine.searcher(); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test")), 0)); MatcherAssert.assertThat( searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits( new TermQuery(new Term("value", "test1")), 1)); searchResult.release(); engine.close(); }
@Test public void testSegments() throws Exception { List<Segment> segments = engine.segments(); assertThat(segments.isEmpty(), equalTo(true)); final boolean defaultCompound = defaultSettings.getAsBoolean(RobinEngine.INDEX_COMPOUND_ON_FLUSH, true); // create a doc and refresh ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); ParsedDocument doc2 = testParsedDocument( "2", "2", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_2, false); engine.create(new Engine.Create(null, newUid("2"), doc2)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).isCommitted(), equalTo(false)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); engine.flush(new Engine.Flush()); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); engineSettingsService.refreshSettings( ImmutableSettings.builder().put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, false).build()); ParsedDocument doc3 = testParsedDocument( "3", "3", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_3, false); engine.create(new Engine.Create(null, newUid("3"), doc3)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); engine.delete(new Engine.Delete("test", "1", newUid("1"))); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); engineSettingsService.refreshSettings( ImmutableSettings.builder().put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, true).build()); ParsedDocument doc4 = testParsedDocument( "4", "4", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_3, false); engine.create(new Engine.Create(null, newUid("4"), doc4)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(3)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); assertThat(segments.get(0).isCompound(), equalTo(defaultCompound)); assertThat(segments.get(1).isCommitted(), equalTo(false)); assertThat(segments.get(1).isSearch(), equalTo(true)); assertThat(segments.get(1).getNumDocs(), equalTo(1)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertThat(segments.get(1).isCompound(), equalTo(false)); assertThat(segments.get(2).isCommitted(), equalTo(false)); assertThat(segments.get(2).isSearch(), equalTo(true)); assertThat(segments.get(2).getNumDocs(), equalTo(1)); assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); assertThat(segments.get(2).isCompound(), equalTo(true)); }
@Test public void testSegments() throws Exception { List<Segment> segments = engine.segments(); assertThat(segments.isEmpty(), equalTo(true)); // create a doc and refresh ParsedDocument doc = testParsedDocument( "1", "1", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_1, false); engine.create(new Engine.Create(null, newUid("1"), doc)); ParsedDocument doc2 = testParsedDocument( "2", "2", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_2, false); engine.create(new Engine.Create(null, newUid("2"), doc2)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).committed(), equalTo(false)); assertThat(segments.get(0).search(), equalTo(true)); assertThat(segments.get(0).numDocs(), equalTo(2)); assertThat(segments.get(0).deletedDocs(), equalTo(0)); engine.flush(new Engine.Flush()); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); assertThat(segments.get(0).committed(), equalTo(true)); assertThat(segments.get(0).search(), equalTo(true)); assertThat(segments.get(0).numDocs(), equalTo(2)); assertThat(segments.get(0).deletedDocs(), equalTo(0)); ParsedDocument doc3 = testParsedDocument( "3", "3", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_3, false); engine.create(new Engine.Create(null, newUid("3"), doc3)); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true)); assertThat(segments.get(0).committed(), equalTo(true)); assertThat(segments.get(0).search(), equalTo(true)); assertThat(segments.get(0).numDocs(), equalTo(2)); assertThat(segments.get(0).deletedDocs(), equalTo(0)); assertThat(segments.get(1).committed(), equalTo(false)); assertThat(segments.get(1).search(), equalTo(true)); assertThat(segments.get(1).numDocs(), equalTo(1)); assertThat(segments.get(1).deletedDocs(), equalTo(0)); engine.delete(new Engine.Delete("test", "1", newUid("1"))); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true)); assertThat(segments.get(0).committed(), equalTo(true)); assertThat(segments.get(0).search(), equalTo(true)); assertThat(segments.get(0).numDocs(), equalTo(1)); assertThat(segments.get(0).deletedDocs(), equalTo(1)); assertThat(segments.get(1).committed(), equalTo(false)); assertThat(segments.get(1).search(), equalTo(true)); assertThat(segments.get(1).numDocs(), equalTo(1)); assertThat(segments.get(1).deletedDocs(), equalTo(0)); }