@Test
public void testVersioningIndexConflictWithFlush() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(2l));
engine.flush(new Engine.Flush());
index = new Engine.Index(null, newUid("1"), doc).version(1l);
try {
engine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
// future versions should not work as well
index = new Engine.Index(null, newUid("1"), doc).version(3l);
try {
engine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testVersioningDeleteConflictWithFlush() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(2l));
engine.flush(new Engine.Flush());
Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")).version(1l);
try {
engine.delete(delete);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
// future versions should not work as well
delete = new Engine.Delete("test", "1", newUid("1")).version(3l);
try {
engine.delete(delete);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
engine.flush(new Engine.Flush());
// now actually delete
delete = new Engine.Delete("test", "1", newUid("1")).version(2l);
engine.delete(delete);
assertThat(delete.version(), equalTo(3l));
engine.flush(new Engine.Flush());
// now check if we can index to a delete doc with version
index = new Engine.Index(null, newUid("1"), doc).version(2l);
try {
engine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
// we shouldn't be able to create as well
Engine.Create create = new Engine.Create(null, newUid("1"), doc).version(2l);
try {
engine.create(create);
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testSimpleRecover() throws Exception {
ParsedDocument doc =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc));
engine.flush(new Engine.Flush());
engine.recover(
new Engine.RecoveryHandler() {
@Override
public void phase1(SnapshotIndexCommit snapshot) throws EngineException {
try {
engine.flush(new Engine.Flush());
assertThat("flush is not allowed in phase 3", false, equalTo(true));
} catch (FlushNotAllowedEngineException e) {
// all is well
}
}
@Override
public void phase2(Translog.Snapshot snapshot) throws EngineException {
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
try {
engine.flush(new Engine.Flush());
assertThat("flush is not allowed in phase 3", false, equalTo(true));
} catch (FlushNotAllowedEngineException e) {
// all is well
}
}
@Override
public void phase3(Translog.Snapshot snapshot) throws EngineException {
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
try {
// we can do this here since we are on the same thread
engine.flush(new Engine.Flush());
assertThat("flush is not allowed in phase 3", false, equalTo(true));
} catch (FlushNotAllowedEngineException e) {
// all is well
}
}
});
engine.flush(new Engine.Flush());
engine.close();
}
public void run() throws Exception {
for (Thread t : searcherThreads) {
t.start();
}
for (Thread t : writerThreads) {
t.start();
}
barrier1.await();
Refresher refresher = new Refresher();
scheduledExecutorService.scheduleWithFixedDelay(
refresher, refreshSchedule.millis(), refreshSchedule.millis(), TimeUnit.MILLISECONDS);
Flusher flusher = new Flusher();
scheduledExecutorService.scheduleWithFixedDelay(
flusher, flushSchedule.millis(), flushSchedule.millis(), TimeUnit.MILLISECONDS);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
barrier2.await();
latch.await();
stopWatch.stop();
System.out.println("Summary");
System.out.println(
" -- Readers ["
+ searcherThreads.length
+ "] with ["
+ searcherIterations
+ "] iterations");
System.out.println(
" -- Writers [" + writerThreads.length + "] with [" + writerIterations + "] iterations");
System.out.println(" -- Took: " + stopWatch.totalTime());
System.out.println(
" -- Refresh [" + refresher.id + "] took: " + refresher.stopWatch.totalTime());
System.out.println(" -- Flush [" + flusher.id + "] took: " + flusher.stopWatch.totalTime());
System.out.println(" -- Store size " + store.estimateSize());
scheduledExecutorService.shutdown();
engine.refresh(new Engine.Refresh(true));
stopWatch = new StopWatch();
stopWatch.start();
Engine.Searcher searcher = engine.searcher();
TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), idGenerator.get() + 1);
stopWatch.stop();
System.out.println(
" -- Indexed ["
+ idGenerator.get()
+ "] docs, found ["
+ topDocs.totalHits
+ "] hits, took "
+ stopWatch.totalTime());
searcher.release();
}
@BeforeMethod
public void setUp() throws Exception {
threadPool = new ThreadPool();
store = createStore();
store.deleteContent();
storeReplica = createStoreReplica();
storeReplica.deleteContent();
engine = createEngine(store, createTranslog());
engine.start();
replicaEngine = createEngine(storeReplica, createTranslogReplica());
replicaEngine.start();
}
@AfterMethod
public void tearDown() throws Exception {
replicaEngine.close();
storeReplica.close();
engine.close();
store.close();
if (threadPool != null) {
threadPool.shutdownNow();
}
}
@Test
public void testVersioningNewCreate() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Create create = new Engine.Create(null, newUid("1"), doc);
engine.create(create);
assertThat(create.version(), equalTo(1l));
create = new Engine.Create(null, newUid("1"), doc).version(create.version()).origin(REPLICA);
replicaEngine.create(create);
assertThat(create.version(), equalTo(1l));
}
@Test
public void testVersioningNewIndex() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
index = new Engine.Index(null, newUid("1"), doc).version(index.version()).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(1l));
}
@Test
public void testVersioningCreateExistsException() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Create create = new Engine.Create(null, newUid("1"), doc);
engine.create(create);
assertThat(create.version(), equalTo(1l));
create = new Engine.Create(null, newUid("1"), doc);
try {
engine.create(create);
assert false;
} catch (DocumentAlreadyExistsException e) {
// all is well
}
}
@Test
public void testCreatedFlagAfterFlush() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertTrue(index.created());
engine.delete(new Engine.Delete(null, "1", newUid("1")));
engine.flush(new Engine.Flush());
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertTrue(index.created());
}
public static void main(String[] args) throws Exception {
ShardId shardId = new ShardId(new Index("index"), 1);
Settings settings = EMPTY_SETTINGS;
// Store store = new RamStore(shardId, settings);
Store store = new ByteBufferStore(shardId, settings, null, new ByteBufferCache(settings));
// Store store = new NioFsStore(shardId, settings);
store.deleteContent();
ThreadPool threadPool = new ScalingThreadPool();
SnapshotDeletionPolicy deletionPolicy =
new SnapshotDeletionPolicy(new KeepOnlyLastDeletionPolicy(shardId, settings));
Engine engine =
new RobinEngine(
shardId,
settings,
store,
deletionPolicy,
new MemoryTranslog(shardId, settings),
new LogByteSizeMergePolicyProvider(store),
new ConcurrentMergeSchedulerProvider(shardId, settings),
new AnalysisService(shardId.index()),
new SimilarityService(shardId.index()));
engine.start();
SimpleEngineBenchmark benchmark =
new SimpleEngineBenchmark(store, engine)
.numberOfContentItems(1000)
.searcherThreads(50)
.searcherIterations(10000)
.writerThreads(10)
.writerIterations(10000)
.refreshSchedule(new TimeValue(1, TimeUnit.SECONDS))
.flushSchedule(new TimeValue(1, TimeUnit.MINUTES))
.create(false)
.build();
benchmark.run();
engine.close();
store.close();
threadPool.shutdown();
}
@Before
public void setUp() throws Exception {
super.setUp();
defaultSettings =
ImmutableSettings.builder()
.put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, getRandom().nextBoolean())
.build(); // TODO randomize more settings
threadPool = new ThreadPool();
store = createStore();
store.deleteContent();
storeReplica = createStoreReplica();
storeReplica.deleteContent();
engineSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
engine = createEngine(engineSettingsService, store, createTranslog());
engine.start();
replicaSettingsService = new IndexSettingsService(shardId.index(), EMPTY_SETTINGS);
replicaEngine = createEngine(replicaSettingsService, storeReplica, createTranslogReplica());
replicaEngine.start();
}
public SimpleEngineBenchmark build() {
for (int i = 0; i < searcherThreads.length; i++) {
searcherThreads[i] = new Thread(new SearcherThread(), "Searcher[" + i + "]");
}
for (int i = 0; i < writerThreads.length; i++) {
writerThreads[i] = new Thread(new WriterThread(), "Writer[" + i + "]");
}
latch = new CountDownLatch(searcherThreads.length + writerThreads.length);
barrier1 = new CyclicBarrier(searcherThreads.length + writerThreads.length + 1);
barrier2 = new CyclicBarrier(searcherThreads.length + writerThreads.length + 1);
// warmup by indexing all content items
StopWatch stopWatch = new StopWatch();
stopWatch.start();
for (String contentItem : contentItems) {
int id = idGenerator.incrementAndGet();
String sId = Integer.toString(id);
Document doc = doc().add(field("_id", sId)).add(field("content", contentItem)).build();
if (create) {
engine.create(
new Engine.Create(doc, Lucene.STANDARD_ANALYZER, "type", sId, TRANSLOG_PAYLOAD));
} else {
engine.index(
new Engine.Index(
new Term("_id", sId),
doc,
Lucene.STANDARD_ANALYZER,
"type",
sId,
TRANSLOG_PAYLOAD));
}
}
engine.refresh(new Engine.Refresh(true));
stopWatch.stop();
System.out.println(
"Warmup of [" + contentItems.length + "] content items, took " + stopWatch.totalTime());
return this;
}
@Test
public void testExternalVersioningIndexConflict() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index =
new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(12);
engine.index(index);
assertThat(index.version(), equalTo(12l));
index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(14);
engine.index(index);
assertThat(index.version(), equalTo(14l));
index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(13l);
try {
engine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testVersioningReplicaConflict1() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(2l));
// apply the second index to the replica, should work fine
index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(2l));
// now, the old one should not work
index = new Engine.Index(null, newUid("1"), doc).version(1l).origin(REPLICA);
try {
replicaEngine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
// second version on replica should fail as well
try {
index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(2l));
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testRecoverWithOperationsBetweenPhase1AndPhase2() throws Exception {
ParsedDocument doc1 =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc1));
engine.flush(new Engine.Flush());
ParsedDocument doc2 =
testParsedDocument(
"2",
"2",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_2,
false);
engine.create(new Engine.Create(null, newUid("2"), doc2));
engine.recover(
new Engine.RecoveryHandler() {
@Override
public void phase1(SnapshotIndexCommit snapshot) throws EngineException {}
@Override
public void phase2(Translog.Snapshot snapshot) throws EngineException {
assertThat(snapshot.hasNext(), equalTo(true));
Translog.Create create = (Translog.Create) snapshot.next();
assertThat(create.source().toBytesArray(), equalTo(B_2));
assertThat(snapshot.hasNext(), equalTo(false));
}
@Override
public void phase3(Translog.Snapshot snapshot) throws EngineException {
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
}
});
engine.flush(new Engine.Flush());
engine.close();
}
@Test
public void testVersioningReplicaConflict2() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
// apply the first index to the replica, should work fine
index = new Engine.Index(null, newUid("1"), doc).version(1l).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(1l));
// index it again
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(2l));
// now delete it
Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"));
engine.delete(delete);
assertThat(delete.version(), equalTo(3l));
// apply the delete on the replica (skipping the second index)
delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA);
replicaEngine.delete(delete);
assertThat(delete.version(), equalTo(3l));
// second time delete with same version should fail
try {
delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA);
replicaEngine.delete(delete);
assertThat(delete.version(), equalTo(3l));
} catch (VersionConflictEngineException e) {
// all is well
}
// now do the second index on the replica, it should fail
try {
index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(2l));
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testSimpleSnapshot() throws Exception {
// create a document
ParsedDocument doc1 =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc1));
final ExecutorService executorService = Executors.newCachedThreadPool();
engine.snapshot(
new Engine.SnapshotHandler<Void>() {
@Override
public Void snapshot(
final SnapshotIndexCommit snapshotIndexCommit1,
final Translog.Snapshot translogSnapshot1) {
MatcherAssert.assertThat(
snapshotIndexCommit1, SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
assertThat(translogSnapshot1.hasNext(), equalTo(true));
Translog.Create create1 = (Translog.Create) translogSnapshot1.next();
assertThat(create1.source().toBytesArray(), equalTo(B_1.toBytesArray()));
assertThat(translogSnapshot1.hasNext(), equalTo(false));
Future<Object> future =
executorService.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
engine.flush(new Engine.Flush());
ParsedDocument doc2 =
testParsedDocument(
"2",
"2",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_2,
false);
engine.create(new Engine.Create(null, newUid("2"), doc2));
engine.flush(new Engine.Flush());
ParsedDocument doc3 =
testParsedDocument(
"3",
"3",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_3,
false);
engine.create(new Engine.Create(null, newUid("3"), doc3));
return null;
}
});
try {
future.get();
} catch (Exception e) {
e.printStackTrace();
assertThat(e.getMessage(), false, equalTo(true));
}
MatcherAssert.assertThat(
snapshotIndexCommit1, SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
engine.snapshot(
new Engine.SnapshotHandler<Void>() {
@Override
public Void snapshot(
SnapshotIndexCommit snapshotIndexCommit2, Translog.Snapshot translogSnapshot2)
throws EngineException {
MatcherAssert.assertThat(
snapshotIndexCommit1,
SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
MatcherAssert.assertThat(
snapshotIndexCommit2,
SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
assertThat(
snapshotIndexCommit2.getSegmentsFileName(),
not(equalTo(snapshotIndexCommit1.getSegmentsFileName())));
assertThat(translogSnapshot2.hasNext(), equalTo(true));
Translog.Create create3 = (Translog.Create) translogSnapshot2.next();
assertThat(create3.source().toBytesArray(), equalTo(B_3.toBytesArray()));
assertThat(translogSnapshot2.hasNext(), equalTo(false));
return null;
}
});
return null;
}
});
engine.close();
}
@Test
public void testSearchResultRelease() throws Exception {
Engine.Searcher searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
searchResult.release();
// create a document
ParsedDocument doc =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc));
// its not there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
searchResult.release();
// refresh and it should be there
engine.refresh(new Engine.Refresh(true));
// now its there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
// don't release the search result yet...
// delete, refresh and do a new search, it should not be there
engine.delete(new Engine.Delete("test", "1", newUid("1")));
engine.refresh(new Engine.Refresh(true));
Engine.Searcher updateSearchResult = engine.searcher();
MatcherAssert.assertThat(
updateSearchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
updateSearchResult.release();
// the non release search result should not see the deleted yet...
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
searchResult.release();
}
@Test
public void testSimpleOperations() throws Exception {
Engine.Searcher searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
searchResult.release();
// create a document
Document document = testDocumentWithTextField();
document.add(
new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.create(new Engine.Create(null, newUid("1"), doc));
// its not there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
searchResult.release();
// but, we can still get it (in realtime)
Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray()));
assertThat(getResult.docIdAndVersion(), nullValue());
// but, not there non realtime
getResult = engine.get(new Engine.Get(false, newUid("1")));
assertThat(getResult.exists(), equalTo(false));
// refresh and it should be there
engine.refresh(new Engine.Refresh(true));
// now its there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
searchResult.release();
// also in non realtime
getResult = engine.get(new Engine.Get(false, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.docIdAndVersion(), notNullValue());
// now do an update
document = testDocument();
document.add(new TextField("value", "test1", Field.Store.YES));
document.add(
new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_2, false);
engine.index(new Engine.Index(null, newUid("1"), doc));
// its not updated yet...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// but, we can still get it (in realtime)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray()));
assertThat(getResult.docIdAndVersion(), nullValue());
// refresh and it should be updated
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
// now delete
engine.delete(new Engine.Delete("test", "1", newUid("1")));
// its not deleted yet
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
// but, get should not see it (in realtime)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(false));
// refresh and it should be deleted
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// add it back
document = testDocumentWithTextField();
document.add(
new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.create(new Engine.Create(null, newUid("1"), doc));
// its not there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// refresh and it should be there
engine.refresh(new Engine.Refresh(true));
// now its there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// now flush
engine.flush(new Engine.Flush());
// and, verify get (in real time)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source(), nullValue());
assertThat(getResult.docIdAndVersion(), notNullValue());
// make sure we can still work with the engine
// now do an update
document = testDocument();
document.add(new TextField("value", "test1", Field.Store.YES));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.index(new Engine.Index(null, newUid("1"), doc));
// its not updated yet...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// refresh and it should be updated
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
engine.close();
}
@Test
public void testSegments() throws Exception {
List<Segment> segments = engine.segments();
assertThat(segments.isEmpty(), equalTo(true));
final boolean defaultCompound =
defaultSettings.getAsBoolean(RobinEngine.INDEX_COMPOUND_ON_FLUSH, true);
// create a doc and refresh
ParsedDocument doc =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc));
ParsedDocument doc2 =
testParsedDocument(
"2",
"2",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_2,
false);
engine.create(new Engine.Create(null, newUid("2"), doc2));
engine.refresh(new Engine.Refresh(true));
segments = engine.segments();
assertThat(segments.size(), equalTo(1));
assertThat(segments.get(0).isCommitted(), equalTo(false));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(2));
assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));
engine.flush(new Engine.Flush());
segments = engine.segments();
assertThat(segments.size(), equalTo(1));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(2));
assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));
engineSettingsService.refreshSettings(
ImmutableSettings.builder().put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, false).build());
ParsedDocument doc3 =
testParsedDocument(
"3",
"3",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_3,
false);
engine.create(new Engine.Create(null, newUid("3"), doc3));
engine.refresh(new Engine.Refresh(true));
segments = engine.segments();
assertThat(segments.size(), equalTo(2));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(2));
assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));
assertThat(segments.get(1).isCommitted(), equalTo(false));
assertThat(segments.get(1).isSearch(), equalTo(true));
assertThat(segments.get(1).getNumDocs(), equalTo(1));
assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
assertThat(segments.get(1).isCompound(), equalTo(false));
engine.delete(new Engine.Delete("test", "1", newUid("1")));
engine.refresh(new Engine.Refresh(true));
segments = engine.segments();
assertThat(segments.size(), equalTo(2));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(1));
assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));
assertThat(segments.get(1).isCommitted(), equalTo(false));
assertThat(segments.get(1).isSearch(), equalTo(true));
assertThat(segments.get(1).getNumDocs(), equalTo(1));
assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
assertThat(segments.get(1).isCompound(), equalTo(false));
engineSettingsService.refreshSettings(
ImmutableSettings.builder().put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, true).build());
ParsedDocument doc4 =
testParsedDocument(
"4",
"4",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_3,
false);
engine.create(new Engine.Create(null, newUid("4"), doc4));
engine.refresh(new Engine.Refresh(true));
segments = engine.segments();
assertThat(segments.size(), equalTo(3));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
assertThat(segments.get(0).isSearch(), equalTo(true));
assertThat(segments.get(0).getNumDocs(), equalTo(1));
assertThat(segments.get(0).getDeletedDocs(), equalTo(1));
assertThat(segments.get(0).isCompound(), equalTo(defaultCompound));
assertThat(segments.get(1).isCommitted(), equalTo(false));
assertThat(segments.get(1).isSearch(), equalTo(true));
assertThat(segments.get(1).getNumDocs(), equalTo(1));
assertThat(segments.get(1).getDeletedDocs(), equalTo(0));
assertThat(segments.get(1).isCompound(), equalTo(false));
assertThat(segments.get(2).isCommitted(), equalTo(false));
assertThat(segments.get(2).isSearch(), equalTo(true));
assertThat(segments.get(2).getNumDocs(), equalTo(1));
assertThat(segments.get(2).getDeletedDocs(), equalTo(0));
assertThat(segments.get(2).isCompound(), equalTo(true));
}
@Test
public void testSegments() throws Exception {
List<Segment> segments = engine.segments();
assertThat(segments.isEmpty(), equalTo(true));
// create a doc and refresh
ParsedDocument doc =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc));
ParsedDocument doc2 =
testParsedDocument(
"2",
"2",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_2,
false);
engine.create(new Engine.Create(null, newUid("2"), doc2));
engine.refresh(new Engine.Refresh(true));
segments = engine.segments();
assertThat(segments.size(), equalTo(1));
assertThat(segments.get(0).committed(), equalTo(false));
assertThat(segments.get(0).search(), equalTo(true));
assertThat(segments.get(0).numDocs(), equalTo(2));
assertThat(segments.get(0).deletedDocs(), equalTo(0));
engine.flush(new Engine.Flush());
segments = engine.segments();
assertThat(segments.size(), equalTo(1));
assertThat(segments.get(0).committed(), equalTo(true));
assertThat(segments.get(0).search(), equalTo(true));
assertThat(segments.get(0).numDocs(), equalTo(2));
assertThat(segments.get(0).deletedDocs(), equalTo(0));
ParsedDocument doc3 =
testParsedDocument(
"3",
"3",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_3,
false);
engine.create(new Engine.Create(null, newUid("3"), doc3));
engine.refresh(new Engine.Refresh(true));
segments = engine.segments();
assertThat(segments.size(), equalTo(2));
assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true));
assertThat(segments.get(0).committed(), equalTo(true));
assertThat(segments.get(0).search(), equalTo(true));
assertThat(segments.get(0).numDocs(), equalTo(2));
assertThat(segments.get(0).deletedDocs(), equalTo(0));
assertThat(segments.get(1).committed(), equalTo(false));
assertThat(segments.get(1).search(), equalTo(true));
assertThat(segments.get(1).numDocs(), equalTo(1));
assertThat(segments.get(1).deletedDocs(), equalTo(0));
engine.delete(new Engine.Delete("test", "1", newUid("1")));
engine.refresh(new Engine.Refresh(true));
segments = engine.segments();
assertThat(segments.size(), equalTo(2));
assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true));
assertThat(segments.get(0).committed(), equalTo(true));
assertThat(segments.get(0).search(), equalTo(true));
assertThat(segments.get(0).numDocs(), equalTo(1));
assertThat(segments.get(0).deletedDocs(), equalTo(1));
assertThat(segments.get(1).committed(), equalTo(false));
assertThat(segments.get(1).search(), equalTo(true));
assertThat(segments.get(1).numDocs(), equalTo(1));
assertThat(segments.get(1).deletedDocs(), equalTo(0));
}
