@AfterMethod
public void tearDown() throws Exception {
replicaEngine.close();
storeReplica.close();
engine.close();
store.close();
if (threadPool != null) {
threadPool.shutdownNow();
}
}
@Test
public void testSimpleRecover() throws Exception {
ParsedDocument doc =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc));
engine.flush(new Engine.Flush());
engine.recover(
new Engine.RecoveryHandler() {
@Override
public void phase1(SnapshotIndexCommit snapshot) throws EngineException {
try {
engine.flush(new Engine.Flush());
assertThat("flush is not allowed in phase 3", false, equalTo(true));
} catch (FlushNotAllowedEngineException e) {
// all is well
}
}
@Override
public void phase2(Translog.Snapshot snapshot) throws EngineException {
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
try {
engine.flush(new Engine.Flush());
assertThat("flush is not allowed in phase 3", false, equalTo(true));
} catch (FlushNotAllowedEngineException e) {
// all is well
}
}
@Override
public void phase3(Translog.Snapshot snapshot) throws EngineException {
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
try {
// we can do this here since we are on the same thread
engine.flush(new Engine.Flush());
assertThat("flush is not allowed in phase 3", false, equalTo(true));
} catch (FlushNotAllowedEngineException e) {
// all is well
}
}
});
engine.flush(new Engine.Flush());
engine.close();
}
@Test
public void testRecoverWithOperationsBetweenPhase1AndPhase2() throws Exception {
ParsedDocument doc1 =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc1));
engine.flush(new Engine.Flush());
ParsedDocument doc2 =
testParsedDocument(
"2",
"2",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_2,
false);
engine.create(new Engine.Create(null, newUid("2"), doc2));
engine.recover(
new Engine.RecoveryHandler() {
@Override
public void phase1(SnapshotIndexCommit snapshot) throws EngineException {}
@Override
public void phase2(Translog.Snapshot snapshot) throws EngineException {
assertThat(snapshot.hasNext(), equalTo(true));
Translog.Create create = (Translog.Create) snapshot.next();
assertThat(create.source().toBytesArray(), equalTo(B_2));
assertThat(snapshot.hasNext(), equalTo(false));
}
@Override
public void phase3(Translog.Snapshot snapshot) throws EngineException {
MatcherAssert.assertThat(snapshot, TranslogSizeMatcher.translogSize(0));
}
});
engine.flush(new Engine.Flush());
engine.close();
}
public static void main(String[] args) throws Exception {
ShardId shardId = new ShardId(new Index("index"), 1);
Settings settings = EMPTY_SETTINGS;
// Store store = new RamStore(shardId, settings);
Store store = new ByteBufferStore(shardId, settings, null, new ByteBufferCache(settings));
// Store store = new NioFsStore(shardId, settings);
store.deleteContent();
ThreadPool threadPool = new ScalingThreadPool();
SnapshotDeletionPolicy deletionPolicy =
new SnapshotDeletionPolicy(new KeepOnlyLastDeletionPolicy(shardId, settings));
Engine engine =
new RobinEngine(
shardId,
settings,
store,
deletionPolicy,
new MemoryTranslog(shardId, settings),
new LogByteSizeMergePolicyProvider(store),
new ConcurrentMergeSchedulerProvider(shardId, settings),
new AnalysisService(shardId.index()),
new SimilarityService(shardId.index()));
engine.start();
SimpleEngineBenchmark benchmark =
new SimpleEngineBenchmark(store, engine)
.numberOfContentItems(1000)
.searcherThreads(50)
.searcherIterations(10000)
.writerThreads(10)
.writerIterations(10000)
.refreshSchedule(new TimeValue(1, TimeUnit.SECONDS))
.flushSchedule(new TimeValue(1, TimeUnit.MINUTES))
.create(false)
.build();
benchmark.run();
engine.close();
store.close();
threadPool.shutdown();
}
@Test
public void testSimpleSnapshot() throws Exception {
// create a document
ParsedDocument doc1 =
testParsedDocument(
"1",
"1",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_1,
false);
engine.create(new Engine.Create(null, newUid("1"), doc1));
final ExecutorService executorService = Executors.newCachedThreadPool();
engine.snapshot(
new Engine.SnapshotHandler<Void>() {
@Override
public Void snapshot(
final SnapshotIndexCommit snapshotIndexCommit1,
final Translog.Snapshot translogSnapshot1) {
MatcherAssert.assertThat(
snapshotIndexCommit1, SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
assertThat(translogSnapshot1.hasNext(), equalTo(true));
Translog.Create create1 = (Translog.Create) translogSnapshot1.next();
assertThat(create1.source().toBytesArray(), equalTo(B_1.toBytesArray()));
assertThat(translogSnapshot1.hasNext(), equalTo(false));
Future<Object> future =
executorService.submit(
new Callable<Object>() {
@Override
public Object call() throws Exception {
engine.flush(new Engine.Flush());
ParsedDocument doc2 =
testParsedDocument(
"2",
"2",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_2,
false);
engine.create(new Engine.Create(null, newUid("2"), doc2));
engine.flush(new Engine.Flush());
ParsedDocument doc3 =
testParsedDocument(
"3",
"3",
"test",
null,
-1,
-1,
testDocumentWithTextField(),
Lucene.STANDARD_ANALYZER,
B_3,
false);
engine.create(new Engine.Create(null, newUid("3"), doc3));
return null;
}
});
try {
future.get();
} catch (Exception e) {
e.printStackTrace();
assertThat(e.getMessage(), false, equalTo(true));
}
MatcherAssert.assertThat(
snapshotIndexCommit1, SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
engine.snapshot(
new Engine.SnapshotHandler<Void>() {
@Override
public Void snapshot(
SnapshotIndexCommit snapshotIndexCommit2, Translog.Snapshot translogSnapshot2)
throws EngineException {
MatcherAssert.assertThat(
snapshotIndexCommit1,
SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
MatcherAssert.assertThat(
snapshotIndexCommit2,
SnapshotIndexCommitExistsMatcher.snapshotIndexCommitExists());
assertThat(
snapshotIndexCommit2.getSegmentsFileName(),
not(equalTo(snapshotIndexCommit1.getSegmentsFileName())));
assertThat(translogSnapshot2.hasNext(), equalTo(true));
Translog.Create create3 = (Translog.Create) translogSnapshot2.next();
assertThat(create3.source().toBytesArray(), equalTo(B_3.toBytesArray()));
assertThat(translogSnapshot2.hasNext(), equalTo(false));
return null;
}
});
return null;
}
});
engine.close();
}
@Test
public void testSimpleOperations() throws Exception {
Engine.Searcher searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
searchResult.release();
// create a document
Document document = testDocumentWithTextField();
document.add(
new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.create(new Engine.Create(null, newUid("1"), doc));
// its not there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
searchResult.release();
// but, we can still get it (in realtime)
Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray()));
assertThat(getResult.docIdAndVersion(), nullValue());
// but, not there non realtime
getResult = engine.get(new Engine.Get(false, newUid("1")));
assertThat(getResult.exists(), equalTo(false));
// refresh and it should be there
engine.refresh(new Engine.Refresh(true));
// now its there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
searchResult.release();
// also in non realtime
getResult = engine.get(new Engine.Get(false, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.docIdAndVersion(), notNullValue());
// now do an update
document = testDocument();
document.add(new TextField("value", "test1", Field.Store.YES));
document.add(
new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_2, false);
engine.index(new Engine.Index(null, newUid("1"), doc));
// its not updated yet...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// but, we can still get it (in realtime)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray()));
assertThat(getResult.docIdAndVersion(), nullValue());
// refresh and it should be updated
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
// now delete
engine.delete(new Engine.Delete("test", "1", newUid("1")));
// its not deleted yet
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
// but, get should not see it (in realtime)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(false));
// refresh and it should be deleted
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// add it back
document = testDocumentWithTextField();
document.add(
new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.create(new Engine.Create(null, newUid("1"), doc));
// its not there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// refresh and it should be there
engine.refresh(new Engine.Refresh(true));
// now its there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// now flush
engine.flush(new Engine.Flush());
// and, verify get (in real time)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source(), nullValue());
assertThat(getResult.docIdAndVersion(), notNullValue());
// make sure we can still work with the engine
// now do an update
document = testDocument();
document.add(new TextField("value", "test1", Field.Store.YES));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.index(new Engine.Index(null, newUid("1"), doc));
// its not updated yet...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// refresh and it should be updated
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
engine.close();
}
