@Test
public void testVersioningIndexConflictWithFlush() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(2l));
engine.flush(new Engine.Flush());
index = new Engine.Index(null, newUid("1"), doc).version(1l);
try {
engine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
// future versions should not work as well
index = new Engine.Index(null, newUid("1"), doc).version(3l);
try {
engine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testVersioningDeleteConflictWithFlush() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(2l));
engine.flush(new Engine.Flush());
Engine.Delete delete = new Engine.Delete("test", "1", newUid("1")).version(1l);
try {
engine.delete(delete);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
// future versions should not work as well
delete = new Engine.Delete("test", "1", newUid("1")).version(3l);
try {
engine.delete(delete);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
engine.flush(new Engine.Flush());
// now actually delete
delete = new Engine.Delete("test", "1", newUid("1")).version(2l);
engine.delete(delete);
assertThat(delete.version(), equalTo(3l));
engine.flush(new Engine.Flush());
// now check if we can index to a delete doc with version
index = new Engine.Index(null, newUid("1"), doc).version(2l);
try {
engine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
// we shouldn't be able to create as well
Engine.Create create = new Engine.Create(null, newUid("1"), doc).version(2l);
try {
engine.create(create);
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testVersioningNewIndex() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
index = new Engine.Index(null, newUid("1"), doc).version(index.version()).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(1l));
}
@Test
public void testVersioningReplicaConflict2() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
// apply the first index to the replica, should work fine
index = new Engine.Index(null, newUid("1"), doc).version(1l).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(1l));
// index it again
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(2l));
// now delete it
Engine.Delete delete = new Engine.Delete("test", "1", newUid("1"));
engine.delete(delete);
assertThat(delete.version(), equalTo(3l));
// apply the delete on the replica (skipping the second index)
delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA);
replicaEngine.delete(delete);
assertThat(delete.version(), equalTo(3l));
// second time delete with same version should fail
try {
delete = new Engine.Delete("test", "1", newUid("1")).version(3l).origin(REPLICA);
replicaEngine.delete(delete);
assertThat(delete.version(), equalTo(3l));
} catch (VersionConflictEngineException e) {
// all is well
}
// now do the second index on the replica, it should fail
try {
index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(2l));
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testCreatedFlagAfterFlush() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertTrue(index.created());
engine.delete(new Engine.Delete(null, "1", newUid("1")));
engine.flush(new Engine.Flush());
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertTrue(index.created());
}
@Test
public void testExternalVersioningIndexConflict() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index =
new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(12);
engine.index(index);
assertThat(index.version(), equalTo(12l));
index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(14);
engine.index(index);
assertThat(index.version(), equalTo(14l));
index = new Engine.Index(null, newUid("1"), doc).versionType(VersionType.EXTERNAL).version(13l);
try {
engine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
}
@Test
public void testVersioningReplicaConflict1() {
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, testDocument(), Lucene.STANDARD_ANALYZER, B_1, false);
Engine.Index index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(1l));
index = new Engine.Index(null, newUid("1"), doc);
engine.index(index);
assertThat(index.version(), equalTo(2l));
// apply the second index to the replica, should work fine
index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(2l));
// now, the old one should not work
index = new Engine.Index(null, newUid("1"), doc).version(1l).origin(REPLICA);
try {
replicaEngine.index(index);
assert false;
} catch (VersionConflictEngineException e) {
// all is well
}
// second version on replica should fail as well
try {
index = new Engine.Index(null, newUid("1"), doc).version(2l).origin(REPLICA);
replicaEngine.index(index);
assertThat(index.version(), equalTo(2l));
} catch (VersionConflictEngineException e) {
// all is well
}
}
public SimpleEngineBenchmark build() {
for (int i = 0; i < searcherThreads.length; i++) {
searcherThreads[i] = new Thread(new SearcherThread(), "Searcher[" + i + "]");
}
for (int i = 0; i < writerThreads.length; i++) {
writerThreads[i] = new Thread(new WriterThread(), "Writer[" + i + "]");
}
latch = new CountDownLatch(searcherThreads.length + writerThreads.length);
barrier1 = new CyclicBarrier(searcherThreads.length + writerThreads.length + 1);
barrier2 = new CyclicBarrier(searcherThreads.length + writerThreads.length + 1);
// warmup by indexing all content items
StopWatch stopWatch = new StopWatch();
stopWatch.start();
for (String contentItem : contentItems) {
int id = idGenerator.incrementAndGet();
String sId = Integer.toString(id);
Document doc = doc().add(field("_id", sId)).add(field("content", contentItem)).build();
if (create) {
engine.create(
new Engine.Create(doc, Lucene.STANDARD_ANALYZER, "type", sId, TRANSLOG_PAYLOAD));
} else {
engine.index(
new Engine.Index(
new Term("_id", sId),
doc,
Lucene.STANDARD_ANALYZER,
"type",
sId,
TRANSLOG_PAYLOAD));
}
}
engine.refresh(new Engine.Refresh(true));
stopWatch.stop();
System.out.println(
"Warmup of [" + contentItems.length + "] content items, took " + stopWatch.totalTime());
return this;
}
@Test
public void testSimpleOperations() throws Exception {
Engine.Searcher searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
searchResult.release();
// create a document
Document document = testDocumentWithTextField();
document.add(
new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
ParsedDocument doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.create(new Engine.Create(null, newUid("1"), doc));
// its not there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
searchResult.release();
// but, we can still get it (in realtime)
Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray()));
assertThat(getResult.docIdAndVersion(), nullValue());
// but, not there non realtime
getResult = engine.get(new Engine.Get(false, newUid("1")));
assertThat(getResult.exists(), equalTo(false));
// refresh and it should be there
engine.refresh(new Engine.Refresh(true));
// now its there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
searchResult.release();
// also in non realtime
getResult = engine.get(new Engine.Get(false, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.docIdAndVersion(), notNullValue());
// now do an update
document = testDocument();
document.add(new TextField("value", "test1", Field.Store.YES));
document.add(
new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_2, false);
engine.index(new Engine.Index(null, newUid("1"), doc));
// its not updated yet...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// but, we can still get it (in realtime)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray()));
assertThat(getResult.docIdAndVersion(), nullValue());
// refresh and it should be updated
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
// now delete
engine.delete(new Engine.Delete("test", "1", newUid("1")));
// its not deleted yet
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
// but, get should not see it (in realtime)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(false));
// refresh and it should be deleted
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// add it back
document = testDocumentWithTextField();
document.add(
new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.create(new Engine.Create(null, newUid("1"), doc));
// its not there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// refresh and it should be there
engine.refresh(new Engine.Refresh(true));
// now its there...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// now flush
engine.flush(new Engine.Flush());
// and, verify get (in real time)
getResult = engine.get(new Engine.Get(true, newUid("1")));
assertThat(getResult.exists(), equalTo(true));
assertThat(getResult.source(), nullValue());
assertThat(getResult.docIdAndVersion(), notNullValue());
// make sure we can still work with the engine
// now do an update
document = testDocument();
document.add(new TextField("value", "test1", Field.Store.YES));
doc =
testParsedDocument(
"1", "1", "test", null, -1, -1, document, Lucene.STANDARD_ANALYZER, B_1, false);
engine.index(new Engine.Index(null, newUid("1"), doc));
// its not updated yet...
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 0));
searchResult.release();
// refresh and it should be updated
engine.refresh(new Engine.Refresh(true));
searchResult = engine.searcher();
MatcherAssert.assertThat(
searchResult, EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(1));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test")), 0));
MatcherAssert.assertThat(
searchResult,
EngineSearcherTotalHitsMatcher.engineSearcherTotalHits(
new TermQuery(new Term("value", "test1")), 1));
searchResult.release();
engine.close();
}