public void testBulk() {
String[] bulkShardActions = new String[] {BulkAction.NAME + "[s]", BulkAction.NAME + "[s][r]"};
interceptTransportActions(bulkShardActions);
List<String> indices = new ArrayList<>();
BulkRequest bulkRequest = new BulkRequest();
int numIndexRequests = iterations(1, 10);
for (int i = 0; i < numIndexRequests; i++) {
String indexOrAlias = randomIndexOrAlias();
bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source("field", "value"));
indices.add(indexOrAlias);
}
int numDeleteRequests = iterations(1, 10);
for (int i = 0; i < numDeleteRequests; i++) {
String indexOrAlias = randomIndexOrAlias();
bulkRequest.add(new DeleteRequest(indexOrAlias, "type", "id"));
indices.add(indexOrAlias);
}
int numUpdateRequests = iterations(1, 10);
for (int i = 0; i < numUpdateRequests; i++) {
String indexOrAlias = randomIndexOrAlias();
bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1"));
indices.add(indexOrAlias);
}
internalCluster().clientNodeClient().bulk(bulkRequest).actionGet();
clearInterceptedActions();
assertIndicesSubset(indices, bulkShardActions);
}
@Test
public void testBenchmarkWithErrors() {
List<SearchRequest> reqList = new ArrayList<>();
int numQueries = scaledRandomIntBetween(20, 100);
int numErrors = scaledRandomIntBetween(1, numQueries);
final boolean containsFatal = randomBoolean();
if (containsFatal) {
ScriptScoreFunctionBuilder scriptFunction =
scriptFunction("DOES NOT COMPILE - fails on any shard");
SearchRequest searchRequest =
searchRequest()
.source(
searchSource()
.query(functionScoreQuery(FilterBuilders.matchAllFilter(), scriptFunction)));
reqList.add(searchRequest);
}
for (int i = 0; reqList.size() < numErrors; i++) {
ScriptScoreFunctionBuilder scriptFunction = scriptFunction("throw new RuntimeException();");
SearchRequest searchRequest =
searchRequest()
.source(
searchSource()
.query(functionScoreQuery(FilterBuilders.matchAllFilter(), scriptFunction)));
reqList.add(searchRequest);
}
logger.info("--> run with [{}] errors ", numErrors);
for (int i = 0; reqList.size() < numQueries; i++) {
reqList.add(BenchmarkTestUtil.randomSearch(client(), indices));
}
Collections.shuffle(reqList, getRandom());
final BenchmarkRequest request =
BenchmarkTestUtil.randomRequest(
client(),
indices,
numExecutorNodes,
competitionSettingsMap,
reqList.toArray(new SearchRequest[0]));
logger.info(
"--> Submitting benchmark - competitors [{}] iterations [{}]",
request.competitors().size(),
request.settings().iterations());
final BenchmarkResponse response = client().bench(request).actionGet();
assertThat(response, notNullValue());
if (response.hasErrors() || containsFatal) {
assertThat(response.state(), equalTo(BenchmarkResponse.State.FAILED));
} else {
assertThat(response.state(), equalTo(BenchmarkResponse.State.COMPLETE));
for (CompetitionResult result : response.competitionResults().values()) {
assertThat(result.nodeResults().size(), equalTo(numExecutorNodes));
validateCompetitionResult(
result, competitionSettingsMap.get(result.competitionName()), true);
}
}
assertThat(response.benchmarkName(), equalTo(BENCHMARK_NAME));
}
@Test // Uses of JMockit API: 1
public void verifyAllInvocationsInOrder() {
mockedList.add("one");
mockedList.size();
mockedList.add("two");
new FullVerificationsInOrder() {
{
mockedList.add("one");
mockedList.size();
mockedList.add("two");
}
};
}
@Test // Uses of JMockit API: 3
public void verifyInOrder(
@Mocked final List<String> firstMock, @Mocked final List<String> secondMock) {
// Using mocks:
firstMock.add("was called first");
secondMock.add("was called second");
new VerificationsInOrder() {
{
// Verifies that firstMock was called before secondMock:
firstMock.add("was called first");
secondMock.add("was called second");
}
};
}
@Override
public void messageReceived(T request, TransportChannel channel) throws Exception {
synchronized (InterceptingTransportService.this) {
if (actions.contains(action)) {
List<TransportRequest> requestList = requests.get(action);
if (requestList == null) {
requestList = new ArrayList<>();
requestList.add(request);
requests.put(action, requestList);
} else {
requestList.add(request);
}
}
}
requestHandler.messageReceived(request, channel);
}
public void beforeExecute(Task task) {
assertTrue(planned.contains(task));
String taskPath = path(task);
if (taskPath.startsWith(":buildSrc:")) {
return;
}
executedTasks.add(taskPath);
}
@Before
public void setup() {
int numIndices = iterations(1, 5);
for (int i = 0; i < numIndices; i++) {
indices.add("test" + i);
}
for (String index : indices) {
assertAcked(prepareCreate(index).addAlias(new Alias(index + "-alias")));
}
ensureGreen();
}
@Test(expected = UnexpectedInvocation.class) // Uses of JMockit API: 2
public void verifyThatInvocationsNeverHappenedWhenTheyDid(@Mocked List<String> mockTwo) {
mockedList.add("one");
mockTwo.size();
new FullVerifications() {
{
mockedList.add("one");
}
};
}
private void extractCauses(Throwable failure, List<Throwable> causes) {
if (failure instanceof MultipleBuildFailures) {
MultipleBuildFailures exception = (MultipleBuildFailures) failure;
for (Throwable componentFailure : exception.getCauses()) {
extractCauses(componentFailure, causes);
}
} else if (failure instanceof LocationAwareException) {
causes.addAll(((LocationAwareException) failure).getReportableCauses());
} else {
causes.add(failure.getCause());
}
}
@Test(expected = MissingInvocation.class) // Uses of JMockit API: 1
public void verifyAllInvocationsInOrderWithOutOfOrderVerifications() {
mockedList.add("one");
mockedList.add("two");
new FullVerificationsInOrder() {
{
mockedList.add("two");
mockedList.add("one");
}
};
}
private static void duelFieldDataGeoPoint(
Random random,
AtomicReaderContext context,
IndexGeoPointFieldData left,
IndexGeoPointFieldData right,
Distance precision)
throws Exception {
AtomicGeoPointFieldData leftData =
random.nextBoolean() ? left.load(context) : left.loadDirect(context);
AtomicGeoPointFieldData rightData =
random.nextBoolean() ? right.load(context) : right.loadDirect(context);
int numDocs = context.reader().maxDoc();
MultiGeoPointValues leftValues = leftData.getGeoPointValues();
MultiGeoPointValues rightValues = rightData.getGeoPointValues();
for (int i = 0; i < numDocs; ++i) {
leftValues.setDocument(i);
final int numValues = leftValues.count();
rightValues.setDocument(i);
;
assertEquals(numValues, rightValues.count());
List<GeoPoint> leftPoints = Lists.newArrayList();
List<GeoPoint> rightPoints = Lists.newArrayList();
for (int j = 0; j < numValues; ++j) {
GeoPoint l = leftValues.valueAt(j);
leftPoints.add(new GeoPoint(l.getLat(), l.getLon()));
GeoPoint r = rightValues.valueAt(j);
rightPoints.add(new GeoPoint(r.getLat(), r.getLon()));
}
for (GeoPoint l : leftPoints) {
assertTrue(
"Couldn't find " + l + " among " + rightPoints, contains(l, rightPoints, precision));
}
for (GeoPoint r : rightPoints) {
assertTrue(
"Couldn't find " + r + " among " + leftPoints, contains(r, leftPoints, precision));
}
}
}
@Test // Uses of JMockit API: 8
public void verifyNumberOfInvocations() {
// Using mock:
mockedList.add("once");
mockedList.add("twice");
mockedList.add("twice");
mockedList.add("three times");
mockedList.add("three times");
mockedList.add("three times");
new Verifications() {
{
// Following two verifications work exactly the same:
mockedList.add("once"); // minTimes == 1 is the default
mockedList.add("once");
times = 1;
// Verifies exact number of invocations:
mockedList.add("twice");
times = 2;
mockedList.add("three times");
times = 3;
// Verifies no invocations occurred:
mockedList.add("never happened");
times = 0;
// Verifies min/max number of invocations:
mockedList.add("three times");
minTimes = 1;
mockedList.add("three times");
minTimes = 2;
mockedList.add("three times");
maxTimes = 5;
}
};
}
@Test(expected = UnexpectedInvocation.class) // Uses of JMockit API: 1
public void verifyAllInvocationsInOrderWhenMoreOfThemHappen() {
mockedList.add("one");
mockedList.add("two");
mockedList.size();
new FullVerificationsInOrder() {
{
mockedList.add("one");
mockedList.add("two");
}
};
}
@Test // Uses of JMockit API: 1
public void verifyAllInvocations() {
mockedList.add("one");
mockedList.add("two");
// Verify all invocations to mockedList.
new FullVerifications() {
{
// Verifies first invocation:
mockedList.add("one");
// Verifies second (and last) invocation:
mockedList.add("two");
}
};
}
@Test
public void dynamicUpdates() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
int recCount = randomIntBetween(200, 600);
int numberOfTypes = randomIntBetween(1, 5);
List<IndexRequestBuilder> indexRequests = Lists.newArrayList();
for (int rec = 0; rec < recCount; rec++) {
String type = "type" + (rec % numberOfTypes);
String fieldName = "field_" + type + "_" + rec;
indexRequests.add(
client()
.prepareIndex("test", type, Integer.toString(rec))
.setSource(fieldName, "some_value"));
}
indexRandom(true, indexRequests);
logger.info("checking all the documents are there");
RefreshResponse refreshResponse =
client().admin().indices().prepareRefresh().execute().actionGet();
assertThat(refreshResponse.getFailedShards(), equalTo(0));
CountResponse response = client().prepareCount("test").execute().actionGet();
assertThat(response.getCount(), equalTo((long) recCount));
logger.info("checking all the fields are in the mappings");
for (int rec = 0; rec < recCount; rec++) {
String type = "type" + (rec % numberOfTypes);
String fieldName = "field_" + type + "_" + rec;
assertConcreteMappingsOnAll("test", type, fieldName);
}
}
public void testMultiGet() {
String multiGetShardAction = MultiGetAction.NAME + "[shard][s]";
interceptTransportActions(multiGetShardAction);
List<String> indices = new ArrayList<>();
MultiGetRequest multiGetRequest = new MultiGetRequest();
int numDocs = iterations(1, 30);
for (int i = 0; i < numDocs; i++) {
String indexOrAlias = randomIndexOrAlias();
multiGetRequest.add(indexOrAlias, "type", Integer.toString(i));
indices.add(indexOrAlias);
}
internalCluster().clientNodeClient().multiGet(multiGetRequest).actionGet();
clearInterceptedActions();
assertIndicesSubset(indices, multiGetShardAction);
}
@Test // Uses of JMockit API: 4
public void verifyThatInvocationsNeverHappened(
@Mocked List<String> mockTwo, @Mocked List<String> mockThree) {
// Using mocks - only mockedList is invoked:
mockedList.add("one");
// Verify that the two other mocks were never invoked.
new FullVerifications() {
{
// Ordinary verification:
mockedList.add("one");
// Verify that method was never called on a mock:
mockedList.add("two");
times = 0;
}
};
}
@Test
public void testNewGetPaymentsApiMethods() throws Exception {
JbillingAPI api = JbillingAPIFactory.getAPI();
// Create a user with balance $1.00
UserWS user = com.sapienter.jbilling.server.user.WSTest.createUser(true, null, null);
List<PaymentWS> payments = new ArrayList<PaymentWS>();
for (int i = 0; i < 5; i++) {
payments.add(
createPaymentWS(
user.getUserId(), new DateTime().plusMonths(i).toDate(), String.valueOf(i)));
}
// get two latest payments except the latest one.
Integer[] paymentsId = api.getLastPaymentsPage(user.getUserId(), 2, 1);
assertEquals(2, paymentsId.length);
assertEquals("3", api.getPayment(paymentsId[0]).getPaymentNotes());
assertEquals("2", api.getPayment(paymentsId[1]).getPaymentNotes());
// get the payments between next month and four months from now.
Integer[] paymentsId2 =
api.getPaymentsByDate(
user.getUserId(),
new DateTime().plusDays(1).toDate(),
new DateTime().plusMonths(3).plusDays(1).toDate());
assertEquals(3, paymentsId2.length);
assertEquals("3", api.getPayment(paymentsId2[0]).getPaymentNotes());
assertEquals("2", api.getPayment(paymentsId2[1]).getPaymentNotes());
assertEquals("1", api.getPayment(paymentsId2[2]).getPaymentNotes());
// Delete orders
for (PaymentWS payment : payments) {
api.deletePayment(payment.getId());
}
// Delete user
api.deleteUser(user.getUserId());
}
private Integer getOrCreateSuspendedStatus(JbillingAPI api) {
List<AgeingWS> steps = Arrays.asList(api.getAgeingConfiguration(LANGUAGE_ID));
for (AgeingWS step : steps) {
if (step.getSuspended().booleanValue()) {
return step.getStatusId();
}
}
AgeingWS suspendStep = new AgeingWS();
suspendStep.setSuspended(Boolean.TRUE);
suspendStep.setDays(Integer.valueOf(180));
suspendStep.setStatusStr("Ageing Step 180");
suspendStep.setFailedLoginMessage("You are suspended");
suspendStep.setWelcomeMessage("Welcome");
steps.add(suspendStep);
api.saveAgeingConfiguration(steps.toArray(new AgeingWS[steps.size()]), LANGUAGE_ID);
return getOrCreateOrderChangeStatusApply(api);
}
private List<Bucket> getBuckets(CollectPhase collectNode)
throws InterruptedException, java.util.concurrent.ExecutionException {
List<Bucket> results = new ArrayList<>();
for (String nodeName : internalCluster().getNodeNames()) {
ContextPreparer contextPreparer =
internalCluster().getInstance(ContextPreparer.class, nodeName);
JobContextService contextService =
internalCluster().getInstance(JobContextService.class, nodeName);
JobExecutionContext.Builder builder = contextService.newBuilder(collectNode.jobId());
ListenableFuture<Bucket> future =
contextPreparer.prepare(
collectNode.jobId(),
NodeOperation.withDownstream(collectNode, mock(ExecutionPhase.class)),
builder);
assert future != null;
JobExecutionContext context = contextService.createContext(builder);
context.start();
results.add(future.get());
}
return results;
}
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
tasks.add(this);
latch.countDown();
return currentState;
}
@Test
public void testAbortByPattern() throws Exception {
final int iters =
between(1, 3); // we run this more than once to make sure metadata is cleaned up propperly
for (int i = 0; i < iters; i++) {
List<BenchmarkRequest> requests = new ArrayList<>();
List<ActionFuture<BenchmarkResponse>> responses = new ArrayList<>();
SearchRequest searchRequest = prepareBlockingScriptQuery();
final int benches = between(1, 3);
String[] names = new String[benches];
for (int k = 0; k < benches; k++) {
final BenchmarkRequest request =
BenchmarkTestUtil.randomRequest(
client(), indices, numExecutorNodes, competitionSettingsMap, searchRequest);
request.settings().iterations(Integer.MAX_VALUE, true); // massive amount of iterations
names[k] = BENCHMARK_NAME + Integer.toString(k);
request.benchmarkName(names[k]);
requests.add(request);
logger.info(
"--> Submitting benchmark - competitors [{}] iterations [{}]",
request.competitors().size(),
request.settings().iterations());
}
boolean aborted = false;
for (BenchmarkRequest r : requests) {
final ActionFuture<BenchmarkResponse> benchmarkResponse = client().bench(r);
responses.add(benchmarkResponse);
}
try {
waitForQuery.await();
if (benches > 1) {
awaitBusy(
new Predicate<Object>() {
@Override
public boolean apply(java.lang.Object input) {
return client().prepareBenchStatus().get().benchmarkResponses().size() == benches;
}
});
}
final String badPatternA = "*z";
final String badPatternB = "xxx";
final String[] patterns;
switch (getRandom().nextInt(3)) {
case 0:
patterns = new String[] {"*"};
break;
case 1:
patterns = new String[] {BENCHMARK_NAME_WILDCARD, badPatternA, badPatternB};
break;
case 2:
patterns = names;
break;
default:
patterns = new String[] {BENCHMARK_NAME_WILDCARD};
}
final AbortBenchmarkResponse abortResponse = client().prepareAbortBench(patterns).get();
aborted = true;
assertAcked(abortResponse);
// Confirm that there are no active benchmarks in the cluster
final BenchmarkStatusResponse statusResponse =
client().prepareBenchStatus().execute().actionGet();
waitForTestLatch.countDown(); // let the queries go - we already aborted and got the status
assertThat(statusResponse.totalActiveBenchmarks(), equalTo(0));
// Confirm that benchmark was indeed aborted
for (ActionFuture<BenchmarkResponse> r : responses) {
assertThat(r.get().state(), is(BenchmarkResponse.State.ABORTED));
}
} finally {
if (waitForTestLatch.getCount() == 1) {
waitForTestLatch.countDown();
}
if (!aborted) {
client().prepareAbortBench(BENCHMARK_NAME).get();
}
assertThat(waitForTestLatch.getCount(), is(0l));
}
}
}
@Test
public void testRecoveryDiff() throws IOException, InterruptedException {
int numDocs = 2 + random().nextInt(100);
List<Document> docs = new ArrayList<>();
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
doc.add(
new StringField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new TextField(
"body",
TestUtil.randomRealisticUnicodeString(random()),
random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
doc.add(
new SortedDocValuesField(
"dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
docs.add(doc);
}
long seed = random().nextLong();
Store.MetadataSnapshot first;
{
Random random = new Random(seed);
IndexWriterConfig iwc =
new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setUseCompoundFile(random.nextBoolean());
iwc.setMaxThreadStates(1);
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random);
Store store =
new Store(
shardId,
ImmutableSettings.EMPTY,
directoryService,
randomDistributor(random, directoryService),
new DummyShardLock(shardId));
IndexWriter writer = new IndexWriter(store.directory(), iwc);
final boolean lotsOfSegments = rarely(random);
for (Document d : docs) {
writer.addDocument(d);
if (lotsOfSegments && random.nextBoolean()) {
writer.commit();
} else if (rarely(random)) {
writer.commit();
}
}
writer.commit();
writer.close();
first = store.getMetadata();
assertDeleteContent(store, directoryService);
store.close();
}
long time = new Date().getTime();
while (time == new Date().getTime()) {
Thread.sleep(10); // bump the time
}
Store.MetadataSnapshot second;
Store store;
{
Random random = new Random(seed);
IndexWriterConfig iwc =
new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setUseCompoundFile(random.nextBoolean());
iwc.setMaxThreadStates(1);
final ShardId shardId = new ShardId(new Index("index"), 1);
DirectoryService directoryService = new LuceneManagedDirectoryService(random);
store =
new Store(
shardId,
ImmutableSettings.EMPTY,
directoryService,
randomDistributor(random, directoryService),
new DummyShardLock(shardId));
IndexWriter writer = new IndexWriter(store.directory(), iwc);
final boolean lotsOfSegments = rarely(random);
for (Document d : docs) {
writer.addDocument(d);
if (lotsOfSegments && random.nextBoolean()) {
writer.commit();
} else if (rarely(random)) {
writer.commit();
}
}
writer.commit();
writer.close();
second = store.getMetadata();
}
Store.RecoveryDiff diff = first.recoveryDiff(second);
assertThat(first.size(), equalTo(second.size()));
for (StoreFileMetaData md : first) {
assertThat(second.get(md.name()), notNullValue());
// si files are different - containing timestamps etc
assertThat(second.get(md.name()).isSame(md), equalTo(false));
}
assertThat(diff.different.size(), equalTo(first.size()));
assertThat(
diff.identical.size(),
equalTo(0)); // in lucene 5 nothing is identical - we use random ids in file headers
assertThat(diff.missing, empty());
// check the self diff
Store.RecoveryDiff selfDiff = first.recoveryDiff(first);
assertThat(selfDiff.identical.size(), equalTo(first.size()));
assertThat(selfDiff.different, empty());
assertThat(selfDiff.missing, empty());
// lets add some deletes
Random random = new Random(seed);
IndexWriterConfig iwc =
new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setUseCompoundFile(random.nextBoolean());
iwc.setMaxThreadStates(1);
iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
IndexWriter writer = new IndexWriter(store.directory(), iwc);
writer.deleteDocuments(new Term("id", Integer.toString(random().nextInt(numDocs))));
writer.commit();
writer.close();
Store.MetadataSnapshot metadata = store.getMetadata();
StoreFileMetaData delFile = null;
for (StoreFileMetaData md : metadata) {
if (md.name().endsWith(".liv")) {
delFile = md;
break;
}
}
Store.RecoveryDiff afterDeleteDiff = metadata.recoveryDiff(second);
if (delFile != null) {
assertThat(
afterDeleteDiff.identical.size(), equalTo(metadata.size() - 2)); // segments_N + del file
assertThat(afterDeleteDiff.different.size(), equalTo(0));
assertThat(afterDeleteDiff.missing.size(), equalTo(2));
} else {
// an entire segment must be missing (single doc segment got dropped)
assertThat(afterDeleteDiff.identical.size(), greaterThan(0));
assertThat(afterDeleteDiff.different.size(), equalTo(0));
assertThat(afterDeleteDiff.missing.size(), equalTo(1)); // the commit file is different
}
// check the self diff
selfDiff = metadata.recoveryDiff(metadata);
assertThat(selfDiff.identical.size(), equalTo(metadata.size()));
assertThat(selfDiff.different, empty());
assertThat(selfDiff.missing, empty());
// add a new commit
iwc = new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setUseCompoundFile(
true); // force CFS - easier to test here since we know it will add 3 files
iwc.setMaxThreadStates(1);
iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
writer = new IndexWriter(store.directory(), iwc);
writer.addDocument(docs.get(0));
writer.close();
Store.MetadataSnapshot newCommitMetaData = store.getMetadata();
Store.RecoveryDiff newCommitDiff = newCommitMetaData.recoveryDiff(metadata);
if (delFile != null) {
assertThat(
newCommitDiff.identical.size(),
equalTo(
newCommitMetaData.size()
- 5)); // segments_N, del file, cfs, cfe, si for the new segment
assertThat(newCommitDiff.different.size(), equalTo(1)); // the del file must be different
assertThat(newCommitDiff.different.get(0).name(), endsWith(".liv"));
assertThat(
newCommitDiff.missing.size(), equalTo(4)); // segments_N,cfs, cfe, si for the new segment
} else {
assertThat(
newCommitDiff.identical.size(),
equalTo(newCommitMetaData.size() - 4)); // segments_N, cfs, cfe, si for the new segment
assertThat(newCommitDiff.different.size(), equalTo(0));
assertThat(
newCommitDiff.missing.size(),
equalTo(
4)); // an entire segment must be missing (single doc segment got dropped) plus the
// commit is different
}
store.deleteContent();
IOUtils.close(store);
}
@Test
@Slow
public void stressUpdateDeleteConcurrency() throws Exception {
// We create an index with merging disabled so that deletes don't get merged away
client()
.admin()
.indices()
.prepareCreate("test")
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.field("store", "yes")
.endObject()
.endObject()
.endObject())
.setSettings(
ImmutableSettings.builder()
.put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class))
.execute()
.actionGet();
ensureGreen();
final int numberOfThreads = scaledRandomIntBetween(5, 10);
final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
final int numberOfUpdatesPerId = scaledRandomIntBetween(100, 200);
final int retryOnConflict = randomIntBetween(0, 1);
final CountDownLatch latch = new CountDownLatch(numberOfThreads);
final CountDownLatch startLatch = new CountDownLatch(1);
final List<Throwable> failures = new CopyOnWriteArrayList<>();
final class UpdateThread extends Thread {
final Map<Integer, Integer> failedMap = new HashMap<>();
final int numberOfIds;
final int updatesPerId;
final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);
private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);
public UpdateThread(int numberOfIds, int updatesPerId) {
this.numberOfIds = numberOfIds;
this.updatesPerId = updatesPerId;
}
final class UpdateListener implements ActionListener<UpdateResponse> {
int id;
public UpdateListener(int id) {
this.id = id;
}
@Override
public void onResponse(UpdateResponse updateResponse) {
updateRequestsOutstanding.release(1);
}
@Override
public void onFailure(Throwable e) {
synchronized (failedMap) {
incrementMapValue(id, failedMap);
}
updateRequestsOutstanding.release(1);
}
}
final class DeleteListener implements ActionListener<DeleteResponse> {
int id;
public DeleteListener(int id) {
this.id = id;
}
@Override
public void onResponse(DeleteResponse deleteResponse) {
deleteRequestsOutstanding.release(1);
}
@Override
public void onFailure(Throwable e) {
synchronized (failedMap) {
incrementMapValue(id, failedMap);
}
deleteRequestsOutstanding.release(1);
}
}
@Override
public void run() {
try {
startLatch.await();
for (int j = 0; j < numberOfIds; j++) {
for (int k = 0; k < numberOfUpdatesPerId; ++k) {
updateRequestsOutstanding.acquire();
UpdateRequest ur =
client()
.prepareUpdate("test", "type1", Integer.toString(j))
.setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
.setRetryOnConflict(retryOnConflict)
.setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
.setListenerThreaded(false)
.request();
client().update(ur, new UpdateListener(j));
deleteRequestsOutstanding.acquire();
DeleteRequest dr =
client()
.prepareDelete("test", "type1", Integer.toString(j))
.setListenerThreaded(false)
.setOperationThreaded(false)
.request();
client().delete(dr, new DeleteListener(j));
}
}
} catch (Throwable e) {
logger.error("Something went wrong", e);
failures.add(e);
} finally {
try {
waitForOutstandingRequests(
TimeValue.timeValueSeconds(60),
updateRequestsOutstanding,
maxUpdateRequests,
"Update");
waitForOutstandingRequests(
TimeValue.timeValueSeconds(60),
deleteRequestsOutstanding,
maxDeleteRequests,
"Delete");
} catch (ElasticsearchTimeoutException ete) {
failures.add(ete);
}
latch.countDown();
}
}
private void incrementMapValue(int j, Map<Integer, Integer> map) {
if (!map.containsKey(j)) {
map.put(j, 0);
}
map.put(j, map.get(j) + 1);
}
private void waitForOutstandingRequests(
TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
long start = System.currentTimeMillis();
do {
long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
logger.info(
"[{}] going to try and aquire [{}] in [{}]ms [{}] available to aquire right now",
name,
maxRequests,
msRemaining,
requestsOutstanding.availablePermits());
try {
requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
return;
} catch (InterruptedException ie) {
// Just keep swimming
}
} while ((System.currentTimeMillis() - start) < timeOut.getMillis());
throw new ElasticsearchTimeoutException(
"Requests were still outstanding after the timeout ["
+ timeOut
+ "] for type ["
+ name
+ "]");
}
}
final List<UpdateThread> threads = new ArrayList<>();
for (int i = 0; i < numberOfThreads; i++) {
UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId);
ut.start();
threads.add(ut);
}
startLatch.countDown();
latch.await();
for (UpdateThread ut : threads) {
ut.join(); // Threads should have finished because of the latch.await
}
// If are no errors every request recieved a response otherwise the test would have timedout
// aquiring the request outstanding semaphores.
for (Throwable throwable : failures) {
logger.info("Captured failure on concurrent update:", throwable);
}
assertThat(failures.size(), equalTo(0));
// Upsert all the ids one last time to make sure they are available at get time
// This means that we add 1 to the expected versions and attempts
// All the previous operations should be complete or failed at this point
for (int i = 0; i < numberOfIdsPerThread; ++i) {
UpdateResponse ur =
client()
.prepareUpdate("test", "type1", Integer.toString(i))
.setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
.setRetryOnConflict(Integer.MAX_VALUE)
.setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
.execute()
.actionGet();
}
refresh();
for (int i = 0; i < numberOfIdsPerThread; ++i) {
int totalFailures = 0;
GetResponse response =
client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
if (response.isExists()) {
assertThat(response.getId(), equalTo(Integer.toString(i)));
int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
for (UpdateThread ut : threads) {
if (ut.failedMap.containsKey(i)) {
totalFailures += ut.failedMap.get(i);
}
}
expectedVersion -= totalFailures;
logger.error(
"Actual version [{}] Expected version [{}] Total failures [{}]",
response.getVersion(),
expectedVersion,
totalFailures);
assertThat(response.getVersion(), equalTo((long) expectedVersion));
assertThat(
response.getVersion() + totalFailures,
equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
}
}
}