public void testBulk() {
    String[] bulkShardActions = new String[] {BulkAction.NAME + "[s]", BulkAction.NAME + "[s][r]"};
    interceptTransportActions(bulkShardActions);

    List<String> indices = new ArrayList<>();
    BulkRequest bulkRequest = new BulkRequest();
    int numIndexRequests = iterations(1, 10);
    for (int i = 0; i < numIndexRequests; i++) {
      String indexOrAlias = randomIndexOrAlias();
      bulkRequest.add(new IndexRequest(indexOrAlias, "type", "id").source("field", "value"));
      indices.add(indexOrAlias);
    }
    int numDeleteRequests = iterations(1, 10);
    for (int i = 0; i < numDeleteRequests; i++) {
      String indexOrAlias = randomIndexOrAlias();
      bulkRequest.add(new DeleteRequest(indexOrAlias, "type", "id"));
      indices.add(indexOrAlias);
    }
    int numUpdateRequests = iterations(1, 10);
    for (int i = 0; i < numUpdateRequests; i++) {
      String indexOrAlias = randomIndexOrAlias();
      bulkRequest.add(new UpdateRequest(indexOrAlias, "type", "id").doc("field1", "value1"));
      indices.add(indexOrAlias);
    }

    internalCluster().clientNodeClient().bulk(bulkRequest).actionGet();

    clearInterceptedActions();
    assertIndicesSubset(indices, bulkShardActions);
  }
  @Test
  public void testBenchmarkWithErrors() {
    List<SearchRequest> reqList = new ArrayList<>();
    int numQueries = scaledRandomIntBetween(20, 100);
    int numErrors = scaledRandomIntBetween(1, numQueries);
    final boolean containsFatal = randomBoolean();
    if (containsFatal) {
      ScriptScoreFunctionBuilder scriptFunction =
          scriptFunction("DOES NOT COMPILE - fails on any shard");
      SearchRequest searchRequest =
          searchRequest()
              .source(
                  searchSource()
                      .query(functionScoreQuery(FilterBuilders.matchAllFilter(), scriptFunction)));
      reqList.add(searchRequest);
    }
    for (int i = 0; reqList.size() < numErrors; i++) {
      ScriptScoreFunctionBuilder scriptFunction = scriptFunction("throw new RuntimeException();");
      SearchRequest searchRequest =
          searchRequest()
              .source(
                  searchSource()
                      .query(functionScoreQuery(FilterBuilders.matchAllFilter(), scriptFunction)));
      reqList.add(searchRequest);
    }
    logger.info("--> run with [{}] errors ", numErrors);
    for (int i = 0; reqList.size() < numQueries; i++) {

      reqList.add(BenchmarkTestUtil.randomSearch(client(), indices));
    }
    Collections.shuffle(reqList, getRandom());

    final BenchmarkRequest request =
        BenchmarkTestUtil.randomRequest(
            client(),
            indices,
            numExecutorNodes,
            competitionSettingsMap,
            reqList.toArray(new SearchRequest[0]));
    logger.info(
        "--> Submitting benchmark - competitors [{}] iterations [{}]",
        request.competitors().size(),
        request.settings().iterations());
    final BenchmarkResponse response = client().bench(request).actionGet();

    assertThat(response, notNullValue());
    if (response.hasErrors() || containsFatal) {
      assertThat(response.state(), equalTo(BenchmarkResponse.State.FAILED));
    } else {
      assertThat(response.state(), equalTo(BenchmarkResponse.State.COMPLETE));
      for (CompetitionResult result : response.competitionResults().values()) {
        assertThat(result.nodeResults().size(), equalTo(numExecutorNodes));
        validateCompetitionResult(
            result, competitionSettingsMap.get(result.competitionName()), true);
      }
    }
    assertThat(response.benchmarkName(), equalTo(BENCHMARK_NAME));
  }
  @Test // Uses of JMockit API: 1
  public void verifyAllInvocationsInOrder() {
    mockedList.add("one");
    mockedList.size();
    mockedList.add("two");

    new FullVerificationsInOrder() {
      {
        mockedList.add("one");
        mockedList.size();
        mockedList.add("two");
      }
    };
  }
  @Test // Uses of JMockit API: 3
  public void verifyInOrder(
      @Mocked final List<String> firstMock, @Mocked final List<String> secondMock) {
    // Using mocks:
    firstMock.add("was called first");
    secondMock.add("was called second");

    new VerificationsInOrder() {
      {
        // Verifies that firstMock was called before secondMock:
        firstMock.add("was called first");
        secondMock.add("was called second");
      }
    };
  }
 @Override
 public void messageReceived(T request, TransportChannel channel) throws Exception {
   synchronized (InterceptingTransportService.this) {
     if (actions.contains(action)) {
       List<TransportRequest> requestList = requests.get(action);
       if (requestList == null) {
         requestList = new ArrayList<>();
         requestList.add(request);
         requests.put(action, requestList);
       } else {
         requestList.add(request);
       }
     }
   }
   requestHandler.messageReceived(request, channel);
 }
    public void beforeExecute(Task task) {
      assertTrue(planned.contains(task));

      String taskPath = path(task);
      if (taskPath.startsWith(":buildSrc:")) {
        return;
      }

      executedTasks.add(taskPath);
    }
 @Before
 public void setup() {
   int numIndices = iterations(1, 5);
   for (int i = 0; i < numIndices; i++) {
     indices.add("test" + i);
   }
   for (String index : indices) {
     assertAcked(prepareCreate(index).addAlias(new Alias(index + "-alias")));
   }
   ensureGreen();
 }
  @Test(expected = UnexpectedInvocation.class) // Uses of JMockit API: 2
  public void verifyThatInvocationsNeverHappenedWhenTheyDid(@Mocked List<String> mockTwo) {
    mockedList.add("one");
    mockTwo.size();

    new FullVerifications() {
      {
        mockedList.add("one");
      }
    };
  }
 private void extractCauses(Throwable failure, List<Throwable> causes) {
   if (failure instanceof MultipleBuildFailures) {
     MultipleBuildFailures exception = (MultipleBuildFailures) failure;
     for (Throwable componentFailure : exception.getCauses()) {
       extractCauses(componentFailure, causes);
     }
   } else if (failure instanceof LocationAwareException) {
     causes.addAll(((LocationAwareException) failure).getReportableCauses());
   } else {
     causes.add(failure.getCause());
   }
 }
  @Test(expected = MissingInvocation.class) // Uses of JMockit API: 1
  public void verifyAllInvocationsInOrderWithOutOfOrderVerifications() {
    mockedList.add("one");
    mockedList.add("two");

    new FullVerificationsInOrder() {
      {
        mockedList.add("two");
        mockedList.add("one");
      }
    };
  }
  private static void duelFieldDataGeoPoint(
      Random random,
      AtomicReaderContext context,
      IndexGeoPointFieldData left,
      IndexGeoPointFieldData right,
      Distance precision)
      throws Exception {
    AtomicGeoPointFieldData leftData =
        random.nextBoolean() ? left.load(context) : left.loadDirect(context);
    AtomicGeoPointFieldData rightData =
        random.nextBoolean() ? right.load(context) : right.loadDirect(context);

    int numDocs = context.reader().maxDoc();
    MultiGeoPointValues leftValues = leftData.getGeoPointValues();
    MultiGeoPointValues rightValues = rightData.getGeoPointValues();
    for (int i = 0; i < numDocs; ++i) {
      leftValues.setDocument(i);
      final int numValues = leftValues.count();
      rightValues.setDocument(i);
      ;
      assertEquals(numValues, rightValues.count());
      List<GeoPoint> leftPoints = Lists.newArrayList();
      List<GeoPoint> rightPoints = Lists.newArrayList();
      for (int j = 0; j < numValues; ++j) {
        GeoPoint l = leftValues.valueAt(j);
        leftPoints.add(new GeoPoint(l.getLat(), l.getLon()));
        GeoPoint r = rightValues.valueAt(j);
        rightPoints.add(new GeoPoint(r.getLat(), r.getLon()));
      }
      for (GeoPoint l : leftPoints) {
        assertTrue(
            "Couldn't find " + l + " among " + rightPoints, contains(l, rightPoints, precision));
      }
      for (GeoPoint r : rightPoints) {
        assertTrue(
            "Couldn't find " + r + " among " + leftPoints, contains(r, leftPoints, precision));
      }
    }
  }
  @Test // Uses of JMockit API: 8
  public void verifyNumberOfInvocations() {
    // Using mock:
    mockedList.add("once");

    mockedList.add("twice");
    mockedList.add("twice");

    mockedList.add("three times");
    mockedList.add("three times");
    mockedList.add("three times");

    new Verifications() {
      {
        // Following two verifications work exactly the same:
        mockedList.add("once"); // minTimes == 1 is the default
        mockedList.add("once");
        times = 1;

        // Verifies exact number of invocations:
        mockedList.add("twice");
        times = 2;
        mockedList.add("three times");
        times = 3;

        // Verifies no invocations occurred:
        mockedList.add("never happened");
        times = 0;

        // Verifies min/max number of invocations:
        mockedList.add("three times");
        minTimes = 1;
        mockedList.add("three times");
        minTimes = 2;
        mockedList.add("three times");
        maxTimes = 5;
      }
    };
  }
  @Test(expected = UnexpectedInvocation.class) // Uses of JMockit API: 1
  public void verifyAllInvocationsInOrderWhenMoreOfThemHappen() {
    mockedList.add("one");
    mockedList.add("two");
    mockedList.size();

    new FullVerificationsInOrder() {
      {
        mockedList.add("one");
        mockedList.add("two");
      }
    };
  }
  @Test // Uses of JMockit API: 1
  public void verifyAllInvocations() {
    mockedList.add("one");
    mockedList.add("two");

    // Verify all invocations to mockedList.
    new FullVerifications() {
      {
        // Verifies first invocation:
        mockedList.add("one");

        // Verifies second (and last) invocation:
        mockedList.add("two");
      }
    };
  }
  @Test
  public void dynamicUpdates() throws Exception {
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(
            settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0))
        .execute()
        .actionGet();
    client()
        .admin()
        .cluster()
        .prepareHealth()
        .setWaitForEvents(Priority.LANGUID)
        .setWaitForGreenStatus()
        .execute()
        .actionGet();

    int recCount = randomIntBetween(200, 600);
    int numberOfTypes = randomIntBetween(1, 5);
    List<IndexRequestBuilder> indexRequests = Lists.newArrayList();
    for (int rec = 0; rec < recCount; rec++) {
      String type = "type" + (rec % numberOfTypes);
      String fieldName = "field_" + type + "_" + rec;
      indexRequests.add(
          client()
              .prepareIndex("test", type, Integer.toString(rec))
              .setSource(fieldName, "some_value"));
    }
    indexRandom(true, indexRequests);

    logger.info("checking all the documents are there");
    RefreshResponse refreshResponse =
        client().admin().indices().prepareRefresh().execute().actionGet();
    assertThat(refreshResponse.getFailedShards(), equalTo(0));
    CountResponse response = client().prepareCount("test").execute().actionGet();
    assertThat(response.getCount(), equalTo((long) recCount));

    logger.info("checking all the fields are in the mappings");

    for (int rec = 0; rec < recCount; rec++) {
      String type = "type" + (rec % numberOfTypes);
      String fieldName = "field_" + type + "_" + rec;
      assertConcreteMappingsOnAll("test", type, fieldName);
    }
  }
  public void testMultiGet() {
    String multiGetShardAction = MultiGetAction.NAME + "[shard][s]";
    interceptTransportActions(multiGetShardAction);

    List<String> indices = new ArrayList<>();
    MultiGetRequest multiGetRequest = new MultiGetRequest();
    int numDocs = iterations(1, 30);
    for (int i = 0; i < numDocs; i++) {
      String indexOrAlias = randomIndexOrAlias();
      multiGetRequest.add(indexOrAlias, "type", Integer.toString(i));
      indices.add(indexOrAlias);
    }
    internalCluster().clientNodeClient().multiGet(multiGetRequest).actionGet();

    clearInterceptedActions();
    assertIndicesSubset(indices, multiGetShardAction);
  }
  @Test // Uses of JMockit API: 4
  public void verifyThatInvocationsNeverHappened(
      @Mocked List<String> mockTwo, @Mocked List<String> mockThree) {
    // Using mocks - only mockedList is invoked:
    mockedList.add("one");

    // Verify that the two other mocks were never invoked.
    new FullVerifications() {
      {
        // Ordinary verification:
        mockedList.add("one");

        // Verify that method was never called on a mock:
        mockedList.add("two");
        times = 0;
      }
    };
  }
Esempio n. 18
0
  @Test
  public void testNewGetPaymentsApiMethods() throws Exception {
    JbillingAPI api = JbillingAPIFactory.getAPI();

    // Create a user with balance $1.00
    UserWS user = com.sapienter.jbilling.server.user.WSTest.createUser(true, null, null);

    List<PaymentWS> payments = new ArrayList<PaymentWS>();

    for (int i = 0; i < 5; i++) {
      payments.add(
          createPaymentWS(
              user.getUserId(), new DateTime().plusMonths(i).toDate(), String.valueOf(i)));
    }

    // get two latest payments except the latest one.
    Integer[] paymentsId = api.getLastPaymentsPage(user.getUserId(), 2, 1);

    assertEquals(2, paymentsId.length);

    assertEquals("3", api.getPayment(paymentsId[0]).getPaymentNotes());
    assertEquals("2", api.getPayment(paymentsId[1]).getPaymentNotes());

    // get the payments between next month and four months from now.
    Integer[] paymentsId2 =
        api.getPaymentsByDate(
            user.getUserId(),
            new DateTime().plusDays(1).toDate(),
            new DateTime().plusMonths(3).plusDays(1).toDate());

    assertEquals(3, paymentsId2.length);

    assertEquals("3", api.getPayment(paymentsId2[0]).getPaymentNotes());
    assertEquals("2", api.getPayment(paymentsId2[1]).getPaymentNotes());
    assertEquals("1", api.getPayment(paymentsId2[2]).getPaymentNotes());

    // Delete orders
    for (PaymentWS payment : payments) {
      api.deletePayment(payment.getId());
    }
    // Delete user
    api.deleteUser(user.getUserId());
  }
Esempio n. 19
0
  private Integer getOrCreateSuspendedStatus(JbillingAPI api) {
    List<AgeingWS> steps = Arrays.asList(api.getAgeingConfiguration(LANGUAGE_ID));

    for (AgeingWS step : steps) {
      if (step.getSuspended().booleanValue()) {
        return step.getStatusId();
      }
    }

    AgeingWS suspendStep = new AgeingWS();
    suspendStep.setSuspended(Boolean.TRUE);
    suspendStep.setDays(Integer.valueOf(180));
    suspendStep.setStatusStr("Ageing Step 180");
    suspendStep.setFailedLoginMessage("You are suspended");
    suspendStep.setWelcomeMessage("Welcome");
    steps.add(suspendStep);
    api.saveAgeingConfiguration(steps.toArray(new AgeingWS[steps.size()]), LANGUAGE_ID);
    return getOrCreateOrderChangeStatusApply(api);
  }
  private List<Bucket> getBuckets(CollectPhase collectNode)
      throws InterruptedException, java.util.concurrent.ExecutionException {
    List<Bucket> results = new ArrayList<>();
    for (String nodeName : internalCluster().getNodeNames()) {
      ContextPreparer contextPreparer =
          internalCluster().getInstance(ContextPreparer.class, nodeName);
      JobContextService contextService =
          internalCluster().getInstance(JobContextService.class, nodeName);

      JobExecutionContext.Builder builder = contextService.newBuilder(collectNode.jobId());
      ListenableFuture<Bucket> future =
          contextPreparer.prepare(
              collectNode.jobId(),
              NodeOperation.withDownstream(collectNode, mock(ExecutionPhase.class)),
              builder);
      assert future != null;

      JobExecutionContext context = contextService.createContext(builder);
      context.start();
      results.add(future.get());
    }
    return results;
  }
 @Override
 public ClusterState execute(ClusterState currentState) throws Exception {
   tasks.add(this);
   latch.countDown();
   return currentState;
 }
  @Test
  public void testAbortByPattern() throws Exception {
    final int iters =
        between(1, 3); // we run this more than once to make sure metadata is cleaned up propperly
    for (int i = 0; i < iters; i++) {
      List<BenchmarkRequest> requests = new ArrayList<>();
      List<ActionFuture<BenchmarkResponse>> responses = new ArrayList<>();

      SearchRequest searchRequest = prepareBlockingScriptQuery();
      final int benches = between(1, 3);
      String[] names = new String[benches];
      for (int k = 0; k < benches; k++) {
        final BenchmarkRequest request =
            BenchmarkTestUtil.randomRequest(
                client(), indices, numExecutorNodes, competitionSettingsMap, searchRequest);
        request.settings().iterations(Integer.MAX_VALUE, true); // massive amount of iterations
        names[k] = BENCHMARK_NAME + Integer.toString(k);
        request.benchmarkName(names[k]);
        requests.add(request);
        logger.info(
            "--> Submitting benchmark - competitors [{}] iterations [{}]",
            request.competitors().size(),
            request.settings().iterations());
      }

      boolean aborted = false;
      for (BenchmarkRequest r : requests) {
        final ActionFuture<BenchmarkResponse> benchmarkResponse = client().bench(r);
        responses.add(benchmarkResponse);
      }
      try {
        waitForQuery.await();
        if (benches > 1) {
          awaitBusy(
              new Predicate<Object>() {
                @Override
                public boolean apply(java.lang.Object input) {
                  return client().prepareBenchStatus().get().benchmarkResponses().size() == benches;
                }
              });
        }
        final String badPatternA = "*z";
        final String badPatternB = "xxx";
        final String[] patterns;
        switch (getRandom().nextInt(3)) {
          case 0:
            patterns = new String[] {"*"};
            break;
          case 1:
            patterns = new String[] {BENCHMARK_NAME_WILDCARD, badPatternA, badPatternB};
            break;
          case 2:
            patterns = names;
            break;
          default:
            patterns = new String[] {BENCHMARK_NAME_WILDCARD};
        }
        final AbortBenchmarkResponse abortResponse = client().prepareAbortBench(patterns).get();
        aborted = true;
        assertAcked(abortResponse);

        // Confirm that there are no active benchmarks in the cluster
        final BenchmarkStatusResponse statusResponse =
            client().prepareBenchStatus().execute().actionGet();
        waitForTestLatch.countDown(); // let the queries go - we already aborted and got the status
        assertThat(statusResponse.totalActiveBenchmarks(), equalTo(0));

        // Confirm that benchmark was indeed aborted
        for (ActionFuture<BenchmarkResponse> r : responses) {
          assertThat(r.get().state(), is(BenchmarkResponse.State.ABORTED));
        }
      } finally {
        if (waitForTestLatch.getCount() == 1) {
          waitForTestLatch.countDown();
        }
        if (!aborted) {
          client().prepareAbortBench(BENCHMARK_NAME).get();
        }
        assertThat(waitForTestLatch.getCount(), is(0l));
      }
    }
  }
Esempio n. 23
0
  @Test
  public void testRecoveryDiff() throws IOException, InterruptedException {
    int numDocs = 2 + random().nextInt(100);
    List<Document> docs = new ArrayList<>();
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      doc.add(
          new StringField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
      doc.add(
          new TextField(
              "body",
              TestUtil.randomRealisticUnicodeString(random()),
              random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
      doc.add(
          new SortedDocValuesField(
              "dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
      docs.add(doc);
    }
    long seed = random().nextLong();
    Store.MetadataSnapshot first;
    {
      Random random = new Random(seed);
      IndexWriterConfig iwc =
          new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
      iwc.setMergePolicy(NoMergePolicy.INSTANCE);
      iwc.setUseCompoundFile(random.nextBoolean());
      iwc.setMaxThreadStates(1);
      final ShardId shardId = new ShardId(new Index("index"), 1);
      DirectoryService directoryService = new LuceneManagedDirectoryService(random);
      Store store =
          new Store(
              shardId,
              ImmutableSettings.EMPTY,
              directoryService,
              randomDistributor(random, directoryService),
              new DummyShardLock(shardId));
      IndexWriter writer = new IndexWriter(store.directory(), iwc);
      final boolean lotsOfSegments = rarely(random);
      for (Document d : docs) {
        writer.addDocument(d);
        if (lotsOfSegments && random.nextBoolean()) {
          writer.commit();
        } else if (rarely(random)) {
          writer.commit();
        }
      }
      writer.commit();
      writer.close();
      first = store.getMetadata();
      assertDeleteContent(store, directoryService);
      store.close();
    }
    long time = new Date().getTime();
    while (time == new Date().getTime()) {
      Thread.sleep(10); // bump the time
    }
    Store.MetadataSnapshot second;
    Store store;
    {
      Random random = new Random(seed);
      IndexWriterConfig iwc =
          new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
      iwc.setMergePolicy(NoMergePolicy.INSTANCE);
      iwc.setUseCompoundFile(random.nextBoolean());
      iwc.setMaxThreadStates(1);
      final ShardId shardId = new ShardId(new Index("index"), 1);
      DirectoryService directoryService = new LuceneManagedDirectoryService(random);
      store =
          new Store(
              shardId,
              ImmutableSettings.EMPTY,
              directoryService,
              randomDistributor(random, directoryService),
              new DummyShardLock(shardId));
      IndexWriter writer = new IndexWriter(store.directory(), iwc);
      final boolean lotsOfSegments = rarely(random);
      for (Document d : docs) {
        writer.addDocument(d);
        if (lotsOfSegments && random.nextBoolean()) {
          writer.commit();
        } else if (rarely(random)) {
          writer.commit();
        }
      }
      writer.commit();
      writer.close();
      second = store.getMetadata();
    }
    Store.RecoveryDiff diff = first.recoveryDiff(second);
    assertThat(first.size(), equalTo(second.size()));
    for (StoreFileMetaData md : first) {
      assertThat(second.get(md.name()), notNullValue());
      // si files are different - containing timestamps etc
      assertThat(second.get(md.name()).isSame(md), equalTo(false));
    }
    assertThat(diff.different.size(), equalTo(first.size()));
    assertThat(
        diff.identical.size(),
        equalTo(0)); // in lucene 5 nothing is identical - we use random ids in file headers
    assertThat(diff.missing, empty());

    // check the self diff
    Store.RecoveryDiff selfDiff = first.recoveryDiff(first);
    assertThat(selfDiff.identical.size(), equalTo(first.size()));
    assertThat(selfDiff.different, empty());
    assertThat(selfDiff.missing, empty());

    // lets add some deletes
    Random random = new Random(seed);
    IndexWriterConfig iwc =
        new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setUseCompoundFile(random.nextBoolean());
    iwc.setMaxThreadStates(1);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
    IndexWriter writer = new IndexWriter(store.directory(), iwc);
    writer.deleteDocuments(new Term("id", Integer.toString(random().nextInt(numDocs))));
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata();
    StoreFileMetaData delFile = null;
    for (StoreFileMetaData md : metadata) {
      if (md.name().endsWith(".liv")) {
        delFile = md;
        break;
      }
    }
    Store.RecoveryDiff afterDeleteDiff = metadata.recoveryDiff(second);
    if (delFile != null) {
      assertThat(
          afterDeleteDiff.identical.size(), equalTo(metadata.size() - 2)); // segments_N + del file
      assertThat(afterDeleteDiff.different.size(), equalTo(0));
      assertThat(afterDeleteDiff.missing.size(), equalTo(2));
    } else {
      // an entire segment must be missing (single doc segment got dropped)
      assertThat(afterDeleteDiff.identical.size(), greaterThan(0));
      assertThat(afterDeleteDiff.different.size(), equalTo(0));
      assertThat(afterDeleteDiff.missing.size(), equalTo(1)); // the commit file is different
    }

    // check the self diff
    selfDiff = metadata.recoveryDiff(metadata);
    assertThat(selfDiff.identical.size(), equalTo(metadata.size()));
    assertThat(selfDiff.different, empty());
    assertThat(selfDiff.missing, empty());

    // add a new commit
    iwc = new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setUseCompoundFile(
        true); // force CFS - easier to test here since we know it will add 3 files
    iwc.setMaxThreadStates(1);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
    writer = new IndexWriter(store.directory(), iwc);
    writer.addDocument(docs.get(0));
    writer.close();

    Store.MetadataSnapshot newCommitMetaData = store.getMetadata();
    Store.RecoveryDiff newCommitDiff = newCommitMetaData.recoveryDiff(metadata);
    if (delFile != null) {
      assertThat(
          newCommitDiff.identical.size(),
          equalTo(
              newCommitMetaData.size()
                  - 5)); // segments_N, del file, cfs, cfe, si for the new segment
      assertThat(newCommitDiff.different.size(), equalTo(1)); // the del file must be different
      assertThat(newCommitDiff.different.get(0).name(), endsWith(".liv"));
      assertThat(
          newCommitDiff.missing.size(), equalTo(4)); // segments_N,cfs, cfe, si for the new segment
    } else {
      assertThat(
          newCommitDiff.identical.size(),
          equalTo(newCommitMetaData.size() - 4)); // segments_N, cfs, cfe, si for the new segment
      assertThat(newCommitDiff.different.size(), equalTo(0));
      assertThat(
          newCommitDiff.missing.size(),
          equalTo(
              4)); // an entire segment must be missing (single doc segment got dropped)  plus the
                   // commit is different
    }

    store.deleteContent();
    IOUtils.close(store);
  }
Esempio n. 24
0
  @Test
  @Slow
  public void stressUpdateDeleteConcurrency() throws Exception {
    // We create an index with merging disabled so that deletes don't get merged away
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .addMapping(
            "type1",
            XContentFactory.jsonBuilder()
                .startObject()
                .startObject("type1")
                .startObject("_timestamp")
                .field("enabled", true)
                .field("store", "yes")
                .endObject()
                .startObject("_ttl")
                .field("enabled", true)
                .field("store", "yes")
                .endObject()
                .endObject()
                .endObject())
        .setSettings(
            ImmutableSettings.builder()
                .put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class))
        .execute()
        .actionGet();
    ensureGreen();

    final int numberOfThreads = scaledRandomIntBetween(5, 10);
    final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
    final int numberOfUpdatesPerId = scaledRandomIntBetween(100, 200);
    final int retryOnConflict = randomIntBetween(0, 1);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();

    final class UpdateThread extends Thread {
      final Map<Integer, Integer> failedMap = new HashMap<>();
      final int numberOfIds;
      final int updatesPerId;
      final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
      final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
      private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);
      private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);

      public UpdateThread(int numberOfIds, int updatesPerId) {
        this.numberOfIds = numberOfIds;
        this.updatesPerId = updatesPerId;
      }

      final class UpdateListener implements ActionListener<UpdateResponse> {
        int id;

        public UpdateListener(int id) {
          this.id = id;
        }

        @Override
        public void onResponse(UpdateResponse updateResponse) {
          updateRequestsOutstanding.release(1);
        }

        @Override
        public void onFailure(Throwable e) {
          synchronized (failedMap) {
            incrementMapValue(id, failedMap);
          }
          updateRequestsOutstanding.release(1);
        }
      }

      final class DeleteListener implements ActionListener<DeleteResponse> {
        int id;

        public DeleteListener(int id) {
          this.id = id;
        }

        @Override
        public void onResponse(DeleteResponse deleteResponse) {
          deleteRequestsOutstanding.release(1);
        }

        @Override
        public void onFailure(Throwable e) {
          synchronized (failedMap) {
            incrementMapValue(id, failedMap);
          }
          deleteRequestsOutstanding.release(1);
        }
      }

      @Override
      public void run() {
        try {
          startLatch.await();
          for (int j = 0; j < numberOfIds; j++) {
            for (int k = 0; k < numberOfUpdatesPerId; ++k) {
              updateRequestsOutstanding.acquire();
              UpdateRequest ur =
                  client()
                      .prepareUpdate("test", "type1", Integer.toString(j))
                      .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
                      .setRetryOnConflict(retryOnConflict)
                      .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
                      .setListenerThreaded(false)
                      .request();
              client().update(ur, new UpdateListener(j));

              deleteRequestsOutstanding.acquire();
              DeleteRequest dr =
                  client()
                      .prepareDelete("test", "type1", Integer.toString(j))
                      .setListenerThreaded(false)
                      .setOperationThreaded(false)
                      .request();
              client().delete(dr, new DeleteListener(j));
            }
          }
        } catch (Throwable e) {
          logger.error("Something went wrong", e);
          failures.add(e);
        } finally {
          try {
            waitForOutstandingRequests(
                TimeValue.timeValueSeconds(60),
                updateRequestsOutstanding,
                maxUpdateRequests,
                "Update");
            waitForOutstandingRequests(
                TimeValue.timeValueSeconds(60),
                deleteRequestsOutstanding,
                maxDeleteRequests,
                "Delete");
          } catch (ElasticsearchTimeoutException ete) {
            failures.add(ete);
          }
          latch.countDown();
        }
      }

      private void incrementMapValue(int j, Map<Integer, Integer> map) {
        if (!map.containsKey(j)) {
          map.put(j, 0);
        }
        map.put(j, map.get(j) + 1);
      }

      private void waitForOutstandingRequests(
          TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
        long start = System.currentTimeMillis();
        do {
          long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
          logger.info(
              "[{}] going to try and aquire [{}] in [{}]ms [{}] available to aquire right now",
              name,
              maxRequests,
              msRemaining,
              requestsOutstanding.availablePermits());
          try {
            requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
            return;
          } catch (InterruptedException ie) {
            // Just keep swimming
          }
        } while ((System.currentTimeMillis() - start) < timeOut.getMillis());
        throw new ElasticsearchTimeoutException(
            "Requests were still outstanding after the timeout ["
                + timeOut
                + "] for type ["
                + name
                + "]");
      }
    }
    final List<UpdateThread> threads = new ArrayList<>();

    for (int i = 0; i < numberOfThreads; i++) {
      UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId);
      ut.start();
      threads.add(ut);
    }

    startLatch.countDown();
    latch.await();

    for (UpdateThread ut : threads) {
      ut.join(); // Threads should have finished because of the latch.await
    }

    // If are no errors every request recieved a response otherwise the test would have timedout
    // aquiring the request outstanding semaphores.
    for (Throwable throwable : failures) {
      logger.info("Captured failure on concurrent update:", throwable);
    }

    assertThat(failures.size(), equalTo(0));

    // Upsert all the ids one last time to make sure they are available at get time
    // This means that we add 1 to the expected versions and attempts
    // All the previous operations should be complete or failed at this point
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
      UpdateResponse ur =
          client()
              .prepareUpdate("test", "type1", Integer.toString(i))
              .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
              .setRetryOnConflict(Integer.MAX_VALUE)
              .setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
              .execute()
              .actionGet();
    }

    refresh();

    for (int i = 0; i < numberOfIdsPerThread; ++i) {
      int totalFailures = 0;
      GetResponse response =
          client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
      if (response.isExists()) {
        assertThat(response.getId(), equalTo(Integer.toString(i)));
        int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
        for (UpdateThread ut : threads) {
          if (ut.failedMap.containsKey(i)) {
            totalFailures += ut.failedMap.get(i);
          }
        }
        expectedVersion -= totalFailures;
        logger.error(
            "Actual version [{}] Expected version [{}] Total failures [{}]",
            response.getVersion(),
            expectedVersion,
            totalFailures);
        assertThat(response.getVersion(), equalTo((long) expectedVersion));
        assertThat(
            response.getVersion() + totalFailures,
            equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
      }
    }
  }