/**
   * Time a multi-threaded access to a cache.
   *
   * @return the timing stopwatch
   */
  private <V> StopWatch timeMultiThreaded(
      String id, final Map<Integer, V> map, ValueFactory<V> factory) throws InterruptedException {

    StopWatch stopWatch = new StopWatch(id);
    for (int i = 0; i < 500; i++) {
      map.put(i, factory.newValue(i));
    }
    Thread[] threads = new Thread[30];
    stopWatch.start("Running threads");
    for (int threadIndex = 0; threadIndex < threads.length; threadIndex++) {
      threads[threadIndex] =
          new Thread("Cache access thread " + threadIndex) {
            @Override
            public void run() {
              for (int j = 0; j < 1000; j++) {
                for (int i = 0; i < 1000; i++) {
                  map.get(i);
                }
              }
            }
          };
    }
    for (Thread thread : threads) {
      thread.start();
    }

    for (Thread thread : threads) {
      if (thread.isAlive()) {
        thread.join(2000);
      }
    }
    stopWatch.stop();
    return stopWatch;
  }
Example #2
0
 @Test
 public void shouldExportFile() throws Exception {
   final File file = File.createTempFile("export", ".csv");
   file.delete();
   Thread thread =
       new Thread() {
         @Override
         public void run() {
           log.info("process download dialog");
           app.getAutoItHelper()
               .winWaitAndActivate("Windows Internet Explorer", "", 5000)
               .click("Button3")
               .winWaitAndActivate("—охранить как", "", 5000)
               .send("Edit1", file.getAbsolutePath())
               .click("Button1");
         }
       };
   thread.start();
   log.info("click export");
   app.getNavigationHelper().clickExport();
   Thread.sleep(2000);
   thread.join();
   // дождатьс¤ сохранени¤ файла
   // проверить содержимое файла
 }
 @Test
 public void we_can_record_the_lifetime_of_a_test_step() throws InterruptedException {
   TestStep step = new TestStep("a narrative description");
   Thread.sleep(150);
   step.recordDuration();
   assertThat(step.getDuration(), is(greaterThanOrEqualTo(100L)));
 }
  @Test
  @Slow
  public void testIndexActions() throws Exception {
    startNode("server1");

    logger.info("Running Cluster Health (waiting for node to startup properly)");
    ClusterHealthResponse clusterHealth =
        client("server1")
            .admin()
            .cluster()
            .health(clusterHealthRequest().waitForGreenStatus())
            .actionGet();
    logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));

    client("server1").admin().indices().create(createIndexRequest("test")).actionGet();

    closeNode("server1");

    startNode("server1");
    Thread.sleep(500);
    try {
      client("server1").admin().indices().create(createIndexRequest("test")).actionGet();
      assert false : "index should exists";
    } catch (IndexAlreadyExistsException e) {
      // all is well
    }
  }
 @Test
 public void we_can_display_the_lifetime_of_a_test_step_in_seconds() throws InterruptedException {
   TestStep step = new TestStep("a narrative description");
   Thread.sleep(150);
   step.recordDuration();
   double expectedDuration = step.getDuration() / 1000.0;
   assertThat(step.getDurationInSeconds(), closeTo(expectedDuration, 0.01));
 }
 @AfterClass
 public static void clean() throws Exception {
   if (sc != null) {
     sc.stop();
     // wait for jetty & spark to properly shutdown
     Thread.sleep(TimeUnit.SECONDS.toMillis(2));
   }
 }
  @Test
  public void testTimeoutUpdateTask() throws Exception {
    Settings settings = settingsBuilder().put("discovery.type", "local").build();
    internalCluster().startNode(settings);
    ClusterService clusterService1 = internalCluster().getInstance(ClusterService.class);
    final CountDownLatch block = new CountDownLatch(1);
    clusterService1.submitStateUpdateTask(
        "test1",
        new ClusterStateUpdateTask() {
          @Override
          public ClusterState execute(ClusterState currentState) {
            try {
              block.await();
            } catch (InterruptedException e) {
              fail();
            }
            return currentState;
          }

          @Override
          public void onFailure(String source, Throwable t) {
            fail();
          }
        });

    final CountDownLatch timedOut = new CountDownLatch(1);
    final AtomicBoolean executeCalled = new AtomicBoolean();
    clusterService1.submitStateUpdateTask(
        "test2",
        new TimeoutClusterStateUpdateTask() {
          @Override
          public TimeValue timeout() {
            return TimeValue.timeValueMillis(2);
          }

          @Override
          public void onFailure(String source, Throwable t) {
            timedOut.countDown();
          }

          @Override
          public ClusterState execute(ClusterState currentState) {
            executeCalled.set(true);
            return currentState;
          }

          @Override
          public void clusterStateProcessed(
              String source, ClusterState oldState, ClusterState newState) {}
        });

    assertThat(timedOut.await(500, TimeUnit.MILLISECONDS), equalTo(true));
    block.countDown();
    Thread.sleep(
        100); // sleep a bit to double check that execute on the timed out update task is not
              // called...
    assertThat(executeCalled.get(), equalTo(false));
  }
  @Ignore
  @Test
  public void testGetNullWorkflowInstances()
      throws RepositoryException, XmlRpcException, IOException, InterruptedException {

    Thread.sleep(3000);
    WorkflowInstance instance = fmc.getWorkflowInstanceById("1234");

    assertThat(instance, is(nullValue()));
  }
Example #9
0
  @After
  public void after() {
    // want to wait a few milliseconds to verify calls because everything is asynchronous

    try {
      Thread.sleep(10);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
  }
  @Test
  public void testUpdateReplicationControllerToZero() throws Exception {
    getClient().createReplicationController(contr);
    getClient().updateReplicationController(contr.getId(), 0);

    Thread.sleep(10000);

    PodList podList = getClient().getSelectedPods(contr.getLabels());
    assertNotNull(podList);
    assertNotNull(podList.getItems());
    assertEquals(0, podList.getItems().size());
  }
  private static void startXmlRpcWorkflowManager() {
    System.setProperty(
        "java.util.logging.config.file",
        new File("./src/main/resources/logging.properties").getAbsolutePath());

    try {
      System.getProperties().load(new FileInputStream("./src/main/resources/workflow.properties"));
    } catch (Exception e) {
      fail(e.getMessage());
    }
    try {
      luceneCatLoc = Files.createTempDirectory("repo").toString();
      LOG.log(Level.INFO, "Lucene instance repository: [" + luceneCatLoc + "]");
    } catch (Exception e) {
      fail(e.getMessage());
    }

    if (new File(luceneCatLoc).exists()) {
      // blow away lucene cat
      LOG.log(Level.INFO, "Removing workflow instance repository: [" + luceneCatLoc + "]");
      try {
        FileUtils.deleteDirectory(new File(luceneCatLoc));
      } catch (IOException e) {
        fail(e.getMessage());
      }
    }

    System.setProperty(
        "workflow.engine.instanceRep.factory",
        "org.apache.oodt.cas.workflow.instrepo.LuceneWorkflowInstanceRepositoryFactory");
    System.setProperty("org.apache.oodt.cas.workflow.instanceRep.lucene.idxPath", luceneCatLoc);

    try {
      System.setProperty(
          "org.apache.oodt.cas.workflow.repo.dirs",
          "file://" + new File("./src/main/resources/examples").getCanonicalPath());
      System.setProperty(
          "org.apache.oodt.cas.workflow.lifecycle.filePath",
          new File("./src/main/resources/examples/workflow-lifecycle.xml").getCanonicalPath());
    } catch (Exception e) {
      fail(e.getMessage());
    }

    try {
      wmgr = new XmlRpcWorkflowManager(WM_PORT);
      Thread.sleep(MILLIS);
    } catch (Exception e) {
      LOG.log(Level.SEVERE, e.getMessage());
      fail(e.getMessage());
    }
  }
  private static void stopXmlRpcWorkflowManager() {
    System.setProperty(
        "java.util.logging.config.file",
        new File("./src/main/resources/logging.properties").getAbsolutePath());

    try {
      System.getProperties().load(new FileInputStream("./src/main/resources/workflow.properties"));
    } catch (Exception e) {
      fail(e.getMessage());
    }
    System.setProperty(
        "workflow.engine.instanceRep.factory",
        "org.apache.oodt.cas.workflow.instrepo.LuceneWorkflowInstanceRepositoryFactory");
    System.setProperty("org.apache.oodt.cas.workflow.instanceRep.lucene.idxPath", luceneCatLoc);

    try {
      System.setProperty(
          "org.apache.oodt.cas.workflow.repo.dirs",
          "file://" + new File("./src/main/resources/examples").getCanonicalPath());
      System.setProperty(
          "org.apache.oodt.cas.workflow.lifecycle.filePath",
          new File("./src/main/resources/examples/workflow-lifecycle.xml").getCanonicalPath());
    } catch (Exception e) {
      fail(e.getMessage());
    }

    try {
      wmgr.shutdown();
    } catch (Exception e) {
      LOG.log(Level.SEVERE, e.getMessage());
      fail(e.getMessage());
    }

    /** Sleep before removing to prevent file not found issues. */
    try {
      Thread.sleep(MILLIS);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }

    if (new File(luceneCatLoc).exists()) {
      // blow away lucene cat
      LOG.log(Level.INFO, "Removing workflow instance repository: [" + luceneCatLoc + "]");
      try {
        FileUtils.deleteDirectory(new File(luceneCatLoc));
      } catch (IOException e) {
        fail(e.getMessage());
      }
    }
  }
  @Test
  public void testDoesNotUseTCCL() {
    Thread.currentThread()
        .setContextClassLoader(
            new ClassLoader() {
              @Override
              public Enumeration<URL> getResources(String name) throws IOException {
                throw new AssertionError();
              }
            });

    ServiceLocator serviceLocator = new ServiceLocator();
    serviceLocator.getService(TestService.class);
  }
  @Test
  public void succeedsWhenJdkProxyAndScheduledMethodIsPresentOnInterface()
      throws InterruptedException {
    AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext();
    ctx.register(Config.class, JdkProxyTxConfig.class, RepoConfigB.class);
    ctx.refresh();

    Thread.sleep(50); // allow @Scheduled method to be called several times

    MyRepositoryWithScheduledMethod repository = ctx.getBean(MyRepositoryWithScheduledMethod.class);
    CallCountingTransactionManager txManager = ctx.getBean(CallCountingTransactionManager.class);
    assertThat("repository is not a proxy", AopUtils.isAopProxy(repository), is(true));
    assertThat("@Scheduled method never called", repository.getInvocationCount(), greaterThan(0));
    assertThat("no transactions were committed", txManager.commits, greaterThan(0));
  }
Example #15
0
 public static StackTraceElement getMyStackTraceElement() {
   int i = 0;
   for (final StackTraceElement ste : Thread.currentThread().getStackTrace()) {
     if ((i++ < 2)
         || ste.getClassName().matches(".*EntityWrapper.*")
         || ste.getClassName().matches(".*TxHandle.*")
         || ste.getMethodName().equals("getEntityWrapper")) {
       continue;
     } else {
       return ste;
     }
   }
   throw new RuntimeException(
       "BUG: Reached bottom of stack trace without finding any relevent frames.");
 }
 @Override
 public synchronized void add(Id id, Record record) throws IOException, InterruptedException {
   switch (record.context.getPhase()) {
     case SETUP:
       assertThat(flowId, is(nullValue()));
       flowId = record.context.getFlowId();
       break;
     case CLEANUP:
       assertThat(flowId, is(record.context.getFlowId()));
       flowId = null;
       break;
     default:
       Thread.sleep(100);
       assertThat(flowId, is(record.context.getFlowId()));
       break;
   }
   super.add(id, record);
 }
  @Test
  public void testGetPages() throws Exception {

    Thread.sleep(3000);
    WorkflowInstancePage page = fmc.getFirstPage();

    assertNotNull(page);

    WorkflowInstancePage lastpage = fmc.getLastPage();

    assertNotNull(lastpage);

    WorkflowInstancePage nextpage = fmc.getNextPage(page);

    assertNotNull(nextpage);

    WorkflowInstancePage prevpage = fmc.getPrevPage(nextpage);

    assertNotNull(prevpage);
  }
  @Test
  public void testFlowDescriptionCache() throws Exception {
    int maxCacheTime = 200;
    JobFlowDescriptionCache cache = new JobFlowDescriptionCache(maxCacheTime);
    DescribeJobFlowsRequest request1 = new DescribeJobFlowsRequest();
    request1.withJobFlowIds("jf1");
    DescribeJobFlowsRequest request2 = new DescribeJobFlowsRequest();
    request1.withJobFlowIds("jf2");

    assertNull(cache.getResponse(request1));
    assertNull(cache.getResponse(request2));

    cache.addResponse(request1, new DescribeJobFlowsResult());
    assertNotNull(cache.getResponse(request1));
    assertNull(cache.getResponse(request2));

    Thread.sleep(maxCacheTime * 2);
    assertNull(cache.getResponse(request1));
    assertNull(cache.getResponse(request2));
  }
  public synchronized List<String> process(byte[] content, int... writes) throws Exception {
    StringBuilder request = new StringBuilder(512);
    request.append("GET /ctx/path/info");
    char s = '?';
    for (int w : writes) {
      request.append(s).append("w=").append(w);
      s = '&';
    }

    request.append(" HTTP/1.1\r\n").append("Host: localhost\r\n").append("Connection: close\r\n");

    if (content != null)
      request
          .append("Content-Length: ")
          .append(content.length)
          .append("\r\n")
          .append("Content-Type: text/plain\r\n");

    request.append("\r\n");

    int port = _port;
    List<String> list = new ArrayList<>();
    try (Socket socket = new Socket("localhost", port)) {
      socket.setSoTimeout(1000000);
      OutputStream out = socket.getOutputStream();
      out.write(request.toString().getBytes(StandardCharsets.ISO_8859_1));

      if (content != null && content.length > 0) {
        Thread.sleep(100);
        out.write(content[0]);
        Thread.sleep(100);
        int half = (content.length - 1) / 2;
        out.write(content, 1, half);
        Thread.sleep(100);
        out.write(content, 1 + half, content.length - half - 1);
      }

      BufferedReader in =
          new BufferedReader(new InputStreamReader(socket.getInputStream()), 102400);

      // response line
      String line = in.readLine();
      LOG.debug("response-line: " + line);
      Assert.assertThat(line, startsWith("HTTP/1.1 200 OK"));

      // Skip headers
      while (line != null) {
        line = in.readLine();
        LOG.debug("header-line:  " + line);
        if (line.length() == 0) break;
      }

      // Get body slowly
      while (true) {
        line = in.readLine();
        if (line == null) break;
        LOG.debug("body:  " + brief(line));
        list.add(line);
        Thread.sleep(50);
      }
    }

    // check lines
    int w = 0;
    for (String line : list) {
      LOG.debug("line:  " + brief(line));
      if ("-".equals(line)) continue;
      assertEquals("Line Length", writes[w], line.length());
      assertEquals("Line Contents", line.charAt(0), '0' + (w % 10));

      w++;
      if (w < writes.length && writes[w] <= 0) w++;
    }

    if (content != null) Assert.assertEquals("Content Length", content.length, _read.get());

    return list;
  }
  @Test
  public void updateMappingConcurrently() throws Throwable {
    createIndex("test1", "test2");

    // This is important. The test assumes all nodes are aware of all indices. Due to initializing
    // shard throttling
    // not all shards are allocated with the initial create index. Wait for it..
    ensureYellow();

    final Throwable[] threadException = new Throwable[1];
    final AtomicBoolean stop = new AtomicBoolean(false);
    Thread[] threads = new Thread[3];
    final CyclicBarrier barrier = new CyclicBarrier(threads.length);
    final ArrayList<Client> clientArray = new ArrayList<>();
    for (Client c : clients()) {
      clientArray.add(c);
    }

    for (int j = 0; j < threads.length; j++) {
      threads[j] =
          new Thread(
              new Runnable() {
                @SuppressWarnings("unchecked")
                @Override
                public void run() {
                  try {
                    barrier.await();

                    for (int i = 0; i < 100; i++) {
                      if (stop.get()) {
                        return;
                      }

                      Client client1 = clientArray.get(i % clientArray.size());
                      Client client2 = clientArray.get((i + 1) % clientArray.size());
                      String indexName = i % 2 == 0 ? "test2" : "test1";
                      String typeName = "type" + (i % 10);
                      String fieldName = Thread.currentThread().getName() + "_" + i;

                      PutMappingResponse response =
                          client1
                              .admin()
                              .indices()
                              .preparePutMapping(indexName)
                              .setType(typeName)
                              .setSource(
                                  JsonXContent.contentBuilder()
                                      .startObject()
                                      .startObject(typeName)
                                      .startObject("properties")
                                      .startObject(fieldName)
                                      .field("type", "string")
                                      .endObject()
                                      .endObject()
                                      .endObject()
                                      .endObject())
                              .get();

                      assertThat(response.isAcknowledged(), equalTo(true));
                      GetMappingsResponse getMappingResponse =
                          client2.admin().indices().prepareGetMappings(indexName).get();
                      ImmutableOpenMap<String, MappingMetaData> mappings =
                          getMappingResponse.getMappings().get(indexName);
                      assertThat(mappings.containsKey(typeName), equalTo(true));
                      assertThat(
                          ((Map<String, Object>)
                                  mappings.get(typeName).getSourceAsMap().get("properties"))
                              .keySet(),
                          Matchers.hasItem(fieldName));
                    }
                  } catch (Throwable t) {
                    threadException[0] = t;
                    stop.set(true);
                  }
                }
              });

      threads[j].setName("t_" + j);
      threads[j].start();
    }

    for (Thread t : threads) t.join();

    if (threadException[0] != null) {
      throw threadException[0];
    }
  }
  @Test
  public void testPendingUpdateTask() throws Exception {
    Settings settings = settingsBuilder().put("discovery.type", "local").build();
    String node_0 = internalCluster().startNode(settings);
    internalCluster().startNodeClient(settings);

    final ClusterService clusterService =
        internalCluster().getInstance(ClusterService.class, node_0);
    final CountDownLatch block1 = new CountDownLatch(1);
    final CountDownLatch invoked1 = new CountDownLatch(1);
    clusterService.submitStateUpdateTask(
        "1",
        new ClusterStateUpdateTask() {
          @Override
          public ClusterState execute(ClusterState currentState) {
            invoked1.countDown();
            try {
              block1.await();
            } catch (InterruptedException e) {
              fail();
            }
            return currentState;
          }

          @Override
          public void onFailure(String source, Throwable t) {
            invoked1.countDown();
            fail();
          }
        });
    invoked1.await();
    final CountDownLatch invoked2 = new CountDownLatch(9);
    for (int i = 2; i <= 10; i++) {
      clusterService.submitStateUpdateTask(
          Integer.toString(i),
          new ProcessedClusterStateUpdateTask() {
            @Override
            public ClusterState execute(ClusterState currentState) {
              return currentState;
            }

            @Override
            public void onFailure(String source, Throwable t) {
              fail();
            }

            @Override
            public void clusterStateProcessed(
                String source, ClusterState oldState, ClusterState newState) {
              invoked2.countDown();
            }
          });
    }

    // there might be other tasks in this node, make sure to only take the ones we add into account
    // in this test

    // The tasks can be re-ordered, so we need to check out-of-order
    Set<String> controlSources =
        new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
    List<PendingClusterTask> pendingClusterTasks = clusterService.pendingTasks();
    assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(10));
    assertThat(pendingClusterTasks.get(0).getSource().string(), equalTo("1"));
    assertThat(pendingClusterTasks.get(0).isExecuting(), equalTo(true));
    for (PendingClusterTask task : pendingClusterTasks) {
      controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());

    controlSources =
        new HashSet<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10"));
    PendingClusterTasksResponse response =
        internalCluster()
            .clientNodeClient()
            .admin()
            .cluster()
            .preparePendingClusterTasks()
            .execute()
            .actionGet();
    assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(10));
    assertThat(response.pendingTasks().get(0).getSource().string(), equalTo("1"));
    assertThat(response.pendingTasks().get(0).isExecuting(), equalTo(true));
    for (PendingClusterTask task : response) {
      controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());
    block1.countDown();
    invoked2.await();

    // whenever we test for no tasks, we need to awaitBusy since this is a live node
    assertTrue(
        awaitBusy(
            new Predicate<Object>() {
              @Override
              public boolean apply(Object input) {
                return clusterService.pendingTasks().isEmpty();
              }
            }));
    waitNoPendingTasksOnAll();

    final CountDownLatch block2 = new CountDownLatch(1);
    final CountDownLatch invoked3 = new CountDownLatch(1);
    clusterService.submitStateUpdateTask(
        "1",
        new ClusterStateUpdateTask() {
          @Override
          public ClusterState execute(ClusterState currentState) {
            invoked3.countDown();
            try {
              block2.await();
            } catch (InterruptedException e) {
              fail();
            }
            return currentState;
          }

          @Override
          public void onFailure(String source, Throwable t) {
            invoked3.countDown();
            fail();
          }
        });
    invoked3.await();

    for (int i = 2; i <= 5; i++) {
      clusterService.submitStateUpdateTask(
          Integer.toString(i),
          new ClusterStateUpdateTask() {
            @Override
            public ClusterState execute(ClusterState currentState) {
              return currentState;
            }

            @Override
            public void onFailure(String source, Throwable t) {
              fail();
            }
          });
    }
    Thread.sleep(100);

    pendingClusterTasks = clusterService.pendingTasks();
    assertThat(pendingClusterTasks.size(), greaterThanOrEqualTo(5));
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
    for (PendingClusterTask task : pendingClusterTasks) {
      controlSources.remove(task.getSource().string());
    }
    assertTrue(controlSources.isEmpty());

    response =
        internalCluster().clientNodeClient().admin().cluster().preparePendingClusterTasks().get();
    assertThat(response.pendingTasks().size(), greaterThanOrEqualTo(5));
    controlSources = new HashSet<>(Arrays.asList("1", "2", "3", "4", "5"));
    for (PendingClusterTask task : response) {
      if (controlSources.remove(task.getSource().string())) {
        assertThat(task.getTimeInQueueInMillis(), greaterThan(0l));
      }
    }
    assertTrue(controlSources.isEmpty());
    block2.countDown();
  }
Example #22
0
  @Test
  @TestLogging("_root:TRACE")
  public void testTimeoutSendExceptionWithDelayedResponse() throws Exception {
    serviceA.registerHandler(
        "sayHelloTimeoutDelayedResponse",
        new BaseTransportRequestHandler<StringMessageRequest>() {
          @Override
          public StringMessageRequest newInstance() {
            return new StringMessageRequest();
          }

          @Override
          public String executor() {
            return ThreadPool.Names.GENERIC;
          }

          @Override
          public void messageReceived(StringMessageRequest request, TransportChannel channel) {
            TimeValue sleep = TimeValue.parseTimeValue(request.message, null);
            try {
              Thread.sleep(sleep.millis());
            } catch (InterruptedException e) {
              // ignore
            }
            try {
              channel.sendResponse(new StringMessageResponse("hello " + request.message));
            } catch (IOException e) {
              e.printStackTrace();
              assertThat(e.getMessage(), false, equalTo(true));
            }
          }
        });

    TransportFuture<StringMessageResponse> res =
        serviceB.submitRequest(
            nodeA,
            "sayHelloTimeoutDelayedResponse",
            new StringMessageRequest("300ms"),
            options().withTimeout(100),
            new BaseTransportResponseHandler<StringMessageResponse>() {
              @Override
              public StringMessageResponse newInstance() {
                return new StringMessageResponse();
              }

              @Override
              public String executor() {
                return ThreadPool.Names.GENERIC;
              }

              @Override
              public void handleResponse(StringMessageResponse response) {
                assertThat("got response instead of exception", false, equalTo(true));
              }

              @Override
              public void handleException(TransportException exp) {
                assertThat(exp, instanceOf(ReceiveTimeoutTransportException.class));
              }
            });

    try {
      StringMessageResponse message = res.txGet();
      assertThat("exception should be thrown", false, equalTo(true));
    } catch (Exception e) {
      assertThat(e, instanceOf(ReceiveTimeoutTransportException.class));
    }

    // sleep for 400 millis to make sure we get back the response
    Thread.sleep(400);

    for (int i = 0; i < 10; i++) {
      final int counter = i;
      // now, try and send another request, this times, with a short timeout
      res =
          serviceB.submitRequest(
              nodeA,
              "sayHelloTimeoutDelayedResponse",
              new StringMessageRequest(counter + "ms"),
              options().withTimeout(100),
              new BaseTransportResponseHandler<StringMessageResponse>() {
                @Override
                public StringMessageResponse newInstance() {
                  return new StringMessageResponse();
                }

                @Override
                public String executor() {
                  return ThreadPool.Names.GENERIC;
                }

                @Override
                public void handleResponse(StringMessageResponse response) {
                  assertThat("hello " + counter + "ms", equalTo(response.message));
                }

                @Override
                public void handleException(TransportException exp) {
                  exp.printStackTrace();
                  assertThat(
                      "got exception instead of a response for "
                          + counter
                          + ": "
                          + exp.getDetailedMessage(),
                      false,
                      equalTo(true));
                }
              });

      StringMessageResponse message = res.txGet();
      assertThat(message.message, equalTo("hello " + counter + "ms"));
    }

    serviceA.removeHandler("sayHelloTimeoutDelayedResponse");
  }
  @Test
  public void testClearAllCaches() throws Exception {
    client().admin().indices().prepareDelete().execute().actionGet();
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(
            ImmutableSettings.settingsBuilder()
                .put("index.number_of_replicas", 0)
                .put("index.number_of_shards", 1))
        .execute()
        .actionGet();
    client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
    client().prepareIndex("test", "type", "1").setSource("field", "value1").execute().actionGet();
    client().prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
    client().admin().indices().prepareRefresh().execute().actionGet();

    NodesStatsResponse nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));

    IndicesStatsResponse indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));

    // sort to load it to field data and filter to load filter cache
    client()
        .prepareSearch()
        .setFilter(FilterBuilders.termFilter("field", "value1"))
        .addSort("field", SortOrder.ASC)
        .execute()
        .actionGet();
    client()
        .prepareSearch()
        .setFilter(FilterBuilders.termFilter("field", "value2"))
        .addSort("field", SortOrder.ASC)
        .execute()
        .actionGet();

    nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(),
        greaterThan(0l));
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(),
        greaterThan(0l));

    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0l));
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), greaterThan(0l));

    client().admin().indices().prepareClearCache().execute().actionGet();
    Thread.sleep(100); // Make sure the filter cache entries have been removed...
    nodesStats =
        client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    assertThat(
        nodesStats.getNodes()[0].getIndices().getFilterCache().getMemorySizeInBytes(), equalTo(0l));

    indicesStats =
        client()
            .admin()
            .indices()
            .prepareStats("test")
            .clear()
            .setFieldData(true)
            .setFilterCache(true)
            .execute()
            .actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l));
    assertThat(indicesStats.getTotal().getFilterCache().getMemorySizeInBytes(), equalTo(0l));
  }
Example #24
0
  @Test
  public void testSimpleTTL() throws Exception {
    assertAcked(
        prepareCreate("test")
            .addMapping(
                "type1",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("type1")
                    .startObject("_timestamp")
                    .field("enabled", true)
                    .field("store", "yes")
                    .endObject()
                    .startObject("_ttl")
                    .field("enabled", true)
                    .endObject()
                    .endObject()
                    .endObject())
            .addMapping(
                "type2",
                XContentFactory.jsonBuilder()
                    .startObject()
                    .startObject("type2")
                    .startObject("_timestamp")
                    .field("enabled", true)
                    .field("store", "yes")
                    .endObject()
                    .startObject("_ttl")
                    .field("enabled", true)
                    .field("default", "1d")
                    .endObject()
                    .endObject()
                    .endObject()));
    ensureYellow("test");

    final NumShards test = getNumShards("test");

    long providedTTLValue = 3000;
    logger.info("--> checking ttl");
    // Index one doc without routing, one doc with routing, one doc with not TTL and no default and
    // one doc with default TTL
    long now = System.currentTimeMillis();
    IndexResponse indexResponse =
        client()
            .prepareIndex("test", "type1", "1")
            .setSource("field1", "value1")
            .setTimestamp(String.valueOf(now))
            .setTTL(providedTTLValue)
            .setRefresh(true)
            .get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client()
            .prepareIndex("test", "type1", "with_routing")
            .setSource("field1", "value1")
            .setTimestamp(String.valueOf(now))
            .setTTL(providedTTLValue)
            .setRouting("routing")
            .setRefresh(true)
            .get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
    assertThat(indexResponse.isCreated(), is(true));
    indexResponse =
        client().prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").get();
    assertThat(indexResponse.isCreated(), is(true));

    // realtime get check
    long currentTime = System.currentTimeMillis();
    GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
    long ttl0;
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // verify the ttl is still decreasing when going to the replica
    currentTime = System.currentTimeMillis();
    getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // non realtime get (stored)
    currentTime = System.currentTimeMillis();
    getResponse =
        client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }
    // non realtime get going the replica
    currentTime = System.currentTimeMillis();
    getResponse =
        client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
    if (getResponse.isExists()) {
      ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
      assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
    } else {
      assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
    }

    // no TTL provided so no TTL fetched
    getResponse =
        client()
            .prepareGet("test", "type1", "no_ttl")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.getField("_ttl"), nullValue());
    // no TTL provided make sure it has default TTL
    getResponse =
        client()
            .prepareGet("test", "type2", "default_ttl")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
    assertThat(ttl0, greaterThan(0L));

    IndicesStatsResponse response =
        client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
    assertThat(
        response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount(),
        equalTo(0L));

    // make sure the purger has done its job for all indexed docs that are expired
    long shouldBeExpiredDate = now + providedTTLValue + PURGE_INTERVAL + 2000;
    currentTime = System.currentTimeMillis();
    if (shouldBeExpiredDate - currentTime > 0) {
      Thread.sleep(shouldBeExpiredDate - currentTime);
    }

    // We can't assume that after waiting for ttl + purgeInterval (waitTime) that the document have
    // actually been deleted.
    // The ttl purging happens in the background in a different thread, and might not have been
    // completed after waiting for waitTime.
    // But we can use index statistics' delete count to be sure that deletes have been executed,
    // that must be incremented before
    // ttl purging has finished.
    logger.info("--> checking purger");
    assertThat(
        awaitBusy(
            new Predicate<Object>() {
              @Override
              public boolean apply(Object input) {
                if (rarely()) {
                  client().admin().indices().prepareFlush("test").get();
                } else if (rarely()) {
                  client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get();
                }
                IndicesStatsResponse response =
                    client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
                // TTL deletes two docs, but it is indexed in the primary shard and replica shard.
                return response
                        .getIndices()
                        .get("test")
                        .getTotal()
                        .getIndexing()
                        .getTotal()
                        .getDeleteCount()
                    == 2L * test.dataCopies;
              }
            },
            5,
            TimeUnit.SECONDS),
        equalTo(true));

    // realtime get check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    // replica realtime get check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(true)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));

    // Need to run a refresh, in order for the non realtime get to work.
    client().admin().indices().prepareRefresh("test").execute().actionGet();

    // non realtime get (stored) check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    // non realtime get going the replica check
    getResponse =
        client()
            .prepareGet("test", "type1", "1")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
    getResponse =
        client()
            .prepareGet("test", "type1", "with_routing")
            .setRouting("routing")
            .setFields("_ttl")
            .setRealtime(false)
            .execute()
            .actionGet();
    assertThat(getResponse.isExists(), equalTo(false));
  }
  public void testDelayedMappingPropagationOnReplica() throws Exception {
    // This is essentially the same thing as testDelayedMappingPropagationOnPrimary
    // but for replicas
    // Here we want to test that everything goes well if the mappings that
    // are needed for a document are not available on the replica at the
    // time of indexing it
    final List<String> nodeNames = internalCluster().startNodesAsync(2).get();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());

    final String master = internalCluster().getMasterName();
    assertThat(nodeNames, hasItem(master));
    String otherNode = null;
    for (String node : nodeNames) {
      if (node.equals(master) == false) {
        otherNode = node;
        break;
      }
    }
    assertNotNull(otherNode);

    // Force allocation of the primary on the master node by first only allocating on the master
    // and then allowing all nodes so that the replica gets allocated on the other node
    assertAcked(
        prepareCreate("index")
            .setSettings(
                Settings.builder()
                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
                    .put("index.routing.allocation.include._name", master))
            .get());
    assertAcked(
        client()
            .admin()
            .indices()
            .prepareUpdateSettings("index")
            .setSettings(Settings.builder().put("index.routing.allocation.include._name", ""))
            .get());
    ensureGreen();

    // Check routing tables
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    assertEquals(master, state.nodes().masterNode().name());
    List<ShardRouting> shards = state.routingTable().allShards("index");
    assertThat(shards, hasSize(2));
    for (ShardRouting shard : shards) {
      if (shard.primary()) {
        // primary must be on the master
        assertEquals(state.nodes().masterNodeId(), shard.currentNodeId());
      } else {
        assertTrue(shard.active());
      }
    }

    // Block cluster state processing on the replica
    BlockClusterStateProcessing disruption =
        new BlockClusterStateProcessing(otherNode, getRandom());
    internalCluster().setDisruptionScheme(disruption);
    disruption.startDisrupting();
    final AtomicReference<Object> putMappingResponse = new AtomicReference<>();
    client()
        .admin()
        .indices()
        .preparePutMapping("index")
        .setType("type")
        .setSource("field", "type=long")
        .execute(
            new ActionListener<PutMappingResponse>() {
              @Override
              public void onResponse(PutMappingResponse response) {
                putMappingResponse.set(response);
              }

              @Override
              public void onFailure(Throwable e) {
                putMappingResponse.set(e);
              }
            });
    // Wait for mappings to be available on master
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            final IndicesService indicesService =
                internalCluster().getInstance(IndicesService.class, master);
            final IndexService indexService = indicesService.indexServiceSafe("index");
            assertNotNull(indexService);
            final MapperService mapperService = indexService.mapperService();
            DocumentMapper mapper = mapperService.documentMapper("type");
            assertNotNull(mapper);
            assertNotNull(mapper.mappers().getMapper("field"));
          }
        });

    final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
    client()
        .prepareIndex("index", "type", "1")
        .setSource("field", 42)
        .execute(
            new ActionListener<IndexResponse>() {
              @Override
              public void onResponse(IndexResponse response) {
                docIndexResponse.set(response);
              }

              @Override
              public void onFailure(Throwable e) {
                docIndexResponse.set(e);
              }
            });

    // Wait for document to be indexed on primary
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            assertTrue(
                client()
                    .prepareGet("index", "type", "1")
                    .setPreference("_primary")
                    .get()
                    .isExists());
          }
        });

    // The mappings have not been propagated to the replica yet as a consequence the document count
    // not be indexed
    // We wait on purpose to make sure that the document is not indexed because the shard operation
    // is stalled
    // and not just because it takes time to replicate the indexing request to the replica
    Thread.sleep(100);
    assertThat(putMappingResponse.get(), equalTo(null));
    assertThat(docIndexResponse.get(), equalTo(null));

    // Now make sure the indexing request finishes successfully
    disruption.stopDisrupting();
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
            PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
            assertTrue(resp.isAcknowledged());
            assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
            IndexResponse docResp = (IndexResponse) docIndexResponse.get();
            assertEquals(
                Arrays.toString(docResp.getShardInfo().getFailures()),
                2,
                docResp.getShardInfo().getTotal()); // both shards should have succeeded
          }
        });
  }
  public void testDelayedMappingPropagationOnPrimary() throws Exception {
    // Here we want to test that things go well if there is a first request
    // that adds mappings but before mappings are propagated to all nodes
    // another index request introduces the same mapping. The master node
    // will reply immediately since it did not change the cluster state
    // but the change might not be on the node that performed the indexing
    // operation yet

    Settings settings = Settings.builder().put(DiscoverySettings.PUBLISH_TIMEOUT, "0ms").build();
    final List<String> nodeNames = internalCluster().startNodesAsync(2, settings).get();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());

    final String master = internalCluster().getMasterName();
    assertThat(nodeNames, hasItem(master));
    String otherNode = null;
    for (String node : nodeNames) {
      if (node.equals(master) == false) {
        otherNode = node;
        break;
      }
    }
    assertNotNull(otherNode);

    // Don't allocate the shard on the master node
    assertAcked(
        prepareCreate("index")
            .setSettings(
                Settings.builder()
                    .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                    .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                    .put("index.routing.allocation.exclude._name", master))
            .get());
    ensureGreen();

    // Check routing tables
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    assertEquals(master, state.nodes().masterNode().name());
    List<ShardRouting> shards = state.routingTable().allShards("index");
    assertThat(shards, hasSize(1));
    for (ShardRouting shard : shards) {
      if (shard.primary()) {
        // primary must not be on the master node
        assertFalse(state.nodes().masterNodeId().equals(shard.currentNodeId()));
      } else {
        fail(); // only primaries
      }
    }

    // Block cluster state processing where our shard is
    BlockClusterStateProcessing disruption =
        new BlockClusterStateProcessing(otherNode, getRandom());
    internalCluster().setDisruptionScheme(disruption);
    disruption.startDisrupting();

    // Add a new mapping...
    final AtomicReference<Object> putMappingResponse = new AtomicReference<>();
    client()
        .admin()
        .indices()
        .preparePutMapping("index")
        .setType("type")
        .setSource("field", "type=long")
        .execute(
            new ActionListener<PutMappingResponse>() {
              @Override
              public void onResponse(PutMappingResponse response) {
                putMappingResponse.set(response);
              }

              @Override
              public void onFailure(Throwable e) {
                putMappingResponse.set(e);
              }
            });
    // ...and wait for mappings to be available on master
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            ImmutableOpenMap<String, MappingMetaData> indexMappings =
                client()
                    .admin()
                    .indices()
                    .prepareGetMappings("index")
                    .get()
                    .getMappings()
                    .get("index");
            assertNotNull(indexMappings);
            MappingMetaData typeMappings = indexMappings.get("type");
            assertNotNull(typeMappings);
            Object properties;
            try {
              properties = typeMappings.getSourceAsMap().get("properties");
            } catch (IOException e) {
              throw new AssertionError(e);
            }
            assertNotNull(properties);
            Object fieldMapping = ((Map<String, Object>) properties).get("field");
            assertNotNull(fieldMapping);
          }
        });

    final AtomicReference<Object> docIndexResponse = new AtomicReference<>();
    client()
        .prepareIndex("index", "type", "1")
        .setSource("field", 42)
        .execute(
            new ActionListener<IndexResponse>() {
              @Override
              public void onResponse(IndexResponse response) {
                docIndexResponse.set(response);
              }

              @Override
              public void onFailure(Throwable e) {
                docIndexResponse.set(e);
              }
            });

    // Wait a bit to make sure that the reason why we did not get a response
    // is that cluster state processing is blocked and not just that it takes
    // time to process the indexing request
    Thread.sleep(100);
    assertThat(putMappingResponse.get(), equalTo(null));
    assertThat(docIndexResponse.get(), equalTo(null));

    // Now make sure the indexing request finishes successfully
    disruption.stopDisrupting();
    assertBusy(
        new Runnable() {
          @Override
          public void run() {
            assertThat(putMappingResponse.get(), instanceOf(PutMappingResponse.class));
            PutMappingResponse resp = (PutMappingResponse) putMappingResponse.get();
            assertTrue(resp.isAcknowledged());
            assertThat(docIndexResponse.get(), instanceOf(IndexResponse.class));
            IndexResponse docResp = (IndexResponse) docIndexResponse.get();
            assertEquals(
                Arrays.toString(docResp.getShardInfo().getFailures()),
                1,
                docResp.getShardInfo().getTotal());
          }
        });
  }
  @Test(timeout = 20000)
  public void testUpdatingThreadPoolSettings() throws Exception {
    internalCluster().startNodesAsync(2).get();
    ThreadPool threadPool = internalCluster().getDataNodeInstance(ThreadPool.class);
    // Check that settings are changed
    assertThat(
        ((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES),
        equalTo(5L));
    client()
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setTransientSettings(settingsBuilder().put("threadpool.search.keep_alive", "10m").build())
        .execute()
        .actionGet();
    assertThat(
        ((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES),
        equalTo(10L));

    // Make sure that threads continue executing when executor is replaced
    final CyclicBarrier barrier = new CyclicBarrier(2);
    Executor oldExecutor = threadPool.executor(Names.SEARCH);
    threadPool
        .executor(Names.SEARCH)
        .execute(
            new Runnable() {
              @Override
              public void run() {
                try {
                  barrier.await();
                } catch (InterruptedException ex) {
                  Thread.currentThread().interrupt();
                } catch (BrokenBarrierException ex) {
                  //
                }
              }
            });
    client()
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setTransientSettings(settingsBuilder().put("threadpool.search.type", "fixed").build())
        .execute()
        .actionGet();
    assertThat(threadPool.executor(Names.SEARCH), not(sameInstance(oldExecutor)));
    assertThat(((ThreadPoolExecutor) oldExecutor).isShutdown(), equalTo(true));
    assertThat(((ThreadPoolExecutor) oldExecutor).isTerminating(), equalTo(true));
    assertThat(((ThreadPoolExecutor) oldExecutor).isTerminated(), equalTo(false));
    barrier.await();

    // Make sure that new thread executor is functional
    threadPool
        .executor(Names.SEARCH)
        .execute(
            new Runnable() {
              @Override
              public void run() {
                try {
                  barrier.await();
                } catch (InterruptedException ex) {
                  Thread.currentThread().interrupt();
                } catch (BrokenBarrierException ex) {
                  //
                }
              }
            });
    client()
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setTransientSettings(settingsBuilder().put("threadpool.search.type", "fixed").build())
        .execute()
        .actionGet();
    barrier.await();
    Thread.sleep(200);

    // Check that node info is correct
    NodesInfoResponse nodesInfoResponse =
        client().admin().cluster().prepareNodesInfo().all().execute().actionGet();
    for (int i = 0; i < 2; i++) {
      NodeInfo nodeInfo = nodesInfoResponse.getNodes()[i];
      boolean found = false;
      for (ThreadPool.Info info : nodeInfo.getThreadPool()) {
        if (info.getName().equals(Names.SEARCH)) {
          assertThat(info.getType(), equalTo("fixed"));
          found = true;
          break;
        }
      }
      assertThat(found, equalTo(true));

      Map<String, Object> poolMap =
          getPoolSettingsThroughJson(nodeInfo.getThreadPool(), Names.SEARCH);
    }
  }
Example #28
0
  @Test
  public void testRecoveryDiff() throws IOException, InterruptedException {
    int numDocs = 2 + random().nextInt(100);
    List<Document> docs = new ArrayList<>();
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      doc.add(
          new StringField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
      doc.add(
          new TextField(
              "body",
              TestUtil.randomRealisticUnicodeString(random()),
              random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
      doc.add(
          new SortedDocValuesField(
              "dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
      docs.add(doc);
    }
    long seed = random().nextLong();
    Store.MetadataSnapshot first;
    {
      Random random = new Random(seed);
      IndexWriterConfig iwc =
          new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
      iwc.setMergePolicy(NoMergePolicy.INSTANCE);
      iwc.setUseCompoundFile(random.nextBoolean());
      iwc.setMaxThreadStates(1);
      final ShardId shardId = new ShardId(new Index("index"), 1);
      DirectoryService directoryService = new LuceneManagedDirectoryService(random);
      Store store =
          new Store(
              shardId,
              ImmutableSettings.EMPTY,
              directoryService,
              randomDistributor(random, directoryService),
              new DummyShardLock(shardId));
      IndexWriter writer = new IndexWriter(store.directory(), iwc);
      final boolean lotsOfSegments = rarely(random);
      for (Document d : docs) {
        writer.addDocument(d);
        if (lotsOfSegments && random.nextBoolean()) {
          writer.commit();
        } else if (rarely(random)) {
          writer.commit();
        }
      }
      writer.commit();
      writer.close();
      first = store.getMetadata();
      assertDeleteContent(store, directoryService);
      store.close();
    }
    long time = new Date().getTime();
    while (time == new Date().getTime()) {
      Thread.sleep(10); // bump the time
    }
    Store.MetadataSnapshot second;
    Store store;
    {
      Random random = new Random(seed);
      IndexWriterConfig iwc =
          new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
      iwc.setMergePolicy(NoMergePolicy.INSTANCE);
      iwc.setUseCompoundFile(random.nextBoolean());
      iwc.setMaxThreadStates(1);
      final ShardId shardId = new ShardId(new Index("index"), 1);
      DirectoryService directoryService = new LuceneManagedDirectoryService(random);
      store =
          new Store(
              shardId,
              ImmutableSettings.EMPTY,
              directoryService,
              randomDistributor(random, directoryService),
              new DummyShardLock(shardId));
      IndexWriter writer = new IndexWriter(store.directory(), iwc);
      final boolean lotsOfSegments = rarely(random);
      for (Document d : docs) {
        writer.addDocument(d);
        if (lotsOfSegments && random.nextBoolean()) {
          writer.commit();
        } else if (rarely(random)) {
          writer.commit();
        }
      }
      writer.commit();
      writer.close();
      second = store.getMetadata();
    }
    Store.RecoveryDiff diff = first.recoveryDiff(second);
    assertThat(first.size(), equalTo(second.size()));
    for (StoreFileMetaData md : first) {
      assertThat(second.get(md.name()), notNullValue());
      // si files are different - containing timestamps etc
      assertThat(second.get(md.name()).isSame(md), equalTo(false));
    }
    assertThat(diff.different.size(), equalTo(first.size()));
    assertThat(
        diff.identical.size(),
        equalTo(0)); // in lucene 5 nothing is identical - we use random ids in file headers
    assertThat(diff.missing, empty());

    // check the self diff
    Store.RecoveryDiff selfDiff = first.recoveryDiff(first);
    assertThat(selfDiff.identical.size(), equalTo(first.size()));
    assertThat(selfDiff.different, empty());
    assertThat(selfDiff.missing, empty());

    // lets add some deletes
    Random random = new Random(seed);
    IndexWriterConfig iwc =
        new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setUseCompoundFile(random.nextBoolean());
    iwc.setMaxThreadStates(1);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
    IndexWriter writer = new IndexWriter(store.directory(), iwc);
    writer.deleteDocuments(new Term("id", Integer.toString(random().nextInt(numDocs))));
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata();
    StoreFileMetaData delFile = null;
    for (StoreFileMetaData md : metadata) {
      if (md.name().endsWith(".liv")) {
        delFile = md;
        break;
      }
    }
    Store.RecoveryDiff afterDeleteDiff = metadata.recoveryDiff(second);
    if (delFile != null) {
      assertThat(
          afterDeleteDiff.identical.size(), equalTo(metadata.size() - 2)); // segments_N + del file
      assertThat(afterDeleteDiff.different.size(), equalTo(0));
      assertThat(afterDeleteDiff.missing.size(), equalTo(2));
    } else {
      // an entire segment must be missing (single doc segment got dropped)
      assertThat(afterDeleteDiff.identical.size(), greaterThan(0));
      assertThat(afterDeleteDiff.different.size(), equalTo(0));
      assertThat(afterDeleteDiff.missing.size(), equalTo(1)); // the commit file is different
    }

    // check the self diff
    selfDiff = metadata.recoveryDiff(metadata);
    assertThat(selfDiff.identical.size(), equalTo(metadata.size()));
    assertThat(selfDiff.different, empty());
    assertThat(selfDiff.missing, empty());

    // add a new commit
    iwc = new IndexWriterConfig(new MockAnalyzer(random)).setCodec(actualDefaultCodec());
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setUseCompoundFile(
        true); // force CFS - easier to test here since we know it will add 3 files
    iwc.setMaxThreadStates(1);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
    writer = new IndexWriter(store.directory(), iwc);
    writer.addDocument(docs.get(0));
    writer.close();

    Store.MetadataSnapshot newCommitMetaData = store.getMetadata();
    Store.RecoveryDiff newCommitDiff = newCommitMetaData.recoveryDiff(metadata);
    if (delFile != null) {
      assertThat(
          newCommitDiff.identical.size(),
          equalTo(
              newCommitMetaData.size()
                  - 5)); // segments_N, del file, cfs, cfe, si for the new segment
      assertThat(newCommitDiff.different.size(), equalTo(1)); // the del file must be different
      assertThat(newCommitDiff.different.get(0).name(), endsWith(".liv"));
      assertThat(
          newCommitDiff.missing.size(), equalTo(4)); // segments_N,cfs, cfe, si for the new segment
    } else {
      assertThat(
          newCommitDiff.identical.size(),
          equalTo(newCommitMetaData.size() - 4)); // segments_N, cfs, cfe, si for the new segment
      assertThat(newCommitDiff.different.size(), equalTo(0));
      assertThat(
          newCommitDiff.missing.size(),
          equalTo(
              4)); // an entire segment must be missing (single doc segment got dropped)  plus the
                   // commit is different
    }

    store.deleteContent();
    IOUtils.close(store);
  }