/* Regression test for DB-3614 */
  @Test
  @Category(SlowTest.class)
  @Ignore("-sf- takes way too long to fail and interferes rest of build")
  public void testTenTableJoinExplainDuration() throws Exception {
    int size = 10;
    List<String> tables = new ArrayList<String>(size);
    List<String> joins = new ArrayList<String>(size - 1);
    for (int i = 0; i < size; i++) {
      methodWatcher.executeUpdate(format("create table tentab%s (c1 int primary key)", i));
      methodWatcher.executeUpdate(format("insert into tentab%s values (1)", i));
      tables.add(format("tentab%s", i));
      if (i > 0) {
        joins.add(format("tentab%s.c1 = tentab%s.c1", i, i - 1));
      }
    }
    System.out.println("Tables created");
    final String fromClause = Joiner.on(", ").join(tables);
    final String joinCriteria = Joiner.on(" AND ").join(joins);

    ExecutorService es =
        Executors.newSingleThreadExecutor(
            new ThreadFactory() {
              @Override
              public Thread newThread(Runnable r) {
                Thread t = new Thread(r);
                t.setDaemon(true);
                return t;
              }
            });
    try {
      final CyclicBarrier startLatch = new CyclicBarrier(2);
      final CountDownLatch finishLatch = new CountDownLatch(1);
      Future<Void> f =
          es.submit(
              new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                  String query =
                      format("EXPLAIN SELECT * FROM %s WHERE %s ", fromClause, joinCriteria);
                  startLatch.await();
                  try {
                    ResultSet rs = methodWatcher.executeQuery(query);
                    // Loose check that explain statement took a few seconds or less,
                    // because we're just making sure the short circuit logic in
                    // OptimizerImpl.checkTimeout() blocks this from taking several minutes.
                    Assert.assertTrue("Explain did not return result!", rs.next());
                  } finally {
                    finishLatch.countDown();
                  }
                  return null;
                }
              });
      System.out.println("Starting wait");
      startLatch.await();
      f.get(1, TimeUnit.SECONDS);
      System.out.println("Finished waiting");
    } finally {
      System.out.println("shutting down");
    }
  }
Exemple #2
0
    public static void main(String[] args) throws Throwable {
      final ReentrantLock lock = new ReentrantLock();
      lock.lock();

      final ReentrantReadWriteLock rwlock = new ReentrantReadWriteLock();
      final ReentrantReadWriteLock.ReadLock readLock = rwlock.readLock();
      final ReentrantReadWriteLock.WriteLock writeLock = rwlock.writeLock();
      rwlock.writeLock().lock();

      final BlockingQueue<Object> q = new LinkedBlockingQueue<Object>();
      final Semaphore fairSem = new Semaphore(0, true);
      final Semaphore unfairSem = new Semaphore(0, false);
      // final int threads =
      // rnd.nextInt(Runtime.getRuntime().availableProcessors() + 1) + 1;
      final int threads = 3;
      // On Linux, this test runs very slowly for some reason,
      // so use a smaller number of iterations.
      // Solaris can handle 1 << 18.
      // On the other hand, jmap is much slower on Solaris...
      final int iterations = 1 << 8;
      final CyclicBarrier cb = new CyclicBarrier(threads + 1);

      for (int i = 0; i < threads; i++)
        new Thread() {
          public void run() {
            try {
              final Random rnd = new Random();
              for (int j = 0; j < iterations; j++) {
                if (j == iterations / 10 || j == iterations - 1) {
                  cb.await(); // Quiesce
                  cb.await(); // Resume
                }
                // int t = rnd.nextInt(2000);
                int t = rnd.nextInt(900);
                check(!lock.tryLock(t, NANOSECONDS));
                check(!readLock.tryLock(t, NANOSECONDS));
                check(!writeLock.tryLock(t, NANOSECONDS));
                equal(null, q.poll(t, NANOSECONDS));
                check(!fairSem.tryAcquire(t, NANOSECONDS));
                check(!unfairSem.tryAcquire(t, NANOSECONDS));
              }
            } catch (Throwable t) {
              unexpected(t);
            }
          }
        }.start();

      cb.await(); // Quiesce
      rendezvousChild(); // Measure
      cb.await(); // Resume

      cb.await(); // Quiesce
      rendezvousChild(); // Measure
      cb.await(); // Resume

      System.exit(failed);
    }
  public void run() throws Exception {
    for (Thread t : searcherThreads) {
      t.start();
    }
    for (Thread t : writerThreads) {
      t.start();
    }
    barrier1.await();

    Refresher refresher = new Refresher();
    scheduledExecutorService.scheduleWithFixedDelay(
        refresher, refreshSchedule.millis(), refreshSchedule.millis(), TimeUnit.MILLISECONDS);
    Flusher flusher = new Flusher();
    scheduledExecutorService.scheduleWithFixedDelay(
        flusher, flushSchedule.millis(), flushSchedule.millis(), TimeUnit.MILLISECONDS);

    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    barrier2.await();

    latch.await();
    stopWatch.stop();

    System.out.println("Summary");
    System.out.println(
        "   -- Readers ["
            + searcherThreads.length
            + "] with ["
            + searcherIterations
            + "] iterations");
    System.out.println(
        "   -- Writers [" + writerThreads.length + "] with [" + writerIterations + "] iterations");
    System.out.println("   -- Took: " + stopWatch.totalTime());
    System.out.println(
        "   -- Refresh [" + refresher.id + "] took: " + refresher.stopWatch.totalTime());
    System.out.println("   -- Flush [" + flusher.id + "] took: " + flusher.stopWatch.totalTime());
    System.out.println("   -- Store size " + store.estimateSize());

    scheduledExecutorService.shutdown();

    engine.refresh(new Engine.Refresh(true));
    stopWatch = new StopWatch();
    stopWatch.start();
    Engine.Searcher searcher = engine.searcher();
    TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), idGenerator.get() + 1);
    stopWatch.stop();
    System.out.println(
        "   -- Indexed ["
            + idGenerator.get()
            + "] docs, found ["
            + topDocs.totalHits
            + "] hits, took "
            + stopWatch.totalTime());
    searcher.release();
  }
Exemple #4
0
  protected void handleTaskSubmittedRequest(
      Runnable runnable, Address source, long requestId, long threadId) {
    // We store in our map so that when that task is
    // finished so that we can send back to the owner
    // with the results
    _running.put(runnable, new Owner(source, requestId));
    // We give the task to the thread that is now waiting for it to be returned
    // If we can't offer then we have to respond back to
    // caller that we can't handle it.  They must have
    // gotten our address when we had a consumer, but
    // they went away between then and now.
    boolean received;
    try {
      _tasks.put(threadId, runnable);

      CyclicBarrier barrier = _taskBarriers.remove(threadId);
      if (received = (barrier != null)) {
        // Only wait 10 milliseconds, in case if the consumer was
        // stopped between when we were told it was available and now
        barrier.await(10, TimeUnit.MILLISECONDS);
      }
    } catch (InterruptedException e) {
      if (log.isDebugEnabled()) log.debug("Interrupted while handing off task");
      Thread.currentThread().interrupt();
      received = false;
    } catch (BrokenBarrierException e) {
      if (log.isDebugEnabled())
        log.debug(
            "Consumer " + threadId + " has been interrupted, " + "must retry to submit elsewhere");
      received = false;
    } catch (TimeoutException e) {
      if (log.isDebugEnabled())
        log.debug("Timeout waiting to hand off to barrier, consumer " + threadId + " must be slow");
      // This should only happen if the consumer put the latch then got
      // interrupted but hadn't yet removed the latch, should almost never
      // happen
      received = false;
    }

    if (!received) {
      // Clean up the tasks request
      _tasks.remove(threadId);
      if (log.isDebugEnabled()) log.debug("Run rejected not able to pass off to consumer");
      // If we couldn't hand off the task we have to tell the client
      // and also reupdate the coordinator that our consumer is ready
      sendRequest(source, Type.RUN_REJECTED, requestId, null);
      _running.remove(runnable);
    }
  }
  @Test(timeout = 20000l)
  @Ignore
  public void testLockWorksUnderContentionDifferentClients() throws Exception {
    int numThreads = 5;
    final int numIterations = 100;
    ExecutorService service = Executors.newFixedThreadPool(numThreads);
    final CyclicBarrier startBarrier = new CyclicBarrier(numThreads + 1);
    final CyclicBarrier endBarrier = new CyclicBarrier(numThreads + 1);

    final UnsafeOperator operator = new UnsafeOperator();
    for (int i = 0; i < numThreads; i++) {
      service.submit(
          new Callable<Void>() {
            @Override
            public Void call() throws Exception {
              startBarrier.await(); // make sure all threads are in the same place before starting

              // create the lock that we're going to use
              ZooKeeper newZk = newZooKeeper();
              try {
                Lock testLock = new ReentrantZkLock(baseLockPath, new BaseZkSessionManager(newZk));
                for (int j = 0; j < numIterations; j++) {
                  testLock.lock();
                  try {
                    operator.increment();
                  } finally {
                    testLock.unlock();
                  }
                }
              } finally {
                closeZooKeeper(newZk);
              }
              // enter the end barrier to ensure that things are finished
              endBarrier.await();
              return null;
            }
          });
    }

    // start the test
    startBarrier.await();

    // wait for the end of the test
    endBarrier.await();

    // check that the number of operations that actually were recorded are correct
    int correctOps = numIterations * numThreads;
    assertEquals("Number of Operations recorded was incorrect!", correctOps, operator.getValue());
  }
    final void test() throws Exception {
      Future[] futures = new Future[nthreads];
      for (int i = 0; i < nthreads; ++i) futures[i] = pool.submit(this);

      barrier.await();
      Thread.sleep(TIMEOUT);
      boolean tooLate = false;
      for (int i = 1; i < nthreads; ++i) {
        if (!futures[i].cancel(true)) tooLate = true;
        // Unbunch some of the cancels
        if ((i & 3) == 0) Thread.sleep(1 + rng.next() % 10);
      }

      Object f0 = futures[0].get();
      if (!tooLate) {
        for (int i = 1; i < nthreads; ++i) {
          if (!futures[i].isDone() || !futures[i].isCancelled())
            throw new Error("Only one thread should complete");
        }
      } else System.out.print("(cancelled too late) ");

      long endTime = System.nanoTime();
      long time = endTime - timer.startTime;
      if (print) {
        double secs = (double) (time) / 1000000000.0;
        System.out.println("\t " + secs + "s run time");
      }
    }
 public Object call() throws Exception {
   _barrier.await(); // barrier, to force racing start
   for (long j = 0; j < _count; j++)
     _map.put(
         j + _offset,
         new TestKey(_rand.nextLong(), _rand.nextInt(), (short) _rand.nextInt(Short.MAX_VALUE)));
   return null;
 }
  /**
   * Testing threads thoroughly seems to be a lot of fun, I am not sure yet how to properly do that.
   * This test tests priority order and total sum received is equal to total sum sent. It also
   * simulates clients random hanging.
   */
  @Test
  public void testReceiveData() {

    DecorateCheckOrderAndCountSumMarshaller sumAppender =
        new DecorateCheckOrderAndCountSumMarshaller();
    QueueWorker worker = new QueueWorker(sumAppender);
    Thread workerThread = new Thread(worker);

    // run 20 clients, 10.000 items will be generated per client as defined in
    // src/test/properties/app.properties
    for (int i = 0; i < 20; i++) {
      Runnable clientHangSimulator = null;
      if (i % 5 == 0) {
        // simulate occasional client hang for four of the total twenty clients and check worker is
        // not biased.
        clientHangSimulator =
            () -> {
              if (ThreadLocalRandom.current().nextInt(1001) % 1000 == 0) {
                try {
                  Thread.sleep(500L);
                } catch (InterruptedException e) {
                  Thread.currentThread().interrupt();
                }
              }
            };
      }
      executorService.execute(new DataGeneratorTask(clientHangSimulator));
    }

    workerThread.start();

    try {
      barrier.await();
      System.out.println("Fired test");
      Thread.sleep(1000);
      executorService.shutdown();
      // runs actually much faster, may need update if item count per client is drastically
      // increased in app.properties
      executorService.awaitTermination(2 * 60, TimeUnit.SECONDS);
      System.out.println("exe service awaited");
    } catch (InterruptedException | BrokenBarrierException e) {
      e.printStackTrace();
    }

    try {
      workerThread.join(Long.MAX_VALUE);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }

    Assert.assertEquals(
        "Sum generated does not match sum received",
        sumAppender.getSum(),
        AppContext.getInstance().getTotalGeneratedAmount());
    Assert.assertFalse("Worker didn't exit successfully", workerThread.isAlive());
  }
  @Test(timeout = 1000l)
  public void testNoReadsWithSingleWrite() throws Exception {
    ReadWriteLock lock = new ReentrantZkReadWriteLock(testPath, zkSessionManager);
    final CyclicBarrier waitBarrier = new CyclicBarrier(2);

    final Lock readLock = lock.readLock();
    final Lock writeLock = lock.writeLock();

    final CountDownLatch latch = new CountDownLatch(1);
    Future<Void> future =
        testService.submit(
            new Callable<Void>() {
              @Override
              public Void call() throws Exception {
                waitBarrier.await();

                readLock.lock();
                try {
                  latch.countDown();
                } finally {
                  readLock.unlock();
                }
                return null; // To change body of implemented methods use File | Settings | File
                             // Templates.
              }
            });

    writeLock.lock();
    waitBarrier.await();

    boolean acquired = latch.await(250, TimeUnit.MILLISECONDS);
    assertTrue("The Read lock was improperly acquired!", !acquired);

    // unlock the read lock, and make sure that the write lock is acquired

    writeLock.unlock();
    acquired = latch.await(300, TimeUnit.MILLISECONDS);
    assertTrue("The Read lock was never acquired!", acquired);

    // make sure no exceptions were thrown
    future.get();
  }
 @Override
 public void run() {
   try {
     LongValue value = new LongValueNative();
     barrier.await();
     for (int i = 0; i < iterations; i++) {
       map.acquireUsing(key, value);
       value.addAtomicValue(1);
     }
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
 public void run() {
   try {
     while (!Thread.interrupted()) {
       synchronized (this) {
         strides += rand.nextInt(3); // Produces 0, 1 or 2
       }
       barrier.await();
     }
   } catch (InterruptedException e) {
     // A legitimate way to exit
   } catch (BrokenBarrierException e) {
     // This one we want to know about
     throw new RuntimeException(e);
   }
 }
 public final Object call() throws Exception {
   barrier.await();
   int sum = v;
   int x = 0;
   int n = ITERS;
   while (n-- > 0) {
     lock.lockInterruptibly();
     try {
       v = x = LoopHelpers.compute1(v);
     } finally {
       lock.unlock();
     }
     sum += LoopHelpers.compute2(LoopHelpers.compute2(x));
   }
   return new Integer(sum);
 }
 public Integer call() throws Exception {
   _barrier.await();
   int count = 0;
   for (TestKey item : _items) {
     if (_map.contains(item._id)) {
       System.err.printf("COLLISION DETECTED: %s exists\n", item.toString());
     }
     final TestKey exists = _map.putIfAbsent(item._id, item);
     if (exists == null) {
       count++;
     } else {
       System.err.printf(
           "COLLISION DETECTED: %s exists as %s\n", item.toString(), exists.toString());
     }
   }
   return count;
 }
Exemple #14
0
 public void run() {
   try {
     while (!Thread.interrupted()) {
       // Blocks until chassis is available:
       car = chassisQueue.take();
       // Hire robots to perform work:
       robotPool.hire(EngineRobot.class, this);
       robotPool.hire(DriveTrainRobot.class, this);
       robotPool.hire(WheelRobot.class, this);
       barrier.await(); // Until the robots finish
       // Put car into finishingQueue for further work
       finishingQueue.put(car);
     }
   } catch (InterruptedException e) {
     print("Exiting Assembler via interrupt");
   } catch (BrokenBarrierException e) {
     // This one we want to know about
     throw new RuntimeException(e);
   }
   print("Assembler off");
 }
 @Override
 public void run() {
   AppContext appContext = AppContext.getInstance();
   try {
     barrier.await();
   } catch (InterruptedException | BrokenBarrierException e) {
     e.printStackTrace();
     Thread.currentThread().interrupt();
   }
   for (int i = 0; i < AppContext.getInstance().getGeneratedItemCountPerConnection(); i++) {
     if (hangingSimulator != null) {
       hangingSimulator.run();
     }
     lastTime += ThreadLocalRandom.current().nextInt(100);
     BigDecimal amount = new BigDecimal(ThreadLocalRandom.current().nextInt(100));
     Item item = new Item(lastTime, amount);
     itemConsumer.accept(item);
     appContext.getClientRegistry().registerLastClientTime(id, lastTime);
     appContext.addToTotalGeneratedAmount(amount);
   }
   appContext.getClientRegistry().deregisterClient(id);
   System.out.println(MessageFormat.format("Client id {0} exited successfully.", id));
   System.out.println("appContext = " + appContext.getWorkQueue().size());
 }
Exemple #16
0
 static void task(CyclicBarrier cb) throws InterruptedException, BrokenBarrierException {
   out.println("first part of the task");
   cb.await();
   out.println("second part of the task");
 }
Exemple #17
0
  public Object down(Event evt) {
    switch (evt.getType()) {
      case ExecutorEvent.TASK_SUBMIT:
        Runnable runnable = evt.getArg();
        // We are limited to a number of concurrent request id's
        // equal to 2^63-1.  This is quite large and if it
        // overflows it will still be positive
        long requestId = Math.abs(counter.getAndIncrement());
        if (requestId == Long.MIN_VALUE) {
          // TODO: need to fix this it isn't safe for concurrent modifications
          counter.set(0);
          requestId = Math.abs(counter.getAndIncrement());
        }

        // Need to make sure to put the requestId in our map before
        // adding the runnable to awaiting consumer in case if
        // coordinator sent a consumer found and their original task
        // is no longer around
        // see https://issues.jboss.org/browse/JGRP-1744
        _requestId.put(runnable, requestId);

        _awaitingConsumer.add(runnable);

        sendToCoordinator(RUN_REQUEST, requestId, local_addr);
        break;
      case ExecutorEvent.CONSUMER_READY:
        Thread currentThread = Thread.currentThread();
        long threadId = currentThread.getId();
        _consumerId.put(threadId, PRESENT);
        try {
          for (; ; ) {
            CyclicBarrier barrier = new CyclicBarrier(2);
            _taskBarriers.put(threadId, barrier);

            // We only send to the coordinator that we are ready after
            // making the barrier, wait for request to come and let
            // us free
            sendToCoordinator(Type.CONSUMER_READY, threadId, local_addr);

            try {
              barrier.await();
              break;
            } catch (BrokenBarrierException e) {
              if (log.isDebugEnabled())
                log.debug(
                    "Producer timed out before we picked up"
                        + " the task, have to tell coordinator"
                        + " we are still good.");
            }
          }
          // This should always be non nullable since the latch
          // was freed
          runnable = _tasks.remove(threadId);
          _runnableThreads.put(runnable, currentThread);
          return runnable;
        } catch (InterruptedException e) {
          if (log.isDebugEnabled()) log.debug("Consumer " + threadId + " stopped via interrupt");
          sendToCoordinator(Type.CONSUMER_UNREADY, threadId, local_addr);
          Thread.currentThread().interrupt();
        } finally {
          // Make sure the barriers are cleaned up as well
          _taskBarriers.remove(threadId);
          _consumerId.remove(threadId);
        }
        break;
      case ExecutorEvent.TASK_COMPLETE:
        Object arg = evt.getArg();
        Throwable throwable = null;
        if (arg instanceof Object[]) {
          Object[] array = (Object[]) arg;
          runnable = (Runnable) array[0];
          throwable = (Throwable) array[1];
        } else {
          runnable = (Runnable) arg;
        }
        Owner owner = _running.remove(runnable);
        // This won't remove anything if owner doesn't come back
        _runnableThreads.remove(runnable);

        Object value = null;
        boolean exception = false;
        if (throwable != null) {
          // InterruptedException is special telling us that
          // we interrupted the thread while waiting but still got
          // a task therefore we have to reject it.
          if (throwable instanceof InterruptedException) {
            if (log.isDebugEnabled())
              log.debug("Run rejected due to interrupted exception returned");
            sendRequest(owner.address, Type.RUN_REJECTED, owner.requestId, null);
            break;
          }
          value = throwable;
          exception = true;
        } else if (runnable instanceof RunnableFuture<?>) {
          RunnableFuture<?> future = (RunnableFuture<?>) runnable;

          boolean interrupted = false;
          boolean gotValue = false;

          // We have the value, before we interrupt at least get it!
          while (!gotValue) {
            try {
              value = future.get();
              gotValue = true;
            } catch (InterruptedException e) {
              interrupted = true;
            } catch (ExecutionException e) {
              value = e.getCause();
              exception = true;
              gotValue = true;
            }
          }

          if (interrupted) {
            Thread.currentThread().interrupt();
          }
        }

        if (owner != null) {
          final Type type;
          final Object valueToSend;
          if (value == null) {
            type = Type.RESULT_SUCCESS;
            valueToSend = value;
          }
          // Both serializable values and exceptions would go in here
          else if (value instanceof Serializable
              || value instanceof Externalizable
              || value instanceof Streamable) {
            type = exception ? Type.RESULT_EXCEPTION : Type.RESULT_SUCCESS;
            valueToSend = value;
          }
          // This would happen if the value wasn't serializable,
          // so we have to send back to the client that the class
          // wasn't serializable
          else {
            type = Type.RESULT_EXCEPTION;
            valueToSend = new NotSerializableException(value.getClass().getName());
          }

          if (local_addr.equals(owner.getAddress())) {
            if (log.isTraceEnabled())
              log.trace(
                  "[redirect] <--> ["
                      + local_addr
                      + "] "
                      + type.name()
                      + " ["
                      + value
                      + (owner.requestId != -1 ? " request id: " + owner.requestId : "")
                      + "]");
            if (type == Type.RESULT_SUCCESS) {
              handleValueResponse(local_addr, owner.requestId, valueToSend);
            } else if (type == Type.RESULT_EXCEPTION) {
              handleExceptionResponse(local_addr, owner.requestId, (Throwable) valueToSend);
            }
          } else {
            sendRequest(owner.getAddress(), type, owner.requestId, valueToSend);
          }
        } else {
          if (log.isTraceEnabled()) {
            log.trace("Could not return result - most likely because it was interrupted");
          }
        }
        break;
      case ExecutorEvent.TASK_CANCEL:
        Object[] array = evt.getArg();
        runnable = (Runnable) array[0];

        if (_awaitingConsumer.remove(runnable)) {
          _requestId.remove(runnable);
          ExecutorNotification notification = notifiers.remove(runnable);
          if (notification != null) {
            notification.interrupted(runnable);
          }
          if (log.isTraceEnabled())
            log.trace("Cancelled task " + runnable + " before it was picked up");
          return Boolean.TRUE;
        }
        // This is guaranteed to not be null so don't take cost of auto unboxing
        else if (array[1] == Boolean.TRUE) {
          owner = removeKeyForValue(_awaitingReturn, runnable);
          if (owner != null) {
            Long requestIdValue = _requestId.remove(runnable);
            // We only cancel if the requestId is still available
            // this means the result hasn't been returned yet and
            // we still have a chance to interrupt
            if (requestIdValue != null) {
              if (requestIdValue != owner.getRequestId()) {
                log.warn("Cancelling requestId didn't match waiting");
              }
              sendRequest(owner.getAddress(), Type.INTERRUPT_RUN, owner.getRequestId(), null);
            }
          } else {
            if (log.isTraceEnabled()) log.warn("Couldn't interrupt server task: " + runnable);
          }
          ExecutorNotification notification = notifiers.remove(runnable);
          if (notification != null) {
            notification.interrupted(runnable);
          }
          return Boolean.TRUE;
        } else {
          return Boolean.FALSE;
        }
      case ExecutorEvent.ALL_TASK_CANCEL:
        array = evt.getArg();

        // This is a RunnableFuture<?> so this cast is okay
        @SuppressWarnings("unchecked")
        Set<Runnable> runnables = (Set<Runnable>) array[0];
        Boolean booleanValue = (Boolean) array[1];

        List<Runnable> notRan = new ArrayList<>();

        for (Runnable cancelRunnable : runnables) {
          // Removed from the consumer
          if (!_awaitingConsumer.remove(cancelRunnable) && booleanValue == Boolean.TRUE) {
            synchronized (_awaitingReturn) {
              owner = removeKeyForValue(_awaitingReturn, cancelRunnable);
              if (owner != null) {
                Long requestIdValue = _requestId.remove(cancelRunnable);
                if (requestIdValue != owner.getRequestId()) {
                  log.warn("Cancelling requestId didn't match waiting");
                }
                sendRequest(owner.getAddress(), Type.INTERRUPT_RUN, owner.getRequestId(), null);
              }
              ExecutorNotification notification = notifiers.remove(cancelRunnable);
              if (notification != null) {
                log.trace("Notifying listener");
                notification.interrupted(cancelRunnable);
              }
            }
          } else {
            _requestId.remove(cancelRunnable);
            notRan.add(cancelRunnable);
          }
        }
        return notRan;
      case Event.SET_LOCAL_ADDRESS:
        local_addr = evt.getArg();
        break;

      case Event.VIEW_CHANGE:
        handleView(evt.getArg());
        break;
    }
    return down_prot.down(evt);
  }
  @Test(timeout = 20000)
  public void testUpdatingThreadPoolSettings() throws Exception {
    internalCluster().startNodesAsync(2).get();
    ThreadPool threadPool = internalCluster().getDataNodeInstance(ThreadPool.class);
    // Check that settings are changed
    assertThat(
        ((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES),
        equalTo(5L));
    client()
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setTransientSettings(settingsBuilder().put("threadpool.search.keep_alive", "10m").build())
        .execute()
        .actionGet();
    assertThat(
        ((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getKeepAliveTime(TimeUnit.MINUTES),
        equalTo(10L));

    // Make sure that threads continue executing when executor is replaced
    final CyclicBarrier barrier = new CyclicBarrier(2);
    Executor oldExecutor = threadPool.executor(Names.SEARCH);
    threadPool
        .executor(Names.SEARCH)
        .execute(
            new Runnable() {
              @Override
              public void run() {
                try {
                  barrier.await();
                } catch (InterruptedException ex) {
                  Thread.currentThread().interrupt();
                } catch (BrokenBarrierException ex) {
                  //
                }
              }
            });
    client()
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setTransientSettings(settingsBuilder().put("threadpool.search.type", "fixed").build())
        .execute()
        .actionGet();
    assertThat(threadPool.executor(Names.SEARCH), not(sameInstance(oldExecutor)));
    assertThat(((ThreadPoolExecutor) oldExecutor).isShutdown(), equalTo(true));
    assertThat(((ThreadPoolExecutor) oldExecutor).isTerminating(), equalTo(true));
    assertThat(((ThreadPoolExecutor) oldExecutor).isTerminated(), equalTo(false));
    barrier.await();

    // Make sure that new thread executor is functional
    threadPool
        .executor(Names.SEARCH)
        .execute(
            new Runnable() {
              @Override
              public void run() {
                try {
                  barrier.await();
                } catch (InterruptedException ex) {
                  Thread.currentThread().interrupt();
                } catch (BrokenBarrierException ex) {
                  //
                }
              }
            });
    client()
        .admin()
        .cluster()
        .prepareUpdateSettings()
        .setTransientSettings(settingsBuilder().put("threadpool.search.type", "fixed").build())
        .execute()
        .actionGet();
    barrier.await();
    Thread.sleep(200);

    // Check that node info is correct
    NodesInfoResponse nodesInfoResponse =
        client().admin().cluster().prepareNodesInfo().all().execute().actionGet();
    for (int i = 0; i < 2; i++) {
      NodeInfo nodeInfo = nodesInfoResponse.getNodes()[i];
      boolean found = false;
      for (ThreadPool.Info info : nodeInfo.getThreadPool()) {
        if (info.getName().equals(Names.SEARCH)) {
          assertThat(info.getType(), equalTo("fixed"));
          found = true;
          break;
        }
      }
      assertThat(found, equalTo(true));

      Map<String, Object> poolMap =
          getPoolSettingsThroughJson(nodeInfo.getThreadPool(), Names.SEARCH);
    }
  }