@Test
 public void testEnlargePool() throws Exception {
   AbstractClientConnectionFactory factory = mock(AbstractClientConnectionFactory.class);
   when(factory.isRunning()).thenReturn(true);
   TcpConnectionSupport mockConn1 = makeMockConnection("conn1");
   TcpConnectionSupport mockConn2 = makeMockConnection("conn2");
   TcpConnectionSupport mockConn3 = makeMockConnection("conn3");
   TcpConnectionSupport mockConn4 = makeMockConnection("conn4");
   when(factory.getConnection()).thenReturn(mockConn1, mockConn2, mockConn3, mockConn4);
   CachingClientConnectionFactory cachingFactory = new CachingClientConnectionFactory(factory, 2);
   cachingFactory.start();
   TcpConnection conn1 = cachingFactory.getConnection();
   TcpConnection conn2 = cachingFactory.getConnection();
   assertNotSame(conn1, conn2);
   Semaphore semaphore =
       TestUtils.getPropertyValue(
           TestUtils.getPropertyValue(cachingFactory, "pool"), "permits", Semaphore.class);
   assertEquals(0, semaphore.availablePermits());
   cachingFactory.setPoolSize(4);
   TcpConnection conn3 = cachingFactory.getConnection();
   TcpConnection conn4 = cachingFactory.getConnection();
   assertEquals(0, semaphore.availablePermits());
   conn1.close();
   conn1.close();
   conn2.close();
   conn3.close();
   conn4.close();
   assertEquals(4, semaphore.availablePermits());
 }
  /**
   * This method checks if the reader thread has finished, and re-throw any exceptions thrown by the
   * reader thread.
   *
   * @throws SqoopException if the consumer thread threw it.
   * @throws RuntimeException if some other exception was thrown.
   */
  private void waitForConsumer() {
    try {
      consumerFuture.get();
    } catch (ExecutionException ex) {
      // In almost all cases, the exception will be SqoopException,
      // because all exceptions are caught and propagated as
      // SqoopExceptions

      // There are race conditions with exceptions where the free sema is
      // no released.  So sense we are in single threaded mode at this point
      // we can ask if there are availablePermits and release if needed
      if (free.availablePermits() == 0) {
        free.release();
      }

      Throwable t = ex.getCause();
      if (t instanceof SqoopException) {
        throw (SqoopException) t;
      }
      // In the rare case, it was not a SqoopException
      Throwables.propagate(t);
    } catch (Exception ex) {

      // There are race conditions with exceptions where the free sema is
      // no released.  So sense we are in single threaded mode at this point
      // we can ask if there are availablePermits and release if needed
      if (free.availablePermits() == 0) {
        free.release();
      }

      throw new SqoopException(SparkExecutionError.SPARK_EXEC_0019, ex);
    }
  }
Example #3
0
 @Override
 public String call() throws Exception {
   semaphore.acquire();
   System.out.println(semaphore.availablePermits());
   Thread.currentThread().sleep(1000);
   barrier.await();
   semaphore.release();
   System.out.println(semaphore.availablePermits());
   return "true";
 }
 private void waitForOutstandingRequests(
     TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
   long start = System.currentTimeMillis();
   do {
     long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
     logger.info(
         "[{}] going to try and aquire [{}] in [{}]ms [{}] available to aquire right now",
         name,
         maxRequests,
         msRemaining,
         requestsOutstanding.availablePermits());
     try {
       requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
       return;
     } catch (InterruptedException ie) {
       // Just keep swimming
     }
   } while ((System.currentTimeMillis() - start) < timeOut.getMillis());
   throw new ElasticsearchTimeoutException(
       "Requests were still outstanding after the timeout ["
           + timeOut
           + "] for type ["
           + name
           + "]");
 }
  @Test
  public void testCancelled() throws InterruptedException, ExecutionException {
    Task<List<?>> t =
        Tasks.sequential(sayTask("1"), sayTask("2a", Duration.THIRTY_SECONDS, "2b"), sayTask("3"));
    ec.submit(t);
    synchronized (messages) {
      while (messages.size() <= 1) messages.wait();
    }
    Assert.assertEquals(messages, Arrays.asList("1", "2a"));
    Time.sleep(Duration.millis(50));
    t.cancel(true);
    Assert.assertTrue(t.isDone());
    // 2 should get cancelled, and invoke the cancellation semaphore
    // 3 should get cancelled and not run at all
    Assert.assertEquals(messages, Arrays.asList("1", "2a"));

    // Need to ensure that 2 has been started; race where we might cancel it before its run method
    // is even begun. Hence doing "2a; pause; 2b" where nothing is interruptable before pause.
    Assert.assertTrue(cancellations.tryAcquire(10, TimeUnit.SECONDS));

    Iterator<Task<?>> ci = ((HasTaskChildren) t).getChildren().iterator();
    Assert.assertEquals(ci.next().get(), "1");
    Task<?> task2 = ci.next();
    Assert.assertTrue(task2.isBegun());
    Assert.assertTrue(task2.isDone());
    Assert.assertTrue(task2.isCancelled());

    Task<?> task3 = ci.next();
    Assert.assertFalse(task3.isBegun());
    Assert.assertTrue(task2.isDone());
    Assert.assertTrue(task2.isCancelled());

    // but we do _not_ get a mutex from task3 as it does not run (is not interrupted)
    Assert.assertEquals(cancellations.availablePermits(), 0);
  }
  @Test
  public void testShutdownLeavesJobRunning() throws InterruptedException {
    OutOfBandScheduledExecutor scheduler = new OutOfBandScheduledExecutor();
    ExecutorService worker = Executors.newSingleThreadExecutor();
    try {
      PartitionedScheduledExecutor executor = new PartitionedScheduledExecutor(scheduler, worker);

      final Semaphore semaphore = new Semaphore(0);
      executor.execute(
          new Runnable() {

            @Override
            public void run() {
              semaphore.acquireUninterruptibly();
            }
          });
      executor.shutdown();
      assertThat(executor.awaitTermination(100, MILLISECONDS), is(false));
      assertThat(executor.isShutdown(), is(true));
      assertThat(executor.isTerminated(), is(false));

      semaphore.release();
      assertThat(executor.awaitTermination(2, MINUTES), is(true));
      assertThat(executor.isShutdown(), is(true));
      assertThat(executor.isTerminated(), is(true));

      assertThat(semaphore.availablePermits(), is(0));
    } finally {
      worker.shutdown();
    }
  }
 private void logState() {
   logger.debug(
       "Available permits: {} thread queue: {} state: {} threshold: {}",
       new Object[] {
         lock.availablePermits(), lock.getQueueLength(), counter.get(), getThreshold()
       });
 }
  protected void smartStepInto(String funcName) {
    XDebugSession currentSession = XDebuggerManager.getInstance(getProject()).getCurrentSession();

    Assert.assertTrue(currentSession.isSuspended());
    Assert.assertEquals(0, myPausedSemaphore.availablePermits());

    myDebugProcess.startSmartStepInto(funcName);
  }
  protected void stepOver() {
    XDebugSession currentSession = XDebuggerManager.getInstance(getProject()).getCurrentSession();

    Assert.assertTrue(currentSession.isSuspended());
    Assert.assertEquals(0, myPausedSemaphore.availablePermits());

    currentSession.stepOver(false);
  }
Example #10
0
  // make sure that log isn't needlessly replayed after a clean close
  @Test
  public void testCleanShutdown() throws Exception {
    DirectUpdateHandler2.commitOnClose = true;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);

    UpdateLog.testing_logReplayHook =
        new Runnable() {
          @Override
          public void run() {
            try {
              assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
            } catch (Exception e) {
              throw new RuntimeException(e);
            }
          }
        };

    UpdateLog.testing_logReplayFinishHook =
        new Runnable() {
          @Override
          public void run() {
            logReplayFinish.release();
          }
        };

    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();

    try {
      clearIndex();
      assertU(commit());

      assertU(adoc("id", "E1", "val_i", "1"));
      assertU(adoc("id", "E2", "val_i", "1"));

      // set to a high enough number so this test won't hang on a bug
      logReplay.release(10);

      h.close();
      createCore();

      // make sure the docs got committed
      assertJQ(req("q", "*:*"), "/response/numFound==2");

      // make sure no replay happened
      assertEquals(10, logReplay.availablePermits());

    } finally {
      DirectUpdateHandler2.commitOnClose = true;
      UpdateLog.testing_logReplayHook = null;
      UpdateLog.testing_logReplayFinishHook = null;

      req().close();
    }
  }
  protected void stepIntoMyCode() {
    XDebugSession currentSession = XDebuggerManager.getInstance(getProject()).getCurrentSession();

    Assert.assertTrue(currentSession.isSuspended());
    Assert.assertEquals(0, myPausedSemaphore.availablePermits());

    PyDebugProcess debugProcess = (PyDebugProcess) currentSession.getDebugProcess();
    debugProcess.startStepIntoMyCode();
  }
 private synchronized void assertInnerState() {
   if (activeConnections < 0) {
     throw new AssertionError();
   }
   if (activeConnections + recycledConnections.size() > maxConnections) {
     throw new AssertionError();
   }
   if (activeConnections + semaphore.availablePermits() > maxConnections) {
     throw new AssertionError();
   }
 }
 @Test
 public void testReducePool() throws Exception {
   AbstractClientConnectionFactory factory = mock(AbstractClientConnectionFactory.class);
   when(factory.isRunning()).thenReturn(true);
   TcpConnectionSupport mockConn1 = makeMockConnection("conn1", true);
   TcpConnectionSupport mockConn2 = makeMockConnection("conn2", true);
   TcpConnectionSupport mockConn3 = makeMockConnection("conn3", true);
   TcpConnectionSupport mockConn4 = makeMockConnection("conn4", true);
   when(factory.getConnection())
       .thenReturn(mockConn1)
       .thenReturn(mockConn2)
       .thenReturn(mockConn3)
       .thenReturn(mockConn4);
   CachingClientConnectionFactory cachingFactory = new CachingClientConnectionFactory(factory, 4);
   cachingFactory.start();
   TcpConnection conn1 = cachingFactory.getConnection();
   TcpConnection conn2 = cachingFactory.getConnection();
   TcpConnection conn3 = cachingFactory.getConnection();
   TcpConnection conn4 = cachingFactory.getConnection();
   Semaphore semaphore =
       TestUtils.getPropertyValue(
           TestUtils.getPropertyValue(cachingFactory, "pool"), "permits", Semaphore.class);
   assertEquals(0, semaphore.availablePermits());
   conn1.close();
   assertEquals(1, semaphore.availablePermits());
   cachingFactory.setPoolSize(2);
   assertEquals(0, semaphore.availablePermits());
   assertEquals(3, cachingFactory.getActiveCount());
   conn2.close();
   assertEquals(0, semaphore.availablePermits());
   assertEquals(2, cachingFactory.getActiveCount());
   conn3.close();
   assertEquals(1, cachingFactory.getActiveCount());
   assertEquals(1, cachingFactory.getIdleCount());
   conn4.close();
   assertEquals(2, semaphore.availablePermits());
   assertEquals(0, cachingFactory.getActiveCount());
   assertEquals(2, cachingFactory.getIdleCount());
   verify(mockConn1).close();
   verify(mockConn2).close();
 }
Example #14
0
  public void writeConfig() throws ExecutionException, TimeoutException, InterruptedException {
    long startTime = System.currentTimeMillis();

    FutureCallback<None> callback = new FutureCallback<None>();
    _store.start(callback);
    callback.get(_timeout, _timeoutUnit);

    final Semaphore outstandingPutSemaphore = new Semaphore(_maxOutstandingWrites);

    for (final String key : _source.keySet()) {

      Map<String, Object> map = merge(_source.get(key), _defaultMap);

      T properties = _builder.fromMap(map);
      Callback<None> putCallback =
          new Callback<None>() {
            @Override
            public void onSuccess(None none) {
              outstandingPutSemaphore.release();
            }

            @Override
            public void onError(Throwable e) {
              _log.error("Put failed for {}", key, e);
              outstandingPutSemaphore.release();
            }
          };

      if (!outstandingPutSemaphore.tryAcquire(_timeout, _timeoutUnit)) {
        _log.error("Put timed out for {}", key);
        throw new TimeoutException();
      }
      _store.put(key, properties, putCallback);
    }

    // Wait until all puts are finished.
    if (!outstandingPutSemaphore.tryAcquire(_maxOutstandingWrites, _timeout, _timeoutUnit)) {
      _log.error(
          "Put timed out with {} outstanding writes",
          _maxOutstandingWrites - outstandingPutSemaphore.availablePermits());
      throw new TimeoutException();
    }

    FutureCallback<None> shutdownCallback = new FutureCallback<None>();
    _store.shutdown(shutdownCallback);
    shutdownCallback.get(_timeout, _timeoutUnit);

    long elapsedTime = System.currentTimeMillis() - startTime;
    _log.info(
        "A total of {}.{}s elapsed to write configs to store.",
        elapsedTime / 1000,
        elapsedTime % 1000);
  }
Example #15
0
  public synchronized void ignore(boolean b) {
    if (b == (0 == ignore.availablePermits())) return;

    if (0 == ignore.availablePermits()) {
      ignore.release();
      serverIDs.add(new Integer(port));
    } else {
      ignore.acquireUninterruptibly();

      if (srv != null) {
        try {
          srv.close();
        } catch (IOException e) {
        }
        srv = null;
      }
      if (connecting != null) connecting.disconnect();

      serverIDs.remove(new Integer(port));
    }
  }
  public void inicializar() {
    ImagemMapa im = imagemMapaService.get(Integer.valueOf(id));
    // Se for QR Code: incrementa.
    if (qr != null && qr.booleanValue()) {
      try {
        qrCodeViewedSemaphore.acquire();
        im.setViewedQrCode(im.getViewedQrCode() + 1);
        imagemMapaService.save(im);
      } catch (InterruptedException e) {
        e.printStackTrace();
      } finally {
        if (qrCodeViewedSemaphore.availablePermits() == 0) qrCodeViewedSemaphore.release();
      }
    }

    Map<String, Object> param = getParams(im);
    List<BaseObject> crimes = crimeService.filter(param);

    // contagem de crimes
    int contRoubos = 0;
    int contFurtos = 0;
    for (BaseObject o : crimes) {
      if (o instanceof Crime) {
        Crime c = (Crime) o;
        String tipo = c.getTipoCrime().getNome();
        if (tipo.equals("tipocrime.roubo")) contRoubos++;
        else if (tipo.equals("tipocrime.furto")) contFurtos++;
      }
    }

    numRoubos = contRoubos + "";
    numFurtos = contFurtos + "";
    numOutros = (crimes.size() - contFurtos - contRoubos) + "";

    // datas
    DateFormat f1 = new SimpleDateFormat("dd,MM,yyyy");
    DateFormat f2 = new SimpleDateFormat("dd/MM/yy");
    String dataIniStr = im.getFiltro().get("dataInicial");
    String dataFimStr = im.getFiltro().get("dataFinal");
    if (dataIniStr == null) dataIniStr = "01,01,1970";
    if (dataFimStr == null) dataFimStr = "01,01,1970";
    try {
      dataIni = f2.format(f1.parse(dataIniStr));
      dataFin = f2.format(f1.parse(dataFimStr));
    } catch (ParseException e) {
      e.printStackTrace();
    }

    // nome do usu�rio
    nomeUsuario = im.getUsuario().getNome();
  }
  @Test
  public void testQueuedJobRunsAfterShutdown() throws InterruptedException {
    OutOfBandScheduledExecutor scheduler = new OutOfBandScheduledExecutor();
    ExecutorService worker = Executors.newSingleThreadExecutor();
    try {
      PartitionedScheduledExecutor executor = new PartitionedScheduledExecutor(scheduler, worker);

      final Semaphore jobSemaphore = new Semaphore(0);
      final Semaphore testSemaphore = new Semaphore(0);

      executor.submit(
          new Callable<Void>() {

            @Override
            public Void call() throws Exception {
              testSemaphore.release();
              jobSemaphore.acquireUninterruptibly();
              return null;
            }
          });
      executor.submit(
          new Callable<Void>() {

            @Override
            public Void call() throws Exception {
              jobSemaphore.acquireUninterruptibly();
              return null;
            }
          });
      testSemaphore.acquireUninterruptibly();
      executor.shutdown();
      assertThat(executor.awaitTermination(100, MILLISECONDS), is(false));
      assertThat(executor.isShutdown(), is(true));
      assertThat(executor.isTerminated(), is(false));

      jobSemaphore.release();
      assertThat(executor.awaitTermination(100, MILLISECONDS), is(false));
      assertThat(executor.isShutdown(), is(true));
      assertThat(executor.isTerminated(), is(false));

      jobSemaphore.release();
      assertThat(executor.awaitTermination(2, MINUTES), is(true));
      assertThat(executor.isShutdown(), is(true));
      assertThat(executor.isTerminated(), is(true));

      assertThat(jobSemaphore.availablePermits(), is(0));
    } finally {
      worker.shutdown();
    }
  }
  @Test
  public void testRunningJobsAreInterruptedAfterShutdownNow() throws InterruptedException {
    final int jobCount = 4;

    OutOfBandScheduledExecutor scheduler = new OutOfBandScheduledExecutor();
    ExecutorService worker = Executors.newCachedThreadPool();
    try {
      PartitionedScheduledExecutor executor = new PartitionedScheduledExecutor(scheduler, worker);

      final Semaphore jobSemaphore = new Semaphore(0);
      final Semaphore testSemaphore = new Semaphore(0);
      final AtomicInteger interrupted = new AtomicInteger();

      for (int i = 0; i < jobCount; i++) {
        executor.submit(
            new Callable<Void>() {

              @Override
              public Void call() throws Exception {
                testSemaphore.release();
                try {
                  jobSemaphore.acquire();
                } catch (InterruptedException e) {
                  interrupted.incrementAndGet();
                }
                return null;
              }
            });
      }

      testSemaphore.acquireUninterruptibly(jobCount);

      assertThat(executor.shutdownNow(), empty());
      assertThat(executor.awaitTermination(2, MINUTES), is(true));
      assertThat(executor.isShutdown(), is(true));
      assertThat(executor.isTerminated(), is(true));

      assertThat(jobSemaphore.availablePermits(), is(0));
      assertThat(interrupted.get(), is(jobCount));
    } finally {
      worker.shutdown();
    }
  }
  @Test
  public void testRunningJobIsInterruptedAfterShutdownNow() throws InterruptedException {
    OutOfBandScheduledExecutor scheduler = new OutOfBandScheduledExecutor();
    ExecutorService worker = Executors.newSingleThreadExecutor();
    try {
      PartitionedScheduledExecutor executor = new PartitionedScheduledExecutor(scheduler, worker);

      final Semaphore jobSemaphore = new Semaphore(0);
      final Semaphore testSemaphore = new Semaphore(0);
      final AtomicBoolean interrupted = new AtomicBoolean();

      executor.submit(
          new Callable<Void>() {

            @Override
            public Void call() throws Exception {
              testSemaphore.release();
              try {
                jobSemaphore.acquire();
              } catch (InterruptedException e) {
                interrupted.set(true);
              }
              return null;
            }
          });
      testSemaphore.acquireUninterruptibly();
      assertThat(executor.shutdownNow(), empty());
      assertThat(executor.awaitTermination(2, MINUTES), is(true));
      assertThat(executor.isShutdown(), is(true));
      assertThat(executor.isTerminated(), is(true));

      assertThat(jobSemaphore.availablePermits(), is(0));
      assertThat(interrupted.get(), is(true));
    } finally {
      worker.shutdown();
    }
  }
Example #20
0
 @Override
 public long get(S source) throws Exception {
   switch (type) {
     case TYPE_PRIMITIVE_LONG:
       return ((Number) method.invoke(source, EMPTY_ARGS)).longValue();
     case TYPE_LONG_NUMBER:
       Number longNumber = (Number) method.invoke(source, EMPTY_ARGS);
       return longNumber == null ? 0 : longNumber.longValue();
     case TYPE_MAP:
       Map<?, ?> map = (Map<?, ?>) method.invoke(source, EMPTY_ARGS);
       return map == null ? 0 : map.size();
     case TYPE_COLLECTION:
       Collection<?> collection = (Collection<?>) method.invoke(source, EMPTY_ARGS);
       return collection == null ? 0 : collection.size();
     case TYPE_COUNTER:
       Counter counter = (Counter) method.invoke(source, EMPTY_ARGS);
       return counter == null ? 0 : counter.get();
     case TYPE_SEMAPHORE:
       Semaphore semaphore = (Semaphore) method.invoke(source, EMPTY_ARGS);
       return semaphore == null ? 0 : semaphore.availablePermits();
     default:
       throw new IllegalStateException("Unrecognized type:" + type);
   }
 }
Example #21
0
  protected void service() {
    initialize();

    ServerSocket serverSocket;
    try {
      InetAddress addr = m_bindAddress == null ? null : InetAddress.getByName(m_bindAddress);
      serverSocket = new ServerSocket(m_port, m_workerSemaphore.availablePermits(), addr);
      serverSocket.setSoTimeout(5000);
      s_logger.info("Listening on " + serverSocket.getLocalSocketAddress());
    } catch (IOException ioe) {
      s_logger.error("Failed to initialize server socket", ioe);
      return;
    }

    // publish stats once to clear out stats from previous instance
    s_logger.info("Publishing zeroed out stats for " + getProxyId());
    m_proxyStats.publish();

    final long initialReservation = m_maxBatchSize;
    while (acquireSlot()) {
      try {
        m_quota.reserve(initialReservation);
        final Socket socket = accept(serverSocket);
        if (socket == null) {
          s_logger.info("Null socket");
          break;
        }

        s_logger.info("Accepting " + socket);
        m_proxyStats.incrementTotalConnections();
        m_activeConnections++;

        final IndexProxy theProxy = this;
        Thread workerThread =
            new Thread(
                new Runnable() {
                  public void run() {
                    try {
                      s_logger.info("handleConnection begin for " + socket);
                      handleConnection(socket, initialReservation);
                      s_logger.info("handleConnection complete for " + socket);
                    } catch (Throwable e) {
                      s_logger.error(
                          "Index proxy connection failure for remote address: "
                              + socket.getRemoteSocketAddress(),
                          e);
                    } finally {
                      m_activeConnections--;

                      try {
                        if (!socket.isClosed()) {
                          socket
                              .close(); // at least let the other side know that we're dead by
                                        // closing the socket.
                        }
                      } catch (IOException ex) {
                        // safe to ignore
                      }
                      m_proxyStats.onProxyWorkerFinished(theProxy);
                    }
                  }
                },
                "IndexProxyWorker for " + socket.getRemoteSocketAddress());

        workerThread.start();
      } catch (IOException ioe) {
        s_logger.error("Exception accepting connection:", ioe);
      } catch (QuotaTimeoutException qte) {
        s_logger.error("Timeout processing quota request", qte);
        suicide(); // signal supervisor to restart this service
      }
    }
  }
Example #22
0
 public void msgBusIsHere() {
   if (waitingAtBus.availablePermits() < 1) {
     getWaitingAtBus().release();
   }
 }
Example #23
0
 public boolean isIgnoring() {
   return 0 == ignore.availablePermits();
 }
Example #24
0
 public void msgAtBusStopDestination() {
   if (beingTransported.availablePermits() < 1) {
     beingTransported.release();
   }
 }
 public int getFreeBytes() {
   return inputSemaphore.availablePermits();
 }
 public int getAvailableBytes() {
   return bufferSize - inputSemaphore.availablePermits();
 }
  /**
   * Returns a list of value constraints and the associated facet counts for each facet field
   * specified in the params.
   *
   * @see FacetParams#FACET_FIELD
   * @see #getFieldMissingCount
   * @see #getFacetTermEnumCounts
   */
  @SuppressWarnings("unchecked")
  public NamedList<Object> getFacetFieldCounts() throws IOException, SyntaxError {

    NamedList<Object> res = new SimpleOrderedMap<>();
    String[] facetFs = global.getParams(FacetParams.FACET_FIELD);
    if (null == facetFs) {
      return res;
    }

    // Passing a negative number for FACET_THREADS implies an unlimited number of threads is
    // acceptable.
    // Also, a subtlety of directExecutor is that no matter how many times you "submit" a job, it's
    // really
    // just a method call in that it's run by the calling thread.
    int maxThreads = req.getParams().getInt(FacetParams.FACET_THREADS, 0);
    Executor executor = maxThreads == 0 ? directExecutor : facetExecutor;
    final Semaphore semaphore = new Semaphore((maxThreads <= 0) ? Integer.MAX_VALUE : maxThreads);
    List<Future<NamedList>> futures = new ArrayList<>(facetFs.length);

    try {
      // Loop over fields; submit to executor, keeping the future
      for (String f : facetFs) {
        final ParsedParams parsed = parseParams(FacetParams.FACET_FIELD, f);
        final SolrParams localParams = parsed.localParams;
        final String termList = localParams == null ? null : localParams.get(CommonParams.TERMS);
        final String key = parsed.key;
        final String facetValue = parsed.facetValue;
        Callable<NamedList> callable =
            new Callable<NamedList>() {
              @Override
              public NamedList call() throws Exception {
                try {
                  NamedList<Object> result = new SimpleOrderedMap<>();
                  if (termList != null) {
                    List<String> terms = StrUtils.splitSmart(termList, ",", true);
                    result.add(key, getListedTermCounts(facetValue, parsed, terms));
                  } else {
                    result.add(key, getTermCounts(facetValue, parsed));
                  }
                  return result;
                } catch (SolrException se) {
                  throw se;
                } catch (Exception e) {
                  throw new SolrException(
                      ErrorCode.SERVER_ERROR, "Exception during facet.field: " + facetValue, e);
                } finally {
                  semaphore.release();
                }
              }
            };

        RunnableFuture<NamedList> runnableFuture = new FutureTask<>(callable);
        semaphore.acquire(); // may block and/or interrupt
        executor.execute(runnableFuture); // releases semaphore when done
        futures.add(runnableFuture);
      } // facetFs loop

      // Loop over futures to get the values. The order is the same as facetFs but shouldn't matter.
      for (Future<NamedList> future : futures) {
        res.addAll(future.get());
      }
      assert semaphore.availablePermits() >= maxThreads;
    } catch (InterruptedException e) {
      throw new SolrException(
          SolrException.ErrorCode.SERVER_ERROR,
          "Error while processing facet fields: InterruptedException",
          e);
    } catch (ExecutionException ee) {
      Throwable e = ee.getCause(); // unwrap
      if (e instanceof RuntimeException) {
        throw (RuntimeException) e;
      }
      throw new SolrException(
          SolrException.ErrorCode.SERVER_ERROR,
          "Error while processing facet fields: " + e.toString(),
          e);
    }

    return res;
  }
Example #28
0
  @Test
  public void testLogReplay() throws Exception {
    try {

      DirectUpdateHandler2.commitOnClose = false;
      final Semaphore logReplay = new Semaphore(0);
      final Semaphore logReplayFinish = new Semaphore(0);

      UpdateLog.testing_logReplayHook =
          new Runnable() {
            @Override
            public void run() {
              try {
                assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
              } catch (Exception e) {
                throw new RuntimeException(e);
              }
            }
          };

      UpdateLog.testing_logReplayFinishHook =
          new Runnable() {
            @Override
            public void run() {
              logReplayFinish.release();
            }
          };

      clearIndex();
      assertU(commit());

      Deque<Long> versions = new ArrayDeque<>();
      versions.addFirst(addAndGetVersion(sdoc("id", "A1"), null));
      versions.addFirst(addAndGetVersion(sdoc("id", "A11"), null));
      versions.addFirst(addAndGetVersion(sdoc("id", "A12"), null));
      versions.addFirst(deleteByQueryAndGetVersion("id:A11", null));
      versions.addFirst(addAndGetVersion(sdoc("id", "A13"), null));

      assertJQ(req("q", "*:*"), "/response/numFound==0");

      assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);

      h.close();
      createCore();
      // Solr should kick this off now
      // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();

      // verify that previous close didn't do a commit
      // recovery should be blocked by our hook
      assertJQ(req("q", "*:*"), "/response/numFound==0");

      // make sure we can still access versions after a restart
      assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);

      // unblock recovery
      logReplay.release(1000);

      // make sure we can still access versions during recovery
      assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);

      // wait until recovery has finished
      assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));

      assertJQ(req("q", "*:*"), "/response/numFound==3");

      // make sure we can still access versions after recovery
      assertJQ(req("qt", "/get", "getVersions", "" + versions.size()), "/versions==" + versions);

      assertU(adoc("id", "A2"));
      assertU(adoc("id", "A3"));
      assertU(delI("A2"));
      assertU(adoc("id", "A4"));

      assertJQ(req("q", "*:*"), "/response/numFound==3");

      h.close();
      createCore();
      // Solr should kick this off now
      // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();

      // wait until recovery has finished
      assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
      assertJQ(req("q", "*:*"), "/response/numFound==5");
      assertJQ(req("q", "id:A2"), "/response/numFound==0");

      // no updates, so insure that recovery does not run
      h.close();
      int permits = logReplay.availablePermits();
      createCore();
      // Solr should kick this off now
      // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();

      assertJQ(req("q", "*:*"), "/response/numFound==5");
      Thread.sleep(100);
      assertEquals(
          permits, logReplay.availablePermits()); // no updates, so insure that recovery didn't run

      assertEquals(
          UpdateLog.State.ACTIVE, h.getCore().getUpdateHandler().getUpdateLog().getState());

    } finally {
      DirectUpdateHandler2.commitOnClose = true;
      UpdateLog.testing_logReplayHook = null;
      UpdateLog.testing_logReplayFinishHook = null;
    }
  }
 /** Test if the lock is available, without blocking. */
 public boolean available() {
   return semaphore.availablePermits() > 0;
 }
Example #30
0
 public int availablePermits() {
   return m_workerSemaphore.availablePermits();
 }