/** * Test that's meant to be run with many iterations to expose a leak of SolrIndexSearcher when a * core is closed due to a reload. Without the fix, this test fails with most iters=1000 runs. */ @Test public void testReloadLeak() throws Exception { final ExecutorService executor = ExecutorUtil.newMDCAwareFixedThreadPool(1, new DefaultSolrThreadFactory("testReloadLeak")); // Continuously open new searcher while core is not closed, and reload core to try to reproduce // searcher leak. // While in practice we never continuously open new searchers, this is trying to make up for the // fact that opening // a searcher in this empty core is very fast by opening new searchers continuously to increase // the likelihood // for race. SolrCore core = h.getCore(); assertTrue("Refcount != 1", core.getOpenCount() == 1); executor.execute(new NewSearcherRunnable(core)); // Since we called getCore() vs getCoreInc() and don't own a refCount, the container should // decRef the core // and close it when we call reload. h.reload(); executor.shutdown(); executor.awaitTermination(1, TimeUnit.MINUTES); // Check that all cores are closed and no searcher references are leaked. assertTrue("SolrCore " + core + " is not closed", core.isClosed()); assertTrue(core.areAllSearcherReferencesEmpty()); }
/** Load the cores defined for this CoreContainer */ public void load() { log.info("Loading cores into CoreContainer [instanceDir={}]", loader.getInstancePath()); // add the sharedLib to the shared resource loader before initializing cfg based plugins String libDir = cfg.getSharedLibDirectory(); if (libDir != null) { Path libPath = loader.getInstancePath().resolve(libDir); try { loader.addToClassLoader(SolrResourceLoader.getURLs(libPath)); loader.reloadLuceneSPI(); } catch (IOException e) { log.warn("Couldn't add files from {} to classpath: {}", libPath, e.getMessage()); } } shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader); updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig()); solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader); logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader); hostName = cfg.getNodeName(); zkSys.initZooKeeper(this, solrHome, cfg.getCloudConfig()); if (isZooKeeperAware()) pkiAuthenticationPlugin = new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName()); ZkStateReader.ConfigData securityConfig = isZooKeeperAware() ? getZkController().getZkStateReader().getSecurityProps(false) : new ZkStateReader.ConfigData(EMPTY_MAP, -1); initializeAuthorizationPlugin((Map<String, Object>) securityConfig.data.get("authorization")); initializeAuthenticationPlugin((Map<String, Object>) securityConfig.data.get("authentication")); this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins()); containerHandlers.put(ZK_PATH, new ZookeeperInfoHandler(this)); securityConfHandler = new SecurityConfHandler(this); collectionsHandler = createHandler(cfg.getCollectionsHandlerClass(), CollectionsHandler.class); containerHandlers.put(COLLECTIONS_HANDLER_PATH, collectionsHandler); infoHandler = createHandler(cfg.getInfoHandlerClass(), InfoHandler.class); containerHandlers.put(INFO_HANDLER_PATH, infoHandler); coreAdminHandler = createHandler(cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class); containerHandlers.put(CORES_HANDLER_PATH, coreAdminHandler); configSetsHandler = createHandler(cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class); containerHandlers.put(CONFIGSETS_HANDLER_PATH, configSetsHandler); containerHandlers.put(AUTHZ_PATH, securityConfHandler); containerHandlers.put(AUTHC_PATH, securityConfHandler); if (pkiAuthenticationPlugin != null) containerHandlers.put( PKIAuthenticationPlugin.PATH, pkiAuthenticationPlugin.getRequestHandler()); coreConfigService = ConfigSetService.createConfigSetService(cfg, loader, zkSys.zkController); containerProperties.putAll(cfg.getSolrProperties()); // setup executor to load cores in parallel ExecutorService coreLoadExecutor = ExecutorUtil.newMDCAwareFixedThreadPool( cfg.getCoreLoadThreadCount( isZooKeeperAware() ? DEFAULT_CORE_LOAD_THREADS_IN_CLOUD : DEFAULT_CORE_LOAD_THREADS), new DefaultSolrThreadFactory("coreLoadExecutor")); final List<Future<SolrCore>> futures = new ArrayList<>(); try { List<CoreDescriptor> cds = coresLocator.discover(this); if (isZooKeeperAware()) { // sort the cores if it is in SolrCloud. In standalone node the order does not matter CoreSorter coreComparator = new CoreSorter().init(this); cds = new ArrayList<>(cds); // make a copy Collections.sort(cds, coreComparator::compare); } checkForDuplicateCoreNames(cds); for (final CoreDescriptor cd : cds) { if (cd.isTransient() || !cd.isLoadOnStartup()) { solrCores.putDynamicDescriptor(cd.getName(), cd); } else if (asyncSolrCoreLoad) { solrCores.markCoreAsLoading(cd); } if (cd.isLoadOnStartup()) { futures.add( coreLoadExecutor.submit( () -> { SolrCore core; try { if (zkSys.getZkController() != null) { zkSys.getZkController().throwErrorIfReplicaReplaced(cd); } core = create(cd, false); } finally { if (asyncSolrCoreLoad) { solrCores.markCoreAsNotLoading(cd); } } try { zkSys.registerInZk(core, true); } catch (RuntimeException e) { SolrException.log(log, "Error registering SolrCore", e); } return core; })); } } // Start the background thread backgroundCloser = new CloserThread(this, solrCores, cfg); backgroundCloser.start(); } finally { if (asyncSolrCoreLoad && futures != null) { coreContainerWorkExecutor.submit( (Runnable) () -> { try { for (Future<SolrCore> future : futures) { try { future.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (ExecutionException e) { log.error("Error waiting for SolrCore to be created", e); } } } finally { ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor); } }); } else { ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor); } } if (isZooKeeperAware()) { zkSys.getZkController().checkOverseerDesignate(); } }
@Test public void testRefCountMT() throws Exception { SolrCore core = h.getCore(); assertTrue("Refcount != 1", core.getOpenCount() == 1); final ClosingRequestHandler handler1 = new ClosingRequestHandler(); handler1.inform(core); String path = "/this/is A path /that won't be registered!"; SolrRequestHandler old = core.registerRequestHandler(path, handler1); assertNull(old); // should not be anything... assertEquals(core.getRequestHandlers().get(path), handler1); final int LOOP = 100; final int MT = 16; ExecutorService service = ExecutorUtil.newMDCAwareFixedThreadPool(MT, new DefaultSolrThreadFactory("refCountMT")); List<Callable<Integer>> callees = new ArrayList<>(MT); final CoreContainer cores = h.getCoreContainer(); for (int i = 0; i < MT; ++i) { Callable<Integer> call = new Callable<Integer>() { void yield(int n) { try { Thread.sleep(0, (n % 13 + 1) * 10); } catch (InterruptedException xint) { } } @Override public Integer call() { SolrCore core = null; int r = 0; try { for (int l = 0; l < LOOP; ++l) { r += 1; core = cores.getCore(SolrTestCaseJ4.DEFAULT_TEST_CORENAME); // sprinkle concurrency hinting... yield(l); assertTrue("Refcount < 1", core.getOpenCount() >= 1); yield(l); assertTrue("Refcount > 17", core.getOpenCount() <= 17); yield(l); assertTrue("Handler is closed", handler1.closed == false); yield(l); core.close(); core = null; yield(l); } return r; } finally { if (core != null) core.close(); } } }; callees.add(call); } List<Future<Integer>> results = service.invokeAll(callees); for (Future<Integer> result : results) { assertTrue("loop=" + result.get() + " < " + LOOP, result.get() >= LOOP); } cores.shutdown(); assertTrue("Refcount != 0", core.getOpenCount() == 0); assertTrue("Handler not closed", core.isClosed() && handler1.closed == true); service.shutdown(); assertTrue("Running for too long...", service.awaitTermination(60, TimeUnit.SECONDS)); }