/**
* Test that's meant to be run with many iterations to expose a leak of SolrIndexSearcher when a
* core is closed due to a reload. Without the fix, this test fails with most iters=1000 runs.
*/
@Test
public void testReloadLeak() throws Exception {
final ExecutorService executor =
ExecutorUtil.newMDCAwareFixedThreadPool(1, new DefaultSolrThreadFactory("testReloadLeak"));
// Continuously open new searcher while core is not closed, and reload core to try to reproduce
// searcher leak.
// While in practice we never continuously open new searchers, this is trying to make up for the
// fact that opening
// a searcher in this empty core is very fast by opening new searchers continuously to increase
// the likelihood
// for race.
SolrCore core = h.getCore();
assertTrue("Refcount != 1", core.getOpenCount() == 1);
executor.execute(new NewSearcherRunnable(core));
// Since we called getCore() vs getCoreInc() and don't own a refCount, the container should
// decRef the core
// and close it when we call reload.
h.reload();
executor.shutdown();
executor.awaitTermination(1, TimeUnit.MINUTES);
// Check that all cores are closed and no searcher references are leaked.
assertTrue("SolrCore " + core + " is not closed", core.isClosed());
assertTrue(core.areAllSearcherReferencesEmpty());
}
/** Load the cores defined for this CoreContainer */
public void load() {
log.info("Loading cores into CoreContainer [instanceDir={}]", loader.getInstancePath());
// add the sharedLib to the shared resource loader before initializing cfg based plugins
String libDir = cfg.getSharedLibDirectory();
if (libDir != null) {
Path libPath = loader.getInstancePath().resolve(libDir);
try {
loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
loader.reloadLuceneSPI();
} catch (IOException e) {
log.warn("Couldn't add files from {} to classpath: {}", libPath, e.getMessage());
}
}
shardHandlerFactory =
ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());
solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);
logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
hostName = cfg.getNodeName();
zkSys.initZooKeeper(this, solrHome, cfg.getCloudConfig());
if (isZooKeeperAware())
pkiAuthenticationPlugin =
new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName());
ZkStateReader.ConfigData securityConfig =
isZooKeeperAware()
? getZkController().getZkStateReader().getSecurityProps(false)
: new ZkStateReader.ConfigData(EMPTY_MAP, -1);
initializeAuthorizationPlugin((Map<String, Object>) securityConfig.data.get("authorization"));
initializeAuthenticationPlugin((Map<String, Object>) securityConfig.data.get("authentication"));
this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());
containerHandlers.put(ZK_PATH, new ZookeeperInfoHandler(this));
securityConfHandler = new SecurityConfHandler(this);
collectionsHandler = createHandler(cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
containerHandlers.put(COLLECTIONS_HANDLER_PATH, collectionsHandler);
infoHandler = createHandler(cfg.getInfoHandlerClass(), InfoHandler.class);
containerHandlers.put(INFO_HANDLER_PATH, infoHandler);
coreAdminHandler = createHandler(cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
containerHandlers.put(CORES_HANDLER_PATH, coreAdminHandler);
configSetsHandler = createHandler(cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
containerHandlers.put(CONFIGSETS_HANDLER_PATH, configSetsHandler);
containerHandlers.put(AUTHZ_PATH, securityConfHandler);
containerHandlers.put(AUTHC_PATH, securityConfHandler);
if (pkiAuthenticationPlugin != null)
containerHandlers.put(
PKIAuthenticationPlugin.PATH, pkiAuthenticationPlugin.getRequestHandler());
coreConfigService = ConfigSetService.createConfigSetService(cfg, loader, zkSys.zkController);
containerProperties.putAll(cfg.getSolrProperties());
// setup executor to load cores in parallel
ExecutorService coreLoadExecutor =
ExecutorUtil.newMDCAwareFixedThreadPool(
cfg.getCoreLoadThreadCount(
isZooKeeperAware()
? DEFAULT_CORE_LOAD_THREADS_IN_CLOUD
: DEFAULT_CORE_LOAD_THREADS),
new DefaultSolrThreadFactory("coreLoadExecutor"));
final List<Future<SolrCore>> futures = new ArrayList<>();
try {
List<CoreDescriptor> cds = coresLocator.discover(this);
if (isZooKeeperAware()) {
// sort the cores if it is in SolrCloud. In standalone node the order does not matter
CoreSorter coreComparator = new CoreSorter().init(this);
cds = new ArrayList<>(cds); // make a copy
Collections.sort(cds, coreComparator::compare);
}
checkForDuplicateCoreNames(cds);
for (final CoreDescriptor cd : cds) {
if (cd.isTransient() || !cd.isLoadOnStartup()) {
solrCores.putDynamicDescriptor(cd.getName(), cd);
} else if (asyncSolrCoreLoad) {
solrCores.markCoreAsLoading(cd);
}
if (cd.isLoadOnStartup()) {
futures.add(
coreLoadExecutor.submit(
() -> {
SolrCore core;
try {
if (zkSys.getZkController() != null) {
zkSys.getZkController().throwErrorIfReplicaReplaced(cd);
}
core = create(cd, false);
} finally {
if (asyncSolrCoreLoad) {
solrCores.markCoreAsNotLoading(cd);
}
}
try {
zkSys.registerInZk(core, true);
} catch (RuntimeException e) {
SolrException.log(log, "Error registering SolrCore", e);
}
return core;
}));
}
}
// Start the background thread
backgroundCloser = new CloserThread(this, solrCores, cfg);
backgroundCloser.start();
} finally {
if (asyncSolrCoreLoad && futures != null) {
coreContainerWorkExecutor.submit(
(Runnable)
() -> {
try {
for (Future<SolrCore> future : futures) {
try {
future.get();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} catch (ExecutionException e) {
log.error("Error waiting for SolrCore to be created", e);
}
}
} finally {
ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
}
});
} else {
ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
}
}
if (isZooKeeperAware()) {
zkSys.getZkController().checkOverseerDesignate();
}
}
@Test
public void testRefCountMT() throws Exception {
SolrCore core = h.getCore();
assertTrue("Refcount != 1", core.getOpenCount() == 1);
final ClosingRequestHandler handler1 = new ClosingRequestHandler();
handler1.inform(core);
String path = "/this/is A path /that won't be registered!";
SolrRequestHandler old = core.registerRequestHandler(path, handler1);
assertNull(old); // should not be anything...
assertEquals(core.getRequestHandlers().get(path), handler1);
final int LOOP = 100;
final int MT = 16;
ExecutorService service =
ExecutorUtil.newMDCAwareFixedThreadPool(MT, new DefaultSolrThreadFactory("refCountMT"));
List<Callable<Integer>> callees = new ArrayList<>(MT);
final CoreContainer cores = h.getCoreContainer();
for (int i = 0; i < MT; ++i) {
Callable<Integer> call =
new Callable<Integer>() {
void yield(int n) {
try {
Thread.sleep(0, (n % 13 + 1) * 10);
} catch (InterruptedException xint) {
}
}
@Override
public Integer call() {
SolrCore core = null;
int r = 0;
try {
for (int l = 0; l < LOOP; ++l) {
r += 1;
core = cores.getCore(SolrTestCaseJ4.DEFAULT_TEST_CORENAME);
// sprinkle concurrency hinting...
yield(l);
assertTrue("Refcount < 1", core.getOpenCount() >= 1);
yield(l);
assertTrue("Refcount > 17", core.getOpenCount() <= 17);
yield(l);
assertTrue("Handler is closed", handler1.closed == false);
yield(l);
core.close();
core = null;
yield(l);
}
return r;
} finally {
if (core != null) core.close();
}
}
};
callees.add(call);
}
List<Future<Integer>> results = service.invokeAll(callees);
for (Future<Integer> result : results) {
assertTrue("loop=" + result.get() + " < " + LOOP, result.get() >= LOOP);
}
cores.shutdown();
assertTrue("Refcount != 0", core.getOpenCount() == 0);
assertTrue("Handler not closed", core.isClosed() && handler1.closed == true);
service.shutdown();
assertTrue("Running for too long...", service.awaitTermination(60, TimeUnit.SECONDS));
}
