@Test
public void testRace() throws Exception {
final List<SolrCore> theCores = new ArrayList<>();
final CoreContainer cc = init();
try {
Thread[] threads = new Thread[15];
for (int idx = 0; idx < threads.length; idx++) {
threads[idx] =
new Thread() {
@Override
public void run() {
SolrCore core = cc.getCore("collectionLazy3");
synchronized (theCores) {
theCores.add(core);
}
}
};
threads[idx].start();
}
for (Thread thread : threads) {
thread.join();
}
for (int idx = 0; idx < theCores.size() - 1; ++idx) {
assertEquals("Cores should be the same!", theCores.get(idx), theCores.get(idx + 1));
}
for (SolrCore core : theCores) {
core.close();
}
} finally {
cc.shutdown();
}
}
@Test
public void testCreateSame() throws Exception {
final CoreContainer cc = init();
try {
// First, try all 4 combinations of load on startup and transient
final CoreAdminHandler admin = new CoreAdminHandler(cc);
SolrCore lc2 = cc.getCore("collectionLazy2");
SolrCore lc4 = cc.getCore("collectionLazy4");
SolrCore lc5 = cc.getCore("collectionLazy5");
SolrCore lc6 = cc.getCore("collectionLazy6");
copyMinConf(new File(solrHomeDirectory, "t2"));
copyMinConf(new File(solrHomeDirectory, "t4"));
copyMinConf(new File(solrHomeDirectory, "t5"));
copyMinConf(new File(solrHomeDirectory, "t6"));
// Should also fail with the same name
tryCreateFail(
admin, "collectionLazy2", "t12", "Core with name", "collectionLazy2", "already exists");
tryCreateFail(
admin, "collectionLazy4", "t14", "Core with name", "collectionLazy4", "already exists");
tryCreateFail(
admin, "collectionLazy5", "t15", "Core with name", "collectionLazy5", "already exists");
tryCreateFail(
admin, "collectionLazy6", "t16", "Core with name", "collectionLazy6", "already exists");
lc2.close();
lc4.close();
lc5.close();
lc6.close();
} finally {
cc.shutdown();
}
}
@Test
public void testRefCount() throws Exception {
SolrCore core = h.getCore();
assertTrue("Refcount != 1", core.getOpenCount() == 1);
final CoreContainer cores = h.getCoreContainer();
SolrCore c1 = cores.getCore(SolrTestCaseJ4.DEFAULT_TEST_CORENAME);
assertTrue("Refcount != 2", core.getOpenCount() == 2);
ClosingRequestHandler handler1 = new ClosingRequestHandler();
handler1.inform(core);
String path = "/this/is A path /that won't be registered!";
SolrRequestHandler old = core.registerRequestHandler(path, handler1);
assertNull(old); // should not be anything...
assertEquals(core.getRequestHandlers().get(path), handler1);
SolrCore c2 = cores.getCore(SolrTestCaseJ4.DEFAULT_TEST_CORENAME);
c1.close();
assertTrue("Refcount < 1", core.getOpenCount() >= 1);
assertTrue("Handler is closed", handler1.closed == false);
c1 = cores.getCore(SolrTestCaseJ4.DEFAULT_TEST_CORENAME);
assertTrue("Refcount < 2", core.getOpenCount() >= 2);
assertTrue("Handler is closed", handler1.closed == false);
c2.close();
assertTrue("Refcount < 1", core.getOpenCount() >= 1);
assertTrue("Handler is closed", handler1.closed == false);
c1.close();
cores.shutdown();
assertTrue("Refcount != 0", core.getOpenCount() == 0);
assertTrue("Handler not closed", core.isClosed() && handler1.closed == true);
}
// Make sure that creating a transient core from the admin handler correctly respects the
// transient limits etc.
@Test
public void testCreateTransientFromAdmin() throws Exception {
final CoreContainer cc = init();
try {
copyMinConf(new File(solrHomeDirectory, "core1"));
copyMinConf(new File(solrHomeDirectory, "core2"));
copyMinConf(new File(solrHomeDirectory, "core3"));
copyMinConf(new File(solrHomeDirectory, "core4"));
copyMinConf(new File(solrHomeDirectory, "core5"));
createViaAdmin(cc, "core1", "./core1", true, true);
createViaAdmin(cc, "core2", "./core2", true, false);
createViaAdmin(cc, "core3", "./core3", true, true);
createViaAdmin(cc, "core4", "./core4", true, false);
createViaAdmin(cc, "core5", "./core5", true, false);
SolrCore c1 = cc.getCore("core1");
SolrCore c2 = cc.getCore("core2");
SolrCore c3 = cc.getCore("core3");
SolrCore c4 = cc.getCore("core4");
SolrCore c5 = cc.getCore("core5");
checkNotInCores(
cc,
"core1",
"collectionLazy2",
"collectionLazy3",
"collectionLazy4",
"collectionLazy6",
"collectionLazy7",
"collectionLazy8",
"collectionLazy9");
checkInCores(cc, "collection1", "collectionLazy5", "core2", "core3", "core4", "core5");
// While we're at it, a test for SOLR-5366, unloading transient core that's been unloaded b/c
// it's
// transient generates a "too many closes" errorl
unloadViaAdmin(cc, "core1");
unloadViaAdmin(cc, "core2");
unloadViaAdmin(cc, "core3");
unloadViaAdmin(cc, "core4");
unloadViaAdmin(cc, "core5");
c1.close();
c2.close();
c3.close();
c4.close();
c5.close();
} finally {
cc.shutdown();
}
}
public void disable() {
if (!enabled) {
return;
}
nakamuraCore.close();
coreContainer.shutdown();
enabled = false;
if (listener != null) {
listener.disabled();
}
}
/**
* Create a new CoreContainer and load its cores
*
* @param solrHome the solr home directory
* @param configFile the file containing this container's configuration
* @return a loaded CoreContainer
*/
public static CoreContainer createAndLoad(Path solrHome, Path configFile) {
SolrResourceLoader loader = new SolrResourceLoader(solrHome);
CoreContainer cc = new CoreContainer(SolrXmlConfig.fromFile(loader, configFile));
try {
cc.load();
} catch (Exception e) {
cc.shutdown();
throw e;
}
return cc;
}
@After
public void cleanUp() throws Exception {
if (cc != null) {
cc.shutdown();
cc = null;
}
if (null != solrHome) {
if (solrHome.exists()) {
FileUtils.deleteDirectory(solrHome);
}
solrHome = null;
}
}
@AfterClass
public static void tearDown() throws Exception {
try {
if (solr != null) {
solr.rollback();
}
} catch (SolrException e) {
}
solrCore.close();
if (coreContainer != null) {
coreContainer.shutdown();
}
FileUtils.cleanDirectory(new File(solrCore.getDataDir() + "/index"));
FileUtils.cleanDirectory(new File(solrCore.getDataDir() + "/tlog"));
}
@Test
public void testClose() throws Exception {
final CoreContainer cores = h.getCoreContainer();
SolrCore core = cores.getCore(SolrTestCaseJ4.DEFAULT_TEST_CORENAME);
ClosingRequestHandler handler1 = new ClosingRequestHandler();
handler1.inform(core);
String path = "/this/is A path /that won't be registered 2!!!!!!!!!!!";
SolrRequestHandler old = core.registerRequestHandler(path, handler1);
assertNull(old); // should not be anything...
assertEquals(core.getRequestHandlers().get(path), handler1);
core.close();
cores.shutdown();
assertTrue("Handler not closed", handler1.closed == true);
}
@Test
public void testLazyLoad() throws Exception {
CoreContainer cc = init();
try {
// NOTE: This checks the initial state for loading, no need to do this elsewhere.
checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy5");
checkNotInCores(
cc,
"collectionLazy3",
"collectionLazy4",
"collectionLazy6",
"collectionLazy7",
"collectionLazy8",
"collectionLazy9");
SolrCore core1 = cc.getCore("collection1");
assertFalse("core1 should not be transient", core1.getCoreDescriptor().isTransient());
assertTrue("core1 should be loadable", core1.getCoreDescriptor().isLoadOnStartup());
assertNotNull(core1.getSolrConfig());
SolrCore core2 = cc.getCore("collectionLazy2");
assertTrue("core2 should be transient", core2.getCoreDescriptor().isTransient());
assertTrue("core2 should be loadable", core2.getCoreDescriptor().isLoadOnStartup());
SolrCore core3 = cc.getCore("collectionLazy3");
assertTrue("core3 should be transient", core3.getCoreDescriptor().isTransient());
assertFalse("core3 should not be loadable", core3.getCoreDescriptor().isLoadOnStartup());
SolrCore core4 = cc.getCore("collectionLazy4");
assertFalse("core4 should not be transient", core4.getCoreDescriptor().isTransient());
assertFalse("core4 should not be loadable", core4.getCoreDescriptor().isLoadOnStartup());
SolrCore core5 = cc.getCore("collectionLazy5");
assertFalse("core5 should not be transient", core5.getCoreDescriptor().isTransient());
assertTrue("core5 should be loadable", core5.getCoreDescriptor().isLoadOnStartup());
core1.close();
core2.close();
core3.close();
core4.close();
core5.close();
} finally {
cc.shutdown();
}
}
@Test
public void testLazySearch() throws Exception {
CoreContainer cc = init();
try {
// Make sure Lazy4 isn't loaded. Should be loaded on the get
checkNotInCores(cc, "collectionLazy4");
SolrCore core4 = cc.getCore("collectionLazy4");
checkSearch(core4);
// Now just insure that the normal searching on "collection1" finds _0_ on the same query that
// found _2_ above.
// Use of makeReq above and req below is tricky, very tricky.
assertQ(
"test raw query", req("q", "{!raw f=v_t}hello", "wt", "xml"), "//result[@numFound='0']");
checkInCores(cc, "collectionLazy4");
core4.close();
} finally {
cc.shutdown();
}
}
public static void validate(File solrCoreDir, String dataDirPropertyName, Fields schemeFields)
throws IOException {
// Verify solrHomeDir exists
if (!solrCoreDir.exists() || !solrCoreDir.isDirectory()) {
throw new TapException("Solr core directory doesn't exist: " + solrCoreDir);
}
File tmpSolrHome = makeTempSolrHome(solrCoreDir);
// Set up a temp location for Solr home, where we're write out a synthetic solr.xml
// that references the core directory.
String coreName = solrCoreDir.getName();
String corePath = solrCoreDir.getAbsolutePath();
String solrXmlContent =
String.format(
"<solr><cores><core name=\"%s\" instanceDir=\"%s\"></core></cores></solr>",
coreName, corePath);
File solrXmlFile = new File(tmpSolrHome, "solr.xml");
FileUtils.write(solrXmlFile, solrXmlContent);
// Set up a temp location for data, so when we instantiate the coreContainer,
// we don't pollute the solr home with a /data sub-dir.
String tmpFolder = System.getProperty("java.io.tmpdir");
File tmpDataDir = new File(tmpFolder, UUID.randomUUID().toString());
tmpDataDir.mkdir();
System.setProperty("solr.solr.home", tmpSolrHome.getAbsolutePath());
System.setProperty(dataDirPropertyName, tmpDataDir.getAbsolutePath());
System.setProperty(
"enable.special-handlers", "false"); // All we need is the update request handler
System.setProperty(
"enable.cache-warming", "false"); // We certainly don't need to warm the cache
CoreContainer.Initializer initializer = new CoreContainer.Initializer();
CoreContainer coreContainer = null;
try {
coreContainer = initializer.initialize();
Collection<SolrCore> cores = coreContainer.getCores();
SolrCore core = null;
if (cores.size() == 0) {
throw new TapException("No Solr cores are available");
} else if (cores.size() > 1) {
throw new TapException("Only one Solr core is supported");
} else {
core = cores.iterator().next();
}
IndexSchema schema = core.getSchema();
Map<String, SchemaField> solrFields = schema.getFields();
Set<String> schemeFieldnames = new HashSet<String>();
for (int i = 0; i < schemeFields.size(); i++) {
String fieldName = schemeFields.get(i).toString();
if (!solrFields.containsKey(fieldName)) {
throw new TapException("Sink field name doesn't exist in Solr schema: " + fieldName);
}
schemeFieldnames.add(fieldName);
}
for (String solrFieldname : solrFields.keySet()) {
SchemaField solrField = solrFields.get(solrFieldname);
if (solrField.isRequired() && !schemeFieldnames.contains(solrFieldname)) {
throw new TapException("No sink field name for required Solr field: " + solrFieldname);
}
}
} finally {
if (coreContainer != null) {
coreContainer.shutdown();
}
}
}
// Test that transient cores
// 1> produce errors as appropriate when the config or schema files are foo'd
// 2> "self heal". That is, if the problem is corrected can the core be reloaded and used?
// 3> that OK cores can be searched even when some cores failed to load.
@Test
public void testBadConfigsGenerateErrors() throws Exception {
final CoreContainer cc =
initGoodAndBad(
Arrays.asList("core1", "core2"),
Arrays.asList("badSchema1", "badSchema2"),
Arrays.asList("badConfig1", "badConfig2"));
try {
// first, did the two good cores load successfully?
checkInCores(cc, "core1", "core2");
// Did the bad cores fail to load?
checkNotInCores(cc, "badSchema1", "badSchema2", "badConfig1", "badConfig2");
// Can we still search the "good" cores even though there were core init failures?
SolrCore core1 = cc.getCore("core1");
checkSearch(core1);
// Did we get the expected message for each of the cores that failed to load? Make sure we
// don't run afoul of
// the dreaded slash/backslash difference on Windows and *nix machines.
testMessage(
cc.getCoreInitFailures(),
"TestLazyCores" + File.separator + "badConfig1" + File.separator + "solrconfig.xml");
testMessage(
cc.getCoreInitFailures(),
"TestLazyCores" + File.separator + "badConfig2" + File.separator + "solrconfig.xml");
testMessage(
cc.getCoreInitFailures(),
"TestLazyCores" + File.separator + "badSchema1" + File.separator + "schema.xml");
testMessage(
cc.getCoreInitFailures(),
"TestLazyCores" + File.separator + "badSchema2" + File.separator + "schema.xml");
// Status should report that there are failure messages for the bad cores and none for the
// good cores.
checkStatus(cc, true, "core1");
checkStatus(cc, true, "core2");
checkStatus(cc, false, "badSchema1");
checkStatus(cc, false, "badSchema2");
checkStatus(cc, false, "badConfig1");
checkStatus(cc, false, "badConfig2");
// Copy good config and schema files in and see if you can then load them (they are transient
// after all)
copyGoodConf("badConfig1", "solrconfig-minimal.xml", "solrconfig.xml");
copyGoodConf("badConfig2", "solrconfig-minimal.xml", "solrconfig.xml");
copyGoodConf("badSchema1", "schema-tiny.xml", "schema.xml");
copyGoodConf("badSchema2", "schema-tiny.xml", "schema.xml");
// This should force a reload of the cores.
SolrCore bc1 = cc.getCore("badConfig1");
SolrCore bc2 = cc.getCore("badConfig2");
SolrCore bs1 = cc.getCore("badSchema1");
SolrCore bs2 = cc.getCore("badSchema2");
// all the cores should be found in the list now.
checkInCores(cc, "core1", "core2", "badSchema1", "badSchema2", "badConfig1", "badConfig2");
// Did we clear out the errors by putting good files in place? And the cores that never were
// bad should be OK too.
checkStatus(cc, true, "core1");
checkStatus(cc, true, "core2");
checkStatus(cc, true, "badSchema1");
checkStatus(cc, true, "badSchema2");
checkStatus(cc, true, "badConfig1");
checkStatus(cc, true, "badConfig2");
// Are the formerly bad cores now searchable? Testing one of each should do.
checkSearch(core1);
checkSearch(bc1);
checkSearch(bs1);
core1.close();
bc1.close();
bc2.close();
bs1.close();
bs2.close();
} finally {
cc.shutdown();
}
}
@Test
public void testPersistence() throws Exception {
final CoreContainer cc = init();
try {
copyMinConf(new File(solrHomeDirectory, "core1"));
copyMinConf(new File(solrHomeDirectory, "core2"));
copyMinConf(new File(solrHomeDirectory, "core3"));
copyMinConf(new File(solrHomeDirectory, "core4"));
final CoreDescriptor cd1 =
buildCoreDescriptor(cc, "core1", "./core1").isTransient(true).loadOnStartup(true).build();
final CoreDescriptor cd2 =
buildCoreDescriptor(cc, "core2", "./core2")
.isTransient(true)
.loadOnStartup(false)
.build();
final CoreDescriptor cd3 =
buildCoreDescriptor(cc, "core3", "./core3")
.isTransient(false)
.loadOnStartup(true)
.build();
final CoreDescriptor cd4 =
buildCoreDescriptor(cc, "core4", "./core4")
.isTransient(false)
.loadOnStartup(false)
.build();
SolrCore core1 = cc.create(cd1);
SolrCore core2 = cc.create(cd2);
SolrCore core3 = cc.create(cd3);
SolrCore core4 = cc.create(cd4);
SolrXMLCoresLocator.NonPersistingLocator locator =
(SolrXMLCoresLocator.NonPersistingLocator) cc.getCoresLocator();
TestHarness.validateXPath(
locator.xml,
"/solr/cores/core[@name='collection1']",
"/solr/cores/core[@name='collectionLazy2']",
"/solr/cores/core[@name='collectionLazy3']",
"/solr/cores/core[@name='collectionLazy4']",
"/solr/cores/core[@name='collectionLazy5']",
"/solr/cores/core[@name='collectionLazy6']",
"/solr/cores/core[@name='collectionLazy7']",
"/solr/cores/core[@name='collectionLazy8']",
"/solr/cores/core[@name='collectionLazy9']",
"/solr/cores/core[@name='core1']",
"/solr/cores/core[@name='core2']",
"/solr/cores/core[@name='core3']",
"/solr/cores/core[@name='core4']",
"13=count(/solr/cores/core)");
removeOne(cc, "collectionLazy2");
removeOne(cc, "collectionLazy3");
removeOne(cc, "collectionLazy4");
removeOne(cc, "collectionLazy5");
removeOne(cc, "collectionLazy6");
removeOne(cc, "collectionLazy7");
removeOne(cc, "core1");
removeOne(cc, "core2");
removeOne(cc, "core3");
removeOne(cc, "core4");
// now test that unloading a core means the core is not persisted
TestHarness.validateXPath(locator.xml, "3=count(/solr/cores/core)");
} finally {
cc.shutdown();
}
}
@Test
public void testCachingLimit() throws Exception {
CoreContainer cc = init();
try {
// First check that all the cores that should be loaded at startup actually are.
checkInCores(cc, "collection1", "collectionLazy2", "collectionLazy5");
checkNotInCores(
cc,
"collectionLazy3",
"collectionLazy4",
"collectionLazy6",
"collectionLazy7",
"collectionLazy8",
"collectionLazy9");
// By putting these in non-alpha order, we're also checking that we're not just seeing an
// artifact.
SolrCore core1 = cc.getCore("collection1");
SolrCore core3 = cc.getCore("collectionLazy3");
SolrCore core4 = cc.getCore("collectionLazy4");
SolrCore core2 = cc.getCore("collectionLazy2");
SolrCore core5 = cc.getCore("collectionLazy5");
checkInCores(
cc,
"collection1",
"collectionLazy2",
"collectionLazy3",
"collectionLazy4",
"collectionLazy5");
checkNotInCores(
cc, "collectionLazy6", "collectionLazy7", "collectionLazy8", "collectionLazy9");
// map should be full up, add one more and verify
SolrCore core6 = cc.getCore("collectionLazy6");
checkInCores(
cc,
"collection1",
"collectionLazy2",
"collectionLazy3",
"collectionLazy4",
"collectionLazy5",
"collectionLazy6");
checkNotInCores(cc, "collectionLazy7", "collectionLazy8", "collectionLazy9");
SolrCore core7 = cc.getCore("collectionLazy7");
checkInCores(
cc,
"collection1",
"collectionLazy2",
"collectionLazy3",
"collectionLazy4",
"collectionLazy5",
"collectionLazy6",
"collectionLazy7");
checkNotInCores(cc, "collectionLazy8", "collectionLazy9");
SolrCore core8 = cc.getCore("collectionLazy8");
checkInCores(
cc,
"collection1",
"collectionLazy2",
"collectionLazy4",
"collectionLazy5",
"collectionLazy6",
"collectionLazy7",
"collectionLazy8");
checkNotInCores(cc, "collectionLazy3", "collectionLazy9");
SolrCore core9 = cc.getCore("collectionLazy9");
checkInCores(
cc,
"collection1",
"collectionLazy4",
"collectionLazy5",
"collectionLazy6",
"collectionLazy7",
"collectionLazy8",
"collectionLazy9");
checkNotInCores(cc, "collectionLazy2", "collectionLazy3");
// Note decrementing the count when the core is removed from the lazyCores list is
// appropriate, since the
// refcount is 1 when constructed. anyone _else_ who's opened up one has to close it.
core1.close();
core2.close();
core3.close();
core4.close();
core5.close();
core6.close();
core7.close();
core8.close();
core9.close();
} finally {
cc.shutdown();
}
}
@Test
public void testRefCountMT() throws Exception {
SolrCore core = h.getCore();
assertTrue("Refcount != 1", core.getOpenCount() == 1);
final ClosingRequestHandler handler1 = new ClosingRequestHandler();
handler1.inform(core);
String path = "/this/is A path /that won't be registered!";
SolrRequestHandler old = core.registerRequestHandler(path, handler1);
assertNull(old); // should not be anything...
assertEquals(core.getRequestHandlers().get(path), handler1);
final int LOOP = 100;
final int MT = 16;
ExecutorService service =
ExecutorUtil.newMDCAwareFixedThreadPool(MT, new DefaultSolrThreadFactory("refCountMT"));
List<Callable<Integer>> callees = new ArrayList<>(MT);
final CoreContainer cores = h.getCoreContainer();
for (int i = 0; i < MT; ++i) {
Callable<Integer> call =
new Callable<Integer>() {
void yield(int n) {
try {
Thread.sleep(0, (n % 13 + 1) * 10);
} catch (InterruptedException xint) {
}
}
@Override
public Integer call() {
SolrCore core = null;
int r = 0;
try {
for (int l = 0; l < LOOP; ++l) {
r += 1;
core = cores.getCore(SolrTestCaseJ4.DEFAULT_TEST_CORENAME);
// sprinkle concurrency hinting...
yield(l);
assertTrue("Refcount < 1", core.getOpenCount() >= 1);
yield(l);
assertTrue("Refcount > 17", core.getOpenCount() <= 17);
yield(l);
assertTrue("Handler is closed", handler1.closed == false);
yield(l);
core.close();
core = null;
yield(l);
}
return r;
} finally {
if (core != null) core.close();
}
}
};
callees.add(call);
}
List<Future<Integer>> results = service.invokeAll(callees);
for (Future<Integer> result : results) {
assertTrue("loop=" + result.get() + " < " + LOOP, result.get() >= LOOP);
}
cores.shutdown();
assertTrue("Refcount != 0", core.getOpenCount() == 0);
assertTrue("Handler not closed", core.isClosed() && handler1.closed == true);
service.shutdown();
assertTrue("Running for too long...", service.awaitTermination(60, TimeUnit.SECONDS));
}
