/**
* Returns human-readable form of the term text. If the term is not unicode, the raw bytes will be
* printed instead.
*/
public static final String toString(BytesRef termText) {
// the term might not be text, but usually is. so we make a best effort
CharsetDecoder decoder =
IOUtils.CHARSET_UTF_8
.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT);
try {
return decoder
.decode(ByteBuffer.wrap(termText.bytes, termText.offset, termText.length))
.toString();
} catch (CharacterCodingException e) {
return termText.toString();
}
}
@Test
public void testArray() throws Exception {
CharBlockArray array = new CharBlockArray();
StringBuilder builder = new StringBuilder();
final int n = 100 * 1000;
byte[] buffer = new byte[50];
for (int i = 0; i < n; i++) {
random().nextBytes(buffer);
int size = 1 + random().nextInt(50);
// This test is turning random bytes into a string,
// this is asking for trouble.
CharsetDecoder decoder =
IOUtils.CHARSET_UTF_8
.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
String s = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
array.append(s);
builder.append(s);
}
for (int i = 0; i < n; i++) {
random().nextBytes(buffer);
int size = 1 + random().nextInt(50);
// This test is turning random bytes into a string,
// this is asking for trouble.
CharsetDecoder decoder =
IOUtils.CHARSET_UTF_8
.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
String s = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
array.append((CharSequence) s);
builder.append(s);
}
for (int i = 0; i < n; i++) {
random().nextBytes(buffer);
int size = 1 + random().nextInt(50);
// This test is turning random bytes into a string,
// this is asking for trouble.
CharsetDecoder decoder =
IOUtils.CHARSET_UTF_8
.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
String s = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
for (int j = 0; j < s.length(); j++) {
array.append(s.charAt(j));
}
builder.append(s);
}
assertEqualsInternal("GrowingCharArray<->StringBuilder mismatch.", builder, array);
File tempDir = _TestUtil.getTempDir("growingchararray");
File f = new File(tempDir, "GrowingCharArrayTest.tmp");
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(f));
array.flush(out);
out.flush();
out.close();
BufferedInputStream in = new BufferedInputStream(new FileInputStream(f));
array = CharBlockArray.open(in);
assertEqualsInternal(
"GrowingCharArray<->StringBuilder mismatch after flush/load.", builder, array);
in.close();
f.delete();
}
@Test
public void testL2O() throws Exception {
LabelToOrdinal map = new LabelToOrdinalMap();
CompactLabelToOrdinal compact = new CompactLabelToOrdinal(2000000, 0.15f, 3);
final int n = atLeast(10 * 1000);
final int numUniqueValues = 50 * 1000;
String[] uniqueValues = new String[numUniqueValues];
byte[] buffer = new byte[50];
Random random = random();
for (int i = 0; i < numUniqueValues; ) {
random.nextBytes(buffer);
int size = 1 + random.nextInt(buffer.length);
// This test is turning random bytes into a string,
// this is asking for trouble.
CharsetDecoder decoder =
IOUtils.CHARSET_UTF_8
.newDecoder()
.onUnmappableCharacter(CodingErrorAction.REPLACE)
.onMalformedInput(CodingErrorAction.REPLACE);
uniqueValues[i] = decoder.decode(ByteBuffer.wrap(buffer, 0, size)).toString();
// we cannot have empty path components, so eliminate all prefix as well
// as middle consecutive delimiter chars.
uniqueValues[i] = uniqueValues[i].replaceAll("/+", "/");
if (uniqueValues[i].startsWith("/")) {
uniqueValues[i] = uniqueValues[i].substring(1);
}
if (uniqueValues[i].indexOf(CompactLabelToOrdinal.TERMINATOR_CHAR) == -1) {
i++;
}
}
File tmpDir = TestUtil.getTempDir("testLableToOrdinal");
File f = new File(tmpDir, "CompactLabelToOrdinalTest.tmp");
int flushInterval = 10;
for (int i = 0; i < n; i++) {
if (i > 0 && i % flushInterval == 0) {
compact.flush(f);
compact = CompactLabelToOrdinal.open(f, 0.15f, 3);
assertTrue(f.delete());
if (flushInterval < (n / 10)) {
flushInterval *= 10;
}
}
int index = random.nextInt(numUniqueValues);
FacetLabel label;
String s = uniqueValues[index];
if (s.length() == 0) {
label = new FacetLabel();
} else {
label = new FacetLabel(s.split("/"));
}
int ord1 = map.getOrdinal(label);
int ord2 = compact.getOrdinal(label);
assertEquals(ord1, ord2);
if (ord1 == LabelToOrdinal.INVALID_ORDINAL) {
ord1 = compact.getNextOrdinal();
map.addLabel(label, ord1);
compact.addLabel(label, ord1);
}
}
for (int i = 0; i < numUniqueValues; i++) {
FacetLabel label;
String s = uniqueValues[i];
if (s.length() == 0) {
label = new FacetLabel();
} else {
label = new FacetLabel(s.split("/"));
}
int ord1 = map.getOrdinal(label);
int ord2 = compact.getOrdinal(label);
assertEquals(ord1, ord2);
}
}
public void testFlowWithEmpty() throws Exception {
// reused state
Map<String, Exception> failures = null;
Collection<String> cores = null;
Exception fail = null;
init("empty_flow");
// solr.xml
File solrXml = new File(solrHome, "solr.xml");
FileUtils.write(solrXml, EMPTY_SOLR_XML, IOUtils.CHARSET_UTF_8.toString());
// ----
// init the CoreContainer
cc.load(solrHome.getAbsolutePath(), solrXml);
// check that we have the cores we expect
cores = cc.getCoreNames();
assertNotNull("core names is null", cores);
assertEquals("wrong number of cores", 0, cores.size());
// check that we have the failures we expect
failures = cc.getCoreInitFailures();
assertNotNull("core failures is a null map", failures);
assertEquals("wrong number of core failures", 0, failures.size());
// -----
// try to add a collection with a path that doesn't exist
final CoreDescriptor bogus = new CoreDescriptor(cc, "bogus", "bogus_path");
try {
ignoreException(Pattern.quote("bogus_path"));
cc.create(bogus);
fail("bogus inst dir failed to trigger exception from create");
} catch (SolrException e) {
assertTrue(
"init exception doesn't mention bogus dir: " + e.getCause().getCause().getMessage(),
0 < e.getCause().getCause().getMessage().indexOf("bogus_path"));
}
// check that we have the cores we expect
cores = cc.getCoreNames();
assertNotNull("core names is null", cores);
assertEquals("wrong number of cores", 0, cores.size());
// check that we have the failures we expect
failures = cc.getCoreInitFailures();
assertNotNull("core failures is a null map", failures);
assertEquals("wrong number of core failures", 1, failures.size());
fail = failures.get("bogus");
assertNotNull("null failure for test core", fail);
assertTrue(
"init failure doesn't mention problem: " + fail.getCause().getMessage(),
0 < fail.getCause().getMessage().indexOf("bogus_path"));
// check that we get null accessing a non-existent core
assertNull(cc.getCore("does_not_exist"));
// check that we get a 500 accessing the core with an init failure
try {
SolrCore c = cc.getCore("bogus");
fail("Failed to get Exception on accessing core with init failure");
} catch (SolrException ex) {
assertEquals(500, ex.code());
// double wrapped
String cause = ex.getCause().getCause().getMessage();
assertTrue(
"getCore() ex cause doesn't mention init fail: " + cause,
0 < cause.indexOf("bogus_path"));
}
// let the test end here, with some recorded failures, and let cleanUp()
// verify that there is no problem shuting down CoreContainer with known
// SolrCore failures
}
public void testFlowBadFromStart() throws Exception {
// reused state
Map<String, Exception> failures = null;
Collection<String> cores = null;
Exception fail = null;
init("bad_flow");
// start with two collections: one valid, and one broken
File solrXml = new File(solrHome, "solr.xml");
FileUtils.write(solrXml, BAD_SOLR_XML, IOUtils.CHARSET_UTF_8.toString());
// our "ok" collection
FileUtils.copyFile(
getFile("solr/collection1/conf/solrconfig-basic.xml"),
FileUtils.getFile(solrHome, "col_ok", "conf", "solrconfig.xml"));
FileUtils.copyFile(
getFile("solr/collection1/conf/schema-minimal.xml"),
FileUtils.getFile(solrHome, "col_ok", "conf", "schema.xml"));
// our "bad" collection
ignoreException(Pattern.quote("DummyMergePolicy"));
FileUtils.copyFile(
getFile("solr/collection1/conf/bad-mp-solrconfig.xml"),
FileUtils.getFile(solrHome, "col_bad", "conf", "solrconfig.xml"));
FileUtils.copyFile(
getFile("solr/collection1/conf/schema-minimal.xml"),
FileUtils.getFile(solrHome, "col_bad", "conf", "schema.xml"));
// -----
// init the CoreContainer with the mix of ok/bad cores
cc.load(solrHome.getAbsolutePath(), solrXml);
// check that we have the cores we expect
cores = cc.getCoreNames();
assertNotNull("core names is null", cores);
assertEquals("wrong number of cores", 1, cores.size());
assertTrue("col_ok not found", cores.contains("col_ok"));
// check that we have the failures we expect
failures = cc.getCoreInitFailures();
assertNotNull("core failures is a null map", failures);
assertEquals("wrong number of core failures", 1, failures.size());
fail = failures.get("col_bad");
assertNotNull("null failure for test core", fail);
assertTrue(
"init failure doesn't mention problem: " + fail.getMessage(),
0 < fail.getMessage().indexOf("DummyMergePolicy"));
// check that we get null accessing a non-existent core
assertNull(cc.getCore("does_not_exist"));
// check that we get a 500 accessing the core with an init failure
try {
SolrCore c = cc.getCore("col_bad");
fail("Failed to get Exception on accessing core with init failure");
} catch (SolrException ex) {
assertEquals(500, ex.code());
// double wrapped
String cause = ex.getCause().getCause().getMessage();
assertTrue(
"getCore() ex cause doesn't mention init fail: " + cause,
0 < cause.indexOf("DummyMergePolicy"));
}
// -----
// "fix" the bad collection
FileUtils.copyFile(
getFile("solr/collection1/conf/solrconfig-basic.xml"),
FileUtils.getFile(solrHome, "col_bad", "conf", "solrconfig.xml"));
final CoreDescriptor fixed = new CoreDescriptor(cc, "col_bad", "col_bad");
cc.register("col_bad", cc.create(fixed), false);
// check that we have the cores we expect
cores = cc.getCoreNames();
assertNotNull("core names is null", cores);
assertEquals("wrong number of cores", 2, cores.size());
assertTrue("col_ok not found", cores.contains("col_ok"));
assertTrue("col_bad not found", cores.contains("col_bad"));
// check that we have the failures we expect
failures = cc.getCoreInitFailures();
assertNotNull("core failures is a null map", failures);
assertEquals("wrong number of core failures", 0, failures.size());
// -----
// try to add a collection with a path that doesn't exist
final CoreDescriptor bogus = new CoreDescriptor(cc, "bogus", "bogus_path");
try {
ignoreException(Pattern.quote("bogus_path"));
cc.create(bogus);
fail("bogus inst dir failed to trigger exception from create");
} catch (SolrException e) {
assertTrue(
"init exception doesn't mention bogus dir: " + e.getCause().getCause().getMessage(),
0 < e.getCause().getCause().getMessage().indexOf("bogus_path"));
}
// check that we have the cores we expect
cores = cc.getCoreNames();
assertNotNull("core names is null", cores);
assertEquals("wrong number of cores", 2, cores.size());
assertTrue("col_ok not found", cores.contains("col_ok"));
assertTrue("col_bad not found", cores.contains("col_bad"));
// check that we have the failures we expect
failures = cc.getCoreInitFailures();
assertNotNull("core failures is a null map", failures);
assertEquals("wrong number of core failures", 1, failures.size());
fail = failures.get("bogus");
assertNotNull("null failure for test core", fail);
assertTrue(
"init failure doesn't mention problem: " + fail.getCause().getMessage(),
0 < fail.getCause().getMessage().indexOf("bogus_path"));
// check that we get null accessing a non-existent core
assertNull(cc.getCore("does_not_exist"));
// check that we get a 500 accessing the core with an init failure
try {
SolrCore c = cc.getCore("bogus");
fail("Failed to get Exception on accessing core with init failure");
} catch (SolrException ex) {
assertEquals(500, ex.code());
// double wrapped
String cause = ex.getCause().getCause().getMessage();
assertTrue(
"getCore() ex cause doesn't mention init fail: " + cause,
0 < cause.indexOf("bogus_path"));
}
// -----
// register bogus as an alias for col_ok and confirm failure goes away
cc.register("bogus", cc.getCore("col_ok"), false);
// check that we have the cores we expect
cores = cc.getCoreNames();
assertNotNull("core names is null", cores);
assertEquals("wrong number of cores", 3, cores.size());
assertTrue("col_ok not found", cores.contains("col_ok"));
assertTrue("col_bad not found", cores.contains("col_bad"));
assertTrue("bogus not found", cores.contains("bogus"));
// check that we have the failures we expect
failures = cc.getCoreInitFailures();
assertNotNull("core failures is a null map", failures);
assertEquals("wrong number of core failures", 0, failures.size());
// -----
// break col_bad's config and try to RELOAD to add failure
final long col_bad_old_start = getCoreStartTime(cc, "col_bad");
FileUtils.write(
FileUtils.getFile(solrHome, "col_bad", "conf", "solrconfig.xml"),
"This is giberish, not valid XML <",
IOUtils.CHARSET_UTF_8.toString());
try {
ignoreException(Pattern.quote("SAX"));
cc.reload("col_bad");
fail("corrupt solrconfig.xml failed to trigger exception from reload");
} catch (SolrException e) {
assertTrue(
"We're supposed to have a wrapped SAXParserException here, but we don't",
e.getCause() instanceof SAXParseException);
SAXParseException se = (SAXParseException) e.getCause();
assertTrue(
"reload exception doesn't refer to slrconfig.xml " + se.getSystemId(),
0 < se.getSystemId().indexOf("solrconfig.xml"));
}
assertEquals(
"Failed core reload should not have changed start time",
col_bad_old_start,
getCoreStartTime(cc, "col_bad"));
// check that we have the cores we expect
cores = cc.getCoreNames();
assertNotNull("core names is null", cores);
assertEquals("wrong number of cores", 3, cores.size());
assertTrue("col_ok not found", cores.contains("col_ok"));
assertTrue("col_bad not found", cores.contains("col_bad"));
assertTrue("bogus not found", cores.contains("bogus"));
// check that we have the failures we expect
failures = cc.getCoreInitFailures();
assertNotNull("core failures is a null map", failures);
assertEquals("wrong number of core failures", 1, failures.size());
fail = failures.get("col_bad");
assertNotNull("null failure for test core", fail);
assertTrue("init failure isn't SAXParseException", fail instanceof SAXParseException);
assertTrue(
"init failure doesn't mention problem: " + fail.toString(),
0 < ((SAXParseException) fail).getSystemId().indexOf("solrconfig.xml"));
// ----
// fix col_bad's config (again) and RELOAD to fix failure
FileUtils.copyFile(
getFile("solr/collection1/conf/solrconfig-basic.xml"),
FileUtils.getFile(solrHome, "col_bad", "conf", "solrconfig.xml"));
cc.reload("col_bad");
assertTrue(
"Core reload should have changed start time",
col_bad_old_start < getCoreStartTime(cc, "col_bad"));
// check that we have the cores we expect
cores = cc.getCoreNames();
assertNotNull("core names is null", cores);
assertEquals("wrong number of cores", 3, cores.size());
assertTrue("col_ok not found", cores.contains("col_ok"));
assertTrue("col_bad not found", cores.contains("col_bad"));
assertTrue("bogus not found", cores.contains("bogus"));
// check that we have the failures we expect
failures = cc.getCoreInitFailures();
assertNotNull("core failures is a null map", failures);
assertEquals("wrong number of core failures", 0, failures.size());
}