@Override
protected void before() throws Exception {
// enable this by default, for IDE consistency with ant tests (as its the default from ant)
// TODO: really should be in solr base classes, but some extend LTC directly.
// we do this in beforeClass, because some tests currently disable it
restoreProperties.put("solr.directoryFactory", System.getProperty("solr.directoryFactory"));
if (System.getProperty("solr.directoryFactory") == null) {
System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockDirectoryFactory");
}
// Restore more Solr properties.
restoreProperties.put("solr.solr.home", System.getProperty("solr.solr.home"));
restoreProperties.put("solr.data.dir", System.getProperty("solr.data.dir"));
// if verbose: print some debugging stuff about which codecs are loaded.
if (VERBOSE) {
Set<String> codecs = Codec.availableCodecs();
for (String codec : codecs) {
System.out.println(
"Loaded codec: '" + codec + "': " + Codec.forName(codec).getClass().getName());
}
Set<String> postingsFormats = PostingsFormat.availablePostingsFormats();
for (String postingsFormat : postingsFormats) {
System.out.println(
"Loaded postingsFormat: '"
+ postingsFormat
+ "': "
+ PostingsFormat.forName(postingsFormat).getClass().getName());
}
}
savedInfoStream = InfoStream.getDefault();
final Random random = RandomizedContext.current().getRandom();
final boolean v = random.nextBoolean();
if (INFOSTREAM) {
InfoStream.setDefault(new ThreadNameFixingPrintStreamInfoStream(System.out));
} else if (v) {
InfoStream.setDefault(new NullInfoStream());
}
Class<?> targetClass = RandomizedContext.current().getTargetClass();
avoidCodecs = new HashSet<String>();
if (targetClass.isAnnotationPresent(SuppressCodecs.class)) {
SuppressCodecs a = targetClass.getAnnotation(SuppressCodecs.class);
avoidCodecs.addAll(Arrays.asList(a.value()));
}
// set back to default
LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = false;
LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = false;
savedCodec = Codec.getDefault();
int randomVal = random.nextInt(10);
if ("Lucene3x".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC)
&& "random".equals(TEST_POSTINGSFORMAT)
&& "random".equals(TEST_DOCVALUESFORMAT)
&& randomVal == 3
&& !shouldAvoidCodec("Lucene3x"))) { // preflex-only setup
codec = Codec.forName("Lucene3x");
assert (codec instanceof PreFlexRWCodec)
: "fix your classpath to have tests-framework.jar before lucene-core.jar";
LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true;
} else if ("Lucene40".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC)
&& "random".equals(TEST_POSTINGSFORMAT)
&& randomVal == 0
&& !shouldAvoidCodec("Lucene40"))) { // 4.0 setup
codec = Codec.forName("Lucene40");
LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
assert codec instanceof Lucene40RWCodec
: "fix your classpath to have tests-framework.jar before lucene-core.jar";
assert (PostingsFormat.forName("Lucene40") instanceof Lucene40RWPostingsFormat)
: "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if ("Lucene41".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC)
&& "random".equals(TEST_POSTINGSFORMAT)
&& "random".equals(TEST_DOCVALUESFORMAT)
&& randomVal == 1
&& !shouldAvoidCodec("Lucene41"))) {
codec = Codec.forName("Lucene41");
LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
assert codec instanceof Lucene41RWCodec
: "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if ("Lucene42".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC)
&& "random".equals(TEST_POSTINGSFORMAT)
&& "random".equals(TEST_DOCVALUESFORMAT)
&& randomVal == 2
&& !shouldAvoidCodec("Lucene42"))) {
codec = Codec.forName("Lucene42");
LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
assert codec instanceof Lucene42RWCodec
: "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if ("Lucene45".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC)
&& "random".equals(TEST_POSTINGSFORMAT)
&& "random".equals(TEST_DOCVALUESFORMAT)
&& randomVal == 5
&& !shouldAvoidCodec("Lucene45"))) {
codec = Codec.forName("Lucene45");
LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
assert codec instanceof Lucene45RWCodec
: "fix your classpath to have tests-framework.jar before lucene-core.jar";
} else if (("random".equals(TEST_POSTINGSFORMAT) == false)
|| ("random".equals(TEST_DOCVALUESFORMAT) == false)) {
// the user wired postings or DV: this is messy
// refactor into RandomCodec....
final PostingsFormat format;
if ("random".equals(TEST_POSTINGSFORMAT)) {
format = PostingsFormat.forName("Lucene41");
} else {
format = PostingsFormat.forName(TEST_POSTINGSFORMAT);
}
final DocValuesFormat dvFormat;
if ("random".equals(TEST_DOCVALUESFORMAT)) {
dvFormat = DocValuesFormat.forName("Lucene45");
} else {
dvFormat = DocValuesFormat.forName(TEST_DOCVALUESFORMAT);
}
codec =
new Lucene46Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return format;
}
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
return dvFormat;
}
@Override
public String toString() {
return super.toString() + ": " + format.toString() + ", " + dvFormat.toString();
}
};
} else if ("SimpleText".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC)
&& randomVal == 9
&& LuceneTestCase.rarely(random)
&& !shouldAvoidCodec("SimpleText"))) {
codec = new SimpleTextCodec();
} else if ("Appending".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC) && randomVal == 8 && !shouldAvoidCodec("Appending"))) {
codec = new AppendingRWCodec();
LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE =
true; // this is really just Lucene40 with some minor changes
} else if ("CheapBastard".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC)
&& randomVal == 8
&& !shouldAvoidCodec("CheapBastard")
&& !shouldAvoidCodec("Lucene41"))) {
// we also avoid this codec if Lucene41 is avoided, since thats the postings format it uses.
codec = new CheapBastardCodec();
} else if ("Asserting".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Asserting"))) {
codec = new AssertingCodec();
} else if ("Compressing".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Compressing"))) {
codec = CompressingCodec.randomInstance(random);
} else if (!"random".equals(TEST_CODEC)) {
codec = Codec.forName(TEST_CODEC);
} else if ("random".equals(TEST_POSTINGSFORMAT)) {
codec = new RandomCodec(random, avoidCodecs);
} else {
assert false;
}
Codec.setDefault(codec);
// Initialize locale/ timezone.
String testLocale = System.getProperty("tests.locale", "random");
String testTimeZone = System.getProperty("tests.timezone", "random");
// Always pick a random one for consistency (whether tests.locale was specified or not).
savedLocale = Locale.getDefault();
Locale randomLocale = randomLocale(random);
locale = testLocale.equals("random") ? randomLocale : localeForName(testLocale);
Locale.setDefault(locale);
// TimeZone.getDefault will set user.timezone to the default timezone of the user's locale.
// So store the original property value and restore it at end.
restoreProperties.put("user.timezone", System.getProperty("user.timezone"));
savedTimeZone = TimeZone.getDefault();
TimeZone randomTimeZone = randomTimeZone(random());
timeZone = testTimeZone.equals("random") ? randomTimeZone : TimeZone.getTimeZone(testTimeZone);
TimeZone.setDefault(timeZone);
similarity =
random().nextBoolean() ? new DefaultSimilarity() : new RandomSimilarityProvider(random());
// Check codec restrictions once at class level.
try {
checkCodecRestrictions(codec);
} catch (AssumptionViolatedException e) {
System.err.println(
"NOTE: "
+ e.getMessage()
+ " Suppressed codecs: "
+ Arrays.toString(avoidCodecs.toArray()));
throw e;
}
}
@Nightly
public void test() throws Exception {
MockDirectoryWrapper dir =
new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("4GBStoredFields")));
dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
iwc.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.setRAMBufferSizeMB(256.0);
iwc.setMergeScheduler(new ConcurrentMergeScheduler());
iwc.setMergePolicy(newLogMergePolicy(false, 10));
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
// TODO: we disable "Compressing" since it likes to pick very extreme values which will be too
// slow for this test.
// maybe we should factor out crazy cases to ExtremeCompressing? then annotations can handle
// this stuff...
if (random().nextBoolean()) {
iwc.setCodec(CompressingCodec.reasonableInstance(random()));
}
IndexWriter w = new IndexWriter(dir, iwc);
MergePolicy mp = w.getConfig().getMergePolicy();
if (mp instanceof LogByteSizeMergePolicy) {
// 1 petabyte:
((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
}
final Document doc = new Document();
final FieldType ft = new FieldType();
ft.setStored(true);
ft.freeze();
final int valueLength = RandomInts.randomIntBetween(random(), 1 << 13, 1 << 20);
final byte[] value = new byte[valueLength];
for (int i = 0; i < valueLength; ++i) {
// random so that even compressing codecs can't compress it
value[i] = (byte) random().nextInt(256);
}
final Field f = new Field("fld", value, ft);
doc.add(f);
final int numDocs = (int) ((1L << 32) / valueLength + 100);
for (int i = 0; i < numDocs; ++i) {
w.addDocument(doc);
if (VERBOSE && i % (numDocs / 10) == 0) {
System.out.println(i + " of " + numDocs + "...");
}
}
w.forceMerge(1);
w.close();
if (VERBOSE) {
boolean found = false;
for (String file : dir.listAll()) {
if (file.endsWith(".fdt")) {
final long fileLength = dir.fileLength(file);
if (fileLength >= 1L << 32) {
found = true;
}
System.out.println("File length of " + file + " : " + fileLength);
}
}
if (!found) {
System.out.println("No .fdt file larger than 4GB, test bug?");
}
}
DirectoryReader rd = DirectoryReader.open(dir);
Document sd = rd.document(numDocs - 1);
assertNotNull(sd);
assertEquals(1, sd.getFields().size());
BytesRef valueRef = sd.getBinaryValue("fld");
assertNotNull(valueRef);
assertEquals(new BytesRef(value), valueRef);
rd.close();
dir.close();
}
