/**
* Get the generation (N) of the current segments_N file in the directory.
*
* @param directory -- directory to search for the latest segments_N file
*/
public static long getCurrentSegmentGeneration(Directory directory) throws IOException {
try {
return getCurrentSegmentGeneration(directory.listAll());
} catch (NoSuchDirectoryException nsde) {
return -1;
}
}
/**
* @param worker
* @param connection
* @throws Exception
*/
private void createIndex(SearchIndexBuilderWorker worker, Connection connection)
throws Exception {
IndexWriter indexWrite = null;
try {
if (worker.isRunning()) {
indexWrite = indexStorage.getIndexWriter(false);
}
if (indexWrite != null) {
Document doc = new Document();
// The date of indexing
String timeStamp = String.valueOf(System.currentTimeMillis());
doc.add(
new Field(
SearchService.DATE_STAMP, timeStamp, Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(
new Field(
SearchService.DATE_STAMP,
CompressionTools.compressString(timeStamp),
Field.Store.YES));
String ref = "---INDEX-CREATED---";
doc.add(
new Field(
SearchService.FIELD_REFERENCE,
CompressionTools.compressString(ref),
Field.Store.YES));
doc.add(
new Field(
SearchService.FIELD_REFERENCE, ref, Field.Store.NO, Field.Index.NOT_ANALYZED));
indexWrite.addDocument(doc);
} else {
log.error("Couldn't get indexWriter to add document!");
}
} catch (Exception ex) {
log.error("Failed to Add Documents ", ex);
throw new Exception(ex);
} finally {
if (indexWrite != null) {
if (log.isDebugEnabled()) {
log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
Directory d = indexWrite.getDirectory();
String[] s = d.listAll();
log.debug("Directory Contains ");
for (int i = 0; i < s.length; i++) {
File f = new File(s[i]);
log.debug(
"\t"
+ String.valueOf(f.length())
+ "\t"
+ new Date(f.lastModified())
+ "\t"
+ s[i]);
}
}
indexStorage.closeIndexWriter(indexWrite);
}
}
}
static Map<String, String> readChecksums(Directory[] dirs, Map<String, String> defaultValue)
throws IOException {
long lastFound = -1;
Directory lastDir = null;
for (Directory dir : dirs) {
for (String name : dir.listAll()) {
if (!isChecksum(name)) {
continue;
}
long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length()));
if (current > lastFound) {
lastFound = current;
lastDir = dir;
}
}
}
if (lastFound == -1) {
return defaultValue;
}
IndexInput indexInput = lastDir.openInput(CHECKSUMS_PREFIX + lastFound, IOContext.READONCE);
try {
indexInput.readInt(); // version
return indexInput.readStringStringMap();
} catch (Exception e) {
// failed to load checksums, ignore and return an empty map
return defaultValue;
} finally {
indexInput.close();
}
}
private void assertNoPrx(Directory dir) throws Throwable {
final String[] files = dir.listAll();
for (int i = 0; i < files.length; i++) {
assertFalse(files[i].endsWith(".prx"));
assertFalse(files[i].endsWith(".pos"));
}
}
/**
* Creates a new <code>IndexHistory</code> from the given <code>dir</code>.
*
* @param dir the directory from where to read the index history.
* @param maxAge the maximum age in milliseconds for unused index infos.
* @throws IOException if an error occurs while reading the index history.
*/
IndexHistory(Directory dir, long maxAge) throws IOException {
this.indexDir = dir;
this.maxAge = maxAge;
// read all index infos
String[] names = dir.listAll();
if (names != null) {
for (String name : names) {
if (name.startsWith(INDEXES)) {
long gen;
if (name.length() == INDEXES.length()) {
gen = 0;
} else if (name.charAt(INDEXES.length()) == '_') {
gen = Long.parseLong(name.substring(INDEXES.length() + 1), Character.MAX_RADIX);
} else {
continue;
}
try {
IndexInfos infos = new IndexInfos(dir, INDEXES, gen);
indexInfosMap.put(gen, infos);
} catch (IOException e) {
log.warn("ignoring invalid index infos file: " + name);
}
}
}
}
}
private void assertNoNrm(Directory dir) throws Throwable {
final String[] files = dir.listAll();
for (int i = 0; i < files.length; i++) {
// TODO: this relies upon filenames
assertFalse(files[i].endsWith(".nrm") || files[i].endsWith(".len"));
}
}
public void assertDeleteContent(Store store, DirectoryService service) throws IOException {
store.deleteContent();
assertThat(
Arrays.toString(store.directory().listAll()),
store.directory().listAll().length,
equalTo(0));
assertThat(store.stats().sizeInBytes(), equalTo(0l));
for (Directory dir : service.build()) {
assertThat(dir.listAll().length, equalTo(0));
}
}
int countExtensions(Directory directory, String ext) throws IOException {
String[] fileNames = directory.listAll();
int fileCount = 0;
for (String fileName : fileNames) {
if (fileName.endsWith("." + ext)) {
fileCount++;
}
}
return fileCount;
}
// LUCENE-2790: tests that the non CFS files were deleted by addIndexes
public void testNonCFSLeftovers() throws Exception {
Directory[] dirs = new Directory[2];
for (int i = 0; i < dirs.length; i++) {
dirs[i] = new RAMDirectory();
IndexWriter w =
new IndexWriter(
dirs[i], new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document d = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.setStoreTermVectors(true);
d.add(new Field("c", "v", customType));
w.addDocument(d);
w.close();
}
IndexReader[] readers =
new IndexReader[] {DirectoryReader.open(dirs[0]), DirectoryReader.open(dirs[1])};
Directory dir = new MockDirectoryWrapper(random(), new RAMDirectory());
IndexWriterConfig conf =
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy(true));
MergePolicy lmp = conf.getMergePolicy();
// Force creation of CFS:
lmp.setNoCFSRatio(1.0);
lmp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
IndexWriter w3 = new IndexWriter(dir, conf);
w3.addIndexes(readers);
w3.close();
// we should now see segments_X,
// segments.gen,_Y.cfs,_Y.cfe, _Z.si
assertEquals(
"Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()),
5,
dir.listAll().length);
dir.close();
}
public LuceneFullTextIndex(AgeQuery qury, Collection<TextFieldExtractor> exts, File path)
throws IOException {
query = qury;
extractors = exts;
defaultFieldName = extractors.iterator().next().getName();
queryParser = new QueryParser(Version.LUCENE_30, defaultFieldName, analyzer);
if (path == null) index = new RAMDirectory();
else {
index = new NIOFSDirectory(path);
if (index.listAll().length != 0) searcher = new IndexSearcher(IndexReader.open(index));
}
}
StoreDirectory(Distributor distributor) throws IOException {
this.distributor = distributor;
synchronized (mutex) {
MapBuilder<String, StoreFileMetaData> builder = MapBuilder.newMapBuilder();
Map<String, String> checksums =
readChecksums(distributor.all(), new HashMap<String, String>());
for (Directory delegate : distributor.all()) {
for (String file : delegate.listAll()) {
String checksum = checksums.get(file);
builder.put(
file, new StoreFileMetaData(file, delegate.fileLength(file), checksum, delegate));
}
}
filesMetadata = builder.immutableMap();
files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]);
}
}
/**
* Load some keys in the collector, excluding some and to a maximum number of collected
* (non-excluded) keys.
*
* @param keysCollector the set where to add loaded keys to
* @param keysToExclude which keys should not be loaded. Warning: can be null! Means all keys are
* to be returned
* @param maxElements upper limit for collection
*/
private void loadSomeKeys(
final HashSet<IndexScopedKey> keysCollector,
final Set<IndexScopedKey> keysToExclude,
final int maxElements) {
if (maxElements <= 0) {
return;
}
int collectedKeys = 0;
// First we collect the (single) FileListCacheKey
FileListCacheKey rootKey = new FileListCacheKey(indexName);
if (keysToExclude == null || !keysToExclude.contains(rootKey)) { // unless it was excluded
if (keysCollector.add(rootKey)) { // unless it was already collected
collectedKeys++;
}
}
try {
// Now we collect first all FileCacheKey (keys for file metadata)
String[] listAll = directory.listAll();
for (String fileName : listAll) {
if (collectedKeys >= maxElements) return;
FileCacheKey key = new FileCacheKey(indexName, fileName);
if (keysToExclude == null || !keysToExclude.contains(key)) {
if (keysCollector.add(key)) {
if (++collectedKeys >= maxElements) return;
}
}
}
// Next we load the ChunkCacheKey (keys for file contents)
for (String fileName : listAll) {
int numChunksInt = figureChunksNumber(fileName);
for (int i = 0; i < numChunksInt; i++) {
// Inner loop: we actually have several Chunks per file name
ChunkCacheKey key = new ChunkCacheKey(indexName, fileName, i, autoChunkSize);
if (keysToExclude == null || !keysToExclude.contains(key)) {
if (keysCollector.add(key)) {
if (++collectedKeys >= maxElements) return;
}
}
}
}
} catch (IOException e) {
throw log.exceptionInCacheLoader(e);
}
}
/** @see org.apache.lucene.index.IndexReader#listCommits */
public static Collection<IndexCommit> listCommits(Directory dir) throws IOException {
final String[] files = dir.listAll();
List<IndexCommit> commits = new ArrayList<IndexCommit>();
SegmentInfos latest = new SegmentInfos();
latest.read(dir);
final long currentGen = latest.getGeneration();
commits.add(new ReaderCommit(latest, dir));
for (int i = 0; i < files.length; i++) {
final String fileName = files[i];
if (fileName.startsWith(IndexFileNames.SEGMENTS)
&& !fileName.equals(IndexFileNames.SEGMENTS_GEN)
&& SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) {
SegmentInfos sis = new SegmentInfos();
try {
// IOException allowed to throw there, in case
// segments_N is corrupt
sis.read(dir, fileName);
} catch (FileNotFoundException fnfe) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
sis = null;
}
if (sis != null) commits.add(new ReaderCommit(sis, dir));
}
}
// Ensure that the commit points are sorted in ascending order.
Collections.sort(commits);
return commits;
}
IndexSearcher getIndexSearcher(String key) {
IndexSearcher searcher = indexSearchers.get(key);
if (searcher == null) {
try {
File fsDirectory = new File(storeDir, key);
if (!fsDirectory.exists()) {
return null;
}
Directory dir = FSDirectory.open(fsDirectory);
if (dir.listAll().length == 0) {
return null;
}
searcher = new IndexSearcher(dir, true);
} catch (IOException e) {
throw new RuntimeException(e);
}
indexSearchers.put(key, searcher);
}
return searcher;
}
public static void checkIndex(ESLogger logger, Store store, ShardId shardId) {
if (store.tryIncRef()) {
logger.info("start check index");
try {
Directory dir = store.directory();
if (!Lucene.indexExists(dir)) {
return;
}
if (IndexWriter.isLocked(dir)) {
ESTestCase.checkIndexFailed = true;
throw new IllegalStateException("IndexWriter is still open on shard " + shardId);
}
try (CheckIndex checkIndex = new CheckIndex(dir)) {
BytesStreamOutput os = new BytesStreamOutput();
PrintStream out = new PrintStream(os, false, StandardCharsets.UTF_8.name());
checkIndex.setInfoStream(out);
out.flush();
CheckIndex.Status status = checkIndex.checkIndex();
if (!status.clean) {
ESTestCase.checkIndexFailed = true;
logger.warn(
"check index [failure] index files={}\n{}",
Arrays.toString(dir.listAll()),
new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
throw new IOException("index check failure");
} else {
if (logger.isDebugEnabled()) {
logger.debug(
"check index [success]\n{}",
new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
}
}
}
} catch (Exception e) {
logger.warn("failed to check index", e);
} finally {
logger.info("end check index");
store.decRef();
}
}
}
public void testOverrideWriteLocker() throws IOException {
Directory dir = null;
try {
dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEX_NAME)
.chunkSize(BUFFER_SIZE)
.overrideWriteLocker(
new LockFactory() {
@Override
public Lock makeLock(String lockName) {
return null;
}
@Override
public void clearLock(String lockName) throws IOException {}
})
.create();
AssertJUnit.assertEquals(0, dir.listAll().length);
} finally {
if (dir != null) dir.close();
}
}
public static void deleteDirectoryContents(Directory directory) throws IOException {
for (String fileName : directory.listAll()) directory.deleteFile(fileName);
}
@Test(enabled = false)
public void testReadChunks() throws Exception {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
// create file headers
FileMetadata file1 = new FileMetadata(5);
FileCacheKey key1 = new FileCacheKey(INDEXNAME, "Hello.txt");
cache.put(key1, file1);
FileMetadata file2 = new FileMetadata(5);
FileCacheKey key2 = new FileCacheKey(INDEXNAME, "World.txt");
cache.put(key2, file2);
// byte array for Hello.txt
String helloText = "Hello world. This is some text.";
cache.put(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE), helloText.getBytes());
// byte array for World.txt - should be in at least 2 chunks.
String worldText =
"This String should contain more than sixty four characters but less than one hundred and twenty eight.";
assert worldText.getBytes().length > BUFFER_SIZE;
assert worldText.getBytes().length < (2 * BUFFER_SIZE);
byte[] buf = new byte[BUFFER_SIZE];
System.arraycopy(worldText.getBytes(), 0, buf, 0, BUFFER_SIZE);
cache.put(new ChunkCacheKey(INDEXNAME, "World.txt", 0, BUFFER_SIZE), buf);
String part1 = new String(buf);
buf = new byte[BUFFER_SIZE];
System.arraycopy(worldText.getBytes(), BUFFER_SIZE, buf, 0, worldText.length() - BUFFER_SIZE);
cache.put(new ChunkCacheKey(INDEXNAME, "World.txt", 1, BUFFER_SIZE), buf);
String part2 = new String(buf);
// make sure the generated bytes do add up!
AssertJUnit.assertEquals(part1 + part2.trim(), worldText);
file1.setSize(helloText.length());
file2.setSize(worldText.length());
Set<String> s = new HashSet<String>();
s.add("Hello.txt");
s.add("World.txt");
Set other = new HashSet(Arrays.asList(dir.listAll()));
// ok, file listing works.
AssertJUnit.assertEquals(s, other);
IndexInput ii = dir.openInput("Hello.txt");
assert ii.length() == helloText.length();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (int i = 0; i < ii.length(); i++) {
baos.write(ii.readByte());
}
assert new String(baos.toByteArray()).equals(helloText);
ii = dir.openInput("World.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
for (int i = 0; i < ii.length(); i++) {
baos.write(ii.readByte());
}
assert new String(baos.toByteArray()).equals(worldText);
// now with buffered reading
ii = dir.openInput("Hello.txt");
assert ii.length() == helloText.length();
baos = new ByteArrayOutputStream();
long toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(helloText);
ii = dir.openInput("World.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(worldText);
dir.deleteFile("Hello.txt");
assert null == cache.get(new FileCacheKey(INDEXNAME, "Hello.txt"));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE));
Object ob1 = cache.get(new FileCacheKey(INDEXNAME, "World.txt"));
Object ob2 = cache.get(new ChunkCacheKey(INDEXNAME, "World.txt", 0, BUFFER_SIZE));
Object ob3 = cache.get(new ChunkCacheKey(INDEXNAME, "World.txt", 1, BUFFER_SIZE));
((DirectoryExtensions) dir).renameFile("World.txt", "HelloWorld.txt");
assert null == cache.get(new FileCacheKey(INDEXNAME, "Hello.txt"));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 1, BUFFER_SIZE));
assert cache.get(new FileCacheKey(INDEXNAME, "HelloWorld.txt")).equals(ob1);
assert cache.get(new ChunkCacheKey(INDEXNAME, "HelloWorld.txt", 0, BUFFER_SIZE)).equals(ob2);
assert cache.get(new ChunkCacheKey(INDEXNAME, "HelloWorld.txt", 1, BUFFER_SIZE)).equals(ob3);
// test that contents survives a move
ii = dir.openInput("HelloWorld.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(worldText);
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
public Object run(IndexCommit commit) throws CorruptIndexException, IOException {
if (commit != null) {
if (directory != commit.getDirectory())
throw new IOException("the specified commit does not match the specified Directory");
return doBody(commit.getSegmentsFileName());
}
String segmentFileName = null;
long lastGen = -1;
long gen = 0;
int genLookaheadCount = 0;
IOException exc = null;
int retryCount = 0;
boolean useFirstMethod = true;
// Loop until we succeed in calling doBody() without
// hitting an IOException. An IOException most likely
// means a commit was in process and has finished, in
// the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a
// true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on
// which generation we are trying to load. If we
// don't, then the original error is real and we throw
// it.
// We have three methods for determining the current
// generation. We try the first two in parallel (when
// useFirstMethod is true), and fall back to the third
// when necessary.
while (true) {
if (useFirstMethod) {
// List the directory and use the highest
// segments_N file. This method works well as long
// as there is no stale caching on the directory
// contents (NOTE: NFS clients often have such stale
// caching):
String[] files = null;
long genA = -1;
files = directory.listAll();
if (files != null) {
genA = getCurrentSegmentGeneration(files);
}
if (infoStream != null) {
message("directory listing genA=" + genA);
}
// Also open segments.gen and read its
// contents. Then we take the larger of the two
// gens. This way, if either approach is hitting
// a stale cache (NFS) we have a better chance of
// getting the right generation.
long genB = -1;
for (int i = 0; i < defaultGenFileRetryCount; i++) {
IndexInput genInput = null;
try {
genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN);
} catch (FileNotFoundException e) {
if (infoStream != null) {
message("segments.gen open: FileNotFoundException " + e);
}
break;
} catch (IOException e) {
if (infoStream != null) {
message("segments.gen open: IOException " + e);
}
}
if (genInput != null) {
try {
int version = genInput.readInt();
if (version == FORMAT_LOCKLESS) {
long gen0 = genInput.readLong();
long gen1 = genInput.readLong();
if (infoStream != null) {
message("fallback check: " + gen0 + "; " + gen1);
}
if (gen0 == gen1) {
// The file is consistent.
genB = gen0;
break;
}
}
} catch (IOException err2) {
// will retry
} finally {
genInput.close();
}
}
try {
Thread.sleep(defaultGenFileRetryPauseMsec);
} catch (InterruptedException ie) {
throw new ThreadInterruptedException(ie);
}
}
if (infoStream != null) {
message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB);
}
// Pick the larger of the two gen's:
if (genA > genB) gen = genA;
else gen = genB;
if (gen == -1) {
// Neither approach found a generation
throw new IndexNotFoundException(
"no segments* file found in " + directory + ": files: " + Arrays.toString(files));
}
}
if (useFirstMethod && lastGen == gen && retryCount >= 2) {
// Give up on first method -- this is 3rd cycle on
// listing directory and checking gen file to
// attempt to locate the segments file.
useFirstMethod = false;
}
// Second method: since both directory cache and
// file contents cache seem to be stale, just
// advance the generation.
if (!useFirstMethod) {
if (genLookaheadCount < defaultGenLookaheadCount) {
gen++;
genLookaheadCount++;
if (infoStream != null) {
message("look ahead increment gen to " + gen);
}
} else {
// All attempts have failed -- throw first exc:
throw exc;
}
} else if (lastGen == gen) {
// This means we're about to try the same
// segments_N last tried.
retryCount++;
} else {
// Segment file has advanced since our last loop
// (we made "progress"), so reset retryCount:
retryCount = 0;
}
lastGen = gen;
segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
try {
Object v = doBody(segmentFileName);
if (infoStream != null) {
message("success on " + segmentFileName);
}
return v;
} catch (IOException err) {
// Save the original root cause:
if (exc == null) {
exc = err;
}
if (infoStream != null) {
message(
"primary Exception on '"
+ segmentFileName
+ "': "
+ err
+ "'; will retry: retryCount="
+ retryCount
+ "; gen = "
+ gen);
}
if (gen > 1 && useFirstMethod && retryCount == 1) {
// This is our second time trying this same segments
// file (because retryCount is 1), and, there is
// possibly a segments_(N-1) (because gen > 1).
// So, check if the segments_(N-1) exists and
// try it if so:
String prevSegmentFileName =
IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1);
final boolean prevExists;
prevExists = directory.fileExists(prevSegmentFileName);
if (prevExists) {
if (infoStream != null) {
message("fallback to prior segment file '" + prevSegmentFileName + "'");
}
try {
Object v = doBody(prevSegmentFileName);
if (infoStream != null) {
message("success on fallback " + prevSegmentFileName);
}
return v;
} catch (IOException err2) {
if (infoStream != null) {
message(
"secondary Exception on '"
+ prevSegmentFileName
+ "': "
+ err2
+ "'; will retry");
}
}
}
}
}
}
}
/**
* Get the generation of the most recent commit to the index in this directory (N in the
* segments_N file).
*
* @param directory -- directory to search for the latest segments_N file
*/
public static long getLastCommitGeneration(Directory directory) throws IOException {
return getLastCommitGeneration(directory.listAll());
}
@Override
public String[] listAll() throws IOException {
return fsDir.listAll();
}
/** Run {@link #doBody} on the provided commit. */
public T run(IndexCommit commit) throws IOException {
if (commit != null) {
if (directory != commit.getDirectory())
throw new IOException("the specified commit does not match the specified Directory");
return doBody(commit.getSegmentsFileName());
}
long lastGen = -1;
long gen = -1;
IOException exc = null;
// Loop until we succeed in calling doBody() without
// hitting an IOException. An IOException most likely
// means an IW deleted our commit while opening
// the time it took us to load the now-old infos files
// (and segments files). It's also possible it's a
// true error (corrupt index). To distinguish these,
// on each retry we must see "forward progress" on
// which generation we are trying to load. If we
// don't, then the original error is real and we throw
// it.
for (; ; ) {
lastGen = gen;
String files[] = directory.listAll();
String files2[] = directory.listAll();
Arrays.sort(files);
Arrays.sort(files2);
if (!Arrays.equals(files, files2)) {
// listAll() is weakly consistent, this means we hit "concurrent modification exception"
continue;
}
gen = getLastCommitGeneration(files);
if (infoStream != null) {
message("directory listing gen=" + gen);
}
if (gen == -1) {
throw new IndexNotFoundException(
"no segments* file found in " + directory + ": files: " + Arrays.toString(files));
} else if (gen > lastGen) {
String segmentFileName =
IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
try {
T t = doBody(segmentFileName);
if (infoStream != null) {
message("success on " + segmentFileName);
}
return t;
} catch (IOException err) {
// Save the original root cause:
if (exc == null) {
exc = err;
}
if (infoStream != null) {
message(
"primary Exception on '"
+ segmentFileName
+ "': "
+ err
+ "'; will retry: gen = "
+ gen);
}
}
} else {
throw exc;
}
}
}
public void testDocsStuckInRAMForever() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
iwc.setRAMBufferSizeMB(.2);
Codec codec = Codec.forName("Lucene49");
iwc.setCodec(codec);
iwc.setMergePolicy(NoMergePolicy.INSTANCE);
final IndexWriter w = new IndexWriter(dir, iwc);
final CountDownLatch startingGun = new CountDownLatch(1);
Thread[] threads = new Thread[2];
for (int i = 0; i < threads.length; i++) {
final int threadID = i;
threads[i] =
new Thread() {
@Override
public void run() {
try {
startingGun.await();
for (int j = 0; j < 1000; j++) {
Document doc = new Document();
doc.add(newStringField("field", "threadID" + threadID, Field.Store.NO));
w.addDocument(doc);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
};
threads[i].start();
}
startingGun.countDown();
for (Thread t : threads) {
t.join();
}
Set<String> segSeen = new HashSet<>();
int thread0Count = 0;
int thread1Count = 0;
// At this point the writer should have 2 thread states w/ docs; now we index with only 1 thread
// until we see all 1000 thread0 & thread1
// docs flushed. If the writer incorrectly holds onto previously indexed docs forever then this
// will run forever:
while (thread0Count < 1000 || thread1Count < 1000) {
Document doc = new Document();
doc.add(newStringField("field", "threadIDmain", Field.Store.NO));
w.addDocument(doc);
for (String fileName : dir.listAll()) {
if (fileName.endsWith(".si")) {
String segName = IndexFileNames.parseSegmentName(fileName);
if (segSeen.contains(segName) == false) {
segSeen.add(segName);
SegmentInfo si =
new Lucene46SegmentInfoFormat()
.getSegmentInfoReader()
.read(dir, segName, IOContext.DEFAULT);
si.setCodec(codec);
SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, -1, -1, -1);
SegmentReader sr = new SegmentReader(sci, 1, IOContext.DEFAULT);
try {
thread0Count += sr.docFreq(new Term("field", "threadID0"));
thread1Count += sr.docFreq(new Term("field", "threadID1"));
} finally {
sr.close();
}
}
}
}
}
w.close();
dir.close();
}
/*
* Test a silly deletion policy that keeps all commits around.
*/
public void testKeepAllDeletionPolicy() throws IOException {
for (int pass = 0; pass < 2; pass++) {
if (VERBOSE) {
System.out.println("TEST: cycle pass="******"TEST: open writer for forceMerge");
}
writer = new IndexWriter(dir, conf);
policy = (KeepAllDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
writer.forceMerge(1);
writer.close();
}
assertEquals(needsMerging ? 2 : 1, policy.numOnInit);
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
assertEquals(1 + (needsMerging ? 1 : 0), policy.numOnCommit);
// Test listCommits
Collection<IndexCommit> commits = DirectoryReader.listCommits(dir);
// 2 from closing writer
assertEquals(1 + (needsMerging ? 1 : 0), commits.size());
// Make sure we can open a reader on each commit:
for (final IndexCommit commit : commits) {
IndexReader r = DirectoryReader.open(commit);
r.close();
}
// Simplistic check: just verify all segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.getLastCommitGeneration(dir);
while (gen > 0) {
IndexReader reader = DirectoryReader.open(dir);
reader.close();
dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
gen--;
if (gen > 0) {
// Now that we've removed a commit point, which
// should have orphan'd at least one index file.
// Open & close a writer and assert that it
// actually removed something:
int preCount = dir.listAll().length;
writer =
new IndexWriter(
dir,
newIndexWriterConfig(new MockAnalyzer(random()))
.setOpenMode(OpenMode.APPEND)
.setIndexDeletionPolicy(policy));
writer.close();
int postCount = dir.listAll().length;
assertTrue(postCount < preCount);
}
}
dir.close();
}
}
private boolean doRestore() throws Exception {
Path backupPath = Paths.get(backupLocation).resolve(backupName);
SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT);
String restoreIndexName = "restore." + dateFormat.format(new Date());
String restoreIndexPath = core.getDataDir() + restoreIndexName;
Directory restoreIndexDir = null;
Directory indexDir = null;
try (Directory backupDir = FSDirectory.open(backupPath)) {
final Version version =
IndexFetcher.checkOldestVersion(SegmentInfos.readLatestCommit(backupDir));
restoreIndexDir =
core.getDirectoryFactory()
.get(
restoreIndexPath,
DirectoryFactory.DirContext.DEFAULT,
core.getSolrConfig().indexConfig.lockType);
// Prefer local copy.
indexDir =
core.getDirectoryFactory()
.get(
core.getIndexDir(),
DirectoryFactory.DirContext.DEFAULT,
core.getSolrConfig().indexConfig.lockType);
// Move all files from backupDir to restoreIndexDir
for (String filename : backupDir.listAll()) {
checkInterrupted();
log.info("Copying file {} to restore directory ", filename);
try (IndexInput indexInput = backupDir.openInput(filename, IOContext.READONCE)) {
Long checksum = null;
try {
checksum = CodecUtil.retrieveChecksum(indexInput);
} catch (Exception e) {
log.warn("Could not read checksum from index file: " + filename, e);
}
long length = indexInput.length();
IndexFetcher.CompareResult compareResult =
IndexFetcher.compareFile(indexDir, version, filename, length, checksum);
if (!compareResult.equal
|| (!compareResult.checkSummed
&& (filename.endsWith(".si")
|| filename.endsWith(".liv")
|| filename.startsWith("segments_")))) {
restoreIndexDir.copyFrom(backupDir, filename, filename, IOContext.READONCE);
} else {
// prefer local copy
restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE);
}
} catch (Exception e) {
throw new SolrException(
SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e);
}
}
log.debug("Switching directories");
IndexFetcher.modifyIndexProps(core, restoreIndexName);
boolean success;
try {
core.getUpdateHandler().newIndexWriter(false);
openNewSearcher();
success = true;
log.info("Successfully restored to the backup index");
} catch (Exception e) {
// Rollback to the old index directory. Delete the restore index directory and mark the
// restore as failed.
log.warn("Could not switch to restored index. Rolling back to the current index");
Directory dir = null;
try {
dir =
core.getDirectoryFactory()
.get(
core.getDataDir(),
DirectoryFactory.DirContext.META_DATA,
core.getSolrConfig().indexConfig.lockType);
dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
} finally {
if (dir != null) {
core.getDirectoryFactory().release(dir);
}
}
core.getDirectoryFactory().doneWithDirectory(restoreIndexDir);
core.getDirectoryFactory().remove(restoreIndexDir);
core.getUpdateHandler().newIndexWriter(false);
openNewSearcher();
throw new SolrException(
SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e);
}
if (success) {
core.getDirectoryFactory().doneWithDirectory(indexDir);
core.getDirectoryFactory().remove(indexDir);
}
return true;
} finally {
if (restoreIndexDir != null) {
core.getDirectoryFactory().release(restoreIndexDir);
}
if (indexDir != null) {
core.getDirectoryFactory().release(indexDir);
}
}
}
/**
* Load implementation for FileListCacheKey; must return a ConcurrentHashSet containing the names
* of all files in this Directory.
*/
private Object loadIntern() throws IOException {
final String[] listAll = directory.listAll();
return new FileListCacheValue(listAll);
}
private void processAdd(
SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
throws Exception {
IndexWriter indexWrite = null;
try {
if (worker.isRunning()) {
indexWrite = indexStorage.getIndexWriter(false);
}
long last = System.currentTimeMillis();
for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
worker.isRunning() && tditer.hasNext(); ) {
Reader contentReader = null;
try {
SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
// only add adds, that have been deleted or are locked
// sucessfully
if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate())
&& !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
continue;
}
// Reference ref =
// entityManager.newReference(sbi.getName());
String ref = sbi.getName();
if (ref == null) {
log.error(
"Unrecognised trigger object presented to index builder " //$NON-NLS-1$
+ sbi);
}
long startDocIndex = System.currentTimeMillis();
worker.setStartDocIndex(startDocIndex);
worker.setNowIndexing(ref);
try {
try {
// Entity entity = ref.getEntity();
EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref);
boolean indexDoc = true;
if (searchIndexBuilder.isOnlyIndexSearchToolSites()) {
try {
String siteId = sep.getSiteId(sbi.getName());
Site s = SiteService.getSite(siteId);
ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$
if (t == null) {
indexDoc = false;
log.debug(
"Not indexing " //$NON-NLS-1$
+ sbi.getName()
+ " as it has no search tool"); //$NON-NLS-1$
}
} catch (Exception ex) {
indexDoc = false;
log.debug(
"Not indexing "
+ sbi.getName() // $NON-NLS-1$
+ " as it has no site",
ex); //$NON-NLS-1$
}
}
if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) {
DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService);
// Reader contentReader = null;
Document doc =
DocumentIndexingUtils.createIndexDocument(
ref,
digestStorageUtil,
sep,
serverConfigurationService.getServerUrl(),
contentReader);
// indexDocTMP(ref, sep);
log.debug("Indexing Document " + doc); // $NON-NLS-1$
indexWrite.addDocument(doc);
log.debug("Done Indexing Document " + doc); // $NON-NLS-1$
processRDF(ref, sep);
} else {
if (log.isDebugEnabled()) {
if (!indexDoc) {
log.debug("Ignored Document: Fileteed out by site " + ref); // $NON-NLS-1$
} else if (sep == null) {
log.debug("Ignored Document: No EntityContentProducer " + ref); // $NON-NLS-1$
} else if (!sep.isForIndex(ref)) {
log.debug("Ignored Document: Marked as Ignore " + ref); // $NON-NLS-1$
} else if (sep.getSiteId(ref) == null) {
log.debug("Ignored Document: No Site ID " + ref); // $NON-NLS-1$
} else {
log.debug("Ignored Document: Reason Unknown " + ref); // $NON-NLS-1$
}
}
}
} catch (Exception e1) {
log.debug(
" Failed to index document for "
+ ref
+ " cause: " //$NON-NLS-1$
+ e1.getMessage(),
e1);
}
sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, sbi);
} catch (Exception e1) {
log.debug(
" Failed to index document cause: " //$NON-NLS-1$
+ e1.getMessage());
}
long endDocIndex = System.currentTimeMillis();
worker.setLastIndex(endDocIndex - startDocIndex);
if ((endDocIndex - startDocIndex) > 60000L) {
log.warn(
"Slow index operation " //$NON-NLS-1$
+ String.valueOf((endDocIndex - startDocIndex) / 1000)
+ " seconds to index " //$NON-NLS-1$
+ ref);
}
// update this node lock to indicate its
// still alove, no document should
// take more than 2 mins to process
// ony do this check once every minute
long now = System.currentTimeMillis();
if ((now - last) > (60L * 1000L)) {
last = System.currentTimeMillis();
if (!worker.getLockTransaction(15L * 60L * 1000L, true)) {
throw new Exception(
"Transaction Lock Expired while indexing " //$NON-NLS-1$
+ ref);
}
}
} finally {
if (contentReader != null) {
try {
contentReader.close();
} catch (IOException ioex) {
log.debug(ioex);
}
}
}
}
worker.setStartDocIndex(System.currentTimeMillis());
worker.setNowIndexing(
Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); // $NON-NLS-1$
} catch (Exception ex) {
log.error("Failed to Add Documents ", ex);
throw new Exception(ex);
} finally {
if (indexWrite != null) {
if (log.isDebugEnabled()) {
log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
Directory d = indexWrite.getDirectory();
String[] s = d.listAll();
log.debug("Directory Contains ");
for (int i = 0; i < s.length; i++) {
File f = new File(s[i]);
log.debug(
"\t"
+ String.valueOf(f.length())
+ "\t"
+ new Date(f.lastModified())
+ "\t"
+ s[i]);
}
}
indexStorage.closeIndexWriter(indexWrite);
}
}
}
public IndexInfo(String indexName, Directory directory, Analyzer analyzer)
throws CorruptIndexException, IOException {
IndexWriter iwriter = null;
IndexReader indexReader = null;
try {
iwriter = new IndexWriter(directory, analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED);
indexReader = iwriter.getReader();
this.index_name = indexName;
this.index_version = indexReader.getVersion();
this.optimized = indexReader.isOptimized();
this.numDocs = indexReader.numDocs();
Date date = new Date(IndexReader.lastModified(directory));
DateFormat formatter = new SimpleDateFormat("EEEE, dd MMMM yyyy, hh:mm:ss a");
this.lastModifyDate = formatter.format(date);
this.files = new HashMap<String, String>();
String[] files = directory.listAll();
for (int i = 0; i < files.length; i++) {
if (files[i].startsWith("_"))
this.files.put(
files[i],
Long.toString(
Math.round(
FileSizeConverter.convertSize(
directory.fileLength(files[i]), FileSizeConverter.KB)))
+ " KB");
}
indexReader.close();
iwriter.close();
iwriter = null;
directory.clearLock(directory.getLockID());
} catch (CorruptIndexException e) {
if (iwriter != null) {
iwriter.close();
iwriter = null;
}
if (indexReader != null) {
indexReader.close();
indexReader = null;
}
throw e;
} catch (LockObtainFailedException e) {
if (iwriter != null) {
iwriter.close();
iwriter = null;
}
if (indexReader != null) {
indexReader.close();
indexReader = null;
}
throw e;
} catch (IOException e) {
if (iwriter != null) {
iwriter.close();
iwriter = null;
}
if (indexReader != null) {
indexReader.close();
indexReader = null;
}
throw e;
}
}
