/** * Get the generation (N) of the current segments_N file in the directory. * * @param directory -- directory to search for the latest segments_N file */ public static long getCurrentSegmentGeneration(Directory directory) throws IOException { try { return getCurrentSegmentGeneration(directory.listAll()); } catch (NoSuchDirectoryException nsde) { return -1; } }
/** * @param worker * @param connection * @throws Exception */ private void createIndex(SearchIndexBuilderWorker worker, Connection connection) throws Exception { IndexWriter indexWrite = null; try { if (worker.isRunning()) { indexWrite = indexStorage.getIndexWriter(false); } if (indexWrite != null) { Document doc = new Document(); // The date of indexing String timeStamp = String.valueOf(System.currentTimeMillis()); doc.add( new Field( SearchService.DATE_STAMP, timeStamp, Field.Store.NO, Field.Index.NOT_ANALYZED)); doc.add( new Field( SearchService.DATE_STAMP, CompressionTools.compressString(timeStamp), Field.Store.YES)); String ref = "---INDEX-CREATED---"; doc.add( new Field( SearchService.FIELD_REFERENCE, CompressionTools.compressString(ref), Field.Store.YES)); doc.add( new Field( SearchService.FIELD_REFERENCE, ref, Field.Store.NO, Field.Index.NOT_ANALYZED)); indexWrite.addDocument(doc); } else { log.error("Couldn't get indexWriter to add document!"); } } catch (Exception ex) { log.error("Failed to Add Documents ", ex); throw new Exception(ex); } finally { if (indexWrite != null) { if (log.isDebugEnabled()) { log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents"); Directory d = indexWrite.getDirectory(); String[] s = d.listAll(); log.debug("Directory Contains "); for (int i = 0; i < s.length; i++) { File f = new File(s[i]); log.debug( "\t" + String.valueOf(f.length()) + "\t" + new Date(f.lastModified()) + "\t" + s[i]); } } indexStorage.closeIndexWriter(indexWrite); } } }
static Map<String, String> readChecksums(Directory[] dirs, Map<String, String> defaultValue) throws IOException { long lastFound = -1; Directory lastDir = null; for (Directory dir : dirs) { for (String name : dir.listAll()) { if (!isChecksum(name)) { continue; } long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length())); if (current > lastFound) { lastFound = current; lastDir = dir; } } } if (lastFound == -1) { return defaultValue; } IndexInput indexInput = lastDir.openInput(CHECKSUMS_PREFIX + lastFound, IOContext.READONCE); try { indexInput.readInt(); // version return indexInput.readStringStringMap(); } catch (Exception e) { // failed to load checksums, ignore and return an empty map return defaultValue; } finally { indexInput.close(); } }
private void assertNoPrx(Directory dir) throws Throwable { final String[] files = dir.listAll(); for (int i = 0; i < files.length; i++) { assertFalse(files[i].endsWith(".prx")); assertFalse(files[i].endsWith(".pos")); } }
/** * Creates a new <code>IndexHistory</code> from the given <code>dir</code>. * * @param dir the directory from where to read the index history. * @param maxAge the maximum age in milliseconds for unused index infos. * @throws IOException if an error occurs while reading the index history. */ IndexHistory(Directory dir, long maxAge) throws IOException { this.indexDir = dir; this.maxAge = maxAge; // read all index infos String[] names = dir.listAll(); if (names != null) { for (String name : names) { if (name.startsWith(INDEXES)) { long gen; if (name.length() == INDEXES.length()) { gen = 0; } else if (name.charAt(INDEXES.length()) == '_') { gen = Long.parseLong(name.substring(INDEXES.length() + 1), Character.MAX_RADIX); } else { continue; } try { IndexInfos infos = new IndexInfos(dir, INDEXES, gen); indexInfosMap.put(gen, infos); } catch (IOException e) { log.warn("ignoring invalid index infos file: " + name); } } } } }
private void assertNoNrm(Directory dir) throws Throwable { final String[] files = dir.listAll(); for (int i = 0; i < files.length; i++) { // TODO: this relies upon filenames assertFalse(files[i].endsWith(".nrm") || files[i].endsWith(".len")); } }
public void assertDeleteContent(Store store, DirectoryService service) throws IOException { store.deleteContent(); assertThat( Arrays.toString(store.directory().listAll()), store.directory().listAll().length, equalTo(0)); assertThat(store.stats().sizeInBytes(), equalTo(0l)); for (Directory dir : service.build()) { assertThat(dir.listAll().length, equalTo(0)); } }
int countExtensions(Directory directory, String ext) throws IOException { String[] fileNames = directory.listAll(); int fileCount = 0; for (String fileName : fileNames) { if (fileName.endsWith("." + ext)) { fileCount++; } } return fileCount; }
// LUCENE-2790: tests that the non CFS files were deleted by addIndexes public void testNonCFSLeftovers() throws Exception { Directory[] dirs = new Directory[2]; for (int i = 0; i < dirs.length; i++) { dirs[i] = new RAMDirectory(); IndexWriter w = new IndexWriter( dirs[i], new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); Document d = new Document(); FieldType customType = new FieldType(TextField.TYPE_STORED); customType.setStoreTermVectors(true); d.add(new Field("c", "v", customType)); w.addDocument(d); w.close(); } IndexReader[] readers = new IndexReader[] {DirectoryReader.open(dirs[0]), DirectoryReader.open(dirs[1])}; Directory dir = new MockDirectoryWrapper(random(), new RAMDirectory()); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setMergePolicy(newLogMergePolicy(true)); MergePolicy lmp = conf.getMergePolicy(); // Force creation of CFS: lmp.setNoCFSRatio(1.0); lmp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY); IndexWriter w3 = new IndexWriter(dir, conf); w3.addIndexes(readers); w3.close(); // we should now see segments_X, // segments.gen,_Y.cfs,_Y.cfe, _Z.si assertEquals( "Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 5, dir.listAll().length); dir.close(); }
public LuceneFullTextIndex(AgeQuery qury, Collection<TextFieldExtractor> exts, File path) throws IOException { query = qury; extractors = exts; defaultFieldName = extractors.iterator().next().getName(); queryParser = new QueryParser(Version.LUCENE_30, defaultFieldName, analyzer); if (path == null) index = new RAMDirectory(); else { index = new NIOFSDirectory(path); if (index.listAll().length != 0) searcher = new IndexSearcher(IndexReader.open(index)); } }
StoreDirectory(Distributor distributor) throws IOException { this.distributor = distributor; synchronized (mutex) { MapBuilder<String, StoreFileMetaData> builder = MapBuilder.newMapBuilder(); Map<String, String> checksums = readChecksums(distributor.all(), new HashMap<String, String>()); for (Directory delegate : distributor.all()) { for (String file : delegate.listAll()) { String checksum = checksums.get(file); builder.put( file, new StoreFileMetaData(file, delegate.fileLength(file), checksum, delegate)); } } filesMetadata = builder.immutableMap(); files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); } }
/** * Load some keys in the collector, excluding some and to a maximum number of collected * (non-excluded) keys. * * @param keysCollector the set where to add loaded keys to * @param keysToExclude which keys should not be loaded. Warning: can be null! Means all keys are * to be returned * @param maxElements upper limit for collection */ private void loadSomeKeys( final HashSet<IndexScopedKey> keysCollector, final Set<IndexScopedKey> keysToExclude, final int maxElements) { if (maxElements <= 0) { return; } int collectedKeys = 0; // First we collect the (single) FileListCacheKey FileListCacheKey rootKey = new FileListCacheKey(indexName); if (keysToExclude == null || !keysToExclude.contains(rootKey)) { // unless it was excluded if (keysCollector.add(rootKey)) { // unless it was already collected collectedKeys++; } } try { // Now we collect first all FileCacheKey (keys for file metadata) String[] listAll = directory.listAll(); for (String fileName : listAll) { if (collectedKeys >= maxElements) return; FileCacheKey key = new FileCacheKey(indexName, fileName); if (keysToExclude == null || !keysToExclude.contains(key)) { if (keysCollector.add(key)) { if (++collectedKeys >= maxElements) return; } } } // Next we load the ChunkCacheKey (keys for file contents) for (String fileName : listAll) { int numChunksInt = figureChunksNumber(fileName); for (int i = 0; i < numChunksInt; i++) { // Inner loop: we actually have several Chunks per file name ChunkCacheKey key = new ChunkCacheKey(indexName, fileName, i, autoChunkSize); if (keysToExclude == null || !keysToExclude.contains(key)) { if (keysCollector.add(key)) { if (++collectedKeys >= maxElements) return; } } } } } catch (IOException e) { throw log.exceptionInCacheLoader(e); } }
/** @see org.apache.lucene.index.IndexReader#listCommits */ public static Collection<IndexCommit> listCommits(Directory dir) throws IOException { final String[] files = dir.listAll(); List<IndexCommit> commits = new ArrayList<IndexCommit>(); SegmentInfos latest = new SegmentInfos(); latest.read(dir); final long currentGen = latest.getGeneration(); commits.add(new ReaderCommit(latest, dir)); for (int i = 0; i < files.length; i++) { final String fileName = files[i]; if (fileName.startsWith(IndexFileNames.SEGMENTS) && !fileName.equals(IndexFileNames.SEGMENTS_GEN) && SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) { SegmentInfos sis = new SegmentInfos(); try { // IOException allowed to throw there, in case // segments_N is corrupt sis.read(dir, fileName); } catch (FileNotFoundException fnfe) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth // between machines, it's very likely that the // dir listing will be stale and will claim a // file segments_X exists when in fact it // doesn't. So, we catch this and handle it // as if the file does not exist sis = null; } if (sis != null) commits.add(new ReaderCommit(sis, dir)); } } // Ensure that the commit points are sorted in ascending order. Collections.sort(commits); return commits; }
IndexSearcher getIndexSearcher(String key) { IndexSearcher searcher = indexSearchers.get(key); if (searcher == null) { try { File fsDirectory = new File(storeDir, key); if (!fsDirectory.exists()) { return null; } Directory dir = FSDirectory.open(fsDirectory); if (dir.listAll().length == 0) { return null; } searcher = new IndexSearcher(dir, true); } catch (IOException e) { throw new RuntimeException(e); } indexSearchers.put(key, searcher); } return searcher; }
public static void checkIndex(ESLogger logger, Store store, ShardId shardId) { if (store.tryIncRef()) { logger.info("start check index"); try { Directory dir = store.directory(); if (!Lucene.indexExists(dir)) { return; } if (IndexWriter.isLocked(dir)) { ESTestCase.checkIndexFailed = true; throw new IllegalStateException("IndexWriter is still open on shard " + shardId); } try (CheckIndex checkIndex = new CheckIndex(dir)) { BytesStreamOutput os = new BytesStreamOutput(); PrintStream out = new PrintStream(os, false, StandardCharsets.UTF_8.name()); checkIndex.setInfoStream(out); out.flush(); CheckIndex.Status status = checkIndex.checkIndex(); if (!status.clean) { ESTestCase.checkIndexFailed = true; logger.warn( "check index [failure] index files={}\n{}", Arrays.toString(dir.listAll()), new String(os.bytes().toBytes(), StandardCharsets.UTF_8)); throw new IOException("index check failure"); } else { if (logger.isDebugEnabled()) { logger.debug( "check index [success]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8)); } } } } catch (Exception e) { logger.warn("failed to check index", e); } finally { logger.info("end check index"); store.decRef(); } } }
public void testOverrideWriteLocker() throws IOException { Directory dir = null; try { dir = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEX_NAME) .chunkSize(BUFFER_SIZE) .overrideWriteLocker( new LockFactory() { @Override public Lock makeLock(String lockName) { return null; } @Override public void clearLock(String lockName) throws IOException {} }) .create(); AssertJUnit.assertEquals(0, dir.listAll().length); } finally { if (dir != null) dir.close(); } }
public static void deleteDirectoryContents(Directory directory) throws IOException { for (String fileName : directory.listAll()) directory.deleteFile(fileName); }
@Test(enabled = false) public void testReadChunks() throws Exception { final int BUFFER_SIZE = 64; Cache cache = cacheManager.getCache(); Directory dir = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME) .chunkSize(BUFFER_SIZE) .create(); // create file headers FileMetadata file1 = new FileMetadata(5); FileCacheKey key1 = new FileCacheKey(INDEXNAME, "Hello.txt"); cache.put(key1, file1); FileMetadata file2 = new FileMetadata(5); FileCacheKey key2 = new FileCacheKey(INDEXNAME, "World.txt"); cache.put(key2, file2); // byte array for Hello.txt String helloText = "Hello world. This is some text."; cache.put(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE), helloText.getBytes()); // byte array for World.txt - should be in at least 2 chunks. String worldText = "This String should contain more than sixty four characters but less than one hundred and twenty eight."; assert worldText.getBytes().length > BUFFER_SIZE; assert worldText.getBytes().length < (2 * BUFFER_SIZE); byte[] buf = new byte[BUFFER_SIZE]; System.arraycopy(worldText.getBytes(), 0, buf, 0, BUFFER_SIZE); cache.put(new ChunkCacheKey(INDEXNAME, "World.txt", 0, BUFFER_SIZE), buf); String part1 = new String(buf); buf = new byte[BUFFER_SIZE]; System.arraycopy(worldText.getBytes(), BUFFER_SIZE, buf, 0, worldText.length() - BUFFER_SIZE); cache.put(new ChunkCacheKey(INDEXNAME, "World.txt", 1, BUFFER_SIZE), buf); String part2 = new String(buf); // make sure the generated bytes do add up! AssertJUnit.assertEquals(part1 + part2.trim(), worldText); file1.setSize(helloText.length()); file2.setSize(worldText.length()); Set<String> s = new HashSet<String>(); s.add("Hello.txt"); s.add("World.txt"); Set other = new HashSet(Arrays.asList(dir.listAll())); // ok, file listing works. AssertJUnit.assertEquals(s, other); IndexInput ii = dir.openInput("Hello.txt"); assert ii.length() == helloText.length(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); for (int i = 0; i < ii.length(); i++) { baos.write(ii.readByte()); } assert new String(baos.toByteArray()).equals(helloText); ii = dir.openInput("World.txt"); assert ii.length() == worldText.length(); baos = new ByteArrayOutputStream(); for (int i = 0; i < ii.length(); i++) { baos.write(ii.readByte()); } assert new String(baos.toByteArray()).equals(worldText); // now with buffered reading ii = dir.openInput("Hello.txt"); assert ii.length() == helloText.length(); baos = new ByteArrayOutputStream(); long toRead = ii.length(); while (toRead > 0) { buf = new byte[19]; // suitably arbitrary int bytesRead = (int) Math.min(toRead, 19); ii.readBytes(buf, 0, bytesRead); toRead = toRead - bytesRead; baos.write(buf, 0, bytesRead); } assert new String(baos.toByteArray()).equals(helloText); ii = dir.openInput("World.txt"); assert ii.length() == worldText.length(); baos = new ByteArrayOutputStream(); toRead = ii.length(); while (toRead > 0) { buf = new byte[19]; // suitably arbitrary int bytesRead = (int) Math.min(toRead, 19); ii.readBytes(buf, 0, bytesRead); toRead = toRead - bytesRead; baos.write(buf, 0, bytesRead); } assert new String(baos.toByteArray()).equals(worldText); dir.deleteFile("Hello.txt"); assert null == cache.get(new FileCacheKey(INDEXNAME, "Hello.txt")); assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE)); Object ob1 = cache.get(new FileCacheKey(INDEXNAME, "World.txt")); Object ob2 = cache.get(new ChunkCacheKey(INDEXNAME, "World.txt", 0, BUFFER_SIZE)); Object ob3 = cache.get(new ChunkCacheKey(INDEXNAME, "World.txt", 1, BUFFER_SIZE)); ((DirectoryExtensions) dir).renameFile("World.txt", "HelloWorld.txt"); assert null == cache.get(new FileCacheKey(INDEXNAME, "Hello.txt")); assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE)); assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 1, BUFFER_SIZE)); assert cache.get(new FileCacheKey(INDEXNAME, "HelloWorld.txt")).equals(ob1); assert cache.get(new ChunkCacheKey(INDEXNAME, "HelloWorld.txt", 0, BUFFER_SIZE)).equals(ob2); assert cache.get(new ChunkCacheKey(INDEXNAME, "HelloWorld.txt", 1, BUFFER_SIZE)).equals(ob3); // test that contents survives a move ii = dir.openInput("HelloWorld.txt"); assert ii.length() == worldText.length(); baos = new ByteArrayOutputStream(); toRead = ii.length(); while (toRead > 0) { buf = new byte[19]; // suitably arbitrary int bytesRead = (int) Math.min(toRead, 19); ii.readBytes(buf, 0, bytesRead); toRead = toRead - bytesRead; baos.write(buf, 0, bytesRead); } assert new String(baos.toByteArray()).equals(worldText); dir.close(); DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME); }
public Object run(IndexCommit commit) throws CorruptIndexException, IOException { if (commit != null) { if (directory != commit.getDirectory()) throw new IOException("the specified commit does not match the specified Directory"); return doBody(commit.getSegmentsFileName()); } String segmentFileName = null; long lastGen = -1; long gen = 0; int genLookaheadCount = 0; IOException exc = null; int retryCount = 0; boolean useFirstMethod = true; // Loop until we succeed in calling doBody() without // hitting an IOException. An IOException most likely // means a commit was in process and has finished, in // the time it took us to load the now-old infos files // (and segments files). It's also possible it's a // true error (corrupt index). To distinguish these, // on each retry we must see "forward progress" on // which generation we are trying to load. If we // don't, then the original error is real and we throw // it. // We have three methods for determining the current // generation. We try the first two in parallel (when // useFirstMethod is true), and fall back to the third // when necessary. while (true) { if (useFirstMethod) { // List the directory and use the highest // segments_N file. This method works well as long // as there is no stale caching on the directory // contents (NOTE: NFS clients often have such stale // caching): String[] files = null; long genA = -1; files = directory.listAll(); if (files != null) { genA = getCurrentSegmentGeneration(files); } if (infoStream != null) { message("directory listing genA=" + genA); } // Also open segments.gen and read its // contents. Then we take the larger of the two // gens. This way, if either approach is hitting // a stale cache (NFS) we have a better chance of // getting the right generation. long genB = -1; for (int i = 0; i < defaultGenFileRetryCount; i++) { IndexInput genInput = null; try { genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN); } catch (FileNotFoundException e) { if (infoStream != null) { message("segments.gen open: FileNotFoundException " + e); } break; } catch (IOException e) { if (infoStream != null) { message("segments.gen open: IOException " + e); } } if (genInput != null) { try { int version = genInput.readInt(); if (version == FORMAT_LOCKLESS) { long gen0 = genInput.readLong(); long gen1 = genInput.readLong(); if (infoStream != null) { message("fallback check: " + gen0 + "; " + gen1); } if (gen0 == gen1) { // The file is consistent. genB = gen0; break; } } } catch (IOException err2) { // will retry } finally { genInput.close(); } } try { Thread.sleep(defaultGenFileRetryPauseMsec); } catch (InterruptedException ie) { throw new ThreadInterruptedException(ie); } } if (infoStream != null) { message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB); } // Pick the larger of the two gen's: if (genA > genB) gen = genA; else gen = genB; if (gen == -1) { // Neither approach found a generation throw new IndexNotFoundException( "no segments* file found in " + directory + ": files: " + Arrays.toString(files)); } } if (useFirstMethod && lastGen == gen && retryCount >= 2) { // Give up on first method -- this is 3rd cycle on // listing directory and checking gen file to // attempt to locate the segments file. useFirstMethod = false; } // Second method: since both directory cache and // file contents cache seem to be stale, just // advance the generation. if (!useFirstMethod) { if (genLookaheadCount < defaultGenLookaheadCount) { gen++; genLookaheadCount++; if (infoStream != null) { message("look ahead increment gen to " + gen); } } else { // All attempts have failed -- throw first exc: throw exc; } } else if (lastGen == gen) { // This means we're about to try the same // segments_N last tried. retryCount++; } else { // Segment file has advanced since our last loop // (we made "progress"), so reset retryCount: retryCount = 0; } lastGen = gen; segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen); try { Object v = doBody(segmentFileName); if (infoStream != null) { message("success on " + segmentFileName); } return v; } catch (IOException err) { // Save the original root cause: if (exc == null) { exc = err; } if (infoStream != null) { message( "primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retryCount=" + retryCount + "; gen = " + gen); } if (gen > 1 && useFirstMethod && retryCount == 1) { // This is our second time trying this same segments // file (because retryCount is 1), and, there is // possibly a segments_(N-1) (because gen > 1). // So, check if the segments_(N-1) exists and // try it if so: String prevSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1); final boolean prevExists; prevExists = directory.fileExists(prevSegmentFileName); if (prevExists) { if (infoStream != null) { message("fallback to prior segment file '" + prevSegmentFileName + "'"); } try { Object v = doBody(prevSegmentFileName); if (infoStream != null) { message("success on fallback " + prevSegmentFileName); } return v; } catch (IOException err2) { if (infoStream != null) { message( "secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry"); } } } } } } }
/** * Get the generation of the most recent commit to the index in this directory (N in the * segments_N file). * * @param directory -- directory to search for the latest segments_N file */ public static long getLastCommitGeneration(Directory directory) throws IOException { return getLastCommitGeneration(directory.listAll()); }
@Override public String[] listAll() throws IOException { return fsDir.listAll(); }
/** Run {@link #doBody} on the provided commit. */ public T run(IndexCommit commit) throws IOException { if (commit != null) { if (directory != commit.getDirectory()) throw new IOException("the specified commit does not match the specified Directory"); return doBody(commit.getSegmentsFileName()); } long lastGen = -1; long gen = -1; IOException exc = null; // Loop until we succeed in calling doBody() without // hitting an IOException. An IOException most likely // means an IW deleted our commit while opening // the time it took us to load the now-old infos files // (and segments files). It's also possible it's a // true error (corrupt index). To distinguish these, // on each retry we must see "forward progress" on // which generation we are trying to load. If we // don't, then the original error is real and we throw // it. for (; ; ) { lastGen = gen; String files[] = directory.listAll(); String files2[] = directory.listAll(); Arrays.sort(files); Arrays.sort(files2); if (!Arrays.equals(files, files2)) { // listAll() is weakly consistent, this means we hit "concurrent modification exception" continue; } gen = getLastCommitGeneration(files); if (infoStream != null) { message("directory listing gen=" + gen); } if (gen == -1) { throw new IndexNotFoundException( "no segments* file found in " + directory + ": files: " + Arrays.toString(files)); } else if (gen > lastGen) { String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen); try { T t = doBody(segmentFileName); if (infoStream != null) { message("success on " + segmentFileName); } return t; } catch (IOException err) { // Save the original root cause: if (exc == null) { exc = err; } if (infoStream != null) { message( "primary Exception on '" + segmentFileName + "': " + err + "'; will retry: gen = " + gen); } } } else { throw exc; } } }
public void testDocsStuckInRAMForever() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); iwc.setRAMBufferSizeMB(.2); Codec codec = Codec.forName("Lucene49"); iwc.setCodec(codec); iwc.setMergePolicy(NoMergePolicy.INSTANCE); final IndexWriter w = new IndexWriter(dir, iwc); final CountDownLatch startingGun = new CountDownLatch(1); Thread[] threads = new Thread[2]; for (int i = 0; i < threads.length; i++) { final int threadID = i; threads[i] = new Thread() { @Override public void run() { try { startingGun.await(); for (int j = 0; j < 1000; j++) { Document doc = new Document(); doc.add(newStringField("field", "threadID" + threadID, Field.Store.NO)); w.addDocument(doc); } } catch (Exception e) { throw new RuntimeException(e); } } }; threads[i].start(); } startingGun.countDown(); for (Thread t : threads) { t.join(); } Set<String> segSeen = new HashSet<>(); int thread0Count = 0; int thread1Count = 0; // At this point the writer should have 2 thread states w/ docs; now we index with only 1 thread // until we see all 1000 thread0 & thread1 // docs flushed. If the writer incorrectly holds onto previously indexed docs forever then this // will run forever: while (thread0Count < 1000 || thread1Count < 1000) { Document doc = new Document(); doc.add(newStringField("field", "threadIDmain", Field.Store.NO)); w.addDocument(doc); for (String fileName : dir.listAll()) { if (fileName.endsWith(".si")) { String segName = IndexFileNames.parseSegmentName(fileName); if (segSeen.contains(segName) == false) { segSeen.add(segName); SegmentInfo si = new Lucene46SegmentInfoFormat() .getSegmentInfoReader() .read(dir, segName, IOContext.DEFAULT); si.setCodec(codec); SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, -1, -1, -1); SegmentReader sr = new SegmentReader(sci, 1, IOContext.DEFAULT); try { thread0Count += sr.docFreq(new Term("field", "threadID0")); thread1Count += sr.docFreq(new Term("field", "threadID1")); } finally { sr.close(); } } } } } w.close(); dir.close(); }
/* * Test a silly deletion policy that keeps all commits around. */ public void testKeepAllDeletionPolicy() throws IOException { for (int pass = 0; pass < 2; pass++) { if (VERBOSE) { System.out.println("TEST: cycle pass="******"TEST: open writer for forceMerge"); } writer = new IndexWriter(dir, conf); policy = (KeepAllDeletionPolicy) writer.getConfig().getIndexDeletionPolicy(); writer.forceMerge(1); writer.close(); } assertEquals(needsMerging ? 2 : 1, policy.numOnInit); // If we are not auto committing then there should // be exactly 2 commits (one per close above): assertEquals(1 + (needsMerging ? 1 : 0), policy.numOnCommit); // Test listCommits Collection<IndexCommit> commits = DirectoryReader.listCommits(dir); // 2 from closing writer assertEquals(1 + (needsMerging ? 1 : 0), commits.size()); // Make sure we can open a reader on each commit: for (final IndexCommit commit : commits) { IndexReader r = DirectoryReader.open(commit); r.close(); } // Simplistic check: just verify all segments_N's still // exist, and, I can open a reader on each: long gen = SegmentInfos.getLastCommitGeneration(dir); while (gen > 0) { IndexReader reader = DirectoryReader.open(dir); reader.close(); dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); gen--; if (gen > 0) { // Now that we've removed a commit point, which // should have orphan'd at least one index file. // Open & close a writer and assert that it // actually removed something: int preCount = dir.listAll().length; writer = new IndexWriter( dir, newIndexWriterConfig(new MockAnalyzer(random())) .setOpenMode(OpenMode.APPEND) .setIndexDeletionPolicy(policy)); writer.close(); int postCount = dir.listAll().length; assertTrue(postCount < preCount); } } dir.close(); } }
private boolean doRestore() throws Exception { Path backupPath = Paths.get(backupLocation).resolve(backupName); SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT); String restoreIndexName = "restore." + dateFormat.format(new Date()); String restoreIndexPath = core.getDataDir() + restoreIndexName; Directory restoreIndexDir = null; Directory indexDir = null; try (Directory backupDir = FSDirectory.open(backupPath)) { final Version version = IndexFetcher.checkOldestVersion(SegmentInfos.readLatestCommit(backupDir)); restoreIndexDir = core.getDirectoryFactory() .get( restoreIndexPath, DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); // Prefer local copy. indexDir = core.getDirectoryFactory() .get( core.getIndexDir(), DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); // Move all files from backupDir to restoreIndexDir for (String filename : backupDir.listAll()) { checkInterrupted(); log.info("Copying file {} to restore directory ", filename); try (IndexInput indexInput = backupDir.openInput(filename, IOContext.READONCE)) { Long checksum = null; try { checksum = CodecUtil.retrieveChecksum(indexInput); } catch (Exception e) { log.warn("Could not read checksum from index file: " + filename, e); } long length = indexInput.length(); IndexFetcher.CompareResult compareResult = IndexFetcher.compareFile(indexDir, version, filename, length, checksum); if (!compareResult.equal || (!compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv") || filename.startsWith("segments_")))) { restoreIndexDir.copyFrom(backupDir, filename, filename, IOContext.READONCE); } else { // prefer local copy restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE); } } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e); } } log.debug("Switching directories"); IndexFetcher.modifyIndexProps(core, restoreIndexName); boolean success; try { core.getUpdateHandler().newIndexWriter(false); openNewSearcher(); success = true; log.info("Successfully restored to the backup index"); } catch (Exception e) { // Rollback to the old index directory. Delete the restore index directory and mark the // restore as failed. log.warn("Could not switch to restored index. Rolling back to the current index"); Directory dir = null; try { dir = core.getDirectoryFactory() .get( core.getDataDir(), DirectoryFactory.DirContext.META_DATA, core.getSolrConfig().indexConfig.lockType); dir.deleteFile(IndexFetcher.INDEX_PROPERTIES); } finally { if (dir != null) { core.getDirectoryFactory().release(dir); } } core.getDirectoryFactory().doneWithDirectory(restoreIndexDir); core.getDirectoryFactory().remove(restoreIndexDir); core.getUpdateHandler().newIndexWriter(false); openNewSearcher(); throw new SolrException( SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e); } if (success) { core.getDirectoryFactory().doneWithDirectory(indexDir); core.getDirectoryFactory().remove(indexDir); } return true; } finally { if (restoreIndexDir != null) { core.getDirectoryFactory().release(restoreIndexDir); } if (indexDir != null) { core.getDirectoryFactory().release(indexDir); } } }
/** * Load implementation for FileListCacheKey; must return a ConcurrentHashSet containing the names * of all files in this Directory. */ private Object loadIntern() throws IOException { final String[] listAll = directory.listAll(); return new FileListCacheValue(listAll); }
private void processAdd( SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo) throws Exception { IndexWriter indexWrite = null; try { if (worker.isRunning()) { indexWrite = indexStorage.getIndexWriter(false); } long last = System.currentTimeMillis(); for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator(); worker.isRunning() && tditer.hasNext(); ) { Reader contentReader = null; try { SearchBuilderItem sbi = (SearchBuilderItem) tditer.next(); // only add adds, that have been deleted or are locked // sucessfully if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate()) && !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) { continue; } // Reference ref = // entityManager.newReference(sbi.getName()); String ref = sbi.getName(); if (ref == null) { log.error( "Unrecognised trigger object presented to index builder " //$NON-NLS-1$ + sbi); } long startDocIndex = System.currentTimeMillis(); worker.setStartDocIndex(startDocIndex); worker.setNowIndexing(ref); try { try { // Entity entity = ref.getEntity(); EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref); boolean indexDoc = true; if (searchIndexBuilder.isOnlyIndexSearchToolSites()) { try { String siteId = sep.getSiteId(sbi.getName()); Site s = SiteService.getSite(siteId); ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$ if (t == null) { indexDoc = false; log.debug( "Not indexing " //$NON-NLS-1$ + sbi.getName() + " as it has no search tool"); //$NON-NLS-1$ } } catch (Exception ex) { indexDoc = false; log.debug( "Not indexing " + sbi.getName() // $NON-NLS-1$ + " as it has no site", ex); //$NON-NLS-1$ } } if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) { DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService); // Reader contentReader = null; Document doc = DocumentIndexingUtils.createIndexDocument( ref, digestStorageUtil, sep, serverConfigurationService.getServerUrl(), contentReader); // indexDocTMP(ref, sep); log.debug("Indexing Document " + doc); // $NON-NLS-1$ indexWrite.addDocument(doc); log.debug("Done Indexing Document " + doc); // $NON-NLS-1$ processRDF(ref, sep); } else { if (log.isDebugEnabled()) { if (!indexDoc) { log.debug("Ignored Document: Fileteed out by site " + ref); // $NON-NLS-1$ } else if (sep == null) { log.debug("Ignored Document: No EntityContentProducer " + ref); // $NON-NLS-1$ } else if (!sep.isForIndex(ref)) { log.debug("Ignored Document: Marked as Ignore " + ref); // $NON-NLS-1$ } else if (sep.getSiteId(ref) == null) { log.debug("Ignored Document: No Site ID " + ref); // $NON-NLS-1$ } else { log.debug("Ignored Document: Reason Unknown " + ref); // $NON-NLS-1$ } } } } catch (Exception e1) { log.debug( " Failed to index document for " + ref + " cause: " //$NON-NLS-1$ + e1.getMessage(), e1); } sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED); updateOrSave(connection, sbi); } catch (Exception e1) { log.debug( " Failed to index document cause: " //$NON-NLS-1$ + e1.getMessage()); } long endDocIndex = System.currentTimeMillis(); worker.setLastIndex(endDocIndex - startDocIndex); if ((endDocIndex - startDocIndex) > 60000L) { log.warn( "Slow index operation " //$NON-NLS-1$ + String.valueOf((endDocIndex - startDocIndex) / 1000) + " seconds to index " //$NON-NLS-1$ + ref); } // update this node lock to indicate its // still alove, no document should // take more than 2 mins to process // ony do this check once every minute long now = System.currentTimeMillis(); if ((now - last) > (60L * 1000L)) { last = System.currentTimeMillis(); if (!worker.getLockTransaction(15L * 60L * 1000L, true)) { throw new Exception( "Transaction Lock Expired while indexing " //$NON-NLS-1$ + ref); } } } finally { if (contentReader != null) { try { contentReader.close(); } catch (IOException ioex) { log.debug(ioex); } } } } worker.setStartDocIndex(System.currentTimeMillis()); worker.setNowIndexing( Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); // $NON-NLS-1$ } catch (Exception ex) { log.error("Failed to Add Documents ", ex); throw new Exception(ex); } finally { if (indexWrite != null) { if (log.isDebugEnabled()) { log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents"); Directory d = indexWrite.getDirectory(); String[] s = d.listAll(); log.debug("Directory Contains "); for (int i = 0; i < s.length; i++) { File f = new File(s[i]); log.debug( "\t" + String.valueOf(f.length()) + "\t" + new Date(f.lastModified()) + "\t" + s[i]); } } indexStorage.closeIndexWriter(indexWrite); } } }
public IndexInfo(String indexName, Directory directory, Analyzer analyzer) throws CorruptIndexException, IOException { IndexWriter iwriter = null; IndexReader indexReader = null; try { iwriter = new IndexWriter(directory, analyzer, false, IndexWriter.MaxFieldLength.UNLIMITED); indexReader = iwriter.getReader(); this.index_name = indexName; this.index_version = indexReader.getVersion(); this.optimized = indexReader.isOptimized(); this.numDocs = indexReader.numDocs(); Date date = new Date(IndexReader.lastModified(directory)); DateFormat formatter = new SimpleDateFormat("EEEE, dd MMMM yyyy, hh:mm:ss a"); this.lastModifyDate = formatter.format(date); this.files = new HashMap<String, String>(); String[] files = directory.listAll(); for (int i = 0; i < files.length; i++) { if (files[i].startsWith("_")) this.files.put( files[i], Long.toString( Math.round( FileSizeConverter.convertSize( directory.fileLength(files[i]), FileSizeConverter.KB))) + " KB"); } indexReader.close(); iwriter.close(); iwriter = null; directory.clearLock(directory.getLockID()); } catch (CorruptIndexException e) { if (iwriter != null) { iwriter.close(); iwriter = null; } if (indexReader != null) { indexReader.close(); indexReader = null; } throw e; } catch (LockObtainFailedException e) { if (iwriter != null) { iwriter.close(); iwriter = null; } if (indexReader != null) { indexReader.close(); indexReader = null; } throw e; } catch (IOException e) { if (iwriter != null) { iwriter.close(); iwriter = null; } if (indexReader != null) { indexReader.close(); indexReader = null; } throw e; } }