public void addSearcher(File luceneDir) { if (isClosed.get()) return; final Searcher searcher = SearcherFactory.getSearcher(luceneDir, searcherType); try { searcher.open(); this.clusterSearchers.add(searcher); this.searchers.clear(); this.searchers.addAll(clusterSearchers); if (extraSearchers != null) this.searchers.addAll(extraSearchers); } catch (FileNotFoundException fnfe) { // this means we didn't add anything to an index, which happens i.e. if we didn't have any // asian (or non-asian) posts in a batch. // so what we'll do is log it and ignore this index. System.err.println(new Date() + ": WARNING : Ignoring empty index at '" + luceneDir + "'!"); } catch (IOException e) { System.err.println( new Date() + ": *** ERROR! " + luceneId + " can't open lucene searcher at '" + luceneDir + "'"); e.printStackTrace(System.err); } }
/** * The ActionListener implementation * * @param event the event. */ public void actionPerformed(ActionEvent event) { String searchText = textField.getText().trim(); if (searchText.equals("") && !saveAs.isSelected() && (fileLength > 10000000)) { textPane.setText("Blank search text is not allowed for large IdTables."); } else { File outputFile = null; if (saveAs.isSelected()) { outputFile = chooser.getSelectedFile(); if (outputFile != null) { String name = outputFile.getName(); int k = name.lastIndexOf("."); if (k != -1) name = name.substring(0, k); name += ".txt"; File parent = outputFile.getAbsoluteFile().getParentFile(); outputFile = new File(parent, name); chooser.setSelectedFile(outputFile); } if (chooser.showSaveDialog(this) != JFileChooser.APPROVE_OPTION) System.exit(0); outputFile = chooser.getSelectedFile(); } textPane.setText(""); Searcher searcher = new Searcher(searchText, event.getSource().equals(searchPHI), outputFile); searcher.start(); } }
/** * Get the searcher at the given position. * * <p>If the searcher's backing directory no longer exists, then close and disable it. A null * entry will then exist at the given index. */ public Searcher get(int index) { Searcher result = index < size() ? searchers.get(index) : null; if (result != null) { if (!result.getDirPath().exists()) { // disable index that has disappeared. searchers.set(index, null); if (index < clusterSearchers.size()) clusterSearchers.set(index, null); else extraSearchers.set(index - clusterSearchers.size(), null); try { result.close(); } catch (IOException e) { System.err.println( new Date() + ": WARNING : MultiSearcher couldn't close disappeared index at '" + result.getDirPath() + "'!"); e.printStackTrace(System.err); } result = null; } } return result; }
public static void callMainSearch(String dir) { try { Searcher s = new Searcher(dir); s.termQuery("treeset"); } catch (IOException e) { e.printStackTrace(); } }
/** * Tries to add new documents to hitDocs. Ensures that the hit numbered <code>min</code> has been * retrieved. */ private final void getMoreDocs(int min) throws IOException { if (hitDocs.size() > min) { min = hitDocs.size(); } int n = min * 2; // double # retrieved TopDocs topDocs = (sort == null) ? searcher.search(weight, filter, n) : searcher.search(weight, filter, n, sort); length = topDocs.totalHits; ScoreDoc[] scoreDocs = topDocs.scoreDocs; float scoreNorm = 1.0f; if (length > 0 && topDocs.getMaxScore() > 1.0f) { scoreNorm = 1.0f / topDocs.getMaxScore(); } int start = hitDocs.size() - nDeletedHits; // any new deletions? int nDels2 = countDeletions(searcher); debugCheckedForDeletions = false; if (nDeletions < 0 || nDels2 > nDeletions) { // either we cannot count deletions, or some "previously valid hits" might have been deleted, // so find exact start point nDeletedHits = 0; debugCheckedForDeletions = true; int i2 = 0; for (int i1 = 0; i1 < hitDocs.size() && i2 < scoreDocs.length; i1++) { int id1 = ((HitDoc) hitDocs.get(i1)).id; int id2 = scoreDocs[i2].doc; if (id1 == id2) { i2++; } else { nDeletedHits++; } } start = i2; } int end = scoreDocs.length < length ? scoreDocs.length : length; length += nDeletedHits; for (int i = start; i < end; i++) { hitDocs.addElement(new HitDoc(scoreDocs[i].score * scoreNorm, scoreDocs[i].doc)); } nDeletions = nDels2; }
public void updateIndex(boolean recreate) throws IOException { try { File logsDir = new File(ConfigurationManager.getInstance().getConfiguration().getLogsPath()); DirectoryTraverser traverser = new DirectoryTraverser(logsDir); indexer.open(directory, recreate); File file = null; while ((file = traverser.getNextFile()) != null) { LogFileParser p = null; try { p = new LogFileParser(file); } catch (Exception e) { log.error(e, e); } if (p != null) { LogEntry entry; while ((entry = p.getNextEntry()) != null) { indexer.add(entry); } } } } finally { indexer.close(); searcher.reopenIndex(); } }
// QHandler handles incoming queries. public void run() { // hmmm.. this seems like potential bug. I want to check that query is not in table. But even // if query table contains key, // that does not necessarily mean it is in table, b/c two queries can have SAME HASHCODE VALUE. // I need to have some other means. // Will talk to Rusty @ this on Monday. if (!qt.containsKey(query)) // check that query is not already in table { Searcher.inform(query); // Give information to the Search Monitor panel NetworkManager.writeButOne(query.getIP(), query); /*Query is forwarded to all connected nodes except one from which query came. */ qt.put((Packet) query, query); // add query to table, indexed by its unique MessageID searchResult = SharedDirectory.search(query.getSearchString()); // check shared directory for query match numHits = searchResult.getSize(); if (numHits != 0) // package a query hit to send out if there is at least one query match { queryID = query.getMessageID(); port = Mine.getPort(); myIP = Mine.getIPAddress(); speed = Mine.getSpeed(); serventID = Mine.getServentIdentifier(); queryHit = new QueryHit(numHits, port, myIP, speed, searchResult, serventID, queryID); NetworkManager.writeToOne( query.getIP(), queryHit); // send qHit back to node that sent original query } } }
/** * Waits for a given view. * * @param view the view to wait for * @param timeout the amount of time in milliseconds to wait * @param scroll {@code true} if scrolling should be performed * @param checkIsShown {@code true} if view.isShown() should be used * @return {@code true} if view is shown and {@code false} if it is not shown before the timeout */ public View waitForView(View view, int timeout, boolean scroll, boolean checkIsShown) { long endTime = SystemClock.uptimeMillis() + timeout; int retry = 0; if (view == null) return null; while (SystemClock.uptimeMillis() < endTime) { final boolean foundAnyMatchingView = searcher.searchFor(view); if (checkIsShown && foundAnyMatchingView && !view.isShown()) { sleeper.sleepMini(); retry++; View identicalView = viewFetcher.getIdenticalView(view); if (identicalView != null && !view.equals(identicalView)) { view = identicalView; } if (retry > 5) { return view; } continue; } if (foundAnyMatchingView) { return view; } if (scroll) scroller.scrollDown(); sleeper.sleep(); } return view; }
/** * Waits for and returns a View. * * @param index the index of the view * @param classToFilterby the class to filter * @return the specified View */ public <T extends View> T waitForAndGetView(int index, Class<T> classToFilterBy) { long endTime = SystemClock.uptimeMillis() + Timeout.getSmallTimeout(); while (SystemClock.uptimeMillis() <= endTime && !waitForView(classToFilterBy, index, true, true)) ; int numberOfUniqueViews = searcher.getNumberOfUniqueViews(); ArrayList<T> views = RobotiumUtils.removeInvisibleViews(viewFetcher.getCurrentViews(classToFilterBy)); if (views.size() < numberOfUniqueViews) { int newIndex = index - (numberOfUniqueViews - views.size()); if (newIndex >= 0) index = newIndex; } T view = null; try { view = views.get(index); } catch (IndexOutOfBoundsException exception) { int match = index + 1; if (match > 1) { Assert.fail(match + " " + classToFilterBy.getSimpleName() + "s" + " are not found!"); } else { Assert.fail(classToFilterBy.getSimpleName() + " is not found!"); } } views = null; return view; }
/** * Waits for a text to be shown. * * @param classToFilterBy the class to filter by * @param text the text that needs to be shown, specified as a regular expression. * @param expectedMinimumNumberOfMatches the minimum number of matches of text that must be shown. * {@code 0} means any number of matches * @param timeout the amount of time in milliseconds to wait * @param scroll {@code true} if scrolling should be performed * @param onlyVisible {@code true} if only visible text views should be waited for * @param hardStoppage {@code true} if search is to be stopped when timeout expires * @return {@code true} if text is found and {@code false} if it is not found before the timeout */ public <T extends TextView> T waitForText( Class<T> classToFilterBy, String text, int expectedMinimumNumberOfMatches, long timeout, boolean scroll, boolean onlyVisible, boolean hardStoppage) { final long endTime = SystemClock.uptimeMillis() + timeout; while (true) { final boolean timedOut = SystemClock.uptimeMillis() > endTime; if (timedOut) { return null; } sleeper.sleep(); if (!hardStoppage) timeout = 0; final T textViewToReturn = searcher.searchFor( classToFilterBy, text, expectedMinimumNumberOfMatches, timeout, scroll, onlyVisible); if (textViewToReturn != null) { return textViewToReturn; } } }
public void init(Directory d) { this.directory = d; searcher = Searcher.create(directory); indexer = new LuceneIndexer(); DateTime dt = new DateTime(); int hours = dt.get(DateTimeFieldType.hourOfDay()); EvaExecutors.getInstance() .getScheduler() .scheduleAtFixedRate( new Runnable() { @Override public void run() { vbotDAOHTMLImplementation.getInstance().flush(); try { SearchManager.getInstanse().updateIndex(true); } catch (IOException e) { log.error(e, e); } } }, 24 - hours, 24, TimeUnit.HOURS); }
// count # deletions, return -1 if unknown. private int countDeletions(Searcher s) throws IOException { int cnt = -1; if (s instanceof IndexSearcher) { cnt = s.maxDoc() - ((IndexSearcher) s).getIndexReader().numDocs(); } return cnt; }
public AssertingSearcher( IndexSearcher indexSearcher, Searcher wrappedSearcher, ShardId shardId) { // we only use the given index searcher here instead of the IS of the wrapped searcher. the IS // might be a wrapped searcher // with a wrapped reader. this.wrappedSearcher = wrappedSearcher; this.shardId = shardId; initialRefCount = wrappedSearcher.reader().getRefCount(); this.indexSearcher = indexSearcher; assert initialRefCount > 0 : "IndexReader#getRefCount() was [" + initialRefCount + "] expected a value > [0] - reader is already closed"; INFLIGHT_ENGINE_SEARCHERS.put( this, new RuntimeException("Unreleased Searcher, source [" + wrappedSearcher.source() + "]")); }
private void searchFor(int n, Searcher searcher) throws Exception { System.out.println("Searching for " + n); Hits hits = searcher.search(QueryParser.parse(intToEnglish(n), "contents", ANALYZER)); System.out.println("Search for " + n + ": total=" + hits.length()); for (int j = 0; j < Math.min(3, hits.length()); j++) { System.out.println("Hit for " + n + ": " + hits.doc(j).get("id")); } }
private Document findByKey(IndexReader reader, Field keyField) throws ParseException, IOException { Searcher searcher = new IndexSearcher(reader); QueryParser queryParser = new QueryParser(luceneVersion, keyField.name(), queryAnalyzer); queryParser.setDefaultOperator(QueryParser.Operator.AND); String queryString = keyField.name() + ":" + keyField.stringValue(); Query query = queryParser.parse(queryString); TopDocs docs = searcher.search(query, 10000); ScoreDoc[] scoreDocs = docs.scoreDocs; if (scoreDocs.length != 1) { return null; } ScoreDoc doc = scoreDocs[0]; return searcher.doc(doc.doc); }
public SearchResult<LogEntry> search(String toSearch, int count, int start) { SearchResult<LogEntry> result = null; try { result = searcher.Result(toSearch, count, start); } catch (Exception e) { log.error(e, e); } return result; }
public void shutDown() { try { indexer.shutDown(); searcher.shutDown(); directory.close(); } catch (IOException e) { // booo } }
public void close() { if (isClosed.compareAndSet(false, true)) { for (Searcher searcher : searchers) { if (searcher != null) { try { searcher.close(); } catch (IOException e) { System.err.println( new Date() + ": *** WARNING: " + luceneId + " unable to close searcher '" + searcher.getDirPath() + "'!"); } } } } }
public void run() { try { String searchQuery = (new BufferedReader(new InputStreamReader(searchSocket.getInputStream()))) .readLine() .trim(); IndexReader reader = writer.getReader(); Searcher searcher = new IndexSearcher(reader); QueryParser indexParser = new QueryParser(Version.LUCENE_30, "data", analyzer); SortField hitSortField = new SortField("date", SortField.LONG); Sort hitSort = new Sort(hitSortField); TopFieldDocs hits = searcher.search(indexParser.parse(searchQuery), null, 1000, hitSort); PrintWriter searchReply = new PrintWriter(searchSocket.getOutputStream(), true); searchReply.println(hits.totalHits + " Hits for " + searchQuery); for (int i = 0; i < hits.totalHits; i++) { Document document = searcher.doc(hits.scoreDocs[i].doc); String host = document.get("hostname"); String date = document.get("date"); String data = document.get("data"); searchReply.print("host: " + host + ", date: " + date + ", data: " + data + "\n\n"); } searchReply.close(); searcher.close(); reader.close(); searchSocket.close(); } catch (Exception ex) { System.out.print("Exception: " + ex + "\n"); } }
/** * After all files are ingested, execute final index commit and final search Cleanup resources, * threads, timers */ @Override public void complete() { if (initialized == false) { return; } // logger.log(Level.INFO, "complete()"); commitTimer.stop(); // handle case if previous search running // cancel it, will re-run after final commit // note: cancellation of Searcher worker is graceful (between keywords) if (currentSearcher != null) { currentSearcher.cancel(false); } // cancel searcher timer, ensure unwanted searcher does not start // before we start the final one if (searchTimer.isRunning()) { searchTimer.stop(); } runSearcher = false; logger.log(Level.INFO, "Running final index commit and search"); // final commit commit(); postIndexSummary(); // run one last search as there are probably some new files committed if (keywords != null && !keywords.isEmpty() && processedFiles == true) { finalSearcher = new Searcher(keywords, true); // final searcher run finalSearcher.execute(); } else { finalSearcherDone = true; managerProxy.postMessage( IngestMessage.createMessage(++messageID, MessageType.INFO, this, "Completed")); } // postSummary(); }
protected void printHits(String test, ScoreDoc[] h, Searcher searcher) throws Exception { System.err.println("------- " + test + " -------"); DecimalFormat f = new DecimalFormat("0.000000000"); for (int i = 0; i < h.length; i++) { Document d = searcher.doc(h[i].doc); float score = h[i].score; System.err.println("#" + i + ": " + f.format(score) + " - " + d.get("id")); } }
// @Override public List<Integer> searchFor(SearchQuery query) { // TODO Auto-generated method stub List<Integer> result = null; try { // System.out.println("searching for "+query.getQ()); result = Searcher.search(indexDir, query); // System.out.println("haha directory for search "+indexDir); } catch (Exception e) { e.printStackTrace(); } return result; }
/** * Waits for a web element. * * @param by the By object. Examples are By.id("id") and By.name("name") * @param minimumNumberOfMatches the minimum number of matches that are expected to be shown. * {@code 0} means any number of matches * @param timeout the the amount of time in milliseconds to wait * @param scroll {@code true} if scrolling should be performed */ public WebElement waitForWebElement( final By by, int minimumNumberOfMatches, int timeout, boolean scroll) { final long endTime = SystemClock.uptimeMillis() + timeout; while (true) { final boolean timedOut = SystemClock.uptimeMillis() > endTime; if (timedOut) { searcher.logMatchesFound(by.getValue()); return null; } sleeper.sleep(); WebElement webElementToReturn = searcher.searchForWebElement(by, minimumNumberOfMatches); if (webElementToReturn != null) return webElementToReturn; if (scroll) { scroller.scrollDown(); } } }
public boolean checkPDF(File pdfFile, String text) { int result = 0; try { IndexItem pdfIndexItem = index(pdfFile); // creating an instance of the indexer class and indexing the items Indexer indexer = new Indexer(INDEX_DIR); indexer.index(pdfIndexItem); indexer.close(); // creating an instance of the Searcher class to the query the index Searcher searcher = new Searcher(INDEX_DIR); result = searcher.findByContent(text, DEFAULT_RESULT_SIZE); searcher.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); } return containsWord(result); }
@Override public void close() throws ElasticsearchException { RuntimeException remove = INFLIGHT_ENGINE_SEARCHERS.remove(this); synchronized (lock) { // make sure we only get this once and store the stack of the first caller! if (remove == null) { assert firstReleaseStack != null; AssertionError error = new AssertionError( "Released Searcher more than once, source [" + wrappedSearcher.source() + "]"); error.initCause(firstReleaseStack); throw error; } else { assert firstReleaseStack == null; firstReleaseStack = new RuntimeException( "Searcher Released first here, source [" + wrappedSearcher.source() + "]"); } } final int refCount = wrappedSearcher.reader().getRefCount(); // this assert seems to be paranoid but given LUCENE-5362 we better add some assertions here // to make sure we catch any potential // problems. assert refCount > 0 : "IndexReader#getRefCount() was [" + refCount + "] expected a value > [0] - reader is already closed. Initial refCount was: [" + initialRefCount + "]"; try { wrappedSearcher.close(); } catch (RuntimeException ex) { logger.debug("Failed to release searcher", ex); throw ex; } }
/** * Returns the correct current Lucene field name to use, based on the complex field name, property * name and list of alternatives. * * @param includeAlternative if true, also includes the default alternative at the end of the * field name (alternatives determine stuff like case-/diacritics-sensitivity). * @return null if field, property or alternative not found; valid Lucene field name otherwise */ public String luceneField(boolean includeAlternative) { // Determine available alternatives based on sensitivity preferences. String[] alternatives = includeAlternative ? getAlternatives() : null; if (searcher == null) { // Mostly for testing. Don't check, just combine field parts. if (alternatives == null || alternatives.length == 0) return ComplexFieldUtil.propertyField(fieldName, propName); return ComplexFieldUtil.propertyField(fieldName, propName, alternatives[0]); } // Find the field and the property. ComplexFieldDesc cfd = searcher.getIndexStructure().getComplexFieldDesc(fieldName); if (cfd == null) return null; if (ComplexFieldUtil.isBookkeepingSubfield(propName)) { // Not a property but a bookkeeping subfield (prob. starttag/endtag); ok, return it // (can be removed when old field naming scheme is removed) return ComplexFieldUtil.bookkeepingField(fieldName, propName); } // Find the property PropertyDesc pd = cfd.getPropertyDesc(propName); if (pd == null) return ComplexFieldUtil.propertyField( fieldName, propName); // doesn't exist? use plain property name if (alternatives == null || alternatives.length == 0) { // Don't use any alternatives return ComplexFieldUtil.propertyField(fieldName, propName); } // Find the first available alternative to use for (String alt : alternatives) { if (pd.hasAlternative(alt)) { // NOTE: is this loop necessary at all? getAlternatives() only // returns available alternatives, so the first one should always // be okay, right? return ComplexFieldUtil.propertyField(fieldName, propName, alt); } } // No valid alternative found. Use plain property. // NOTE: should never happen, and doesn't make sense anymore as there are // no 'plain properties' anymore. return ComplexFieldUtil.propertyField(fieldName, propName); }
/** * Return alternatives for the current field/prop that exist and are appropriate for our current * settings. * * @return the alternatives that exist, in order of appropriateness */ private String[] getAlternatives() { if (searcher == null) { // Test if (caseSensitive) return new String[] {"s", "i"}; return new String[] {"i", "s"}; } final String s = ComplexFieldUtil.SENSITIVE_ALT_NAME; final String i = ComplexFieldUtil.INSENSITIVE_ALT_NAME; final String ci = ComplexFieldUtil.CASE_INSENSITIVE_ALT_NAME; final String di = ComplexFieldUtil.DIACRITICS_INSENSITIVE_ALT_NAME; ComplexFieldDesc cfd = searcher.getIndexStructure().getComplexFieldDesc(fieldName); if (cfd == null) return null; // Find the property PropertyDesc pd = cfd.getPropertyDesc(propName); SensitivitySetting sensitivity = pd.getSensitivity(); // Collection<String> availableAlternatives = Collections.emptyList(); // if (pd != null) { // availableAlternatives = pd.getAlternatives(); // } // New alternative naming scheme (every alternative has a name) List<String> validAlternatives = new ArrayList<>(); if (!caseSensitive && !diacriticsSensitive) { // search insensitive if available if (sensitivity != SensitivitySetting.ONLY_SENSITIVE) validAlternatives.add(i); if (sensitivity != SensitivitySetting.ONLY_INSENSITIVE) validAlternatives.add(s); } else if (caseSensitive && diacriticsSensitive) { // search fully-sensitive if available if (sensitivity != SensitivitySetting.ONLY_INSENSITIVE) validAlternatives.add(s); if (sensitivity != SensitivitySetting.ONLY_SENSITIVE) validAlternatives.add(i); } else if (!diacriticsSensitive) { // search case-sensitive if available if (sensitivity == SensitivitySetting.CASE_AND_DIACRITICS_SEPARATE) validAlternatives.add(di); if (sensitivity != SensitivitySetting.ONLY_INSENSITIVE) validAlternatives.add(s); if (sensitivity != SensitivitySetting.ONLY_SENSITIVE) validAlternatives.add(i); } else { // search diacritics-sensitive if available if (sensitivity == SensitivitySetting.CASE_AND_DIACRITICS_SEPARATE) validAlternatives.add(ci); if (sensitivity != SensitivitySetting.ONLY_SENSITIVE) validAlternatives.add(i); if (sensitivity != SensitivitySetting.ONLY_INSENSITIVE) validAlternatives.add(s); } return validAlternatives.toArray(new String[] {}); }
/** * Waits for a view to be shown. * * @param viewClass the {@code View} class to wait for * @param index the index of the view that is expected to be shown. * @param timeout the amount of time in milliseconds to wait * @param scroll {@code true} if scrolling should be performed * @return {@code true} if view is shown and {@code false} if it is not shown before the timeout */ public <T extends View> boolean waitForView( final Class<T> viewClass, final int index, final int timeout, final boolean scroll) { Set<T> uniqueViews = new HashSet<T>(); final long endTime = SystemClock.uptimeMillis() + timeout; boolean foundMatchingView; while (SystemClock.uptimeMillis() < endTime) { sleeper.sleep(); foundMatchingView = searcher.searchFor(uniqueViews, viewClass, index); if (foundMatchingView) return true; if (scroll) scroller.scrollDown(); } return false; }
/** * Returns the stored fields of the n<sup>th</sup> document in this set. * * <p>Documents are cached, so that repeated requests for the same element may return the same * Document object. * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ public final Document doc(int n) throws CorruptIndexException, IOException { HitDoc hitDoc = hitDoc(n); // Update LRU cache of documents remove(hitDoc); // remove from list, if there addToFront(hitDoc); // add to front of list if (numDocs > maxDocs) { // if cache is full HitDoc oldLast = last; remove(last); // flush last oldLast.doc = null; // let doc get gc'd } if (hitDoc.doc == null) { hitDoc.doc = searcher.doc(hitDoc.id); // cache miss: read document } return hitDoc.doc; }
/** * Waits for a view to be shown. * * @param viewClass the {@code View} class to wait for * @param index the index of the view that is expected to be shown * @param sleep true if should sleep * @param scroll {@code true} if scrolling should be performed * @return {@code true} if view is shown and {@code false} if it is not shown before the timeout */ public <T extends View> boolean waitForView( final Class<T> viewClass, final int index, boolean sleep, boolean scroll) { Set<T> uniqueViews = new HashSet<T>(); boolean foundMatchingView; while (true) { if (sleep) sleeper.sleep(); foundMatchingView = searcher.searchFor(uniqueViews, viewClass, index); if (foundMatchingView) return true; if (scroll && !scroller.scrollDown()) return false; if (!scroll) return false; } }