private Fields generateTermVectorsFromDoc(TermVectorRequest request, boolean doAllFields) throws IOException { // parse the document, at the moment we do update the mapping, just like percolate ParsedDocument parsedDocument = parseDocument(indexShard.shardId().getIndex(), request.type(), request.doc()); // select the right fields and generate term vectors ParseContext.Document doc = parsedDocument.rootDoc(); Collection<String> seenFields = new HashSet<>(); Collection<GetField> getFields = new HashSet<>(); for (IndexableField field : doc.getFields()) { FieldMapper fieldMapper = indexShard.mapperService().smartNameFieldMapper(field.name()); if (seenFields.contains(field.name())) { continue; } else { seenFields.add(field.name()); } if (!isValidField(fieldMapper)) { continue; } if (request.selectedFields() == null && !doAllFields && !fieldMapper.fieldType().storeTermVectors()) { continue; } if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) { continue; } String[] values = doc.getValues(field.name()); getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values))); } return generateTermVectors(getFields, request.offsets(), request.perFieldAnalyzer()); }
/** * Return a query that will return docs like the passed lucene document ID. * * @param docNum the documentID of the lucene doc to generate the 'More Like This" query for. * @return a query that will return docs like the passed lucene document ID. */ public Query like(int docNum) throws IOException { if (fieldNames == null) { // gather list of valid fields from lucene Collection<String> fields = MultiFields.getIndexedFields(ir); fieldNames = fields.toArray(new String[fields.size()]); } return createQuery(retrieveTerms(docNum)); }
/** * Indexes page using the given IndexWriter. * * @param page WikiPage * @param text Page text to index * @param writer The Lucene IndexWriter to use for indexing * @return the created index Document * @throws IOException If there's an indexing problem */ protected Document luceneIndexPage(WikiPage page, String text, IndexWriter writer) throws IOException { if (log.isDebugEnabled()) log.debug("Indexing " + page.getName() + "..."); // make a new, empty document Document doc = new Document(); if (text == null) return doc; // Raw name is the keyword we'll use to refer to this document for updates. Field field = new Field(LUCENE_ID, page.getName(), Field.Store.YES, Field.Index.NOT_ANALYZED); doc.add(field); // Body text. It is stored in the doc for search contexts. field = new Field( LUCENE_PAGE_CONTENTS, text, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); doc.add(field); // Allow searching by page name. Both beautified and raw String unTokenizedTitle = StringUtils.replaceChars( page.getName(), MarkupParser.PUNCTUATION_CHARS_ALLOWED, c_punctuationSpaces); field = new Field( LUCENE_PAGE_NAME, TextUtil.beautifyString(page.getName()) + " " + unTokenizedTitle, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); doc.add(field); // Allow searching by authorname if (page.getAuthor() != null) { field = new Field( LUCENE_AUTHOR, page.getAuthor(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); doc.add(field); } // Now add the names of the attachments of this page try { Collection attachments = m_engine.getAttachmentManager().listAttachments(page); String attachmentNames = ""; for (Iterator it = attachments.iterator(); it.hasNext(); ) { Attachment att = (Attachment) it.next(); attachmentNames += att.getName() + ";"; } field = new Field( LUCENE_ATTACHMENTS, attachmentNames, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); doc.add(field); } catch (ProviderException e) { // Unable to read attachments log.error("Failed to get attachments for page", e); } writer.addDocument(doc); return doc; }
/** * Performs a full Lucene reindex, if necessary. * * @throws IOException If there's a problem during indexing */ protected void doFullLuceneReindex() throws IOException { File dir = new File(m_luceneDirectory); String[] filelist = dir.list(); if (filelist == null) { throw new IOException( "Invalid Lucene directory: cannot produce listing: " + dir.getAbsolutePath()); } try { if (filelist.length == 0) { // // No files? Reindex! // Date start = new Date(); IndexWriter writer = null; log.info("Starting Lucene reindexing, this can take a couple minutes..."); Directory luceneDir = new SimpleFSDirectory(dir, null); try { writer = getIndexWriter(luceneDir); Collection allPages = m_engine.getPageManager().getAllPages(); for (Iterator iterator = allPages.iterator(); iterator.hasNext(); ) { WikiPage page = (WikiPage) iterator.next(); try { String text = m_engine .getPageManager() .getPageText(page.getName(), WikiProvider.LATEST_VERSION); luceneIndexPage(page, text, writer); } catch (IOException e) { log.warn("Unable to index page " + page.getName() + ", continuing to next ", e); } } Collection allAttachments = m_engine.getAttachmentManager().getAllAttachments(); for (Iterator iterator = allAttachments.iterator(); iterator.hasNext(); ) { Attachment att = (Attachment) iterator.next(); try { String text = getAttachmentContent(att.getName(), WikiProvider.LATEST_VERSION); luceneIndexPage(att, text, writer); } catch (IOException e) { log.warn("Unable to index attachment " + att.getName() + ", continuing to next", e); } } } finally { close(writer); } Date end = new Date(); log.info( "Full Lucene index finished in " + (end.getTime() - start.getTime()) + " milliseconds."); } else { log.info("Files found in Lucene directory, not reindexing."); } } catch (NoClassDefFoundError e) { log.info("Lucene libraries do not exist - not using Lucene."); } catch (IOException e) { log.error("Problem while creating Lucene index - not using Lucene.", e); } catch (ProviderException e) { log.error("Problem reading pages while creating Lucene index (JSPWiki won't start.)", e); throw new IllegalArgumentException("unable to create Lucene index"); } catch (Exception e) { log.error("Unable to start lucene", e); } }