/** * @param worker * @param connection * @throws Exception */ private void createIndex(SearchIndexBuilderWorker worker, Connection connection) throws Exception { IndexWriter indexWrite = null; try { if (worker.isRunning()) { indexWrite = indexStorage.getIndexWriter(false); } if (indexWrite != null) { Document doc = new Document(); // The date of indexing String timeStamp = String.valueOf(System.currentTimeMillis()); doc.add( new Field( SearchService.DATE_STAMP, timeStamp, Field.Store.NO, Field.Index.NOT_ANALYZED)); doc.add( new Field( SearchService.DATE_STAMP, CompressionTools.compressString(timeStamp), Field.Store.YES)); String ref = "---INDEX-CREATED---"; doc.add( new Field( SearchService.FIELD_REFERENCE, CompressionTools.compressString(ref), Field.Store.YES)); doc.add( new Field( SearchService.FIELD_REFERENCE, ref, Field.Store.NO, Field.Index.NOT_ANALYZED)); indexWrite.addDocument(doc); } else { log.error("Couldn't get indexWriter to add document!"); } } catch (Exception ex) { log.error("Failed to Add Documents ", ex); throw new Exception(ex); } finally { if (indexWrite != null) { if (log.isDebugEnabled()) { log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents"); Directory d = indexWrite.getDirectory(); String[] s = d.listAll(); log.debug("Directory Contains "); for (int i = 0; i < s.length; i++) { File f = new File(s[i]); log.debug( "\t" + String.valueOf(f.length()) + "\t" + new Date(f.lastModified()) + "\t" + s[i]); } } indexStorage.closeIndexWriter(indexWrite); } } }
private void processAdd( SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo) throws Exception { IndexWriter indexWrite = null; try { if (worker.isRunning()) { indexWrite = indexStorage.getIndexWriter(false); } long last = System.currentTimeMillis(); for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator(); worker.isRunning() && tditer.hasNext(); ) { Reader contentReader = null; try { SearchBuilderItem sbi = (SearchBuilderItem) tditer.next(); // only add adds, that have been deleted or are locked // sucessfully if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate()) && !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) { continue; } // Reference ref = // entityManager.newReference(sbi.getName()); String ref = sbi.getName(); if (ref == null) { log.error( "Unrecognised trigger object presented to index builder " //$NON-NLS-1$ + sbi); } long startDocIndex = System.currentTimeMillis(); worker.setStartDocIndex(startDocIndex); worker.setNowIndexing(ref); try { try { // Entity entity = ref.getEntity(); EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref); boolean indexDoc = true; if (searchIndexBuilder.isOnlyIndexSearchToolSites()) { try { String siteId = sep.getSiteId(sbi.getName()); Site s = SiteService.getSite(siteId); ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$ if (t == null) { indexDoc = false; log.debug( "Not indexing " //$NON-NLS-1$ + sbi.getName() + " as it has no search tool"); //$NON-NLS-1$ } } catch (Exception ex) { indexDoc = false; log.debug( "Not indexing " + sbi.getName() // $NON-NLS-1$ + " as it has no site", ex); //$NON-NLS-1$ } } if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) { DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService); // Reader contentReader = null; Document doc = DocumentIndexingUtils.createIndexDocument( ref, digestStorageUtil, sep, serverConfigurationService.getServerUrl(), contentReader); // indexDocTMP(ref, sep); log.debug("Indexing Document " + doc); // $NON-NLS-1$ indexWrite.addDocument(doc); log.debug("Done Indexing Document " + doc); // $NON-NLS-1$ processRDF(ref, sep); } else { if (log.isDebugEnabled()) { if (!indexDoc) { log.debug("Ignored Document: Fileteed out by site " + ref); // $NON-NLS-1$ } else if (sep == null) { log.debug("Ignored Document: No EntityContentProducer " + ref); // $NON-NLS-1$ } else if (!sep.isForIndex(ref)) { log.debug("Ignored Document: Marked as Ignore " + ref); // $NON-NLS-1$ } else if (sep.getSiteId(ref) == null) { log.debug("Ignored Document: No Site ID " + ref); // $NON-NLS-1$ } else { log.debug("Ignored Document: Reason Unknown " + ref); // $NON-NLS-1$ } } } } catch (Exception e1) { log.debug( " Failed to index document for " + ref + " cause: " //$NON-NLS-1$ + e1.getMessage(), e1); } sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED); updateOrSave(connection, sbi); } catch (Exception e1) { log.debug( " Failed to index document cause: " //$NON-NLS-1$ + e1.getMessage()); } long endDocIndex = System.currentTimeMillis(); worker.setLastIndex(endDocIndex - startDocIndex); if ((endDocIndex - startDocIndex) > 60000L) { log.warn( "Slow index operation " //$NON-NLS-1$ + String.valueOf((endDocIndex - startDocIndex) / 1000) + " seconds to index " //$NON-NLS-1$ + ref); } // update this node lock to indicate its // still alove, no document should // take more than 2 mins to process // ony do this check once every minute long now = System.currentTimeMillis(); if ((now - last) > (60L * 1000L)) { last = System.currentTimeMillis(); if (!worker.getLockTransaction(15L * 60L * 1000L, true)) { throw new Exception( "Transaction Lock Expired while indexing " //$NON-NLS-1$ + ref); } } } finally { if (contentReader != null) { try { contentReader.close(); } catch (IOException ioex) { log.debug(ioex); } } } } worker.setStartDocIndex(System.currentTimeMillis()); worker.setNowIndexing( Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); // $NON-NLS-1$ } catch (Exception ex) { log.error("Failed to Add Documents ", ex); throw new Exception(ex); } finally { if (indexWrite != null) { if (log.isDebugEnabled()) { log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents"); Directory d = indexWrite.getDirectory(); String[] s = d.listAll(); log.debug("Directory Contains "); for (int i = 0; i < s.length; i++) { File f = new File(s[i]); log.debug( "\t" + String.valueOf(f.length()) + "\t" + new Date(f.lastModified()) + "\t" + s[i]); } } indexStorage.closeIndexWriter(indexWrite); } } }