/** @param indexWriter */
protected void closeIndexWrite(IndexWriter indexWriter) {
if (indexWriter != null) {
try {
indexWriter.close();
if (IndexWriter.isLocked(indexWriter.getDirectory())) {
IndexWriter.unlock(indexWriter.getDirectory());
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
public DirectoryReader getReader(boolean applyDeletions) throws IOException {
getReaderCalled = true;
if (r.nextInt(20) == 2) {
doRandomForceMerge();
}
if (!applyDeletions || r.nextBoolean()) {
if (LuceneTestCase.VERBOSE) {
System.out.println("RIW.getReader: use NRT reader");
}
if (r.nextInt(5) == 1) {
w.commit();
}
return w.getReader(applyDeletions);
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("RIW.getReader: open new reader");
}
w.commit();
if (r.nextBoolean()) {
return DirectoryReader.open(w.getDirectory());
} else {
return w.getReader(applyDeletions);
}
}
}
/**
* @param worker
* @param connection
* @throws Exception
*/
private void createIndex(SearchIndexBuilderWorker worker, Connection connection)
throws Exception {
IndexWriter indexWrite = null;
try {
if (worker.isRunning()) {
indexWrite = indexStorage.getIndexWriter(false);
}
if (indexWrite != null) {
Document doc = new Document();
// The date of indexing
String timeStamp = String.valueOf(System.currentTimeMillis());
doc.add(
new Field(
SearchService.DATE_STAMP, timeStamp, Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(
new Field(
SearchService.DATE_STAMP,
CompressionTools.compressString(timeStamp),
Field.Store.YES));
String ref = "---INDEX-CREATED---";
doc.add(
new Field(
SearchService.FIELD_REFERENCE,
CompressionTools.compressString(ref),
Field.Store.YES));
doc.add(
new Field(
SearchService.FIELD_REFERENCE, ref, Field.Store.NO, Field.Index.NOT_ANALYZED));
indexWrite.addDocument(doc);
} else {
log.error("Couldn't get indexWriter to add document!");
}
} catch (Exception ex) {
log.error("Failed to Add Documents ", ex);
throw new Exception(ex);
} finally {
if (indexWrite != null) {
if (log.isDebugEnabled()) {
log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
Directory d = indexWrite.getDirectory();
String[] s = d.listAll();
log.debug("Directory Contains ");
for (int i = 0; i < s.length; i++) {
File f = new File(s[i]);
log.debug(
"\t"
+ String.valueOf(f.length())
+ "\t"
+ new Date(f.lastModified())
+ "\t"
+ s[i]);
}
}
indexStorage.closeIndexWriter(indexWrite);
}
}
}
private void processAdd(
SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
throws Exception {
IndexWriter indexWrite = null;
try {
if (worker.isRunning()) {
indexWrite = indexStorage.getIndexWriter(false);
}
long last = System.currentTimeMillis();
for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
worker.isRunning() && tditer.hasNext(); ) {
Reader contentReader = null;
try {
SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
// only add adds, that have been deleted or are locked
// sucessfully
if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate())
&& !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
continue;
}
// Reference ref =
// entityManager.newReference(sbi.getName());
String ref = sbi.getName();
if (ref == null) {
log.error(
"Unrecognised trigger object presented to index builder " //$NON-NLS-1$
+ sbi);
}
long startDocIndex = System.currentTimeMillis();
worker.setStartDocIndex(startDocIndex);
worker.setNowIndexing(ref);
try {
try {
// Entity entity = ref.getEntity();
EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref);
boolean indexDoc = true;
if (searchIndexBuilder.isOnlyIndexSearchToolSites()) {
try {
String siteId = sep.getSiteId(sbi.getName());
Site s = SiteService.getSite(siteId);
ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$
if (t == null) {
indexDoc = false;
log.debug(
"Not indexing " //$NON-NLS-1$
+ sbi.getName()
+ " as it has no search tool"); //$NON-NLS-1$
}
} catch (Exception ex) {
indexDoc = false;
log.debug(
"Not indexing "
+ sbi.getName() // $NON-NLS-1$
+ " as it has no site",
ex); //$NON-NLS-1$
}
}
if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) {
DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService);
// Reader contentReader = null;
Document doc =
DocumentIndexingUtils.createIndexDocument(
ref,
digestStorageUtil,
sep,
serverConfigurationService.getServerUrl(),
contentReader);
// indexDocTMP(ref, sep);
log.debug("Indexing Document " + doc); // $NON-NLS-1$
indexWrite.addDocument(doc);
log.debug("Done Indexing Document " + doc); // $NON-NLS-1$
processRDF(ref, sep);
} else {
if (log.isDebugEnabled()) {
if (!indexDoc) {
log.debug("Ignored Document: Fileteed out by site " + ref); // $NON-NLS-1$
} else if (sep == null) {
log.debug("Ignored Document: No EntityContentProducer " + ref); // $NON-NLS-1$
} else if (!sep.isForIndex(ref)) {
log.debug("Ignored Document: Marked as Ignore " + ref); // $NON-NLS-1$
} else if (sep.getSiteId(ref) == null) {
log.debug("Ignored Document: No Site ID " + ref); // $NON-NLS-1$
} else {
log.debug("Ignored Document: Reason Unknown " + ref); // $NON-NLS-1$
}
}
}
} catch (Exception e1) {
log.debug(
" Failed to index document for "
+ ref
+ " cause: " //$NON-NLS-1$
+ e1.getMessage(),
e1);
}
sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, sbi);
} catch (Exception e1) {
log.debug(
" Failed to index document cause: " //$NON-NLS-1$
+ e1.getMessage());
}
long endDocIndex = System.currentTimeMillis();
worker.setLastIndex(endDocIndex - startDocIndex);
if ((endDocIndex - startDocIndex) > 60000L) {
log.warn(
"Slow index operation " //$NON-NLS-1$
+ String.valueOf((endDocIndex - startDocIndex) / 1000)
+ " seconds to index " //$NON-NLS-1$
+ ref);
}
// update this node lock to indicate its
// still alove, no document should
// take more than 2 mins to process
// ony do this check once every minute
long now = System.currentTimeMillis();
if ((now - last) > (60L * 1000L)) {
last = System.currentTimeMillis();
if (!worker.getLockTransaction(15L * 60L * 1000L, true)) {
throw new Exception(
"Transaction Lock Expired while indexing " //$NON-NLS-1$
+ ref);
}
}
} finally {
if (contentReader != null) {
try {
contentReader.close();
} catch (IOException ioex) {
log.debug(ioex);
}
}
}
}
worker.setStartDocIndex(System.currentTimeMillis());
worker.setNowIndexing(
Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); // $NON-NLS-1$
} catch (Exception ex) {
log.error("Failed to Add Documents ", ex);
throw new Exception(ex);
} finally {
if (indexWrite != null) {
if (log.isDebugEnabled()) {
log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
Directory d = indexWrite.getDirectory();
String[] s = d.listAll();
log.debug("Directory Contains ");
for (int i = 0; i < s.length; i++) {
File f = new File(s[i]);
log.debug(
"\t"
+ String.valueOf(f.length())
+ "\t"
+ new Date(f.lastModified())
+ "\t"
+ s[i]);
}
}
indexStorage.closeIndexWriter(indexWrite);
}
}
}
public void convert(Application app, File dbhome) throws Exception {
FSDirectory indexDir = FSDirectory.getDirectory(dbhome, false);
if (indexDir instanceof TransFSDirectory) {
FSDirectory.setDisableLocks(true);
TransFSDirectory d = (TransFSDirectory) indexDir;
TransSource source = app.getTransSource();
d.setDriverClass(source.getDriverClass());
d.setUrl(source.getUrl());
d.setUser(source.getUser());
d.setPassword(source.getPassword());
}
File ndbhome = new File(dbhome.getParentFile(), dbhome.getName() + "_tmp");
File olddbhome = new File(dbhome.getParentFile(), dbhome.getName() + "_old");
FSDirectory nindexDir = FSDirectory.getDirectory(ndbhome, true);
if (nindexDir instanceof TransFSDirectory) {
FSDirectory.setDisableLocks(true);
TransFSDirectory d = (TransFSDirectory) nindexDir;
TransSource source = app.getTransSource();
d.setDriverClass(source.getDriverClass());
d.setUrl(source.getUrl());
d.setUser(source.getUser());
d.setPassword(source.getPassword());
}
IndexSearcher searcher = null;
IndexWriter writer = null;
LuceneManager lmgr = null;
try {
searcher = new IndexSearcher(indexDir);
PerFieldAnalyzerWrapper a = LuceneManager.buildAnalyzer();
writer = IndexWriterManager.getWriter(nindexDir, a, true);
final int numDocs = searcher.getIndexReader().numDocs();
HashSet deldocs = new HashSet();
HashMap infos = new HashMap();
for (int i = 0; i < numDocs; i++) {
Document doc = searcher.doc(i);
String delprop = doc.get(DeletedInfos.DELETED);
String layerStr = doc.get(LuceneManager.LAYER_OF_SAVE);
int layer = -1;
try {
layer = Integer.parseInt(layerStr);
} catch (Exception ex) {
layer = -1;
}
final String id =
doc.get(LuceneManager.ID)
+ DeletedInfos.KEY_SEPERATOR
+ doc.get(LuceneManager.LAYER_OF_SAVE);
if (delprop != null && "true".equals(delprop) /* && layer == DbKey.LIVE_LAYER*/) {
deldocs.add(id);
} else {
Object v;
if ((v = infos.get(id)) == null) {
infos.put(id, new Integer(i));
} else {
final String lmod = doc.get(LuceneManager.LASTMODIFIED);
final String lmod_prev = searcher.doc(((Integer) v).intValue()).get("_lastmodified");
if (lmod_prev == null || (lmod != null && lmod.compareTo(lmod_prev) > 0)) {
infos.put(id, new Integer(i));
}
}
}
}
ArrayList listOfMaps = new ArrayList();
for (int i = 0; i < numDocs; i++) {
Document doc = searcher.doc(i);
String delprop = doc.get(DeletedInfos.DELETED);
String layerStr = doc.get(LuceneManager.LAYER_OF_SAVE);
int layer = -1;
try {
layer = Integer.parseInt(layerStr);
} catch (Exception ex) {
layer = -1;
}
final String id =
doc.get(LuceneManager.ID)
+ DeletedInfos.KEY_SEPERATOR
+ doc.get(LuceneManager.LAYER_OF_SAVE);
if (delprop != null && "true".equals(delprop)) {
continue;
} else if (id != null && deldocs.contains(id) /* && layer == DbKey.LIVE_LAYER*/) {
continue;
}
Integer idx = (Integer) infos.get(id);
if (idx != null && i != idx.intValue()) {
continue;
}
Document ndoc = convertDocument(doc);
if (this.recordNodes) {
listOfMaps.add(LuceneManager.luceneDocumentToMap(doc));
}
if (ndoc != null) {
writer.addDocument(ndoc);
}
}
if (this.recordNodes) {
lmgr = new LuceneManager(this.app, false, true);
this.allNodes = new HashMap();
final int size = listOfMaps.size();
for (int i = 0; i < size; i++) {
HashMap m = (HashMap) listOfMaps.get(i);
INode n = lmgr.mapToNode(m);
this.allNodes.put(n.getID(), getPath(n));
n = null;
}
}
} catch (Exception ex) {
ex.printStackTrace();
throw new RuntimeException(ex);
} finally {
if (searcher != null) {
try {
searcher.close();
} catch (Exception ex) {
app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), ex);
}
}
if (lmgr != null) {
lmgr.shutdown();
lmgr = null;
}
indexDir.close();
SegmentInfos sinfos = IndexObjectsFactory.getFSSegmentInfos(indexDir);
sinfos.clear();
IndexObjectsFactory.removeDeletedInfos(indexDir);
}
Connection conn = null;
boolean exceptionOccured = false;
try {
if (writer != null) {
TransSource ts = app.getTransSource();
conn = ts.getConnection();
DatabaseMetaData dmd = conn.getMetaData();
ResultSet rs = dmd.getColumns(null, null, "Lucene", "version");
if (!rs.next()) {
final String alterTbl = "ALTER TABLE Lucene ADD version INT NOT NULL DEFAULT 1";
PreparedStatement pstmt = null;
try {
pstmt = conn.prepareStatement(alterTbl);
pstmt.execute();
} catch (SQLException sqle) {
app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), sqle);
} finally {
if (pstmt != null) {
pstmt.close();
pstmt = null;
}
}
}
rs.close();
rs = null;
writer.close();
writer.flushCache(); // TODO:writer.writeSegmentsFile();
LuceneManager.commitSegments(conn, app, writer.getDirectory());
writer.finalizeTrans();
this.updateSQL(conn);
}
} catch (Exception ex) {
ex.printStackTrace();
exceptionOccured = true;
throw new RuntimeException(ex);
} finally {
if (conn != null) {
try {
if (!conn.getAutoCommit()) {
if (!exceptionOccured) {
conn.commit();
} else {
conn.rollback();
}
}
conn.close();
} catch (Exception ex) {
app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), ex);
}
conn = null;
}
nindexDir.close();
SegmentInfos sinfos = IndexObjectsFactory.getFSSegmentInfos(nindexDir);
sinfos.clear();
IndexObjectsFactory.removeDeletedInfos(nindexDir);
}
if (!dbhome.renameTo(olddbhome)) {
throw new Exception("Could not move the old version of the db into " + olddbhome);
}
if (!ndbhome.renameTo(dbhome)) {
throw new Exception("Could not move the newer version of the db into " + dbhome);
}
File oldBlobDir = new File(olddbhome, "blob");
File newBlobDir = new File(ndbhome, "blob");
oldBlobDir.renameTo(newBlobDir);
if (!FileUtils.deleteDir(olddbhome)) {
throw new Exception("Could not delete the old version of the db at " + olddbhome);
}
}
