/*
* (non-Javadoc)
*
* @see org.sakaiproject.search.dao.SearchIndexBuilderWorkerDao#createIndexTransaction(org.sakaiproject.search.api.SearchIndexBuilderWorker)
*/
public void createIndexTransaction(SearchIndexBuilderWorker worker) {
Connection connection = null;
try {
connection = dataSource.getConnection();
long startTime = System.currentTimeMillis();
int totalDocs = 0;
log.debug("Preupdate Start");
indexStorage.doPreIndexUpdate();
log.debug("Preupdate End");
createIndex(worker, connection);
// get lock
try {
log.debug("Post update Start");
indexStorage.doPostIndexUpdate();
log.debug("Post update End");
} catch (IOException e) {
log.error("Failed to do Post Index Update", e); // $NON-NLS-1$
}
log.info("Created Index"); // $NON-NLS-1$
} catch (Exception ex) {
log.warn("Failed to create Index " + ex.getMessage()); // $NON-NLS-1$
log.debug("Traceback is ", ex); // $NON-NLS-1$
} finally {
try {
connection.close();
} catch (Exception ex) {
log.debug(ex);
}
}
}
/**
* @param worker
* @param connection
* @throws Exception
*/
private void createIndex(SearchIndexBuilderWorker worker, Connection connection)
throws Exception {
IndexWriter indexWrite = null;
try {
if (worker.isRunning()) {
indexWrite = indexStorage.getIndexWriter(false);
}
if (indexWrite != null) {
Document doc = new Document();
// The date of indexing
String timeStamp = String.valueOf(System.currentTimeMillis());
doc.add(
new Field(
SearchService.DATE_STAMP, timeStamp, Field.Store.NO, Field.Index.NOT_ANALYZED));
doc.add(
new Field(
SearchService.DATE_STAMP,
CompressionTools.compressString(timeStamp),
Field.Store.YES));
String ref = "---INDEX-CREATED---";
doc.add(
new Field(
SearchService.FIELD_REFERENCE,
CompressionTools.compressString(ref),
Field.Store.YES));
doc.add(
new Field(
SearchService.FIELD_REFERENCE, ref, Field.Store.NO, Field.Index.NOT_ANALYZED));
indexWrite.addDocument(doc);
} else {
log.error("Couldn't get indexWriter to add document!");
}
} catch (Exception ex) {
log.error("Failed to Add Documents ", ex);
throw new Exception(ex);
} finally {
if (indexWrite != null) {
if (log.isDebugEnabled()) {
log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
Directory d = indexWrite.getDirectory();
String[] s = d.listAll();
log.debug("Directory Contains ");
for (int i = 0; i < s.length; i++) {
File f = new File(s[i]);
log.debug(
"\t"
+ String.valueOf(f.length())
+ "\t"
+ new Date(f.lastModified())
+ "\t"
+ s[i]);
}
}
indexStorage.closeIndexWriter(indexWrite);
}
}
}
private void processDeletes(
SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
throws SQLException, IOException {
if (indexStorage.indexExists()) {
IndexReader indexReader = null;
try {
indexReader = indexStorage.getIndexReader();
// Open the index
for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
worker.isRunning() && tditer.hasNext(); ) {
SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
if (!SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
// should only be getting pending
// items
log.warn(
" Found Item that was not pending " //$NON-NLS-1$
+ sbi.getName());
continue;
}
if (SearchBuilderItem.ACTION_UNKNOWN.equals(sbi.getSearchaction())) {
sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, sbi);
continue;
}
// remove document
// if this is mult segment it might not work.
try {
indexReader.deleteDocuments(new Term(SearchService.FIELD_REFERENCE, sbi.getName()));
if (SearchBuilderItem.ACTION_DELETE.equals(sbi.getSearchaction())) {
sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, sbi);
} else {
sbi.setSearchstate(SearchBuilderItem.STATE_PENDING_2);
}
} catch (IOException ex) {
log.warn("Failed to delete Page ", ex); // $NON-NLS-1$
}
}
} finally {
if (indexReader != null) {
indexStorage.closeIndexReader(indexReader);
indexReader = null;
}
}
}
}
/*
* (non-Javadoc)
*
* @see org.sakaiproject.search.component.dao.impl.SearchIndexBuilderWorkerDao#processToDoListTransaction()
*/
public void processToDoListTransaction(SearchIndexBuilderWorker worker, int indexBatchSize) {
Connection connection = null;
try {
connection = dataSource.getConnection();
long startTime = System.currentTimeMillis();
int totalDocs = 0;
// Load the list
List runtimeToDo = findPending(indexBatchSize, connection, worker);
totalDocs = runtimeToDo.size();
log.debug("Processing " + totalDocs + " documents"); // $NON-NLS-1$ //$NON-NLS-2$
if (totalDocs > 0) {
log.debug("Preupdate Start");
indexStorage.doPreIndexUpdate();
log.debug("Preupdate End");
// get lock
// this needs to be exclusive
log.debug("Process Deletes Start");
processDeletes(worker, connection, runtimeToDo);
log.debug("Process Deletes End");
// upate and release lock
// after a process Deletes the index needs to updated
// can be parallel
log.debug("Process Add Start");
processAdd(worker, connection, runtimeToDo);
log.debug("Process Add End");
log.debug("Complete Update Start");
completeUpdate(worker, connection, runtimeToDo);
log.debug("Complete Update End");
// get lock
try {
log.debug("Post update Start");
indexStorage.doPostIndexUpdate();
log.debug("Post update End");
} catch (IOException e) {
log.error("Failed to do Post Index Update", e); // $NON-NLS-1$
}
// release lock
}
if (worker.isRunning()) {
long endTime = System.currentTimeMillis();
float totalTime = endTime - startTime;
float ndocs = totalDocs;
if (totalDocs > 0) {
float docspersec = 1000 * ndocs / totalTime;
log.info(
"Completed Process List of "
+ totalDocs
+ " at " //$NON-NLS-1$ //$NON-NLS-2$
+ docspersec
+ " documents/per second "); //$NON-NLS-1$
}
}
} catch (Exception ex) {
log.warn("Failed to perform index cycle " + ex.getMessage(), ex); // $NON-NLS-1$
// rollback any uncommitted transactions
try {
connection.rollback();
} catch (SQLException e) {
log.debug(e);
}
} finally {
try {
connection.close();
} catch (Exception ex) {
log.debug(ex);
}
}
}
private void processAdd(
SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
throws Exception {
IndexWriter indexWrite = null;
try {
if (worker.isRunning()) {
indexWrite = indexStorage.getIndexWriter(false);
}
long last = System.currentTimeMillis();
for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
worker.isRunning() && tditer.hasNext(); ) {
Reader contentReader = null;
try {
SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
// only add adds, that have been deleted or are locked
// sucessfully
if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate())
&& !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
continue;
}
// Reference ref =
// entityManager.newReference(sbi.getName());
String ref = sbi.getName();
if (ref == null) {
log.error(
"Unrecognised trigger object presented to index builder " //$NON-NLS-1$
+ sbi);
}
long startDocIndex = System.currentTimeMillis();
worker.setStartDocIndex(startDocIndex);
worker.setNowIndexing(ref);
try {
try {
// Entity entity = ref.getEntity();
EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref);
boolean indexDoc = true;
if (searchIndexBuilder.isOnlyIndexSearchToolSites()) {
try {
String siteId = sep.getSiteId(sbi.getName());
Site s = SiteService.getSite(siteId);
ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$
if (t == null) {
indexDoc = false;
log.debug(
"Not indexing " //$NON-NLS-1$
+ sbi.getName()
+ " as it has no search tool"); //$NON-NLS-1$
}
} catch (Exception ex) {
indexDoc = false;
log.debug(
"Not indexing "
+ sbi.getName() // $NON-NLS-1$
+ " as it has no site",
ex); //$NON-NLS-1$
}
}
if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) {
DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService);
// Reader contentReader = null;
Document doc =
DocumentIndexingUtils.createIndexDocument(
ref,
digestStorageUtil,
sep,
serverConfigurationService.getServerUrl(),
contentReader);
// indexDocTMP(ref, sep);
log.debug("Indexing Document " + doc); // $NON-NLS-1$
indexWrite.addDocument(doc);
log.debug("Done Indexing Document " + doc); // $NON-NLS-1$
processRDF(ref, sep);
} else {
if (log.isDebugEnabled()) {
if (!indexDoc) {
log.debug("Ignored Document: Fileteed out by site " + ref); // $NON-NLS-1$
} else if (sep == null) {
log.debug("Ignored Document: No EntityContentProducer " + ref); // $NON-NLS-1$
} else if (!sep.isForIndex(ref)) {
log.debug("Ignored Document: Marked as Ignore " + ref); // $NON-NLS-1$
} else if (sep.getSiteId(ref) == null) {
log.debug("Ignored Document: No Site ID " + ref); // $NON-NLS-1$
} else {
log.debug("Ignored Document: Reason Unknown " + ref); // $NON-NLS-1$
}
}
}
} catch (Exception e1) {
log.debug(
" Failed to index document for "
+ ref
+ " cause: " //$NON-NLS-1$
+ e1.getMessage(),
e1);
}
sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, sbi);
} catch (Exception e1) {
log.debug(
" Failed to index document cause: " //$NON-NLS-1$
+ e1.getMessage());
}
long endDocIndex = System.currentTimeMillis();
worker.setLastIndex(endDocIndex - startDocIndex);
if ((endDocIndex - startDocIndex) > 60000L) {
log.warn(
"Slow index operation " //$NON-NLS-1$
+ String.valueOf((endDocIndex - startDocIndex) / 1000)
+ " seconds to index " //$NON-NLS-1$
+ ref);
}
// update this node lock to indicate its
// still alove, no document should
// take more than 2 mins to process
// ony do this check once every minute
long now = System.currentTimeMillis();
if ((now - last) > (60L * 1000L)) {
last = System.currentTimeMillis();
if (!worker.getLockTransaction(15L * 60L * 1000L, true)) {
throw new Exception(
"Transaction Lock Expired while indexing " //$NON-NLS-1$
+ ref);
}
}
} finally {
if (contentReader != null) {
try {
contentReader.close();
} catch (IOException ioex) {
log.debug(ioex);
}
}
}
}
worker.setStartDocIndex(System.currentTimeMillis());
worker.setNowIndexing(
Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); // $NON-NLS-1$
} catch (Exception ex) {
log.error("Failed to Add Documents ", ex);
throw new Exception(ex);
} finally {
if (indexWrite != null) {
if (log.isDebugEnabled()) {
log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
Directory d = indexWrite.getDirectory();
String[] s = d.listAll();
log.debug("Directory Contains ");
for (int i = 0; i < s.length; i++) {
File f = new File(s[i]);
log.debug(
"\t"
+ String.valueOf(f.length())
+ "\t"
+ new Date(f.lastModified())
+ "\t"
+ s[i]);
}
}
indexStorage.closeIndexWriter(indexWrite);
}
}
}
/*
* (non-Javadoc)
*
* @see org.sakaiproject.search.dao.SearchIndexBuilderWorkerDao#indexExists()
*/
public boolean indexExists() {
return indexStorage.centralIndexExists();
}
public boolean isLockRequired() {
return !indexStorage.isMultipleIndexers();
}
