private int completeUpdate(
SearchIndexBuilderWorker worker, Connection connection, List runtimeToDo) throws Exception {
try {
for (Iterator tditer = runtimeToDo.iterator(); worker.isRunning() && tditer.hasNext(); ) {
SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
if (SearchBuilderItem.STATE_COMPLETED.equals(sbi.getSearchstate())) {
if (SearchBuilderItem.ACTION_DELETE.equals(sbi.getSearchaction())) {
delete(connection, sbi);
connection.commit();
} else {
updateOrSave(connection, sbi);
connection.commit();
}
}
}
return runtimeToDo.size();
} catch (Exception ex) {
log.warn(
"Failed to update state in database due to " //$NON-NLS-1$
+ ex.getMessage()
+ " this will be corrected on the next run of the IndexBuilder, no cause for alarm"); //$NON-NLS-1$
}
return 0;
}
/**
* get the master action of known master item
*
* @param master
* @return
*/
private Integer getMasterAction(SearchBuilderItem master) {
if (master.getName().equals(SearchBuilderItem.GLOBAL_MASTER)) {
if (SearchBuilderItem.STATE_PENDING.equals(master.getSearchstate())) {
return master.getSearchaction();
}
}
return SearchBuilderItem.STATE_UNKNOWN;
}
/**
* Get the site that the siteMaster references
*
* @param siteMaster
* @return
*/
private String getSiteMasterSite(SearchBuilderItem siteMaster) {
if (siteMaster.getName().startsWith(SearchBuilderItem.INDEX_MASTER)
&& !SearchBuilderItem.GLOBAL_CONTEXT.equals(siteMaster.getContext())) {
// this depends on the pattern, perhapse it should be a parse
return siteMaster.getName().substring(SearchBuilderItem.INDEX_MASTER.length() + 1);
}
return null;
}
/**
* get the action for the site master
*
* @param siteMaster
* @return
*/
private Integer getSiteMasterAction(SearchBuilderItem siteMaster) {
if (siteMaster.getName().startsWith(SearchBuilderItem.INDEX_MASTER)
&& !SearchBuilderItem.GLOBAL_CONTEXT.equals(siteMaster.getContext())) {
if (SearchBuilderItem.STATE_PENDING.equals(siteMaster.getSearchstate())) {
return siteMaster.getSearchaction();
}
}
return SearchBuilderItem.STATE_UNKNOWN;
}
private void processDeletes(
SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
throws SQLException, IOException {
if (indexStorage.indexExists()) {
IndexReader indexReader = null;
try {
indexReader = indexStorage.getIndexReader();
// Open the index
for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
worker.isRunning() && tditer.hasNext(); ) {
SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
if (!SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
// should only be getting pending
// items
log.warn(
" Found Item that was not pending " //$NON-NLS-1$
+ sbi.getName());
continue;
}
if (SearchBuilderItem.ACTION_UNKNOWN.equals(sbi.getSearchaction())) {
sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, sbi);
continue;
}
// remove document
// if this is mult segment it might not work.
try {
indexReader.deleteDocuments(new Term(SearchService.FIELD_REFERENCE, sbi.getName()));
if (SearchBuilderItem.ACTION_DELETE.equals(sbi.getSearchaction())) {
sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, sbi);
} else {
sbi.setSearchstate(SearchBuilderItem.STATE_PENDING_2);
}
} catch (IOException ex) {
log.warn("Failed to delete Page ", ex); // $NON-NLS-1$
}
}
} finally {
if (indexReader != null) {
indexStorage.closeIndexReader(indexReader);
indexReader = null;
}
}
}
}
private int populateStatement(PreparedStatement pst, SearchBuilderItem sbi) throws SQLException {
pst.setString(1, sbi.getName());
pst.setString(2, sbi.getContext());
pst.setInt(3, sbi.getSearchaction().intValue());
pst.setInt(4, sbi.getSearchstate().intValue());
pst.setDate(5, new Date(sbi.getVersion().getTime()));
pst.setInt(6, sbi.getItemscope().intValue());
pst.setString(7, sbi.getId());
return 7;
}
private void refreshIndex(Connection connection, SearchBuilderItem controlItem)
throws SQLException {
// delete all and return the master action only
// the caller will then rebuild the index from scratch
log.debug(
"UPDATE ALL RECORDS =========================================================="); //$NON-NLS-1$
Statement stm = null;
try {
stm = connection.createStatement();
if (SearchBuilderItem.GLOBAL_CONTEXT.equals(controlItem.getContext())) {
stm.execute(
"update searchbuilderitem set searchstate = " //$NON-NLS-1$
+ SearchBuilderItem.STATE_PENDING
+ " where itemscope = "
+ SearchBuilderItem.ITEM); // $NON-NLS-1$
} else {
stm.execute(
"update searchbuilderitem set searchstate = " //$NON-NLS-1$
+ SearchBuilderItem.STATE_PENDING
+ " where itemscope = "
+ SearchBuilderItem.ITEM_SITE_MASTER
+ " and context = '"
+ controlItem.getContext() // $NON-NLS-1$
+ "' and name <> '"
+ controlItem.getName()
+ "'"); //$NON-NLS-1$ //$NON-NLS-2$
}
controlItem.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, controlItem);
connection.commit();
} finally {
try {
stm.close();
} catch (Exception ex) {
log.debug(ex);
}
;
}
}
private void delete(Connection connection, SearchBuilderItem sbi) throws SQLException {
PreparedStatement pst = null;
try {
pst =
connection.prepareStatement(
" delete from " //$NON-NLS-1$
+ SEARCH_BUILDER_ITEM_T
+ " where id = ? "); //$NON-NLS-1$
pst.clearParameters();
pst.setString(1, sbi.getId());
pst.execute();
} catch (SQLException ex) {
log.warn("Failed ", ex); // $NON-NLS-1$
throw ex;
} finally {
try {
pst.close();
} catch (Exception ex) {
log.debug(ex);
}
}
}
private List findPending(int batchSize, Connection connection, SearchIndexBuilderWorker worker)
throws SQLException {
// Pending is the first 100 items
// State == PENDING
// Action != Unknown
long start = System.currentTimeMillis();
try {
log.debug("TXFind pending with " + connection); // $NON-NLS-1$
SearchBuilderItem masterItem = getMasterItem(connection);
Integer masterAction = getMasterAction(masterItem);
log.debug(
" Master Item is "
+ masterItem.getName()
+ ":" //$NON-NLS-1$ //$NON-NLS-2$
+ masterItem.getSearchaction()
+ ":" //$NON-NLS-1$
+ masterItem.getSearchstate()
+ "::" //$NON-NLS-1$
+ masterItem.getVersion());
if (SearchBuilderItem.ACTION_REFRESH.equals(masterAction)) {
log.debug(" Master Action is " + masterAction); // $NON-NLS-1$
log.debug(" REFRESH = " + SearchBuilderItem.ACTION_REFRESH); // $NON-NLS-1$
log.debug(" RELOAD = " + SearchBuilderItem.ACTION_REBUILD); // $NON-NLS-1$
// get a complete list of all items, before the master
// action version
// if there are none, update the master action action to
// completed
// and return a blank list
refreshIndex(connection, masterItem);
} else if (SearchBuilderItem.ACTION_REBUILD.equals(masterAction)) {
rebuildIndex(connection, masterItem, worker);
} else {
// get all site masters and perform the required action.
List siteMasters = getSiteMasterItems(connection);
for (Iterator i = siteMasters.iterator(); i.hasNext(); ) {
SearchBuilderItem siteMaster = (SearchBuilderItem) i.next();
Integer action = getSiteMasterAction(siteMaster);
if (SearchBuilderItem.ACTION_REBUILD.equals(action)) {
rebuildIndex(connection, siteMaster, worker);
} else if (SearchBuilderItem.ACTION_REFRESH.equals(action)) {
refreshIndex(connection, siteMaster);
}
}
}
PreparedStatement pst = null;
PreparedStatement lockedPst = null;
ResultSet rst = null;
try {
pst =
connection.prepareStatement(
"select " //$NON-NLS-1$
+ SEARCH_BUILDER_ITEM_FIELDS
+ " from " //$NON-NLS-1$
+ SEARCH_BUILDER_ITEM_T
+ " where searchstate = ? and " //$NON-NLS-1$
+ " itemscope = ? order by version "); //$NON-NLS-1$
lockedPst =
connection.prepareStatement(
"update " //$NON-NLS-1$
+ SEARCH_BUILDER_ITEM_T
+ " set searchstate = ? " //$NON-NLS-1$
+ " where id = ? and searchstate = ? "); //$NON-NLS-1$
pst.clearParameters();
pst.setInt(1, SearchBuilderItem.STATE_PENDING.intValue());
pst.setInt(2, SearchBuilderItem.ITEM.intValue());
rst = pst.executeQuery();
ArrayList<SearchBuilderItemImpl> a = new ArrayList<SearchBuilderItemImpl>();
while (rst.next() && a.size() < batchSize) {
SearchBuilderItemImpl sbi = new SearchBuilderItemImpl();
populateSearchBuilderItem(rst, sbi);
if (!SearchBuilderItem.ACTION_UNKNOWN.equals(sbi.getSearchaction())) {
lockedPst.clearParameters();
lockedPst.setInt(1, SearchBuilderItem.STATE_LOCKED.intValue());
lockedPst.setString(2, sbi.getId());
lockedPst.setInt(3, SearchBuilderItem.STATE_PENDING.intValue());
if (lockedPst.executeUpdate() == 1) {
sbi.setSearchstate(SearchBuilderItem.STATE_LOCKED);
a.add(sbi);
}
connection.commit();
}
}
return a;
} finally {
try {
rst.close();
} catch (Exception ex) {
log.debug(ex);
}
try {
pst.close();
} catch (Exception ex) {
log.debug(ex);
}
}
} finally {
long finish = System.currentTimeMillis();
log.debug(" findPending took " + (finish - start) + " ms"); // $NON-NLS-1$ //$NON-NLS-2$
}
}
private void processAdd(
SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
throws Exception {
IndexWriter indexWrite = null;
try {
if (worker.isRunning()) {
indexWrite = indexStorage.getIndexWriter(false);
}
long last = System.currentTimeMillis();
for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
worker.isRunning() && tditer.hasNext(); ) {
Reader contentReader = null;
try {
SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
// only add adds, that have been deleted or are locked
// sucessfully
if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate())
&& !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
continue;
}
// Reference ref =
// entityManager.newReference(sbi.getName());
String ref = sbi.getName();
if (ref == null) {
log.error(
"Unrecognised trigger object presented to index builder " //$NON-NLS-1$
+ sbi);
}
long startDocIndex = System.currentTimeMillis();
worker.setStartDocIndex(startDocIndex);
worker.setNowIndexing(ref);
try {
try {
// Entity entity = ref.getEntity();
EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref);
boolean indexDoc = true;
if (searchIndexBuilder.isOnlyIndexSearchToolSites()) {
try {
String siteId = sep.getSiteId(sbi.getName());
Site s = SiteService.getSite(siteId);
ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$
if (t == null) {
indexDoc = false;
log.debug(
"Not indexing " //$NON-NLS-1$
+ sbi.getName()
+ " as it has no search tool"); //$NON-NLS-1$
}
} catch (Exception ex) {
indexDoc = false;
log.debug(
"Not indexing "
+ sbi.getName() // $NON-NLS-1$
+ " as it has no site",
ex); //$NON-NLS-1$
}
}
if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) {
DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService);
// Reader contentReader = null;
Document doc =
DocumentIndexingUtils.createIndexDocument(
ref,
digestStorageUtil,
sep,
serverConfigurationService.getServerUrl(),
contentReader);
// indexDocTMP(ref, sep);
log.debug("Indexing Document " + doc); // $NON-NLS-1$
indexWrite.addDocument(doc);
log.debug("Done Indexing Document " + doc); // $NON-NLS-1$
processRDF(ref, sep);
} else {
if (log.isDebugEnabled()) {
if (!indexDoc) {
log.debug("Ignored Document: Fileteed out by site " + ref); // $NON-NLS-1$
} else if (sep == null) {
log.debug("Ignored Document: No EntityContentProducer " + ref); // $NON-NLS-1$
} else if (!sep.isForIndex(ref)) {
log.debug("Ignored Document: Marked as Ignore " + ref); // $NON-NLS-1$
} else if (sep.getSiteId(ref) == null) {
log.debug("Ignored Document: No Site ID " + ref); // $NON-NLS-1$
} else {
log.debug("Ignored Document: Reason Unknown " + ref); // $NON-NLS-1$
}
}
}
} catch (Exception e1) {
log.debug(
" Failed to index document for "
+ ref
+ " cause: " //$NON-NLS-1$
+ e1.getMessage(),
e1);
}
sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, sbi);
} catch (Exception e1) {
log.debug(
" Failed to index document cause: " //$NON-NLS-1$
+ e1.getMessage());
}
long endDocIndex = System.currentTimeMillis();
worker.setLastIndex(endDocIndex - startDocIndex);
if ((endDocIndex - startDocIndex) > 60000L) {
log.warn(
"Slow index operation " //$NON-NLS-1$
+ String.valueOf((endDocIndex - startDocIndex) / 1000)
+ " seconds to index " //$NON-NLS-1$
+ ref);
}
// update this node lock to indicate its
// still alove, no document should
// take more than 2 mins to process
// ony do this check once every minute
long now = System.currentTimeMillis();
if ((now - last) > (60L * 1000L)) {
last = System.currentTimeMillis();
if (!worker.getLockTransaction(15L * 60L * 1000L, true)) {
throw new Exception(
"Transaction Lock Expired while indexing " //$NON-NLS-1$
+ ref);
}
}
} finally {
if (contentReader != null) {
try {
contentReader.close();
} catch (IOException ioex) {
log.debug(ioex);
}
}
}
}
worker.setStartDocIndex(System.currentTimeMillis());
worker.setNowIndexing(
Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); // $NON-NLS-1$
} catch (Exception ex) {
log.error("Failed to Add Documents ", ex);
throw new Exception(ex);
} finally {
if (indexWrite != null) {
if (log.isDebugEnabled()) {
log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
Directory d = indexWrite.getDirectory();
String[] s = d.listAll();
log.debug("Directory Contains ");
for (int i = 0; i < s.length; i++) {
File f = new File(s[i]);
log.debug(
"\t"
+ String.valueOf(f.length())
+ "\t"
+ new Date(f.lastModified())
+ "\t"
+ s[i]);
}
}
indexStorage.closeIndexWriter(indexWrite);
}
}
}
private void rebuildIndex(
Connection connection, SearchBuilderItem controlItem, SearchIndexBuilderWorker worker)
throws SQLException {
// delete all and return the master action only
// the caller will then rebuild the index from scratch
log.info(
"DELETE ALL RECORDS =========================================================="); //$NON-NLS-1$
Statement stm = null;
try {
stm = connection.createStatement();
if (SearchBuilderItem.GLOBAL_CONTEXT.equals(controlItem.getContext())) {
stm.execute(
"delete from searchbuilderitem where itemscope = "
+ SearchBuilderItem.ITEM
+ " or itemscope = "
+ SearchBuilderItem.ITEM_SITE_MASTER); // $NON-NLS-1$
} else {
stm.execute("delete from searchbuilderitem where itemscope = " + SearchBuilderItem.ITEM);
stm.execute(
"delete from searchbuilderitem where context = '" //$NON-NLS-1$
+ controlItem.getContext()
+ "' and name <> '" //$NON-NLS-1$
+ controlItem.getName()
+ "' "); //$NON-NLS-1$
}
log.debug(
"DONE DELETE ALL RECORDS ==========================================================="); //$NON-NLS-1$
connection.commit();
log.debug(
"ADD ALL RECORDS ==========================================================="); //$NON-NLS-1$
long lastupdate = System.currentTimeMillis();
List<String> contextList = new ArrayList<String>();
if (SearchBuilderItem.GLOBAL_CONTEXT.equals(controlItem.getContext())) {
for (Iterator<Site> i =
SiteService.getSites(SelectionType.ANY, null, null, null, SortType.NONE, null)
.iterator();
i.hasNext(); ) {
Site s = (Site) i.next();
if (!SiteService.isSpecialSite(s.getId())) {
if (searchIndexBuilder.isOnlyIndexSearchToolSites()) {
ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$
if (t != null) {
contextList.add(s.getId());
}
} else if (!(searchIndexBuilder.isExcludeUserSites()
&& SiteService.isUserSite(s.getId()))) {
contextList.add(s.getId());
}
}
}
} else {
contextList.add(controlItem.getContext());
}
for (Iterator<String> c = contextList.iterator(); c.hasNext(); ) {
String siteContext = (String) c.next();
log.debug("Rebuild for " + siteContext); // $NON-NLS-1$
for (Iterator<EntityContentProducer> i =
searchIndexBuilder.getContentProducers().iterator();
i.hasNext(); ) {
EntityContentProducer ecp = (EntityContentProducer) i.next();
Iterator<String> contentIterator = null;
contentIterator = ecp.getSiteContentIterator(siteContext);
log.debug("Using ECP " + ecp); // $NON-NLS-1$
int added = 0;
for (; contentIterator.hasNext(); ) {
if ((System.currentTimeMillis() - lastupdate) > 60000L) {
lastupdate = System.currentTimeMillis();
if (!worker.getLockTransaction(15L * 60L * 1000L, true)) {
throw new RuntimeException(
"Transaction Lock Expired while Rebuilding Index "); //$NON-NLS-1$
}
}
String resourceName = (String) contentIterator.next();
log.debug("Checking " + resourceName); // $NON-NLS-1$
if (resourceName == null || resourceName.length() > 255) {
log.warn(
"Entity Reference Longer than 255 characters, ignored: Reference=" //$NON-NLS-1$
+ resourceName);
continue;
}
SearchBuilderItem sbi = new SearchBuilderItemImpl();
sbi.setName(resourceName);
sbi.setSearchaction(SearchBuilderItem.ACTION_ADD);
sbi.setSearchstate(SearchBuilderItem.STATE_PENDING);
sbi.setId(UUID.randomUUID().toString());
sbi.setVersion(new Date(System.currentTimeMillis()));
sbi.setItemscope(SearchBuilderItem.ITEM);
String context = null;
try {
context = ecp.getSiteId(resourceName);
} catch (Exception ex) {
log.debug(
"No context for resource "
+ resourceName //$NON-NLS-1$
+ " defaulting to none"); //$NON-NLS-1$
}
if (context == null || context.length() == 0) {
context = "none"; // $NON-NLS-1$
}
sbi.setContext(context);
try {
updateOrSave(connection, sbi);
} catch (SQLException sqlex) {
log.error("Failed to update " + sqlex.getMessage()); // $NON-NLS-1$
}
connection.commit();
}
log.debug(" Added " + added); // $NON-NLS-1$
}
}
log.info(
"DONE ADD ALL RECORDS ==========================================================="); //$NON-NLS-1$
controlItem.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
updateOrSave(connection, controlItem);
connection.commit();
} finally {
try {
stm.close();
} catch (Exception ex) {
log.debug(ex);
}
}
}