/**
   * @param worker
   * @param connection
   * @throws Exception
   */
  private void createIndex(SearchIndexBuilderWorker worker, Connection connection)
      throws Exception {
    IndexWriter indexWrite = null;
    try {
      if (worker.isRunning()) {
        indexWrite = indexStorage.getIndexWriter(false);
      }
      if (indexWrite != null) {
        Document doc = new Document();
        // The date of indexing
        String timeStamp = String.valueOf(System.currentTimeMillis());
        doc.add(
            new Field(
                SearchService.DATE_STAMP, timeStamp, Field.Store.NO, Field.Index.NOT_ANALYZED));
        doc.add(
            new Field(
                SearchService.DATE_STAMP,
                CompressionTools.compressString(timeStamp),
                Field.Store.YES));

        String ref = "---INDEX-CREATED---";
        doc.add(
            new Field(
                SearchService.FIELD_REFERENCE,
                CompressionTools.compressString(ref),
                Field.Store.YES));
        doc.add(
            new Field(
                SearchService.FIELD_REFERENCE, ref, Field.Store.NO, Field.Index.NOT_ANALYZED));

        indexWrite.addDocument(doc);
      } else {
        log.error("Couldn't get indexWriter to add document!");
      }

    } catch (Exception ex) {
      log.error("Failed to Add Documents ", ex);
      throw new Exception(ex);
    } finally {
      if (indexWrite != null) {
        if (log.isDebugEnabled()) {
          log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
          Directory d = indexWrite.getDirectory();
          String[] s = d.listAll();
          log.debug("Directory Contains ");
          for (int i = 0; i < s.length; i++) {
            File f = new File(s[i]);
            log.debug(
                "\t"
                    + String.valueOf(f.length())
                    + "\t"
                    + new Date(f.lastModified())
                    + "\t"
                    + s[i]);
          }
        }
        indexStorage.closeIndexWriter(indexWrite);
      }
    }
  }
  private int completeUpdate(
      SearchIndexBuilderWorker worker, Connection connection, List runtimeToDo) throws Exception {
    try {

      for (Iterator tditer = runtimeToDo.iterator(); worker.isRunning() && tditer.hasNext(); ) {
        SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
        if (SearchBuilderItem.STATE_COMPLETED.equals(sbi.getSearchstate())) {
          if (SearchBuilderItem.ACTION_DELETE.equals(sbi.getSearchaction())) {
            delete(connection, sbi);
            connection.commit();
          } else {
            updateOrSave(connection, sbi);
            connection.commit();
          }
        }
      }
      return runtimeToDo.size();
    } catch (Exception ex) {
      log.warn(
          "Failed to update state in database due to " //$NON-NLS-1$
              + ex.getMessage()
              + " this will be corrected on the next run of the IndexBuilder, no cause for alarm"); //$NON-NLS-1$
    }
    return 0;
  }
  private void processDeletes(
      SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
      throws SQLException, IOException {

    if (indexStorage.indexExists()) {
      IndexReader indexReader = null;
      try {
        indexReader = indexStorage.getIndexReader();

        // Open the index
        for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
            worker.isRunning() && tditer.hasNext(); ) {
          SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
          if (!SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
            // should only be getting pending
            // items
            log.warn(
                " Found Item that was not pending " //$NON-NLS-1$
                    + sbi.getName());
            continue;
          }
          if (SearchBuilderItem.ACTION_UNKNOWN.equals(sbi.getSearchaction())) {
            sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
            updateOrSave(connection, sbi);

            continue;
          }
          // remove document
          // if this is mult segment it might not work.
          try {
            indexReader.deleteDocuments(new Term(SearchService.FIELD_REFERENCE, sbi.getName()));
            if (SearchBuilderItem.ACTION_DELETE.equals(sbi.getSearchaction())) {
              sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
              updateOrSave(connection, sbi);

            } else {
              sbi.setSearchstate(SearchBuilderItem.STATE_PENDING_2);
            }

          } catch (IOException ex) {
            log.warn("Failed to delete Page ", ex); // $NON-NLS-1$
          }
        }
      } finally {
        if (indexReader != null) {
          indexStorage.closeIndexReader(indexReader);
          indexReader = null;
        }
      }
    }
  }
  /*
   * (non-Javadoc)
   *
   * @see org.sakaiproject.search.component.dao.impl.SearchIndexBuilderWorkerDao#processToDoListTransaction()
   */
  public void processToDoListTransaction(SearchIndexBuilderWorker worker, int indexBatchSize) {
    Connection connection = null;
    try {
      connection = dataSource.getConnection();
      long startTime = System.currentTimeMillis();
      int totalDocs = 0;

      // Load the list

      List runtimeToDo = findPending(indexBatchSize, connection, worker);

      totalDocs = runtimeToDo.size();

      log.debug("Processing " + totalDocs + " documents"); // $NON-NLS-1$ //$NON-NLS-2$

      if (totalDocs > 0) {
        log.debug("Preupdate Start");
        indexStorage.doPreIndexUpdate();
        log.debug("Preupdate End");

        // get lock

        // this needs to be exclusive
        log.debug("Process Deletes Start");

        processDeletes(worker, connection, runtimeToDo);
        log.debug("Process Deletes End");

        // upate and release lock
        // after a process Deletes the index needs to updated

        // can be parallel
        log.debug("Process Add Start");

        processAdd(worker, connection, runtimeToDo);
        log.debug("Process Add End");
        log.debug("Complete Update Start");

        completeUpdate(worker, connection, runtimeToDo);
        log.debug("Complete Update End");

        // get lock
        try {
          log.debug("Post update Start");
          indexStorage.doPostIndexUpdate();
          log.debug("Post update End");
        } catch (IOException e) {
          log.error("Failed to do Post Index Update", e); // $NON-NLS-1$
        }
        // release lock

      }

      if (worker.isRunning()) {
        long endTime = System.currentTimeMillis();
        float totalTime = endTime - startTime;
        float ndocs = totalDocs;
        if (totalDocs > 0) {
          float docspersec = 1000 * ndocs / totalTime;
          log.info(
              "Completed Process List of "
                  + totalDocs
                  + " at " //$NON-NLS-1$ //$NON-NLS-2$
                  + docspersec
                  + " documents/per second "); //$NON-NLS-1$
        }
      }
    } catch (Exception ex) {
      log.warn("Failed to perform index cycle " + ex.getMessage(), ex); // $NON-NLS-1$
      // rollback any uncommitted transactions
      try {
        connection.rollback();
      } catch (SQLException e) {
        log.debug(e);
      }
    } finally {
      try {
        connection.close();
      } catch (Exception ex) {
        log.debug(ex);
      }
    }
  }
  private void processAdd(
      SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
      throws Exception {
    IndexWriter indexWrite = null;
    try {
      if (worker.isRunning()) {
        indexWrite = indexStorage.getIndexWriter(false);
      }
      long last = System.currentTimeMillis();

      for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
          worker.isRunning() && tditer.hasNext(); ) {

        Reader contentReader = null;
        try {
          SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
          // only add adds, that have been deleted or are locked
          // sucessfully
          if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate())
              && !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
            continue;
          }
          // Reference ref =
          // entityManager.newReference(sbi.getName());
          String ref = sbi.getName();
          if (ref == null) {
            log.error(
                "Unrecognised trigger object presented to index builder " //$NON-NLS-1$
                    + sbi);
          }

          long startDocIndex = System.currentTimeMillis();
          worker.setStartDocIndex(startDocIndex);
          worker.setNowIndexing(ref);

          try {
            try {
              // Entity entity = ref.getEntity();
              EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref);
              boolean indexDoc = true;
              if (searchIndexBuilder.isOnlyIndexSearchToolSites()) {
                try {
                  String siteId = sep.getSiteId(sbi.getName());
                  Site s = SiteService.getSite(siteId);
                  ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$
                  if (t == null) {
                    indexDoc = false;
                    log.debug(
                        "Not indexing " //$NON-NLS-1$
                            + sbi.getName()
                            + " as it has no search tool"); //$NON-NLS-1$
                  }
                } catch (Exception ex) {
                  indexDoc = false;
                  log.debug(
                      "Not indexing  "
                          + sbi.getName() // $NON-NLS-1$
                          + " as it has no site",
                      ex); //$NON-NLS-1$
                }
              }
              if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) {

                DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService);
                // Reader contentReader = null;
                Document doc =
                    DocumentIndexingUtils.createIndexDocument(
                        ref,
                        digestStorageUtil,
                        sep,
                        serverConfigurationService.getServerUrl(),
                        contentReader);
                // indexDocTMP(ref, sep);

                log.debug("Indexing Document " + doc); // $NON-NLS-1$

                indexWrite.addDocument(doc);

                log.debug("Done Indexing Document " + doc); // $NON-NLS-1$

                processRDF(ref, sep);

              } else {
                if (log.isDebugEnabled()) {
                  if (!indexDoc) {
                    log.debug("Ignored Document: Fileteed out by site " + ref); // $NON-NLS-1$
                  } else if (sep == null) {
                    log.debug("Ignored Document: No EntityContentProducer " + ref); // $NON-NLS-1$

                  } else if (!sep.isForIndex(ref)) {
                    log.debug("Ignored Document: Marked as Ignore " + ref); // $NON-NLS-1$

                  } else if (sep.getSiteId(ref) == null) {
                    log.debug("Ignored Document: No Site ID " + ref); // $NON-NLS-1$

                  } else {
                    log.debug("Ignored Document: Reason Unknown " + ref); // $NON-NLS-1$
                  }
                }
              }
            } catch (Exception e1) {
              log.debug(
                  " Failed to index document for "
                      + ref
                      + " cause: " //$NON-NLS-1$
                      + e1.getMessage(),
                  e1);
            }
            sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
            updateOrSave(connection, sbi);

          } catch (Exception e1) {
            log.debug(
                " Failed to index document cause: " //$NON-NLS-1$
                    + e1.getMessage());
          }
          long endDocIndex = System.currentTimeMillis();
          worker.setLastIndex(endDocIndex - startDocIndex);
          if ((endDocIndex - startDocIndex) > 60000L) {
            log.warn(
                "Slow index operation " //$NON-NLS-1$
                    + String.valueOf((endDocIndex - startDocIndex) / 1000)
                    + " seconds to index " //$NON-NLS-1$
                    + ref);
          }
          // update this node lock to indicate its
          // still alove, no document should
          // take more than 2 mins to process
          // ony do this check once every minute
          long now = System.currentTimeMillis();
          if ((now - last) > (60L * 1000L)) {
            last = System.currentTimeMillis();
            if (!worker.getLockTransaction(15L * 60L * 1000L, true)) {
              throw new Exception(
                  "Transaction Lock Expired while indexing " //$NON-NLS-1$
                      + ref);
            }
          }

        } finally {
          if (contentReader != null) {
            try {
              contentReader.close();
            } catch (IOException ioex) {
              log.debug(ioex);
            }
          }
        }
      }
      worker.setStartDocIndex(System.currentTimeMillis());
      worker.setNowIndexing(
          Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); // $NON-NLS-1$
    } catch (Exception ex) {
      log.error("Failed to Add Documents ", ex);
      throw new Exception(ex);
    } finally {
      if (indexWrite != null) {
        if (log.isDebugEnabled()) {
          log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
          Directory d = indexWrite.getDirectory();
          String[] s = d.listAll();
          log.debug("Directory Contains ");
          for (int i = 0; i < s.length; i++) {
            File f = new File(s[i]);
            log.debug(
                "\t"
                    + String.valueOf(f.length())
                    + "\t"
                    + new Date(f.lastModified())
                    + "\t"
                    + s[i]);
          }
        }
        indexStorage.closeIndexWriter(indexWrite);
      }
    }
  }
  private void rebuildIndex(
      Connection connection, SearchBuilderItem controlItem, SearchIndexBuilderWorker worker)
      throws SQLException {
    // delete all and return the master action only
    // the caller will then rebuild the index from scratch
    log.info(
        "DELETE ALL RECORDS =========================================================="); //$NON-NLS-1$
    Statement stm = null;
    try {
      stm = connection.createStatement();
      if (SearchBuilderItem.GLOBAL_CONTEXT.equals(controlItem.getContext())) {
        stm.execute(
            "delete from searchbuilderitem where itemscope = "
                + SearchBuilderItem.ITEM
                + " or itemscope = "
                + SearchBuilderItem.ITEM_SITE_MASTER); // $NON-NLS-1$
      } else {
        stm.execute("delete from searchbuilderitem where itemscope = " + SearchBuilderItem.ITEM);
        stm.execute(
            "delete from searchbuilderitem where context = '" //$NON-NLS-1$
                + controlItem.getContext()
                + "' and name <> '" //$NON-NLS-1$
                + controlItem.getName()
                + "' "); //$NON-NLS-1$
      }

      log.debug(
          "DONE DELETE ALL RECORDS ==========================================================="); //$NON-NLS-1$
      connection.commit();
      log.debug(
          "ADD ALL RECORDS ==========================================================="); //$NON-NLS-1$
      long lastupdate = System.currentTimeMillis();
      List<String> contextList = new ArrayList<String>();
      if (SearchBuilderItem.GLOBAL_CONTEXT.equals(controlItem.getContext())) {

        for (Iterator<Site> i =
                SiteService.getSites(SelectionType.ANY, null, null, null, SortType.NONE, null)
                    .iterator();
            i.hasNext(); ) {
          Site s = (Site) i.next();
          if (!SiteService.isSpecialSite(s.getId())) {
            if (searchIndexBuilder.isOnlyIndexSearchToolSites()) {
              ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$
              if (t != null) {
                contextList.add(s.getId());
              }
            } else if (!(searchIndexBuilder.isExcludeUserSites()
                && SiteService.isUserSite(s.getId()))) {
              contextList.add(s.getId());
            }
          }
        }
      } else {
        contextList.add(controlItem.getContext());
      }
      for (Iterator<String> c = contextList.iterator(); c.hasNext(); ) {
        String siteContext = (String) c.next();
        log.debug("Rebuild for " + siteContext); // $NON-NLS-1$
        for (Iterator<EntityContentProducer> i =
                searchIndexBuilder.getContentProducers().iterator();
            i.hasNext(); ) {
          EntityContentProducer ecp = (EntityContentProducer) i.next();

          Iterator<String> contentIterator = null;
          contentIterator = ecp.getSiteContentIterator(siteContext);
          log.debug("Using ECP " + ecp); // $NON-NLS-1$

          int added = 0;
          for (; contentIterator.hasNext(); ) {
            if ((System.currentTimeMillis() - lastupdate) > 60000L) {
              lastupdate = System.currentTimeMillis();
              if (!worker.getLockTransaction(15L * 60L * 1000L, true)) {
                throw new RuntimeException(
                    "Transaction Lock Expired while Rebuilding Index "); //$NON-NLS-1$
              }
            }
            String resourceName = (String) contentIterator.next();
            log.debug("Checking " + resourceName); // $NON-NLS-1$
            if (resourceName == null || resourceName.length() > 255) {
              log.warn(
                  "Entity Reference Longer than 255 characters, ignored: Reference=" //$NON-NLS-1$
                      + resourceName);
              continue;
            }
            SearchBuilderItem sbi = new SearchBuilderItemImpl();
            sbi.setName(resourceName);
            sbi.setSearchaction(SearchBuilderItem.ACTION_ADD);
            sbi.setSearchstate(SearchBuilderItem.STATE_PENDING);
            sbi.setId(UUID.randomUUID().toString());
            sbi.setVersion(new Date(System.currentTimeMillis()));
            sbi.setItemscope(SearchBuilderItem.ITEM);
            String context = null;
            try {
              context = ecp.getSiteId(resourceName);
            } catch (Exception ex) {
              log.debug(
                  "No context for resource "
                      + resourceName //$NON-NLS-1$
                      + " defaulting to none"); //$NON-NLS-1$
            }
            if (context == null || context.length() == 0) {
              context = "none"; // $NON-NLS-1$
            }
            sbi.setContext(context);
            try {
              updateOrSave(connection, sbi);
            } catch (SQLException sqlex) {
              log.error("Failed to update " + sqlex.getMessage()); // $NON-NLS-1$
            }
            connection.commit();
          }
          log.debug(" Added " + added); // $NON-NLS-1$
        }
      }
      log.info(
          "DONE ADD ALL RECORDS ==========================================================="); //$NON-NLS-1$
      controlItem.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
      updateOrSave(connection, controlItem);
      connection.commit();
    } finally {
      try {
        stm.close();
      } catch (Exception ex) {
        log.debug(ex);
      }
    }
  }