Ejemplo n.º 1
0
 /** @param indexWriter */
 protected void closeIndexWrite(IndexWriter indexWriter) {
   if (indexWriter != null) {
     try {
       indexWriter.close();
       if (IndexWriter.isLocked(indexWriter.getDirectory())) {
         IndexWriter.unlock(indexWriter.getDirectory());
       }
     } catch (IOException e) {
       e.printStackTrace();
     }
   }
 }
Ejemplo n.º 2
0
 public DirectoryReader getReader(boolean applyDeletions) throws IOException {
   getReaderCalled = true;
   if (r.nextInt(20) == 2) {
     doRandomForceMerge();
   }
   if (!applyDeletions || r.nextBoolean()) {
     if (LuceneTestCase.VERBOSE) {
       System.out.println("RIW.getReader: use NRT reader");
     }
     if (r.nextInt(5) == 1) {
       w.commit();
     }
     return w.getReader(applyDeletions);
   } else {
     if (LuceneTestCase.VERBOSE) {
       System.out.println("RIW.getReader: open new reader");
     }
     w.commit();
     if (r.nextBoolean()) {
       return DirectoryReader.open(w.getDirectory());
     } else {
       return w.getReader(applyDeletions);
     }
   }
 }
  /**
   * @param worker
   * @param connection
   * @throws Exception
   */
  private void createIndex(SearchIndexBuilderWorker worker, Connection connection)
      throws Exception {
    IndexWriter indexWrite = null;
    try {
      if (worker.isRunning()) {
        indexWrite = indexStorage.getIndexWriter(false);
      }
      if (indexWrite != null) {
        Document doc = new Document();
        // The date of indexing
        String timeStamp = String.valueOf(System.currentTimeMillis());
        doc.add(
            new Field(
                SearchService.DATE_STAMP, timeStamp, Field.Store.NO, Field.Index.NOT_ANALYZED));
        doc.add(
            new Field(
                SearchService.DATE_STAMP,
                CompressionTools.compressString(timeStamp),
                Field.Store.YES));

        String ref = "---INDEX-CREATED---";
        doc.add(
            new Field(
                SearchService.FIELD_REFERENCE,
                CompressionTools.compressString(ref),
                Field.Store.YES));
        doc.add(
            new Field(
                SearchService.FIELD_REFERENCE, ref, Field.Store.NO, Field.Index.NOT_ANALYZED));

        indexWrite.addDocument(doc);
      } else {
        log.error("Couldn't get indexWriter to add document!");
      }

    } catch (Exception ex) {
      log.error("Failed to Add Documents ", ex);
      throw new Exception(ex);
    } finally {
      if (indexWrite != null) {
        if (log.isDebugEnabled()) {
          log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
          Directory d = indexWrite.getDirectory();
          String[] s = d.listAll();
          log.debug("Directory Contains ");
          for (int i = 0; i < s.length; i++) {
            File f = new File(s[i]);
            log.debug(
                "\t"
                    + String.valueOf(f.length())
                    + "\t"
                    + new Date(f.lastModified())
                    + "\t"
                    + s[i]);
          }
        }
        indexStorage.closeIndexWriter(indexWrite);
      }
    }
  }
  private void processAdd(
      SearchIndexBuilderWorker worker, Connection connection, List<SearchBuilderItem> runtimeToDo)
      throws Exception {
    IndexWriter indexWrite = null;
    try {
      if (worker.isRunning()) {
        indexWrite = indexStorage.getIndexWriter(false);
      }
      long last = System.currentTimeMillis();

      for (Iterator<SearchBuilderItem> tditer = runtimeToDo.iterator();
          worker.isRunning() && tditer.hasNext(); ) {

        Reader contentReader = null;
        try {
          SearchBuilderItem sbi = (SearchBuilderItem) tditer.next();
          // only add adds, that have been deleted or are locked
          // sucessfully
          if (!SearchBuilderItem.STATE_PENDING_2.equals(sbi.getSearchstate())
              && !SearchBuilderItem.STATE_LOCKED.equals(sbi.getSearchstate())) {
            continue;
          }
          // Reference ref =
          // entityManager.newReference(sbi.getName());
          String ref = sbi.getName();
          if (ref == null) {
            log.error(
                "Unrecognised trigger object presented to index builder " //$NON-NLS-1$
                    + sbi);
          }

          long startDocIndex = System.currentTimeMillis();
          worker.setStartDocIndex(startDocIndex);
          worker.setNowIndexing(ref);

          try {
            try {
              // Entity entity = ref.getEntity();
              EntityContentProducer sep = searchIndexBuilder.newEntityContentProducer(ref);
              boolean indexDoc = true;
              if (searchIndexBuilder.isOnlyIndexSearchToolSites()) {
                try {
                  String siteId = sep.getSiteId(sbi.getName());
                  Site s = SiteService.getSite(siteId);
                  ToolConfiguration t = s.getToolForCommonId("sakai.search"); // $NON-NLS-1$
                  if (t == null) {
                    indexDoc = false;
                    log.debug(
                        "Not indexing " //$NON-NLS-1$
                            + sbi.getName()
                            + " as it has no search tool"); //$NON-NLS-1$
                  }
                } catch (Exception ex) {
                  indexDoc = false;
                  log.debug(
                      "Not indexing  "
                          + sbi.getName() // $NON-NLS-1$
                          + " as it has no site",
                      ex); //$NON-NLS-1$
                }
              }
              if (indexDoc && sep != null && sep.isForIndex(ref) && sep.getSiteId(ref) != null) {

                DigestStorageUtil digestStorageUtil = new DigestStorageUtil(searchService);
                // Reader contentReader = null;
                Document doc =
                    DocumentIndexingUtils.createIndexDocument(
                        ref,
                        digestStorageUtil,
                        sep,
                        serverConfigurationService.getServerUrl(),
                        contentReader);
                // indexDocTMP(ref, sep);

                log.debug("Indexing Document " + doc); // $NON-NLS-1$

                indexWrite.addDocument(doc);

                log.debug("Done Indexing Document " + doc); // $NON-NLS-1$

                processRDF(ref, sep);

              } else {
                if (log.isDebugEnabled()) {
                  if (!indexDoc) {
                    log.debug("Ignored Document: Fileteed out by site " + ref); // $NON-NLS-1$
                  } else if (sep == null) {
                    log.debug("Ignored Document: No EntityContentProducer " + ref); // $NON-NLS-1$

                  } else if (!sep.isForIndex(ref)) {
                    log.debug("Ignored Document: Marked as Ignore " + ref); // $NON-NLS-1$

                  } else if (sep.getSiteId(ref) == null) {
                    log.debug("Ignored Document: No Site ID " + ref); // $NON-NLS-1$

                  } else {
                    log.debug("Ignored Document: Reason Unknown " + ref); // $NON-NLS-1$
                  }
                }
              }
            } catch (Exception e1) {
              log.debug(
                  " Failed to index document for "
                      + ref
                      + " cause: " //$NON-NLS-1$
                      + e1.getMessage(),
                  e1);
            }
            sbi.setSearchstate(SearchBuilderItem.STATE_COMPLETED);
            updateOrSave(connection, sbi);

          } catch (Exception e1) {
            log.debug(
                " Failed to index document cause: " //$NON-NLS-1$
                    + e1.getMessage());
          }
          long endDocIndex = System.currentTimeMillis();
          worker.setLastIndex(endDocIndex - startDocIndex);
          if ((endDocIndex - startDocIndex) > 60000L) {
            log.warn(
                "Slow index operation " //$NON-NLS-1$
                    + String.valueOf((endDocIndex - startDocIndex) / 1000)
                    + " seconds to index " //$NON-NLS-1$
                    + ref);
          }
          // update this node lock to indicate its
          // still alove, no document should
          // take more than 2 mins to process
          // ony do this check once every minute
          long now = System.currentTimeMillis();
          if ((now - last) > (60L * 1000L)) {
            last = System.currentTimeMillis();
            if (!worker.getLockTransaction(15L * 60L * 1000L, true)) {
              throw new Exception(
                  "Transaction Lock Expired while indexing " //$NON-NLS-1$
                      + ref);
            }
          }

        } finally {
          if (contentReader != null) {
            try {
              contentReader.close();
            } catch (IOException ioex) {
              log.debug(ioex);
            }
          }
        }
      }
      worker.setStartDocIndex(System.currentTimeMillis());
      worker.setNowIndexing(
          Messages.getString("SearchIndexBuilderWorkerDaoJdbcImpl.33")); // $NON-NLS-1$
    } catch (Exception ex) {
      log.error("Failed to Add Documents ", ex);
      throw new Exception(ex);
    } finally {
      if (indexWrite != null) {
        if (log.isDebugEnabled()) {
          log.debug("Closing Index Writer With " + indexWrite.maxDoc() + " documents");
          Directory d = indexWrite.getDirectory();
          String[] s = d.listAll();
          log.debug("Directory Contains ");
          for (int i = 0; i < s.length; i++) {
            File f = new File(s[i]);
            log.debug(
                "\t"
                    + String.valueOf(f.length())
                    + "\t"
                    + new Date(f.lastModified())
                    + "\t"
                    + s[i]);
          }
        }
        indexStorage.closeIndexWriter(indexWrite);
      }
    }
  }
Ejemplo n.º 5
0
  public void convert(Application app, File dbhome) throws Exception {
    FSDirectory indexDir = FSDirectory.getDirectory(dbhome, false);
    if (indexDir instanceof TransFSDirectory) {
      FSDirectory.setDisableLocks(true);
      TransFSDirectory d = (TransFSDirectory) indexDir;
      TransSource source = app.getTransSource();
      d.setDriverClass(source.getDriverClass());
      d.setUrl(source.getUrl());
      d.setUser(source.getUser());
      d.setPassword(source.getPassword());
    }
    File ndbhome = new File(dbhome.getParentFile(), dbhome.getName() + "_tmp");
    File olddbhome = new File(dbhome.getParentFile(), dbhome.getName() + "_old");
    FSDirectory nindexDir = FSDirectory.getDirectory(ndbhome, true);
    if (nindexDir instanceof TransFSDirectory) {
      FSDirectory.setDisableLocks(true);
      TransFSDirectory d = (TransFSDirectory) nindexDir;
      TransSource source = app.getTransSource();
      d.setDriverClass(source.getDriverClass());
      d.setUrl(source.getUrl());
      d.setUser(source.getUser());
      d.setPassword(source.getPassword());
    }

    IndexSearcher searcher = null;
    IndexWriter writer = null;
    LuceneManager lmgr = null;

    try {
      searcher = new IndexSearcher(indexDir);
      PerFieldAnalyzerWrapper a = LuceneManager.buildAnalyzer();
      writer = IndexWriterManager.getWriter(nindexDir, a, true);
      final int numDocs = searcher.getIndexReader().numDocs();

      HashSet deldocs = new HashSet();
      HashMap infos = new HashMap();
      for (int i = 0; i < numDocs; i++) {
        Document doc = searcher.doc(i);
        String delprop = doc.get(DeletedInfos.DELETED);
        String layerStr = doc.get(LuceneManager.LAYER_OF_SAVE);
        int layer = -1;
        try {
          layer = Integer.parseInt(layerStr);
        } catch (Exception ex) {
          layer = -1;
        }
        final String id =
            doc.get(LuceneManager.ID)
                + DeletedInfos.KEY_SEPERATOR
                + doc.get(LuceneManager.LAYER_OF_SAVE);
        if (delprop != null && "true".equals(delprop) /* && layer == DbKey.LIVE_LAYER*/) {
          deldocs.add(id);
        } else {
          Object v;
          if ((v = infos.get(id)) == null) {
            infos.put(id, new Integer(i));
          } else {
            final String lmod = doc.get(LuceneManager.LASTMODIFIED);
            final String lmod_prev = searcher.doc(((Integer) v).intValue()).get("_lastmodified");
            if (lmod_prev == null || (lmod != null && lmod.compareTo(lmod_prev) > 0)) {
              infos.put(id, new Integer(i));
            }
          }
        }
      }

      ArrayList listOfMaps = new ArrayList();

      for (int i = 0; i < numDocs; i++) {
        Document doc = searcher.doc(i);
        String delprop = doc.get(DeletedInfos.DELETED);
        String layerStr = doc.get(LuceneManager.LAYER_OF_SAVE);
        int layer = -1;
        try {
          layer = Integer.parseInt(layerStr);
        } catch (Exception ex) {
          layer = -1;
        }
        final String id =
            doc.get(LuceneManager.ID)
                + DeletedInfos.KEY_SEPERATOR
                + doc.get(LuceneManager.LAYER_OF_SAVE);
        if (delprop != null && "true".equals(delprop)) {
          continue;
        } else if (id != null && deldocs.contains(id) /* && layer == DbKey.LIVE_LAYER*/) {
          continue;
        }

        Integer idx = (Integer) infos.get(id);
        if (idx != null && i != idx.intValue()) {
          continue;
        }

        Document ndoc = convertDocument(doc);

        if (this.recordNodes) {
          listOfMaps.add(LuceneManager.luceneDocumentToMap(doc));
        }

        if (ndoc != null) {
          writer.addDocument(ndoc);
        }
      }

      if (this.recordNodes) {
        lmgr = new LuceneManager(this.app, false, true);
        this.allNodes = new HashMap();
        final int size = listOfMaps.size();
        for (int i = 0; i < size; i++) {
          HashMap m = (HashMap) listOfMaps.get(i);
          INode n = lmgr.mapToNode(m);
          this.allNodes.put(n.getID(), getPath(n));
          n = null;
        }
      }

    } catch (Exception ex) {
      ex.printStackTrace();
      throw new RuntimeException(ex);
    } finally {
      if (searcher != null) {
        try {
          searcher.close();
        } catch (Exception ex) {
          app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), ex);
        }
      }

      if (lmgr != null) {
        lmgr.shutdown();
        lmgr = null;
      }

      indexDir.close();
      SegmentInfos sinfos = IndexObjectsFactory.getFSSegmentInfos(indexDir);
      sinfos.clear();
      IndexObjectsFactory.removeDeletedInfos(indexDir);
    }

    Connection conn = null;
    boolean exceptionOccured = false;

    try {
      if (writer != null) {
        TransSource ts = app.getTransSource();
        conn = ts.getConnection();

        DatabaseMetaData dmd = conn.getMetaData();
        ResultSet rs = dmd.getColumns(null, null, "Lucene", "version");
        if (!rs.next()) {
          final String alterTbl = "ALTER TABLE Lucene ADD version INT NOT NULL DEFAULT 1";
          PreparedStatement pstmt = null;
          try {
            pstmt = conn.prepareStatement(alterTbl);
            pstmt.execute();
          } catch (SQLException sqle) {
            app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), sqle);
          } finally {
            if (pstmt != null) {
              pstmt.close();
              pstmt = null;
            }
          }
        }
        rs.close();
        rs = null;

        writer.close();
        writer.flushCache(); // TODO:writer.writeSegmentsFile();
        LuceneManager.commitSegments(conn, app, writer.getDirectory());
        writer.finalizeTrans();

        this.updateSQL(conn);
      }
    } catch (Exception ex) {
      ex.printStackTrace();
      exceptionOccured = true;
      throw new RuntimeException(ex);
    } finally {
      if (conn != null) {
        try {
          if (!conn.getAutoCommit()) {
            if (!exceptionOccured) {
              conn.commit();
            } else {
              conn.rollback();
            }
          }
          conn.close();
        } catch (Exception ex) {
          app.logError(ErrorReporter.errorMsg(this.getClass(), "convert"), ex);
        }
        conn = null;
      }

      nindexDir.close();
      SegmentInfos sinfos = IndexObjectsFactory.getFSSegmentInfos(nindexDir);
      sinfos.clear();
      IndexObjectsFactory.removeDeletedInfos(nindexDir);
    }

    if (!dbhome.renameTo(olddbhome)) {
      throw new Exception("Could not move the old version of the db into " + olddbhome);
    }

    if (!ndbhome.renameTo(dbhome)) {
      throw new Exception("Could not move the newer version of the db into " + dbhome);
    }

    File oldBlobDir = new File(olddbhome, "blob");
    File newBlobDir = new File(ndbhome, "blob");
    oldBlobDir.renameTo(newBlobDir);

    if (!FileUtils.deleteDir(olddbhome)) {
      throw new Exception("Could not delete the old version of the db at " + olddbhome);
    }
  }