Пример #1
0
  @Override
  public void init(MemeDB memeDB) {
    super.init(memeDB);

    String path = memeDB.getProperty(STATE_PATH);
    if (path == null) {
      throw new RuntimeException(
          "You must include a " + STATE_PATH + " element in memedb.properties");
    }
    this.stateDir = new File(path);
    if (!stateDir.exists()) {
      log.info("Creating lucene directory " + path);
      stateDir.mkdirs();
    } else if (!stateDir.isDirectory()) {
      log.error("Path: {} not valid!", path);
      throw new RuntimeException("Path: " + path + " not valid!");
    }
    for (String db : memeDB.getBackend().getDatabaseNames()) {
      log.debug("Opening fulltext results for: {}", db);
      try {
        openWriterForDatabase(db);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    }
  }
Пример #2
0
  public synchronized JSONObject runQuery(
      String db, String defaultField, String queryString, Map<String, String> options) {
    log.debug("runQuery {} {} {}", db, defaultField, queryString);
    JSONObject o = new JSONObject();
    o.put("ok", false);
    JSONArray rows = new JSONArray();
    long count = 0;
    try {
      Hits hits = query(db, defaultField, queryString);
      log.debug("Query returned {} results", hits.length());
      ArrayList<JSONObject> d = queryFilter(hits, options);

      for (int i = 0; i < hits.length(); i++) {
        Document doc = hits.doc(i);
        String id = doc.get("_id");
        if (id != null) {
          String rev = doc.get("_rev");
          System.out.println("Query found doc " + id + "/" + rev);
        } else {
          log.warn("No id field for Lucene document");
        }
      }
      /*
      //			for(Object o : reader.getFieldNames(IndexReader.FieldOption.ALL)) {
      //				log.warn("FIELD: {}", o);
      //			}
      			*/
      count = d.size();
      rows = new JSONArray(d);
      o.put("ok", true);
    } catch (ParseException pe) {
      log.warn("Query parse exception {}", pe);
      o.put("error", "Query parse error");
    } catch (IOException e) {
      log.warn("Query IO error {}", e);
      o.put("error", "Internal error");
    } finally {
      o.put("rows", rows);
      o.put("total_rows", count);
      return o;
    }
  }
Пример #3
0
 public void onFulltextResult(
     memedb.document.Document doc, org.apache.lucene.document.Document luceneDoc)
     throws FulltextException {
   if (luceneDoc == null) return;
   //		log.debug("onFulltextResult : {} luceneDoc {}", doc.getDatabase(), luceneDoc.toString());
   log.debug("onFulltextResult : {}", doc.getDatabase());
   String db = doc.getDatabase();
   IndexWriter writer = writers.get(db);
   if (writer == null) {
     log.error("Missing IndexWriter for database {}", db);
     return;
   }
   try {
     writer.addDocument(luceneDoc);
     writer.flush();
   } catch (CorruptIndexException e) {
     throw new IndexCorruptException(db);
   } catch (IOException e) {
     throw new FulltextException(db);
   }
 }
Пример #4
0
 public void removeResult(String db, String id) {
   IndexWriter writer = writers.get(db);
   if (writer == null) {
     log.error("Missing IndexWriter for database {}", db);
     return;
   }
   try {
     writer.deleteDocuments(new Term("_id", id));
   } catch (CorruptIndexException e) {
   } catch (IOException e) {
   }
 }
Пример #5
0
 public void onDatabaseCreated(String db, long seq) {
   this.abort(db);
   File path = indexPath(db);
   onDatabaseDeleted(db, -1);
   if (!path.exists()) {
     log.info("Creating lucene directory {} for db {}", path.getPath(), db);
     path.mkdirs();
   }
   try {
     Directory directory = FSDirectory.getDirectory(path.getPath());
     writers.put(db, new IndexWriter(directory, true, new StandardAnalyzer(), true));
   } catch (CorruptIndexException e) {
   } catch (LockObtainFailedException e) {
   } catch (IOException e) {
   }
 }
Пример #6
0
/**
 * @todo The analyzer should be configurable for language in the memedb.properties
 * @author Russell Weir
 */
public class Lucene extends FulltextEngine {
  public static final String STATE_PATH = "fulltext.lucene.path";

  private final Logger log = Logger.get(Lucene.class);
  private HashMap<String, IndexWriter> writers = new HashMap<String, IndexWriter>();
  private File stateDir;

  @Override
  public void init(MemeDB memeDB) {
    super.init(memeDB);

    String path = memeDB.getProperty(STATE_PATH);
    if (path == null) {
      throw new RuntimeException(
          "You must include a " + STATE_PATH + " element in memedb.properties");
    }
    this.stateDir = new File(path);
    if (!stateDir.exists()) {
      log.info("Creating lucene directory " + path);
      stateDir.mkdirs();
    } else if (!stateDir.isDirectory()) {
      log.error("Path: {} not valid!", path);
      throw new RuntimeException("Path: " + path + " not valid!");
    }
    for (String db : memeDB.getBackend().getDatabaseNames()) {
      log.debug("Opening fulltext results for: {}", db);
      try {
        openWriterForDatabase(db);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    }
  }

  public void onDatabaseDeleted(String db, long seq) {
    this.abort(db);
    File path = this.indexPath(db);
    memedb.utils.FileUtils.deleteRecursive(path);
  }

  public void onDatabaseCreated(String db, long seq) {
    this.abort(db);
    File path = indexPath(db);
    onDatabaseDeleted(db, -1);
    if (!path.exists()) {
      log.info("Creating lucene directory {} for db {}", path.getPath(), db);
      path.mkdirs();
    }
    try {
      Directory directory = FSDirectory.getDirectory(path.getPath());
      writers.put(db, new IndexWriter(directory, true, new StandardAnalyzer(), true));
    } catch (CorruptIndexException e) {
    } catch (LockObtainFailedException e) {
    } catch (IOException e) {
    }
  }

  public void onDocumentDeleted(String db, String id, long seq) {
    removeResult(db, id);
  }

  public void removeResult(String db, String id) {
    IndexWriter writer = writers.get(db);
    if (writer == null) {
      log.error("Missing IndexWriter for database {}", db);
      return;
    }
    try {
      writer.deleteDocuments(new Term("_id", id));
    } catch (CorruptIndexException e) {
    } catch (IOException e) {
    }
  }

  public void onFulltextResult(
      memedb.document.Document doc, org.apache.lucene.document.Document luceneDoc)
      throws FulltextException {
    if (luceneDoc == null) return;
    //		log.debug("onFulltextResult : {} luceneDoc {}", doc.getDatabase(), luceneDoc.toString());
    log.debug("onFulltextResult : {}", doc.getDatabase());
    String db = doc.getDatabase();
    IndexWriter writer = writers.get(db);
    if (writer == null) {
      log.error("Missing IndexWriter for database {}", db);
      return;
    }
    try {
      writer.addDocument(luceneDoc);
      writer.flush();
    } catch (CorruptIndexException e) {
      throw new IndexCorruptException(db);
    } catch (IOException e) {
      throw new FulltextException(db);
    }
  }

  protected void openWriterForDatabase(String db) throws IOException, CorruptIndexException {
    File indexDir = indexPath(db);
    if (!indexDir.exists()) {
      onDatabaseCreated(db, -1);
    } else {
      Directory directory = FSDirectory.getDirectory(indexDir.getPath());
      writers.put(db, new IndexWriter(directory, true, new StandardAnalyzer(), false));
    }
  }

  protected Hits query(String db, String defaultField, String queryString)
      throws IOException, CorruptIndexException, ParseException {
    Directory directory = FSDirectory.getDirectory(indexPath(db));
    IndexReader reader = IndexReader.open(directory);

    Searcher searcher = new IndexSearcher(reader);
    Analyzer analyzer = new StandardAnalyzer();
    QueryParser qp = new QueryParser(defaultField, analyzer);
    Query query = qp.parse(queryString);
    return searcher.search(query);
  }

  protected ArrayList<JSONObject> queryFilter(Hits hits, Map<String, String> options)
      throws CorruptIndexException, IOException {
    Long key = findLongValue("key", options);
    Long startkey = findLongValue("startkey", options);
    Long endkey = findLongValue("endkey", options);
    boolean startInclusive = "false".equals(options.get("startkey_inclusive")) ? false : true;
    boolean endInclusive = "false".equals(options.get("endkey_inclusive")) ? false : true;
    boolean descending = "true".equals(options.get("descending"));
    Long skip = findLongValue("skip", options);
    Long count = findLongValue("count", options);

    ArrayList<JSONObject> rv = new ArrayList<JSONObject>();
    if (hits.length() == 0) return rv;
    int start = 0;
    int end = hits.length();

    if (key != null) {
      int v = key.intValue();
      startkey = key;
      endkey = key;
      startInclusive = true;
      endInclusive = true;
      skip = new Long(0);
      count = new Long(0);
    }
    if (startkey != null) start = startkey.intValue();
    if (endkey != null) end = endkey.intValue();
    if (!descending) {
      if (!startInclusive) start++;
      if (!endInclusive) end--;
      if (skip != null) start += skip.intValue();
      if (count != null) {
        int c = count.intValue();
        if (c >= 0) {
          if (start + c < end) end = start + c;
        } else {
          start = end + c;
        }
      }
      if (start < 0) start = 0;
      if (start >= hits.length()) return rv;
      if (end < 0) end = 0;
      if (end > hits.length()) end = hits.length();
      for (int c = start; c < end; c++) {
        Document doc = hits.doc(c);
        JSONObject o = new JSONObject();
        try {
          o.put("key", c);
          o.put("id", doc.get("_id"));
          o.put("rev", doc.get("_rev"));
          rv.add(o);
        } catch (JSONException e) {
        }
      }
    } else {
      int c = start;
      start = end;
      end = c;
      if (!startInclusive) start--;
      if (!endInclusive) end++;
      if (skip != null) start -= skip.intValue();
      if (count != null) {
        c = count.intValue();
        if (c >= 0) {
          if (start - c > end) end = start - c;
        } else {
          start = end - c;
        }
      }
      if (start >= hits.length()) start = hits.length() - 1;
      if (start < 0 || end >= start) return rv;

      if (end < -1) end = -1;
      if (end >= hits.length()) end = hits.length() - 1;
      for (c = start; c > end; c--) {
        Document doc = hits.doc(c);
        JSONObject o = new JSONObject();
        try {
          o.put("key", c);
          o.put("id", doc.get("_id"));
          o.put("rev", doc.get("_rev"));
          rv.add(o);
        } catch (JSONException e) {
        }
      }
    }
    return rv;
  }

  public synchronized JSONObject runQuery(
      String db, String defaultField, String queryString, Map<String, String> options) {
    log.debug("runQuery {} {} {}", db, defaultField, queryString);
    JSONObject o = new JSONObject();
    o.put("ok", false);
    JSONArray rows = new JSONArray();
    long count = 0;
    try {
      Hits hits = query(db, defaultField, queryString);
      log.debug("Query returned {} results", hits.length());
      ArrayList<JSONObject> d = queryFilter(hits, options);

      for (int i = 0; i < hits.length(); i++) {
        Document doc = hits.doc(i);
        String id = doc.get("_id");
        if (id != null) {
          String rev = doc.get("_rev");
          System.out.println("Query found doc " + id + "/" + rev);
        } else {
          log.warn("No id field for Lucene document");
        }
      }
      /*
      //			for(Object o : reader.getFieldNames(IndexReader.FieldOption.ALL)) {
      //				log.warn("FIELD: {}", o);
      //			}
      			*/
      count = d.size();
      rows = new JSONArray(d);
      o.put("ok", true);
    } catch (ParseException pe) {
      log.warn("Query parse exception {}", pe);
      o.put("error", "Query parse error");
    } catch (IOException e) {
      log.warn("Query IO error {}", e);
      o.put("error", "Internal error");
    } finally {
      o.put("rows", rows);
      o.put("total_rows", count);
      return o;
    }
  }

  public void shutdown() {
    for (String db : writers.keySet()) {
      IndexWriter writer = writers.get(db);
      try {
        if (writer != null) writer.close();
      } catch (IOException e) {
      }
    }
  }

  /**
   * Returns the base path for the indexing results for the database specified
   *
   * @param db Database name, unencoded
   * @return new File object
   */
  protected File indexPath(String db) {
    return new File(stateDir, memedb.utils.FileUtils.fsEncode(db));
  }

  /**
   * Forcibly shuts down readers and writers for a db and removes the results path
   *
   * @param db Database name
   */
  protected void abort(String db) {
    File path = indexPath(db);
    IndexWriter writer = writers.get(db);
    try {
      if (writer != null) writer.close();
    } catch (IOException e) {
    }
    memedb.utils.FileUtils.deleteRecursive(path);
  }

  /**
   * Gracefully shuts down readers and writers for a db
   *
   * @param db Database name
   */
  protected void close(String db) {
    IndexWriter writer = writers.get(db);
    try {
      if (writer != null) writer.close();
    } catch (IOException e) {
    }
  }

  private static Long findLongValue(String opt, Map<String, String> options) {
    if (!options.containsKey(opt)) return null;
    try {
      return new Long(options.get(opt));
    } catch (Exception e) {
    }
    return null;
  }
}