public void buildIndex(JSONObject indexData) {
try {
Directory dir = FSDirectory.open(new File(indexDir));
IKAnalyzer analyzer = new IKAnalyzer();
analyzer.setUseSmart(true);
IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_35, analyzer);
indexWriter = new IndexWriter(dir, iwc);
indexWriter.deleteAll();
JSONArray statusData = indexData.getJSONArray("statusData");
for (int i = 0; i < statusData.length(); i++) {
String text = statusData.getString(i);
Document doc = new Document();
doc.add(
new Field(
"text",
text,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(doc);
}
JSONArray userData = indexData.getJSONArray("userData");
for (int i = 0; i < userData.length(); i++) {
String text = userData.getString(i);
Document doc = new Document();
doc.add(
new Field(
"text",
text,
Field.Store.YES,
Field.Index.ANALYZED,
Field.TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(doc);
}
// indexWriter.commit();
System.out.println("Index is done");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (JSONException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
indexWriter.close();
} catch (CorruptIndexException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
public NewIndexer(String indexDir2) throws IOException {
// create the index
if (indexWriter2 == null) {
indexWriter2 =
new IndexWriter(
FSDirectory.open(new File("resultsList")),
new IndexWriterConfig(
Version.LUCENE_36,
new EnglishAnalyzer(Version.LUCENE_36, StandardAnalyzer.STOP_WORDS_SET)));
indexWriter2.deleteAll();
}
}
public void createSenIndex(JSONArray jsonArray, String indexPath, String stopwordsFile)
throws Exception {
if (jsonArray == null) {
System.out.println("error: jsonArray is null!\n");
return;
}
Analyzer analyzer = null;
if (stopwordsFile == null) {
analyzer = new SimpleAnalyzer();
} else {
analyzer = new StopAnalyzer(Paths.get(stopwordsFile));
}
IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
IndexWriter indexWriter = new IndexWriter(FSDirectory.open(Paths.get(indexPath)), iwc);
indexWriter.deleteAll();
TextUtil textUtil = new TextUtil();
long startTime = new Date().getTime();
System.out.println("jsonArray size: " + jsonArray.size());
long num_sentence = 0;
for (JSONObject jsonObj : (List<JSONObject>) jsonArray) {
long id = (long) jsonObj.get(idKey);
String review = (String) jsonObj.get(reviewKey);
if (review == null || review.isEmpty()) {
continue;
}
if (review.matches(".*[^\\x00-\\x7F].*")) {
continue;
}
String[] tokens = textUtil.tokenize(review);
if (tokens.length <= TERM_MIN_THRESHOLD) {
continue;
}
String[] sentences = textUtil.sentenceDetect(review);
// System.out.println(body.toLowerCase() + "\n");
num_sentence = 0;
for (int i = 0; i < sentences.length; i++) {
if (sentences[i] == null || sentences[i].isEmpty()) {
continue;
}
// System.out.println(sentences[i]);
Document doc = new Document();
Field idField = new LongField(idKey, id, Field.Store.YES);
Field numField = new LongField("num", num_sentence, Field.Store.NO);
Field contentField =
new TextField(
reviewKey,
sentences[i].replaceAll("[_'.,]", " ").replaceAll("[0-9]", ""),
Field.Store.YES);
doc.add(idField);
doc.add(numField);
doc.add(contentField);
indexWriter.addDocument(doc);
num_sentence++;
}
}
indexWriter.commit();
indexWriter.close();
long endTime = new Date().getTime();
System.out.println("\n\ncreate index time: " + (endTime - startTime) + "ms");
System.out.println("\n sentence num: " + num_sentence + "\n");
}