private IndexWriter createWriter() throws IOException {
IndexWriter indexWriter = null;
try {
// release locks when started
if (IndexWriter.isLocked(store.directory())) {
logger.warn("shard is locked, releasing lock");
IndexWriter.unlock(store.directory());
}
boolean create = !IndexReader.indexExists(store.directory());
indexWriter =
new IndexWriter(
store.directory(),
analysisService.defaultIndexAnalyzer(),
create,
deletionPolicy,
IndexWriter.MaxFieldLength.UNLIMITED);
indexWriter.setMergeScheduler(mergeScheduler.newMergeScheduler());
indexWriter.setMergePolicy(mergePolicyProvider.newMergePolicy(indexWriter));
indexWriter.setSimilarity(similarityService.defaultIndexSimilarity());
indexWriter.setRAMBufferSizeMB(indexingBufferSize.mbFrac());
indexWriter.setTermIndexInterval(termIndexInterval);
} catch (IOException e) {
safeClose(indexWriter);
throw e;
}
return indexWriter;
}
@Override
public void updateIndexingBufferSize(ByteSizeValue indexingBufferSize) {
rwl.readLock().lock();
try {
// LUCENE MONITOR - If this restriction is removed from Lucene, remove it from here
if (indexingBufferSize.mbFrac() > 2048.0) {
this.indexingBufferSize = new ByteSizeValue(2048, ByteSizeUnit.MB);
} else {
this.indexingBufferSize = indexingBufferSize;
}
IndexWriter indexWriter = this.indexWriter;
if (indexWriter != null) {
indexWriter.setRAMBufferSizeMB(this.indexingBufferSize.mbFrac());
}
} finally {
rwl.readLock().unlock();
}
}
public static void main(String[] args) {
Date start = new Date();
try {
String path;
BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
System.out.println("Enter the merged index path:");
path = br.readLine();
Analyzer analyzer = new SimpleAnalyzer(Version.LUCENE_30);
File file = new File(path);
Directory directory = new SimpleFSDirectory(file);
IndexWriter iwriter =
new IndexWriter(directory, analyzer, new IndexWriter.MaxFieldLength(Integer.MAX_VALUE));
iwriter.setMergeFactor(10000);
iwriter.setMaxMergeDocs(Integer.MAX_VALUE);
iwriter.setRAMBufferSizeMB(100);
String index_path;
Directory index;
while (true) {
System.out.println("Enter the index Directory or 'exit' to quit");
index_path = br.readLine();
if (index_path.equals("exit")) break;
File folder = new File(index_path);
String filepath = "";
File[] listOfFiles = folder.listFiles();
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isDirectory()) {
filepath = index_path + listOfFiles[i].getName();
try {
index = new SimpleFSDirectory(new File(filepath));
System.out.println("Merging " + filepath + " optimize");
iwriter.addIndexes(index);
System.out.println("Merging " + filepath + " done");
} catch (Exception e) {
System.out.println("Index creation/merge failed for directiory " + filepath);
e.printStackTrace();
}
}
}
}
System.out.print("Optimizing index...");
try {
iwriter.optimize();
System.out.println("Optimzation successful ...");
} catch (Exception e) {
System.out.println("Optimzation failed ...");
e.printStackTrace();
}
try {
iwriter.close();
System.out.println("Close successful ...");
} catch (Exception e) {
System.out.println("Close failed ...");
}
System.out.println("done all merging");
Date end = new Date();
System.out.println(
"Total indexing time with optimize: " + ((end.getTime() - start.getTime()) / 1000));
} catch (IOException e) {
e.printStackTrace();
}
}