private void mergeIndexByPart(
Path minorPath,
Path inputPath,
Path outputPath,
int partId,
int localIndexVer,
int maxVersion,
int partNo)
throws CorruptIndexException, IOException {
List<IndexReader> mergeIndexArray = new ArrayList<IndexReader>();
if (minorPath != null && PathUtil.exists(minorPath)) {
if (PathUtil.exists(minorPath.cat(partId + ""))) {
mergeIndexArray.add(
IndexReader.open(
FSDirectory.open(
minorPath
.cat(MailConstants.PART_PRE + partId)
.cat(IndexBuilder.LUCENE_INDEX_DIR)
.asFile())));
}
}
for (int i = localIndexVer + 1; i <= maxVersion; i++) {
Path segPath = inputPath.cat(i + "");
Path[] userPathes = segPath.listPathes();
for (Path userPath : userPathes) {
if (!userPath.getName().equals("built")) {
int shouldInPart = LSUtils.genPartId(userPath.getName(), partNo);
if (PathUtil.exists(segPath) && shouldInPart == partId) {
mergeIndexArray.add(
IndexReader.open(
FSDirectory.open(userPath.cat(IndexBuilder.LUCENE_INDEX_DIR).asFile())));
}
}
}
}
IndexWriter indexWriter =
new IndexWriter(
FSDirectory.open(
outputPath
.cat(MailConstants.PART_PRE + partId)
.cat(IndexBuilder.LUCENE_INDEX_DIR)
.asFile()),
new IKAnalyzer(true),
true,
IndexWriter.MaxFieldLength.LIMITED);
indexWriter.setMaxMergeDocs(1024);
indexWriter.setMergeFactor(100);
indexWriter.addIndexes(mergeIndexArray.toArray(new IndexReader[0]));
indexWriter.close();
}
public static void main(String[] args) {
Date start = new Date();
try {
String path;
BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
System.out.println("Enter the merged index path:");
path = br.readLine();
Analyzer analyzer = new SimpleAnalyzer(Version.LUCENE_30);
File file = new File(path);
Directory directory = new SimpleFSDirectory(file);
IndexWriter iwriter =
new IndexWriter(directory, analyzer, new IndexWriter.MaxFieldLength(Integer.MAX_VALUE));
iwriter.setMergeFactor(10000);
iwriter.setMaxMergeDocs(Integer.MAX_VALUE);
iwriter.setRAMBufferSizeMB(100);
String index_path;
Directory index;
while (true) {
System.out.println("Enter the index Directory or 'exit' to quit");
index_path = br.readLine();
if (index_path.equals("exit")) break;
File folder = new File(index_path);
String filepath = "";
File[] listOfFiles = folder.listFiles();
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isDirectory()) {
filepath = index_path + listOfFiles[i].getName();
try {
index = new SimpleFSDirectory(new File(filepath));
System.out.println("Merging " + filepath + " optimize");
iwriter.addIndexes(index);
System.out.println("Merging " + filepath + " done");
} catch (Exception e) {
System.out.println("Index creation/merge failed for directiory " + filepath);
e.printStackTrace();
}
}
}
}
System.out.print("Optimizing index...");
try {
iwriter.optimize();
System.out.println("Optimzation successful ...");
} catch (Exception e) {
System.out.println("Optimzation failed ...");
e.printStackTrace();
}
try {
iwriter.close();
System.out.println("Close successful ...");
} catch (Exception e) {
System.out.println("Close failed ...");
}
System.out.println("done all merging");
Date end = new Date();
System.out.println(
"Total indexing time with optimize: " + ((end.getTime() - start.getTime()) / 1000));
} catch (IOException e) {
e.printStackTrace();
}
}
