private void mergeIndexByPart(
Path minorPath,
Path inputPath,
Path outputPath,
int partId,
int localIndexVer,
int maxVersion,
int partNo)
throws CorruptIndexException, IOException {
List<IndexReader> mergeIndexArray = new ArrayList<IndexReader>();
if (minorPath != null && PathUtil.exists(minorPath)) {
if (PathUtil.exists(minorPath.cat(partId + ""))) {
mergeIndexArray.add(
IndexReader.open(
FSDirectory.open(
minorPath
.cat(MailConstants.PART_PRE + partId)
.cat(IndexBuilder.LUCENE_INDEX_DIR)
.asFile())));
}
}
for (int i = localIndexVer + 1; i <= maxVersion; i++) {
Path segPath = inputPath.cat(i + "");
Path[] userPathes = segPath.listPathes();
for (Path userPath : userPathes) {
if (!userPath.getName().equals("built")) {
int shouldInPart = LSUtils.genPartId(userPath.getName(), partNo);
if (PathUtil.exists(segPath) && shouldInPart == partId) {
mergeIndexArray.add(
IndexReader.open(
FSDirectory.open(userPath.cat(IndexBuilder.LUCENE_INDEX_DIR).asFile())));
}
}
}
}
IndexWriter indexWriter =
new IndexWriter(
FSDirectory.open(
outputPath
.cat(MailConstants.PART_PRE + partId)
.cat(IndexBuilder.LUCENE_INDEX_DIR)
.asFile()),
new IKAnalyzer(true),
true,
IndexWriter.MaxFieldLength.LIMITED);
indexWriter.setMaxMergeDocs(1024);
indexWriter.setMergeFactor(100);
indexWriter.addIndexes(mergeIndexArray.toArray(new IndexReader[0]));
indexWriter.close();
}
public boolean merge(
Path inputPath, Path outputPath, Path localInputPath, int partNo, File segInfoFile)
throws InvalidPropertiesFormatException, IOException {
// 读取minor库中版本信息
int localIndexVer = -1;
if (localInputPath != null && PathUtil.exists(localInputPath)) {
InputStream in = null;
try {
in = new FileInputStream(segInfoFile);
} catch (FileNotFoundException e) {
LOG.error("SegInfoFile missed", e);
return false;
}
Properties prop = new Properties();
try {
prop.loadFromXML(in);
} finally {
in.close();
}
localIndexVer = Integer.parseInt(prop.getProperty(LOCAL_DIST_VER, "-1"));
}
Segments seg = new Segments(inputPath);
int maxVersion = seg.getLastSegment();
if (localIndexVer >= maxVersion) {
// do nothing
return false;
} else {
long mergeStartTime = System.currentTimeMillis();
for (int i = 0; i < partNo; i++) {
mergeIndexByPart(
localInputPath,
inputPath,
outputPath.cat(maxVersion + ""),
i,
localIndexVer,
maxVersion,
partNo);
}
long mergeEndTime = System.currentTimeMillis();
// TODO:
// XMLUtil.writeSegFile(maxVersion + "", outputPath);
PathUtil.mkdirs(outputPath.cat(maxVersion + "").cat(MailConstants.MERGED_TAG));
LOG.info("merge index use " + (mergeEndTime - mergeStartTime) / 1000 + " s.");
}
return true;
}
