Ejemplo n.º 1
0
  private void mergeIndexByPart(
      Path minorPath,
      Path inputPath,
      Path outputPath,
      int partId,
      int localIndexVer,
      int maxVersion,
      int partNo)
      throws CorruptIndexException, IOException {
    List<IndexReader> mergeIndexArray = new ArrayList<IndexReader>();
    if (minorPath != null && PathUtil.exists(minorPath)) {
      if (PathUtil.exists(minorPath.cat(partId + ""))) {
        mergeIndexArray.add(
            IndexReader.open(
                FSDirectory.open(
                    minorPath
                        .cat(MailConstants.PART_PRE + partId)
                        .cat(IndexBuilder.LUCENE_INDEX_DIR)
                        .asFile())));
      }
    }

    for (int i = localIndexVer + 1; i <= maxVersion; i++) {
      Path segPath = inputPath.cat(i + "");
      Path[] userPathes = segPath.listPathes();
      for (Path userPath : userPathes) {
        if (!userPath.getName().equals("built")) {
          int shouldInPart = LSUtils.genPartId(userPath.getName(), partNo);
          if (PathUtil.exists(segPath) && shouldInPart == partId) {
            mergeIndexArray.add(
                IndexReader.open(
                    FSDirectory.open(userPath.cat(IndexBuilder.LUCENE_INDEX_DIR).asFile())));
          }
        }
      }
    }
    IndexWriter indexWriter =
        new IndexWriter(
            FSDirectory.open(
                outputPath
                    .cat(MailConstants.PART_PRE + partId)
                    .cat(IndexBuilder.LUCENE_INDEX_DIR)
                    .asFile()),
            new IKAnalyzer(true),
            true,
            IndexWriter.MaxFieldLength.LIMITED);
    indexWriter.setMaxMergeDocs(1024);
    indexWriter.setMergeFactor(100);
    indexWriter.addIndexes(mergeIndexArray.toArray(new IndexReader[0]));
    indexWriter.close();
  }
Ejemplo n.º 2
0
  public static void main(String[] args) {
    Date start = new Date();
    try {

      String path;

      BufferedReader br = new BufferedReader(new InputStreamReader(System.in));
      System.out.println("Enter the merged index path:");
      path = br.readLine();

      Analyzer analyzer = new SimpleAnalyzer(Version.LUCENE_30);
      File file = new File(path);

      Directory directory = new SimpleFSDirectory(file);

      IndexWriter iwriter =
          new IndexWriter(directory, analyzer, new IndexWriter.MaxFieldLength(Integer.MAX_VALUE));
      iwriter.setMergeFactor(10000);
      iwriter.setMaxMergeDocs(Integer.MAX_VALUE);
      iwriter.setRAMBufferSizeMB(100);

      String index_path;
      Directory index;

      while (true) {
        System.out.println("Enter the index Directory or 'exit' to quit");
        index_path = br.readLine();
        if (index_path.equals("exit")) break;

        File folder = new File(index_path);
        String filepath = "";
        File[] listOfFiles = folder.listFiles();
        for (int i = 0; i < listOfFiles.length; i++) {
          if (listOfFiles[i].isDirectory()) {
            filepath = index_path + listOfFiles[i].getName();
            try {
              index = new SimpleFSDirectory(new File(filepath));
              System.out.println("Merging " + filepath + " optimize");
              iwriter.addIndexes(index);
              System.out.println("Merging " + filepath + " done");
            } catch (Exception e) {
              System.out.println("Index creation/merge failed for directiory " + filepath);
              e.printStackTrace();
            }
          }
        }
      }

      System.out.print("Optimizing index...");
      try {
        iwriter.optimize();
        System.out.println("Optimzation successful ...");
      } catch (Exception e) {
        System.out.println("Optimzation failed ...");
        e.printStackTrace();
      }
      try {
        iwriter.close();
        System.out.println("Close successful ...");
      } catch (Exception e) {
        System.out.println("Close failed ...");
      }
      System.out.println("done all merging");

      Date end = new Date();
      System.out.println(
          "Total indexing time with optimize: " + ((end.getTime() - start.getTime()) / 1000));

    } catch (IOException e) {
      e.printStackTrace();
    }
  }