/**
   * Creates the {@link Document} for {@link KnowledgeBasePathIndex}
   *
   * @param txtFileDir where the path txt files are
   * @param indexDir the directory where the index is created
   * @throws IOException
   */
  public static void createIndex(File txtFileDir, File indexDir) throws IOException {
    if (!indexDir.exists()) indexDir.mkdirs();
    else if (!Files.isDirectoryEmpty(indexDir)) {
      log.warn(indexDir + " IS NOT EMPTY, BAILING OUT ...");
      return;
    }

    IndexWriter indexWriter =
        new IndexWriter(
            new SimpleFSDirectory(indexDir),
            new StandardAnalyzer(Version.LUCENE_29),
            true,
            IndexWriter.MaxFieldLength.UNLIMITED);

    int counter = 0;
    FilenameFilter filter =
        new FilenameFilter() {
          public boolean accept(File dir, String name) {
            return name.endsWith(".txt");
          }
        };
    Collection<File> files = Files.listFiles(txtFileDir, filter, true);

    for (File file : files) {
      BufferedReader reader = new BufferedReader(new FileReader(file));

      Set<List<String>> paths = new HashSet<List<String>>();
      while (reader.ready()) {
        String line = reader.readLine();
        paths.add(Arrays.asList(line.split(" ")));
      }
      reader.close();

      List<Document> docs = getRecords(paths);
      for (Document doc : docs) indexWriter.addDocument(doc);

      if ((counter % 1000) == 0)
        log.info("\tINDEXED PATHS FOR " + counter + " CONCEPTS SO FAR ... ");
      counter++;
    }
    log.info("\tINDEXED " + counter + " CONCEPTS, DONE!");

    // closes the index writers
    indexWriter.optimize();
    indexWriter.close();
  }
  /**
   * Creates the {@link Document} for {@link KnowledgeBasePathIndex} for a bunch of input concepts
   * only
   *
   * @param kb
   * @throws IOException
   */
  public static void createIndex(KnowledgeBase kb, Collection<String> concepts) throws IOException {
    File txtFileDir =
        new File(
            KnowledgeConfiguration.getInstance().getKnowledgeBasePathIndexDir(kb) + TXT_FILE_DIR);
    // init the index dirs and the index writer
    File indexDir =
        new File(KnowledgeConfiguration.getInstance().getKnowledgeBasePathIndexDir(kb) + INDEX_DIR);

    if (!indexDir.exists()) indexDir.mkdirs();
    else if (!Files.isDirectoryEmpty(indexDir)) {
      log.warn(indexDir + " IS NOT EMPTY, BAILING OUT ...");
      return;
    }

    IndexWriter indexWriter =
        new IndexWriter(
            new SimpleFSDirectory(indexDir),
            new StandardAnalyzer(Version.LUCENE_29),
            true,
            IndexWriter.MaxFieldLength.UNLIMITED);

    int counter = 0;
    FilenameFilter filter =
        new FilenameFilter() {
          public boolean accept(File dir, String name) {
            return name.endsWith(".txt");
          }
        };
    Collection<File> files = Files.listFiles(txtFileDir, filter, true);

    // the intermediate concepts @depth 2 from the starting nodes
    Set<String> intermediate = new HashSet<String>();

    // PHASE 1. index all paths starting with concepts of interest
    for (File file : files) {
      String fileName = file.getName();
      String concept = fileName.substring(0, fileName.indexOf("."));
      if (!concepts.contains(concept)) continue;

      BufferedReader reader = new BufferedReader(new FileReader(file));

      Set<List<String>> paths = new HashSet<List<String>>();
      while (reader.ready()) {
        String line = reader.readLine();
        String[] path = line.split(" ");
        paths.add(Arrays.asList(path));
        intermediate.add(path[path.length - 1]);
      }
      reader.close();

      List<Document> docs = getRecords(paths);
      for (Document doc : docs) indexWriter.addDocument(doc);

      if ((counter % 1000) == 0)
        log.info("\tINDEXED STARTING PATHS FOR " + counter + " CONCEPTS SO FAR ... ");
      counter++;
    }
    log.info("\tINDEXED " + counter + " STARTING CONCEPTS, DONE!");

    log.info("NOW INDEXING ENDING PATHS FOR " + intermediate.size() + " INTERMEDIATE CONCEPTS");

    // PHASE 2. index all paths ending with concepts of interest
    counter = 0;
    for (File file : files) {
      String fileName = file.getName();
      String concept = fileName.substring(0, fileName.indexOf("."));
      if (concepts.contains(concept)) continue; // already seen
      if (!intermediate.contains(concept)) continue; // interested only
      // in intermediate

      BufferedReader reader = new BufferedReader(new FileReader(file));

      Set<List<String>> paths = new HashSet<List<String>>();
      while (reader.ready()) {
        String line = reader.readLine();
        String[] path = line.split(" ");
        if (concepts.contains(path[path.length - 1])) paths.add(Arrays.asList(path));
      }
      reader.close();

      List<Document> docs = getRecords(paths);
      for (Document doc : docs) indexWriter.addDocument(doc);

      if ((counter % 1000) == 0)
        log.info("\tINDEXED ENDING PATHS FOR " + counter + " CONCEPTS SO FAR ... ");
      counter++;
    }
    log.info("\tINDEXED " + counter + " ENDING CONCEPTS, DONE!");

    // closes the index writers
    indexWriter.optimize();
    indexWriter.close();
  }