/**
   * Creates the files containing all paths for all concepts up to a certain depth
   *
   * @param kb
   * @param concepts
   * @throws IOException
   */
  public static void createIndexFiles(KnowledgeBase kb, Collection<String> concepts)
      throws IOException {
    Iterator<String> conceptIterator = kb.getConceptIterator();
    String outputDir =
        KnowledgeConfiguration.getInstance().getKnowledgeBasePathIndexDir(kb) + TXT_FILE_DIR;
    if (!Files.isDirectoryEmpty(new File(outputDir))) {
      log.warn(outputDir + " IS NOT EMPTY, BAILING OUT ...");
      return;
    }

    int maxSearchDepth = getMaxSearchDepth();
    DirectoryFileManager dirManager = new DirectoryFileManager(outputDir, 1000);
    log.info("CREATING THE INDEXABLE PATH FILES FOR: " + kb.name() + " INTO " + outputDir);

    int idx = 0;
    List<String> goodConcepts = new ArrayList<String>();
    while (conceptIterator.hasNext()) {
      String concept = conceptIterator.next();
      if (concepts.isEmpty() || concepts.contains(concept)) goodConcepts.add(concept);

      idx++;
      if ((idx % 100000) == 0) log.info("ITERATED THROUGH " + idx + " CONCEPTS SO FAR ...");
    }

    List<List<String>> splitConcepts =
        Collections.split(goodConcepts, goodConcepts.size() / NSPLITS);
    int nThreads = splitConcepts.size();
    ExecutorService threadExecutor = Executors.newFixedThreadPool(nThreads);
    CountDownLatch doneSignal = new CountDownLatch(nThreads);

    for (int i = 0; i < nThreads; i++) {
      ConceptIndexer worker =
          new ConceptIndexer(splitConcepts.get(i), i, dirManager, kb, maxSearchDepth, doneSignal);
      threadExecutor.execute(worker);
    }

    Timer timer = new Timer();
    try {
      doneSignal.await();
      threadExecutor.shutdown();
    } catch (InterruptedException ie) {
      ie.printStackTrace();
    }
    timer.tick("TO CREATE INDEXABLE FILES");
  }
    public void getConcepts() throws IOException {
      int counter = 0;
      Timer timer = new Timer();
      for (String concept : concepts) {
        Set<List<String>> paths = kb.getAllPathsFrom(concept, maxSearchDepth);

        FileWriter writerFrom = dirManager.getFileWriter(concept + ".txt");

        // saves the paths starting from the concept
        for (List<String> path : paths) writerFrom.write(Strings.join(path) + "\n");

        writerFrom.close();

        if ((counter % 100) == 0)
          timer.tick(
              "\t[WORKER] "
                  + id
                  + " TO CREATE INDEXABLE FILES FOR "
                  + counter
                  + " CONCEPTS SO FAR ... ");
        counter++;
      }
    }