/**
  * Loads a similarity metric if it isn't already loaded.
  *
  * @param name
  * @return
  * @throws ConfigurationException
  * @throws IOException
  */
 public SimilarityMetric loadMetric(String name) throws ConfigurationException, IOException {
   if (env.hasMetric(name)) {
     return env.getMetric(name);
   }
   info("loading metric " + name);
   String type = requireString(configuration.getMetric(name), "type");
   SimilarityMetric metric;
   if (type.equals("category")) {
     metric = createCategorySimilarity(name);
   } else if (type.equals("text")) {
     metric = createTextSimilarity(name);
   } else if (type.equals("esa")) {
     metric = createEsaSimilarity(name);
   } else if (type.equals("links")) {
     metric = createLinkSimilarity(name);
   } else if (type.equals("pairwise")) {
     metric = createPairwiseSimilarity(name);
   } else if (type.equals("ensemble")) {
     metric = loadEnsembleMetric(name);
   } else {
     throw new ConfigurationException("Unknown metric type: " + type);
   }
   metric.setName(name);
   JSONObject params = configuration.getMetric(name);
   if (params.containsKey("normalizer")) {
     Normalizer norm = parseNormalizer(requireString(params, "normalizer"));
     if (!norm.equals(null)) {
       metric.setNormalizer(norm);
     }
   }
   env.addMetric(name, metric);
   return metric;
 }
 /**
  * Loads an index if it is not already loaded.
  *
  * @param name - The name of the
  * @return
  * @throws IOException
  * @throws ConfigurationException
  */
 private IndexHelper loadIndex(String name) throws IOException, ConfigurationException {
   if (env.hasIndex(name)) {
     return env.getIndex(name);
   }
   info("loading index " + name);
   JSONObject indexConfig = configuration.getIndex(name);
   File parentDir = requireDirectory(configuration.getIndexes(), "outputDir");
   IndexHelper helper = new IndexHelper(new File(parentDir, name), true);
   if (indexConfig.containsKey("similarity")) {
     String sim = requireString(indexConfig, "similarity");
     if (sim.equals("ESA")) {
       helper.getSearcher().setSimilarity(new ESASimilarity.LuceneSimilarity());
     } else {
       throw new ConfigurationException("unknown similarity type: " + sim);
     }
   }
   if (indexConfig.containsKey("analyzer")) {
     String analyzer = requireString(indexConfig, "analyzer");
     if (analyzer.equals("ESA")) {
       helper.setAnalyzer(new ESAAnalyzer());
     } else {
       throw new ConfigurationException("unknown analyzer type: " + analyzer);
     }
   }
   env.addIndex(name, helper);
   return helper;
 }
 private List<KnownSim> loadGold() throws ConfigurationException, IOException {
   JSONObject params = configuration.getGold();
   String path = requireString(params, "path");
   List<KnownSim> g = KnownSim.read(new File(path));
   env.setGold(g);
   return g;
 }
 private SimilarityMetric createLinkSimilarity(String name)
     throws ConfigurationException, IOException {
   JSONObject params = configuration.getMetric(name);
   SimilarityMetric metric;
   String field = requireString(params, "field");
   LinkSimilarity lmetric =
       new LinkSimilarity(
           loadMainMapper(),
           loadIndex(requireString(params, "lucene")),
           env.getMainIndex(),
           field);
   if (params.containsKey("similarity")) {
     String sim = requireString(params, "similarity");
     if (sim.equals("tfidf")) {
       lmetric.setSimilarity(LinkSimilarity.SimFn.TFIDF);
     } else if (sim.equals("google")) {
       lmetric.setSimilarity(LinkSimilarity.SimFn.GOOGLE);
     } else if (sim.equals("logodds")) {
       lmetric.setSimilarity(LinkSimilarity.SimFn.LOGODDS);
     } else if (sim.equals("jacard")) {
       lmetric.setSimilarity(LinkSimilarity.SimFn.JACARD);
     } else if (sim.equals("lucene")) {
       lmetric.setSimilarity(LinkSimilarity.SimFn.LUCENE);
     } else {
       throw new IllegalArgumentException("unknown similarity: " + sim);
     }
   }
   if (params.containsKey("minDocFreq")) {
     lmetric.setMinDocFreq(requireInteger(params, "minDocFreq"));
   }
   metric = lmetric;
   return metric;
 }
 private ConceptMapper getDictionaryMapper(String name)
     throws IOException, ConfigurationException {
   try {
     JSONObject params = configuration.getMapper(name);
     return new DictionaryMapper(
         requireDirectory(params, "dictionary"), loadIndex(requireString(params, "indexName")));
   } catch (DatabaseException e) {
     throw new IOException(e);
   }
 }
 private SimilarityMetric createCategorySimilarity(String name)
     throws ConfigurationException, IOException {
   JSONObject params = configuration.getMetric(name);
   SimilarityMetric metric;
   IndexHelper helper = loadIndex(requireString(params, "lucene"));
   CategoryGraph graph = new CategoryGraph(helper);
   graph.init();
   metric = new CategorySimilarity(loadMainMapper(), graph, helper);
   return metric;
 }
 private SimilarityMetric createEsaSimilarity(String name)
     throws ConfigurationException, IOException {
   JSONObject params = configuration.getMetric(name);
   ESASimilarity metric =
       new ESASimilarity(loadMainMapper(), loadIndex(requireString(params, "lucene")));
   if (params.containsKey("textLucene")) {
     metric.setTextHelper(loadIndex(requireString(params, "textLucene")));
   }
   return metric;
 }
 /**
  * Loads indexes and puts them in the environment.
  *
  * @throws ConfigurationException
  * @throws IOException
  */
 public void loadIndexes() throws ConfigurationException, IOException {
   info("loading indexes");
   Collection<String> namesToSkip = Arrays.asList("inputDir", "outputDir");
   for (String name : (Set<String>) configuration.getIndexes().keySet()) {
     if (namesToSkip.contains(name)) {
       continue;
     }
     loadIndex(name);
   }
 }
 /**
  * Loads metrics and puts them in the environment.
  *
  * @return
  * @throws IOException
  * @throws ConfigurationException
  */
 public List<SimilarityMetric> loadMetrics() throws IOException, ConfigurationException {
   info("loading metrics");
   Set<String> ensembleKeys = new HashSet<String>();
   List<SimilarityMetric> metrics = new ArrayList<SimilarityMetric>();
   for (String key : (Set<String>) configuration.getMetrics().keySet()) {
     String type = requireString(configuration.getMetric(key), "type");
     if (type.equals("ensemble")) {
       ensembleKeys.add(key);
     } else if (type.equals("pairwise") && !doPairwise) {
       // do nothing
     } else {
       metrics.add(loadMetric(key));
     }
   }
   if (doEnsembles) {
     for (String key : ensembleKeys) {
       metrics.add(loadMetric(key));
     }
   }
   return metrics;
 }
 private SimilarityMetric createPairwiseSimilarity(String name)
     throws IOException, ConfigurationException {
   JSONObject params = configuration.getMetric(name);
   PairwiseCosineSimilarity metric;
   SparseMatrix m = new SparseMatrix(requireFile(params, "matrix"));
   SparseMatrix mt = new SparseMatrix(requireFile(params, "transpose"));
   metric = new PairwiseCosineSimilarity(loadMainMapper(), loadMainIndex(), m, mt);
   if (params.containsKey("basedOn")) {
     metric.setBasedOn(loadMetric(requireString(params, "basedOn")));
   }
   if (params.containsKey("buildPhraseVectors")) {
     metric.setBuildPhraseVectors(requireBoolean(params, "buildPhraseVectors"));
   }
   return metric;
 }
 private SimilarityMetric loadEnsembleMetric(String key)
     throws IOException, ConfigurationException {
   info("loading ensemble metric " + key);
   Map<String, Object> params = (Map<String, Object>) configuration.getMetric(key);
   setDoEnsembles(false);
   List<SimilarityMetric> metrics = loadMetrics();
   EnsembleSimilarity similarity =
       new EnsembleSimilarity(new SvmEnsemble(), loadMainMapper(), env.getMainIndex());
   similarity.setComponents(metrics);
   similarity.read(requireDirectory(params, "model"));
   similarity.setName(key);
   if (params.containsKey("minComponents")) {
     similarity.setMinComponents(requireInteger(params, "minComponents"));
   }
   return similarity;
 }
 private SimilarityMetric createTextSimilarity(String name)
     throws ConfigurationException, IOException {
   JSONObject params = configuration.getMetric(name);
   SimilarityMetric metric;
   IndexHelper helper = loadIndex(requireString(params, "lucene"));
   String field = requireString(params, "field");
   metric = new TextSimilarity(loadMainMapper(), helper, field);
   if (params.containsKey("maxPercentage")) {
     ((TextSimilarity) metric).setMaxPercentage(requireInteger(params, "maxPercentage"));
   }
   if (params.containsKey("minTermFreq")) {
     ((TextSimilarity) metric).setMinTermFreq(requireInteger(params, "minTermFreq"));
   }
   if (params.containsKey("minDocFreq")) {
     ((TextSimilarity) metric).setMinDocFreq(requireInteger(params, "minDocFreq"));
   }
   if (params.containsKey("useInternalMapper")) {
     ((TextSimilarity) metric).setUseInternalMapper(requireBoolean(params, "useInternalMapper"));
   }
   return metric;
 }
 /**
  * Loads the mapper with the specified name if it is not already loaded.
  *
  * @param name
  * @return The requested mapper.
  * @throws ConfigurationException
  * @throws IOException
  */
 public synchronized ConceptMapper loadMapper(String name)
     throws ConfigurationException, IOException {
   if (env.hasMapper(name)) {
     return env.getMapper(name);
   }
   info("loading mapper " + name);
   JSONObject params = configuration.getMapper(name);
   String type = requireString(params, "type");
   ConceptMapper mapper;
   if (type.equals("dictionary")) {
     mapper = getDictionaryMapper(name);
   } else if (type.equals("lucene")) {
     mapper = getLuceneMapper(name);
   } else if (type.equals("ensemble")) {
     mapper = getEnsembleMapper(name);
   } else if (type.equals("title")) {
     mapper = getTitleMapper(name);
   } else {
     throw new ConfigurationException("unknown type for mapper " + name + ": " + type);
   }
   env.addMapper(name, mapper);
   return mapper;
 }
 /**
  * Loads mappers and puts them in the environment.
  *
  * @throws IOException
  * @throws ConfigurationException
  */
 public void loadMappers() throws IOException, ConfigurationException {
   info("loading mappers");
   for (String name : (Set<String>) configuration.getMappers().keySet()) {
     loadMapper(name);
   }
 }
 private void info(String message) {
   LOG.info("configurator for " + configuration.getPath() + ": " + message);
 }
 private ConceptMapper getTitleMapper(String name) throws IOException, ConfigurationException {
   JSONObject params = configuration.getMapper(name);
   IndexHelper helper = loadIndex(requireString(params, "indexName"));
   return new TitleMapper(helper);
 }