/**
  * @param statType
  * @param statisticsStorage
  * @return Multiset<Integer> containing experiment counts across all efo attributes
  */
 public static Multiset<Integer> getScoresAcrossAllEfos(
     final StatisticsType statType, final StatisticsStorage statisticsStorage) {
   List<Attribute> efoAttrs = new ArrayList<Attribute>();
   for (String efo : statisticsStorage.getEfos()) {
     efoAttrs.add(new EfoAttribute(efo));
   }
   StatisticsQueryCondition statsQuery = new StatisticsQueryCondition(statType);
   statsQuery.and(getStatisticsOrQuery(efoAttrs, statType, 1, statisticsStorage));
   return getExperimentCounts(statsQuery, statisticsStorage, null);
 }
 /**
  * If no experiments were specified, inject into statisticsQuery a superset of all experiments for
  * which stats exist across all attributes
  *
  * @param statisticsQuery
  * @param statisticsStorage
  */
 private static void setQueryExperiments(
     StatisticsQueryCondition statisticsQuery, StatisticsStorage statisticsStorage) {
   Set<ExperimentInfo> exps = statisticsQuery.getExperiments();
   if (exps
       .isEmpty()) { // No experiments conditions were specified - assemble a superset of all
                     // experiments for which stats exist across all attributes
     for (EfAttribute attr : statisticsQuery.getAttributes()) {
       Map<ExperimentInfo, ConciseSet> expsToStats =
           getStatisticsForAttribute(statisticsQuery.getStatisticsType(), attr, statisticsStorage);
       if (expsToStats != null) exps.addAll(expsToStats.keySet());
     }
     statisticsQuery.inExperiments(exps);
   }
 }
  /**
   * @param orConditions StatisticsQueryOrConditions<StatisticsQueryCondition>
   * @param statisticsStorage
   * @param scoringExps Set of experiments that have at least one non-zero score for
   *     statisticsQuery. This is used retrieving efos to be displayed in heatmap when no query efvs
   *     exist (c.f. atlasStatisticsQueryService.getScoringAttributesForGenes())
   * @return Multiset<Integer> containing experiment counts corresponding to all attributes in each
   *     StatisticsQueryCondition in orConditions
   */
  private static Multiset<Integer> getScoresForOrConditions(
      final StatisticsQueryOrConditions<StatisticsQueryCondition> orConditions,
      StatisticsStorage statisticsStorage,
      Set<ExperimentInfo> scoringExps) {

    Multiset<Integer> scores = HashMultiset.create();
    for (StatisticsQueryCondition orCondition : orConditions.getConditions()) {
      orCondition.setBioEntityIdRestrictionSet(orConditions.getBioEntityIdRestrictionSet());
      scores.addAll(scoreQuery(orCondition, statisticsStorage, scoringExps));
    }

    // Now apply orConditions' min experiments restriction to scores
    Multiset<Integer> qualifyingScores = HashMultiset.create();
    for (Multiset.Entry<Integer> entry : scores.entrySet()) {
      if (entry.getCount() >= orConditions.getMinExperiments()) {
        qualifyingScores.setCount(entry.getElement(), entry.getCount());
      }
    }

    return qualifyingScores;
  }
  /**
   * @param orAttributes
   * @param statType
   * @param minExperiments minimum experiment count restriction for this clause
   * @param statisticsStorage - used to retrieve orAttributes, needed finding experiment counts in
   *     bit index
   * @return StatisticsQueryOrConditions representing orAttributes
   */
  public static StatisticsQueryOrConditions<StatisticsQueryCondition> getStatisticsOrQuery(
      final List<Attribute> orAttributes,
      final StatisticsType statType,
      int minExperiments,
      final StatisticsStorage statisticsStorage) {

    StatisticsQueryOrConditions<StatisticsQueryCondition> orConditions =
        new StatisticsQueryOrConditions<StatisticsQueryCondition>();

    orConditions.setMinExperiments(minExperiments);

    // LinkedHashMap used to maintain ordering of processing of experiments in multi-Attribute,
    // multi-Experiment bit index queries to
    // retrieve sorted lists of experiments to be plotted on the gene page.
    Map<ExperimentInfo, Set<EfAttribute>> allExpsToAttrs =
        new LinkedHashMap<ExperimentInfo, Set<EfAttribute>>();

    for (Attribute attr : orAttributes) {
      attr.getAttributeToExperimentMappings(statisticsStorage, allExpsToAttrs);
    }

    // Now process allExpsToAttrs - for all efo terms in orAttributes, grouping into one
    // StatisticsQueryCondition
    // attributes from potentially different efoTerms for one experiment. This has the effect of
    // counting a given
    // experiment only once for an OR collection of Attributes.
    for (Map.Entry<ExperimentInfo, Set<EfAttribute>> expToAttr : allExpsToAttrs.entrySet()) {
      StatisticsQueryCondition cond = new StatisticsQueryCondition(statType);
      if (expToAttr.getKey() != EfAttribute.ALL_EXPERIMENTS)
        // For efv Attributes we span all experiments
        cond.inExperiments(Collections.singletonList(expToAttr.getKey()));
      for (EfAttribute attr : expToAttr.getValue()) {
        cond.inAttribute(attr);
      }
      orConditions.orCondition(cond);
    }

    return orConditions;
  }
 /**
  * @param statsQuery StatisticsQueryCondition
  * @param statisticsStorage
  * @param scoringExps Set of experiments that have at least one non-zero score for
  *     statisticsQuery. This is used retrieving efos to be displayed in heatmap when no query efvs
  *     exist (c.f. atlasStatisticsQueryService.getScoringAttributesForGenes())
  * @return experiment counts corresponding for statsQuery
  */
 public static Multiset<Integer> getExperimentCounts(
     StatisticsQueryCondition statsQuery,
     StatisticsStorage statisticsStorage,
     Set<ExperimentInfo> scoringExps) {
   long start = System.currentTimeMillis();
   Multiset<Integer> counts =
       StatisticsQueryUtils.scoreQuery(statsQuery, statisticsStorage, scoringExps);
   long dur = System.currentTimeMillis() - start;
   int numOfGenesWithCounts = counts.elementSet().size();
   if (numOfGenesWithCounts > 0) {
     log.debug(
         "StatisticsQuery: "
             + statsQuery.prettyPrint()
             + " ==> result set size: "
             + numOfGenesWithCounts
             + " (duration: "
             + dur
             + " ms)");
   }
   return counts;
 }
  /**
   * The core scoring method for statistics queries
   *
   * @param statisticsQuery query to be peformed on statisticsStorage
   * @param statisticsStorage core data for Statistics qeries
   * @param scoringExps an out parameter.
   *     <p>- If null, experiment counts result of statisticsQuery should be returned. if - If
   *     non-null, it serves as a flag that an optimised statisticsQuery should be performed to just
   *     collect Experiments for which non-zero counts exist for Statistics query. A typical call
   *     scenario in this case is just one efv per statisticsQuery, in which we can both: 1. check
   *     if the efv Attribute itself is a scoring one 2. map this Attribute and Experimeants in
   *     scoringExps to efo terms - via the reverse mapping efv-experiment-> efo term in EfoIndex
   *     (c.f. atlasStatisticsQueryService.getScoringAttributesForGenes())
   * @return Multiset of aggregated experiment counts, where the set of scores genes is intersected
   *     across statisticsQuery.getConditions(), and union-ed across attributes within each
   *     condition in statisticsQuery.getConditions().
   */
  public static Multiset<Integer> scoreQuery(
      StatisticsQueryCondition statisticsQuery,
      final StatisticsStorage statisticsStorage,
      Set<ExperimentInfo> scoringExps) {

    // gatherScoringExpsOnly -> experiment counts should be calculated for statisticsQuery
    // !gatherScoringExpsOnly -> scoring experiments should be collected (into scoringExps) only
    boolean gatherScoringExpsOnly = scoringExps != null;
    Set<StatisticsQueryOrConditions<StatisticsQueryCondition>> andStatisticsQueryConditions =
        statisticsQuery.getConditions();

    Multiset<Integer> results = null;

    if (andStatisticsQueryConditions.isEmpty()) { // End of recursion
      Set<Integer> bioEntityIdRestrictionSet = statisticsQuery.getBioEntityIdRestrictionSet();

      Set<EfAttribute> attributes = statisticsQuery.getAttributes();
      if (attributes.isEmpty()) {

        // No attributes were provided - we have to use pre-computed scores across all attributes
        Multiset<Integer> scoresAcrossAllEfos =
            statisticsStorage.getScoresAcrossAllEfos(statisticsQuery.getStatisticsType());
        results = intersect(scoresAcrossAllEfos, bioEntityIdRestrictionSet);
      } else {
        results = HashMultiset.create();
        setQueryExperiments(statisticsQuery, statisticsStorage);

        // For each experiment in the query, traverse through all attributes and add all gene
        // indexes into one ConciseSet. This way a gene can score
        // only once for a single experiment - across all OR attributes in this query. Once all
        // attributes have been traversed for a single experiment,
        // add ConciseSet to Multiset results
        for (ExperimentInfo exp : statisticsQuery.getExperiments()) {
          FastSet statsForExperiment = new FastSet();
          for (EfAttribute attr : attributes) {
            Map<ExperimentInfo, ConciseSet> expsToStats =
                getStatisticsForAttribute(
                    statisticsQuery.getStatisticsType(), attr, statisticsStorage);
            if (expsToStats != null) {
              if (expsToStats.isEmpty()) {
                log.debug(
                    "Failed to retrieve stats for stat: "
                        + statisticsQuery.getStatisticsType()
                        + " and attr: "
                        + attr);
              } else {
                if (expsToStats.get(exp) != null) {
                  if (!gatherScoringExpsOnly) {
                    statsForExperiment.addAll(
                        intersect(expsToStats.get(exp), bioEntityIdRestrictionSet));
                  } else if (containsAtLeastOne(expsToStats.get(exp), bioEntityIdRestrictionSet)) {
                    // exp contains at least one non-zero score for at least one gene index in
                    // bioEntityIdRestrictionSet -> add it to scoringExps
                    scoringExps.add(exp);
                  }
                } else {
                  log.debug(
                      "Failed to retrieve stats for stat: "
                          + statisticsQuery.getStatisticsType()
                          + " exp: "
                          + exp.getAccession()
                          + " and attr: "
                          + attr);
                }
              }
            }
          }
          if (!gatherScoringExpsOnly) {
            results.addAll(statsForExperiment);
          }
        }
      }
    } else {
      // run over all AND conditions, do "OR" inside (cf. scoreOrStatisticsQueryConditions()) ,
      // "AND"'ing over the whole thing
      for (StatisticsQueryOrConditions<StatisticsQueryCondition> orConditions :
          andStatisticsQueryConditions) {

        // Pass gene restriction set down to orConditions
        orConditions.setGeneRestrictionSet(statisticsQuery.getBioEntityIdRestrictionSet());
        // process OR conditions
        Multiset<Integer> condGenes =
            getScoresForOrConditions(orConditions, statisticsStorage, scoringExps);

        if (results == null) results = condGenes;
        else {
          Iterator<Multiset.Entry<Integer>> resultGenes = results.entrySet().iterator();

          while (resultGenes.hasNext()) {
            Multiset.Entry<Integer> entry = resultGenes.next();
            if (!condGenes.contains(
                entry.getElement())) // AND operation between different top query conditions
            resultGenes.remove();
            else
              // for all gene ids belonging to intersection of all conditions seen so far, we
              // accumulate experiment counts
              results.setCount(
                  entry.getElement(), entry.getCount() + condGenes.count(entry.getElement()));
          }
        }
      }
    }

    if (results == null) {
      results = HashMultiset.create();
    }
    return results;
  }