/**
   * Returns KC distance between event sets es1 and es2
   *
   * @param es1 The first EventSet
   * @param es2 The second EventSet
   * @return the KC distance between them
   */
  @Override
  public double distance(EventSet es1, EventSet es2) {

    EventHistogram h1 = es1.getHistogram();
    EventHistogram h2 = es2.getHistogram();

    Set<Event> s = new HashSet<Event>();

    List<Pair<Event, Double>> l1 = new ArrayList<Pair<Event, Double>>();
    List<Pair<Event, Double>> l2 = new ArrayList<Pair<Event, Double>>();

    HashMap<Event, Integer> hm1 = new HashMap<Event, Integer>();
    HashMap<Event, Integer> hm2 = new HashMap<Event, Integer>();

    double oldfreq = Double.POSITIVE_INFINITY;

    double correlation = 0.0;

    s.addAll(es1.uniqueEvents());
    s.addAll(es2.uniqueEvents());

    // System.out.println(h1.toString());
    // System.out.println(h2.toString());

    /* make lists of the histograms */
    for (Event e : h1) {
      l1.add(new Pair<Event, Double>(e, h1.getRelativeFrequency(e), 2));
    }
    for (Event e : h2) {
      l2.add(new Pair<Event, Double>(e, h2.getRelativeFrequency(e), 2));
    }

    /* sort the list so the most frequent items are at the top */
    /* NOTE : THIS MAY BE USEFUL ELSEWHERE : SAVE THIS CODE */
    Collections.sort(l1);
    Collections.reverse(l1);
    Collections.sort(l2);
    Collections.reverse(l2);

    /* DEBUGGING STUFF
    for (Pair <Event,Double> p : l1) {
    	System.out.println("L1: " + p.toString());
    }
    for (Pair <Event,Double> p : l1) {
    	System.out.println("L2: " + p.toString());
    }
    */

    /* Convert lists into a hashmap of event:rank pairs */
    int rank = 0;
    int count = 0;
    for (Pair<Event, Double> p : l1) {
      Event e = (Event) (p.getFirst());
      double f = (Double) (p.getSecond());
      count++;
      if (f != oldfreq) {
        rank = count;
        oldfreq = f;
      }
      hm1.put(e, rank);
    }

    /* reset and do second list */
    rank = 0;
    count = 0;
    for (Pair<Event, Double> p : l2) {
      Event e = (Event) (p.getFirst());
      double f = (Double) (p.getSecond());
      count++;
      if (f != oldfreq) {
        rank = count;
        oldfreq = f;
      }
      hm2.put(e, rank);
    }

    /* More debugging stuff
    System.out.println(hm1.toString());
    System.out.println(hm2.toString());
    System.out.println(s.toString());
    */

    Integer x1, x2, y1, y2;
    Set<Event> s2 = new HashSet<Event>(s);
    for (Event e1 : s) {
      // s2.remove(e1);
      for (Event e2 : s2) {

        if (e1.equals(e2)) continue;

        /* get ranks of events e1 and e2 in both x and y distributions */
        x1 = hm1.get(e1);
        /* if not present, rank is size + 1 */
        if (x1 == null) x1 = hm1.size() + 1;

        x2 = hm2.get(e1);
        if (x2 == null) x2 = hm2.size() + 1;

        y1 = hm1.get(e2);
        /* if not present, rank is size + 1 */
        // broke because if (y1 == null) x1 = hm1.size()+1; x1 should be y1
        if (y1 == null) y1 = hm1.size() + 1;

        y2 = hm2.get(e2);
        if (y2 == null) y2 = hm2.size() + 1;

        /* more debugging stuff
        System.out.println(e1.toString() + " is ("+x1+","+x2+")");
        System.out.println(e2.toString() + " is ("+y1+","+y2+")");
        System.out.println(sgn(x1.compareTo(y1)) + " " +
        		   sgn(x2.compareTo(y2)) );
        System.out.println("");
        */

        correlation += (sgn(x1.compareTo(y1)) * sgn(x2.compareTo(y2)));
        //				System.out.println(correlation);
      }
    }

    // System.out.println(correlation);
    correlation /= (hm1.size() * (hm2.size() - 1));
    // System.out.println(correlation);
    // System.out.println("---");

    return 1.0 - correlation;
  }
  /**
   * Converts the extracted document information into a JStylo DataMap
   *
   * @param features
   * @param relevantEvents
   * @param cumulativeFeatureDriver
   * @param documentData
   * @return
   */
  public ConcurrentHashMap<Integer, FeatureData> createDocMap(
      List<String> features,
      List<EventSet> relevantEvents,
      CumulativeFeatureDriver cumulativeFeatureDriver,
      List<EventSet> documentData) {

    // generate training instances
    ConcurrentHashMap<Integer, FeatureData> documentMap =
        new ConcurrentHashMap<Integer, FeatureData>();

    // remove metadata event
    EventSet metadata = documentData.remove(documentData.size() - 1);

    // go through all eventSets in the document
    for (EventSet es : documentData) {

      // initialize relevant information
      ArrayList<Integer> indices = new ArrayList<Integer>();
      ArrayList<Event> events = new ArrayList<Event>();
      EventHistogram currHistogram = new EventHistogram();

      // whether or not we actually need this eventSet
      boolean eventSetIsRelevant = false;

      // find out if it is a histogram or not
      if (cumulativeFeatureDriver.featureDriverAt(documentData.indexOf(es)).isCalcHist()) {

        // find the event set in the list of relevant events
        for (EventSet res : relevantEvents) {
          if (es.getEventSetID().equals(res.getEventSetID())) {
            eventSetIsRelevant = true;
            break;
          }
        }

        // if it is relevant
        if (eventSetIsRelevant) {

          // find the indices of the events
          // and count all of the events
          for (Event e : es) {
            int currIndex = 0;
            boolean hasInner = false;

            // for the events n the set
            for (EventSet res : relevantEvents) {
              boolean found = false;
              for (Event re : res) {
                hasInner = true;

                // if they are the same event
                if (e.getEvent().equals(re.getEvent())) {
                  boolean inList = false;
                  for (Event el : events) {
                    if (el.getEvent().equals(e.getEvent())) {
                      inList = true;
                      break;
                    }
                  }

                  if (!inList) {
                    indices.add(currIndex);
                    events.add(e);
                  }
                  // Old location revert if change breaks
                  currHistogram.add(e);
                  found = true;
                }
                if (found) {
                  break;
                }
                currIndex++;
              }
              if (found) {
                break;
              }

              // if there's no inner, it was a non-hist feature.
              // increment by one
              if (!hasInner) {
                currIndex++;
              }
            }
          }
          // calculate/add the histograms
          int index = 0;
          for (Integer i : indices) {
            documentMap.put(
                i,
                new FeatureData(
                    cumulativeFeatureDriver.featureDriverAt(documentData.indexOf(es)).getName(),
                    cumulativeFeatureDriver
                        .featureDriverAt(documentData.indexOf(es))
                        .getNormBaseline()
                        .getTitle(),
                    currHistogram.getAbsoluteFrequency(events.get(index))));
            index++;
          }
        }
      } else { // non histogram feature

        // initialize the index
        int nonHistIndex = 0;

        // find the indices of the events
        // and count all of the events
        for (EventSet res : relevantEvents) {

          if (es.getEventSetID().equals(res.getEventSetID())) {
            break;
          }

          // count to find the index
          boolean hasInner = false;
          for (@SuppressWarnings("unused") Event re : res) {
            hasInner = true;
            nonHistIndex++;
          }

          // if ther's no inner feature, increment by one; we just passed a non-histogram
          if (!hasInner) nonHistIndex++;
        }

        // Extract and add the event
        String eventString = es.eventAt(0).getEvent();
        int startIndex = eventString.indexOf("{");
        int endIndex = eventString.indexOf("}");
        eventString = eventString.substring(startIndex + 1, endIndex);

        FeatureData fd =
            new FeatureData(
                cumulativeFeatureDriver.featureDriverAt(documentData.indexOf(es)).getName(),
                cumulativeFeatureDriver
                    .featureDriverAt(documentData.indexOf(es))
                    .getNormBaseline()
                    .getTitle(),
                Math.round((float) Double.parseDouble(eventString)));
        documentMap.put(nonHistIndex, fd);
      }
    }
    // add metadata back. Not sure if necessary
    documentData.add(metadata);

    return documentMap;
  }