Ejemplo n.º 1
0
  public TopicScores getDistanceFromUniform() {
    int[] tokensPerTopic = model.tokensPerTopic;

    TopicScores scores = new TopicScores("uniform_dist", numTopics, numTopWords);
    scores.wordScoresDefined = true;

    int numTypes = alphabet.size();

    for (int topic = 0; topic < numTopics; topic++) {

      double topicScore = 0.0;
      int position = 0;
      TreeSet<IDSorter> sortedWords = topicSortedWords.get(topic);

      for (IDSorter info : sortedWords) {
        int type = info.getID();
        double count = info.getWeight();

        double score =
            (count / tokensPerTopic[topic]) * Math.log((count * numTypes) / tokensPerTopic[topic]);

        if (position < numTopWords) {
          scores.setTopicWordScore(topic, position, score);
        }

        topicScore += score;
        position++;
      }

      scores.setTopicScore(topic, topicScore);
    }

    return scores;
  }
Ejemplo n.º 2
0
  public TopicScores getCoherence() {
    TopicScores scores = new TopicScores("coherence", numTopics, numTopWords);
    scores.wordScoresDefined = true;

    for (int topic = 0; topic < numTopics; topic++) {
      int[][] matrix = topicCodocumentMatrices[topic];

      double topicScore = 0.0;

      for (int row = 0; row < numTopWords; row++) {
        double rowScore = 0.0;
        double minScore = 0.0;
        for (int col = 0; col < row; col++) {
          double score =
              Math.log((matrix[row][col] + model.beta) / (matrix[col][col] + model.beta));
          rowScore += score;
          if (score < minScore) {
            minScore = score;
          }
        }
        topicScore += rowScore;
        scores.setTopicWordScore(topic, row, minScore);
      }

      scores.setTopicScore(topic, topicScore);
    }

    return scores;
  }
Ejemplo n.º 3
0
  /** Low-quality topics often have lots of unusually short words. */
  public TopicScores getWordLengthStandardDeviation() {

    TopicScores scores = new TopicScores("word-length-sd", numTopics, numTopWords);
    scores.wordScoresDefined = true;

    // Get the mean length

    double meanLength = 0.0;
    int totalWords = 0;

    for (int topic = 0; topic < numTopics; topic++) {
      for (int position = 0; position < topicTopWords[topic].length; position++) {
        // Some topics may not have all N words
        if (topicTopWords[topic][position] == null) {
          break;
        }
        meanLength += topicTopWords[topic][position].length();
        totalWords++;
      }
    }

    meanLength /= totalWords;

    // Now calculate the standard deviation

    double lengthVariance = 0.0;

    for (int topic = 0; topic < numTopics; topic++) {
      for (int position = 0; position < topicTopWords[topic].length; position++) {
        if (topicTopWords[topic][position] == null) {
          break;
        }

        int length = topicTopWords[topic][position].length();

        lengthVariance += (length - meanLength) * (length - meanLength);
      }
    }
    lengthVariance /= (totalWords - 1);

    // Finally produce an overall topic score

    double lengthSD = Math.sqrt(lengthVariance);
    for (int topic = 0; topic < numTopics; topic++) {
      for (int position = 0; position < topicTopWords[topic].length; position++) {
        if (topicTopWords[topic][position] == null) {
          break;
        }

        int length = topicTopWords[topic][position].length();

        scores.addToTopicScore(topic, (length - meanLength) / lengthSD);
        scores.setTopicWordScore(topic, position, (length - meanLength) / lengthSD);
      }
    }

    return scores;
  }
Ejemplo n.º 4
0
  public TopicScores getRank1Percent() {
    TopicScores scores = new TopicScores("rank_1_docs", numTopics, numTopWords);

    for (int topic = 0; topic < numTopics; topic++) {
      scores.setTopicScore(topic, (double) numRank1Documents[topic] / numNonZeroDocuments[topic]);
    }

    return scores;
  }
Ejemplo n.º 5
0
  public TopicScores getTokensPerTopic(int[] tokensPerTopic) {
    TopicScores scores = new TopicScores("tokens", numTopics, numTopWords);

    for (int topic = 0; topic < numTopics; topic++) {
      scores.setTopicScore(topic, tokensPerTopic[topic]);
    }

    return scores;
  }
Ejemplo n.º 6
0
  /** Low-quality topics may have words that are also prominent in other topics. */
  public TopicScores getExclusivity() {

    int[] tokensPerTopic = model.tokensPerTopic;

    TopicScores scores = new TopicScores("exclusivity", numTopics, numTopWords);
    scores.wordScoresDefined = true;

    double sumDefaultProbs = 0.0;
    for (int topic = 0; topic < numTopics; topic++) {
      sumDefaultProbs += model.beta / (model.betaSum + tokensPerTopic[topic]);
    }

    for (int topic = 0; topic < numTopics; topic++) {

      double topicScore = 0.0;
      int position = 0;
      TreeSet<IDSorter> sortedWords = topicSortedWords.get(topic);

      for (IDSorter info : sortedWords) {
        int type = info.getID();
        double count = info.getWeight();

        double sumTypeProbs = sumDefaultProbs;
        int[] topicCounts = model.typeTopicCounts[type];

        int index = 0;
        while (index < topicCounts.length && topicCounts[index] > 0) {

          int otherTopic = topicCounts[index] & model.topicMask;
          int otherCount = topicCounts[index] >> model.topicBits;

          // We've already accounted for the smoothing parameter,
          //  now we need to add the actual count for the non-zero
          //  topics.
          sumTypeProbs += ((double) otherCount) / (model.betaSum + tokensPerTopic[otherTopic]);

          index++;
        }

        double score =
            ((model.beta + count) / (model.betaSum + tokensPerTopic[topic])) / sumTypeProbs;
        scores.setTopicWordScore(topic, position, score);
        topicScore += score;

        position++;
        if (position == numTopWords) {
          break;
        }
      }

      scores.setTopicScore(topic, topicScore / numTopWords);
    }

    return scores;
  }
Ejemplo n.º 7
0
  public TopicScores getDocumentEntropy(int[] tokensPerTopic) {
    TopicScores scores = new TopicScores("document_entropy", numTopics, numTopWords);

    for (int topic = 0; topic < numTopics; topic++) {
      scores.setTopicScore(
          topic,
          -sumCountTimesLogCount[topic] / tokensPerTopic[topic] + Math.log(tokensPerTopic[topic]));
    }

    return scores;
  }
Ejemplo n.º 8
0
  public TopicScores getTokenDocumentDiscrepancies() {
    TopicScores scores = new TopicScores("token-doc-diff", numTopics, numTopWords);
    scores.wordScoresDefined = true;

    for (int topic = 0; topic < numTopics; topic++) {
      int[][] matrix = topicCodocumentMatrices[topic];
      TreeSet<IDSorter> sortedWords = topicSortedWords.get(topic);

      double topicScore = 0.0;

      double[] wordDistribution = new double[numTopWords];
      double[] docDistribution = new double[numTopWords];

      double wordSum = 0.0;
      double docSum = 0.0;

      int position = 0;
      Iterator<IDSorter> iterator = sortedWords.iterator();
      while (iterator.hasNext() && position < numTopWords) {
        IDSorter info = iterator.next();

        wordDistribution[position] = info.getWeight();
        docDistribution[position] = matrix[position][position];

        wordSum += wordDistribution[position];
        docSum += docDistribution[position];

        position++;
      }

      for (position = 0; position < numTopWords; position++) {
        double p = wordDistribution[position] / wordSum;
        double q = docDistribution[position] / docSum;
        double meanProb = 0.5 * (p + q);

        double score = 0.0;
        if (p > 0) {
          score += 0.5 * p * Math.log(p / meanProb);
        }
        if (q > 0) {
          score += 0.5 * q * Math.log(q / meanProb);
        }

        scores.setTopicWordScore(topic, position, score);
        topicScore += score;
      }

      scores.setTopicScore(topic, topicScore);
    }

    return scores;
  }
Ejemplo n.º 9
0
  public TopicScores getDocumentPercent(int i) {
    TopicScores scores = new TopicScores("allocation_count", numTopics, numTopWords);

    if (i > numDocumentsAtProportions[0].length) {
      System.err.println(
          "Invalid proportion indices (max "
              + (numDocumentsAtProportions[0].length - 1)
              + ") : "
              + i);
      return scores;
    }

    for (int topic = 0; topic < numTopics; topic++) {
      scores.setTopicScore(
          topic, (double) numDocumentsAtProportions[topic][i] / numNonZeroDocuments[topic]);
    }

    return scores;
  }
Ejemplo n.º 10
0
  /** Low-quality topics often have lots of unusually short words. */
  public TopicScores getWordLengthScores() {

    TopicScores scores = new TopicScores("word-length", numTopics, numTopWords);
    scores.wordScoresDefined = true;

    for (int topic = 0; topic < numTopics; topic++) {
      int total = 0;
      for (int position = 0; position < topicTopWords[topic].length; position++) {
        if (topicTopWords[topic][position] == null) {
          break;
        }

        int length = topicTopWords[topic][position].length();
        total += length;

        scores.setTopicWordScore(topic, position, length);
      }
      scores.setTopicScore(topic, (double) total / topicTopWords[topic].length);
    }

    return scores;
  }
Ejemplo n.º 11
0
  /** Low-quality topics may be very similar to the global distribution. */
  public TopicScores getDistanceFromCorpus() {

    int[] tokensPerTopic = model.tokensPerTopic;

    TopicScores scores = new TopicScores("corpus_dist", numTopics, numTopWords);
    scores.wordScoresDefined = true;

    for (int topic = 0; topic < numTopics; topic++) {

      double coefficient = (double) numTokens / tokensPerTopic[topic];

      double topicScore = 0.0;
      int position = 0;
      TreeSet<IDSorter> sortedWords = topicSortedWords.get(topic);

      for (IDSorter info : sortedWords) {
        int type = info.getID();
        double count = info.getWeight();

        double score =
            (count / tokensPerTopic[topic]) * Math.log(coefficient * count / wordTypeCounts[type]);

        if (position < numTopWords) {
          // System.out.println(alphabet.lookupObject(type) + ": " + count + " * " + numTokens + " /
          // " + wordTypeCounts[type] + " * " + tokensPerTopic[topic] + " = " + (coefficient * count
          // / wordTypeCounts[type]));
          scores.setTopicWordScore(topic, position, score);
        }

        topicScore += score;

        position++;
      }

      scores.setTopicScore(topic, topicScore);
    }

    return scores;
  }
Ejemplo n.º 12
0
  public TopicScores getDocumentPercentRatio(int numeratorIndex, int denominatorIndex) {
    TopicScores scores = new TopicScores("allocation_ratio", numTopics, numTopWords);

    if (numeratorIndex > numDocumentsAtProportions[0].length
        || denominatorIndex > numDocumentsAtProportions[0].length) {
      System.err.println(
          "Invalid proportion indices (max "
              + (numDocumentsAtProportions[0].length - 1)
              + ") : "
              + numeratorIndex
              + ", "
              + denominatorIndex);
      return scores;
    }

    for (int topic = 0; topic < numTopics; topic++) {
      scores.setTopicScore(
          topic,
          (double) numDocumentsAtProportions[topic][numeratorIndex]
              / numDocumentsAtProportions[topic][denominatorIndex]);
    }

    return scores;
  }
Ejemplo n.º 13
0
  public TopicScores getEffectiveNumberOfWords() {
    int[] tokensPerTopic = model.tokensPerTopic;

    TopicScores scores = new TopicScores("eff_num_words", numTopics, numTopWords);

    int numTypes = alphabet.size();

    for (int topic = 0; topic < numTopics; topic++) {

      double sumSquaredProbabilities = 0.0;
      TreeSet<IDSorter> sortedWords = topicSortedWords.get(topic);

      for (IDSorter info : sortedWords) {
        int type = info.getID();
        double probability = info.getWeight() / tokensPerTopic[topic];

        sumSquaredProbabilities += probability * probability;
      }

      scores.setTopicScore(topic, 1.0 / sumSquaredProbabilities);
    }

    return scores;
  }