Example #1
0
    /** {@inheritDoc} */
    public void processSpace(Properties properties) {
      SparseMatrix cleanedMatrix = (SparseMatrix) transform.transform(cooccurrenceMatrix);
      for (String term : basis.keySet()) {
        int index = basis.getDimension(term);
        SparseDoubleVector sdv = cleanedMatrix.getRowVector(index);

        double score = 0;
        for (int i : sdv.getNonZeroIndices()) score += sdv.get(i);

        wordScores.put(term, score);
      }
    }
Example #2
0
  @Override
  public ScoreResult score(SparseMatrix m) {
    ScoreResult hitsResult = hits.score(m);
    double[] colScore = hitsResult.getColScore();
    SingularValueDecompositionLibJ svd = new SingularValueDecompositionLibJ();
    svd.factorize(m, Math.min(20, Math.min(m.rows(), m.columns()) * 7 / 11));

    Matrix u = svd.dataClasses();
    Matrix v = new TransposedMatrix(svd.classFeatures());

    if (hasNaN(u) || hasNaN(v)) return hitsResult;

    double[] svs = svd.singularValues();

    PriorityQueue<Double, Integer> cos2dim =
        PriorityQueue.make(
            Math.min(5, svs.length),
            SortUtils.reverse(Double.class),
            SortUtils.comp(Integer.class));
    for (int j = 0; j < v.columns(); j++) {
      double cos = MatrixUtils.cosine(colScore, v.getColumn(j));
      cos2dim.add(cos, j);
    }

    List<Integer> dims = cos2dim.values();

    double[] hub = new double[u.rows()];
    for (int k = 0; k < u.rows(); k++) {
      double sum = 0;
      for (int i : dims) sum += u.get(k, i) * u.get(k, i) * svs[i] * svs[i];
      hub[k] = Math.sqrt(sum);
    }
    MatrixUtils.normalize(hub);

    double[] authority = new double[v.rows()];
    for (int k = 0; k < v.rows(); k++) {
      double sum = 0;
      for (int i : dims) sum += v.get(k, i) * v.get(k, i) * svs[i] * svs[i];
      authority[k] = Math.sqrt(sum);
    }
    MatrixUtils.normalize(authority);

    return new ScoreResult(hub, authority);
  }
Example #3
0
 /**
  * Returns an array containing the row indices of the neighbors of the impost node and the row
  * index of the impost node itself.
  */
 private static int[] getImpostNeighbors(SparseMatrix sm, int rowIndex) {
   int[] impost1edges = sm.getRowVector(rowIndex).getNonZeroIndices();
   int[] neighbors = Arrays.copyOf(impost1edges, impost1edges.length + 1);
   neighbors[neighbors.length - 1] = rowIndex;
   return neighbors;
 }
Example #4
0
  /**
   * {@inheritDoc}
   *
   * @throws IllegalArgumentException if {@code matrix} is not square, or is not an instance of
   *     {@link SparseMatrix}
   */
  public Assignments cluster(Matrix matrix, Properties props) {
    if (matrix.rows() != matrix.columns())
      throw new IllegalArgumentException(
          "Input matrix is not square. "
              + "Matrix is expected to be a square matrix whose values (i,j) "
              + "denote an edge from row i to row j");
    if (!(matrix instanceof SparseMatrix)) {
      throw new IllegalArgumentException("Input matrix must be a " + "sparse matrix.");
    }
    SparseMatrix sm = (SparseMatrix) matrix;

    String inMemProp = props.getProperty(KEEP_SIMILARITY_MATRIX_IN_MEMORY_PROPERTY);
    boolean keepSimMatrixInMem = (inMemProp != null) ? Boolean.parseBoolean(inMemProp) : true;

    // IMPLEMENTATION NOTE: Ahn et al. used single-linkage HAC, which can be
    // efficiently implemented in O(n^2) time as a special case of HAC.
    // However, we currently don't optimize for this special case and
    // instead use our HAC class.  Because of the complexity of the edge
    // similarity function, we build our own similarity matrix and then pass
    // it in, rather than passing in the edge matrix directly.

    final int rows = sm.rows();
    numRows = rows;
    LOGGER.fine("Generating link similarity matrix for " + rows + " nodes");

    //  Rather than create an O(row^3) matrix for representing the edges,
    // compress the edge matrix by getting a mapping for each edge to a row
    // in the new matrix.
    final List<Edge> edgeList = new ArrayList<Edge>();
    this.edgeList = edgeList;

    for (int r = 0; r < rows; ++r) {
      SparseDoubleVector row = sm.getRowVector(r);
      int[] edges = row.getNonZeroIndices();
      for (int col : edges) {
        // Always add edges from the upper triangular
        if (r > col) edgeList.add(new Edge(r, col));
        // Otherwise, we only add the edge from the lower triangular if
        // it wasn't present in the upper.  This avoids counting
        // duplicate edges.
        else if (r < col && sm.get(col, r) == 0) edgeList.add(new Edge(r, col));
      }
    }

    final int numEdges = edgeList.size();
    LOGGER.fine("Number of edges to cluster: " + numEdges);

    Matrix edgeSimMatrix = getEdgeSimMatrix(edgeList, sm, keepSimMatrixInMem);

    LOGGER.fine("Computing single linkage link clustering");

    final List<Merge> mergeOrder =
        new HierarchicalAgglomerativeClustering()
            .buildDendrogram(edgeSimMatrix, ClusterLinkage.SINGLE_LINKAGE);
    this.mergeOrder = mergeOrder;

    LOGGER.fine("Calculating partition densitities");

    // Set up a concurrent map that each thread will update once it has
    // calculated the densitites of each of its partitions.  This map is
    // only written to once per thread.
    final ConcurrentNavigableMap<Double, Integer> partitionDensities =
        new ConcurrentSkipListMap<Double, Integer>();

    // Register a task group for calculating all of the partition
    // densitities
    Object key = WORK_QUEUE.registerTaskGroup(mergeOrder.size());
    for (int p = 0; p < mergeOrder.size(); ++p) {
      final int part = p;
      WORK_QUEUE.add(
          key,
          new Runnable() {
            public void run() {
              // Get the merges for this particular partitioning of
              // the links
              List<Merge> mergeSteps = mergeOrder.subList(0, part);

              // Convert the merges to a specific cluster labeling
              MultiMap<Integer, Integer> clusterToElements =
                  convertMergesToAssignments(mergeSteps, numEdges);

              // Based on the link partitioning, calculate the
              // partition density for each cluster
              double partitionDensitySum = 0d;
              for (Integer cluster : clusterToElements.keySet()) {
                Set<Integer> linkPartition = clusterToElements.get(cluster);
                int numLinks = linkPartition.size();
                BitSet nodesInPartition = new BitSet(rows);
                for (Integer linkIndex : linkPartition) {
                  Edge link = edgeList.get(linkIndex);
                  nodesInPartition.set(link.from);
                  nodesInPartition.set(link.to);
                }
                int numNodes = nodesInPartition.cardinality();
                // This reflects the density of this particular
                // cluster
                double partitionDensity =
                    (numLinks - (numNodes - 1d))
                        / (((numNodes * (numNodes - 1d)) / 2d) - (numLinks - 1));
                partitionDensitySum += partitionDensity;
              }
              // Compute the density for the total partitioning
              // solution
              double partitionDensity = (2d / numEdges) * partitionDensitySum;
              LOGGER.log(
                  Level.FINER,
                  "Partition solution {0} had " + "density {1}",
                  new Object[] {part, partitionDensity});

              // Update the thread-shared partition density map with
              // this task's calculation
              partitionDensities.put(partitionDensity, part);
            }
          });
    }

    // Wait for all the partition densities to be calculated
    WORK_QUEUE.await(key);

    Map.Entry<Double, Integer> densest = partitionDensities.lastEntry();
    LOGGER.fine(
        "Partition " + densest.getValue() + " had the highest density: " + densest.getKey());
    int partitionWithMaxDensity = densest.getValue();

    // Select the solution with the highest partition density and assign
    // nodes accordingly
    MultiMap<Integer, Integer> bestEdgeAssignment =
        convertMergesToAssignments(mergeOrder.subList(0, partitionWithMaxDensity), numEdges);

    List<Set<Integer>> nodeClusters = new ArrayList<Set<Integer>>(rows);
    for (int i = 0; i < rows; ++i) nodeClusters.add(new HashSet<Integer>());

    // Ignore the original partition labeling, and use our own cluster
    // labeling to ensure that the IDs are contiguous.
    int clusterId = 0;

    // For each of the partitions, add the partion's cluster ID to all the
    // nodes that are connected by one of the partition's edges
    for (Integer cluster : bestEdgeAssignment.keySet()) {
      Set<Integer> edgePartition = bestEdgeAssignment.get(cluster);
      for (Integer edgeId : edgePartition) {
        Edge e = edgeList.get(edgeId);
        nodeClusters.get(e.from).add(clusterId);
        nodeClusters.get(e.to).add(clusterId);
      }
      // Update the cluster id
      clusterId++;
    }

    int numClusters = 0;
    Assignment[] nodeAssignments = new Assignment[rows];
    for (int i = 0; i < nodeAssignments.length; ++i) {
      nodeAssignments[i] = new SoftAssignment(nodeClusters.get(i));
    }
    return new Assignments(numClusters, nodeAssignments, matrix);
  }