Ejemplo n.º 1
0
  /**
   * Based on "Details of the Adjusted Rand index and Clustering algorithms Supplement to the paper
   * “An empirical study on Principal Component Analysis for clustering gene expression data” (to
   * appear in Bioinformatics)"
   *
   * @return
   */
  public Clustering<Instance, Cluster<Instance>> pcaData() {
    Clustering<Instance, Cluster<Instance>> clustering = new ClusterList<>(3);
    Random rand = new Random();
    int size = 10;
    Dataset<? extends Instance> data = new ArrayDataset<>(size, 2);
    data.attributeBuilder().create("x1", "NUMERIC");
    data.attributeBuilder().create("x2", "NUMERIC");

    InstanceBuilder<? extends Instance> builder = data.builder();
    BaseCluster c1 = new BaseCluster(2);
    clustering.add(c1);
    BaseCluster c2 = new BaseCluster(3);
    clustering.add(c2);
    BaseCluster c3 = new BaseCluster(5);
    clustering.add(c3);

    c1.add(next(rand, builder, "u1"));
    c1.add(next(rand, builder, "u2"));
    c2.add(next(rand, builder, "u1"));
    c2.add(next(rand, builder, "u2"));
    c2.add(next(rand, builder, "u2"));
    c3.add(next(rand, builder, "u2"));
    c3.add(next(rand, builder, "u3"));
    c3.add(next(rand, builder, "u3"));
    c3.add(next(rand, builder, "u3"));
    c3.add(next(rand, builder, "u3"));

    clustering.lookupAdd(data);
    return clustering;
  }
Ejemplo n.º 2
0
 /** Dummy mapping for debugging purposes */
 public void createMapping() {
   treeData = new DynamicTreeData();
   mapping = new int[dataset.size()];
   for (int i = 0; i < dataset.size(); i++) {
     mapping[i] = i;
   }
 }
Ejemplo n.º 3
0
  @Override
  public Clustering getClustering(Dataset<E> parent) {
    setDataset(parent);

    int estClusters = (int) Math.sqrt(dataset.size());
    Clustering result = new ClusterList(estClusters);

    // estimated capacity
    int perCluster = (int) (parent.size() / (float) estClusters);
    int[] assign = getMapping();
    if (assign != null) {
      int id;
      Cluster clust;
      for (int i = 0; i < assign.length; i++) {
        id = assign[i];
        clust = result.createCluster(id, perCluster);
        clust.add(dataset.get(i));
      }
    } else {
      // try some cutoff method?
      throw new RuntimeException("don't know how to get clusters..");
    }

    // proximity.printLower(5, 2);
    // similarity.print(4, 2);
    result.lookupAdd(dataset);
    if (props != null) {
      result.setParams(props);
    }
    return result;
  }
Ejemplo n.º 4
0
  /**
   * Sample covariance
   *
   * @param dataset
   * @return
   */
  protected double covariance(Dataset<E> dataset) {
    Matrix m = dataset.asMatrix();
    Matrix cov = new SymmetricMatrixDiag(m.columnsCount());

    DenseVector mean = new DenseVector(dataset.attributeCount());
    for (int i = 0; i < mean.size(); i++) {
      mean.set(i, dataset.getAttribute(i).statistics(StatsNum.MEAN));
    }

    Vector v;
    double res, sum = 0.0;
    for (int i = 0; i < m.rowsCount(); i++) {
      v = m.getRowVector(i).minus(mean);
      res = v.dot(v);
      sum += res;
    }
    return sum / (m.rowsCount() - 1);

    /* for (int i = 0; i < m.columnsCount(); i++) {            mean = dataset.getAttribute(i).statistics(StatsNum.AVG);
        cov.set(i, i, dataset.getAttribute(i).statistics(StatsNum.VARIANCE));
        for (int j = 0; j < i; j++) {
            //cov.set(i, j, mean);
        }
    } */

    // return cov;
  }
Ejemplo n.º 5
0
  @Test
  public void testSingleLinkage() {
    Dataset<? extends Instance> dataset = FakeClustering.kumarData();
    assertEquals(6, dataset.size());
    Props pref = new Props();
    pref.put(AlgParams.LINKAGE, SingleLinkage.name);
    pref.put(AlgParams.CLUSTERING_TYPE, ClusteringType.ROWS_CLUSTERING);
    pref.put(PropType.PERFORMANCE, AlgParams.KEEP_PROXIMITY, true);
    HierarchicalResult result = subject.hierarchy(dataset, pref);
    Matrix similarityMatrix = result.getProximityMatrix();
    assertNotNull(similarityMatrix);
    assertEquals(similarityMatrix.rowsCount(), dataset.size());
    assertEquals(similarityMatrix.columnsCount(), dataset.size());
    System.out.println("kumar - single");
    DendroTreeData tree = result.getTreeData();
    tree.print();
    assertEquals(dataset.size(), tree.numLeaves());
    DendroNode root = tree.getRoot();
    assertEquals(0.21587033144922904, root.getHeight(), DELTA);

    int levels = tree.distinctHeights(1e-7);
    // TODO: in this example nodes #7 and #8 are on different level,
    // but their height is the same. should we consider those as different
    assertEquals(4, levels);
  }
Ejemplo n.º 6
0
  public static Clustering wineCorrect() {
    if (simpleClustering == null) {

      simpleClustering = new ClusterList(3);
      Cluster a = new BaseCluster(12);
      a.setName("cabernet");
      a.attributeBuilder().create("x", BasicAttrType.INTEGER);
      Cluster b = new BaseCluster(9);
      b.setName("syrah");
      b.attributeBuilder().create("x", BasicAttrType.INTEGER);
      Cluster c = new BaseCluster(6);
      c.setName("pinot");
      c.attributeBuilder().create("x", BasicAttrType.INTEGER);

      Dataset<Instance> data = wine();
      for (int i = 0; i < 13; i++) {
        a.add(data.instance(i));
      }

      for (int i = 13; i < 22; i++) {
        b.add(data.instance(i));
      }

      for (int i = 22; i < 27; i++) {
        c.add(data.instance(i));
      }

      simpleClustering.add(a);
      simpleClustering.add(b);
      simpleClustering.add(c);
    }

    return simpleClustering;
  }
Ejemplo n.º 7
0
 @Override
 public int size() {
   switch (resultType) {
     case COLUMNS_CLUSTERING:
       return dataset.attributeCount();
     case ROWS_CLUSTERING:
       return dataset.size();
   }
   throw new RuntimeException("Don't know wether cluster rows or columns.");
 }
Ejemplo n.º 8
0
  public static Clustering irisWrong() {
    if (irisWrong == null) {
      irisWrong = new ClusterList(3);
      Cluster a = new BaseCluster(50);
      a.setName("cluster 1");
      a.setAttributes(irisData.getAttributes());
      // add few instances to first cluster
      a.add(irisData.instance(0));
      a.add(irisData.instance(1));
      a.add(irisData.instance(2));
      a.add(irisData.instance(149));

      Cluster b = new BaseCluster(50);
      b.setName("cluster 2");
      b.setAttributes(irisData.getAttributes());
      b.add(irisData.instance(3));
      b.add(irisData.instance(4));
      b.add(irisData.instance(5));
      b.add(irisData.instance(6));
      Cluster c = new BaseCluster(50);
      c.setName("cluster 3");
      c.setAttributes(irisData.getAttributes());
      // rest goes to the last cluster
      for (int i = 7; i < 149; i++) {
        c.add(irisData.instance(i));
      }

      irisWrong.add(a);
      irisWrong.add(b);
      irisWrong.add(c);
    }
    return irisWrong;
  }
 @Test
 public void testCompleteLinkage() {
   Dataset<? extends Instance> dataset = FakeDatasets.irisDataset();
   for (AgglomerativeClustering alg : algorithms) {
     NanoBench.create()
         .cpuAndMemory()
         .measurements(2)
         .measure(
             alg.getName() + " complete link - " + dataset.getName(),
             new HclustBenchmark().completeLinkage(alg, dataset));
   }
 }
Ejemplo n.º 10
0
 /**
  * Testing dataset from Kumar (chapter 8, page 519)
  *
  * @return
  */
 public static Dataset<? extends Instance> kumarData() {
   if (kumar == null) {
     kumar = new ArrayDataset<>(4, 2);
     kumar.attributeBuilder().create("x", BasicAttrType.NUMERIC);
     kumar.attributeBuilder().create("y", BasicAttrType.NUMERIC);
     kumar.builder().create(new double[] {0.40, 0.53}, "1");
     kumar.builder().create(new double[] {0.22, 0.38}, "2");
     kumar.builder().create(new double[] {0.35, 0.32}, "3");
     kumar.builder().create(new double[] {0.26, 0.19}, "4");
     kumar.builder().create(new double[] {0.08, 0.41}, "5");
     kumar.builder().create(new double[] {0.45, 0.30}, "6");
   }
   return kumar;
 }
Ejemplo n.º 11
0
  @Test
  public void testColumnClustering() throws IOException {
    Dataset<? extends Instance> dataset = FakeClustering.schoolData();
    Props pref = new Props();
    pref.put(AlgParams.LINKAGE, SingleLinkage.name);
    pref.put(AlgParams.CLUSTERING_TYPE, ClusteringType.COLUMNS_CLUSTERING);
    pref.put(PropType.PERFORMANCE, AlgParams.KEEP_PROXIMITY, true);
    HierarchicalResult result = subject.hierarchy(dataset, pref);
    Matrix similarityMatrix = result.getProximityMatrix();
    assertNotNull(similarityMatrix);
    assertEquals(similarityMatrix.rowsCount(), dataset.attributeCount());
    assertEquals(similarityMatrix.columnsCount(), dataset.attributeCount());

    result.getTreeData().print();
  }
Ejemplo n.º 12
0
 @Override
 public E getInstance(int index) {
   if (dataset != null) {
     return dataset.instance(getMappedIndex(index));
   } else {
     throw new RuntimeException("dataset is null");
   }
 }
Ejemplo n.º 13
0
  public Clustering<Instance, Cluster<Instance>> oneClassPerCluster() {
    Clustering<Instance, Cluster<Instance>> oneClass = new ClusterList(3);
    int size = 10;
    Random rand = new Random();
    Dataset<? extends Instance> data = new ArrayDataset<>(size, 2);
    data.attributeBuilder().create("x1", "NUMERIC");
    data.attributeBuilder().create("x2", "NUMERIC");

    for (int i = 0; i < size; i++) {
      Instance inst = next(rand, data.builder(), "same class");
      // cluster with single class
      BaseCluster clust = new BaseCluster(1);
      clust.add(inst);
      oneClass.add(clust);
    }
    oneClass.lookupAdd(data);
    return oneClass;
  }
Ejemplo n.º 14
0
  @Test
  public void testSingleLinkageSchool() {
    Dataset<? extends Instance> dataset = FakeClustering.schoolData();
    assertEquals(17, dataset.size());
    Props pref = new Props();
    pref.put(AlgParams.LINKAGE, SingleLinkage.name);
    pref.put(AlgParams.CLUSTERING_TYPE, ClusteringType.ROWS_CLUSTERING);
    HierarchicalResult result = subject.hierarchy(dataset, pref);
    System.out.println("school - single");
    DendroTreeData tree = result.getTreeData();
    tree.print();
    assertEquals(dataset.size(), tree.numLeaves());
    DendroNode root = tree.getRoot();
    assertEquals(32.542734980330046, root.getHeight(), DELTA);
    assertEquals(2 * dataset.size() - 1, tree.numNodes());

    assertEquals(16, tree.distinctHeights());
    assertEquals(8, tree.treeLevels());
  }
Ejemplo n.º 15
0
  @Test
  public void testInverseSorting() {
    Dataset<? extends Instance> dataset = FakeClustering.kumarData();
    assertEquals(6, dataset.size());
    Props pref = new Props();
    pref.put(AlgParams.LINKAGE, SingleLinkage.name);
    pref.put(AlgParams.CLUSTERING_TYPE, ClusteringType.ROWS_CLUSTERING);
    // inverse ordering
    pref.put(AlgParams.SMALLEST_FIRST, false);
    HierarchicalResult result = subject.hierarchy(dataset, pref);
    System.out.println("kumar - inverse");
    DendroTreeData tree = result.getTreeData();
    tree.print();
    assertEquals(dataset.size(), tree.numLeaves());
    DendroNode root = tree.getRoot();
    assertEquals(0.10198039027185574, root.getHeight(), DELTA);

    assertEquals(5, tree.distinctHeights());
    assertEquals(4, tree.treeLevels());
  }
Ejemplo n.º 16
0
  @Before
  public void setUp() {
    irisDataset(); // preload
    irisClusters = new ClusterList(3);
    Cluster a = new BaseCluster(50);
    a.setName("cluster 1");
    a.setClusterId(0);
    a.setAttributes(irisData.getAttributes());
    Cluster b = new BaseCluster(50);
    b.setName("cluster 2");
    b.setAttributes(irisData.getAttributes());
    b.setClusterId(1);
    Cluster c = new BaseCluster(50);
    c.setName("cluster 3");
    c.setAttributes(irisData.getAttributes());
    c.setClusterId(2);
    for (int i = 0; i < 50; i++) {
      a.add(irisData.instance(i));
      b.add(irisData.instance(i + 50));
      c.add(irisData.instance(i + 100));
    }

    irisClusters.add(a);
    irisClusters.add(b);
    irisClusters.add(c);
  }
Ejemplo n.º 17
0
  /**
   * Pretty bad clustering result, one class contained in two clusters
   *
   * @return
   */
  public static Clustering irisWrong2() {
    if (irisWrong2 == null) {
      irisWrong2 = new ClusterList(3);
      Cluster a = new BaseCluster(50);
      a.setName("cluster 1");
      // will contain 30 elements of first class
      a.setAttributes(irisData.getAttributes());
      for (int i = 0; i < 30; i++) {
        a.add(irisData.instance(i));
      }

      Cluster b = new BaseCluster(50);
      b.setName("cluster 2");
      // will contain 20 elements of first class
      b.setAttributes(irisData.getAttributes());
      for (int i = 30; i < 50; i++) {
        b.add(irisData.instance(i));
      }
      Cluster c = new BaseCluster(50);
      c.setName("cluster 3");
      c.setAttributes(irisData.getAttributes());
      // the rest (100) goes to the last cluster
      for (int i = 50; i < 150; i++) {
        c.add(irisData.instance(i));
      }

      irisWrong2.add(a);
      irisWrong2.add(b);
      irisWrong2.add(c);
    }
    return irisWrong2;
  }
Ejemplo n.º 18
0
  public static Clustering iris() {
    if (irisClusters == null) {
      irisDataset();
      /** fictive clustering, create iris cluster based on class labels (the dataset is sorted) */
      irisClusters = new ClusterList(3);
      Cluster a = new BaseCluster(50);
      a.setName("cluster 1");
      a.setAttributes(irisData.getAttributes());
      Cluster b = new BaseCluster(50);
      b.setName("cluster 2");
      b.setAttributes(irisData.getAttributes());
      Cluster c = new BaseCluster(50);
      c.setName("cluster 3");
      c.setAttributes(irisData.getAttributes());
      for (int i = 0; i < 50; i++) {
        a.add(irisData.instance(i));
        b.add(irisData.instance(i + 50));
        c.add(irisData.instance(i + 100));
      }

      irisClusters.add(a);
      irisClusters.add(b);
      irisClusters.add(c);
    }
    return irisClusters;
  }
Ejemplo n.º 19
0
 @Override
 public Clustering updateCutoff(double cutoff) {
   this.cutoff = cutoff;
   int[] assign = new int[dataset.size()];
   int estClusters = (int) Math.sqrt(dataset.size());
   colorGenerator.reset();
   num = 0; // human readable
   Clustering clusters = new ClusterList(estClusters);
   DendroNode root = treeData.getRoot();
   if (root != null) {
     checkCutoff(root, cutoff, clusters, assign);
     if (clusters.size() > 0) {
       mapping = assign;
     } else {
       LOG.info("failed to cutoff dendrogram, cut = {}", cutoff);
     }
   }
   // add input dataset to clustering lookup
   if (noise != null) {
     Cluster clust = new BaseCluster<>(noise.size());
     clust.setColor(colorGenerator.next());
     clust.setClusterId(num++);
     clust.setParent(getDataset());
     clust.setName("Noise");
     clust.setAttributes(getDataset().getAttributes());
     for (Instance ins : noise) {
       clust.add(ins);
       mapping[ins.getIndex()] = num - 1;
     }
     clusters.add(clust);
   }
   clusters.lookupAdd(dataset);
   if (dendroMapping != null) {
     clusters.lookupAdd(dendroMapping);
   }
   clusters.lookupAdd(this);
   return clusters;
 }
Ejemplo n.º 20
0
  public static Dataset<Instance> wine() {
    if (wine == null) {
      wine = new SampleDataset(27);
      wine.attributeBuilder().create("x", BasicAttrType.INTEGER);

      String klass = "cabernet";
      for (int i = 0; i < 13; i++) {
        wine.builder().create(new double[] {i}, klass);
      }

      String klass2 = "syrah";
      for (int i = 0; i < 9; i++) {
        wine.builder().create(new double[] {i * 3 + 13}, klass2);
      }

      String klass3 = "pinot";
      for (int i = 0; i < 5; i++) {
        wine.builder().create(new double[] {i * 4 + 50}, klass3);
      }
    }

    return wine;
  }
Ejemplo n.º 21
0
 public static Dataset<? extends Instance> schoolData() {
   if (school == null) {
     CsvLoader loader = new CsvLoader();
     school = new ArrayDataset(17, 4);
     school.setName("school data");
     loader.setClassIndex(4);
     loader.setSeparator(' ');
     try {
       loader.load(fixture.schoolData(), school);
     } catch (IOException ex) {
       Exceptions.printStackTrace(ex);
     }
   }
   return school;
 }
Ejemplo n.º 22
0
 /** TODO: make sure this test is correct */
 @Ignore
 public void testScore() throws ScoreException {
   Clustering c = new ClusterList(2);
   Dataset<? extends Instance> d = new ArrayDataset(8, 2);
   d.builder().create(new double[] {0, 0}, "0");
   d.builder().create(new double[] {0, 0}, "0");
   d.builder().create(new double[] {0, 0}, "0");
   d.builder().create(new double[] {1, 1}, "0");
   d.builder().create(new double[] {1, 1}, "1");
   d.builder().create(new double[] {1, 1}, "1");
   d.builder().create(new double[] {1, 1}, "1");
   d.builder().create(new double[] {1, 1}, "1");
   assertEquals(8, d.size());
   Cluster a = c.createCluster(0, 4);
   Cluster b = c.createCluster(1, 4);
   for (int i = 0; i < 4; i++) {
     a.add(d.get(i));
     b.add(d.get(i + 4));
   }
   assertEquals(2, c.size());
   assertEquals(0.14039740914097984, subject.score(c), delta);
 }
Ejemplo n.º 23
0
  /**
   * @see http://alias-i.com/lingpipe/docs/api/com/aliasi/classify/PrecisionRecallEvaluation.html
   * @return
   */
  public static Clustering wineClustering() {

    if (simpleResponse == null) {
      simpleResponse = new ClusterList(3);
      Cluster a = new BaseCluster(13);
      a.setName("cluster A");
      a.setAttribute(0, a.attributeBuilder().create("x", BasicAttrType.INTEGER));
      Cluster b = new BaseCluster(9);
      b.setName("cluster B");
      b.setAttribute(0, b.attributeBuilder().create("x", BasicAttrType.INTEGER));

      Cluster c = new BaseCluster(5);
      c.setName("cluster C");
      c.setAttribute(0, c.attributeBuilder().create("x", BasicAttrType.INTEGER));

      Dataset<Instance> data = wine();
      System.out.println("dataset size " + data.size());
      // cabernet 9x -> a
      for (int i = 0; i < 9; i++) {
        a.add(data.instance(i));
      }

      // cabernet 2x => b
      b.add(data.instance(9));
      // cabernet 1x => c
      c.add(data.instance(10));
      b.add(data.instance(11));
      b.add(data.instance(12));

      // syrah 2x -> a
      for (int i = 13; i < 15; i++) {
        a.add(data.instance(i));
      }

      // syrah 2x -> c
      c.add(data.instance(15));

      // syrah 5x -> b
      for (int i = 16; i < 21; i++) {
        b.add(data.instance(i));
      }
      a.add(data.instance(21));
      // pinot 4x -> c
      for (int i = 22; i < 26; i++) {
        c.add(data.instance(i));
      }

      // pinot -> cabernet cluster
      b.add(data.instance(26));

      simpleResponse.add(a);
      simpleResponse.add(b);
      simpleResponse.add(c);
    }

    return simpleResponse;
  }