private void calculateArrowsForward(Node x, Node y, Graph graph) {
clearArrow(x, y);
if (!knowledgeEmpty()) {
if (getKnowledge().isForbidden(x.getName(), y.getName())) {
return;
}
}
List<Node> naYX = getNaYX(x, y, graph);
List<Node> t = getTNeighbors(x, y, graph);
DepthChoiceGenerator gen = new DepthChoiceGenerator(t.size(), t.size());
int[] choice;
while ((choice = gen.next()) != null) {
List<Node> s = GraphUtils.asList(choice, t);
if (!knowledgeEmpty()) {
if (!validSetByKnowledge(y, s)) {
continue;
}
}
double bump = insertEval(x, y, s, naYX, graph);
if (bump > 0.0) {
Arrow arrow = new Arrow(bump, x, y, s, naYX);
sortedArrows.add(arrow);
addLookupArrow(x, y, arrow);
}
}
}
private Graph changeLatentNames(Graph full, Clusters measurements, List<String> latentVarList) {
Graph g2 = null;
try {
g2 = (Graph) new MarshalledObject(full).get();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
for (int i = 0; i < measurements.getNumClusters(); i++) {
List<String> d = measurements.getCluster(i);
String latentName = latentVarList.get(i);
for (Node node : full.getNodes()) {
if (!(node.getNodeType() == NodeType.LATENT)) {
continue;
}
List<Node> _children = full.getChildren(node);
_children.removeAll(ReidentifyVariables.getLatents(full));
List<String> childNames = getNames(_children);
if (new HashSet<String>(childNames).equals(new HashSet<String>(d))) {
g2.getNode(node.getName()).setName(latentName);
}
}
}
return g2;
}
private String clusterSizes(List<List<Node>> partition, List<List<Node>> trueClusters) {
String s = "";
FOR:
for (int i = 0; i < partition.size(); i++) {
List<Node> cluster = partition.get(i);
s += cluster.size();
for (List<Node> trueCluster : trueClusters) {
if (trueCluster.containsAll(cluster)) {
// Collections.sort(trueCluster);
// Collections.sort(cluster);
// System.out.println(trueCluster + " " + cluster);
s += "p";
if (i < partition.size() - 1) {
s += ",";
}
continue FOR;
}
}
if (i < partition.size() - 1) {
s += ",";
}
}
return s;
}
private List<String> getNames(List<Node> nodes) {
List<String> names = new ArrayList<String>();
for (Node node : nodes) {
names.add(node.getName());
}
return names;
}
public static Graph erdosRenyiGraph(int n, int e) {
List<Node> nodes = new ArrayList<Node>();
for (int i = 0; i < n; i++) nodes.add(new GraphNode("X" + i));
Graph graph = new EdgeListGraph(nodes);
for (int e0 = 0; e0 < e; e0++) {
int i1 = RandomUtil.getInstance().nextInt(n);
int i2 = RandomUtil.getInstance().nextInt(n);
if (i1 == i2) {
e0--;
continue;
}
Edge edge = Edges.undirectedEdge(nodes.get(i1), nodes.get(i2));
if (graph.containsEdge(edge)) {
e0--;
continue;
}
graph.addEdge(edge);
}
return graph;
}
private int numClustered(List<List<Node>> partition) {
int sum = 0;
for (int i = 0; i < partition.size(); i++) {
List<Node> cluster = partition.get(i);
sum += cluster.size();
}
return sum;
}
/** Returns true iif the given set forms a clique in the given graph. */
private static boolean isClique(List<Node> nodes, Graph graph) {
for (int i = 0; i < nodes.size() - 1; i++) {
for (int j = i + 1; j < nodes.size(); j++) {
if (!graph.isAdjacentTo(nodes.get(i), nodes.get(j))) {
return false;
}
}
}
return true;
}
private Graph structure(Graph mim) {
List<Node> latents = new ArrayList<Node>();
for (Node node : mim.getNodes()) {
if (node.getNodeType() == NodeType.LATENT) {
latents.add(node);
}
}
return mim.subgraph(latents);
}
private boolean dConnected(Graph graph, String x, String y, String... z) {
Node _x = graph.getNode(x);
Node _y = graph.getNode(y);
List<Node> _z = new ArrayList<Node>();
for (String name : z) {
_z.add(graph.getNode(name));
}
return graph.isDConnectedTo(_x, _y, _z);
}
private static double characteristicPathLength(Graph g) {
List<Node> nodes = g.getNodes();
int total = 0;
int count = 0;
for (int i = 0; i < nodes.size(); i++) {
for (int j = i; j < nodes.size(); j++) {
int shortest = shortestPath(nodes.get(i), nodes.get(j), g);
total += shortest;
count++;
}
}
return total / (double) count;
}
private static int weight(List<Node> nodes, Graph graph, int total, int b) {
double p = 1;
int degree = graph.getNumEdges(nodes.get(b));
int t = degree + 1;
total += pow((double) t, p);
return total;
}
private boolean existsUnblockedSemiDirectedPath(Node from, Node to, List<Node> cond, Graph G) {
Queue<Node> Q = new LinkedList<Node>();
Set<Node> V = new HashSet<Node>();
Q.offer(from);
V.add(from);
while (!Q.isEmpty()) {
Node t = Q.remove();
if (t == to) return true;
for (Node u : G.getAdjacentNodes(t)) {
Edge edge = G.getEdge(t, u);
Node c = Edges.traverseSemiDirected(t, edge);
if (c == null) continue;
if (cond.contains(c)) continue;
if (c == to) return true;
if (!V.contains(c)) {
V.add(c);
Q.offer(c);
}
}
}
return false;
}
// ===========================SCORING METHODS===================//
public double scoreDag(Graph graph) {
Graph dag = new EdgeListGraphSingleConnections(graph);
buildIndexing(graph);
double score = 0.0;
for (Node y : dag.getNodes()) {
Set<Node> parents = new HashSet<Node>(dag.getParents(y));
int nextIndex = -1;
for (int i = 0; i < getVariables().size(); i++) {
nextIndex = hashIndices.get(variables.get(i));
}
int parentIndices[] = new int[parents.size()];
Iterator<Node> pi = parents.iterator();
int count = 0;
while (pi.hasNext()) {
Node nextParent = pi.next();
parentIndices[count++] = hashIndices.get(nextParent);
}
if (this.isDiscrete()) {
score += localDiscreteScore(nextIndex, parentIndices);
} else {
score += localSemScore(nextIndex, parentIndices);
}
}
return score;
}
public void testAlternativeGraphs() {
// UniformGraphGenerator gen = new UniformGraphGenerator(UniformGraphGenerator.ANY_DAG);
// gen.setNumNodes(100);
// gen.setMaxEdges(200);
// gen.setMaxDegree(30);
// gen.setMaxInDegree(30);
// gen.setMaxOutDegree(30);
//// gen.setNumIterations(3000000);
// gen.setResamplingDegree(10);
//
// gen.generate();
//
// Graph graph = gen.getDag();
Graph graph = weightedRandomGraph(250, 400);
List<Integer> degreeCounts = new ArrayList<Integer>();
Map<Integer, Integer> degreeCount = new HashMap<Integer, Integer>();
for (Node node : graph.getNodes()) {
int degree = graph.getNumEdges(node);
degreeCounts.add(degree);
if (degreeCount.get(degree) == null) {
degreeCount.put(degree, 0);
}
degreeCount.put(degree, degreeCount.get(degree) + 1);
}
Collections.sort(degreeCounts);
System.out.println(degreeCounts);
List<Integer> _degrees = new ArrayList<Integer>(degreeCount.keySet());
Collections.sort(_degrees);
for (int i : _degrees) {
int j = degreeCount.get(i);
// System.out.println(i + " " + j);
System.out.println(log(i + 1) + " " + log(j));
}
System.out.println("\nCPL = " + characteristicPathLength(graph));
Graph erGraph = erdosRenyiGraph(200, 200);
System.out.println("\n ER CPL = " + characteristicPathLength(erGraph));
}
private boolean containsImpureCluster(List<List<Node>> partition, List<List<Node>> trueClusters) {
FOR:
for (int i = 0; i < partition.size(); i++) {
List<Node> cluster = partition.get(i);
for (List<Node> trueCluster : trueClusters) {
if (trueCluster.containsAll(cluster)) {
continue FOR;
}
}
return true;
}
return false;
}
/** Get all nodes that are connected to Y by an undirected edge and not adjacent to X. */
private static List<Node> getTNeighbors(Node x, Node y, Graph graph) {
List<Edge> yEdges = graph.getEdges(y);
List<Node> tNeighbors = new ArrayList<Node>();
for (Edge edge : yEdges) {
if (!Edges.isUndirectedEdge(edge)) {
continue;
}
Node z = edge.getDistalNode(y);
if (graph.isAdjacentTo(z, x)) {
continue;
}
tNeighbors.add(z);
}
return tNeighbors;
}
/**
* Find all nodes that are connected to Y by an undirected edge that are adjacent to X (that is,
* by undirected or directed edge).
*/
private static List<Node> getNaYX(Node x, Node y, Graph graph) {
List<Edge> yEdges = graph.getEdges(y);
List<Node> nayx = new ArrayList<Node>();
for (Edge edge : yEdges) {
if (!Edges.isUndirectedEdge(edge)) {
continue;
}
Node z = edge.getDistalNode(y);
if (!graph.isAdjacentTo(z, x)) {
continue;
}
nayx.add(z);
}
return nayx;
}
// Invalid if then nodes or graph changes.
private void calculateArrowsBackward(Node x, Node y, Graph graph) {
if (x == y) {
return;
}
if (!graph.isAdjacentTo(x, y)) {
return;
}
if (!knowledgeEmpty()) {
if (!getKnowledge().noEdgeRequired(x.getName(), y.getName())) {
return;
}
}
List<Node> naYX = getNaYX(x, y, graph);
clearArrow(x, y);
List<Node> _naYX = new ArrayList<Node>(naYX);
DepthChoiceGenerator gen = new DepthChoiceGenerator(_naYX.size(), _naYX.size());
int[] choice;
while ((choice = gen.next()) != null) {
List<Node> H = GraphUtils.asList(choice, _naYX);
if (!knowledgeEmpty()) {
if (!validSetByKnowledge(y, H)) {
continue;
}
}
double bump = deleteEval(x, y, H, naYX, graph);
if (bump > 0.0) {
Arrow arrow = new Arrow(bump, x, y, H, naYX);
sortedArrows.add(arrow);
addLookupArrow(x, y, arrow);
}
}
}
/** Tests to see if d separation facts are symmetric. */
public void testDSeparation() {
EdgeListGraphSingleConnections graph =
new EdgeListGraphSingleConnections(
new Dag(GraphUtils.randomGraph(7, 0, 7, 30, 15, 15, true)));
System.out.println(graph);
List<Node> nodes = graph.getNodes();
int depth = -1;
for (int i = 0; i < nodes.size(); i++) {
for (int j = i + 1; j < nodes.size(); j++) {
Node x = nodes.get(i);
Node y = nodes.get(j);
List<Node> theRest = new ArrayList<Node>(nodes);
theRest.remove(x);
theRest.remove(y);
DepthChoiceGenerator gen = new DepthChoiceGenerator(theRest.size(), depth);
int[] choice;
while ((choice = gen.next()) != null) {
List<Node> z = new LinkedList<Node>();
for (int k = 0; k < choice.length; k++) {
z.add(theRest.get(choice[k]));
}
if (graph.isDSeparatedFrom(x, y, z) != graph.isDSeparatedFrom(y, x, z)) {
fail(
SearchLogUtils.independenceFact(x, y, z)
+ " should have same d-sep result as "
+ SearchLogUtils.independenceFact(y, x, z));
}
}
}
}
}
/** Tests to see if d separation facts are symmetric. */
public void testDSeparation2() {
EdgeListGraphSingleConnections graph =
new EdgeListGraphSingleConnections(
new Dag(GraphUtils.randomGraph(7, 0, 14, 30, 15, 15, true)));
List<Node> nodes = graph.getNodes();
int depth = -1;
for (int i = 0; i < nodes.size(); i++) {
for (int j = i; j < nodes.size(); j++) {
Node x = nodes.get(i);
Node y = nodes.get(j);
List<Node> theRest = new ArrayList<Node>(nodes);
// theRest.remove(x);
// theRest.remove(y);
DepthChoiceGenerator gen = new DepthChoiceGenerator(theRest.size(), depth);
int[] choice;
while ((choice = gen.next()) != null) {
List<Node> z = new LinkedList<Node>();
for (int k = 0; k < choice.length; k++) {
z.add(theRest.get(choice[k]));
}
boolean dConnectedTo = graph.isDConnectedTo(x, y, z);
boolean dConnectedTo1 = graph.isDConnectedTo(y, x, z);
if (dConnectedTo != dConnectedTo1) {
System.out.println(x + " d connected to " + y + " given " + z);
System.out.println(graph);
System.out.println("dconnectedto = " + dConnectedTo);
System.out.println("dconnecteto1 = " + dConnectedTo1);
fail();
}
}
}
}
}
public static Graph weightedRandomGraph(int n, int e) {
List<Node> nodes = new ArrayList<Node>();
for (int i = 0; i < n; i++) nodes.add(new GraphNode("X" + i));
Graph graph = new EdgeListGraph(nodes);
for (int e0 = 0; e0 < e; e0++) {
int i1 = weightedRandom(nodes, graph);
// int i2 = RandomUtil.getInstance().nextInt(n);
int i2 = weightedRandom(nodes, graph);
if (!(shortestPath(nodes.get(i1), nodes.get(i2), graph) < 9)) {
e0--;
continue;
}
if (i1 == i2) {
e0--;
continue;
}
Edge edge = Edges.undirectedEdge(nodes.get(i1), nodes.get(i2));
if (graph.containsEdge(edge)) {
e0--;
continue;
}
graph.addEdge(edge);
}
for (Edge edge : graph.getEdges()) {
Node n1 = edge.getNode1();
Node n2 = edge.getNode2();
if (!graph.isAncestorOf(n2, n1)) {
graph.removeEdge(edge);
graph.addDirectedEdge(n1, n2);
} else {
graph.removeEdge(edge);
graph.addDirectedEdge(n2, n1);
}
}
return graph;
}
private static int weightedRandom(List<Node> nodes, Graph graph) {
int total = 0;
int n = nodes.size();
for (int b = 0; b < n; b++) {
total = weight(nodes, graph, total, b);
}
int r = RandomUtil.getInstance().nextInt(total);
int count = 0;
int index = 0;
for (int b = 0; b < n; b++) {
count = weight(nodes, graph, count, b);
if (r <= count) {
index = b;
break;
}
}
return index;
}
private boolean validInsert(Node x, Node y, List<Node> t, List<Node> naYX, Graph graph) {
List<Node> union = new ArrayList<Node>(t); // t and nayx are disjoint
union.addAll(naYX);
return isClique(union, graph) && !existsUnblockedSemiDirectedPath(y, x, union, graph);
}
public void rtest3() {
Node x = new GraphNode("X");
Node y = new GraphNode("Y");
Node z = new GraphNode("Z");
Node w = new GraphNode("W");
List<Node> nodes = new ArrayList<Node>();
nodes.add(x);
nodes.add(y);
nodes.add(z);
nodes.add(w);
Graph g = new EdgeListGraph(nodes);
g.addDirectedEdge(x, y);
g.addDirectedEdge(x, z);
g.addDirectedEdge(y, w);
g.addDirectedEdge(z, w);
Graph maxGraph = null;
double maxPValue = -1.0;
ICovarianceMatrix maxLatentCov = null;
Graph mim = DataGraphUtils.randomMim(g, 8, 0, 0, 0, true);
// Graph mim = DataGraphUtils.randomSingleFactorModel(5, 5, 8, 0, 0, 0);
Graph mimStructure = structure(mim);
SemPm pm = new SemPm(mim);
System.out.println("\n\nTrue graph:");
System.out.println(mimStructure);
SemImInitializationParams params = new SemImInitializationParams();
params.setCoefRange(0.5, 1.5);
SemIm im = new SemIm(pm, params);
int N = 1000;
DataSet data = im.simulateData(N, false);
CovarianceMatrix cov = new CovarianceMatrix(data);
for (int i = 0; i < 1; i++) {
ICovarianceMatrix _cov = DataUtils.reorderColumns(cov);
List<List<Node>> partition;
FindOneFactorClusters fofc = new FindOneFactorClusters(_cov, TestType.TETRAD_WISHART, .001);
fofc.search();
partition = fofc.getClusters();
System.out.println(partition);
List<String> latentVarList = reidentifyVariables(mim, data, partition, 2);
Mimbuild2 mimbuild = new Mimbuild2();
mimbuild.setAlpha(0.001);
// mimbuild.setMinimumSize(5);
// To test knowledge.
// Knowledge knowledge = new Knowledge2();
// knowledge.setEdgeForbidden("L.Y", "L.W", true);
// knowledge.setEdgeRequired("L.Y", "L.Z", true);
// mimbuild.setKnowledge(knowledge);
Graph mimbuildStructure = mimbuild.search(partition, latentVarList, _cov);
double pValue = mimbuild.getpValue();
System.out.println(mimbuildStructure);
System.out.println("P = " + pValue);
System.out.println("Latent Cov = " + mimbuild.getLatentsCov());
if (pValue > maxPValue) {
maxPValue = pValue;
maxGraph = new EdgeListGraph(mimbuildStructure);
maxLatentCov = mimbuild.getLatentsCov();
}
}
System.out.println("\n\nTrue graph:");
System.out.println(mimStructure);
System.out.println("\nBest graph:");
System.out.println(maxGraph);
System.out.println("P = " + maxPValue);
System.out.println("Latent Cov = " + maxLatentCov);
System.out.println();
}
private void buildIndexing(Graph graph) {
this.hashIndices = new HashMap<Node, Integer>();
for (Node node : graph.getNodes()) {
this.hashIndices.put(node, variables.indexOf(node));
}
}
/** Test if the candidate deletion is a valid operation (Theorem 17 from Chickering, 2002). */
private static boolean validDelete(List<Node> h, List<Node> naXY, Graph graph) {
List<Node> list = new ArrayList<Node>(naXY);
list.removeAll(h);
return isClique(list, graph);
}