public void generateTestInference() {
if (lda == null) {
System.out.println("Should run lda estimation first.");
System.exit(1);
return;
}
if (testTopicDistribution == null) testTopicDistribution = new double[test.size()][];
TopicInferencer infer = lda.getInferencer();
int iterations = 800;
int thinning = 5;
int burnIn = 100;
for (int ti = 0; ti < test.size(); ti++) {
testTopicDistribution[ti] =
infer.getSampledDistribution(test.get(ti), iterations, thinning, burnIn);
}
}
/** This is (mostly) copied from CRF4.java */
public boolean[][] labelConnectionsIn(
Alphabet outputAlphabet, InstanceList trainingSet, String start) {
int numLabels = outputAlphabet.size();
boolean[][] connections = new boolean[numLabels][numLabels];
for (int i = 0; i < trainingSet.size(); i++) {
Instance instance = trainingSet.getInstance(i);
FeatureSequence output = (FeatureSequence) instance.getTarget();
for (int j = 1; j < output.size(); j++) {
int sourceIndex = outputAlphabet.lookupIndex(output.get(j - 1));
int destIndex = outputAlphabet.lookupIndex(output.get(j));
assert (sourceIndex >= 0 && destIndex >= 0);
connections[sourceIndex][destIndex] = true;
}
}
// Handle start state
if (start != null) {
int startIndex = outputAlphabet.lookupIndex(start);
for (int j = 0; j < outputAlphabet.size(); j++) {
connections[startIndex][j] = true;
}
}
return connections;
}
public static void clearInstances(ModelRoot modelRoot) {
InstanceList instances = modelRoot.getInstanceList(InformalArgument_c.class);
synchronized (instances) {
for (int i = instances.size() - 1; i >= 0; i--) {
((NonRootModelElement) instances.get(i)).delete_unchecked();
}
}
}
private static InformalArgument_c findInformalArgumentInstance(
ModelRoot modelRoot, ClassQueryInterface_c test, boolean loadComponent) {
InstanceList instances = modelRoot.getInstanceList(InformalArgument_c.class);
synchronized (instances) {
for (int i = 0; i < instances.size(); ++i) {
InformalArgument_c x = (InformalArgument_c) instances.get(i);
if (test == null || test.evaluate(x)) {
if (x.ensureLoaded(loadComponent)) return x;
}
}
}
return null;
}
private static Graphnode_c findGraphnodeInstance(
ModelRoot modelRoot, ClassQueryInterface_c test, boolean loadComponent) {
InstanceList instances = modelRoot.getInstanceList(Graphnode_c.class);
synchronized (instances) {
for (int i = 0; i < instances.size(); ++i) {
Graphnode_c x = (Graphnode_c) instances.get(i);
if (test == null || test.evaluate(x)) {
return x;
}
}
}
return null;
}
/**
* Initialize this separate model using a complete list.
*
* @param documents
* @param testStartIndex
*/
public void divideDocuments(InstanceList documents, int testStartIndex) {
Alphabet dataAlpha = documents.getDataAlphabet();
Alphabet targetAlpha = documents.getTargetAlphabet();
this.training = new InstanceList(dataAlpha, targetAlpha);
this.test = new InstanceList(dataAlpha, targetAlpha);
int di = 0;
for (di = 0; di < testStartIndex; di++) {
training.add(documents.get(di));
}
for (di = testStartIndex; di < documents.size(); di++) {
test.add(documents.get(di));
}
}
public void doInference() {
try {
ParallelTopicModel model = ParallelTopicModel.read(new File(inferencerFile));
TopicInferencer inferencer = model.getInferencer();
// TopicInferencer inferencer =
// TopicInferencer.read(new File(inferencerFile));
// InstanceList testing = readFile();
readFile();
InstanceList testing = generateInstanceList(); // readFile();
for (int i = 0; i < testing.size(); i++) {
StringBuilder probabilities = new StringBuilder();
double[] testProbabilities = inferencer.getSampledDistribution(testing.get(i), 10, 1, 5);
ArrayList probabilityList = new ArrayList();
for (int j = 0; j < testProbabilities.length; j++) {
probabilityList.add(new Pair<Integer, Double>(j, testProbabilities[j]));
}
Collections.sort(probabilityList, new CustomComparator());
for (int j = 0; j < testProbabilities.length && j < topN; j++) {
if (j > 0) probabilities.append(" ");
probabilities.append(
((Pair<Integer, Double>) probabilityList.get(j)).getFirst().toString()
+ ","
+ ((Pair<Integer, Double>) probabilityList.get(j)).getSecond().toString());
}
System.out.println(docIds.get(i) + "," + probabilities.toString());
}
} catch (Exception e) {
e.printStackTrace();
System.err.println(e.getMessage());
}
}
public static Graphnode_c[] GraphnodeInstances(
ModelRoot modelRoot, ClassQueryInterface_c test, boolean loadComponent) {
InstanceList instances = modelRoot.getInstanceList(Graphnode_c.class);
Vector matches = new Vector();
synchronized (instances) {
for (int i = 0; i < instances.size(); ++i) {
Graphnode_c x = (Graphnode_c) instances.get(i);
if (test == null || test.evaluate(x)) {
matches.add(x);
}
}
if (matches.size() > 0) {
Graphnode_c[] ret_set = new Graphnode_c[matches.size()];
matches.copyInto(ret_set);
return ret_set;
} else {
return new Graphnode_c[0];
}
}
}
public static InformalArgument_c[] InformalArgumentInstances(
ModelRoot modelRoot, ClassQueryInterface_c test, boolean loadComponent) {
if (loadComponent) {
PersistenceManager.ensureAllInstancesLoaded(modelRoot, InformalArgument_c.class);
}
InstanceList instances = modelRoot.getInstanceList(InformalArgument_c.class);
Vector matches = new Vector();
synchronized (instances) {
for (int i = 0; i < instances.size(); ++i) {
InformalArgument_c x = (InformalArgument_c) instances.get(i);
if (test == null || test.evaluate(x)) {
if (x.ensureLoaded(loadComponent)) matches.add(x);
}
}
if (matches.size() > 0) {
InformalArgument_c[] ret_set = new InformalArgument_c[matches.size()];
matches.copyInto(ret_set);
return ret_set;
} else {
return new InformalArgument_c[0];
}
}
}