public void generateTestInference() {
if (lda == null) {
System.out.println("Should run lda estimation first.");
System.exit(1);
return;
}
if (testTopicDistribution == null) testTopicDistribution = new double[test.size()][];
TopicInferencer infer = lda.getInferencer();
int iterations = 800;
int thinning = 5;
int burnIn = 100;
for (int ti = 0; ti < test.size(); ti++) {
testTopicDistribution[ti] =
infer.getSampledDistribution(test.get(ti), iterations, thinning, burnIn);
}
}
public static void main(String[] args) {
// String malletFile = "dataset/vlc_lectures.all.en.f8.mallet";
// String simFile = "dataset/vlc/sim5p.csv";
// String solutionFile = "dataset/vlc/task1_solution.en.f8.lm.txt";
// String queryFile = "dataset/task1_query.en.f8.txt";
// String targetFile = "dataset/task1_target.en.f8.txt";
String malletFile = "dataset/vlc/folds/all.0.4189.mallet";
String trainMalletFile = "dataset/vlc/folds/training.0.mallet";
String testMalletFile = "dataset/vlc/folds/test.0.mallet";
String queryFile = "dataset/vlc/folds/query.0.csv";
String linkFile = "dataset/vlc/folds/trainingPairs.0.csv";
String targetFile = "dataset/vlc/folds/target.0.csv";
String solutionFile = "dataset/vlc/task1_solution.en.f8.lm.txt";
int numTopics = 160;
int numIterations = 200;
double alpha = 0.0016;
double beta = 0.0001;
InstanceList train = InstanceList.load(new File(trainMalletFile));
InstanceList test = InstanceList.load(new File(testMalletFile));
SeparateParallelLda spl = new SeparateParallelLda(train, test);
spl.trainDocuments(numTopics, numIterations, alpha, beta);
spl.generateTestInference();
spl.lda.printTopWords(System.out, 10, true);
BasicTask1Solution solver = new Task1SolutionWithSeparateData(spl);
double precision;
try {
solver.retrieveTask1Solution(queryFile, solutionFile);
precision = Task1Solution.evaluateResult(targetFile, solutionFile);
System.out.println(
String.format(
"SeparateParallelLda: iteration: %d, precisoion: %f", numIterations, precision));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
* Initialize this separate model using a complete list.
*
* @param documents
* @param testStartIndex
*/
public void divideDocuments(InstanceList documents, int testStartIndex) {
Alphabet dataAlpha = documents.getDataAlphabet();
Alphabet targetAlpha = documents.getTargetAlphabet();
this.training = new InstanceList(dataAlpha, targetAlpha);
this.test = new InstanceList(dataAlpha, targetAlpha);
int di = 0;
for (di = 0; di < testStartIndex; di++) {
training.add(documents.get(di));
}
for (di = testStartIndex; di < documents.size(); di++) {
test.add(documents.get(di));
}
}