private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
int featuresLength;
int version = in.readInt();
ilist = (InstanceList) in.readObject();
numTopics = in.readInt();
alpha = in.readDouble();
beta = in.readDouble();
tAlpha = in.readDouble();
vBeta = in.readDouble();
int numDocs = ilist.size();
topics = new int[numDocs][];
for (int di = 0; di < ilist.size(); di++) {
int docLen = ((FeatureSequence) ilist.get(di).getData()).getLength();
topics[di] = new int[docLen];
for (int si = 0; si < docLen; si++) topics[di][si] = in.readInt();
}
docTopicCounts = new int[numDocs][numTopics];
for (int di = 0; di < ilist.size(); di++)
for (int ti = 0; ti < numTopics; ti++) docTopicCounts[di][ti] = in.readInt();
int numTypes = ilist.getDataAlphabet().size();
typeTopicCounts = new int[numTypes][numTopics];
for (int fi = 0; fi < numTypes; fi++)
for (int ti = 0; ti < numTopics; ti++) typeTopicCounts[fi][ti] = in.readInt();
tokensPerTopic = new int[numTopics];
for (int ti = 0; ti < numTopics; ti++) tokensPerTopic[ti] = in.readInt();
}
public void generateTestInference() {
if (lda == null) {
System.out.println("Should run lda estimation first.");
System.exit(1);
return;
}
if (testTopicDistribution == null) testTopicDistribution = new double[test.size()][];
TopicInferencer infer = lda.getInferencer();
int iterations = 800;
int thinning = 5;
int burnIn = 100;
for (int ti = 0; ti < test.size(); ti++) {
testTopicDistribution[ti] =
infer.getSampledDistribution(test.get(ti), iterations, thinning, burnIn);
}
}
/** This is (mostly) copied from CRF4.java */
public boolean[][] labelConnectionsIn(
Alphabet outputAlphabet, InstanceList trainingSet, String start) {
int numLabels = outputAlphabet.size();
boolean[][] connections = new boolean[numLabels][numLabels];
for (int i = 0; i < trainingSet.size(); i++) {
Instance instance = trainingSet.getInstance(i);
FeatureSequence output = (FeatureSequence) instance.getTarget();
for (int j = 1; j < output.size(); j++) {
int sourceIndex = outputAlphabet.lookupIndex(output.get(j - 1));
int destIndex = outputAlphabet.lookupIndex(output.get(j));
assert (sourceIndex >= 0 && destIndex >= 0);
connections[sourceIndex][destIndex] = true;
}
}
// Handle start state
if (start != null) {
int startIndex = outputAlphabet.lookupIndex(start);
for (int j = 0; j < outputAlphabet.size(); j++) {
connections[startIndex][j] = true;
}
}
return connections;
}
public static void clearInstances(ModelRoot modelRoot) {
InstanceList instances = modelRoot.getInstanceList(InformalArgument_c.class);
synchronized (instances) {
for (int i = instances.size() - 1; i >= 0; i--) {
((NonRootModelElement) instances.get(i)).delete_unchecked();
}
}
}
public void count() {
TIntIntHashMap docCounts = new TIntIntHashMap();
int index = 0;
if (instances.size() == 0) {
logger.info("Instance list is empty");
return;
}
if (instances.get(0).getData() instanceof FeatureSequence) {
for (Instance instance : instances) {
FeatureSequence features = (FeatureSequence) instance.getData();
for (int i = 0; i < features.getLength(); i++) {
docCounts.adjustOrPutValue(features.getIndexAtPosition(i), 1, 1);
}
int[] keys = docCounts.keys();
for (int i = 0; i < keys.length - 1; i++) {
int feature = keys[i];
featureCounts[feature] += docCounts.get(feature);
documentFrequencies[feature]++;
}
docCounts = new TIntIntHashMap();
index++;
if (index % 1000 == 0) {
System.err.println(index);
}
}
} else if (instances.get(0).getData() instanceof FeatureVector) {
for (Instance instance : instances) {
FeatureVector features = (FeatureVector) instance.getData();
for (int location = 0; location < features.numLocations(); location++) {
int feature = features.indexAtLocation(location);
double value = features.valueAtLocation(location);
documentFrequencies[feature]++;
featureCounts[feature] += value;
}
index++;
if (index % 1000 == 0) {
System.err.println(index);
}
}
} else {
logger.info("Unsupported data class: " + instances.get(0).getData().getClass().getName());
}
}
public void estimate(
InstanceList documents,
int numIterations,
int showTopicsInterval,
int outputModelInterval,
String outputModelFilename,
Randoms r) {
ilist = documents.shallowClone();
numTypes = ilist.getDataAlphabet().size();
int numDocs = ilist.size();
topics = new int[numDocs][];
docTopicCounts = new int[numDocs][numTopics];
typeTopicCounts = new int[numTypes][numTopics];
tokensPerTopic = new int[numTopics];
tAlpha = alpha * numTopics;
vBeta = beta * numTypes;
long startTime = System.currentTimeMillis();
// Initialize with random assignments of tokens to topics
// and finish allocating this.topics and this.tokens
int topic, seqLen;
FeatureSequence fs;
for (int di = 0; di < numDocs; di++) {
try {
fs = (FeatureSequence) ilist.get(di).getData();
} catch (ClassCastException e) {
System.err.println(
"LDA and other topic models expect FeatureSequence data, not FeatureVector data. "
+ "With text2vectors, you can obtain such data with --keep-sequence or --keep-bisequence.");
throw e;
}
seqLen = fs.getLength();
numTokens += seqLen;
topics[di] = new int[seqLen];
// Randomly assign tokens to topics
for (int si = 0; si < seqLen; si++) {
topic = r.nextInt(numTopics);
topics[di][si] = topic;
docTopicCounts[di][topic]++;
typeTopicCounts[fs.getIndexAtPosition(si)][topic]++;
tokensPerTopic[topic]++;
}
}
this.estimate(
0, numDocs, numIterations, showTopicsInterval, outputModelInterval, outputModelFilename, r);
// 124.5 seconds
// 144.8 seconds after using FeatureSequence instead of tokens[][] array
// 121.6 seconds after putting "final" on FeatureSequence.getIndexAtPosition()
// 106.3 seconds after avoiding array lookup in inner loop with a temporary variable
}
private static Graphnode_c findGraphnodeInstance(
ModelRoot modelRoot, ClassQueryInterface_c test, boolean loadComponent) {
InstanceList instances = modelRoot.getInstanceList(Graphnode_c.class);
synchronized (instances) {
for (int i = 0; i < instances.size(); ++i) {
Graphnode_c x = (Graphnode_c) instances.get(i);
if (test == null || test.evaluate(x)) {
return x;
}
}
}
return null;
}
private static InformalArgument_c findInformalArgumentInstance(
ModelRoot modelRoot, ClassQueryInterface_c test, boolean loadComponent) {
InstanceList instances = modelRoot.getInstanceList(InformalArgument_c.class);
synchronized (instances) {
for (int i = 0; i < instances.size(); ++i) {
InformalArgument_c x = (InformalArgument_c) instances.get(i);
if (test == null || test.evaluate(x)) {
if (x.ensureLoaded(loadComponent)) return x;
}
}
}
return null;
}
public void addDocuments(
InstanceList additionalDocuments,
int numIterations,
int showTopicsInterval,
int outputModelInterval,
String outputModelFilename,
Randoms r) {
if (ilist == null) throw new IllegalStateException("Must already have some documents first.");
for (Instance inst : additionalDocuments) ilist.add(inst);
assert (ilist.getDataAlphabet() == additionalDocuments.getDataAlphabet());
assert (additionalDocuments.getDataAlphabet().size() >= numTypes);
numTypes = additionalDocuments.getDataAlphabet().size();
int numNewDocs = additionalDocuments.size();
int numOldDocs = topics.length;
int numDocs = numOldDocs + numNewDocs;
// Expand various arrays to make space for the new data.
int[][] newTopics = new int[numDocs][];
for (int i = 0; i < topics.length; i++) newTopics[i] = topics[i];
topics = newTopics; // The rest of this array will be initialized below.
int[][] newDocTopicCounts = new int[numDocs][numTopics];
for (int i = 0; i < docTopicCounts.length; i++) newDocTopicCounts[i] = docTopicCounts[i];
docTopicCounts = newDocTopicCounts; // The rest of this array will be initialized below.
int[][] newTypeTopicCounts = new int[numTypes][numTopics];
for (int i = 0; i < typeTopicCounts.length; i++)
for (int j = 0; j < numTopics; j++)
newTypeTopicCounts[i][j] = typeTopicCounts[i][j]; // This array further populated below
FeatureSequence fs;
for (int di = numOldDocs; di < numDocs; di++) {
try {
fs = (FeatureSequence) additionalDocuments.get(di - numOldDocs).getData();
} catch (ClassCastException e) {
System.err.println(
"LDA and other topic models expect FeatureSequence data, not FeatureVector data. "
+ "With text2vectors, you can obtain such data with --keep-sequence or --keep-bisequence.");
throw e;
}
int seqLen = fs.getLength();
numTokens += seqLen;
topics[di] = new int[seqLen];
// Randomly assign tokens to topics
for (int si = 0; si < seqLen; si++) {
int topic = r.nextInt(numTopics);
topics[di][si] = topic;
docTopicCounts[di][topic]++;
typeTopicCounts[fs.getIndexAtPosition(si)][topic]++;
tokensPerTopic[topic]++;
}
}
}
/**
* Initialize this separate model using a complete list.
*
* @param documents
* @param testStartIndex
*/
public void divideDocuments(InstanceList documents, int testStartIndex) {
Alphabet dataAlpha = documents.getDataAlphabet();
Alphabet targetAlpha = documents.getTargetAlphabet();
this.training = new InstanceList(dataAlpha, targetAlpha);
this.test = new InstanceList(dataAlpha, targetAlpha);
int di = 0;
for (di = 0; di < testStartIndex; di++) {
training.add(documents.get(di));
}
for (di = testStartIndex; di < documents.size(); di++) {
test.add(documents.get(di));
}
}
public void doInference() {
try {
ParallelTopicModel model = ParallelTopicModel.read(new File(inferencerFile));
TopicInferencer inferencer = model.getInferencer();
// TopicInferencer inferencer =
// TopicInferencer.read(new File(inferencerFile));
// InstanceList testing = readFile();
readFile();
InstanceList testing = generateInstanceList(); // readFile();
for (int i = 0; i < testing.size(); i++) {
StringBuilder probabilities = new StringBuilder();
double[] testProbabilities = inferencer.getSampledDistribution(testing.get(i), 10, 1, 5);
ArrayList probabilityList = new ArrayList();
for (int j = 0; j < testProbabilities.length; j++) {
probabilityList.add(new Pair<Integer, Double>(j, testProbabilities[j]));
}
Collections.sort(probabilityList, new CustomComparator());
for (int j = 0; j < testProbabilities.length && j < topN; j++) {
if (j > 0) probabilities.append(" ");
probabilities.append(
((Pair<Integer, Double>) probabilityList.get(j)).getFirst().toString()
+ ","
+ ((Pair<Integer, Double>) probabilityList.get(j)).getSecond().toString());
}
System.out.println(docIds.get(i) + "," + probabilities.toString());
}
} catch (Exception e) {
e.printStackTrace();
System.err.println(e.getMessage());
}
}
public static Graphnode_c[] GraphnodeInstances(
ModelRoot modelRoot, ClassQueryInterface_c test, boolean loadComponent) {
InstanceList instances = modelRoot.getInstanceList(Graphnode_c.class);
Vector matches = new Vector();
synchronized (instances) {
for (int i = 0; i < instances.size(); ++i) {
Graphnode_c x = (Graphnode_c) instances.get(i);
if (test == null || test.evaluate(x)) {
matches.add(x);
}
}
if (matches.size() > 0) {
Graphnode_c[] ret_set = new Graphnode_c[matches.size()];
matches.copyInto(ret_set);
return ret_set;
} else {
return new Graphnode_c[0];
}
}
}
public double dataLogLikelihood(InstanceList ilist) {
double logLikelihood = 0;
for (int ii = 0; ii < ilist.size(); ii++) {
double instanceWeight = ilist.getInstanceWeight(ii);
Instance inst = ilist.get(ii);
Labeling labeling = inst.getLabeling();
if (labeling != null)
logLikelihood += instanceWeight * dataLogProbability(inst, labeling.getBestIndex());
else {
Labeling predicted = this.classify(inst).getLabeling();
// System.err.println ("label = \n"+labeling);
// System.err.println ("predicted = \n"+predicted);
for (int lpos = 0; lpos < predicted.numLocations(); lpos++) {
int li = predicted.indexAtLocation(lpos);
double labelWeight = predicted.valueAtLocation(lpos);
// System.err.print (", "+labelWeight);
if (labelWeight == 0) continue;
logLikelihood += instanceWeight * labelWeight * dataLogProbability(inst, li);
}
}
}
return logLikelihood;
}
public static InformalArgument_c[] InformalArgumentInstances(
ModelRoot modelRoot, ClassQueryInterface_c test, boolean loadComponent) {
if (loadComponent) {
PersistenceManager.ensureAllInstancesLoaded(modelRoot, InformalArgument_c.class);
}
InstanceList instances = modelRoot.getInstanceList(InformalArgument_c.class);
Vector matches = new Vector();
synchronized (instances) {
for (int i = 0; i < instances.size(); ++i) {
InformalArgument_c x = (InformalArgument_c) instances.get(i);
if (test == null || test.evaluate(x)) {
if (x.ensureLoaded(loadComponent)) matches.add(x);
}
}
if (matches.size() > 0) {
InformalArgument_c[] ret_set = new InformalArgument_c[matches.size()];
matches.copyInto(ret_set);
return ret_set;
} else {
return new InformalArgument_c[0];
}
}
}
public double labelLogLikelihood(InstanceList ilist) {
double logLikelihood = 0;
for (int ii = 0; ii < ilist.size(); ii++) {
double instanceWeight = ilist.getInstanceWeight(ii);
Instance inst = ilist.get(ii);
Labeling labeling = inst.getLabeling();
if (labeling == null) continue;
Labeling predicted = this.classify(inst).getLabeling();
// System.err.println ("label = \n"+labeling);
// System.err.println ("predicted = \n"+predicted);
if (labeling.numLocations() == 1) {
logLikelihood += instanceWeight * Math.log(predicted.value(labeling.getBestIndex()));
} else {
for (int lpos = 0; lpos < labeling.numLocations(); lpos++) {
int li = labeling.indexAtLocation(lpos);
double labelWeight = labeling.valueAtLocation(lpos);
// System.err.print (", "+labelWeight);
if (labelWeight == 0) continue;
logLikelihood += instanceWeight * labelWeight * Math.log(predicted.value(li));
}
}
}
return logLikelihood;
}
public void test(
Transducer transducer,
InstanceList data,
String description,
PrintStream viterbiOutputStream) {
int[] ntrue = new int[segmentTags.length];
int[] npred = new int[segmentTags.length];
int[] ncorr = new int[segmentTags.length];
LabelAlphabet dict = (LabelAlphabet) transducer.getInputPipe().getTargetAlphabet();
for (int i = 0; i < data.size(); i++) {
Instance instance = data.getInstance(i);
Sequence input = (Sequence) instance.getData();
Sequence trueOutput = (Sequence) instance.getTarget();
assert (input.size() == trueOutput.size());
Sequence predOutput = transducer.viterbiPath(input).output();
assert (predOutput.size() == trueOutput.size());
List trueSegs = new ArrayList();
List predSegs = new ArrayList();
addSegs(trueSegs, trueOutput);
addSegs(predSegs, predOutput);
// System.out.println("FieldF1Evaluator instance "+instance.getName ());
// printSegs(dict, trueSegs, "True");
// printSegs(dict, predSegs, "Pred");
for (Iterator it = predSegs.iterator(); it.hasNext(); ) {
Segment seg = (Segment) it.next();
npred[seg.tag]++;
if (trueSegs.contains(seg)) {
ncorr[seg.tag]++;
}
}
for (Iterator it = trueSegs.iterator(); it.hasNext(); ) {
Segment seg = (Segment) it.next();
ntrue[seg.tag]++;
}
}
DecimalFormat f = new DecimalFormat("0.####");
logger.info(description + " per-field F1");
for (int tag = 0; tag < segmentTags.length; tag++) {
double precision = ((double) ncorr[tag]) / npred[tag];
double recall = ((double) ncorr[tag]) / ntrue[tag];
double f1 = (2 * precision * recall) / (precision + recall);
Label name = dict.lookupLabel(segmentTags[tag]);
logger.info(
" segments "
+ name
+ " true = "
+ ntrue[tag]
+ " pred = "
+ npred[tag]
+ " correct = "
+ ncorr[tag]);
logger.info(
" precision="
+ f.format(precision)
+ " recall="
+ f.format(recall)
+ " f1="
+ f.format(f1));
}
}
public void estimate(
InstanceList documents,
int numIterations,
int showTopicsInterval,
int outputModelInterval,
String outputModelFilename,
Randoms r) {
ilist = documents;
uniAlphabet = ilist.getDataAlphabet();
biAlphabet = ((FeatureSequenceWithBigrams) ilist.get(0).getData()).getBiAlphabet();
numTypes = uniAlphabet.size();
numBitypes = biAlphabet.size();
int numDocs = ilist.size();
topics = new int[numDocs][];
grams = new int[numDocs][];
docTopicCounts = new int[numDocs][numTopics];
typeNgramTopicCounts = new int[numTypes][2][numTopics];
unitypeTopicCounts = new int[numTypes][numTopics];
bitypeTopicCounts = new int[numBitypes][numTopics];
tokensPerTopic = new int[numTopics];
bitokensPerTopic = new int[numTypes][numTopics];
tAlpha = alpha * numTopics;
vBeta = beta * numTypes;
vGamma = gamma * numTypes;
long startTime = System.currentTimeMillis();
// Initialize with random assignments of tokens to topics
// and finish allocating this.topics and this.tokens
int topic, gram, seqLen, fi;
for (int di = 0; di < numDocs; di++) {
FeatureSequenceWithBigrams fs = (FeatureSequenceWithBigrams) ilist.get(di).getData();
seqLen = fs.getLength();
numTokens += seqLen;
topics[di] = new int[seqLen];
grams[di] = new int[seqLen];
// Randomly assign tokens to topics
int prevFi = -1, prevTopic = -1;
for (int si = 0; si < seqLen; si++) {
// randomly sample a topic for the word at position si
topic = r.nextInt(numTopics);
// if a bigram is allowed at position si, then sample a gram status for it.
gram = (fs.getBiIndexAtPosition(si) == -1 ? 0 : r.nextInt(2));
if (gram != 0) biTokens++;
topics[di][si] = topic;
grams[di][si] = gram;
docTopicCounts[di][topic]++;
fi = fs.getIndexAtPosition(si);
if (prevFi != -1) typeNgramTopicCounts[prevFi][gram][prevTopic]++;
if (gram == 0) {
unitypeTopicCounts[fi][topic]++;
tokensPerTopic[topic]++;
} else {
bitypeTopicCounts[fs.getBiIndexAtPosition(si)][topic]++;
bitokensPerTopic[prevFi][topic]++;
}
prevFi = fi;
prevTopic = topic;
}
}
for (int iterations = 0; iterations < numIterations; iterations++) {
sampleTopicsForAllDocs(r);
if (iterations % 10 == 0) System.out.print(iterations);
else System.out.print(".");
System.out.flush();
if (showTopicsInterval != 0 && iterations % showTopicsInterval == 0 && iterations > 0) {
System.out.println();
printTopWords(5, false);
}
if (outputModelInterval != 0 && iterations % outputModelInterval == 0 && iterations > 0) {
this.write(new File(outputModelFilename + '.' + iterations));
}
}
System.out.println(
"\nTotal time (sec): " + ((System.currentTimeMillis() - startTime) / 1000.0));
}