public void testGetDistributionFromLogValues() {
Counter<String> c1 = new DefaultCounter<String>();
c1.setCount("p", 1.0);
c1.setCount("q", 2.0);
c1.setCount("r", 3.0);
c1.setCount("s", 4.0);
// take log
Counters.logInPlace(c1);
// now call distribution
Distribution<String> distribution = Distribution.getDistributionFromLogValues(c1);
// test
assertEquals(distribution.keySet().size(), 4); // size
// keys
assertEquals(distribution.containsKey("p"), true);
assertEquals(distribution.containsKey("q"), true);
assertEquals(distribution.containsKey("r"), true);
assertEquals(distribution.containsKey("s"), true);
// values
assertEquals(distribution.getCount("p"), 1.0E-1, 1E-10);
assertEquals(distribution.getCount("q"), 2.0E-1, 1E-10);
assertEquals(distribution.getCount("r"), 3.0E-1, 1E-10);
assertEquals(distribution.getCount("s"), 4.0E-1, 1E-10);
}
/**
* GT smoothing with least squares interpolation. This follows the procedure in Jurafsky and
* Martin sect. 4.5.3.
*/
public void smoothAndNormalize() {
Counter<Integer> cntCounter = new Counter<Integer>();
for (K tok : lm.keySet()) {
int cnt = (int) lm.getCount(tok);
cntCounter.incrementCount(cnt);
}
final double[] coeffs = runLogSpaceRegression(cntCounter);
UNK_PROB = cntCounter.getCount(1) / lm.totalCount();
for (K tok : lm.keySet()) {
double tokCnt = lm.getCount(tok);
if (tokCnt <= unkCutoff) // Treat as unknown
unkTokens.add(tok);
if (tokCnt <= kCutoff) { // Smooth
double cSmooth = katzEstimate(cntCounter, tokCnt, coeffs);
lm.setCount(tok, cSmooth);
}
}
// Normalize
// Counters.normalize(lm);
// MY COUNTER IS ALWAYS NORMALIZED AND AWESOME
}
public static <E> Counter<E> normalize(Counter<E> counter) {
Counter<E> normalizedCounter = new Counter<E>();
double total = counter.totalCount();
for (E key : counter.keySet()) {
normalizedCounter.setCount(key, counter.getCount(key) / total);
}
return normalizedCounter;
}
public static void main(String[] args) {
/* The following test code is just an example of how you can write
* a simple test program to check the basic functionality of your class
* works as you expect. The primary aim is to test each constructor and
* method - there are many combinations of how you could go about this.
* For more complex classes and/or methods you may write several different
* tests for each method to check boundary cases for example.
*/
System.out.println("Testing default constructor initialisation and get method...");
Counter c = new Counter();
System.out.println("Expected 0: Actual: " + c.getCount());
System.out.println("Testing increment method...");
c.increment();
c.increment();
System.out.println("Expected 2: Actual: " + c.getCount());
System.out.println("Testing set method...");
c.setCount(10);
System.out.println("Expected 10: Actual: " + c.getCount());
System.out.println("Testing custom constructor initialisation...");
Counter c1 =
new Counter(
5); // Note: creating a second object instance of the Counter class (which has its own
// state)
System.out.println("Expected 5: Actual: " + c1.getCount());
System.out.println("Testing reset method...");
c1.reset();
System.out.println("Expected 0: Actual: " + c1.getCount());
System.out.println("Testing toString method...");
System.out.println("Expected count=0: Actual: " + c1.toString());
System.out.println("Testing decrement method..");
c.decrement();
c.decrement();
System.out.println("Expected 1: Actual:" + c.getCount());
Counter launch = new Counter();
launch.incremendBy(10);
while (!launch.isZero()) {
System.out.print(launch.getCount() + " ");
launch.decrement();
}
System.out.print("Blast off!");
}
@Override
public void run() {
synchronized (counter) {
int i = 0;
while (true) {
counter.setCount(i);
System.out.println("Counter value seted to: " + i);
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
return;
}
i++;
}
}
}
public Counter<String> getLogScoreCounter(LocalTrigramContext localTrigramContext) {
int position = localTrigramContext.getPosition();
String word = localTrigramContext.getWords().get(position);
Counter<String> tagCounter = unknownWordTags;
if (wordsToTags.keySet().contains(word)) {
tagCounter = wordsToTags.getCounter(word);
}
Set<String> allowedFollowingTags =
allowedFollowingTags(
tagCounter.keySet(),
localTrigramContext.getPreviousPreviousTag(),
localTrigramContext.getPreviousTag());
Counter<String> logScoreCounter = new Counter<String>();
for (String tag : tagCounter.keySet()) {
double logScore = Math.log(tagCounter.getCount(tag));
if (!restrictTrigrams
|| allowedFollowingTags.isEmpty()
|| allowedFollowingTags.contains(tag)) logScoreCounter.setCount(tag, logScore);
}
return logScoreCounter;
}
