/** * GT smoothing with least squares interpolation. This follows the procedure in Jurafsky and * Martin sect. 4.5.3. */ public void smoothAndNormalize() { Counter<Integer> cntCounter = new Counter<Integer>(); for (K tok : lm.keySet()) { int cnt = (int) lm.getCount(tok); cntCounter.incrementCount(cnt); } final double[] coeffs = runLogSpaceRegression(cntCounter); UNK_PROB = cntCounter.getCount(1) / lm.totalCount(); for (K tok : lm.keySet()) { double tokCnt = lm.getCount(tok); if (tokCnt <= unkCutoff) // Treat as unknown unkTokens.add(tok); if (tokCnt <= kCutoff) { // Smooth double cSmooth = katzEstimate(cntCounter, tokCnt, coeffs); lm.setCount(tok, cSmooth); } } // Normalize // Counters.normalize(lm); // MY COUNTER IS ALWAYS NORMALIZED AND AWESOME }
/** * Simple sparse dot product method. Try to put the sparser <code>Counter</code> as the <code>x * </code> parameter since we iterate over those keys and search for them in the <code>y</code> * parameter. * * @param x * @param y * @return dotProduct */ public static <E> double dotProduct(Counter<E> x, Counter<E> y) { double total = 0.0; for (E keyX : x.keySet()) { total += x.getCount(keyX) * y.getCount(keyX); } return total; }
private double getDiceCoefficient(String f, String e) { double intersection = collocationCountSentences.getCount(f,e); double cardinalityF = fCountSentences.getCount(f); double cardinalityE = eCountSentences.getCount(e); double dice = 2*intersection / (cardinalityF + cardinalityE); return dice; }
/* Returns a smoothed estimate of P(word|tag) */ public double scoreTagging(String word, String tag) { double p_tag = tagCounter.getCount(tag) / totalTokens; double c_word = wordCounter.getCount(word); double c_tag_and_word = wordToTagCounters.getCount(word, tag); if (c_word < 10) { // rare or unknown c_word += 1.0; c_tag_and_word += typeTagCounter.getCount(tag) / totalWordTypes; } double p_word = (1.0 + c_word) / (totalTokens + totalWordTypes); double p_tag_given_word = c_tag_and_word / c_word; return p_tag_given_word / p_tag * p_word; }
@Test public void testScheduleFixedRateCallable() throws Exception { assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isZero(); assertThat(completed.getCount()).isZero(); assertThat(duration.getCount()).isZero(); assertThat(scheduledOnce.getCount()).isZero(); assertThat(scheduledRepetitively.getCount()).isZero(); assertThat(scheduledOverrun.getCount()).isZero(); assertThat(percentOfPeriod.getCount()).isZero(); ScheduledFuture<?> theFuture = instrumentedScheduledExecutor.scheduleAtFixedRate( new Runnable() { public void run() { assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isEqualTo(1); assertThat(scheduledOnce.getCount()).isEqualTo(0); assertThat(scheduledRepetitively.getCount()).isEqualTo(1); try { TimeUnit.MILLISECONDS.sleep(50); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } return; } }, 10L, 10L, TimeUnit.MILLISECONDS); TimeUnit.MILLISECONDS.sleep(100); theFuture.cancel(true); TimeUnit.MILLISECONDS.sleep(100); assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isZero(); assertThat(completed.getCount()).isNotEqualTo(0); assertThat(duration.getCount()).isNotEqualTo(0); assertThat(duration.getSnapshot().size()).isNotEqualTo(0); assertThat(scheduledOnce.getCount()).isZero(); assertThat(scheduledRepetitively.getCount()).isEqualTo(1); assertThat(scheduledOverrun.getCount()).isNotEqualTo(0); assertThat(percentOfPeriod.getCount()).isNotEqualTo(0); }
@Test public void testScheduleCallable() throws Exception { assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isZero(); assertThat(completed.getCount()).isZero(); assertThat(duration.getCount()).isZero(); assertThat(scheduledOnce.getCount()).isZero(); assertThat(scheduledRepetitively.getCount()).isZero(); assertThat(scheduledOverrun.getCount()).isZero(); assertThat(percentOfPeriod.getCount()).isZero(); final Object obj = new Object(); ScheduledFuture<Object> theFuture = instrumentedScheduledExecutor.schedule( new Callable<Object>() { public Object call() { assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isEqualTo(1); assertThat(completed.getCount()).isZero(); assertThat(duration.getCount()).isZero(); assertThat(scheduledOnce.getCount()).isEqualTo(1); assertThat(scheduledRepetitively.getCount()).isZero(); assertThat(scheduledOverrun.getCount()).isZero(); assertThat(percentOfPeriod.getCount()).isZero(); return obj; } }, 10L, TimeUnit.MILLISECONDS); assertThat(theFuture.get()).isEqualTo(obj); assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isZero(); assertThat(completed.getCount()).isEqualTo(1); assertThat(duration.getCount()).isEqualTo(1); assertThat(duration.getSnapshot().size()).isEqualTo(1); assertThat(scheduledOnce.getCount()).isEqualTo(1); assertThat(scheduledRepetitively.getCount()).isZero(); assertThat(scheduledOverrun.getCount()).isZero(); assertThat(percentOfPeriod.getCount()).isZero(); }
private double katzEstimate(Counter<Integer> cnt, double c, double[] coeffs) { double nC = cnt.getCount((int) c); double nC1 = cnt.getCount(((int) c) + 1); if (nC1 == 0.0) nC1 = Math.exp(coeffs[0] + (coeffs[1] * (c + 1.0))); double n1 = cnt.getCount(1); double nK1 = cnt.getCount(((int) kCutoff) + 1); if (nK1 == 0.0) nK1 = Math.exp(coeffs[0] + (coeffs[1] * (kCutoff + 1.0))); double kTerm = (kCutoff + 1.0) * (nK1 / n1); double cTerm = (c + 1.0) * (nC1 / nC); double cSmooth = (cTerm - (c * kTerm)) / (1.0 - kTerm); return cSmooth; }
public Alignment alignSentencePair(SentencePair sentencePair) { Alignment alignment = new Alignment(); List<String> frenchWords = sentencePair.getFrenchWords(); List<String> englishWords = sentencePair.getEnglishWords(); int numFrenchWords = frenchWords.size(); int numEnglishWords = englishWords.size(); for (int frenchPosition = 0; frenchPosition < numFrenchWords; frenchPosition++) { String f = frenchWords.get(frenchPosition); int englishMaxPosition = frenchPosition; if (englishMaxPosition >= numEnglishWords) englishMaxPosition = -1; // map French word to BASELINE if c(f,e) = 0 for all English words double maxConditionalProb = 0; for (int englishPosition = 0; englishPosition < numEnglishWords; englishPosition++) { String e = englishWords.get(englishPosition); double conditionalGivenEnglish = collocationCounts.getCount(f, e) / (eCounts.getCount(e)); if (conditionalGivenEnglish > maxConditionalProb) { maxConditionalProb = conditionalGivenEnglish; englishMaxPosition = englishPosition; } } alignment.addAlignment(englishMaxPosition, frenchPosition, true); } return alignment; }
/** * Tests that invocation on a stateful session bean fails, if a session hasn't been created * * @throws Exception */ @Test @Ignore( "No longer appropriate, since a proxy can no longer be created without a session, for a SFSB. " + "Need to think if there's a different way to test this. Else just remove this test") public void testSFSBAccessFailureWithoutSession() throws Exception { // create a locator without a session final StatefulEJBLocator<Counter> locator = new StatefulEJBLocator<Counter>( Counter.class, APP_NAME, MODULE_NAME, CounterBean.class.getSimpleName(), "", null, Affinity.NONE, null); final Counter counter = EJBClient.createProxy(locator); Assert.assertNotNull("Received a null proxy", counter); // invoke the bean without creating a session try { final int initialCount = counter.getCount(); Assert.fail( "Expected a EJBException for calling a stateful session bean without creating a session"); } catch (EJBException ejbe) { // expected logger.info("Received the expected exception", ejbe); } }
public double getCount(K token) { if (!lm.keySet().contains(token)) { System.err.println(lm.keySet().size()); throw new RuntimeException("token not in keyset"); } return lm.getCount(token); }
/** * Builds a Trellis over a sentence, by starting at the state State, and advancing through all * legal extensions of each state already in the trellis. You should not have to modify this * code (or even read it, really). */ private Trellis<State> buildTrellis(List<String> sentence) { Trellis<State> trellis = new Trellis<State>(); trellis.setStartState(State.getStartState()); State stopState = State.getStopState(sentence.size() + 2); trellis.setStopState(stopState); Set<State> states = Collections.singleton(State.getStartState()); for (int position = 0; position <= sentence.size() + 1; position++) { Set<State> nextStates = new HashSet<State>(); for (State state : states) { if (state.equals(stopState)) continue; LocalTrigramContext localTrigramContext = new LocalTrigramContext( sentence, position, state.getPreviousPreviousTag(), state.getPreviousTag()); Counter<String> tagScores = localTrigramScorer.getLogScoreCounter(localTrigramContext); for (String tag : tagScores.keySet()) { double score = tagScores.getCount(tag); State nextState = state.getNextState(tag); trellis.setTransitionCount(state, nextState, score); nextStates.add(nextState); } } // System.out.println("States: "+nextStates); states = nextStates; } return trellis; }
@Test public void testScheduleRunnable() throws Exception { assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isZero(); assertThat(completed.getCount()).isZero(); assertThat(duration.getCount()).isZero(); assertThat(scheduledOnce.getCount()).isZero(); assertThat(scheduledRepetitively.getCount()).isZero(); assertThat(scheduledOverrun.getCount()).isZero(); assertThat(percentOfPeriod.getCount()).isZero(); ScheduledFuture<?> theFuture = instrumentedScheduledExecutor.schedule( new Runnable() { public void run() { assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isEqualTo(1); assertThat(completed.getCount()).isZero(); assertThat(duration.getCount()).isZero(); assertThat(scheduledOnce.getCount()).isEqualTo(1); assertThat(scheduledRepetitively.getCount()).isZero(); assertThat(scheduledOverrun.getCount()).isZero(); assertThat(percentOfPeriod.getCount()).isZero(); } }, 10L, TimeUnit.MILLISECONDS); theFuture.get(); assertThat(submitted.getCount()).isZero(); assertThat(running.getCount()).isZero(); assertThat(completed.getCount()).isEqualTo(1); assertThat(duration.getCount()).isEqualTo(1); assertThat(duration.getSnapshot().size()).isEqualTo(1); assertThat(scheduledOnce.getCount()).isEqualTo(1); assertThat(scheduledRepetitively.getCount()).isZero(); assertThat(scheduledOverrun.getCount()).isZero(); assertThat(percentOfPeriod.getCount()).isZero(); }
@Override public void end() { final int count = childCounter.getCount(); collectionStart(count, valuesType.getThriftType()); parentEvents.addAll(listEvents); listEvents.clear(); collectionEnd(); }
public static <E> Counter<E> normalize(Counter<E> counter) { Counter<E> normalizedCounter = new Counter<E>(); double total = counter.totalCount(); for (E key : counter.keySet()) { normalizedCounter.setCount(key, counter.getCount(key) / total); } return normalizedCounter; }
// loads the Storage object and updates the displayed count protected void recoverData() { storage = loadStorage(); if (storage == null) storage = new Storage(); else { Counter item = storage.getCounter(id); titleText.setText(String.format("%s - Counter", item.getName())); setCurrCount(item.getCount()); } }
@Override public void finishTree(DetailAST rootAST) { // pop counter from the stack final Counter counter = counters.pop(); final int count = counter.getCount(); if (count > fileMaximum) { log(rootAST.getLineNo(), rootAST.getColumnNo(), MSG_FILE, count, fileMaximum); } }
/* A builds PCFG using the observed counts of binary and unary * productions in the training trees to estimate the probabilities * for those rules. */ public Grammar(List<Tree<String>> trainTrees) { Counter<UnaryRule> unaryRuleCounter = new Counter<UnaryRule>(); Counter<BinaryRule> binaryRuleCounter = new Counter<BinaryRule>(); Counter<String> symbolCounter = new Counter<String>(); for (Tree<String> trainTree : trainTrees) { tallyTree(trainTree, symbolCounter, unaryRuleCounter, binaryRuleCounter); } for (UnaryRule unaryRule : unaryRuleCounter.keySet()) { double unaryProbability = unaryRuleCounter.getCount(unaryRule) / symbolCounter.getCount(unaryRule.getParent()); unaryRule.setScore(unaryProbability); addUnary(unaryRule); } for (BinaryRule binaryRule : binaryRuleCounter.keySet()) { double binaryProbability = binaryRuleCounter.getCount(binaryRule) / symbolCounter.getCount(binaryRule.getParent()); binaryRule.setScore(binaryProbability); addBinary(binaryRule); } }
public static <K, V> CounterMap<K, V> conditionalNormalize(CounterMap<K, V> counterMap) { CounterMap<K, V> normalizedCounterMap = new CounterMap<K, V>(); for (K key : counterMap.keySet()) { Counter<V> normalizedSubCounter = normalize(counterMap.getCounter(key)); for (V value : normalizedSubCounter.keySet()) { double count = normalizedSubCounter.getCount(value); normalizedCounterMap.setCount(key, value, count); } } return normalizedCounterMap; }
/** * @param <E> * @param x * @param y * @return */ public static <E> double jensenShannonDivergence(Counter<E> x, Counter<E> y) { double sum = 0.0; double xTotal = x.totalCount(); double yTotal = y.totalCount(); for (E key : x.keySet()) { // x -> x+y/2 double xVal = x.getCount(key) / xTotal; double yVal = y.getCount(key) / yTotal; double avg = 0.5 * (xVal + yVal); sum += xVal * Math.log(xVal / avg); } for (E key : y.keySet()) { // y -> x+y/2 double xVal = x.getCount(key) / xTotal; double yVal = y.getCount(key) / yTotal; double avg = 0.5 * (xVal + yVal); sum += yVal * Math.log(yVal / avg); } return sum / 0.5; }
/** * Tests that invocations on a stateful session bean work after a session is created and the * stateful session bean really acts as a stateful bean * * @throws Exception */ @Test public void testSFSBInvocation() throws Exception { final StatefulEJBLocator<Counter> locator = EJBClient.createSession( Counter.class, APP_NAME, MODULE_NAME, CounterBean.class.getSimpleName(), ""); final Counter counter = EJBClient.createProxy(locator); Assert.assertNotNull("Received a null proxy", counter); // invoke the bean final int initialCount = counter.getCount(); logger.info("Got initial count " + initialCount); Assert.assertEquals("Unexpected initial count from stateful bean", 0, initialCount); final int NUM_TIMES = 50; for (int i = 1; i <= NUM_TIMES; i++) { final int count = counter.incrementAndGetCount(); logger.info("Got next count " + count); Assert.assertEquals("Unexpected count after increment", i, count); } final int finalCount = counter.getCount(); logger.info("Got final count " + finalCount); Assert.assertEquals("Unexpected final count", NUM_TIMES, finalCount); }
private void execute() { // setup a counter Counter counter = new Counter(THREAD_COUNT); // create a bunch of threads for (int i = 0; i < THREAD_COUNT; i++) { Thread myThread = new Thread(new MyThread(i, counter)); myThread.start(); } // wait for the threads to finish int justForSureCounter = 10; // give the threads 10 seconds to finish while ((counter.getCount() > 0) && (justForSureCounter > 0)) { waitASecond(); justForSureCounter--; int liveCount = counter.getCount(); System.out.println( "------------- Waiting for threads to finish (count = " + liveCount + ") -------------"); } // print the final value of counter - if > 0 then race condition occured System.out.println( "================== Main finished (count = " + counter.getCount() + ") =================="); }
/** @param args */ public static void main(String[] args) { Counter counter = new Counter(); Adder adder1 = new Adder("Adder 1", counter, 1000, 1, 1); Adder adder2 = new Adder("Adder 2", counter, 1000, 1, 2); Adder adder3 = new Adder("Adder 3", counter, 1000, 1, 3); Adder adder4 = new Adder("Adder 4", counter, 1000, 1, 4); Adder adder5 = new Adder("Adder 5", counter, 1000, 1, 5); Adder adder6 = new Adder("Adder 1", counter, 1000, 1, 6); Adder adder7 = new Adder("Adder 2", counter, 1000, 1, 7); Adder adder8 = new Adder("Adder 3", counter, 1000, 1, 8); Adder adder9 = new Adder("Adder 4", counter, 1000, 1, 9); Adder adder10 = new Adder("Adder 5", counter, 1000, 1, 10); // use executor Executor executor = Executors.newCachedThreadPool(); executor.execute(adder1); executor.execute(adder2); executor.execute(adder3); executor.execute(adder4); executor.execute(adder5); executor.execute(adder6); executor.execute(adder7); executor.execute(adder8); executor.execute(adder9); executor.execute(adder10); while (!adder1.isFinished() || !adder2.isFinished() || !adder3.isFinished() || !adder4.isFinished() || !adder5.isFinished() || !adder6.isFinished() || !adder7.isFinished() || !adder8.isFinished() || !adder9.isFinished() || !adder10.isFinished()) {} System.out.println("Adder 1 status: " + adder1.isFinished()); System.out.println("Adder 2 status: " + adder2.isFinished()); System.out.println("Adder 3 status: " + adder3.isFinished()); System.out.println("Adder 4 status: " + adder4.isFinished()); System.out.println("Adder 5 status: " + adder5.isFinished()); System.out.println("Adder 6 status: " + adder6.isFinished()); System.out.println("Adder 7 status: " + adder7.isFinished()); System.out.println("Adder 8 status: " + adder8.isFinished()); System.out.println("Adder 9 status: " + adder9.isFinished()); System.out.println("Adder 10 status: " + adder10.isFinished()); System.out.println("Adders are Finished and count is " + counter.getCount() + "!"); System.exit(0); }
public static void main(String[] args) { /* The following test code is just an example of how you can write * a simple test program to check the basic functionality of your class * works as you expect. The primary aim is to test each constructor and * method - there are many combinations of how you could go about this. * For more complex classes and/or methods you may write several different * tests for each method to check boundary cases for example. */ System.out.println("Testing default constructor initialisation and get method..."); Counter c = new Counter(); System.out.println("Expected 0: Actual: " + c.getCount()); System.out.println("Testing increment method..."); c.increment(); c.increment(); System.out.println("Expected 2: Actual: " + c.getCount()); System.out.println("Testing set method..."); c.setCount(10); System.out.println("Expected 10: Actual: " + c.getCount()); System.out.println("Testing custom constructor initialisation..."); Counter c1 = new Counter( 5); // Note: creating a second object instance of the Counter class (which has its own // state) System.out.println("Expected 5: Actual: " + c1.getCount()); System.out.println("Testing reset method..."); c1.reset(); System.out.println("Expected 0: Actual: " + c1.getCount()); System.out.println("Testing toString method..."); System.out.println("Expected count=0: Actual: " + c1.toString()); System.out.println("Testing decrement method.."); c.decrement(); c.decrement(); System.out.println("Expected 1: Actual:" + c.getCount()); Counter launch = new Counter(); launch.incremendBy(10); while (!launch.isZero()) { System.out.print(launch.getCount() + " "); launch.decrement(); } System.out.print("Blast off!"); }
private double[] runLogSpaceRegression(Counter<Integer> cntCounter) { SimpleRegression reg = new SimpleRegression(); for (int cnt : cntCounter.keySet()) { reg.addData(cnt, Math.log(cntCounter.getCount(cnt))); } // System.out.println(reg.getIntercept()); // System.out.println(reg.getSlope()); // System.out.println(regression.getSlopeStdErr()); double[] coeffs = new double[] {reg.getIntercept(), reg.getSlope()}; return coeffs; }
@Override public void leaveToken(DetailAST ast) { final int tokenType = ast.getType(); if (tokenType == TokenTypes.METHOD_DEF || tokenType == TokenTypes.CTOR_DEF || tokenType == TokenTypes.STATIC_INIT || tokenType == TokenTypes.INSTANCE_INIT) { // pop counter from the stack final Counter counter = counters.pop(); final int count = counter.getCount(); if (count > methodMaximum) { log(ast.getLineNo(), ast.getColumnNo(), MSG_METHOD, count, methodMaximum); } } else if (tokenType == TokenTypes.CLASS_DEF) { // pop counter from the stack final Counter counter = counters.pop(); final int count = counter.getCount(); if (count > classMaximum) { log(ast.getLineNo(), ast.getColumnNo(), MSG_CLASS, count, classMaximum); } } }
/** * Scores a tagging for a sentence. Note that a tag sequence not accepted by the markov process * should receive a log score of Double.NEGATIVE_INFINITY. */ public double scoreTagging(TaggedSentence taggedSentence) { double logScore = 0.0; List<LabeledLocalTrigramContext> labeledLocalTrigramContexts = extractLabeledLocalTrigramContexts(taggedSentence); for (LabeledLocalTrigramContext labeledLocalTrigramContext : labeledLocalTrigramContexts) { Counter<String> logScoreCounter = localTrigramScorer.getLogScoreCounter(labeledLocalTrigramContext); String currentTag = labeledLocalTrigramContext.getCurrentTag(); if (logScoreCounter.containsKey(currentTag)) { logScore += logScoreCounter.getCount(currentTag); } else { logScore += Double.NEGATIVE_INFINITY; } } return logScore; }
@Test public void reportsCounterValues() throws Exception { final Counter counter = mock(Counter.class); when(counter.getCount()).thenReturn(100L); reporter.report( this.<Gauge>map(), map("test.counter", counter), this.<Histogram>map(), this.<Meter>map(), this.<Timer>map()); verify(ganglia) .announce( "test.counter.count", "100", GMetricType.DOUBLE, "", GMetricSlope.BOTH, 60, 0, "test"); verifyNoMoreInteractions(ganglia); }
public static <E> E sample(Counter<E> counter) { double total = counter.totalCount(); double rand = random.nextDouble(); double sum = 0.0; if (total <= 0.0) { throw new RuntimeException("Non-positive counter total: " + total); } for (E key : counter.keySet()) { double count = counter.getCount(key); if (count < 0.0) { throw new RuntimeException("Negative count in counter: " + key + " => " + count); } double prob = count / total; sum += prob; if (rand < sum) { return key; } } throw new RuntimeException("Shouldn't Reach Here"); }
public Counter<String> getLogScoreCounter(LocalTrigramContext localTrigramContext) { int position = localTrigramContext.getPosition(); String word = localTrigramContext.getWords().get(position); Counter<String> tagCounter = unknownWordTags; if (wordsToTags.keySet().contains(word)) { tagCounter = wordsToTags.getCounter(word); } Set<String> allowedFollowingTags = allowedFollowingTags( tagCounter.keySet(), localTrigramContext.getPreviousPreviousTag(), localTrigramContext.getPreviousTag()); Counter<String> logScoreCounter = new Counter<String>(); for (String tag : tagCounter.keySet()) { double logScore = Math.log(tagCounter.getCount(tag)); if (!restrictTrigrams || allowedFollowingTags.isEmpty() || allowedFollowingTags.contains(tag)) logScoreCounter.setCount(tag, logScore); } return logScoreCounter; }
public synchronized int rPiece(byte[] bitfield) { Counter[] val = iterate(); Arrays.sort(val); Counter rare = finder(val, bitfield); return rare.getCount(); }