Beispiel #1
0
 /**
  * Builds a Trellis over a sentence, by starting at the state State, and advancing through all
  * legal extensions of each state already in the trellis. You should not have to modify this
  * code (or even read it, really).
  */
 private Trellis<State> buildTrellis(List<String> sentence) {
   Trellis<State> trellis = new Trellis<State>();
   trellis.setStartState(State.getStartState());
   State stopState = State.getStopState(sentence.size() + 2);
   trellis.setStopState(stopState);
   Set<State> states = Collections.singleton(State.getStartState());
   for (int position = 0; position <= sentence.size() + 1; position++) {
     Set<State> nextStates = new HashSet<State>();
     for (State state : states) {
       if (state.equals(stopState)) continue;
       LocalTrigramContext localTrigramContext =
           new LocalTrigramContext(
               sentence, position, state.getPreviousPreviousTag(), state.getPreviousTag());
       Counter<String> tagScores = localTrigramScorer.getLogScoreCounter(localTrigramContext);
       for (String tag : tagScores.keySet()) {
         double score = tagScores.getCount(tag);
         State nextState = state.getNextState(tag);
         trellis.setTransitionCount(state, nextState, score);
         nextStates.add(nextState);
       }
     }
     //        System.out.println("States: "+nextStates);
     states = nextStates;
   }
   return trellis;
 }
Beispiel #2
0
  public void act() // sean
      {

    if (!played) {
      burningSteppes.playLoop();
      played = !played;
    }
    // makeSmokeFireball();
    counterDelay++;
    if (Greenfoot.isKeyDown("h") && delay > 10) {
      clickSound.play();
      Menu menu = new Menu(getThisWorld());
      Greenfoot.setWorld(menu);
      delay = 0;
    }
    if (getObjects(Ninja.class).size() != 0 && counterDelay >= 10) {
      healthCounter.setValue(ninja.getNINJAHP());
      shurikenCounter.setValue(ninja.getSHURIKENNUMBER());
      powerCounter.setValue(ninja.getPOWERBAR());
      checkDoor();
      counterDelay -= 10;
      /**/
      // TEMPORAY FUNCTIONS FOR HAYDEN TO CHANGE LEVELS TO MAKE THEM /**/

      /**/
      // TEMPORAY FUNCTIONS FOR HAYDEN TO CHANGE LEVELS TO MAKE THEM /**/
    }
    delay++;
    fireballDelay++;
  }
Beispiel #3
0
 public void add(int r, int c, double v) {
   Counter newRow = new Counter();
   newRow.add(c, v);
   this.addRow(r, newRow);
   rows.add(r);
   cols.add(c);
 }
Beispiel #4
0
 /*
  * Takes a set of sketch nodes, and returns an ArrayList<Integer> such that
  * arr.get(i) gives the index of the sketch node that node i is closest too.
  *
  * Need to work the return values a little bit. Make a proper data
  * structure.
  */
 public ArrayList<ArrayList<Integer>> distSketch(int len, Counter sketchNodes) {
   ArrayList<Integer> closestIndex = new ArrayList<Integer>();
   for (int i = 0; i < len; i++) closestIndex.set(i, -1);
   ArrayList<Double> closestDist = new ArrayList<Double>();
   for (int i = 0; i < len; i++) closestDist.set(i, Double.MAX_VALUE);
   ArrayList<ArrayList<Integer>> sketchReverseIndex = new ArrayList<ArrayList<Integer>>();
   for (int index : sketchNodes.keySet()) {
     Counter distances = this.bfs(index);
     for (int j = 0; j < len; j++) {
       double curDist = closestDist.get(j);
       double dist = distances.getPath(index);
       if (dist < curDist) {
         closestIndex.set(j, index);
       }
     }
     sketchReverseIndex.add(new ArrayList<Integer>());
   }
   for (int j = 0; j < len; j++) {
     int closest = closestIndex.get(j);
     sketchReverseIndex.get(closest).add(j);
   }
   // Return sketchReverseIndex, closestIndex forward index, and index
   // correspondence bimap
   return sketchReverseIndex;
 }
 private Word findTrg(Word tg, int pos2, BasicChunk bs) {
   int pos1 = tg.pos;
   if (pos2 - pos1 > 10) {
     return null;
   }
   Chunk c1 = bs.getChunk(pos1);
   Chunk c2 = bs.getChunk(pos2);
   int begin = c1.begin;
   int end = c2.end;
   for (Chunk c : bs.chunkList) {
     if (c.begin >= begin && c.end <= end) {
       for (Word w : c.trigs) {
         if (!validTG.contains(w) && w.pos > pos1) {
           return w;
         } else if (validTG.contains(w) && w.pos > pos1) {
           String key = tg.word + tg.pos_tag;
           Map<String, Counter> ct = sharedTG.get(key);
           if (ct == null) {
             ct = new HashMap<String, Counter>();
             sharedTG.put(key, ct);
           }
           Counter count = ct.get(w.word + w.pos_tag);
           if (count == null) {
             count = new Counter(1);
             ct.put(w.word + w.pos_tag, count);
           } else {
             count.inc();
           }
         }
       }
     }
   }
   return null;
 }
Beispiel #6
0
 public double getCount(K token) {
   if (!lm.keySet().contains(token)) {
     System.err.println(lm.keySet().size());
     throw new RuntimeException("token not in keyset");
   }
   return lm.getCount(token);
 }
 private void tallyTree(
     Tree<String> tree,
     Counter<String> symbolCounter,
     Counter<UnaryRule> unaryRuleCounter,
     Counter<BinaryRule> binaryRuleCounter) {
   if (tree.isLeaf()) return;
   if (tree.isPreTerminal()) return;
   if (tree.getChildren().size() == 1) {
     UnaryRule unaryRule = makeUnaryRule(tree);
     symbolCounter.incrementCount(tree.getLabel(), 1.0);
     unaryRuleCounter.incrementCount(unaryRule, 1.0);
   }
   if (tree.getChildren().size() == 2) {
     BinaryRule binaryRule = makeBinaryRule(tree);
     symbolCounter.incrementCount(tree.getLabel(), 1.0);
     binaryRuleCounter.incrementCount(binaryRule, 1.0);
   }
   if (tree.getChildren().size() < 1 || tree.getChildren().size() > 2) {
     throw new RuntimeException(
         "Attempted to construct a Grammar with an illegal tree: " + tree);
   }
   for (Tree<String> child : tree.getChildren()) {
     tallyTree(child, symbolCounter, unaryRuleCounter, binaryRuleCounter);
   }
 }
Beispiel #8
0
 /*
  * Matrix mult but with min-plus, and iterative. Each min-plus operation
  * that changes the path inserts it into a new queue
  */
 public SparseMatrix apsp() {
   SparseMatrix shortestPaths = new SparseMatrix(this);
   SparseMatrix currentPairs = new SparseMatrix(this.rowDim, this.colDim);
   SparseMatrix newPairs = new SparseMatrix(this.rowDim, this.colDim);
   newPairs = new SparseMatrix(this);
   for (int d = 0; d < this.rowDim; d++) {
     shortestPaths.set(d, d, 0.0);
   }
   for (int d = 0; d < this.rowDim; d++) {
     newPairs.set(d, d, 0.0);
   }
   while (!newPairs.isEmpty()) {
     currentPairs = new SparseMatrix(newPairs);
     newPairs = new SparseMatrix(this.rowDim, this.colDim);
     for (int r : currentPairs.rows) {
       Counter row = currentPairs.getRow(r);
       for (int c : row.keySet()) {
         Counter oRow = this.getRow(c);
         for (int oc : oRow.keySet()) {
           double pathLength = currentPairs.get(r, c) + oRow.get(oc);
           if (pathLength < shortestPaths.getPath(r, oc)) {
             newPairs.set(r, oc, pathLength);
             shortestPaths.set(r, oc, pathLength);
           }
         }
       }
     }
   }
   return shortestPaths;
 }
	  private double getDiceCoefficient(String f, String e) {
		  double intersection = collocationCountSentences.getCount(f,e);
		  double cardinalityF = fCountSentences.getCount(f);
		  double cardinalityE = eCountSentences.getCount(e);
		  
		  double dice = 2*intersection / (cardinalityF + cardinalityE);
		  return dice;
	  }
Beispiel #10
0
 public void removeEntries(SparseMatrix redundant) {
   for (int r : redundant.getRows()) {
     Counter row = redundant.getRow(r);
     for (int c : row.keySet()) {
       this.remove(r, c);
     }
   }
 }
Beispiel #11
0
 public SparseMatrix multiply(double f) {
   SparseMatrix multMat = new SparseMatrix(this.rowDim, this.colDim);
   for (int r : this.rows) {
     Counter row = this.getRow(r);
     multMat.addRow(r, row.multiplyImmutable(f));
   }
   return multMat;
 }
 /** {@inheritDoc} */
 @Override
 public synchronized void incrAllCounters(AbstractCounters<Counter, CounterGroup> other) {
   for (CounterGroup group : other) {
     for (Counter counter : group) {
       findCounter(group.getName(), counter.getName()).increment(counter.getValue());
     }
   }
 }
Beispiel #13
0
 public Counter getCol(int c) {
   Counter col = new Counter();
   for (int r : rows) {
     if (this.getRow(r).containsKey(c)) {
       col.put(r, this.get(r, c));
     }
   }
   return col;
 }
Beispiel #14
0
 public void set(int r, int c, double v) {
   Counter row = this.getRow(r);
   if (row.isEmpty()) {
     mat.put(r, row);
   }
   row.put(c, v);
   rows.add(r);
   cols.add(c);
 }
 private void tallyTagging(String word, String tag) {
   if (!isKnown(word)) {
     totalWordTypes += 1.0;
     typeTagCounter.incrementCount(tag, 1.0);
   }
   totalTokens += 1.0;
   tagCounter.incrementCount(tag, 1.0);
   wordCounter.incrementCount(word, 1.0);
   wordToTagCounters.incrementCount(word, tag, 1.0);
 }
Beispiel #16
0
 public Counter harmonicAvg(Counter vector) {
   Counter currentVec = new Counter();
   Counter newVec = new Counter(vector);
   double dist = currentVec.dist(newVec);
   while (dist > 0.00001) {
     currentVec = new Counter(newVec);
     dist = currentVec.dist(newVec);
   }
   return currentVec;
 }
Beispiel #17
0
  /**
   * GT smoothing with least squares interpolation. This follows the procedure in Jurafsky and
   * Martin sect. 4.5.3.
   */
  public void smoothAndNormalize() {
    Counter<Integer> cntCounter = new Counter<Integer>();
    for (K tok : lm.keySet()) {
      int cnt = (int) lm.getCount(tok);
      cntCounter.incrementCount(cnt);
    }

    final double[] coeffs = runLogSpaceRegression(cntCounter);

    UNK_PROB = cntCounter.getCount(1) / lm.totalCount();

    for (K tok : lm.keySet()) {
      double tokCnt = lm.getCount(tok);
      if (tokCnt <= unkCutoff) // Treat as unknown
      unkTokens.add(tok);
      if (tokCnt <= kCutoff) { // Smooth
        double cSmooth = katzEstimate(cntCounter, tokCnt, coeffs);
        lm.setCount(tok, cSmooth);
      }
    }

    // Normalize
    // Counters.normalize(lm);
    // MY COUNTER IS ALWAYS NORMALIZED AND AWESOME
  }
Beispiel #18
0
 public SparseMatrix transpose() {
   SparseMatrix transp = new SparseMatrix(this.rowDim, this.colDim);
   for (int r : rows) {
     Counter row = this.getRow(r);
     for (int c : row.keySet()) {
       double v = row.get(c);
       transp.set(c, r, v);
     }
   }
   return transp;
 }
Beispiel #19
0
 public SparseMatrix makeLaplacian() {
   SparseMatrix laplacian = new SparseMatrix(this.rowDim, this.colDim);
   for (int r : this.getRows()) {
     Counter row = this.getRow(r);
     laplacian.set(r, r, row.sum());
     for (int c : row.keySet()) {
       laplacian.set(r, c, -1 * row.get(c));
     }
   }
   return laplacian;
 }
Beispiel #20
0
 public List<S> getBestPath(Trellis<S> trellis) {
   List<S> states = new ArrayList<S>();
   S currentState = trellis.getStartState();
   states.add(currentState);
   while (!currentState.equals(trellis.getEndState())) {
     Counter<S> transitions = trellis.getForwardTransitions(currentState);
     S nextState = transitions.argMax();
     states.add(nextState);
     currentState = nextState;
   }
   return states;
 }
Beispiel #21
0
 public void addRow(int r, Counter other) {
   // System.out.println("MSG: added row "+r);
   Counter row = this.getRow(r);
   if (row.isEmpty()) {
     mat.put(r, row);
     rows.add(r);
   }
   for (int c : other.keySet()) {
     cols.add(c);
   }
   row.addAll(other);
 }
 /* Returns a smoothed estimate of P(word|tag) */
 public double scoreTagging(String word, String tag) {
   double p_tag = tagCounter.getCount(tag) / totalTokens;
   double c_word = wordCounter.getCount(word);
   double c_tag_and_word = wordToTagCounters.getCount(word, tag);
   if (c_word < 10) { // rare or unknown
     c_word += 1.0;
     c_tag_and_word += typeTagCounter.getCount(tag) / totalWordTypes;
   }
   double p_word = (1.0 + c_word) / (totalTokens + totalWordTypes);
   double p_tag_given_word = c_tag_and_word / c_word;
   return p_tag_given_word / p_tag * p_word;
 }
Beispiel #23
0
  private double[] runLogSpaceRegression(Counter<Integer> cntCounter) {
    SimpleRegression reg = new SimpleRegression();

    for (int cnt : cntCounter.keySet()) {
      reg.addData(cnt, Math.log(cntCounter.getCount(cnt)));
    }

    // System.out.println(reg.getIntercept());
    // System.out.println(reg.getSlope());
    // System.out.println(regression.getSlopeStdErr());

    double[] coeffs = new double[] {reg.getIntercept(), reg.getSlope()};

    return coeffs;
  }
 private void add2Map(String pos, String tg, String type) {
   Map<String, Counter> ct = TGCount.get(tg);
   if (ct == null) {
     ct = new HashMap<String, Counter>();
     TGCount.put(tg, ct);
   }
   String key = pos + type;
   Counter c = ct.get(key);
   if (c == null) {
     c = new Counter(1);
     ct.put(key, c);
   } else {
     c.inc();
   }
 }
Beispiel #25
0
  private double katzEstimate(Counter<Integer> cnt, double c, double[] coeffs) {
    double nC = cnt.getCount((int) c);
    double nC1 = cnt.getCount(((int) c) + 1);
    if (nC1 == 0.0) nC1 = Math.exp(coeffs[0] + (coeffs[1] * (c + 1.0)));

    double n1 = cnt.getCount(1);
    double nK1 = cnt.getCount(((int) kCutoff) + 1);
    if (nK1 == 0.0) nK1 = Math.exp(coeffs[0] + (coeffs[1] * (kCutoff + 1.0)));

    double kTerm = (kCutoff + 1.0) * (nK1 / n1);
    double cTerm = (c + 1.0) * (nC1 / nC);

    double cSmooth = (cTerm - (c * kTerm)) / (1.0 - kTerm);

    return cSmooth;
  }
	  public Alignment alignSentencePair(SentencePair sentencePair) {
		  Alignment alignment = new Alignment();
	      List<String> frenchWords = sentencePair.getFrenchWords();
	      List<String> englishWords = sentencePair.getEnglishWords();     
	      int numFrenchWords = frenchWords.size();
	      int numEnglishWords = englishWords.size();
	      
	      for (int frenchPosition = 0; frenchPosition < numFrenchWords; frenchPosition++) {
	    	  String f = frenchWords.get(frenchPosition);
	    	  int englishMaxPosition = frenchPosition;
	    	  if (englishMaxPosition >= numEnglishWords)
	    		  englishMaxPosition = -1; // map French word to BASELINE if c(f,e) = 0 for all English words
	    	  double maxConditionalProb = 0;
	    	  for (int englishPosition = 0; englishPosition < numEnglishWords; englishPosition++) {
	    		  String e = englishWords.get(englishPosition);
	    		  double conditionalGivenEnglish = collocationCounts.getCount(f, e) / (eCounts.getCount(e));
	    		  if (conditionalGivenEnglish > maxConditionalProb) {
	    			  maxConditionalProb = conditionalGivenEnglish;
	    			  englishMaxPosition = englishPosition;
	    		  }
	    	  }	
	    	  alignment.addAlignment(englishMaxPosition, frenchPosition, true);
	      }
		  return alignment;
	  }
Beispiel #27
0
 /**
  * Scores a tagging for a sentence. Note that a tag sequence not accepted by the markov process
  * should receive a log score of Double.NEGATIVE_INFINITY.
  */
 public double scoreTagging(TaggedSentence taggedSentence) {
   double logScore = 0.0;
   List<LabeledLocalTrigramContext> labeledLocalTrigramContexts =
       extractLabeledLocalTrigramContexts(taggedSentence);
   for (LabeledLocalTrigramContext labeledLocalTrigramContext : labeledLocalTrigramContexts) {
     Counter<String> logScoreCounter =
         localTrigramScorer.getLogScoreCounter(labeledLocalTrigramContext);
     String currentTag = labeledLocalTrigramContext.getCurrentTag();
     if (logScoreCounter.containsKey(currentTag)) {
       logScore += logScoreCounter.getCount(currentTag);
     } else {
       logScore += Double.NEGATIVE_INFINITY;
     }
   }
   return logScore;
 }
	  private void trainCounters() {
		  for (SentencePair sentencePair : trainingSentencePairs) {
			  List<String> frenchWords = sentencePair.getFrenchWords();
		      List<String> englishWords = sentencePair.getEnglishWords();
		      Set<String> frenchSet = new HashSet<String>(frenchWords);
		      Set<String> englishSet = new HashSet<String>(englishWords);
		      
		      fCountSentences.incrementAll(frenchSet, 1.0); 
		      eCountSentences.incrementAll(englishSet, 1.0);
		      
		      for (String f: frenchSet) {
		    	  for (String e: englishSet)
		    		  collocationCountSentences.incrementCount(f, e, 1.0);
		      }
		  }
		  System.out.println("Trained!");
	  }
Beispiel #29
0
  public Counter multiply(Counter vector) {
    Counter resVec = new Counter();

    //		for(int r: vector.keySet()){
    //			Counter row = this.getRow(r);
    //			double sum = 0;
    //			for(int c: row.keySet()){
    ////				resVec.add(r, row.get(c)*vector.get(c));
    //				sum += row.get(c)*vector.get(c);
    //			}
    //			resVec.add(r, sum);
    //		}

    for (int r : rows) {
      resVec.set(r, this.getRow(r).dot(vector));
    }
    return resVec;
  }
Beispiel #30
0
 public SparseMatrix stochasticizeRows() {
   SparseMatrix stochasticMat = new SparseMatrix(this.rowDim, this.colDim);
   double[] rowSums = new double[this.rowDim];
   for (int r : this.rows) {
     Counter row = this.getRow(r);
     for (int c : row.keySet()) {
       rowSums[r] += row.get(c);
     }
   }
   for (int r : this.rows) {
     Counter row = this.getRow(r);
     for (int c : row.keySet()) {
       double value = 0;
       if (rowSums[r] != 0) {
         //				if(true){
         value = this.get(r, c) / rowSums[r];
       }
       stochasticMat.set(r, c, value);
     }
   }
   return stochasticMat;
 }