コード例 #1
0
ファイル: Redactor.java プロジェクト: ryannk/open-eval
 /**
  * Removes every {@code Relation} from the {@code RELATIONVIEW} in the list of text annotations.
  */
 private static List<TextAnnotation> removeRelationsFromPredicateArgumentView(
     List<TextAnnotation> uncleansedAnnotations) {
   List<String> relationExtractionViews = new ArrayList<>();
   relationExtractionViews.add(ViewNames.SENTENCE);
   relationExtractionViews.add(ViewNames.TOKENS);
   relationExtractionViews.add("RELATIONVIEW");
   List<TextAnnotation> textAnnotations =
       removeViews(uncleansedAnnotations, relationExtractionViews);
   for (TextAnnotation textAnnotation : textAnnotations) {
     Set<String> viewNames = textAnnotation.getAvailableViews();
     for (String viewName : viewNames) {
       View view = textAnnotation.getView(viewName);
       if (view instanceof PredicateArgumentView) {
         PredicateArgumentView predicateArgumentView = (PredicateArgumentView) view;
         predicateArgumentView.removeAllRelations();
         for (Constituent c : predicateArgumentView.getConstituents()) {
           predicateArgumentView.removeConstituent(c);
           int start = c.getStartSpan();
           int end = c.getEndSpan();
           view.addConstituent(new Constituent("", "RELATIONVIEW", textAnnotation, start, end));
         }
       }
     }
   }
   return textAnnotations;
 }
        @Override
        public List<Constituent> transform(Constituent c) {

          Constituent c1 = c.cloneForNewView(c.getViewName());
          new Relation("", c, c1, 0.0);
          return Collections.singletonList(c1);
        }
  private static Constituent getHead(Constituent c, TreeView dependency) {
    Constituent end = null;
    List<Constituent> constituentsCovering = dependency.getConstituentsCovering(c);
    for (Constituent d : constituentsCovering) {
      List<Relation> in = d.getIncomingRelations();
      if (in.size() == 0) {
        end = d;
        break;
      } else {
        Constituent parent = in.get(0).getSource();

        int parentToken = parent.getStartSpan();
        if (c.getStartSpan() <= parentToken && parentToken < c.getEndSpan()) continue;

        if (end == null) {
          end = d;
        } else if (end.getStartSpan() < d.getStartSpan()) {
          end = d;
        }
      }
    }

    Constituent c1;
    if (end == null) c1 = constituentsCovering.get(0).cloneForNewView("");
    else c1 = end.cloneForNewView("");

    return addPointerToSource(c, c1);
  }
        @Override
        public List<Constituent> transform(Constituent input) {

          TextAnnotation ta = input.getTextAnnotation();
          int tokenPosition = input.getStartSpan();
          TreeView dependency = (TreeView) ta.getView(ViewNames.DEPENDENCY);

          Constituent verbNode = dependency.getConstituentsCoveringToken(tokenPosition).get(0);
          boolean done = false;

          while (!done) {
            String pos = WordHelpers.getPOS(ta, verbNode.getStartSpan());

            if (POSUtils.isPOSVerb(pos)) {
              done = true;
            } else {
              List<Relation> incoming = verbNode.getIncomingRelations();
              if (incoming == null || incoming.size() == 0) {
                return new ArrayList<>();
              } else verbNode = incoming.get(0).getSource();
            }
          }

          return Collections.singletonList(addPointerToSource(input, verbNode));
        }
 @Override
 public List<Constituent> transform(Constituent input) {
   int tokenId = input.getStartSpan();
   if (tokenId > 0) {
     Constituent c =
         new Constituent("", "", input.getTextAnnotation(), tokenId - 1, tokenId);
     return Collections.singletonList(addPointerToSource(input, c));
   } else return new ArrayList<>();
 }
コード例 #6
0
 public Constituent getDependencyConstituentCoveringTokenId(Problem prob, int tokenId) {
   for (int i = 0; i <= 2; ++i) {
     for (Constituent cons : prob.dependency) {
       if (tokenId + i >= cons.getStartSpan() && tokenId + i < cons.getEndSpan()) {
         return cons;
       }
     }
   }
   return null;
 }
 @Override
 public List<Constituent> transform(Constituent input) {
   int tokenId = input.getEndSpan();
   TextAnnotation ta = input.getTextAnnotation();
   Sentence sentence = ta.getSentence(input.getSentenceId());
   if (tokenId < sentence.size()) {
     Constituent c = new Constituent("", "", ta, tokenId, tokenId + 1);
     return Collections.singletonList(addPointerToSource(input, c));
   } else return new ArrayList<>();
 }
 @Override
 public List<Constituent> transform(Constituent input) {
   return Collections.singletonList(
       new Constituent(
           "",
           "",
           input.getTextAnnotation(),
           input.getStartSpan(),
           input.getStartSpan() + 1));
 }
コード例 #9
0
 /**
  * Add an array of objects reporting View's Constituents' surface form and character offsets. May
  * make deserialization to TextAnnotation problematic, as the relevant methods deduce token
  * character offsets directly from list of token strings and raw text.
  *
  * @param fieldName name to give to this field
  * @param view view whose character offsets will be serialized
  * @param json Json object to which resulting array will be added
  */
 private static void writeTokenOffsets(String fieldName, View view, JsonObject json) {
   JsonArray offsetArray = new JsonArray();
   for (Constituent c : view.getConstituents()) {
     JsonObject cJ = new JsonObject();
     writeString(FORM, c.getSurfaceForm(), cJ);
     writeInt(STARTCHAROFFSET, c.getStartCharOffset(), cJ);
     writeInt(ENDCHAROFFSET, c.getEndCharOffset(), cJ);
     offsetArray.add(cJ);
   }
   json.add(fieldName, offsetArray);
 }
  @Override
  /**
   * This feature extractor assumes that the TOKEN View has been generated in the Constituents
   * TextAnnotation. It generate a feature for a window [-2, +2] of Forms (original text) for each
   * constituent.
   */
  public Set<Feature> getFeatures(Constituent c) throws EdisonException {
    TextAnnotation ta = c.getTextAnnotation();
    TOKENS = ta.getView(ViewNames.TOKENS);

    // We can assume that the constituent in this case is a Word(Token)
    int startspan = c.getStartSpan();
    int endspan = c.getEndSpan();
    // k is 3 since we need up to 3-grams
    int k = 3;
    int window = 2;

    // All our constituents are words(tokens)
    String[] forms = getWindowK(TOKENS, startspan, endspan, window);

    String id, value;
    String classifier = "WordConjunctionOneTwoThreeGramWindowTwo";
    Set<Feature> result = new LinkedHashSet<>();

    for (int j = 0; j < k; j++) {
      // k = 3, j goes from 0 to 2

      for (int i = 0; i < forms.length; i++) {
        // forms.length = 5, So i goes from 0 to 4, for each String in the forms array.

        StringBuilder f = new StringBuilder();

        // Starts with context = 0 and then increments context as long as it is below
        // the current value of j and is not out of index of the forms array.
        // This is basically creating a discrete feature for each combination of one, two
        // and three word combinations within [-2,2] window or words.
        for (int context = 0; context <= j && i + context < forms.length; context++) {
          // add a '_' between words to conjoin them together
          if (context != 0) {
            f.append("_");
          }
          f.append(forms[i + context]);
        }

        // 2 is the center object in the array so i should go from -2 to +2 (with 0 being
        // the center)
        // j is the size of the n-gram so it goes 1 to 3
        id = classifier + ":" + ((i - window) + "_" + (j + 1));
        value = "(" + (f.toString()) + ")";
        result.add(new DiscreteFeature(id + value));
      }
    }
    return result;
  }
コード例 #11
0
        @Override
        public List<Constituent> transform(Constituent input) {
          List<Constituent> list = new ArrayList<>();
          TextAnnotation ta = input.getTextAnnotation();

          for (int i = input.getStartSpan(); i < input.getEndSpan(); i++) {
            list.add(new Constituent("", "", ta, i, i + 1));
          }

          return list;
        }
コード例 #12
0
 public static int getChunkIndex(Problem prob, int tokenId) {
   for (int i = 0; i <= 2; ++i) {
     for (int j = 0; j < prob.chunks.size(); ++j) {
       Constituent cons = prob.chunks.get(j);
       if (tokenId + i >= cons.getStartSpan() && tokenId + i < cons.getEndSpan()) {
         return j;
       }
     }
   }
   return -1;
 }
コード例 #13
0
  private static List<Constituent> getGovernor(Constituent input, TreeView dependency) {
    List<Constituent> constituentsCovering = dependency.getConstituentsCovering(input);

    if (constituentsCovering.size() == 0) return new ArrayList<>();

    Constituent c = constituentsCovering.get(0);

    List<Relation> incomingRelations = c.getIncomingRelations();

    if (incomingRelations == null || incomingRelations.size() == 0) return new ArrayList<>();
    else
      return Collections.singletonList(
          addPointerToSource(input, incomingRelations.get(0).getSource()));
  }
コード例 #14
0
        @Override
        public List<Constituent> transform(Constituent input) {

          TextAnnotation ta = input.getTextAnnotation();

          return getGovernor(input, getDependencyView(ta, BERKELEY_DEPENDENCIES));
        }
コード例 #15
0
        @Override
        public List<Constituent> transform(Constituent input) {

          TreeView dependency = (TreeView) input.getTextAnnotation().getView(ViewNames.DEPENDENCY);

          return getObject(input, dependency, "obj");
        }
コード例 #16
0
        @Override
        public List<Constituent> transform(Constituent input) {

          TreeView dependency =
              (TreeView) input.getTextAnnotation().getView(ViewNames.DEPENDENCY_STANFORD);

          return getGovernor(input, dependency);
        }
コード例 #17
0
        @Override
        public List<Constituent> transform(Constituent input) {

          TextAnnotation ta = input.getTextAnnotation();

          TreeView dependency = getDependencyView(ta, STANFORD_DEPENDENCIES);

          return getGovernor(input, dependency);
        }
コード例 #18
0
        @Override
        public List<Constituent> transform(Constituent input) {

          List<Constituent> c = new ArrayList<>();
          for (Relation r : input.getOutgoingRelations()) {
            c.add(addPointerToSource(input, r.getTarget()));
          }
          return c;
        }
コード例 #19
0
 protected String getNERString() {
   List<Constituent> constituents = new ArrayList<>(view.getConstituents());
   Collections.sort(constituents, TextAnnotationUtilities.constituentStartComparator);
   StringBuilder sb = new StringBuilder();
   String text = textAnnotation.getText();
   int where = 0;
   for (Constituent c : constituents) {
     int start = c.getStartCharOffset();
     String startstring = text.substring(where, start);
     sb.append(startstring)
         .append("[")
         .append(c.getLabel())
         .append(" ")
         .append(c.getTokenizedSurfaceForm())
         .append(" ] ");
     where = c.getEndCharOffset();
   }
   return sb.toString();
 }
コード例 #20
0
ファイル: Redactor.java プロジェクト: ryannk/open-eval
 /** Removes the label from the NER_GOLD_EXTENT_SPAN. */
 private static List<TextAnnotation> removeLabelsForNER(List<TextAnnotation> cleansedAnnotations) {
   List<String> nerViews = new ArrayList<>();
   nerViews.add(ViewNames.SENTENCE);
   nerViews.add(ViewNames.TOKENS);
   nerViews.add("NER_GOLD_EXTENT_SPAN");
   List<TextAnnotation> textAnnotations = removeViews(cleansedAnnotations, nerViews);
   for (TextAnnotation textAnnotation : textAnnotations) {
     View view = textAnnotation.getView("NER_GOLD_EXTENT_SPAN");
     List<Constituent> constituents = view.getConstituents();
     for (Constituent c : constituents) {
       view.removeConstituent(c);
       int start = c.getStartSpan();
       int end = c.getEndSpan();
       view.addConstituent(
           new Constituent("", "NER_GOLD_EXTENT_SPAN", textAnnotation, start, end));
     }
     textAnnotation.addView(view.getViewName(), view);
   }
   return textAnnotations;
 }
  public void evaluate(ClassificationTester tester, View gold, View prediction) {
    super.cleanAttributes(gold, prediction);
    Set<IntPair> goldSpans = new HashSet<>();
    for (Constituent cons : gold.getConstituents()) {
      goldSpans.add(cons.getSpan());
    }

    Set<IntPair> predictedSpans = new HashSet<>();
    for (Constituent cons : prediction.getConstituents()) {
      predictedSpans.add(cons.getSpan());
    }

    Set<IntPair> spanIntersection = new HashSet<>(goldSpans);
    spanIntersection.retainAll(predictedSpans);

    tester.recordCount(
        "" /* label doesn't matter */,
        goldSpans.size(),
        predictedSpans.size(),
        spanIntersection.size());
  }
コード例 #22
0
  private static List<Constituent> getModifiers(Constituent input, TreeView dependency) {
    List<Constituent> constituentsCovering = dependency.getConstituentsCovering(input);

    if (constituentsCovering.size() == 0) return new ArrayList<>();

    Constituent c = constituentsCovering.get(0);

    List<Relation> outgoingRelations = c.getOutgoingRelations();

    if (outgoingRelations == null || outgoingRelations.size() == 0) return new ArrayList<>();
    else {

      for (Relation r : outgoingRelations) {
        if (r.getRelationName().contains("mod"))
          return Collections.singletonList(addPointerToSource(input, r.getTarget()));
      }

      return Collections.singletonList(
          addPointerToSource(input, outgoingRelations.get(0).getTarget()));
    }
  }
コード例 #23
0
 public Constituent getDependentVerb(Problem prob, QuantSpan qs) {
   Constituent result =
       getDependencyConstituentCoveringTokenId(
           prob, prob.ta.getTokenIdFromCharacterOffset(qs.start));
   if (result == null) {
     System.out.println(
         "Text : "
             + prob.question
             + " Token : "
             + prob.ta.getTokenIdFromCharacterOffset(qs.start));
     Tools.printCons(prob.dependency);
   }
   while (result != null) {
     if (result.getIncomingRelations().size() == 0) break;
     //			System.out.println(result.getIncomingRelations().get(0).getSource()+" --> "+result);
     result = result.getIncomingRelations().get(0).getSource();
     if (prob.posTags.get(result.getStartSpan()).getLabel().startsWith("VB")) {
       return result;
     }
   }
   return result;
 }
コード例 #24
0
        @Override
        public List<Constituent> transform(Constituent c) {
          TextAnnotation ta = c.getTextAnnotation();
          TreeView tree = (TreeView) ta.getView(ViewNames.PARSE_CHARNIAK);
          try {
            Constituent phrase = tree.getParsePhrase(c);
            int head = CollinsHeadFinder.getInstance().getHeadWordPosition(phrase);
            Constituent c1 = new Constituent("", "", ta, head, head + 1);

            return Collections.singletonList(addPointerToSource(c, c1));

          } catch (Exception e) {
            throw new RuntimeException(e);
          }
        }
コード例 #25
0
  private static void writeConstituent(Constituent c, JsonObject cJ) {
    writeString("label", c.getLabel(), cJ);

    if (c.getConstituentScore() != 0) writeDouble("score", c.getConstituentScore(), cJ);
    writeInt("start", c.getStartSpan(), cJ);
    writeInt("end", c.getEndSpan(), cJ);

    writeAttributes(c, cJ);
    Map<String, Double> labelsToScores = c.getLabelsToScores();

    if (null != labelsToScores) writeLabelsToScores(labelsToScores, cJ);
  }
コード例 #26
0
        @Override
        public List<Constituent> transform(Constituent c) {
          TextAnnotation ta = c.getTextAnnotation();
          int tokenPosition = c.getStartSpan();
          TreeView dependency = (TreeView) ta.getView(ViewNames.DEPENDENCY);

          Constituent prepositionDepConstituent =
              dependency.getConstituentsCoveringToken(tokenPosition).get(0);

          List<Relation> incomingRelations = prepositionDepConstituent.getIncomingRelations();

          List<Constituent> list = new ArrayList<>();
          if (incomingRelations != null && incomingRelations.size() > 0) {

            Constituent parent = incomingRelations.get(0).getSource();

            for (Relation out : parent.getOutgoingRelations()) {
              if (out == incomingRelations.get(0)) continue;

              String label = out.getRelationName();

              if (label.contains("prep")) {
                Constituent ppNode = out.getTarget();

                list.add(addPointerToSource(c, ppNode));

                // get the first child of the pp and add this
                List<Relation> ppOut = ppNode.getOutgoingRelations();

                if (ppOut != null && ppOut.size() != 0) {

                  Constituent child = ppOut.get(0).getTarget();
                  list.add(addPointerToSource(c, child));
                }
              }
            }
          }

          return list;
        }
コード例 #27
0
 public Constituent getRateUnit(Problem prob) {
   for (Constituent cons : connectedNPs) {
     if (cons.getSurfaceForm().toLowerCase().contains("each")) {
       return cons;
     }
   }
   for (Constituent cons : connectedNPs) {
     if (cons.getSurfaceForm().toLowerCase().contains("every")) {
       return cons;
     }
   }
   if (quantPhrase.getSurfaceForm().contains("each")
       || quantPhrase.getSurfaceForm().contains("every")) {
     return quantPhrase;
   }
   int chunkId = getChunkIndex(prob, quantPhrase.getStartSpan());
   if (chunkId + 2 < prob.chunks.size()
       && prob.chunks.get(chunkId + 1).getSurfaceForm().equals("per")) {
     return prob.chunks.get(chunkId + 2);
   }
   return null;
 }
コード例 #28
0
 public List<Constituent> getConnectedNPs(Problem prob) {
   List<Constituent> npList = new ArrayList<>();
   List<Constituent> npListQuantRemoved = new ArrayList<>();
   boolean onlyQuantityInSentence = true;
   int sentId = prob.ta.getSentenceFromToken(quantPhrase.getStartSpan()).getSentenceId();
   for (QuantSpan qs : prob.quantities) {
     int tokenId = prob.ta.getTokenIdFromCharacterOffset(qs.start);
     if (prob.ta.getSentenceFromToken(tokenId).getSentenceId() == sentId
         && !(quantPhrase.getStartSpan() <= tokenId && quantPhrase.getEndSpan() > tokenId)) {
       onlyQuantityInSentence = false;
       break;
     }
   }
   // Find NPs from children of verb
   if (verbPhrase != null) {
     List<Relation> relations = verbPhrase.getOutgoingRelations();
     for (Relation relation : relations) {
       if (!relation.getRelationName().equals("nsubj")) continue;
       Constituent dst = relation.getTarget();
       for (Constituent cons : prob.chunks) {
         if (cons.getStartSpan() <= dst.getStartSpan()
             && cons.getEndSpan() > dst.getStartSpan()
             && cons.getLabel().equals("NP")
             && !npList.contains(cons)) {
           npList.add(cons);
           subject = cons;
           break;
         }
       }
     }
   }
   // Find NPs from PP NP connection
   int quantPhraseId = getChunkIndex(prob, quantPhrase.getStartSpan());
   if (quantPhraseId + 2 < prob.chunks.size()
       && !prob.chunks.get(quantPhraseId + 1).getSurfaceForm().trim().equals("of")
       && prob.chunks.get(quantPhraseId + 1).getLabel().equals("PP")
       && prob.chunks.get(quantPhraseId + 2).getLabel().equals("NP")
       && !npList.contains(prob.chunks.get(quantPhraseId + 2))) {
     npList.add(prob.chunks.get(quantPhraseId + 2));
   }
   if (quantPhraseId - 2 >= 0
       && prob.chunks.get(quantPhraseId - 1).getLabel().equals("PP")
       && prob.chunks.get(quantPhraseId - 2).getLabel().equals("NP")
       && !npList.contains(prob.chunks.get(quantPhraseId - 2))) {
     npList.add(prob.chunks.get(quantPhraseId - 2));
   }
   // Get preceding NP
   if (quantPhraseId - 1 >= 0
       && prob.chunks.get(quantPhraseId - 1).getLabel().equals("NP")
       && !prob.posTags
           .get(prob.chunks.get(quantPhraseId - 1).getEndSpan())
           .getLabel()
           .equals("CC")
       && !npList.contains(prob.chunks.get(quantPhraseId - 1))) {
     npList.add(prob.chunks.get(quantPhraseId - 1));
   }
   // Get succeeding NP
   if (quantPhraseId + 1 < prob.chunks.size()
       && prob.chunks.get(quantPhraseId + 1).getLabel().equals("NP")
       && !prob.posTags.get(prob.chunks.get(quantPhraseId).getEndSpan()).getLabel().equals("CC")
       && !npList.contains(prob.chunks.get(quantPhraseId + 1))) {
     npList.add(prob.chunks.get(quantPhraseId + 1));
   }
   //		 If only quantity in sentence, all NPs are connected
   if (onlyQuantityInSentence) {
     for (int i = 0; i < prob.chunks.size(); ++i) {
       Constituent cons = prob.chunks.get(i);
       if (cons.getSentenceId() == sentId
           && (i > quantPhraseId + 2 || i < quantPhraseId - 2)
           && !npList.contains(cons)
           && cons.getLabel().equals("NP")) {
         npList.add(cons);
       }
     }
   }
   // Remove quantity phrases from npList
   for (Constituent cons : npList) {
     boolean allow = true;
     for (QuantSpan qs : prob.quantities) {
       int index = prob.ta.getTokenIdFromCharacterOffset(qs.start);
       if (index >= cons.getStartSpan() && index < cons.getEndSpan()) {
         allow = false;
         break;
       }
     }
     if (allow) {
       npListQuantRemoved.add(cons);
     }
   }
   return npListQuantRemoved;
 }
コード例 #29
0
  @Override
  public Set<Feature> getFeatures(Constituent c) throws EdisonException {
    TextAnnotation ta = c.getTextAnnotation();
    Set<Feature> features = new LinkedHashSet<>();
    TreeView parse = (TreeView) ta.getView(dependencyViewName);
    // get equivalent of c in the parse view
    Constituent c2 = parse.getConstituentsCoveringToken(c.getStartSpan()).get(0);
    List<Relation> incomingRelations = c2.getIncomingRelations();
    if (incomingRelations.size() > 0) {
      Constituent c1 =
          parse
              .getConstituentsCoveringToken(incomingRelations.get(0).getSource().getStartSpan())
              .get(0);

      Pair<List<Constituent>, List<Constituent>> paths =
          PathFeatureHelper.getPathsToCommonAncestor(c1, c2, 400);

      List<String> path = new ArrayList<>();
      List<String> pos = new ArrayList<>();

      for (int i = 0; i < paths.getFirst().size() - 1; i++) {
        Constituent cc = paths.getFirst().get(i);
        path.add(
            cc.getIncomingRelations().get(0).getRelationName() + PathFeatureHelper.PATH_UP_STRING);

        pos.add(
            WordHelpers.getPOS(ta, cc.getStartSpan())
                + ":"
                + cc.getIncomingRelations().get(0).getRelationName()
                + PathFeatureHelper.PATH_UP_STRING);
      }

      Constituent top = paths.getFirst().get(paths.getFirst().size() - 1);

      pos.add(WordHelpers.getPOS(ta, top.getStartSpan()) + ":*");
      path.add("*");

      if (paths.getSecond().size() > 1) {
        for (int i = paths.getSecond().size() - 2; i >= 0; i--) {
          Constituent cc = paths.getSecond().get(i);

          pos.add(
              WordHelpers.getPOS(ta, cc.getStartSpan()) + ":" + PathFeatureHelper.PATH_DOWN_STRING);
          path.add(PathFeatureHelper.PATH_DOWN_STRING);
        }
      }

      features.addAll(getNgrams(path, ""));
      features.addAll(getNgrams(pos, "pos"));
    }
    return features;
  }
コード例 #30
0
 public Pair<String, Constituent> getUnit(Problem prob, int quantIndex) {
   String unit = "";
   int tokenId = prob.ta.getTokenIdFromCharacterOffset(prob.quantities.get(quantIndex).start);
   int quantPhraseId = getChunkIndex(prob, tokenId);
   Constituent quantPhrase = prob.chunks.get(quantPhraseId);
   // Detect cases like 4 red and 6 blue balls
   int numQuantInChunk = 0;
   for (QuantSpan qs : prob.quantities) {
     int index = prob.ta.getTokenIdFromCharacterOffset(qs.start);
     if (index >= quantPhrase.getStartSpan() && index < quantPhrase.getEndSpan()) {
       numQuantInChunk++;
     }
   }
   int start = quantPhrase.getStartSpan();
   int end = quantPhrase.getEndSpan();
   boolean addEndNoun = false;
   if (numQuantInChunk > 1) {
     for (int i = quantPhrase.getStartSpan(); i < quantPhrase.getEndSpan(); ++i) {
       if (prob.posTags.get(i).getLabel().equals("CC")) {
         if (tokenId < i) {
           end = i;
           addEndNoun = true;
         } else {
           start = i + 1;
         }
         break;
       }
     }
   }
   for (int i = start; i < end; ++i) {
     if (i != tokenId) {
       if (prob.ta.getToken(i).equals("$")) {
         unit += "dollar ";
       } else {
         unit += prob.lemmas.get(i) + " ";
       }
     }
   }
   // Connecting disconnected units, as in, 5 red and 6 green apples
   if (addEndNoun
       && quantPhrase.getEndSpan() <= prob.ta.size()
       && prob.posTags.get(quantPhrase.getEndSpan() - 1).getLabel().startsWith("N")) {
     unit += prob.lemmas.get(quantPhrase.getEndSpan() - 1) + " ";
   }
   // Unit from neighboring phrases
   if (quantPhraseId + 2 < prob.chunks.size()
       && prob.chunks.get(quantPhraseId + 1).getSurfaceForm().trim().equals("of")
       && prob.chunks.get(quantPhraseId + 2).getLabel().equals("NP")) {
     Constituent cons = prob.chunks.get(quantPhraseId + 2);
     for (int j = cons.getStartSpan(); j < cons.getEndSpan(); ++j) {
       unit += prob.lemmas.get(j) + " ";
     }
     quantPhraseId += 2;
   }
   return new Pair<String, Constituent>(unit, quantPhrase);
 }