示例#1
0
 /*
  * 停用词过滤并且修正词性
  */
 public static List<Term> modifResult(List<Term> all) {
   List<Term> result = new ArrayList<Term>();
   try {
     for (Term term : all) {
       if (FILTER.size() > 0
           && (FILTER.contains(term.getName())
               || (isTag && FILTER.contains(TAG + term.natrue().natureStr)))) {
         continue;
       }
       // 添加对正则停用词的支持
       if ((stopwordPattern != null) && stopwordPattern.matcher(term.getName()).matches()) {
         continue;
       }
       String[] params = UserDefineLibrary.getParams(term.getName());
       if (params != null) {
         term.setNature(new Nature(params[0]));
       }
       result.add(term);
     }
   } catch (Exception e) {
     MyStaticValue.LIBRARYLOG.warn(
         "FilterStopWord.updateDic can not be null , "
             + "you must use set FilterStopWord.setUpdateDic(map) or use method set map");
   }
   return result;
 }
  @Test
  public void test() {
    String newWord = "爸爸去哪儿";
    String nature = "aaaaa";
    String str = "上海电力2012年财务报表如下怎爸爸去哪儿么办";

    // 增加新词
    UserDefineLibrary.insertWord(newWord, nature, 1000);

    List<Term> parse = ToAnalysis.parse(str);
    HashMap<String, Term> hs = new HashMap<String, Term>();
    for (Term term : parse) {
      hs.put(term.getName(), term);
    }

    Assert.assertTrue(hs.containsKey(newWord));

    Assert.assertEquals(hs.get(newWord).natrue().natureStr, nature);

    // 删除词
    UserDefineLibrary.removeWord(newWord);
    parse = ToAnalysis.parse(str);
    hs = new HashMap<String, Term>();
    for (Term term : parse) {
      hs.put(term.getName(), term);
    }

    Assert.assertTrue(!hs.containsKey(newWord));
  }
示例#3
0
  /*
   * 修正词性
   */
  public static List<Term> updateNature(List<Term> all, Forest... forests) {

    if (forests == null) {
      if (UserDefineLibrary.FOREST != null) {
        forests = new Forest[] {UserDefineLibrary.FOREST};
      } else {
        return all;
      }
    }

    List<Term> result = new ArrayList<Term>();

    for (Term term : all) {

      // 添加对正则停用词的支持
      if ((stopwordPattern != null) && stopwordPattern.matcher(term.getName()).matches()) {
        continue;
      }
      for (Forest forest : forests) {
        String[] params = UserDefineLibrary.getParams(forest, term.getName());
        if (params != null) {
          term.setNature(new Nature(params[0]));
        }
      }
      result.add(term);
    }

    return result;
  }
示例#4
0
 public static void main(String[] args) {
   Term from = new Term("阿", 0, new TermNatures(TermNature.NULL));
   from.getTermNatures().id = InitDictionary.getWordId(from.getName());
   Term to = new Term("全国", 0, new TermNatures(TermNature.NULL));
   to.getTermNatures().id = InitDictionary.getWordId(to.getName());
   System.out.println(getTwoWordFreq(from, to));
 }
  @Override
  public EmotionResponse emotionJudge(EmotionRequest req) throws TException {
    long s1 = System.currentTimeMillis();
    String text = req.getText();
    List<Term> terms = ToAnalysis.parse(text);
    List<String> words = new ArrayList<>();
    for (Term term : terms) {
      words.add(term.getName());
    }
    //        logger.info("{} is cut into {}", text, words);

    double score = fbsc.classify(words);
    EmotionResponse response = new EmotionResponse();
    response.setReq(req);
    response.setScore(score);
    int emotion = 0;
    if (score > 0.5) {
      emotion = 1;
    } else if (score < -0.5) {
      emotion = 2;
    } else {
      emotion = 3;
    }
    response.setEmotion(emotion);
    long s2 = System.currentTimeMillis();
    logger.info("emotion: {}, used:{}ms", emotion, (s2 - s1));
    return response;
  }
示例#6
0
  public List<String> cut(String text) {
    List<Term> terms = ToAnalysis.parse(text);
    List<String> array = new ArrayList<>();
    StringBuilder sbDebug = new StringBuilder();
    for (Term term : terms) {
      String word = term.getName().trim();
      if (word.length() == 0) {
        continue;
      }
      //            System.out.println(word);
      if (filterSingleWord) { // 过滤单字
        if (word.length() < 2) {
          continue;
        }
      }
      if (filterStopWords) {
        if (stopWordsSet.contains(word)) {
          continue;
        }
      }
      Matcher matcher = pureNum.matcher(word); // 含数字的词都不要
      if (matcher.find()) {
        continue;
      }
      array.add(word);

      if (debug) {
        sbDebug.append(word + ",");
      }
    }
    if (debug) {
      logger.info("cut {} into {}", text, sbDebug.toString());
    }
    return array;
  }
示例#7
0
 public void test() {
   List<Term> terms = ToAnalysis.parse(line);
   for (Term term : terms) {
     String name = term.getName();
     System.out.println(name);
   }
 }
  /**
   * 数字+数字合并,zheng
   *
   * @param terms
   */
  public static void recognition(Term[] terms) {
    int length = terms.length - 1;
    Term from = null;
    Term to = null;
    Term temp = null;
    for (int i = 0; i < length; i++) {
      if (terms[i] == null) {
        continue;
      } else if (".".equals(terms[i].getName())) {
        // 如果是.前后都为数字进行特殊处理
        to = terms[i].getTo();
        from = terms[i].getFrom();
        if (from.getTermNatures().numAttr.flag && to.getTermNatures().numAttr.flag) {
          from.setName(from.getName() + "." + to.getName());
          TermUtil.termLink(from, to.getTo());
          terms[to.getOffe()] = null;
          terms[i] = null;
          i = from.getOffe() - 1;
        }
        continue;
      } else if (!terms[i].getTermNatures().numAttr.flag) {
        continue;
      }

      temp = terms[i];
      // 将所有的数字合并
      while ((temp = temp.getTo()).getTermNatures().numAttr.flag) {
        terms[i].setName(terms[i].getName() + temp.getName());
      }
      // 如果是数字结尾
      if (temp.getTermNatures().numAttr.numEndFreq > 0) {
        terms[i].setName(terms[i].getName() + temp.getName());
        temp = temp.getTo();
      }

      // 如果不等,说明terms[i]发生了改变
      if (terms[i].getTo() != temp) {
        TermUtil.termLink(terms[i], temp);
        // 将中间无用元素设置为null
        for (int j = i + 1; j < temp.getOffe(); j++) {
          terms[j] = null;
        }
        i = temp.getOffe() - 1;
      }
    }
  }
 public List<NewWord> getNewWords() {
   // TODO Auto-generated method stub
   List<NewWord> all = new ArrayList<NewWord>();
   List<Term> termList = recogntion_();
   for (Term term2 : termList) {
     all.add(new NewWord(term2.getName(), TermNatures.NR, term2.selfScore, 1));
   }
   return all;
 }
  @Override
  public HashMap<String, TermScore> tokenizeTerm(String input_str) {
    tokens = ToAnalysis.parse(input_str);
    token_iterator = tokens.listIterator();

    HashMap<String, TermScore> hash = new HashMap<String, TermScore>();
    while (token_iterator.hasNext()) {
      Term term = token_iterator.next();
      if (hash.get(term.getName()) == null)
        hash.put(term.getName(), new TermScore(term.getName(), 0));
      else {
        TermScore exist_term = hash.get(term.getName());
        int new_score = exist_term.getScore() + 1;
        exist_term.setScore(new_score);
        hash.put(term.getName(), exist_term);
      }
    }

    return hash;
  }
示例#11
0
 public String segAnsi(String text) {
   String segs = "";
   List<Term> parse2 = ToAnalysis.parse(text);
   StringBuilder sb = new StringBuilder();
   for (Term term : parse2) {
     sb.append(term.getName());
     sb.append(" ");
   }
   segs = sb.toString();
   segs = segs.trim();
   return segs;
 }
 /*
  * 停用词过滤并且修正词性
  */
 public static List<Term> modifResult(List<Term> all) {
   List<Term> result = new ArrayList<Term>();
   try {
     for (Term term : all) {
       if (FILTER.size() > 0
           && (FILTER.contains(term.getName())
               || FILTER.contains(TAG + term.getNatrue().natureStr))) {
         continue;
       }
       String[] params = UserDefineLibrary.getParams(term.getName());
       if (params != null) {
         term.setNature(new Nature(params[0]));
       }
       result.add(term);
     }
   } catch (Exception e) {
     System.err.println(
         "FilterStopWord.updateDic can not be null , "
             + "you must use set FilterStopWord.setUpdateDic(map) or use method set map");
   }
   return result;
 }
示例#13
0
  @Override
  public void run() {
    String[] strings = line.split("\t");

    if (strings.length == 8) {
      int hfd = Integer.parseInt(strings[3]);
      if (hfd > 1) {
        String content = strings[7];
        List<Term> terms = ToAnalysis.parse(content);
        for (Term term : terms) {
          String name = term.getName();
          System.out.println(name);
        }
      }
    }
  }
示例#14
0
  /** 人名消歧,比如.邓颖超生前->邓颖 超生 前 fix to 丁颖超 生 前! 规则的方式增加如果两个人名之间连接是- , ·,•则连接 */
  public static void nameAmbiguity(Term[] terms) {
    Term from = null;
    Term term = null;
    Term next = null;
    for (int i = 0; i < terms.length - 1; i++) {
      term = terms[i];
      if (term != null && term.termNatures() == TermNatures.NR && term.getName().length() == 2) {
        next = terms[i + 2];
        if (next.termNatures().personAttr.split > 0) {
          term.setName(term.getName() + next.getName().charAt(0));
          terms[i + 2] = null;
          terms[i + 3] = new Term(next.getName().substring(1), next.getOffe(), TermNatures.NW);
          TermUtil.termLink(term, terms[i + 3]);
          TermUtil.termLink(terms[i + 3], next.to());
        }
      }
    }

    // 外国人名修正
    for (int i = 0; i < terms.length; i++) {
      term = terms[i];
      if (term != null
          && term.getName().length() == 1
          && i > 0
          && WordAlert.CharCover(term.getName().charAt(0)) == '·') {
        from = term.from();
        next = term.to();

        if (from.natrue().natureStr.startsWith("nr") && next.natrue().natureStr.startsWith("nr")) {
          from.setName(from.getName() + term.getName() + next.getName());
          TermUtil.termLink(from, next.to());
          terms[i] = null;
          terms[i + 1] = null;
        }
      }
    }
  }
  public void recognition(Result result) {
    List<Term> terms = result.getTerms();
    String end = null;
    String name;

    LinkedList<Term> mergeList = null;

    List<Term> list = new LinkedList<Term>();

    for (Term term : terms) {
      name = term.getName();
      if (end == null) {
        if ((end = ruleMap.get(name)) != null) {
          mergeList = new LinkedList<Term>();
          mergeList.add(term);
        } else {
          list.add(term);
        }
      } else {
        mergeList.add(term);
        if (end.equals(name)) {

          Term ft = mergeList.pollFirst();
          for (Term sub : mergeList) {
            ft.merage(sub);
          }
          ft.setNature(nature);
          list.add(ft);
          mergeList = null;
          end = null;
        }
      }
    }

    if (mergeList != null) {
      for (Term term : list) {
        list.add(term);
      }
    }

    result.setTerms(list);
  }
 private List<Term> recogntion_() {
   Term term = null;
   Term tempTerm = null;
   List<Term> termList = new ArrayList<Term>();
   int beginFreq = 10;
   for (int i = 0; i < terms.length; i++) {
     term = terms[i];
     if (term == null || !term.getTermNatures().personAttr.flag) {
       continue;
     }
     term.score = 0;
     term.selfScore = 0;
     int freq = 0;
     for (int j = 2; j > -1; j--) {
       freq = term.getTermNatures().personAttr.getFreq(j, 0);
       if ((freq > 10) || (term.getName().length() == 2 && freq > 10)) {
         tempTerm = nameFind(i, beginFreq, j);
         if (tempTerm != null) {
           termList.add(tempTerm);
           // 如果是无争议性识别
           if (skip) {
             for (int j2 = i; j2 < tempTerm.getToValue(); j2++) {
               if (terms[j2] != null) {
                 terms[j2].score = 0;
                 terms[j2].selfScore = 0;
               }
             }
             i = tempTerm.getToValue() - 1;
             break;
           }
         }
       }
     }
     beginFreq = term.getTermNatures().personAttr.begin + 1;
   }
   return termList;
 }
  /**
   * 人名识别
   *
   * @param term
   * @param offe
   * @param freq
   */
  private Term nameFind(int offe, int beginFreq, int size) {
    // TODO Auto-generated method stub
    StringBuilder sb = new StringBuilder();
    int undefinite = 0;
    skip = false;
    PersonNatureAttr pna = null;
    int index = 0;
    int freq = 0;
    double allFreq = 0;
    Term term = null;
    int i = offe;
    for (; i < terms.length; i++) {
      // 走到结尾处识别出来一个名字.
      if (terms[i] == null) {
        continue;
      }
      term = terms[i];
      pna = term.getTermNatures().personAttr;
      // 在这个长度的这个位置的词频,如果没有可能就干掉,跳出循环
      if ((freq = pna.getFreq(size, index)) == 0) {
        return null;
      }

      if (pna.allFreq > 0) {
        undefinite++;
      }
      sb.append(term.getName());
      allFreq += Math.log(term.getTermNatures().allFreq + 1);
      allFreq += -Math.log((freq));
      index++;

      if (index == size + 2) {
        break;
      }
    }

    double score = -Math.log(FACTORY[size]);
    score += allFreq;
    double endFreq = 0;
    // 开始寻找结尾词
    boolean flag = true;
    while (flag) {
      i++;
      if (i >= terms.length) {
        endFreq = 10;
        flag = false;
      } else if (terms[i] != null) {
        int twoWordFreq = NgramLibrary.getTwoWordFreq(term, terms[i]);
        if (twoWordFreq > 3) {
          return null;
        }
        endFreq = terms[i].getTermNatures().personAttr.end + 1;
        flag = false;
      }
    }

    score -= Math.log(endFreq);
    score -= Math.log(beginFreq);

    if (score > -3) {
      return null;
    }

    if (allFreq > 0 && undefinite > 0) {
      return null;
    }

    skip = undefinite == 0;

    term = new Term(sb.toString(), offe, TermNatures.NR);
    term.selfScore = score;

    return term;
  }
  public void recognition() {
    if (branch == null) {
      return;
    }
    int length = terms.length - 1;

    Term term = null;
    for (int i = 0; i < length; i++) {
      if (terms[i] == null) {
        continue;
      } else {
        from = terms[i].getFrom();
        terms[i].score = 0;
        terms[i].selfScore = 0;
      }

      branch = branch.getBranch(terms[i].getName());

      if (branch == null || branch.getStatus() == 3) {
        reset();
        continue;
      }

      offe = i;

      // 循环查找添加
      term = terms[i];
      sb.append(term.getName());
      if (branch.getStatus() == 2) {
        term.selfScore = branch.getParam().getScore();
      }
      boolean flag = true;
      while (flag) {
        term = term.getTo();
        branch = branch.getBranch(term.getName());
        // 如果没有找到跳出
        if (branch == null) {
          break;
        }

        switch (branch.getStatus()) {
          case 1:
            sb.append(term.getName());
            continue;
          case 2:
            sb.append(term.getName());
            score = branch.getParam().getScore();
            tempNature = branch.getParam().getNature();
            to = term.getTo();
            makeNewTerm();
            continue;
          case 3:
            sb.append(term.getName());
            score = branch.getParam().getScore();
            tempNature = branch.getParam().getNature();
            to = term.getTo();
            makeNewTerm();
            flag = false;
            break;
          default:
            System.out.println("怎么能出现0呢?");
            break;
        }
      }
      reset();
    }
  }