Example #1
0
  /**
   * @author dengxiang.liu
   * @description update jieba user dictionary with securities and themes just came up recently;
   *     should be called before getThemeSecPair methods;
   * @see
   */
  public void updateLocalJiebaDictWithSecAndTheme() {
    String jiebaUserDictPath = JiebaUserDictPath;

    logger.info("jieba user dictionary file path: {}", jiebaUserDictPath);
    try {
      List<DatayesdbpSecurity> securityList = datayesdbpMapper.getSecurityList();
      Map<String, String> tokenMap = FileIOUtil.readJiebaDict(jiebaUserDictPath);
      logger.info("original jieba dictionary size: {}", tokenMap == null ? 0 : tokenMap.size());
      Integer addedNewsWordsCount = 0;
      for (DatayesdbpSecurity datayesdbpSecurity : securityList) {
        if (!tokenMap.containsKey(datayesdbpSecurity.getSecShortName())) {
          tokenMap.put(datayesdbpSecurity.getSecShortName(), "n");
          addedNewsWordsCount++;
        }
      }

      List<BigdataTheme> bigdataThemeList = bigdataMapper.getThemeList();
      for (BigdataTheme bigdataTheme : bigdataThemeList) {
        if (!tokenMap.containsKey(bigdataTheme.getThemeName())) {
          tokenMap.put(bigdataTheme.getThemeName(), "n");
        }
      }

      FileIOUtil.writeJiebaDict(jiebaUserDictPath, tokenMap);
      logger.info("Add {} new words into jieba dictionary.", addedNewsWordsCount);
    } catch (Exception e) {
      logger.error("error occurs when update jieba dictionary, {}", e.toString());
    }
  }
Example #2
0
 /**
  * @author dengxiang.liu
  * @description 将等待QA的(theme-security-type...)对写入本地文件;
  */
 public void writeThemeSecPairToBeQAFile(
     Map<String, List<ThemeSecurityObjPair>> globalThemeSecPairMapToBeQA, String currentDateStr) {
   if (globalThemeSecPairMapToBeQA == null) {
     logger.info("No theme security pair to be qa today");
     return;
   }
   String globalQAFolderPath = "data/qa/";
   logger.info(
       "{} themes have theme-security pair information need to be QA today",
       globalThemeSecPairMapToBeQA.size());
   try {
     globalQAFolderPath = ConfigConst.QA_FOLDER_PATH;
   } catch (Exception e) {
     logger.error(e.toString());
   }
   for (String themeName : globalThemeSecPairMapToBeQA.keySet()) {
     String qaFolderPath = globalQAFolderPath + currentDateStr;
     File qaFolderToBeCreate = new File(qaFolderPath);
     if (!qaFolderToBeCreate.exists()) {
       Boolean success = qaFolderToBeCreate.mkdirs();
       if (!success) {
         logger.info("create folder failed.");
         return;
       }
     }
     List<ThemeSecurityObjPair> themeSecPairMapToBeQA = globalThemeSecPairMapToBeQA.get(themeName);
     if (null != themeSecPairMapToBeQA && !themeSecPairMapToBeQA.isEmpty())
       FileIOUtil.writeThemeSecPair(themeSecPairMapToBeQA, themeName, qaFolderPath);
   }
 }
Example #3
0
  /**
   * @Author: dengxiang.liu @Desc: get unhandled themes from the different between themes list in
   * database and themes list in local file; @See: updateHandledThemeFile()
   */
  public List<BigdataTheme> getUnhandledThemeList(List<BigdataTheme> bigdataThemeList) {

    /** get themes have been handled from the file */
    Map<Long, BigdataTheme> handledThemeMap = FileIOUtil.readThemeMap(HANDLED_THEMES_FILE_PATH);
    /** themes to be handled */
    List<BigdataTheme> unHandledThemeList = new ArrayList<BigdataTheme>();

    if (null == bigdataThemeList) return unHandledThemeList;
    for (BigdataTheme bigdataTheme : bigdataThemeList) {
      try {
        Long themeID = bigdataTheme.getThemeID();
        if (!handledThemeMap.containsKey(themeID)) {
          unHandledThemeList.add(bigdataTheme);
        }
      } catch (Exception e) {
        logger.error(e.toString());
      }
    }
    logger.info("get unhandled theme list size: {}", unHandledThemeList.size());
    return unHandledThemeList;
  }