/** * @author dengxiang.liu * @description update jieba user dictionary with securities and themes just came up recently; * should be called before getThemeSecPair methods; * @see */ public void updateLocalJiebaDictWithSecAndTheme() { String jiebaUserDictPath = JiebaUserDictPath; logger.info("jieba user dictionary file path: {}", jiebaUserDictPath); try { List<DatayesdbpSecurity> securityList = datayesdbpMapper.getSecurityList(); Map<String, String> tokenMap = FileIOUtil.readJiebaDict(jiebaUserDictPath); logger.info("original jieba dictionary size: {}", tokenMap == null ? 0 : tokenMap.size()); Integer addedNewsWordsCount = 0; for (DatayesdbpSecurity datayesdbpSecurity : securityList) { if (!tokenMap.containsKey(datayesdbpSecurity.getSecShortName())) { tokenMap.put(datayesdbpSecurity.getSecShortName(), "n"); addedNewsWordsCount++; } } List<BigdataTheme> bigdataThemeList = bigdataMapper.getThemeList(); for (BigdataTheme bigdataTheme : bigdataThemeList) { if (!tokenMap.containsKey(bigdataTheme.getThemeName())) { tokenMap.put(bigdataTheme.getThemeName(), "n"); } } FileIOUtil.writeJiebaDict(jiebaUserDictPath, tokenMap); logger.info("Add {} new words into jieba dictionary.", addedNewsWordsCount); } catch (Exception e) { logger.error("error occurs when update jieba dictionary, {}", e.toString()); } }
/** * @author dengxiang.liu * @description 将等待QA的(theme-security-type...)对写入本地文件; */ public void writeThemeSecPairToBeQAFile( Map<String, List<ThemeSecurityObjPair>> globalThemeSecPairMapToBeQA, String currentDateStr) { if (globalThemeSecPairMapToBeQA == null) { logger.info("No theme security pair to be qa today"); return; } String globalQAFolderPath = "data/qa/"; logger.info( "{} themes have theme-security pair information need to be QA today", globalThemeSecPairMapToBeQA.size()); try { globalQAFolderPath = ConfigConst.QA_FOLDER_PATH; } catch (Exception e) { logger.error(e.toString()); } for (String themeName : globalThemeSecPairMapToBeQA.keySet()) { String qaFolderPath = globalQAFolderPath + currentDateStr; File qaFolderToBeCreate = new File(qaFolderPath); if (!qaFolderToBeCreate.exists()) { Boolean success = qaFolderToBeCreate.mkdirs(); if (!success) { logger.info("create folder failed."); return; } } List<ThemeSecurityObjPair> themeSecPairMapToBeQA = globalThemeSecPairMapToBeQA.get(themeName); if (null != themeSecPairMapToBeQA && !themeSecPairMapToBeQA.isEmpty()) FileIOUtil.writeThemeSecPair(themeSecPairMapToBeQA, themeName, qaFolderPath); } }
/** * @Author: dengxiang.liu @Desc: get unhandled themes from the different between themes list in * database and themes list in local file; @See: updateHandledThemeFile() */ public List<BigdataTheme> getUnhandledThemeList(List<BigdataTheme> bigdataThemeList) { /** get themes have been handled from the file */ Map<Long, BigdataTheme> handledThemeMap = FileIOUtil.readThemeMap(HANDLED_THEMES_FILE_PATH); /** themes to be handled */ List<BigdataTheme> unHandledThemeList = new ArrayList<BigdataTheme>(); if (null == bigdataThemeList) return unHandledThemeList; for (BigdataTheme bigdataTheme : bigdataThemeList) { try { Long themeID = bigdataTheme.getThemeID(); if (!handledThemeMap.containsKey(themeID)) { unHandledThemeList.add(bigdataTheme); } } catch (Exception e) { logger.error(e.toString()); } } logger.info("get unhandled theme list size: {}", unHandledThemeList.size()); return unHandledThemeList; }