/**
* @author dengxiang.liu
* @description update jieba user dictionary with securities and themes just came up recently;
* should be called before getThemeSecPair methods;
* @see
*/
public void updateLocalJiebaDictWithSecAndTheme() {
String jiebaUserDictPath = JiebaUserDictPath;
logger.info("jieba user dictionary file path: {}", jiebaUserDictPath);
try {
List<DatayesdbpSecurity> securityList = datayesdbpMapper.getSecurityList();
Map<String, String> tokenMap = FileIOUtil.readJiebaDict(jiebaUserDictPath);
logger.info("original jieba dictionary size: {}", tokenMap == null ? 0 : tokenMap.size());
Integer addedNewsWordsCount = 0;
for (DatayesdbpSecurity datayesdbpSecurity : securityList) {
if (!tokenMap.containsKey(datayesdbpSecurity.getSecShortName())) {
tokenMap.put(datayesdbpSecurity.getSecShortName(), "n");
addedNewsWordsCount++;
}
}
List<BigdataTheme> bigdataThemeList = bigdataMapper.getThemeList();
for (BigdataTheme bigdataTheme : bigdataThemeList) {
if (!tokenMap.containsKey(bigdataTheme.getThemeName())) {
tokenMap.put(bigdataTheme.getThemeName(), "n");
}
}
FileIOUtil.writeJiebaDict(jiebaUserDictPath, tokenMap);
logger.info("Add {} new words into jieba dictionary.", addedNewsWordsCount);
} catch (Exception e) {
logger.error("error occurs when update jieba dictionary, {}", e.toString());
}
}
