public void store(final DictionaryFile<? extends DictionaryItem> dictFile, final File file) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
getDictionaryFile(dictFile.getId())
.ifPresent(
currentFile -> {
if (currentFile.getTimestamp().getTime() > dictFile.getTimestamp().getTime()) {
throw new DictionaryException(dictFile.getPath() + " was updated.");
}
// TODO use stream
try (CurlResponse response =
Curl.post(fessConfig.getElasticsearchUrl() + "/_configsync/file")
.param("path", dictFile.getPath())
.body(FileUtil.readUTF8(file))
.execute()) {
final Map<String, Object> contentMap = response.getContentAsMap();
if (!Constants.TRUE.equalsIgnoreCase(contentMap.get("acknowledged").toString())) {
throw new DictionaryException("Failed to update " + dictFile.getPath());
}
} catch (final IOException e) {
throw new DictionaryException("Failed to update " + dictFile.getPath(), e);
}
})
.orElse(
() -> {
throw new DictionaryException(dictFile.getPath() + " does not exist.");
});
}
@Override
public int getPageSize() {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (num == null) {
num = fessConfig.getPagingSearchPageSizeAsInteger();
}
if (num > fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue() || num <= 0) {
num = fessConfig.getPagingSearchPageMaxSizeAsInteger();
}
return num;
}
public void initialize() {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (start == null) {
start = fessConfig.getPagingSearchPageStartAsInteger();
}
if (num == null) {
num = fessConfig.getPagingSearchPageSizeAsInteger();
} else if (num > fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue()) {
num = fessConfig.getPagingSearchPageMaxSizeAsInteger();
}
}
public InputStream getContentInputStream(
final DictionaryFile<? extends DictionaryItem> dictFile) {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
try {
return Curl.get(fessConfig.getElasticsearchUrl() + "/_configsync/file")
.param("path", dictFile.getPath())
.execute()
.getContentAsStream();
} catch (final IOException e) {
throw new DictionaryException("Failed to access " + dictFile.getPath(), e);
}
}
private void deleteOldDocs() {
if (Constants.FALSE.equals(initParamMap.get(DELETE_OLD_DOCS))) {
return;
}
final String sessionId = initParamMap.get(Constants.SESSION_ID);
if (StringUtil.isBlank(sessionId)) {
logger.warn("Invalid sessionId at " + dataConfig);
return;
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final QueryBuilder queryBuilder =
QueryBuilders.boolQuery()
.must(
QueryBuilders.termQuery(
fessConfig.getIndexFieldConfigId(), dataConfig.getConfigId()))
.must(
QueryBuilders.boolQuery()
.should(
QueryBuilders.rangeQuery(fessConfig.getIndexFieldExpires()).lte("now"))
.should(QueryBuilders.missingQuery(fessConfig.getIndexFieldExpires())))
.mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
try {
final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
final String index = fessConfig.getIndexDocumentUpdateIndex();
fessEsClient.admin().indices().prepareRefresh(index).execute().actionGet();
final int numOfDeleted =
fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder);
logger.info("Deleted {} old docs.", numOfDeleted);
} catch (final Exception e) {
logger.error("Could not delete old docs at " + dataConfig, e);
}
}
public DictionaryFile<? extends DictionaryItem>[] getDictionaryFiles() {
final FessConfig fessConfig = ComponentUtil.getFessConfig();
try (CurlResponse response =
Curl.get(fessConfig.getElasticsearchUrl() + "/_configsync/file")
.param("fields", "path,@timestamp")
.execute()) {
final Map<String, Object> contentMap = response.getContentAsMap();
@SuppressWarnings("unchecked")
final List<Map<String, Object>> fileList = (List<Map<String, Object>>) contentMap.get("file");
return fileList
.stream()
.map(
fileMap -> {
try {
final String path = fileMap.get("path").toString();
final Date timestamp =
new SimpleDateFormat(Constants.DATE_FORMAT_ISO_8601_EXTEND_UTC)
.parse(fileMap.get("@timestamp").toString());
for (final DictionaryCreator creator : creatorList) {
final DictionaryFile<? extends DictionaryItem> file =
creator.create(path, timestamp);
if (file != null) {
return file;
}
}
} catch (final Exception e) {
logger.warn("Failed to load " + fileMap, e);
}
return null;
})
.filter(file -> file != null)
.toArray(n -> new DictionaryFile<?>[n]);
} catch (final IOException e) {
throw new DictionaryException("Failed to access dictionaries", e);
}
}
