private void deleteOldDocs() {
if (Constants.FALSE.equals(initParamMap.get(DELETE_OLD_DOCS))) {
return;
}
final String sessionId = initParamMap.get(Constants.SESSION_ID);
if (StringUtil.isBlank(sessionId)) {
logger.warn("Invalid sessionId at " + dataConfig);
return;
}
final FessConfig fessConfig = ComponentUtil.getFessConfig();
final QueryBuilder queryBuilder =
QueryBuilders.boolQuery()
.must(
QueryBuilders.termQuery(
fessConfig.getIndexFieldConfigId(), dataConfig.getConfigId()))
.must(
QueryBuilders.boolQuery()
.should(
QueryBuilders.rangeQuery(fessConfig.getIndexFieldExpires()).lte("now"))
.should(QueryBuilders.missingQuery(fessConfig.getIndexFieldExpires())))
.mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
try {
final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
final String index = fessConfig.getIndexDocumentUpdateIndex();
fessEsClient.admin().indices().prepareRefresh(index).execute().actionGet();
final int numOfDeleted =
fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder);
logger.info("Deleted {} old docs.", numOfDeleted);
} catch (final Exception e) {
logger.error("Could not delete old docs at " + dataConfig, e);
}
}
protected void process() {
final DataStoreFactory dataStoreFactory = ComponentUtil.getDataStoreFactory();
dataStore = dataStoreFactory.getDataStore(dataConfig.getHandlerName());
if (dataStore == null) {
logger.error("DataStore(" + dataConfig.getHandlerName() + ") is not found.");
} else {
try {
dataStore.store(dataConfig, indexUpdateCallback, initParamMap);
} catch (final Throwable e) {
logger.error("Failed to process a data crawling: " + dataConfig.getName(), e);
ComponentUtil.getComponent(FailureUrlService.class)
.store(
dataConfig,
e.getClass().getCanonicalName(),
dataConfig.getConfigId() + ":" + dataConfig.getName(),
e);
} finally {
indexUpdateCallback.commit();
deleteOldDocs();
}
}
}