Beispiel #1
0
 private void deleteOldDocs() {
   if (Constants.FALSE.equals(initParamMap.get(DELETE_OLD_DOCS))) {
     return;
   }
   final String sessionId = initParamMap.get(Constants.SESSION_ID);
   if (StringUtil.isBlank(sessionId)) {
     logger.warn("Invalid sessionId at " + dataConfig);
     return;
   }
   final FessConfig fessConfig = ComponentUtil.getFessConfig();
   final QueryBuilder queryBuilder =
       QueryBuilders.boolQuery()
           .must(
               QueryBuilders.termQuery(
                   fessConfig.getIndexFieldConfigId(), dataConfig.getConfigId()))
           .must(
               QueryBuilders.boolQuery()
                   .should(
                       QueryBuilders.rangeQuery(fessConfig.getIndexFieldExpires()).lte("now"))
                   .should(QueryBuilders.missingQuery(fessConfig.getIndexFieldExpires())))
           .mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
   try {
     final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
     final String index = fessConfig.getIndexDocumentUpdateIndex();
     fessEsClient.admin().indices().prepareRefresh(index).execute().actionGet();
     final int numOfDeleted =
         fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder);
     logger.info("Deleted {} old docs.", numOfDeleted);
   } catch (final Exception e) {
     logger.error("Could not delete old docs at " + dataConfig, e);
   }
 }
Beispiel #2
0
 protected void process() {
   final DataStoreFactory dataStoreFactory = ComponentUtil.getDataStoreFactory();
   dataStore = dataStoreFactory.getDataStore(dataConfig.getHandlerName());
   if (dataStore == null) {
     logger.error("DataStore(" + dataConfig.getHandlerName() + ") is not found.");
   } else {
     try {
       dataStore.store(dataConfig, indexUpdateCallback, initParamMap);
     } catch (final Throwable e) {
       logger.error("Failed to process a data crawling: " + dataConfig.getName(), e);
       ComponentUtil.getComponent(FailureUrlService.class)
           .store(
               dataConfig,
               e.getClass().getCanonicalName(),
               dataConfig.getConfigId() + ":" + dataConfig.getName(),
               e);
     } finally {
       indexUpdateCallback.commit();
       deleteOldDocs();
     }
   }
 }