Пример #1
0
 private void deleteOldDocs() {
   if (Constants.FALSE.equals(initParamMap.get(DELETE_OLD_DOCS))) {
     return;
   }
   final String sessionId = initParamMap.get(Constants.SESSION_ID);
   if (StringUtil.isBlank(sessionId)) {
     logger.warn("Invalid sessionId at " + dataConfig);
     return;
   }
   final FessConfig fessConfig = ComponentUtil.getFessConfig();
   final QueryBuilder queryBuilder =
       QueryBuilders.boolQuery()
           .must(
               QueryBuilders.termQuery(
                   fessConfig.getIndexFieldConfigId(), dataConfig.getConfigId()))
           .must(
               QueryBuilders.boolQuery()
                   .should(
                       QueryBuilders.rangeQuery(fessConfig.getIndexFieldExpires()).lte("now"))
                   .should(QueryBuilders.missingQuery(fessConfig.getIndexFieldExpires())))
           .mustNot(QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId));
   try {
     final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
     final String index = fessConfig.getIndexDocumentUpdateIndex();
     fessEsClient.admin().indices().prepareRefresh(index).execute().actionGet();
     final int numOfDeleted =
         fessEsClient.deleteByQuery(index, fessConfig.getIndexDocumentType(), queryBuilder);
     logger.info("Deleted {} old docs.", numOfDeleted);
   } catch (final Exception e) {
     logger.error("Could not delete old docs at " + dataConfig, e);
   }
 }
Пример #2
0
  public void store(final DictionaryFile<? extends DictionaryItem> dictFile, final File file) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    getDictionaryFile(dictFile.getId())
        .ifPresent(
            currentFile -> {
              if (currentFile.getTimestamp().getTime() > dictFile.getTimestamp().getTime()) {
                throw new DictionaryException(dictFile.getPath() + " was updated.");
              }

              // TODO use stream
              try (CurlResponse response =
                  Curl.post(fessConfig.getElasticsearchUrl() + "/_configsync/file")
                      .param("path", dictFile.getPath())
                      .body(FileUtil.readUTF8(file))
                      .execute()) {
                final Map<String, Object> contentMap = response.getContentAsMap();
                if (!Constants.TRUE.equalsIgnoreCase(contentMap.get("acknowledged").toString())) {
                  throw new DictionaryException("Failed to update " + dictFile.getPath());
                }
              } catch (final IOException e) {
                throw new DictionaryException("Failed to update " + dictFile.getPath(), e);
              }
            })
        .orElse(
            () -> {
              throw new DictionaryException(dictFile.getPath() + " does not exist.");
            });
  }
Пример #3
0
 protected OptionalEntity<FileConfig> getFileConfig(final CreateForm form) {
   final String username = systemHelper.getUsername();
   final long currentTime = systemHelper.getCurrentTimeAsLong();
   return getEntity(form, username, currentTime)
       .map(
           entity -> {
             entity.setUpdatedBy(username);
             entity.setUpdatedTime(currentTime);
             copyBeanToBean(
                 form,
                 entity,
                 op ->
                     op.exclude(
                         Stream.concat(
                                 Stream.of(Constants.COMMON_CONVERSION_RULE),
                                 Stream.of(Constants.PERMISSIONS))
                             .toArray(n -> new String[n])));
             final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
             entity.setPermissions(
                 split(form.permissions, "\n")
                     .get(
                         stream ->
                             stream
                                 .map(s -> permissionHelper.encode(s))
                                 .filter(StringUtil::isNotBlank)
                                 .distinct()
                                 .toArray(n -> new String[n])));
             return entity;
           });
 }
Пример #4
0
 @Override
 public int getStartPosition() {
   if (start == null) {
     start = ComponentUtil.getFessConfig().getPagingSearchPageStartAsInteger();
   }
   return start;
 }
Пример #5
0
 public FileConfig getFileConfig() {
   if (fileConfig == null) {
     final FileConfigService fileConfigService =
         ComponentUtil.getComponent(FileConfigService.class);
     fileConfig = fileConfigService.getFileConfig(getFileConfigId()).get();
   }
   return fileConfig;
 }
Пример #6
0
 @PostConstruct
 public void init() {
   StreamUtil.of(ComponentUtil.getFessConfig().getSearchDefaultRoles().split(","))
       .filter(name -> StringUtil.isNotBlank(name))
       .forEach(
           name -> {
             defaultRoleList.add(name);
           });
 }
Пример #7
0
 @Override
 public int getPageSize() {
   final FessConfig fessConfig = ComponentUtil.getFessConfig();
   if (num == null) {
     num = fessConfig.getPagingSearchPageSizeAsInteger();
   }
   if (num > fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue() || num <= 0) {
     num = fessConfig.getPagingSearchPageMaxSizeAsInteger();
   }
   return num;
 }
Пример #8
0
 public void initialize() {
   final FessConfig fessConfig = ComponentUtil.getFessConfig();
   if (start == null) {
     start = fessConfig.getPagingSearchPageStartAsInteger();
   }
   if (num == null) {
     num = fessConfig.getPagingSearchPageSizeAsInteger();
   } else if (num > fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue()) {
     num = fessConfig.getPagingSearchPageMaxSizeAsInteger();
   }
 }
Пример #9
0
 public InputStream getContentInputStream(
     final DictionaryFile<? extends DictionaryItem> dictFile) {
   final FessConfig fessConfig = ComponentUtil.getFessConfig();
   try {
     return Curl.get(fessConfig.getElasticsearchUrl() + "/_configsync/file")
         .param("path", dictFile.getPath())
         .execute()
         .getContentAsStream();
   } catch (final IOException e) {
     throw new DictionaryException("Failed to access " + dictFile.getPath(), e);
   }
 }
Пример #10
0
 @PostConstruct
 public void init() {
   final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
   cacheMap.put(Range.ONE_DAY, getHotSearchWordListByFromDate(now - Constants.ONE_DAY_IN_MILLIS));
   cacheMap.put(
       Range.ONE_WEEK, getHotSearchWordListByFromDate(now - Constants.ONE_DAY_IN_MILLIS * 7));
   cacheMap.put(
       Range.ONE_MONTH, getHotSearchWordListByFromDate(now - Constants.ONE_DAY_IN_MILLIS * 30));
   cacheMap.put(
       Range.ONE_YEAR, getHotSearchWordListByFromDate(now - Constants.ONE_DAY_IN_MILLIS * 365));
   cacheMap.put(Range.ENTIRE, getHotSearchWordListByFromDate(null));
 }
Пример #11
0
  /* (non-Javadoc)
   * @see org.codelibs.fess.helper.impl.RoleQueryHelper#build()
   */
  @Override
  public Set<String> build() {
    final Set<String> roleList = new HashSet<>();
    final HttpServletRequest request = LaRequestUtil.getOptionalRequest().orElse(null);

    // request parameter
    if (request != null && StringUtil.isNotBlank(parameterKey)) {
      roleList.addAll(buildByParameter(request));
    }

    // request header
    if (request != null && StringUtil.isNotBlank(headerKey)) {
      roleList.addAll(buildByHeader(request));
    }

    // cookie
    if (request != null && StringUtil.isNotBlank(cookieKey)) {
      roleList.addAll(buildByCookie(request));
    }

    // cookie mapping
    if (cookieNameMap != null) {
      roleList.addAll(buildByCookieNameMapping(request));
    }

    final FessLoginAssist fessLoginAssist = ComponentUtil.getComponent(FessLoginAssist.class);
    fessLoginAssist
        .getSessionUserBean()
        .ifPresent(
            fessUserBean ->
                StreamUtil.of(fessUserBean.getRoles())
                    .map(role -> Base64.getDecoder().decode(role))
                    .map(
                        role -> {
                          try {
                            return Optional.of(new String(role, Constants.UTF_8));
                          } catch (final IOException e) {
                            return null;
                          }
                        })
                    .forEach(role -> role.ifPresent(roleList::add)));

    if (defaultRoleList != null) {
      roleList.addAll(defaultRoleList);
    }

    if (logger.isDebugEnabled()) {
      logger.debug("roleList: " + roleList);
    }

    return roleList;
  }
Пример #12
0
 protected void process() {
   final DataStoreFactory dataStoreFactory = ComponentUtil.getDataStoreFactory();
   dataStore = dataStoreFactory.getDataStore(dataConfig.getHandlerName());
   if (dataStore == null) {
     logger.error("DataStore(" + dataConfig.getHandlerName() + ") is not found.");
   } else {
     try {
       dataStore.store(dataConfig, indexUpdateCallback, initParamMap);
     } catch (final Throwable e) {
       logger.error("Failed to process a data crawling: " + dataConfig.getName(), e);
       ComponentUtil.getComponent(FailureUrlService.class)
           .store(
               dataConfig,
               e.getClass().getCanonicalName(),
               dataConfig.getConfigId() + ":" + dataConfig.getName(),
               e);
     } finally {
       indexUpdateCallback.commit();
       deleteOldDocs();
     }
   }
 }
Пример #13
0
 // -----------------------------------------------------
 //                                               Details
 //                                               -------
 @Execute
 public HtmlResponse details(final int crudMode, final String id) {
   verifyCrudMode(crudMode, CrudMode.DETAILS);
   saveToken();
   return asHtml(path_AdminFileconfig_AdminFileconfigDetailsJsp)
       .useForm(
           EditForm.class,
           op ->
               op.setup(
                   form -> {
                     fileConfigService
                         .getFileConfig(id)
                         .ifPresent(
                             entity -> {
                               copyBeanToBean(
                                   entity,
                                   form,
                                   copyOp -> {
                                     copyOp.excludeNull();
                                     copyOp.exclude(Constants.PERMISSIONS);
                                   });
                               final PermissionHelper permissionHelper =
                                   ComponentUtil.getPermissionHelper();
                               form.permissions =
                                   stream(entity.getPermissions())
                                       .get(
                                           stream ->
                                               stream
                                                   .map(s -> permissionHelper.decode(s))
                                                   .filter(StringUtil::isNotBlank)
                                                   .distinct()
                                                   .collect(Collectors.joining("\n")));
                               form.crudMode = crudMode;
                             })
                         .orElse(
                             () ->
                                 throwValidationError(
                                     messages ->
                                         messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, id),
                                     () -> asListHtml()));
                   }))
       .renderWith(data -> registerRolesAndLabels(data));
 }
Пример #14
0
 @Execute
 public HtmlResponse edit(final EditForm form) {
   validate(form, messages -> {}, () -> asListHtml());
   final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
   final String id = form.id;
   fileConfigService
       .getFileConfig(id)
       .ifPresent(
           entity -> {
             copyBeanToBean(
                 entity,
                 form,
                 copyOp -> {
                   copyOp.excludeNull();
                   copyOp.exclude(Constants.PERMISSIONS);
                 });
             form.permissions =
                 stream(entity.getPermissions())
                     .get(
                         stream ->
                             stream
                                 .map(permissionHelper::decode)
                                 .filter(StringUtil::isNotBlank)
                                 .distinct()
                                 .collect(Collectors.joining("\n")));
           })
       .orElse(
           () ->
               throwValidationError(
                   messages -> messages.addErrorsCrudCouldNotFindCrudTable(GLOBAL, id),
                   () -> asListHtml()));
   saveToken();
   if (form.crudMode.intValue() == CrudMode.EDIT) {
     // back
     form.crudMode = CrudMode.DETAILS;
     return asDetailsHtml();
   } else {
     form.crudMode = CrudMode.EDIT;
     return asEditHtml();
   }
 }
Пример #15
0
 public DictionaryFile<? extends DictionaryItem>[] getDictionaryFiles() {
   final FessConfig fessConfig = ComponentUtil.getFessConfig();
   try (CurlResponse response =
       Curl.get(fessConfig.getElasticsearchUrl() + "/_configsync/file")
           .param("fields", "path,@timestamp")
           .execute()) {
     final Map<String, Object> contentMap = response.getContentAsMap();
     @SuppressWarnings("unchecked")
     final List<Map<String, Object>> fileList = (List<Map<String, Object>>) contentMap.get("file");
     return fileList
         .stream()
         .map(
             fileMap -> {
               try {
                 final String path = fileMap.get("path").toString();
                 final Date timestamp =
                     new SimpleDateFormat(Constants.DATE_FORMAT_ISO_8601_EXTEND_UTC)
                         .parse(fileMap.get("@timestamp").toString());
                 for (final DictionaryCreator creator : creatorList) {
                   final DictionaryFile<? extends DictionaryItem> file =
                       creator.create(path, timestamp);
                   if (file != null) {
                     return file;
                   }
                 }
               } catch (final Exception e) {
                 logger.warn("Failed to load " + fileMap, e);
               }
               return null;
             })
         .filter(file -> file != null)
         .toArray(n -> new DictionaryFile<?>[n]);
   } catch (final IOException e) {
     throw new DictionaryException("Failed to access dictionaries", e);
   }
 }
Пример #16
0
  protected void doCrawl(final String sessionId, final List<DataConfig> configList) {
    final int multiprocessCrawlingCount = ComponentUtil.getFessConfig().getCrawlingThreadCount();

    final long startTime = System.currentTimeMillis();

    final IndexUpdateCallback indexUpdateCallback =
        ComponentUtil.getComponent(IndexUpdateCallback.class);

    final List<String> sessionIdList = new ArrayList<>();
    final Map<String, String> initParamMap = new HashMap<>();
    dataCrawlingThreadList.clear();
    final List<String> dataCrawlingThreadStatusList = new ArrayList<>();
    for (final DataConfig dataConfig : configList) {
      final String sid = crawlingConfigHelper.store(sessionId, dataConfig);
      sessionIdList.add(sid);

      initParamMap.put(Constants.SESSION_ID, sessionId);
      initParamMap.put(Constants.CRAWLING_INFO_ID, sid);

      final DataCrawlingThread dataCrawlingThread =
          new DataCrawlingThread(dataConfig, indexUpdateCallback, initParamMap);
      dataCrawlingThread.setPriority(crawlerPriority);
      dataCrawlingThread.setName(sid);
      dataCrawlingThread.setDaemon(true);

      dataCrawlingThreadList.add(dataCrawlingThread);
      dataCrawlingThreadStatusList.add(Constants.READY);
    }

    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();

    int startedCrawlerNum = 0;
    int activeCrawlerNum = 0;
    while (startedCrawlerNum < dataCrawlingThreadList.size()) {
      // Force to stop crawl
      if (systemHelper.isForceStop()) {
        for (final DataCrawlingThread crawlerThread : dataCrawlingThreadList) {
          crawlerThread.stopCrawling();
        }
        break;
      }

      if (activeCrawlerNum < multiprocessCrawlingCount) {
        // start crawling
        dataCrawlingThreadList.get(startedCrawlerNum).start();
        dataCrawlingThreadStatusList.set(startedCrawlerNum, Constants.RUNNING);
        startedCrawlerNum++;
        activeCrawlerNum++;
        try {
          Thread.sleep(crawlingExecutionInterval);
        } catch (final InterruptedException e) {
          if (logger.isDebugEnabled()) {
            logger.debug("Interrupted.", e);
          }
        }
        continue;
      }

      // check status
      for (int i = 0; i < startedCrawlerNum; i++) {
        if (!dataCrawlingThreadList.get(i).isRunning()
            && dataCrawlingThreadStatusList.get(i).equals(Constants.RUNNING)) {
          dataCrawlingThreadList.get(i).awaitTermination();
          dataCrawlingThreadStatusList.set(i, Constants.DONE);
          activeCrawlerNum--;
        }
      }
      try {
        Thread.sleep(crawlingExecutionInterval);
      } catch (final InterruptedException e) {
        if (logger.isDebugEnabled()) {
          logger.debug("Interrupted.", e);
        }
      }
    }

    boolean finishedAll = false;
    while (!finishedAll) {
      finishedAll = true;
      for (int i = 0; i < dataCrawlingThreadList.size(); i++) {
        dataCrawlingThreadList.get(i).awaitTermination(crawlingExecutionInterval);
        if (!dataCrawlingThreadList.get(i).isRunning()
            && dataCrawlingThreadStatusList.get(i).equals(Constants.RUNNING)) {
          dataCrawlingThreadStatusList.set(i, Constants.DONE);
        }
        if (!dataCrawlingThreadStatusList.get(i).equals(Constants.DONE)) {
          finishedAll = false;
        }
      }
    }
    dataCrawlingThreadList.clear();
    dataCrawlingThreadStatusList.clear();

    // put cralwing info
    final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();

    final long execTime = System.currentTimeMillis() - startTime;
    crawlingInfoHelper.putToInfoMap(Constants.DATA_CRAWLING_EXEC_TIME, Long.toString(execTime));
    if (logger.isInfoEnabled()) {
      logger.info("[EXEC TIME] crawling time: " + execTime + "ms");
    }

    crawlingInfoHelper.putToInfoMap(
        Constants.DATA_INDEX_EXEC_TIME, Long.toString(indexUpdateCallback.getExecuteTime()));
    crawlingInfoHelper.putToInfoMap(
        Constants.DATA_INDEX_SIZE, Long.toString(indexUpdateCallback.getDocumentSize()));

    for (final String sid : sessionIdList) {
      // remove config
      crawlingConfigHelper.remove(sid);
    }
  }