Ejemplo n.º 1
0
  /** 1. 得到需要处理的file,可能多个 2. copy 3. rename 4. 异步压缩 5. 删除之前文件 */
  @Override
  public void run() {
    String lastTime;
    if (StringUtils.equals(rollUnit, "day")) {
      lastTime = FlumeUtil.getLastDayWithDate(fileDateFormat);
    } else {
      lastTime = FlumeUtil.getLastHourWithDate(fileDateFormat);
    }

    List<String> files = FileUtil.getFiles(logDir, filePrefix, fileCompresionMode, lastTime, false);
    if (files == null || files.size() < 1) {
      LOGGER.warn("No matched logs found in {}, fileDateFormat={}", logDir, fileDateFormat);
      return;
    }
    for (String file : files) {
      String src = logDir + "/" + file;
      String copyDst = spoolDir + "/" + file + completedSuffix;
      String moveDst = spoolDir + "/" + file;

      File srcFile = new File(src);
      File copyDstFile = new File(copyDst);
      File moveDstFile = new File(moveDst);

      // copy: 从logDir中拷贝file到spoolDir中,文件名要加上completedSuffix
      try {
        Files.copy(srcFile, copyDstFile);
        LOGGER.info("Copy file {} to {}.", src, copyDst);
      } catch (IOException e) {
        LOGGER.error("Connot copy file {} to {}.", src, copyDst);
      }

      // rename: 复制完成后,将file去掉completeSuffix进行重命名
      try {
        Files.move(copyDstFile, moveDstFile);
        LOGGER.info("Move file {} to {}.", copyDst, moveDst);
      } catch (IOException e) {
        LOGGER.error("Connot move file {} to {}.", copyDst, moveDst);
      }

      // 异步压缩
      if (needCompress()) {
        String innerEntryName = FileUtil.afterLastSlash(src);
        compressFuture = compressAsynchronously(src, src, innerEntryName, fileCompresionMode, 60);
      }
    }

    // 删除之前文件
    if (needDeletePastFile()) {
      deleteFiles(logDir, filePrefix, fileCompresionMode, fileMaxHistory);
    }
  }
Ejemplo n.º 2
0
 public void deleteFiles(String path, String prefix, String suffix, int maxHistory) {
   int dayBefore = maxHistory;
   while (true) {
     String filterDate = FlumeUtil.getDayBefore(dayBefore);
     List<String> files = FileUtil.getFiles(path, prefix, suffix, filterDate, true);
     // 一直上溯到没有日志的那天
     if (files == null || files.size() < 1) {
       break;
     }
     for (String file : files) {
       FileUtil.delete(new File(path + "/" + file));
     }
     LOGGER.info("delete files before {} days", dayBefore);
     dayBefore++;
   }
 }