/** Check if the file/folder exists before proceeding and retrying */
  private boolean initFileCheck() {
    boolean wasError = true;
    int retryCount = 0;

    fileObject = null;
    while (wasError) {
      try {
        retryCount++;
        fileObject = fsManager.resolveFile(fileURI, fso);
        if (fileObject == null) {
          log.error("fileObject is null");
          throw new FileSystemException("fileObject is null");
        }
        wasError = false;
      } catch (FileSystemException e) {
        if (retryCount >= maxRetryCount) {
          log.error(
              "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI), e);
          return false;
        } else {
          log.warn(
              "Failed to resolve the file URI: "
                  + VFSUtils.maskURLPassword(fileURI)
                  + ", in attempt "
                  + retryCount
                  + ", "
                  + e.getMessage()
                  + " Retrying in "
                  + reconnectionTimeout
                  + " milliseconds.");
        }
      }
      if (wasError) {
        try {
          Thread.sleep(reconnectionTimeout);
        } catch (InterruptedException e2) {
          log.error("Thread was interrupted while waiting to reconnect.", e2);
        }
      }
    }
    return true;
  }
Пример #2
0
 @Override
 protected void doInit() throws AxisFault {
   super.doInit();
   try {
     StandardFileSystemManager fsm = new StandardFileSystemManager();
     fsm.setConfiguration(getClass().getClassLoader().getResource("providers.xml"));
     fsm.init();
     this.workerPool = super.workerPool;
     fsManager = fsm;
     Parameter lockFlagParam =
         getTransportInDescription().getParameter(VFSConstants.TRANSPORT_FILE_LOCKING);
     if (lockFlagParam != null) {
       String strLockingFlag = lockFlagParam.getValue().toString();
       // by-default enabled, if explicitly specified as "disable" make it disable
       if (VFSConstants.TRANSPORT_FILE_LOCKING_DISABLED.equals(strLockingFlag)) {
         globalFileLockingFlag = false;
       }
     }
   } catch (FileSystemException e) {
     handleException("Error initializing the file transport : " + e.getMessage(), e);
   }
 }
Пример #3
0
  public void connect(MessageContext messageContext) throws ConnectException {
    System.out.println("File deletion started...");
    String fileLocation =
        getParameter(messageContext, "filelocation") == null
            ? ""
            : getParameter(messageContext, "filelocation").toString();
    String filename =
        getParameter(messageContext, "file") == null
            ? ""
            : getParameter(messageContext, "file").toString();

    String filebeforepprocess =
        getParameter(messageContext, "filebeforeprocess") == null
            ? ""
            : getParameter(messageContext, "filebeforeprocess").toString();
    String fileafterprocess =
        getParameter(messageContext, "fileafterprocess") == null
            ? ""
            : getParameter(messageContext, "fileafterprocess").toString();
    String newFileLocation =
        getParameter(messageContext, "newfilelocation") == null
            ? ""
            : getParameter(messageContext, "newfilelocation").toString();
    if (log.isDebugEnabled()) {
      log.info("File deletion started..." + filename.toString());
      log.info("File Location..." + fileLocation);
    }

    boolean resultStatus = false;
    try {
      resultStatus =
          moveFile(fileLocation, filename, filebeforepprocess, fileafterprocess, newFileLocation);
    } catch (FileSystemException e) {
      handleException(e.getMessage(), messageContext);
    }

    generateResults(messageContext, resultStatus);
  }
  /**
   * Do the file processing operation for the given set of properties. Do the checks and pass the
   * control to processFile method
   */
  public FileObject poll() {
    if (fileURI == null || fileURI.trim().equals("")) {
      log.error(
          "Invalid file url. Check the inbound endpoint configuration. Endpoint Name : "
              + name
              + ", File URL : "
              + fileURI);
      return null;
    }

    if (log.isDebugEnabled()) {
      log.debug("Start : Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
    }

    if (!initFileCheck()) {
      // Unable to read from the source location provided.
      return null;
    }

    // If file/folder found proceed to the processing stage
    try {
      lastCycle = 0;
      if (fileObject.exists() && fileObject.isReadable()) {
        FileObject[] children = null;
        try {
          children = fileObject.getChildren();
        } catch (FileNotFolderException ignored) {
          if (log.isDebugEnabled()) {
            log.debug("No Folder found. Only file found on : " + VFSUtils.maskURLPassword(fileURI));
          }
        } catch (FileSystemException ex) {
          log.error(ex.getMessage(), ex);
        }

        // if this is a file that would translate to a single message
        if (children == null || children.length == 0) {
          // Fail record is a one that is processed but was not moved
          // or deleted due to an error.
          boolean isFailedRecord = VFSUtils.isFailRecord(fsManager, fileObject);
          if (!isFailedRecord) {
            fileHandler();
            if (injectHandler == null) {
              return fileObject;
            }
          } else {
            try {
              lastCycle = 2;
              moveOrDeleteAfterProcessing(fileObject);
            } catch (SynapseException synapseException) {
              log.error(
                  "File object '"
                      + fileObject.getURL().toString()
                      + "' "
                      + "cloud not be moved after first attempt",
                  synapseException);
            }
            if (fileLock) {
              // TODO: passing null to avoid build break. Fix properly
              VFSUtils.releaseLock(fsManager, fileObject, fso);
            }
            if (log.isDebugEnabled()) {
              log.debug(
                  "File '"
                      + fileObject.getURL()
                      + "' has been marked as a failed"
                      + " record, it will not process");
            }
          }
        } else {
          FileObject fileObject = directoryHandler(children);
          if (fileObject != null) {
            return fileObject;
          }
        }
      } else {
        log.warn(
            "Unable to access or read file or directory : "
                + VFSUtils.maskURLPassword(fileURI)
                + "."
                + " Reason: "
                + (fileObject.exists()
                    ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!")
                    : "The file does not exists!"));
        return null;
      }
    } catch (FileSystemException e) {
      log.error(
          "Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI), e);
      return null;
    } catch (Exception e) {
      log.error(
          "Error while processing the file/folder in URL : " + VFSUtils.maskURLPassword(fileURI),
          e);
      return null;
    } finally {
      try {
        fsManager.closeFileSystem(fileObject.getParent().getFileSystem());
        fileObject.close();
      } catch (Exception e) {
        log.error("Unable to close the file system. " + e.getMessage());
        log.error(e);
      }
    }
    if (log.isDebugEnabled()) {
      log.debug("End : Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
    }
    return null;
  }
  private void killLogstashAgent(int logAgentNumber, String logstashLogPath) {

    FileObject listendir;
    CustomFileListener listener = new CustomFileListener();
    long TIMEOUT_BETWEEN_FILE_QUERYING = 1000;
    long LOOP_TIMEOUT_IN_MILLIS = 10 * 1000;

    try {
      FileSystemManager fileSystemManager = VFS.getManager();
      listendir = fileSystemManager.resolveFile(logstashLogPath);
    } catch (FileSystemException e) {
      e.printStackTrace();
      return;
    }

    DefaultFileMonitor fm = new DefaultFileMonitor(listener);
    fm.setRecursive(true);
    fm.addFile(listendir);
    fm.setDelay(TIMEOUT_BETWEEN_FILE_QUERYING);
    fm.start();

    LogUtils.log("waiting to destroy logger");
    long startTimeMillis = System.currentTimeMillis();

    while (true) {

      if (!listener.isProcessUp()) {
        break;
      }

      listener.setProcessUp(false);

      try {
        TimeUnit.MILLISECONDS.sleep(LOOP_TIMEOUT_IN_MILLIS);
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }

    long endTimeMillis = System.currentTimeMillis();
    LogUtils.log("destroying logstash agent " + logAgentNumber);
    LogUtils.log("waited " + (endTimeMillis - startTimeMillis) / 1000 + " seconds");
    fm.stop();

    //        File logstashOutputFile = new File(logstashLogPath);

    if (logAgentNumber == 1) {

      process.destroy();
      process = null;
    } else {

      process2.destroy();
      process2 = null;
    }

    //        try {
    //            TimeUnit.SECONDS.sleep(5);
    //
    //            LogUtils.log("returning logstash config file to initial state");
    //            if(logAgentNumber == 1){
    //                IOUtils.replaceFileWithMove(new File(confFilePath), new File(backupFilePath));
    //                FileUtils.deleteQuietly(new File(backupFilePath));
    //            }
    //            else{
    //                IOUtils.replaceFileWithMove(new File(confFilePath2), new
    // File(backupFilePath2));
    //                FileUtils.deleteQuietly(new File(backupFilePath2));
    //
    //            }
    //        } catch (IOException e) {
    //            e.printStackTrace();
    //        } catch (InterruptedException e) {
    //            e.printStackTrace();
    //        }

    //        if(logstashOutputFile.exists()){
    //            FileUtils.deleteQuietly(logstashOutputFile);
    //        }
  }
Пример #6
0
  /**
   * Search for files that match the given regex pattern and create a list Then process each of
   * these files and update the status of the scan on the poll table
   *
   * @param entry the poll table entry for the scan
   * @param fileURI the file or directory to be scanned
   */
  private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) {

    FileObject fileObject = null;

    if (log.isDebugEnabled()) {
      log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
    }

    boolean wasError = true;
    int retryCount = 0;
    int maxRetryCount = entry.getMaxRetryCount();
    long reconnectionTimeout = entry.getReconnectTimeout();

    while (wasError) {
      try {
        retryCount++;
        fileObject = fsManager.resolveFile(fileURI);

        if (fileObject == null) {
          log.error("fileObject is null");
          throw new FileSystemException("fileObject is null");
        }

        wasError = false;

      } catch (FileSystemException e) {
        if (retryCount >= maxRetryCount) {
          processFailure(
              "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI),
              e,
              entry);
          return;
        } else {
          log.warn(
              "Failed to resolve the file URI: "
                  + VFSUtils.maskURLPassword(fileURI)
                  + ", in attempt "
                  + retryCount
                  + ", "
                  + e.getMessage()
                  + " Retrying in "
                  + reconnectionTimeout
                  + " milliseconds.");
        }
      }

      if (wasError) {
        try {
          Thread.sleep(reconnectionTimeout);
        } catch (InterruptedException e2) {
          log.error("Thread was interrupted while waiting to reconnect.", e2);
        }
      }
    }

    try {
      if (fileObject.exists() && fileObject.isReadable()) {

        entry.setLastPollState(PollTableEntry.NONE);
        FileObject[] children = null;
        try {
          children = fileObject.getChildren();
        } catch (FileSystemException ignore) {
        }

        // if this is a file that would translate to a single message
        if (children == null || children.length == 0) {
          boolean isFailedRecord = false;
          if (entry.getMoveAfterMoveFailure() != null) {
            isFailedRecord = isFailedRecord(fileObject, entry);
          }

          if (fileObject.getType() == FileType.FILE && !isFailedRecord) {
            if (!entry.isFileLockingEnabled()
                || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) {
              try {
                processFile(entry, fileObject);
                entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
                metrics.incrementMessagesReceived();

              } catch (AxisFault e) {
                logException("Error processing File URI : " + fileObject.getName(), e);
                entry.setLastPollState(PollTableEntry.FAILED);
                metrics.incrementFaultsReceiving();
              }

              try {
                moveOrDeleteAfterProcessing(entry, fileObject);
              } catch (AxisFault axisFault) {
                logException(
                    "File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved",
                    axisFault);
                entry.setLastPollState(PollTableEntry.FAILED);
                String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
                addFailedRecord(entry, fileObject, timeStamp);
              }
              if (entry.isFileLockingEnabled()) {
                VFSUtils.releaseLock(fsManager, fileObject);
                if (log.isDebugEnabled()) {
                  log.debug(
                      "Removed the lock file '"
                          + fileObject.toString()
                          + ".lock' of the file '"
                          + fileObject.toString());
                }
              }
            } else if (log.isDebugEnabled()) {
              log.debug("Couldn't get the lock for processing the file : " + fileObject.getName());
            } else if (isFailedRecord) {
              if (entry.isFileLockingEnabled()) {
                VFSUtils.releaseLock(fsManager, fileObject);
              }
              // schedule a cleanup task if the file is there
              if (fsManager.resolveFile(fileObject.getURL().toString()) != null
                  && removeTaskState == STATE_STOPPED
                  && entry.getMoveAfterMoveFailure() != null) {
                workerPool.execute(new FileRemoveTask(entry, fileObject));
              }
              if (log.isDebugEnabled()) {
                log.debug(
                    "File '"
                        + fileObject.getURL()
                        + "' has been marked as a failed"
                        + " record, it will not process");
              }
            }
          }

        } else {
          int failCount = 0;
          int successCount = 0;

          if (log.isDebugEnabled()) {
            log.debug("File name pattern : " + entry.getFileNamePattern());
          }
          for (FileObject child : children) {
            boolean isFailedRecord = false;
            if (entry.getMoveAfterMoveFailure() != null) {
              isFailedRecord = isFailedRecord(child, entry);
            }
            if (log.isDebugEnabled()) {
              log.debug("Matching file : " + child.getName().getBaseName());
            }
            if ((entry.getFileNamePattern() != null)
                && (child.getName().getBaseName().matches(entry.getFileNamePattern()))
                && (!entry.isFileLockingEnabled()
                    || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child)))
                && !isFailedRecord) {
              try {
                if (log.isDebugEnabled()) {
                  log.debug("Processing file :" + child);
                }
                processFile(entry, child);
                successCount++;
                // tell moveOrDeleteAfterProcessing() file was success
                entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
                metrics.incrementMessagesReceived();

              } catch (Exception e) {
                logException("Error processing File URI : " + child.getName(), e);
                failCount++;
                // tell moveOrDeleteAfterProcessing() file failed
                entry.setLastPollState(PollTableEntry.FAILED);
                metrics.incrementFaultsReceiving();
              }

              try {
                moveOrDeleteAfterProcessing(entry, child);
              } catch (AxisFault axisFault) {
                logException(
                    "File object '" + child.getURL().toString() + "'cloud not be moved", axisFault);
                failCount++;
                entry.setLastPollState(PollTableEntry.FAILED);
                String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
                addFailedRecord(entry, child, timeStamp);
              }
              // if there is a failure or not we'll try to release the lock
              if (entry.isFileLockingEnabled()) {
                VFSUtils.releaseLock(fsManager, child);
              }
            } else if (!(!entry.isFileLockingEnabled()
                    || (entry.isFileLockingEnabled()
                        && VFSUtils.acquireLock(fsManager, fileObject)))
                && log.isDebugEnabled()) {
              log.debug("Couldn't get the lock for processing the file : " + child.getName());
            } else if (isFailedRecord) {
              if (entry.isFileLockingEnabled()) {
                VFSUtils.releaseLock(fsManager, child);
                VFSUtils.releaseLock(fsManager, fileObject);
              }
              if (fsManager.resolveFile(child.getURL().toString()) != null
                  && removeTaskState == STATE_STOPPED
                  && entry.getMoveAfterMoveFailure() != null) {
                workerPool.execute(new FileRemoveTask(entry, child));
              }
              if (log.isDebugEnabled()) {
                log.debug(
                    "File '"
                        + fileObject.getURL()
                        + "' has been marked as a failed record, it will not "
                        + "process");
              }
            }
          }

          if (failCount == 0 && successCount > 0) {
            entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
          } else if (successCount == 0 && failCount > 0) {
            entry.setLastPollState(PollTableEntry.FAILED);
          } else {
            entry.setLastPollState(PollTableEntry.WITH_ERRORS);
          }
        }

        // processing of this poll table entry is complete
        long now = System.currentTimeMillis();
        entry.setLastPollTime(now);
        entry.setNextPollTime(now + entry.getPollInterval());

      } else if (log.isDebugEnabled()) {
        log.debug(
            "Unable to access or read file or directory : "
                + VFSUtils.maskURLPassword(fileURI)
                + "."
                + " Reason: "
                + (fileObject.exists()
                    ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!")
                    : "The file does not exists!"));
      }
      onPollCompletion(entry);
    } catch (FileSystemException e) {
      processFailure(
          "Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI),
          e,
          entry);
    }
  }