Example #1
0
  public void testSimpleFileObject() throws IOException {

    FileSystemManager fs = new FileSystemManager();
    //
    String pathFile = "com/sinosoft/one/mvc/scanning/vfs/SimpleFileObject.class";
    URL urlFile = loader.getResource(pathFile);
    assertNotNull(urlFile);
    FileObject fileObjectFile = fs.resolveFile(urlFile);
    assertEquals(SimpleFileObject.class, fileObjectFile.getClass());
    //
    String pathDir = new File(urlFile.getPath()).getParent().replace('\\', '/');
    pathDir = StringUtils.removeEnd(pathDir, "/");
    URL urlDir = ResourceUtils.getURL(pathDir);
    assertNotNull(urlDir);
    File fileDir = new File(urlDir.getFile());
    assertTrue(fileDir.exists());
    FileObject fileObjectDir = fs.resolveFile(urlDir);
    assertEquals(SimpleFileObject.class, fileObjectDir.getClass());

    File dirFile = ResourceUtils.getFile(urlDir);
    assertTrue(urlDir.toString().endsWith("/"));
    assertTrue(urlDir.getPath().endsWith("/"));
    assertFalse(dirFile.getPath().endsWith("/"));
    assertTrue(fileObjectDir.toString().endsWith("/"));

    // exists
    assertTrue(fileObjectFile.exists());
    assertTrue(fileObjectDir.exists());
    assertFalse(fileObjectDir.getChild("a_not_exists_file.txt").exists());

    // getName
    assertEquals("vfs", fileObjectDir.getName().getBaseName());
    assertEquals("SimpleFileObject.class", fileObjectFile.getName().getBaseName());

    // getRelativeName
    assertEquals(
        "SimpleFileObject.class",
        fileObjectDir.getName().getRelativeName(fileObjectFile.getName()));
    assertEquals("", fileObjectDir.getName().getRelativeName(fileObjectDir.getName()));
    assertEquals("", fileObjectFile.getName().getRelativeName(fileObjectFile.getName()));

    // getType
    assertSame(FileType.FOLDER, fileObjectDir.getType());
    assertSame(FileType.FILE, fileObjectFile.getType());

    // getChild, getParent, and equals, getChildren
    assertEquals(fileObjectFile, fileObjectDir.getChild("SimpleFileObject.class"));
    assertEquals(fileObjectDir, fileObjectFile.getParent());
    assertSame(fileObjectFile, fileObjectDir.getChild("SimpleFileObject.class"));
    assertSame(fileObjectDir, fileObjectFile.getParent());
    assertTrue(ArrayUtils.contains(fileObjectDir.getChildren(), fileObjectFile));

    // getURL
    assertEquals(urlDir, fileObjectDir.getURL());
    assertEquals(urlFile, fileObjectFile.getURL());
  }
 @Override
 public FileObject[] getChildren() throws MalformedURLException, IOException {
   File[] files = file.listFiles();
   FileObject[] children = new FileObject[files.length];
   for (int i = 0; i < children.length; i++) {
     if (files[i].isDirectory()) {
       children[i] = fs.resolveFile(urlString + files[i].getName() + "/");
     } else {
       children[i] = fs.resolveFile(urlString + files[i].getName());
     }
   }
   return children;
 }
Example #3
0
  public void testJarFileRootObject() throws IOException {

    FileSystemManager fs = new FileSystemManager();

    URL urlFile = loader.getResource("org/apache/commons/lang/StringUtils.class");
    String urlString = urlFile.toString();
    int index = urlString.indexOf("!/");
    assertTrue(index > 0);
    String root = urlString.substring(0, index + 2);
    FileObject rootObject = fs.resolveFile(root);
    assertTrue(root, rootObject.exists());

    assertTrue(rootObject.getChildren().length > 0);
    assertTrue(rootObject.getChild("org/").exists());
    assertTrue(rootObject.getChild("org/") == fs.resolveFile(root + "org/"));
  }
  /**
   * Take specified action to either move or delete the processed file, depending on the outcome
   *
   * @param entry the PollTableEntry for the file that has been processed
   * @param fileObject the FileObject representing the file to be moved or deleted
   */
  private void moveOrDeleteAfterProcessing(final PollTableEntry entry, FileObject fileObject)
      throws AxisFault {

    String moveToDirectoryURI = null;
    try {
      switch (entry.getLastPollState()) {
        case PollTableEntry.SUCCSESSFUL:
          if (entry.getActionAfterProcess() == PollTableEntry.MOVE) {
            moveToDirectoryURI = entry.getMoveAfterProcess();
          }
          break;

        case PollTableEntry.FAILED:
          if (entry.getActionAfterFailure() == PollTableEntry.MOVE) {
            moveToDirectoryURI = entry.getMoveAfterFailure();
          }
          break;

        default:
          return;
      }

      if (moveToDirectoryURI != null) {
        FileObject moveToDirectory = fsManager.resolveFile(moveToDirectoryURI);
        String prefix;
        if (entry.getMoveTimestampFormat() != null) {
          prefix = entry.getMoveTimestampFormat().format(new Date());
        } else {
          prefix = "";
        }
        FileObject dest = moveToDirectory.resolveFile(prefix + fileObject.getName().getBaseName());
        if (log.isDebugEnabled()) {
          log.debug("Moving to file :" + dest.getName().getURI());
        }
        try {
          fileObject.moveTo(dest);
        } catch (FileSystemException e) {
          handleException("Error moving file : " + fileObject + " to " + moveToDirectoryURI, e);
        }
      } else {
        try {
          if (log.isDebugEnabled()) {
            log.debug("Deleting file :" + fileObject);
          }
          fileObject.close();
          if (!fileObject.delete()) {
            String msg = "Cannot delete file : " + fileObject;
            log.error(msg);
            throw new AxisFault(msg);
          }
        } catch (FileSystemException e) {
          log.error("Error deleting file : " + fileObject, e);
        }
      }

    } catch (FileSystemException e) {
      handleException(
          "Error resolving directory to move after processing : " + moveToDirectoryURI, e);
    }
  }
 @Override
 public FileObject getParent() throws MalformedURLException, IOException {
   File parent = file.getParentFile();
   if (parent == null) {
     return null;
   }
   return fs.resolveFile(parent.toURI().toURL());
 }
Example #6
0
  public void testJarFileObject() throws IOException {

    FileSystemManager fs = new FileSystemManager();

    URL urlFile = loader.getResource("org/apache/commons/lang/StringUtils.class");
    assertTrue(urlFile.toString().startsWith("jar:"));
    FileObject fileObjectFile = fs.resolveFile(urlFile);
    assertEquals(JarFileObject.class, fileObjectFile.getClass());
    //

    URL urlDir = new URL(urlFile.toString().substring(0, urlFile.toString().lastIndexOf('/') + 1));
    assertNotNull(urlDir);
    FileObject fileObjectDir = fs.resolveFile(urlDir);
    assertEquals(JarFileObject.class, fileObjectDir.getClass());

    // exists
    assertTrue(fileObjectFile.exists());
    assertTrue(fileObjectDir.exists());
    assertFalse(fileObjectDir.getChild("a_not_exists_file.txt").exists());

    // getName
    assertEquals("lang", fileObjectDir.getName().getBaseName());
    assertEquals("StringUtils.class", fileObjectFile.getName().getBaseName());

    // getRelativeName
    assertEquals(
        "StringUtils.class", fileObjectDir.getName().getRelativeName(fileObjectFile.getName()));
    assertEquals("", fileObjectDir.getName().getRelativeName(fileObjectDir.getName()));
    assertEquals("", fileObjectFile.getName().getRelativeName(fileObjectFile.getName()));

    // getType
    assertSame(FileType.FOLDER, fileObjectDir.getType());
    assertSame(FileType.FILE, fileObjectFile.getType());

    // getChild, getParent, and equals
    assertEquals(fileObjectFile, fileObjectDir.getChild("StringUtils.class"));
    assertEquals(fileObjectDir, fileObjectFile.getParent());
    assertSame(fileObjectFile, fileObjectDir.getChild("StringUtils.class"));
    assertSame(fileObjectDir, fileObjectFile.getParent());
    assertTrue(ArrayUtils.contains(fileObjectDir.getChildren(), fileObjectFile));

    // getURL
    assertEquals(urlDir, fileObjectDir.getURL());
    assertEquals(urlFile, fileObjectFile.getURL());
  }
Example #7
0
  public void testSimpleFileObjectDirEnd() throws IOException {
    FileSystemManager fs = new FileSystemManager();
    String pathDir = "com/sinosoft/one/mvc/testcases";
    URL urlDir = loader.getResource(pathDir);
    FileObject parent = fs.resolveFile(urlDir);
    assertTrue(parent.getURL().toString().endsWith("/"));
    assertTrue(parent.getURL().getPath().endsWith("/"));
    FileObject childDir = parent.getChild("scanning");
    assertTrue(childDir.getURL().toString().endsWith("/"));

    // getChildren
    assertTrue(ArrayUtils.contains(parent.getChildren(), childDir));
  }
Example #8
0
  public void testJarFileObjectDirEnd() throws IOException {
    FileSystemManager fs = new FileSystemManager();
    String pathDir = "org/apache/commons/lang/";
    URL urlDir = loader.getResource(pathDir);
    FileObject parent = fs.resolveFile(urlDir);
    assertTrue(parent.exists());
    assertTrue(parent.getURL().toString().endsWith("/"));
    FileObject childDir = parent.getChild("math");
    FileObject childDir2 = parent.getChild("math/");
    assertTrue(childDir.exists());
    assertTrue(childDir.getURL().toString().endsWith("/"));
    assertSame(childDir, childDir2);

    assertEquals("math/", parent.getName().getRelativeName(childDir.getName()));
    assertEquals("math/", parent.getName().getRelativeName(childDir2.getName()));
  }
 private FileObject getChildFolder(
     final FileSystemManager fileSystem, final FileObject baseFolder, final String baseName)
     throws FileSystemException {
   for (int counter = 0; counter < TEMPORARY_FOLDER_ATTEMPTS; ++counter) {
     final FileObject temporaryFolder = fileSystem.resolveFile(baseFolder, baseName + counter);
     if (!temporaryFolder.exists()) {
       temporaryFolder.createFolder();
       return temporaryFolder;
     }
   }
   throw new FileSystemException(
       "Failed to create directory within "
           + TEMPORARY_FOLDER_ATTEMPTS
           + " attempts (tried "
           + baseName
           + "0 to "
           + baseName
           + (TEMPORARY_FOLDER_ATTEMPTS - 1)
           + ')');
 }
 @Override
 public FileObject getChild(final String child) throws IOException {
   return fs.resolveFile(urlString + child);
 }
 private FileObject getTemporaryFolder(final FileSystemManager fileSystem)
     throws FileSystemException {
   return fileSystem.resolveFile(System.getProperty("java.io.tmpdir"));
 }
Example #12
0
  /**
   * Search for files that match the given regex pattern and create a list Then process each of
   * these files and update the status of the scan on the poll table
   *
   * @param entry the poll table entry for the scan
   * @param fileURI the file or directory to be scanned
   */
  private void scanFileOrDirectory(final PollTableEntry entry, String fileURI) {

    FileObject fileObject = null;

    if (log.isDebugEnabled()) {
      log.debug("Scanning directory or file : " + VFSUtils.maskURLPassword(fileURI));
    }

    boolean wasError = true;
    int retryCount = 0;
    int maxRetryCount = entry.getMaxRetryCount();
    long reconnectionTimeout = entry.getReconnectTimeout();

    while (wasError) {
      try {
        retryCount++;
        fileObject = fsManager.resolveFile(fileURI);

        if (fileObject == null) {
          log.error("fileObject is null");
          throw new FileSystemException("fileObject is null");
        }

        wasError = false;

      } catch (FileSystemException e) {
        if (retryCount >= maxRetryCount) {
          processFailure(
              "Repeatedly failed to resolve the file URI: " + VFSUtils.maskURLPassword(fileURI),
              e,
              entry);
          return;
        } else {
          log.warn(
              "Failed to resolve the file URI: "
                  + VFSUtils.maskURLPassword(fileURI)
                  + ", in attempt "
                  + retryCount
                  + ", "
                  + e.getMessage()
                  + " Retrying in "
                  + reconnectionTimeout
                  + " milliseconds.");
        }
      }

      if (wasError) {
        try {
          Thread.sleep(reconnectionTimeout);
        } catch (InterruptedException e2) {
          log.error("Thread was interrupted while waiting to reconnect.", e2);
        }
      }
    }

    try {
      if (fileObject.exists() && fileObject.isReadable()) {

        entry.setLastPollState(PollTableEntry.NONE);
        FileObject[] children = null;
        try {
          children = fileObject.getChildren();
        } catch (FileSystemException ignore) {
        }

        // if this is a file that would translate to a single message
        if (children == null || children.length == 0) {
          boolean isFailedRecord = false;
          if (entry.getMoveAfterMoveFailure() != null) {
            isFailedRecord = isFailedRecord(fileObject, entry);
          }

          if (fileObject.getType() == FileType.FILE && !isFailedRecord) {
            if (!entry.isFileLockingEnabled()
                || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, fileObject))) {
              try {
                processFile(entry, fileObject);
                entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
                metrics.incrementMessagesReceived();

              } catch (AxisFault e) {
                logException("Error processing File URI : " + fileObject.getName(), e);
                entry.setLastPollState(PollTableEntry.FAILED);
                metrics.incrementFaultsReceiving();
              }

              try {
                moveOrDeleteAfterProcessing(entry, fileObject);
              } catch (AxisFault axisFault) {
                logException(
                    "File object '" + fileObject.getURL().toString() + "' " + "cloud not be moved",
                    axisFault);
                entry.setLastPollState(PollTableEntry.FAILED);
                String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
                addFailedRecord(entry, fileObject, timeStamp);
              }
              if (entry.isFileLockingEnabled()) {
                VFSUtils.releaseLock(fsManager, fileObject);
                if (log.isDebugEnabled()) {
                  log.debug(
                      "Removed the lock file '"
                          + fileObject.toString()
                          + ".lock' of the file '"
                          + fileObject.toString());
                }
              }
            } else if (log.isDebugEnabled()) {
              log.debug("Couldn't get the lock for processing the file : " + fileObject.getName());
            } else if (isFailedRecord) {
              if (entry.isFileLockingEnabled()) {
                VFSUtils.releaseLock(fsManager, fileObject);
              }
              // schedule a cleanup task if the file is there
              if (fsManager.resolveFile(fileObject.getURL().toString()) != null
                  && removeTaskState == STATE_STOPPED
                  && entry.getMoveAfterMoveFailure() != null) {
                workerPool.execute(new FileRemoveTask(entry, fileObject));
              }
              if (log.isDebugEnabled()) {
                log.debug(
                    "File '"
                        + fileObject.getURL()
                        + "' has been marked as a failed"
                        + " record, it will not process");
              }
            }
          }

        } else {
          int failCount = 0;
          int successCount = 0;

          if (log.isDebugEnabled()) {
            log.debug("File name pattern : " + entry.getFileNamePattern());
          }
          for (FileObject child : children) {
            boolean isFailedRecord = false;
            if (entry.getMoveAfterMoveFailure() != null) {
              isFailedRecord = isFailedRecord(child, entry);
            }
            if (log.isDebugEnabled()) {
              log.debug("Matching file : " + child.getName().getBaseName());
            }
            if ((entry.getFileNamePattern() != null)
                && (child.getName().getBaseName().matches(entry.getFileNamePattern()))
                && (!entry.isFileLockingEnabled()
                    || (entry.isFileLockingEnabled() && VFSUtils.acquireLock(fsManager, child)))
                && !isFailedRecord) {
              try {
                if (log.isDebugEnabled()) {
                  log.debug("Processing file :" + child);
                }
                processFile(entry, child);
                successCount++;
                // tell moveOrDeleteAfterProcessing() file was success
                entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
                metrics.incrementMessagesReceived();

              } catch (Exception e) {
                logException("Error processing File URI : " + child.getName(), e);
                failCount++;
                // tell moveOrDeleteAfterProcessing() file failed
                entry.setLastPollState(PollTableEntry.FAILED);
                metrics.incrementFaultsReceiving();
              }

              try {
                moveOrDeleteAfterProcessing(entry, child);
              } catch (AxisFault axisFault) {
                logException(
                    "File object '" + child.getURL().toString() + "'cloud not be moved", axisFault);
                failCount++;
                entry.setLastPollState(PollTableEntry.FAILED);
                String timeStamp = VFSUtils.getSystemTime(entry.getFailedRecordTimestampFormat());
                addFailedRecord(entry, child, timeStamp);
              }
              // if there is a failure or not we'll try to release the lock
              if (entry.isFileLockingEnabled()) {
                VFSUtils.releaseLock(fsManager, child);
              }
            } else if (!(!entry.isFileLockingEnabled()
                    || (entry.isFileLockingEnabled()
                        && VFSUtils.acquireLock(fsManager, fileObject)))
                && log.isDebugEnabled()) {
              log.debug("Couldn't get the lock for processing the file : " + child.getName());
            } else if (isFailedRecord) {
              if (entry.isFileLockingEnabled()) {
                VFSUtils.releaseLock(fsManager, child);
                VFSUtils.releaseLock(fsManager, fileObject);
              }
              if (fsManager.resolveFile(child.getURL().toString()) != null
                  && removeTaskState == STATE_STOPPED
                  && entry.getMoveAfterMoveFailure() != null) {
                workerPool.execute(new FileRemoveTask(entry, child));
              }
              if (log.isDebugEnabled()) {
                log.debug(
                    "File '"
                        + fileObject.getURL()
                        + "' has been marked as a failed record, it will not "
                        + "process");
              }
            }
          }

          if (failCount == 0 && successCount > 0) {
            entry.setLastPollState(PollTableEntry.SUCCSESSFUL);
          } else if (successCount == 0 && failCount > 0) {
            entry.setLastPollState(PollTableEntry.FAILED);
          } else {
            entry.setLastPollState(PollTableEntry.WITH_ERRORS);
          }
        }

        // processing of this poll table entry is complete
        long now = System.currentTimeMillis();
        entry.setLastPollTime(now);
        entry.setNextPollTime(now + entry.getPollInterval());

      } else if (log.isDebugEnabled()) {
        log.debug(
            "Unable to access or read file or directory : "
                + VFSUtils.maskURLPassword(fileURI)
                + "."
                + " Reason: "
                + (fileObject.exists()
                    ? (fileObject.isReadable() ? "Unknown reason" : "The file can not be read!")
                    : "The file does not exists!"));
      }
      onPollCompletion(entry);
    } catch (FileSystemException e) {
      processFailure(
          "Error checking for existence and readability : " + VFSUtils.maskURLPassword(fileURI),
          e,
          entry);
    }
  }