@Override
public boolean onFileFilter(File file) {
logger.log(Level.FINER, "- +File {0}", file);
startFileProperties = fileVersionComparator.captureFileProperties(file, null, false);
// Check if file has vanished
if (!startFileProperties.exists() || startFileProperties.isLocked()) {
logger.log(Level.FINER, "- /File: {0}", file);
logger.log(
Level.INFO,
" * NOT ADDING because file has VANISHED (exists = {0}) or is LOCKED (locked = {1}).",
new Object[] {startFileProperties.exists(), startFileProperties.isLocked()});
resetFileEnd();
return false;
}
// Content
if (startFileProperties.getType() == FileType.FILE) {
logger.log(Level.FINER, "- +FileContent: {0}", file);
fileContent = new FileContent();
}
return true;
}
private PartialFileHistory guessLastFileHistoryForFileWithMatchingChecksum(
FileProperties fileProperties,
Collection<PartialFileHistory> fileHistoriesWithSameChecksum) {
PartialFileHistory lastFileHistory = null;
// Check if they do not exist anymore --> assume it has moved!
// We choose the best fileHistory to base on as follows:
// 1. Ensure that it was modified at the same time and is the same size
// 2. Check the fileHistory was deleted and the file does not actually exists
// 3. Choose the one with the longest matching tail of the path to the new path
for (PartialFileHistory fileHistoryWithSameChecksum : fileHistoriesWithSameChecksum) {
FileVersion lastVersion = fileHistoryWithSameChecksum.getLastVersion();
if (fileProperties.getLastModified() != lastVersion.getLastModified().getTime()
|| fileProperties.getSize() != lastVersion.getSize()) {
continue;
}
File lastVersionOnLocalDisk =
new File(config.getLocalDir() + File.separator + lastVersion.getPath());
if (lastVersion.getStatus() != FileStatus.DELETED
&& !FileUtil.exists(lastVersionOnLocalDisk)) {
if (lastFileHistory == null) {
lastFileHistory = fileHistoryWithSameChecksum;
} else {
String filePath = fileProperties.getRelativePath();
String currentPreviousPath = lastFileHistory.getLastVersion().getPath();
String candidatePreviousPath = fileHistoryWithSameChecksum.getLastVersion().getPath();
for (int i = 0; i < filePath.length(); i++) {
if (!filePath.regionMatches(
filePath.length() - i,
candidatePreviousPath,
candidatePreviousPath.length() - i,
i)) {
// The candidate no longer matches, take the current path.
break;
}
if (!filePath.regionMatches(
filePath.length() - i,
currentPreviousPath,
currentPreviousPath.length() - i,
i)) {
// The current previous path no longer matches, take the new candidate
lastFileHistory = fileHistoryWithSameChecksum;
break;
}
}
}
}
}
return lastFileHistory;
}
private PartialFileHistory guessLastFileHistory(FileProperties fileProperties) {
if (fileProperties.getType() == FileType.FILE) {
return guessLastFileHistoryForFile(fileProperties);
} else if (fileProperties.getType() == FileType.SYMLINK) {
return guessLastFileHistoryForSymlink(fileProperties);
} else if (fileProperties.getType() == FileType.FOLDER) {
return guessLastFileHistoryForFolder(fileProperties);
} else {
throw new RuntimeException("This should not happen.");
}
}
private void addFileVersion(FileProperties fileProperties) {
if (fileProperties.getChecksum() != null) {
logger.log(
Level.FINER,
"- /File: {0} (checksum {1})",
new Object[] {fileProperties.getRelativePath(), fileProperties.getChecksum()});
} else {
logger.log(
Level.FINER,
"- /File: {0} (directory/symlink/0-byte-file)",
fileProperties.getRelativePath());
}
// 1. Determine if file already exists in database
PartialFileHistory lastFileHistory = guessLastFileHistory(fileProperties);
FileVersion lastFileVersion =
(lastFileHistory != null) ? lastFileHistory.getLastVersion() : null;
// 2. Create new file history/version
PartialFileHistory fileHistory = createNewFileHistory(lastFileHistory);
FileVersion fileVersion = createNewFileVersion(lastFileVersion, fileProperties);
// 3. Compare new and last version
FileProperties lastFileVersionProperties =
fileVersionComparator.captureFileProperties(lastFileVersion);
FileVersionComparison lastToNewFileVersionComparison =
fileVersionComparator.compare(fileProperties, lastFileVersionProperties, true);
boolean newVersionDiffersFromToLastVersion = !lastToNewFileVersionComparison.equals();
if (newVersionDiffersFromToLastVersion) {
fileHistory.addFileVersion(fileVersion);
newDatabaseVersion.addFileHistory(fileHistory);
logger.log(Level.INFO, " * Added file version: " + fileVersion);
logger.log(Level.INFO, " based on file version: " + lastFileVersion);
} else {
logger.log(Level.INFO, " * NOT ADDING file version: " + fileVersion);
logger.log(Level.INFO, " b/c IDENTICAL prev.: " + lastFileVersion);
}
// 4. Add file content (if not a directory)
if (fileProperties.getChecksum() != null && fileContent != null) {
fileContent.setSize(fileProperties.getSize());
fileContent.setChecksum(fileProperties.getChecksum());
// Check if content already exists, throw gathered content away if it does!
FileContent existingContent =
localDatabase.getFileContent(fileProperties.getChecksum(), false);
if (existingContent == null) {
newDatabaseVersion.addFileContent(fileContent);
} else {
// Uses existing content (already in database); ref. by checksum
}
}
}
@Override
public boolean onFileStart(File file, int fileIndex) {
boolean processFile =
startFileProperties.getType() == FileType.FILE; // Ignore directories and symlinks!
if (processFile && listener != null) {
listener.onIndexFile(file.getName(), fileIndex);
}
return processFile;
}
private FileVersion createNewFileVersion(
FileVersion lastFileVersion, FileProperties fileProperties) {
FileVersion fileVersion = null;
// Version
if (lastFileVersion == null) {
fileVersion = new FileVersion();
fileVersion.setVersion(1L);
fileVersion.setStatus(FileStatus.NEW);
} else {
fileVersion = lastFileVersion.clone();
fileVersion.setVersion(lastFileVersion.getVersion() + 1);
}
// Simple attributes
fileVersion.setPath(fileProperties.getRelativePath());
fileVersion.setLinkTarget(fileProperties.getLinkTarget());
fileVersion.setType(fileProperties.getType());
fileVersion.setSize(fileProperties.getSize());
fileVersion.setChecksum(fileProperties.getChecksum());
fileVersion.setLastModified(new Date(fileProperties.getLastModified()));
fileVersion.setUpdated(new Date());
// Permissions
if (EnvironmentUtil.isWindows()) {
fileVersion.setDosAttributes(fileProperties.getDosAttributes());
fileVersion.setPosixPermissions(DEFAULT_POSIX_PERMISSIONS);
} else if (EnvironmentUtil.isUnixLikeOperatingSystem()) {
fileVersion.setPosixPermissions(fileProperties.getPosixPermissions());
fileVersion.setDosAttributes(DEFAULT_DOS_ATTRIBUTES);
}
// Status
if (lastFileVersion != null) {
if (fileVersion.getType() == FileType.FILE
&& FileChecksum.fileChecksumEquals(
fileVersion.getChecksum(), lastFileVersion.getChecksum())) {
fileVersion.setStatus(FileStatus.CHANGED);
} else if (!fileVersion.getPath().equals(lastFileVersion.getPath())) {
fileVersion.setStatus(FileStatus.RENAMED);
} else {
fileVersion.setStatus(FileStatus.CHANGED);
}
}
return fileVersion;
}
private PartialFileHistory guessLastFileHistoryForFolderOrSymlink(
FileProperties fileProperties) {
PartialFileHistory lastFileHistory = filePathCache.get(fileProperties.getRelativePath());
if (lastFileHistory == null) {
logger.log(
Level.FINER,
" * No old file history found, starting new history (path: "
+ fileProperties.getRelativePath()
+ ", "
+ fileProperties.getType()
+ ")");
return null;
} else {
FileVersion lastFileVersion = lastFileHistory.getLastVersion();
if (lastFileVersion.getStatus() != FileStatus.DELETED
&& lastFileVersion.getType() == fileProperties.getType()) {
logger.log(
Level.FINER,
" * Found old file history "
+ lastFileHistory.getFileHistoryId()
+ " (by path: "
+ fileProperties.getRelativePath()
+ "), "
+ fileProperties.getType()
+ ", appending new version.");
return lastFileHistory;
} else {
logger.log(
Level.FINER,
" * No old file history found, starting new history (path: "
+ fileProperties.getRelativePath()
+ ", "
+ fileProperties.getType()
+ ")");
return null;
}
}
}
@Override
public void onFileEnd(File file, byte[] rawFileChecksum) {
// Get file attributes (get them while file exists)
// Note: Do NOT move any File-methods (file.anything()) below the file.exists()-part,
// because the file could vanish!
FileChecksum fileChecksum =
(rawFileChecksum != null) ? new FileChecksum(rawFileChecksum) : null;
endFileProperties = fileVersionComparator.captureFileProperties(file, fileChecksum, false);
// Check if file has vanished
boolean fileIsLocked = endFileProperties.isLocked();
boolean fileVanished = !endFileProperties.exists();
boolean fileHasChanged =
startFileProperties.getSize() != endFileProperties.getSize()
|| startFileProperties.getLastModified() != endFileProperties.getLastModified();
if (fileVanished || fileIsLocked || fileHasChanged) {
logger.log(Level.FINER, "- /File: {0}", file);
logger.log(
Level.INFO,
" * NOT ADDING because file has VANISHED ("
+ !endFileProperties.exists()
+ "), is LOCKED ("
+ endFileProperties.isLocked()
+ "), or has CHANGED ("
+ fileHasChanged
+ ")");
resetFileEnd();
return;
}
// If it's still there, add it to the database
addFileVersion(endFileProperties);
// Reset
resetFileEnd();
}
/**
* Tries to guess a matching file history, first by path and then by matching checksum.
*
* <p>If the path matches the path of an existing file in the database, the file history from
* the database is used, and a new file version is appended. If there is no file in the database
* with that path, checksums are compared.
*
* <p>If there are more than one file with the same checksum (potential matches), the file with
* the closest path is chosen.
*/
private PartialFileHistory guessLastFileHistoryForFile(FileProperties fileProperties) {
PartialFileHistory lastFileHistory = null;
// a) Try finding a file history for which the last version has the same path
lastFileHistory = filePathCache.get(fileProperties.getRelativePath());
// b) If that fails, try finding files with a matching checksum
if (lastFileHistory == null) {
if (fileProperties.getChecksum() != null) {
Collection<PartialFileHistory> fileHistoriesWithSameChecksum =
fileChecksumCache.get(fileProperties.getChecksum());
if (fileHistoriesWithSameChecksum != null && fileHistoriesWithSameChecksum.size() > 0) {
lastFileHistory =
guessLastFileHistoryForFileWithMatchingChecksum(
fileProperties, fileHistoriesWithSameChecksum);
// Remove the lastFileHistory we are basing this one on from the
// cache, so no other history will be
fileHistoriesWithSameChecksum.remove(lastFileHistory);
if (fileHistoriesWithSameChecksum.isEmpty()) {
fileChecksumCache.remove(fileProperties.getChecksum());
}
}
}
if (lastFileHistory == null) {
logger.log(
Level.FINER,
" * No old file history found, starting new history (path: "
+ fileProperties.getRelativePath()
+ ", checksum: "
+ fileProperties.getChecksum()
+ ")");
return null;
} else {
logger.log(
Level.FINER,
" * Found old file history "
+ lastFileHistory.getFileHistoryId()
+ " (by checksum: "
+ fileProperties.getChecksum()
+ "), appending new version.");
return lastFileHistory;
}
} else {
if (fileProperties.getType() != lastFileHistory.getLastVersion().getType()) {
logger.log(
Level.FINER,
" * No old file history found, starting new history (path: "
+ fileProperties.getRelativePath()
+ ", checksum: "
+ fileProperties.getChecksum()
+ ")");
return null;
} else {
logger.log(
Level.FINER,
" * Found old file history "
+ lastFileHistory.getFileHistoryId()
+ " (by path: "
+ fileProperties.getRelativePath()
+ "), appending new version.");
return lastFileHistory;
}
}
}