public static void assertDatabaseEquals(
MemoryDatabase expectedDatabase, MemoryDatabase actualDatabase) {
logger.log(Level.INFO, "--");
logger.log(Level.INFO, "Now comparing two databases.");
logger.log(
Level.INFO, "DON'T WORRY. This can take a long time or even overload the heap space.");
List<DatabaseVersion> writtenDatabaseVersions = expectedDatabase.getDatabaseVersions();
List<DatabaseVersion> readDatabaseVersions = actualDatabase.getDatabaseVersions();
assertEquals(
"Different number of database versions.",
writtenDatabaseVersions.size(),
readDatabaseVersions.size());
for (DatabaseVersion writtenDatabaseVersion : writtenDatabaseVersions) {
DatabaseVersion readDatabaseVersion = null;
for (DatabaseVersion aReadDatabaseVersion : readDatabaseVersions) {
if (aReadDatabaseVersion.equals(writtenDatabaseVersion)) {
readDatabaseVersion = aReadDatabaseVersion;
break;
}
}
assertNotNull(
"Database version " + writtenDatabaseVersion + " does not exist in read database.",
readDatabaseVersion);
assertDatabaseVersionEquals(writtenDatabaseVersion, readDatabaseVersion);
}
logger.log(Level.INFO, "End of comparing databases");
logger.log(Level.INFO, "--");
}
@Test
public void testContentChecksumCache() throws IOException {
Database database = new Database();
// Round 1: Add file history & version
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
// - history 1, version 1
FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("samechecksum1.jpg");
fileVersion1.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0});
PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);
databaseVersion1.addFileHistory(fileHistory1);
databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);
database.addDatabaseVersion(databaseVersion1);
assertNotNull(database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}));
assertEquals(1, database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).size());
// Round 2: Add two other versions with same checksum to new database version
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
// - history 1, version 2
FileVersion fileVersion11 =
TestDatabaseUtil.createFileVersion("samechecksum2-renamed.jpg", fileVersion1);
fileVersion11.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!
fileVersion11.setStatus(FileStatus.RENAMED);
PartialFileHistory fileHistory11 =
new PartialFileHistory(11111111111111111L); // same ID as above
databaseVersion2.addFileHistory(fileHistory11);
databaseVersion2.addFileVersionToHistory(fileHistory11.getFileId(), fileVersion11);
// - history 2, version 1
FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("samechecksum2.jpg");
fileVersion2.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!
PartialFileHistory fileHistory2 = new PartialFileHistory(22222222222222222L); // different ID
databaseVersion2.addFileHistory(fileHistory2);
databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);
// - history 3, version 1
FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("samechecksum3.jpg");
fileVersion3.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!
PartialFileHistory fileHistory3 = new PartialFileHistory(33333333333333333L); // different ID
databaseVersion2.addFileHistory(fileHistory3);
databaseVersion2.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);
database.addDatabaseVersion(databaseVersion2);
assertNotNull(database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}));
assertEquals(3, database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).size());
}
private void addFileVersion(FileProperties fileProperties) {
if (fileProperties.getChecksum() != null) {
logger.log(
Level.FINER,
"- /File: {0} (checksum {1})",
new Object[] {fileProperties.getRelativePath(), fileProperties.getChecksum()});
} else {
logger.log(
Level.FINER,
"- /File: {0} (directory/symlink/0-byte-file)",
fileProperties.getRelativePath());
}
// 1. Determine if file already exists in database
PartialFileHistory lastFileHistory = guessLastFileHistory(fileProperties);
FileVersion lastFileVersion =
(lastFileHistory != null) ? lastFileHistory.getLastVersion() : null;
// 2. Create new file history/version
PartialFileHistory fileHistory = createNewFileHistory(lastFileHistory);
FileVersion fileVersion = createNewFileVersion(lastFileVersion, fileProperties);
// 3. Compare new and last version
FileProperties lastFileVersionProperties =
fileVersionComparator.captureFileProperties(lastFileVersion);
FileVersionComparison lastToNewFileVersionComparison =
fileVersionComparator.compare(fileProperties, lastFileVersionProperties, true);
boolean newVersionDiffersFromToLastVersion = !lastToNewFileVersionComparison.equals();
if (newVersionDiffersFromToLastVersion) {
fileHistory.addFileVersion(fileVersion);
newDatabaseVersion.addFileHistory(fileHistory);
logger.log(Level.INFO, " * Added file version: " + fileVersion);
logger.log(Level.INFO, " based on file version: " + lastFileVersion);
} else {
logger.log(Level.INFO, " * NOT ADDING file version: " + fileVersion);
logger.log(Level.INFO, " b/c IDENTICAL prev.: " + lastFileVersion);
}
// 4. Add file content (if not a directory)
if (fileProperties.getChecksum() != null && fileContent != null) {
fileContent.setSize(fileProperties.getSize());
fileContent.setChecksum(fileProperties.getChecksum());
// Check if content already exists, throw gathered content away if it does!
FileContent existingContent =
localDatabase.getFileContent(fileProperties.getChecksum(), false);
if (existingContent == null) {
newDatabaseVersion.addFileContent(fileContent);
} else {
// Uses existing content (already in database); ref. by checksum
}
}
}
public static void assertDatabaseVersionEquals(
DatabaseVersion expectedDatabaseVersion, DatabaseVersion actualDatabaseVersion) {
assertVectorClockEquals(
expectedDatabaseVersion.getVectorClock(), actualDatabaseVersion.getVectorClock());
compareDatabaseVersionChunks(
expectedDatabaseVersion.getChunks(), actualDatabaseVersion.getChunks());
compareDatabaseVersionMultiChunks(
expectedDatabaseVersion.getMultiChunks(), actualDatabaseVersion.getMultiChunks());
compareDatabaseVersionFileContents(
expectedDatabaseVersion.getFileContents(), actualDatabaseVersion.getFileContents());
compareDatabaseVersionFileHistories(
expectedDatabaseVersion.getFileHistories(), actualDatabaseVersion.getFileHistories());
}
public static DatabaseVersion createDatabaseVersion(
DatabaseVersionHeader basedOnDatabaseVersionHeader, Date date) {
VectorClock vectorClock =
(basedOnDatabaseVersionHeader != null)
? basedOnDatabaseVersionHeader.getVectorClock().clone()
: new VectorClock();
vectorClock.incrementClock("someclient");
DatabaseVersion databaseVersion = new DatabaseVersion();
databaseVersion.setClient("someclient");
databaseVersion.setTimestamp(date);
databaseVersion.setVectorClock(vectorClock);
return databaseVersion;
}
public static DatabaseVersion createDatabaseVersion(DatabaseVersion basedOnDatabaseVersion) {
if (basedOnDatabaseVersion == null) {
return createDatabaseVersion(null, new Date());
} else {
return createDatabaseVersion(basedOnDatabaseVersion.getHeader(), new Date());
}
}
@Override
public void onMultiChunkClose(MultiChunk multiChunk) {
logger.log(Level.FINER, "- /MultiChunk {0}", multiChunk.getId());
multiChunkEntry.setSize(multiChunk.getSize());
newDatabaseVersion.addMultiChunk(multiChunkEntry);
multiChunkEntry = null;
}
private void removeDeletedFiles(
DatabaseVersion newDatabaseVersion, List<PartialFileHistory> fileHistoriesWithLastVersion) {
logger.log(Level.FINER, "- Looking for deleted files ...");
for (PartialFileHistory fileHistory : fileHistoriesWithLastVersion) {
// Ignore this file history if it has been updated in this database version before (file
// probably renamed!)
if (newDatabaseVersion.getFileHistory(fileHistory.getFileHistoryId()) != null) {
continue;
}
// Check if file exists, remove if it doesn't
FileVersion lastLocalVersion = fileHistory.getLastVersion();
File lastLocalVersionOnDisk =
new File(config.getLocalDir() + File.separator + lastLocalVersion.getPath());
// Ignore this file history if the last version is marked "DELETED"
if (lastLocalVersion.getStatus() == FileStatus.DELETED) {
continue;
}
// Add this file history if a new file with this name has been added (file type change)
PartialFileHistory newFileWithSameName =
getFileHistoryByPathFromDatabaseVersion(
newDatabaseVersion, fileHistory.getLastVersion().getPath());
// If file has VANISHED, mark as DELETED
if (!FileUtil.exists(lastLocalVersionOnDisk) || newFileWithSameName != null) {
PartialFileHistory deletedFileHistory =
new PartialFileHistory(fileHistory.getFileHistoryId());
FileVersion deletedVersion = lastLocalVersion.clone();
deletedVersion.setStatus(FileStatus.DELETED);
deletedVersion.setVersion(fileHistory.getLastVersion().getVersion() + 1);
deletedVersion.setUpdated(new Date());
logger.log(Level.FINER, " + Deleted: Adding DELETED version: {0}", deletedVersion);
logger.log(Level.FINER, " based on: {0}", lastLocalVersion);
deletedFileHistory.addFileVersion(deletedVersion);
newDatabaseVersion.addFileHistory(deletedFileHistory);
}
}
}
@Test
public void testFilenameCache() throws IOException {
Database database = new Database();
// Round 1: Add file history & version
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("file1.jpg");
PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);
databaseVersion1.addFileHistory(fileHistory1);
databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(fileHistory1, database.getFileHistory("file1.jpg"));
// Round 2: Add new version
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("file2.jpg", fileVersion1);
PartialFileHistory fileHistory2 = new PartialFileHistory(11111111111111111L); // same ID
databaseVersion2.addFileHistory(fileHistory2);
databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);
database.addDatabaseVersion(databaseVersion2);
assertNotNull(database.getFileHistory("file2.jpg"));
assertEquals(2, database.getFileHistory("file2.jpg").getFileVersions().size());
assertNull(database.getFileHistory("file1.jpg"));
// Round 3: Add deleted version
DatabaseVersion databaseVersion3 = TestDatabaseUtil.createDatabaseVersion(databaseVersion2);
FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("file2.jpg", fileVersion2);
fileVersion3.setStatus(FileStatus.DELETED);
PartialFileHistory fileHistory3 = new PartialFileHistory(11111111111111111L); // same ID
databaseVersion3.addFileHistory(fileHistory3);
databaseVersion3.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);
database.addDatabaseVersion(databaseVersion3);
assertNull(database.getFileHistory("file2.jpg"));
}
/**
* Checks if chunk already exists in all database versions Afterwards checks if chunk exists in
* new introduced database version.
*/
@Override
public boolean onChunk(Chunk chunk) {
ChunkChecksum chunkChecksum = new ChunkChecksum(chunk.getChecksum());
chunkEntry = localDatabase.getChunk(chunkChecksum);
if (chunkEntry == null) {
chunkEntry = newDatabaseVersion.getChunk(chunkChecksum);
if (chunkEntry == null) {
logger.log(Level.FINER, "- Chunk new: {0}", chunkChecksum.toString());
chunkEntry = new ChunkEntry(chunkChecksum, chunk.getSize());
newDatabaseVersion.addChunk(chunkEntry);
return true;
}
}
logger.log(Level.FINER, "- Chunk exists: {0}", StringUtil.toHex(chunk.getChecksum()));
return false;
}
@Test
public void testChunkCache() throws IOException {
Database database = new Database();
// Round 1: Add chunk to new database version, then add database version
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12);
databaseVersion1.addChunk(chunkA1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
// Round 2: Add chunk to new database version, then add database version
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}, 112);
databaseVersion2.addChunk(chunkA2);
database.addDatabaseVersion(databaseVersion2);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA2, database.getChunk(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}));
// Round 3: Add chunk to new database version, then add database version
DatabaseVersion databaseVersion3 = TestDatabaseUtil.createDatabaseVersion(databaseVersion2);
ChunkEntry chunkA3 = new ChunkEntry(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1}, 192);
databaseVersion3.addChunk(chunkA3);
database.addDatabaseVersion(databaseVersion3);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA2, database.getChunk(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}));
assertEquals(chunkA3, database.getChunk(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1}));
}
@Test
public void testMultiChunkCache() throws IOException {
Database database = new Database();
// Round 1: Add chunk to multichunk
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
MultiChunkEntry multiChunkP1 = new MultiChunkEntry(new byte[] {8, 8, 8, 8, 8, 8, 8, 8});
ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12);
multiChunkP1.addChunk(new ChunkEntryId(chunkA1.getChecksum()));
databaseVersion1.addChunk(chunkA1);
databaseVersion1.addMultiChunk(multiChunkP1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}));
// Round 2: Add chunk to multichunk
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
MultiChunkEntry multiChunkP2 = new MultiChunkEntry(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7});
MultiChunkEntry multiChunkP3 = new MultiChunkEntry(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5});
ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}, 912);
ChunkEntry chunkA3 = new ChunkEntry(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}, 812);
ChunkEntry chunkA4 = new ChunkEntry(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}, 712);
multiChunkP2.addChunk(new ChunkEntryId(chunkA2.getChecksum()));
multiChunkP2.addChunk(new ChunkEntryId(chunkA3.getChecksum()));
multiChunkP3.addChunk(new ChunkEntryId(chunkA4.getChecksum()));
databaseVersion2.addChunk(chunkA2);
databaseVersion2.addChunk(chunkA3);
databaseVersion2.addChunk(chunkA4);
databaseVersion2.addMultiChunk(multiChunkP2);
databaseVersion2.addMultiChunk(multiChunkP3);
database.addDatabaseVersion(databaseVersion2);
// fail("xx");
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA2, database.getChunk(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA3, database.getChunk(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA4, database.getChunk(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}));
assertEquals(multiChunkP2, database.getMultiChunk(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7}));
assertEquals(multiChunkP3, database.getMultiChunk(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5}));
}
private PartialFileHistory getFileHistoryByPathFromDatabaseVersion(
DatabaseVersion databaseVersion, String path) {
// TODO [medium] Extremely performance intensive, because this is called inside a loop above.
// Implement better caching for database version!!!
for (PartialFileHistory fileHistory : databaseVersion.getFileHistories()) {
FileVersion lastVersion = fileHistory.getLastVersion();
if (lastVersion.getStatus() != FileStatus.DELETED && lastVersion.getPath().equals(path)) {
return fileHistory;
}
}
return null;
}
private Branches readUnknownDatabaseVersionHeaders(List<File> remoteDatabases)
throws IOException {
logger.log(Level.INFO, "Loading database headers, creating branches ...");
// Sort files (db-a-1 must be before db-a-2 !)
Collections.sort(
remoteDatabases); // TODO [medium] natural sort is a workaround, database file names should
// be centrally managed, db-name-0000000009 avoids natural sort
// Read database files
Branches unknownRemoteBranches = new Branches();
DatabaseDAO dbDAO = new XmlDatabaseDAO(config.getTransformer());
for (File remoteDatabaseFileInCache : remoteDatabases) {
Database remoteDatabase =
new Database(); // Database cannot be reused, since these might be different clients
RemoteDatabaseFile remoteDatabaseFile = new RemoteDatabaseFile(remoteDatabaseFileInCache);
dbDAO.load(
remoteDatabase,
remoteDatabaseFile
.getFile()); // TODO [medium] Performance: This is very, very, very inefficient, DB is
// loaded and then discarded
List<DatabaseVersion> remoteDatabaseVersions = remoteDatabase.getDatabaseVersions();
// Populate branches
Branch remoteClientBranch =
unknownRemoteBranches.getBranch(remoteDatabaseFile.getClientName(), true);
for (DatabaseVersion remoteDatabaseVersion : remoteDatabaseVersions) {
DatabaseVersionHeader header = remoteDatabaseVersion.getHeader();
remoteClientBranch.add(header);
}
}
return unknownRemoteBranches;
}
@Test
public void testFilenameCacheDeleteAndNewOfSameFileInOneDatabaseVersion() throws IOException {
Database database = new Database();
// Round 1: Add file history & version
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("file1.jpg");
PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);
databaseVersion1.addFileHistory(fileHistory1);
databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(fileHistory1, database.getFileHistory("file1.jpg"));
// Round 2: Add new version
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
// - delete file1.jpg
FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("file1.jpg", fileVersion1);
fileVersion2.setStatus(FileStatus.DELETED);
PartialFileHistory fileHistory2 = new PartialFileHistory(11111111111111111L); // same ID
databaseVersion2.addFileHistory(fileHistory2);
databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);
// - add file1.jpg (as FOLDER!)
FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("file1.jpg"); // new file!
fileVersion3.setType(FileType.FOLDER);
PartialFileHistory fileHistory3 = new PartialFileHistory(222222222L); // new ID !
databaseVersion2.addFileHistory(fileHistory3);
databaseVersion2.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);
// - add datbase version
database.addDatabaseVersion(databaseVersion2);
assertNotNull(database.getFileHistory("file1.jpg"));
assertEquals(1, database.getFileHistory("file1.jpg").getFileVersions().size());
assertEquals(fileHistory3, database.getFileHistory("file1.jpg"));
}