@Test
public void testChunkCache() throws IOException {
Database database = new Database();
// Round 1: Add chunk to new database version, then add database version
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12);
databaseVersion1.addChunk(chunkA1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
// Round 2: Add chunk to new database version, then add database version
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}, 112);
databaseVersion2.addChunk(chunkA2);
database.addDatabaseVersion(databaseVersion2);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA2, database.getChunk(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}));
// Round 3: Add chunk to new database version, then add database version
DatabaseVersion databaseVersion3 = TestDatabaseUtil.createDatabaseVersion(databaseVersion2);
ChunkEntry chunkA3 = new ChunkEntry(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1}, 192);
databaseVersion3.addChunk(chunkA3);
database.addDatabaseVersion(databaseVersion3);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA2, database.getChunk(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}));
assertEquals(chunkA3, database.getChunk(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1}));
}
private void initOperationVariables() throws Exception {
localDatabase = (localDatabase != null) ? localDatabase : loadLocalDatabase();
localBranch = localDatabase.getBranch();
transferManager = config.getConnection().createTransferManager();
databaseReconciliator = new DatabaseReconciliator();
}
@Test
public void testMultiChunkCache() throws IOException {
Database database = new Database();
// Round 1: Add chunk to multichunk
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
MultiChunkEntry multiChunkP1 = new MultiChunkEntry(new byte[] {8, 8, 8, 8, 8, 8, 8, 8});
ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12);
multiChunkP1.addChunk(new ChunkEntryId(chunkA1.getChecksum()));
databaseVersion1.addChunk(chunkA1);
databaseVersion1.addMultiChunk(multiChunkP1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}));
// Round 2: Add chunk to multichunk
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
MultiChunkEntry multiChunkP2 = new MultiChunkEntry(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7});
MultiChunkEntry multiChunkP3 = new MultiChunkEntry(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5});
ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}, 912);
ChunkEntry chunkA3 = new ChunkEntry(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}, 812);
ChunkEntry chunkA4 = new ChunkEntry(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}, 712);
multiChunkP2.addChunk(new ChunkEntryId(chunkA2.getChecksum()));
multiChunkP2.addChunk(new ChunkEntryId(chunkA3.getChecksum()));
multiChunkP3.addChunk(new ChunkEntryId(chunkA4.getChecksum()));
databaseVersion2.addChunk(chunkA2);
databaseVersion2.addChunk(chunkA3);
databaseVersion2.addChunk(chunkA4);
databaseVersion2.addMultiChunk(multiChunkP2);
databaseVersion2.addMultiChunk(multiChunkP3);
database.addDatabaseVersion(databaseVersion2);
// fail("xx");
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA2, database.getChunk(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA3, database.getChunk(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA4, database.getChunk(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}));
assertEquals(multiChunkP2, database.getMultiChunk(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7}));
assertEquals(multiChunkP3, database.getMultiChunk(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5}));
}
private void applyWinnersBranch(Branch winnersBranch, List<File> unknownRemoteDatabasesInCache)
throws Exception {
Branch winnersApplyBranch =
databaseReconciliator.findWinnersApplyBranch(localBranch, winnersBranch);
logger.log(Level.INFO, "- Database versions to APPLY locally: " + winnersApplyBranch);
if (winnersApplyBranch.size() == 0) {
logger.log(Level.WARNING, " + Nothing to update. Nice!");
result.setResultCode(DownResultCode.OK_NO_REMOTE_CHANGES);
} else {
logger.log(Level.INFO, "- Loading winners database ...");
Database winnersDatabase =
readWinnersDatabase(winnersApplyBranch, unknownRemoteDatabasesInCache);
FileSystemActionReconciliator actionReconciliator =
new FileSystemActionReconciliator(config, localDatabase, result);
List<FileSystemAction> actions =
actionReconciliator.determineFileSystemActions(winnersDatabase);
Set<MultiChunkEntry> unknownMultiChunks =
determineRequiredMultiChunks(actions, winnersDatabase);
downloadAndDecryptMultiChunks(unknownMultiChunks);
applyFileSystemActions(actions);
// Add winners database to local database
// Note: This must happen AFTER the file system stuff, because we compare the winners database
// with the local database!
for (DatabaseVersionHeader applyDatabaseVersionHeader : winnersApplyBranch.getAll()) {
logger.log(
Level.INFO,
" + Applying database version " + applyDatabaseVersionHeader.getVectorClock());
DatabaseVersion applyDatabaseVersion =
winnersDatabase.getDatabaseVersion(applyDatabaseVersionHeader.getVectorClock());
localDatabase.addDatabaseVersion(applyDatabaseVersion);
}
logger.log(Level.INFO, "- Saving local database to " + config.getDatabaseFile() + " ...");
saveLocalDatabase(localDatabase, config.getDatabaseFile());
result.setResultCode(DownResultCode.OK_WITH_REMOTE_CHANGES);
}
}
@Test
public void testContentChecksumCache() throws IOException {
Database database = new Database();
// Round 1: Add file history & version
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
// - history 1, version 1
FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("samechecksum1.jpg");
fileVersion1.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0});
PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);
databaseVersion1.addFileHistory(fileHistory1);
databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);
database.addDatabaseVersion(databaseVersion1);
assertNotNull(database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}));
assertEquals(1, database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).size());
// Round 2: Add two other versions with same checksum to new database version
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
// - history 1, version 2
FileVersion fileVersion11 =
TestDatabaseUtil.createFileVersion("samechecksum2-renamed.jpg", fileVersion1);
fileVersion11.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!
fileVersion11.setStatus(FileStatus.RENAMED);
PartialFileHistory fileHistory11 =
new PartialFileHistory(11111111111111111L); // same ID as above
databaseVersion2.addFileHistory(fileHistory11);
databaseVersion2.addFileVersionToHistory(fileHistory11.getFileId(), fileVersion11);
// - history 2, version 1
FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("samechecksum2.jpg");
fileVersion2.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!
PartialFileHistory fileHistory2 = new PartialFileHistory(22222222222222222L); // different ID
databaseVersion2.addFileHistory(fileHistory2);
databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);
// - history 3, version 1
FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("samechecksum3.jpg");
fileVersion3.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!
PartialFileHistory fileHistory3 = new PartialFileHistory(33333333333333333L); // different ID
databaseVersion2.addFileHistory(fileHistory3);
databaseVersion2.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);
database.addDatabaseVersion(databaseVersion2);
assertNotNull(database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}));
assertEquals(3, database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).size());
}
@Test
public void testFilenameCache() throws IOException {
Database database = new Database();
// Round 1: Add file history & version
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("file1.jpg");
PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);
databaseVersion1.addFileHistory(fileHistory1);
databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(fileHistory1, database.getFileHistory("file1.jpg"));
// Round 2: Add new version
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("file2.jpg", fileVersion1);
PartialFileHistory fileHistory2 = new PartialFileHistory(11111111111111111L); // same ID
databaseVersion2.addFileHistory(fileHistory2);
databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);
database.addDatabaseVersion(databaseVersion2);
assertNotNull(database.getFileHistory("file2.jpg"));
assertEquals(2, database.getFileHistory("file2.jpg").getFileVersions().size());
assertNull(database.getFileHistory("file1.jpg"));
// Round 3: Add deleted version
DatabaseVersion databaseVersion3 = TestDatabaseUtil.createDatabaseVersion(databaseVersion2);
FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("file2.jpg", fileVersion2);
fileVersion3.setStatus(FileStatus.DELETED);
PartialFileHistory fileHistory3 = new PartialFileHistory(11111111111111111L); // same ID
databaseVersion3.addFileHistory(fileHistory3);
databaseVersion3.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);
database.addDatabaseVersion(databaseVersion3);
assertNull(database.getFileHistory("file2.jpg"));
}
private void initOperationVariables() throws Exception {
localDatabase =
(localDatabase != null)
? localDatabase
: ((LoadDatabaseOperationResult) new LoadDatabaseOperation(config).execute())
.getDatabase();
localBranch = localDatabase.getBranch();
transferManager = config.getConnection().createTransferManager();
databaseReconciliator = new DatabaseReconciliator();
}
private Collection<MultiChunkEntry> determineMultiChunksToDownload(
FileVersion fileVersion, Database localDatabase, Database winnersDatabase) {
Set<MultiChunkEntry> multiChunksToDownload = new HashSet<MultiChunkEntry>();
FileContent winningFileContent = localDatabase.getContent(fileVersion.getChecksum());
if (winningFileContent == null) {
winningFileContent = winnersDatabase.getContent(fileVersion.getChecksum());
}
boolean winningFileHasContent = winningFileContent != null;
if (winningFileHasContent) { // File can be empty!
Collection<ChunkEntryId> fileChunks =
winningFileContent
.getChunks(); // TODO [medium] Instead of just looking for multichunks to download
// here, we should look for chunks in local files as well and return the
// chunk positions in the local files ChunkPosition (chunk123 at file12,
// offset 200, size 250)
for (ChunkEntryId chunkChecksum : fileChunks) {
MultiChunkEntry multiChunkForChunk = localDatabase.getMultiChunkForChunk(chunkChecksum);
if (multiChunkForChunk == null) {
multiChunkForChunk = winnersDatabase.getMultiChunkForChunk(chunkChecksum);
}
if (!multiChunksToDownload.contains(multiChunkForChunk)) {
logger.log(
Level.INFO,
" + Adding multichunk "
+ StringUtil.toHex(multiChunkForChunk.getId())
+ " to download list ...");
multiChunksToDownload.add(multiChunkForChunk);
}
}
}
return multiChunksToDownload;
}
public OperationResult execute() throws Exception {
logger.log(Level.INFO, "");
logger.log(Level.INFO, "Running 'Log' at client " + config.getMachineName() + " ...");
logger.log(Level.INFO, "--------------------------------------------");
Database database = loadLocalDatabase();
DatabaseVersion currentDatabaseVersion = database.getLastDatabaseVersion();
if (currentDatabaseVersion == null) {
throw new Exception("No database versions yet locally. Nothing to show here.");
}
List<PartialFileHistory> fileHistories = null;
if (options.getPaths().isEmpty()) {
fileHistories = new ArrayList<PartialFileHistory>(database.getFileHistories());
} else {
fileHistories = getFileHistoriesByPath(options.getPaths(), database);
}
return new LogOperationResult(fileHistories, options.getFormat());
}
private void pruneConflictingLocalBranch(Branch winnersBranch) throws Exception {
Branch localPruneBranch =
databaseReconciliator.findLosersPruneBranch(localBranch, winnersBranch);
logger.log(Level.INFO, "- Database versions to REMOVE locally: " + localPruneBranch);
if (localPruneBranch.size() == 0) {
logger.log(Level.INFO, " + Nothing to prune locally. No conflicts. Only updates. Nice!");
} else {
// Load dirty database (if existent)
logger.log(Level.INFO, " + Pruning databases locally ...");
Database dirtyDatabase = new Database();
for (DatabaseVersionHeader databaseVersionHeader : localPruneBranch.getAll()) {
// Database version
DatabaseVersion databaseVersion =
localDatabase.getDatabaseVersion(databaseVersionHeader.getVectorClock());
dirtyDatabase.addDatabaseVersion(databaseVersion);
// Remove database version locally
logger.log(Level.INFO, " * Removing " + databaseVersionHeader + " ...");
localDatabase.removeDatabaseVersion(databaseVersion);
DatabaseRemoteFile remoteFileToPrune =
new DatabaseRemoteFile(
"db-"
+ config.getMachineName()
+ "-"
+ databaseVersionHeader.getVectorClock().get(config.getMachineName()));
logger.log(Level.INFO, " * Deleting remote database file " + remoteFileToPrune + " ...");
transferManager.delete(remoteFileToPrune);
}
logger.log(
Level.INFO, " * Saving dirty database to " + config.getDirtyDatabaseFile() + " ...");
saveLocalDatabase(dirtyDatabase, config.getDirtyDatabaseFile());
}
}
private Branches readUnknownDatabaseVersionHeaders(List<File> remoteDatabases)
throws IOException {
logger.log(Level.INFO, "Loading database headers, creating branches ...");
// Sort files (db-a-1 must be before db-a-2 !)
Collections.sort(
remoteDatabases); // TODO [medium] natural sort is a workaround, database file names should
// be centrally managed, db-name-0000000009 avoids natural sort
// Read database files
Branches unknownRemoteBranches = new Branches();
DatabaseDAO dbDAO = new XmlDatabaseDAO(config.getTransformer());
for (File remoteDatabaseFileInCache : remoteDatabases) {
Database remoteDatabase =
new Database(); // Database cannot be reused, since these might be different clients
RemoteDatabaseFile remoteDatabaseFile = new RemoteDatabaseFile(remoteDatabaseFileInCache);
dbDAO.load(
remoteDatabase,
remoteDatabaseFile
.getFile()); // TODO [medium] Performance: This is very, very, very inefficient, DB is
// loaded and then discarded
List<DatabaseVersion> remoteDatabaseVersions = remoteDatabase.getDatabaseVersions();
// Populate branches
Branch remoteClientBranch =
unknownRemoteBranches.getBranch(remoteDatabaseFile.getClientName(), true);
for (DatabaseVersion remoteDatabaseVersion : remoteDatabaseVersions) {
DatabaseVersionHeader header = remoteDatabaseVersion.getHeader();
remoteClientBranch.add(header);
}
}
return unknownRemoteBranches;
}
private List<PartialFileHistory> getFileHistoriesByPath(
List<String> filePaths, Database database) {
List<PartialFileHistory> fileHistories = new ArrayList<PartialFileHistory>();
for (String filePath : filePaths) {
PartialFileHistory fileHistory = database.getFileHistory(filePath);
if (fileHistory != null) {
fileHistories.add(fileHistory);
} else {
logger.log(Level.INFO, "Cannot find file history for file " + filePath);
}
}
return fileHistories;
}
@Test
public void testFilenameCacheDeleteAndNewOfSameFileInOneDatabaseVersion() throws IOException {
Database database = new Database();
// Round 1: Add file history & version
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("file1.jpg");
PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);
databaseVersion1.addFileHistory(fileHistory1);
databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(fileHistory1, database.getFileHistory("file1.jpg"));
// Round 2: Add new version
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
// - delete file1.jpg
FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("file1.jpg", fileVersion1);
fileVersion2.setStatus(FileStatus.DELETED);
PartialFileHistory fileHistory2 = new PartialFileHistory(11111111111111111L); // same ID
databaseVersion2.addFileHistory(fileHistory2);
databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);
// - add file1.jpg (as FOLDER!)
FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("file1.jpg"); // new file!
fileVersion3.setType(FileType.FOLDER);
PartialFileHistory fileHistory3 = new PartialFileHistory(222222222L); // new ID !
databaseVersion2.addFileHistory(fileHistory3);
databaseVersion2.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);
// - add datbase version
database.addDatabaseVersion(databaseVersion2);
assertNotNull(database.getFileHistory("file1.jpg"));
assertEquals(1, database.getFileHistory("file1.jpg").getFileVersions().size());
assertEquals(fileHistory3, database.getFileHistory("file1.jpg"));
}
@Test
public void testChangedModifiedDate() throws Exception {
// Setup
Connection testConnection = TestConfigUtil.createTestLocalConnection();
TestClient clientA = new TestClient("A", testConnection);
TestClient clientB = new TestClient("B", testConnection);
// Run
// A, create two files with identical content and change mod. date of one of them
clientA.createNewFile("A-file1.jpg", 50 * 1024);
clientA.copyFile("A-file1.jpg", "A-file1-with-different-modified-date.jpg");
clientA.getLocalFile("A-file1.jpg").setLastModified(0);
clientA.up();
// B, down, then move BOTH files
clientB.down();
assertFileListEquals(clientA.getLocalFiles(), clientB.getLocalFiles());
assertDatabaseFileEquals(
clientA.getLocalDatabaseFile(),
clientB.getLocalDatabaseFile(),
clientA.getConfig().getTransformer());
clientB.moveFile("A-file1.jpg", "A-file1-moved.jpg");
clientB.moveFile(
"A-file1-with-different-modified-date.jpg",
"A-file1-with-different-modified-date-moved.jpg");
clientB.up();
Database clientDatabaseB = clientB.loadLocalDatabase();
PartialFileHistory file1Orig = clientDatabaseB.getFileHistory("A-file1-moved.jpg");
PartialFileHistory file1WithDiffLastModDate =
clientDatabaseB.getFileHistory("A-file1-with-different-modified-date-moved.jpg");
assertNotNull(file1Orig);
assertNotNull(file1WithDiffLastModDate);
FileVersion fileVersion1OrigV1 = file1Orig.getFileVersion(1);
FileVersion fileVersion1OrigV2 = file1Orig.getFileVersion(2);
FileVersion fileVersion1WithDiffLastModDateV1 = file1WithDiffLastModDate.getFileVersion(1);
FileVersion fileVersion1WithDiffLastModDateV2 = file1WithDiffLastModDate.getFileVersion(2);
assertNotNull(fileVersion1OrigV1);
assertNotNull(fileVersion1OrigV2);
assertNotNull(fileVersion1WithDiffLastModDateV1);
assertNotNull(fileVersion1WithDiffLastModDateV2);
assertEquals("A-file1.jpg", fileVersion1OrigV1.getName());
assertEquals("A-file1-moved.jpg", fileVersion1OrigV2.getName());
assertEquals(
"A-file1-with-different-modified-date.jpg", fileVersion1WithDiffLastModDateV1.getName());
assertEquals(
"A-file1-with-different-modified-date-moved.jpg",
fileVersion1WithDiffLastModDateV2.getName());
// Tear down
clientA.cleanup();
clientB.cleanup();
}