@Test public void testChunkCache() throws IOException { Database database = new Database(); // Round 1: Add chunk to new database version, then add database version DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion(); ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12); databaseVersion1.addChunk(chunkA1); database.addDatabaseVersion(databaseVersion1); assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0})); // Round 2: Add chunk to new database version, then add database version DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1); ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}, 112); databaseVersion2.addChunk(chunkA2); database.addDatabaseVersion(databaseVersion2); assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0})); assertEquals(chunkA2, database.getChunk(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1})); // Round 3: Add chunk to new database version, then add database version DatabaseVersion databaseVersion3 = TestDatabaseUtil.createDatabaseVersion(databaseVersion2); ChunkEntry chunkA3 = new ChunkEntry(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1}, 192); databaseVersion3.addChunk(chunkA3); database.addDatabaseVersion(databaseVersion3); assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0})); assertEquals(chunkA2, database.getChunk(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1})); assertEquals(chunkA3, database.getChunk(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1})); }
private void initOperationVariables() throws Exception { localDatabase = (localDatabase != null) ? localDatabase : loadLocalDatabase(); localBranch = localDatabase.getBranch(); transferManager = config.getConnection().createTransferManager(); databaseReconciliator = new DatabaseReconciliator(); }
@Test public void testMultiChunkCache() throws IOException { Database database = new Database(); // Round 1: Add chunk to multichunk DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion(); MultiChunkEntry multiChunkP1 = new MultiChunkEntry(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}); ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12); multiChunkP1.addChunk(new ChunkEntryId(chunkA1.getChecksum())); databaseVersion1.addChunk(chunkA1); databaseVersion1.addMultiChunk(multiChunkP1); database.addDatabaseVersion(databaseVersion1); assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0})); assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8})); // Round 2: Add chunk to multichunk DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1); MultiChunkEntry multiChunkP2 = new MultiChunkEntry(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7}); MultiChunkEntry multiChunkP3 = new MultiChunkEntry(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5}); ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}, 912); ChunkEntry chunkA3 = new ChunkEntry(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}, 812); ChunkEntry chunkA4 = new ChunkEntry(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}, 712); multiChunkP2.addChunk(new ChunkEntryId(chunkA2.getChecksum())); multiChunkP2.addChunk(new ChunkEntryId(chunkA3.getChecksum())); multiChunkP3.addChunk(new ChunkEntryId(chunkA4.getChecksum())); databaseVersion2.addChunk(chunkA2); databaseVersion2.addChunk(chunkA3); databaseVersion2.addChunk(chunkA4); databaseVersion2.addMultiChunk(multiChunkP2); databaseVersion2.addMultiChunk(multiChunkP3); database.addDatabaseVersion(databaseVersion2); // fail("xx"); assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0})); assertEquals(chunkA2, database.getChunk(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0})); assertEquals(chunkA3, database.getChunk(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0})); assertEquals(chunkA4, database.getChunk(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0})); assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8})); assertEquals(multiChunkP2, database.getMultiChunk(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7})); assertEquals(multiChunkP3, database.getMultiChunk(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5})); }
private void applyWinnersBranch(Branch winnersBranch, List<File> unknownRemoteDatabasesInCache) throws Exception { Branch winnersApplyBranch = databaseReconciliator.findWinnersApplyBranch(localBranch, winnersBranch); logger.log(Level.INFO, "- Database versions to APPLY locally: " + winnersApplyBranch); if (winnersApplyBranch.size() == 0) { logger.log(Level.WARNING, " + Nothing to update. Nice!"); result.setResultCode(DownResultCode.OK_NO_REMOTE_CHANGES); } else { logger.log(Level.INFO, "- Loading winners database ..."); Database winnersDatabase = readWinnersDatabase(winnersApplyBranch, unknownRemoteDatabasesInCache); FileSystemActionReconciliator actionReconciliator = new FileSystemActionReconciliator(config, localDatabase, result); List<FileSystemAction> actions = actionReconciliator.determineFileSystemActions(winnersDatabase); Set<MultiChunkEntry> unknownMultiChunks = determineRequiredMultiChunks(actions, winnersDatabase); downloadAndDecryptMultiChunks(unknownMultiChunks); applyFileSystemActions(actions); // Add winners database to local database // Note: This must happen AFTER the file system stuff, because we compare the winners database // with the local database! for (DatabaseVersionHeader applyDatabaseVersionHeader : winnersApplyBranch.getAll()) { logger.log( Level.INFO, " + Applying database version " + applyDatabaseVersionHeader.getVectorClock()); DatabaseVersion applyDatabaseVersion = winnersDatabase.getDatabaseVersion(applyDatabaseVersionHeader.getVectorClock()); localDatabase.addDatabaseVersion(applyDatabaseVersion); } logger.log(Level.INFO, "- Saving local database to " + config.getDatabaseFile() + " ..."); saveLocalDatabase(localDatabase, config.getDatabaseFile()); result.setResultCode(DownResultCode.OK_WITH_REMOTE_CHANGES); } }
@Test public void testContentChecksumCache() throws IOException { Database database = new Database(); // Round 1: Add file history & version DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion(); // - history 1, version 1 FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("samechecksum1.jpg"); fileVersion1.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L); databaseVersion1.addFileHistory(fileHistory1); databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1); database.addDatabaseVersion(databaseVersion1); assertNotNull(database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0})); assertEquals(1, database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).size()); // Round 2: Add two other versions with same checksum to new database version DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1); // - history 1, version 2 FileVersion fileVersion11 = TestDatabaseUtil.createFileVersion("samechecksum2-renamed.jpg", fileVersion1); fileVersion11.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum! fileVersion11.setStatus(FileStatus.RENAMED); PartialFileHistory fileHistory11 = new PartialFileHistory(11111111111111111L); // same ID as above databaseVersion2.addFileHistory(fileHistory11); databaseVersion2.addFileVersionToHistory(fileHistory11.getFileId(), fileVersion11); // - history 2, version 1 FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("samechecksum2.jpg"); fileVersion2.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum! PartialFileHistory fileHistory2 = new PartialFileHistory(22222222222222222L); // different ID databaseVersion2.addFileHistory(fileHistory2); databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2); // - history 3, version 1 FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("samechecksum3.jpg"); fileVersion3.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum! PartialFileHistory fileHistory3 = new PartialFileHistory(33333333333333333L); // different ID databaseVersion2.addFileHistory(fileHistory3); databaseVersion2.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3); database.addDatabaseVersion(databaseVersion2); assertNotNull(database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0})); assertEquals(3, database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).size()); }
@Test public void testFilenameCache() throws IOException { Database database = new Database(); // Round 1: Add file history & version DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion(); FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("file1.jpg"); PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L); databaseVersion1.addFileHistory(fileHistory1); databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1); database.addDatabaseVersion(databaseVersion1); assertEquals(fileHistory1, database.getFileHistory("file1.jpg")); // Round 2: Add new version DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1); FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("file2.jpg", fileVersion1); PartialFileHistory fileHistory2 = new PartialFileHistory(11111111111111111L); // same ID databaseVersion2.addFileHistory(fileHistory2); databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2); database.addDatabaseVersion(databaseVersion2); assertNotNull(database.getFileHistory("file2.jpg")); assertEquals(2, database.getFileHistory("file2.jpg").getFileVersions().size()); assertNull(database.getFileHistory("file1.jpg")); // Round 3: Add deleted version DatabaseVersion databaseVersion3 = TestDatabaseUtil.createDatabaseVersion(databaseVersion2); FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("file2.jpg", fileVersion2); fileVersion3.setStatus(FileStatus.DELETED); PartialFileHistory fileHistory3 = new PartialFileHistory(11111111111111111L); // same ID databaseVersion3.addFileHistory(fileHistory3); databaseVersion3.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3); database.addDatabaseVersion(databaseVersion3); assertNull(database.getFileHistory("file2.jpg")); }
private void initOperationVariables() throws Exception { localDatabase = (localDatabase != null) ? localDatabase : ((LoadDatabaseOperationResult) new LoadDatabaseOperation(config).execute()) .getDatabase(); localBranch = localDatabase.getBranch(); transferManager = config.getConnection().createTransferManager(); databaseReconciliator = new DatabaseReconciliator(); }
private Collection<MultiChunkEntry> determineMultiChunksToDownload( FileVersion fileVersion, Database localDatabase, Database winnersDatabase) { Set<MultiChunkEntry> multiChunksToDownload = new HashSet<MultiChunkEntry>(); FileContent winningFileContent = localDatabase.getContent(fileVersion.getChecksum()); if (winningFileContent == null) { winningFileContent = winnersDatabase.getContent(fileVersion.getChecksum()); } boolean winningFileHasContent = winningFileContent != null; if (winningFileHasContent) { // File can be empty! Collection<ChunkEntryId> fileChunks = winningFileContent .getChunks(); // TODO [medium] Instead of just looking for multichunks to download // here, we should look for chunks in local files as well and return the // chunk positions in the local files ChunkPosition (chunk123 at file12, // offset 200, size 250) for (ChunkEntryId chunkChecksum : fileChunks) { MultiChunkEntry multiChunkForChunk = localDatabase.getMultiChunkForChunk(chunkChecksum); if (multiChunkForChunk == null) { multiChunkForChunk = winnersDatabase.getMultiChunkForChunk(chunkChecksum); } if (!multiChunksToDownload.contains(multiChunkForChunk)) { logger.log( Level.INFO, " + Adding multichunk " + StringUtil.toHex(multiChunkForChunk.getId()) + " to download list ..."); multiChunksToDownload.add(multiChunkForChunk); } } } return multiChunksToDownload; }
public OperationResult execute() throws Exception { logger.log(Level.INFO, ""); logger.log(Level.INFO, "Running 'Log' at client " + config.getMachineName() + " ..."); logger.log(Level.INFO, "--------------------------------------------"); Database database = loadLocalDatabase(); DatabaseVersion currentDatabaseVersion = database.getLastDatabaseVersion(); if (currentDatabaseVersion == null) { throw new Exception("No database versions yet locally. Nothing to show here."); } List<PartialFileHistory> fileHistories = null; if (options.getPaths().isEmpty()) { fileHistories = new ArrayList<PartialFileHistory>(database.getFileHistories()); } else { fileHistories = getFileHistoriesByPath(options.getPaths(), database); } return new LogOperationResult(fileHistories, options.getFormat()); }
private void pruneConflictingLocalBranch(Branch winnersBranch) throws Exception { Branch localPruneBranch = databaseReconciliator.findLosersPruneBranch(localBranch, winnersBranch); logger.log(Level.INFO, "- Database versions to REMOVE locally: " + localPruneBranch); if (localPruneBranch.size() == 0) { logger.log(Level.INFO, " + Nothing to prune locally. No conflicts. Only updates. Nice!"); } else { // Load dirty database (if existent) logger.log(Level.INFO, " + Pruning databases locally ..."); Database dirtyDatabase = new Database(); for (DatabaseVersionHeader databaseVersionHeader : localPruneBranch.getAll()) { // Database version DatabaseVersion databaseVersion = localDatabase.getDatabaseVersion(databaseVersionHeader.getVectorClock()); dirtyDatabase.addDatabaseVersion(databaseVersion); // Remove database version locally logger.log(Level.INFO, " * Removing " + databaseVersionHeader + " ..."); localDatabase.removeDatabaseVersion(databaseVersion); DatabaseRemoteFile remoteFileToPrune = new DatabaseRemoteFile( "db-" + config.getMachineName() + "-" + databaseVersionHeader.getVectorClock().get(config.getMachineName())); logger.log(Level.INFO, " * Deleting remote database file " + remoteFileToPrune + " ..."); transferManager.delete(remoteFileToPrune); } logger.log( Level.INFO, " * Saving dirty database to " + config.getDirtyDatabaseFile() + " ..."); saveLocalDatabase(dirtyDatabase, config.getDirtyDatabaseFile()); } }
private Branches readUnknownDatabaseVersionHeaders(List<File> remoteDatabases) throws IOException { logger.log(Level.INFO, "Loading database headers, creating branches ..."); // Sort files (db-a-1 must be before db-a-2 !) Collections.sort( remoteDatabases); // TODO [medium] natural sort is a workaround, database file names should // be centrally managed, db-name-0000000009 avoids natural sort // Read database files Branches unknownRemoteBranches = new Branches(); DatabaseDAO dbDAO = new XmlDatabaseDAO(config.getTransformer()); for (File remoteDatabaseFileInCache : remoteDatabases) { Database remoteDatabase = new Database(); // Database cannot be reused, since these might be different clients RemoteDatabaseFile remoteDatabaseFile = new RemoteDatabaseFile(remoteDatabaseFileInCache); dbDAO.load( remoteDatabase, remoteDatabaseFile .getFile()); // TODO [medium] Performance: This is very, very, very inefficient, DB is // loaded and then discarded List<DatabaseVersion> remoteDatabaseVersions = remoteDatabase.getDatabaseVersions(); // Populate branches Branch remoteClientBranch = unknownRemoteBranches.getBranch(remoteDatabaseFile.getClientName(), true); for (DatabaseVersion remoteDatabaseVersion : remoteDatabaseVersions) { DatabaseVersionHeader header = remoteDatabaseVersion.getHeader(); remoteClientBranch.add(header); } } return unknownRemoteBranches; }
private List<PartialFileHistory> getFileHistoriesByPath( List<String> filePaths, Database database) { List<PartialFileHistory> fileHistories = new ArrayList<PartialFileHistory>(); for (String filePath : filePaths) { PartialFileHistory fileHistory = database.getFileHistory(filePath); if (fileHistory != null) { fileHistories.add(fileHistory); } else { logger.log(Level.INFO, "Cannot find file history for file " + filePath); } } return fileHistories; }
@Test public void testFilenameCacheDeleteAndNewOfSameFileInOneDatabaseVersion() throws IOException { Database database = new Database(); // Round 1: Add file history & version DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion(); FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("file1.jpg"); PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L); databaseVersion1.addFileHistory(fileHistory1); databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1); database.addDatabaseVersion(databaseVersion1); assertEquals(fileHistory1, database.getFileHistory("file1.jpg")); // Round 2: Add new version DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1); // - delete file1.jpg FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("file1.jpg", fileVersion1); fileVersion2.setStatus(FileStatus.DELETED); PartialFileHistory fileHistory2 = new PartialFileHistory(11111111111111111L); // same ID databaseVersion2.addFileHistory(fileHistory2); databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2); // - add file1.jpg (as FOLDER!) FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("file1.jpg"); // new file! fileVersion3.setType(FileType.FOLDER); PartialFileHistory fileHistory3 = new PartialFileHistory(222222222L); // new ID ! databaseVersion2.addFileHistory(fileHistory3); databaseVersion2.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3); // - add datbase version database.addDatabaseVersion(databaseVersion2); assertNotNull(database.getFileHistory("file1.jpg")); assertEquals(1, database.getFileHistory("file1.jpg").getFileVersions().size()); assertEquals(fileHistory3, database.getFileHistory("file1.jpg")); }
@Test public void testChangedModifiedDate() throws Exception { // Setup Connection testConnection = TestConfigUtil.createTestLocalConnection(); TestClient clientA = new TestClient("A", testConnection); TestClient clientB = new TestClient("B", testConnection); // Run // A, create two files with identical content and change mod. date of one of them clientA.createNewFile("A-file1.jpg", 50 * 1024); clientA.copyFile("A-file1.jpg", "A-file1-with-different-modified-date.jpg"); clientA.getLocalFile("A-file1.jpg").setLastModified(0); clientA.up(); // B, down, then move BOTH files clientB.down(); assertFileListEquals(clientA.getLocalFiles(), clientB.getLocalFiles()); assertDatabaseFileEquals( clientA.getLocalDatabaseFile(), clientB.getLocalDatabaseFile(), clientA.getConfig().getTransformer()); clientB.moveFile("A-file1.jpg", "A-file1-moved.jpg"); clientB.moveFile( "A-file1-with-different-modified-date.jpg", "A-file1-with-different-modified-date-moved.jpg"); clientB.up(); Database clientDatabaseB = clientB.loadLocalDatabase(); PartialFileHistory file1Orig = clientDatabaseB.getFileHistory("A-file1-moved.jpg"); PartialFileHistory file1WithDiffLastModDate = clientDatabaseB.getFileHistory("A-file1-with-different-modified-date-moved.jpg"); assertNotNull(file1Orig); assertNotNull(file1WithDiffLastModDate); FileVersion fileVersion1OrigV1 = file1Orig.getFileVersion(1); FileVersion fileVersion1OrigV2 = file1Orig.getFileVersion(2); FileVersion fileVersion1WithDiffLastModDateV1 = file1WithDiffLastModDate.getFileVersion(1); FileVersion fileVersion1WithDiffLastModDateV2 = file1WithDiffLastModDate.getFileVersion(2); assertNotNull(fileVersion1OrigV1); assertNotNull(fileVersion1OrigV2); assertNotNull(fileVersion1WithDiffLastModDateV1); assertNotNull(fileVersion1WithDiffLastModDateV2); assertEquals("A-file1.jpg", fileVersion1OrigV1.getName()); assertEquals("A-file1-moved.jpg", fileVersion1OrigV2.getName()); assertEquals( "A-file1-with-different-modified-date.jpg", fileVersion1WithDiffLastModDateV1.getName()); assertEquals( "A-file1-with-different-modified-date-moved.jpg", fileVersion1WithDiffLastModDateV2.getName()); // Tear down clientA.cleanup(); clientB.cleanup(); }