public static void assertDatabaseEquals(
      MemoryDatabase expectedDatabase, MemoryDatabase actualDatabase) {
    logger.log(Level.INFO, "--");
    logger.log(Level.INFO, "Now comparing two databases.");
    logger.log(
        Level.INFO, "DON'T WORRY. This can take a long time or even overload the heap space.");

    List<DatabaseVersion> writtenDatabaseVersions = expectedDatabase.getDatabaseVersions();
    List<DatabaseVersion> readDatabaseVersions = actualDatabase.getDatabaseVersions();

    assertEquals(
        "Different number of database versions.",
        writtenDatabaseVersions.size(),
        readDatabaseVersions.size());

    for (DatabaseVersion writtenDatabaseVersion : writtenDatabaseVersions) {
      DatabaseVersion readDatabaseVersion = null;

      for (DatabaseVersion aReadDatabaseVersion : readDatabaseVersions) {
        if (aReadDatabaseVersion.equals(writtenDatabaseVersion)) {
          readDatabaseVersion = aReadDatabaseVersion;
          break;
        }
      }

      assertNotNull(
          "Database version " + writtenDatabaseVersion + " does not exist in read database.",
          readDatabaseVersion);

      assertDatabaseVersionEquals(writtenDatabaseVersion, readDatabaseVersion);
    }

    logger.log(Level.INFO, "End of comparing databases");
    logger.log(Level.INFO, "--");
  }
Example #2
0
  @Test
  public void testContentChecksumCache() throws IOException {
    Database database = new Database();

    // Round 1: Add file history & version
    DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();

    // - history 1, version 1
    FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("samechecksum1.jpg");
    fileVersion1.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0});

    PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);

    databaseVersion1.addFileHistory(fileHistory1);
    databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);

    database.addDatabaseVersion(databaseVersion1);

    assertNotNull(database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}));
    assertEquals(1, database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).size());

    // Round 2: Add two other versions with same checksum to new database version
    DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);

    // - history 1, version 2
    FileVersion fileVersion11 =
        TestDatabaseUtil.createFileVersion("samechecksum2-renamed.jpg", fileVersion1);
    fileVersion11.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!
    fileVersion11.setStatus(FileStatus.RENAMED);

    PartialFileHistory fileHistory11 =
        new PartialFileHistory(11111111111111111L); // same ID as above	

    databaseVersion2.addFileHistory(fileHistory11);
    databaseVersion2.addFileVersionToHistory(fileHistory11.getFileId(), fileVersion11);

    // - history 2, version 1
    FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("samechecksum2.jpg");
    fileVersion2.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!

    PartialFileHistory fileHistory2 = new PartialFileHistory(22222222222222222L); // different ID	

    databaseVersion2.addFileHistory(fileHistory2);
    databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);

    // - history 3, version 1
    FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("samechecksum3.jpg");
    fileVersion3.setChecksum(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}); // same checksum!

    PartialFileHistory fileHistory3 = new PartialFileHistory(33333333333333333L); // different ID	

    databaseVersion2.addFileHistory(fileHistory3);
    databaseVersion2.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);

    database.addDatabaseVersion(databaseVersion2);

    assertNotNull(database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}));
    assertEquals(3, database.getFileHistories(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).size());
  }
Example #3
0
    private void addFileVersion(FileProperties fileProperties) {
      if (fileProperties.getChecksum() != null) {
        logger.log(
            Level.FINER,
            "- /File: {0} (checksum {1})",
            new Object[] {fileProperties.getRelativePath(), fileProperties.getChecksum()});
      } else {
        logger.log(
            Level.FINER,
            "- /File: {0} (directory/symlink/0-byte-file)",
            fileProperties.getRelativePath());
      }

      // 1. Determine if file already exists in database
      PartialFileHistory lastFileHistory = guessLastFileHistory(fileProperties);
      FileVersion lastFileVersion =
          (lastFileHistory != null) ? lastFileHistory.getLastVersion() : null;

      // 2. Create new file history/version
      PartialFileHistory fileHistory = createNewFileHistory(lastFileHistory);
      FileVersion fileVersion = createNewFileVersion(lastFileVersion, fileProperties);

      // 3. Compare new and last version
      FileProperties lastFileVersionProperties =
          fileVersionComparator.captureFileProperties(lastFileVersion);
      FileVersionComparison lastToNewFileVersionComparison =
          fileVersionComparator.compare(fileProperties, lastFileVersionProperties, true);

      boolean newVersionDiffersFromToLastVersion = !lastToNewFileVersionComparison.equals();

      if (newVersionDiffersFromToLastVersion) {
        fileHistory.addFileVersion(fileVersion);
        newDatabaseVersion.addFileHistory(fileHistory);

        logger.log(Level.INFO, "   * Added file version:    " + fileVersion);
        logger.log(Level.INFO, "     based on file version: " + lastFileVersion);
      } else {
        logger.log(Level.INFO, "   * NOT ADDING file version: " + fileVersion);
        logger.log(Level.INFO, "         b/c IDENTICAL prev.: " + lastFileVersion);
      }

      // 4. Add file content (if not a directory)
      if (fileProperties.getChecksum() != null && fileContent != null) {
        fileContent.setSize(fileProperties.getSize());
        fileContent.setChecksum(fileProperties.getChecksum());

        // Check if content already exists, throw gathered content away if it does!
        FileContent existingContent =
            localDatabase.getFileContent(fileProperties.getChecksum(), false);

        if (existingContent == null) {
          newDatabaseVersion.addFileContent(fileContent);
        } else {
          // Uses existing content (already in database); ref. by checksum
        }
      }
    }
 public static void assertDatabaseVersionEquals(
     DatabaseVersion expectedDatabaseVersion, DatabaseVersion actualDatabaseVersion) {
   assertVectorClockEquals(
       expectedDatabaseVersion.getVectorClock(), actualDatabaseVersion.getVectorClock());
   compareDatabaseVersionChunks(
       expectedDatabaseVersion.getChunks(), actualDatabaseVersion.getChunks());
   compareDatabaseVersionMultiChunks(
       expectedDatabaseVersion.getMultiChunks(), actualDatabaseVersion.getMultiChunks());
   compareDatabaseVersionFileContents(
       expectedDatabaseVersion.getFileContents(), actualDatabaseVersion.getFileContents());
   compareDatabaseVersionFileHistories(
       expectedDatabaseVersion.getFileHistories(), actualDatabaseVersion.getFileHistories());
 }
Example #5
0
  public static DatabaseVersion createDatabaseVersion(
      DatabaseVersionHeader basedOnDatabaseVersionHeader, Date date) {
    VectorClock vectorClock =
        (basedOnDatabaseVersionHeader != null)
            ? basedOnDatabaseVersionHeader.getVectorClock().clone()
            : new VectorClock();
    vectorClock.incrementClock("someclient");

    DatabaseVersion databaseVersion = new DatabaseVersion();

    databaseVersion.setClient("someclient");
    databaseVersion.setTimestamp(date);
    databaseVersion.setVectorClock(vectorClock);

    return databaseVersion;
  }
Example #6
0
 public static DatabaseVersion createDatabaseVersion(DatabaseVersion basedOnDatabaseVersion) {
   if (basedOnDatabaseVersion == null) {
     return createDatabaseVersion(null, new Date());
   } else {
     return createDatabaseVersion(basedOnDatabaseVersion.getHeader(), new Date());
   }
 }
Example #7
0
    @Override
    public void onMultiChunkClose(MultiChunk multiChunk) {
      logger.log(Level.FINER, "- /MultiChunk {0}", multiChunk.getId());

      multiChunkEntry.setSize(multiChunk.getSize());

      newDatabaseVersion.addMultiChunk(multiChunkEntry);
      multiChunkEntry = null;
    }
Example #8
0
  private void removeDeletedFiles(
      DatabaseVersion newDatabaseVersion, List<PartialFileHistory> fileHistoriesWithLastVersion) {
    logger.log(Level.FINER, "- Looking for deleted files ...");

    for (PartialFileHistory fileHistory : fileHistoriesWithLastVersion) {
      // Ignore this file history if it has been updated in this database version before (file
      // probably renamed!)
      if (newDatabaseVersion.getFileHistory(fileHistory.getFileHistoryId()) != null) {
        continue;
      }

      // Check if file exists, remove if it doesn't
      FileVersion lastLocalVersion = fileHistory.getLastVersion();
      File lastLocalVersionOnDisk =
          new File(config.getLocalDir() + File.separator + lastLocalVersion.getPath());

      // Ignore this file history if the last version is marked "DELETED"
      if (lastLocalVersion.getStatus() == FileStatus.DELETED) {
        continue;
      }

      // Add this file history if a new file with this name has been added (file type change)
      PartialFileHistory newFileWithSameName =
          getFileHistoryByPathFromDatabaseVersion(
              newDatabaseVersion, fileHistory.getLastVersion().getPath());

      // If file has VANISHED, mark as DELETED
      if (!FileUtil.exists(lastLocalVersionOnDisk) || newFileWithSameName != null) {
        PartialFileHistory deletedFileHistory =
            new PartialFileHistory(fileHistory.getFileHistoryId());
        FileVersion deletedVersion = lastLocalVersion.clone();

        deletedVersion.setStatus(FileStatus.DELETED);
        deletedVersion.setVersion(fileHistory.getLastVersion().getVersion() + 1);
        deletedVersion.setUpdated(new Date());

        logger.log(Level.FINER, "  + Deleted: Adding DELETED version: {0}", deletedVersion);
        logger.log(Level.FINER, "                           based on: {0}", lastLocalVersion);

        deletedFileHistory.addFileVersion(deletedVersion);
        newDatabaseVersion.addFileHistory(deletedFileHistory);
      }
    }
  }
Example #9
0
  @Test
  public void testFilenameCache() throws IOException {
    Database database = new Database();

    // Round 1: Add file history & version
    DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();

    FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("file1.jpg");
    PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);

    databaseVersion1.addFileHistory(fileHistory1);
    databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);

    database.addDatabaseVersion(databaseVersion1);

    assertEquals(fileHistory1, database.getFileHistory("file1.jpg"));

    // Round 2: Add new version
    DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);

    FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("file2.jpg", fileVersion1);
    PartialFileHistory fileHistory2 = new PartialFileHistory(11111111111111111L); // same ID	

    databaseVersion2.addFileHistory(fileHistory2);
    databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);

    database.addDatabaseVersion(databaseVersion2);

    assertNotNull(database.getFileHistory("file2.jpg"));
    assertEquals(2, database.getFileHistory("file2.jpg").getFileVersions().size());
    assertNull(database.getFileHistory("file1.jpg"));

    // Round 3: Add deleted version
    DatabaseVersion databaseVersion3 = TestDatabaseUtil.createDatabaseVersion(databaseVersion2);

    FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("file2.jpg", fileVersion2);
    fileVersion3.setStatus(FileStatus.DELETED);

    PartialFileHistory fileHistory3 = new PartialFileHistory(11111111111111111L); // same ID	

    databaseVersion3.addFileHistory(fileHistory3);
    databaseVersion3.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);

    database.addDatabaseVersion(databaseVersion3);

    assertNull(database.getFileHistory("file2.jpg"));
  }
Example #10
0
    /**
     * Checks if chunk already exists in all database versions Afterwards checks if chunk exists in
     * new introduced database version.
     */
    @Override
    public boolean onChunk(Chunk chunk) {
      ChunkChecksum chunkChecksum = new ChunkChecksum(chunk.getChecksum());
      chunkEntry = localDatabase.getChunk(chunkChecksum);

      if (chunkEntry == null) {
        chunkEntry = newDatabaseVersion.getChunk(chunkChecksum);

        if (chunkEntry == null) {
          logger.log(Level.FINER, "- Chunk new: {0}", chunkChecksum.toString());

          chunkEntry = new ChunkEntry(chunkChecksum, chunk.getSize());
          newDatabaseVersion.addChunk(chunkEntry);

          return true;
        }
      }

      logger.log(Level.FINER, "- Chunk exists: {0}", StringUtil.toHex(chunk.getChecksum()));
      return false;
    }
Example #11
0
  @Test
  public void testChunkCache() throws IOException {
    Database database = new Database();

    // Round 1: Add chunk to new database version, then add database version
    DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();

    ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12);
    databaseVersion1.addChunk(chunkA1);

    database.addDatabaseVersion(databaseVersion1);
    assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));

    // Round 2: Add chunk to new database version, then add database version
    DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);

    ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}, 112);
    databaseVersion2.addChunk(chunkA2);

    database.addDatabaseVersion(databaseVersion2);
    assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
    assertEquals(chunkA2, database.getChunk(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}));

    // Round 3: Add chunk to new database version, then add database version
    DatabaseVersion databaseVersion3 = TestDatabaseUtil.createDatabaseVersion(databaseVersion2);

    ChunkEntry chunkA3 = new ChunkEntry(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1}, 192);
    databaseVersion3.addChunk(chunkA3);

    database.addDatabaseVersion(databaseVersion3);
    assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
    assertEquals(chunkA2, database.getChunk(new byte[] {9, 8, 7, 6, 5, 4, 3, 2, 1}));
    assertEquals(chunkA3, database.getChunk(new byte[] {1, 1, 1, 1, 1, 1, 1, 1, 1}));
  }
Example #12
0
  @Test
  public void testMultiChunkCache() throws IOException {
    Database database = new Database();

    // Round 1: Add chunk to multichunk
    DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();

    MultiChunkEntry multiChunkP1 = new MultiChunkEntry(new byte[] {8, 8, 8, 8, 8, 8, 8, 8});
    ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12);

    multiChunkP1.addChunk(new ChunkEntryId(chunkA1.getChecksum()));
    databaseVersion1.addChunk(chunkA1);
    databaseVersion1.addMultiChunk(multiChunkP1);

    database.addDatabaseVersion(databaseVersion1);

    assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
    assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}));

    // Round 2: Add chunk to multichunk
    DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);

    MultiChunkEntry multiChunkP2 = new MultiChunkEntry(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7});
    MultiChunkEntry multiChunkP3 = new MultiChunkEntry(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5});

    ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}, 912);
    ChunkEntry chunkA3 = new ChunkEntry(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}, 812);
    ChunkEntry chunkA4 = new ChunkEntry(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}, 712);

    multiChunkP2.addChunk(new ChunkEntryId(chunkA2.getChecksum()));
    multiChunkP2.addChunk(new ChunkEntryId(chunkA3.getChecksum()));
    multiChunkP3.addChunk(new ChunkEntryId(chunkA4.getChecksum()));

    databaseVersion2.addChunk(chunkA2);
    databaseVersion2.addChunk(chunkA3);
    databaseVersion2.addChunk(chunkA4);

    databaseVersion2.addMultiChunk(multiChunkP2);
    databaseVersion2.addMultiChunk(multiChunkP3);

    database.addDatabaseVersion(databaseVersion2);

    // fail("xx");

    assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
    assertEquals(chunkA2, database.getChunk(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}));
    assertEquals(chunkA3, database.getChunk(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}));
    assertEquals(chunkA4, database.getChunk(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}));
    assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}));
    assertEquals(multiChunkP2, database.getMultiChunk(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7}));
    assertEquals(multiChunkP3, database.getMultiChunk(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5}));
  }
Example #13
0
  private PartialFileHistory getFileHistoryByPathFromDatabaseVersion(
      DatabaseVersion databaseVersion, String path) {
    // TODO [medium] Extremely performance intensive, because this is called inside a loop above.
    // Implement better caching for database version!!!

    for (PartialFileHistory fileHistory : databaseVersion.getFileHistories()) {
      FileVersion lastVersion = fileHistory.getLastVersion();

      if (lastVersion.getStatus() != FileStatus.DELETED && lastVersion.getPath().equals(path)) {
        return fileHistory;
      }
    }

    return null;
  }
Example #14
0
  private Branches readUnknownDatabaseVersionHeaders(List<File> remoteDatabases)
      throws IOException {
    logger.log(Level.INFO, "Loading database headers, creating branches ...");
    // Sort files (db-a-1 must be before db-a-2 !)
    Collections.sort(
        remoteDatabases); // TODO [medium] natural sort is a workaround, database file names should
    // be centrally managed, db-name-0000000009 avoids natural sort

    // Read database files
    Branches unknownRemoteBranches = new Branches();
    DatabaseDAO dbDAO = new XmlDatabaseDAO(config.getTransformer());

    for (File remoteDatabaseFileInCache : remoteDatabases) {
      Database remoteDatabase =
          new Database(); // Database cannot be reused, since these might be different clients

      RemoteDatabaseFile remoteDatabaseFile = new RemoteDatabaseFile(remoteDatabaseFileInCache);
      dbDAO.load(
          remoteDatabase,
          remoteDatabaseFile
              .getFile()); // TODO [medium] Performance: This is very, very, very inefficient, DB is
      // loaded and then discarded
      List<DatabaseVersion> remoteDatabaseVersions = remoteDatabase.getDatabaseVersions();

      // Populate branches
      Branch remoteClientBranch =
          unknownRemoteBranches.getBranch(remoteDatabaseFile.getClientName(), true);

      for (DatabaseVersion remoteDatabaseVersion : remoteDatabaseVersions) {
        DatabaseVersionHeader header = remoteDatabaseVersion.getHeader();
        remoteClientBranch.add(header);
      }
    }

    return unknownRemoteBranches;
  }
Example #15
0
  @Test
  public void testFilenameCacheDeleteAndNewOfSameFileInOneDatabaseVersion() throws IOException {
    Database database = new Database();

    // Round 1: Add file history & version
    DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();

    FileVersion fileVersion1 = TestDatabaseUtil.createFileVersion("file1.jpg");
    PartialFileHistory fileHistory1 = new PartialFileHistory(11111111111111111L);

    databaseVersion1.addFileHistory(fileHistory1);
    databaseVersion1.addFileVersionToHistory(fileHistory1.getFileId(), fileVersion1);

    database.addDatabaseVersion(databaseVersion1);

    assertEquals(fileHistory1, database.getFileHistory("file1.jpg"));

    // Round 2: Add new version
    DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);

    // - delete file1.jpg
    FileVersion fileVersion2 = TestDatabaseUtil.createFileVersion("file1.jpg", fileVersion1);
    fileVersion2.setStatus(FileStatus.DELETED);

    PartialFileHistory fileHistory2 = new PartialFileHistory(11111111111111111L); // same ID	

    databaseVersion2.addFileHistory(fileHistory2);
    databaseVersion2.addFileVersionToHistory(fileHistory2.getFileId(), fileVersion2);

    // - add file1.jpg (as FOLDER!)
    FileVersion fileVersion3 = TestDatabaseUtil.createFileVersion("file1.jpg"); // new file!
    fileVersion3.setType(FileType.FOLDER);

    PartialFileHistory fileHistory3 = new PartialFileHistory(222222222L); // new ID	!

    databaseVersion2.addFileHistory(fileHistory3);
    databaseVersion2.addFileVersionToHistory(fileHistory3.getFileId(), fileVersion3);

    // - add datbase version
    database.addDatabaseVersion(databaseVersion2);

    assertNotNull(database.getFileHistory("file1.jpg"));
    assertEquals(1, database.getFileHistory("file1.jpg").getFileVersions().size());
    assertEquals(fileHistory3, database.getFileHistory("file1.jpg"));
  }