@Override
public void onMultiChunkWrite(MultiChunk multiChunk, Chunk chunk) {
logger.log(
Level.FINER,
"- Chunk > MultiChunk: {0} > {1}",
new Object[] {StringUtil.toHex(chunk.getChecksum()), multiChunk.getId()});
multiChunkEntry.addChunk(chunkEntry.getChecksum());
}
@Override
public void onMultiChunkClose(MultiChunk multiChunk) {
logger.log(Level.FINER, "- /MultiChunk {0}", multiChunk.getId());
multiChunkEntry.setSize(multiChunk.getSize());
newDatabaseVersion.addMultiChunk(multiChunkEntry);
multiChunkEntry = null;
}
private Collection<MultiChunkEntry> determineMultiChunksToDownload(
FileVersion fileVersion, Database localDatabase, Database winnersDatabase) {
Set<MultiChunkEntry> multiChunksToDownload = new HashSet<MultiChunkEntry>();
FileContent winningFileContent = localDatabase.getContent(fileVersion.getChecksum());
if (winningFileContent == null) {
winningFileContent = winnersDatabase.getContent(fileVersion.getChecksum());
}
boolean winningFileHasContent = winningFileContent != null;
if (winningFileHasContent) { // File can be empty!
Collection<ChunkEntryId> fileChunks =
winningFileContent
.getChunks(); // TODO [medium] Instead of just looking for multichunks to download
// here, we should look for chunks in local files as well and return the
// chunk positions in the local files ChunkPosition (chunk123 at file12,
// offset 200, size 250)
for (ChunkEntryId chunkChecksum : fileChunks) {
MultiChunkEntry multiChunkForChunk = localDatabase.getMultiChunkForChunk(chunkChecksum);
if (multiChunkForChunk == null) {
multiChunkForChunk = winnersDatabase.getMultiChunkForChunk(chunkChecksum);
}
if (!multiChunksToDownload.contains(multiChunkForChunk)) {
logger.log(
Level.INFO,
" + Adding multichunk "
+ StringUtil.toHex(multiChunkForChunk.getId())
+ " to download list ...");
multiChunksToDownload.add(multiChunkForChunk);
}
}
}
return multiChunksToDownload;
}
@Test
public void testMultiChunkCache() throws IOException {
Database database = new Database();
// Round 1: Add chunk to multichunk
DatabaseVersion databaseVersion1 = TestDatabaseUtil.createDatabaseVersion();
MultiChunkEntry multiChunkP1 = new MultiChunkEntry(new byte[] {8, 8, 8, 8, 8, 8, 8, 8});
ChunkEntry chunkA1 = new ChunkEntry(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}, 12);
multiChunkP1.addChunk(new ChunkEntryId(chunkA1.getChecksum()));
databaseVersion1.addChunk(chunkA1);
databaseVersion1.addMultiChunk(multiChunkP1);
database.addDatabaseVersion(databaseVersion1);
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}));
// Round 2: Add chunk to multichunk
DatabaseVersion databaseVersion2 = TestDatabaseUtil.createDatabaseVersion(databaseVersion1);
MultiChunkEntry multiChunkP2 = new MultiChunkEntry(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7});
MultiChunkEntry multiChunkP3 = new MultiChunkEntry(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5});
ChunkEntry chunkA2 = new ChunkEntry(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}, 912);
ChunkEntry chunkA3 = new ChunkEntry(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}, 812);
ChunkEntry chunkA4 = new ChunkEntry(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}, 712);
multiChunkP2.addChunk(new ChunkEntryId(chunkA2.getChecksum()));
multiChunkP2.addChunk(new ChunkEntryId(chunkA3.getChecksum()));
multiChunkP3.addChunk(new ChunkEntryId(chunkA4.getChecksum()));
databaseVersion2.addChunk(chunkA2);
databaseVersion2.addChunk(chunkA3);
databaseVersion2.addChunk(chunkA4);
databaseVersion2.addMultiChunk(multiChunkP2);
databaseVersion2.addMultiChunk(multiChunkP3);
database.addDatabaseVersion(databaseVersion2);
// fail("xx");
assertEquals(chunkA1, database.getChunk(new byte[] {1, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA2, database.getChunk(new byte[] {9, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA3, database.getChunk(new byte[] {8, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(chunkA4, database.getChunk(new byte[] {7, 2, 3, 4, 5, 7, 8, 9, 0}));
assertEquals(multiChunkP1, database.getMultiChunk(new byte[] {8, 8, 8, 8, 8, 8, 8, 8}));
assertEquals(multiChunkP2, database.getMultiChunk(new byte[] {7, 7, 7, 7, 7, 7, 7, 7, 7}));
assertEquals(multiChunkP3, database.getMultiChunk(new byte[] {5, 5, 5, 5, 5, 5, 5, 5, 5}));
}
private void downloadAndDecryptMultiChunks(Set<MultiChunkEntry> unknownMultiChunks)
throws StorageException, IOException {
logger.log(Level.INFO, "- Downloading and extracting multichunks ...");
TransferManager transferManager = config.getConnection().createTransferManager();
// TODO [medium] Check existing files by checksum and do NOT download them if they exist
// locally, or copy them
for (MultiChunkEntry multiChunkEntry : unknownMultiChunks) {
File localEncryptedMultiChunkFile =
config.getCache().getEncryptedMultiChunkFile(multiChunkEntry.getId());
File localDecryptedMultiChunkFile =
config.getCache().getDecryptedMultiChunkFile(multiChunkEntry.getId());
MultiChunkRemoteFile remoteMultiChunkFile =
new MultiChunkRemoteFile(
localEncryptedMultiChunkFile
.getName()); // TODO [low] Make MultiChunkRemoteFile class, or something like that
logger.log(
Level.INFO,
" + Downloading multichunk " + StringUtil.toHex(multiChunkEntry.getId()) + " ...");
transferManager.download(remoteMultiChunkFile, localEncryptedMultiChunkFile);
result.downloadedMultiChunks.add(multiChunkEntry);
logger.log(
Level.INFO,
" + Decrypting multichunk " + StringUtil.toHex(multiChunkEntry.getId()) + " ...");
InputStream multiChunkInputStream =
config
.getTransformer()
.createInputStream(new FileInputStream(localEncryptedMultiChunkFile));
OutputStream decryptedMultiChunkOutputStream =
new FileOutputStream(localDecryptedMultiChunkFile);
// TODO [medium] Calculate checksum while writing file, to verify correct content
FileUtil.appendToOutputStream(multiChunkInputStream, decryptedMultiChunkOutputStream);
decryptedMultiChunkOutputStream.close();
multiChunkInputStream.close();
logger.log(
Level.FINE,
" + Locally deleting multichunk " + StringUtil.toHex(multiChunkEntry.getId()) + " ...");
localEncryptedMultiChunkFile.delete();
}
transferManager.disconnect();
}
