@Override
public void run() {
try {
// check whether the own client is also in the list (should be usually, but you never know...)
this.clientCounter = this.receivers.size();
for (NodeLocation location : this.receivers) {
if (location.getPeerAddress().equals(this.node.getPeerAddress())) {
this.clientCounter--;
break;
}
}
this.chunkCountDownLatch = new CountDownLatch(this.clientCounter);
this.initReceiverLatch.countDown();
// check whether we got access to the file
this.fileId = null;
this.owner = null;
try {
PathObject pathObject =
this.objectStore.getObjectManager().getObjectForPath(this.relativeFilePath);
// if we are not the owner but have access to the file
if (null != pathObject.getOwner()
&& !this.node.getUser().getUserName().equals(pathObject.getOwner())
&& AccessType.WRITE.equals(pathObject.getAccessType())) {
try {
this.fileId = this.node.getIdentifierManager().getValue(this.relativeFilePath);
this.owner = pathObject.getOwner();
} catch (InputOutputException e) {
logger.error(
"Failed to get file id for "
+ this.relativeFilePath
+ ". Message: "
+ e.getMessage());
}
}
// add file id also if the path is shared
if (pathObject.isShared()) {
for (Sharer entry : pathObject.getSharers()) {
try {
// ask sharer's clients to get the changes too
List<NodeLocation> sharerLocations =
this.nodeManager.getNodeLocations(entry.getUsername());
// only add one client of the sharer. He may propagate the change then
// to his clients, and if a conflict occurs, there will be a new file
if (!sharerLocations.isEmpty()) {
fileId = super.node.getIdentifierManager().getValue(pathObject.getAbsolutePath());
// Note that we do not add the sharer location again since these
// are assembled in FileOfferExchangeHandlerResult
}
} catch (InputOutputException e) {
logger.error(
"Could not get client locations of sharer "
+ entry.getUsername()
+ ". This sharer's clients do not get the file (change)");
}
}
}
} catch (InputOutputException e) {
logger.error(
"Failed to read path object for "
+ this.relativeFilePath
+ ". Message: "
+ e.getMessage());
}
// the owner of a file is only added on a share request
for (NodeLocation location : this.receivers) {
UUID uuid = UUID.randomUUID();
logger.info(
"Sending first chunk as subRequest of "
+ this.exchangeId
+ " with id "
+ uuid
+ " to client "
+ location.getPeerAddress().inetAddress().getHostName()
+ ":"
+ location.getPeerAddress().tcpPort());
// add callback handler for sub request
super.node.getObjectDataReplyHandler().addResponseCallbackHandler(uuid, this);
this.sendChunk(
0, // first chunk
this.fileId,
this.owner,
uuid,
location);
}
} catch (Exception e) {
logger.error("Failed to execute FilePushExchangeHandler. Message: " + e.getMessage(), e);
}
}
@BeforeClass
public static void setUpChild() throws IOException, InputOutputException {
// create some test files and dirs to move
// -- testDir1 // mv to targetDir
// | |
// | |--- myFile.txt
// |
// - testDir2
// | |
// | |--- myFile2.txt // mv to targetDir
// | |--- myFile3.txt // mv to targetDir after creating conflict manually
// |
// - targetDir
// |
// - dirToDelete
// | |
// | |--- fileToDeleteInDir.txt
// |
// |--- fileToDelete.txt
Files.createDirectory(ROOT_TEST_DIR1.resolve(TEST_DIR_1));
Files.createDirectory(ROOT_TEST_DIR2.resolve(TEST_DIR_1));
Files.createDirectory(ROOT_TEST_DIR1.resolve(TEST_DIR_3));
Files.createDirectory(ROOT_TEST_DIR2.resolve(TEST_DIR_3));
Files.createDirectory(ROOT_TEST_DIR1.resolve(TEST_DIR_2));
Files.createDirectory(ROOT_TEST_DIR2.resolve(TEST_DIR_2));
Files.createFile(ROOT_TEST_DIR1.resolve(TEST_FILE_1));
Files.createFile(ROOT_TEST_DIR2.resolve(TEST_FILE_1));
Files.createFile(ROOT_TEST_DIR1.resolve(TEST_FILE_2));
Files.createFile(ROOT_TEST_DIR2.resolve(TEST_FILE_2));
Files.createFile(ROOT_TEST_DIR1.resolve(TEST_FILE_3));
Files.createFile(ROOT_TEST_DIR2.resolve(TEST_FILE_3));
Files.createFile(ROOT_TEST_DIR1.resolve(TEST_FILE_4));
Files.createFile(ROOT_TEST_DIR2.resolve(TEST_FILE_4));
Files.createFile(ROOT_TEST_DIR1.resolve(TEST_FILE_5));
Files.createFile(ROOT_TEST_DIR2.resolve(TEST_FILE_5));
// create directory to where the files should be moved
Files.createDirectory(ROOT_TEST_DIR1.resolve(TARGET_DIR));
Files.createDirectory(ROOT_TEST_DIR2.resolve(TARGET_DIR));
// force recreation of object store
OBJECT_STORE_1.sync();
OBJECT_STORE_2.sync();
// do not start the event aggregator but manually sync the event
PathObject pathObject =
OBJECT_STORE_1.getObjectManager().getObjectForPath(TEST_DIR_1.toString());
moveDirEvent =
new MoveEvent(
TEST_DIR_1,
TARGET_DIR,
TARGET_DIR.getFileName().toString(),
pathObject
.getVersions()
.get(Math.max(pathObject.getVersions().size() - 1, 0))
.getHash(),
System.currentTimeMillis());
PathObject fileObject =
OBJECT_STORE_1.getObjectManager().getObjectForPath(TEST_FILE_2.toString());
moveFileEvent =
new MoveEvent(
TEST_FILE_2,
TARGET_DIR.resolve(TEST_FILE_2.getFileName().toString()),
TEST_FILE_2.getFileName().toString(),
fileObject
.getVersions()
.get(Math.max(fileObject.getVersions().size() - 1, 0))
.getHash(),
System.currentTimeMillis());
moveConflictFileEvent =
new MoveEvent(
TEST_FILE_3,
TARGET_DIR.resolve(TEST_FILE_3.getFileName().toString()),
TEST_FILE_3.getFileName().toString(),
fileObject
.getVersions()
.get(Math.max(fileObject.getVersions().size() - 1, 0))
.getHash(),
System.currentTimeMillis());
}
@Override
public void run() {
try {
UUID exchangeId = UUID.randomUUID();
logger.info("Starting non blocking background syncer (Exchange: " + exchangeId + ")");
this.eventAggregator.stop();
FetchObjectStoreExchangeHandler fetchObjectStoreExchangeHandler =
new FetchObjectStoreExchangeHandler(this.node, this.nodeManager, exchangeId);
int nrOfRunningResponseCallbackExchanges =
this.node.getObjectDataReplyHandler().getResponseCallbackHandlers().size();
if (0 < nrOfRunningResponseCallbackExchanges
|| this.node.getObjectDataReplyHandler().areRequestCallbacksRunning()) {
// other callbacks are currently in progress
// -> postpone this run to next interval
logger.info(
"Skipping execution of background synchronisation since other exchanges are currently running");
return;
}
this.node
.getObjectDataReplyHandler()
.addResponseCallbackHandler(exchangeId, fetchObjectStoreExchangeHandler);
Thread fetchObjectStoreExchangeHandlerThread = new Thread(fetchObjectStoreExchangeHandler);
fetchObjectStoreExchangeHandlerThread.setName(
"FetchObjectStoreExchangeHandler-" + exchangeId);
fetchObjectStoreExchangeHandlerThread.start();
logger.info("Waiting for fetching of object stores to complete");
try {
fetchObjectStoreExchangeHandler.await();
} catch (InterruptedException e) {
logger.error("Got interrupted while waiting for fetching all object stores");
}
this.node.getObjectDataReplyHandler().removeResponseCallbackHandler(exchangeId);
if (!fetchObjectStoreExchangeHandler.isCompleted()) {
logger.error(
"FetchObjectStoreExchangeHandler should be completed after awaiting. Since we do not know about the other clients object store, we abort background sync for exchange "
+ exchangeId);
return;
}
FetchObjectStoreExchangeHandlerResult result = fetchObjectStoreExchangeHandler.getResult();
Map<ClientDevice, IObjectStore> objectStores = Zip.unzipObjectStore(this.objectStore, result);
// use a tree map for guaranteed ordering
Map<String, ClientDevice> deletedPaths = new TreeMap<>(new StringLengthComparator());
Map<String, ClientDevice> updatedPaths = new TreeMap<>(new StringLengthComparator());
Map<String, ClientDevice> conflictPaths = new TreeMap<>(new StringLengthComparator());
for (Map.Entry<ClientDevice, IObjectStore> entry : objectStores.entrySet()) {
HashMap<ObjectStore.MergedObjectType, Set<String>> outdatedOrDeletedPaths =
this.objectStore.mergeObjectStore(entry.getValue());
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.CHANGED)
.stream()
.filter(outDatedPath -> !this.isIgnored(outDatedPath))
.forEach(outDatedPath -> updatedPaths.put(outDatedPath, entry.getKey()));
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.DELETED)
.stream()
.filter(deletedPath -> !this.isIgnored(deletedPath))
.forEach(deletedPath -> deletedPaths.put(deletedPath, entry.getKey()));
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.CONFLICT)
.stream()
.filter(conflictPath -> !this.isIgnored(conflictPath))
.forEach(conflictPath -> conflictPaths.put(conflictPath, entry.getKey()));
entry.getValue().getObjectManager().getStorageAdapater().delete(new TreePathElement("./"));
this.objectStore
.getObjectManager()
.getStorageAdapater()
.delete(new TreePathElement(entry.getKey().getClientDeviceId().toString()));
}
// delete all removed files
logger.info("Removing all (" + deletedPaths.size() + ") deleted files");
for (Map.Entry<String, ClientDevice> entry : deletedPaths.entrySet()) {
logger.debug("Removing deleted path " + entry.getKey());
TreePathElement elementToDelete = new TreePathElement(entry.getKey());
// only delete the file on disk if it actually exists
if (this.storageAdapter.exists(StorageType.DIRECTORY, elementToDelete)
|| this.storageAdapter.exists(StorageType.FILE, elementToDelete)) {
this.storageAdapter.delete(elementToDelete);
}
}
logger.info("Creating all (" + conflictPaths.size() + ") conflict files");
for (Map.Entry<String, ClientDevice> entry : conflictPaths.entrySet()) {
logger.debug("Creating conflict file " + entry.getKey());
Path conflictFilePath =
ConflictHandler.createConflictFile(
this.globalEventBus,
this.node.getClientDeviceId().toString(),
this.objectStore,
this.storageAdapter,
new TreePathElement(entry.getKey()));
// we have to emit an ignore event here for the file syncer
// otherwise the final difference calculation will get an updated
// path (i.e. actually an create event) and will try to sync it to the other clients
if (null != conflictFilePath) {
this.globalEventBus.publish(
new IgnoreBusEvent(
new ModifyEvent(
conflictFilePath,
conflictFilePath.getFileName().toString(),
"weIgnoreTheHash",
System.currentTimeMillis())));
}
// now add this to the updated paths, so that we can fetch the original again
updatedPaths.put(entry.getKey(), entry.getValue());
}
// fetch all missing files
logger.info("Fetching all (" + updatedPaths.size() + ") missing files");
for (Map.Entry<String, ClientDevice> entry : updatedPaths.entrySet()) {
UUID subExchangeId = UUID.randomUUID();
logger.debug(
"Starting to fetch file "
+ entry.getKey()
+ " with subExchangeId "
+ subExchangeId
+ " (non blocking background sync "
+ exchangeId
+ ")");
// before updating, check the actual content hash on disk
// to prevent data loss during sync
PathObject mergedPathObject =
this.objectStore.getObjectManager().getObjectForPath(entry.getKey());
Version lastVersion =
mergedPathObject
.getVersions()
.get(Math.max(0, mergedPathObject.getVersions().size() - 1));
PathType pathType = mergedPathObject.getPathType();
StorageType storageType =
pathType.equals(PathType.DIRECTORY) ? StorageType.DIRECTORY : StorageType.FILE;
// only check version, if the file does exist on our disk,
// if not, we have to fetch it anyway
TreePathElement mergedTreeElement = new TreePathElement(mergedPathObject.getAbsolutePath());
if (this.storageAdapter.exists(storageType, mergedTreeElement)) {
this.objectStore.syncFile(mergedTreeElement);
PathObject modifiedPathObject =
this.objectStore.getObjectManager().getObjectForPath(entry.getKey());
Version modifiedLastVersion =
modifiedPathObject
.getVersions()
.get(Math.max(0, modifiedPathObject.getVersions().size() - 1));
if (!modifiedLastVersion.equals(lastVersion)) {
// we just changed the file on this client while syncing...
// therefore we use this state and do not request an outdated state from another client
logger.info(
"Detected file change while merging object store (from other client or end user)... Using our state");
continue;
}
}
// add owner, access type and sharers for object store to prevent overwriting when a file
// is fetched which does not exist yet
this.globalEventBus.publish(
new AddOwnerAndAccessTypeToObjectStoreBusEvent(
mergedPathObject.getOwner(), mergedPathObject.getAccessType(), entry.getKey()));
this.globalEventBus.publish(
new AddSharerToObjectStoreBusEvent(entry.getKey(), mergedPathObject.getSharers()));
FileDemandExchangeHandler fileDemandExchangeHandler =
new FileDemandExchangeHandler(
this.storageAdapter,
this.node,
this.nodeManager,
this.globalEventBus,
new NodeLocation(
entry.getValue().getUserName(),
entry.getValue().getClientDeviceId(),
entry.getValue().getPeerAddress()),
entry.getKey(),
subExchangeId);
this.node
.getObjectDataReplyHandler()
.addResponseCallbackHandler(subExchangeId, fileDemandExchangeHandler);
Thread fileDemandExchangeHandlerThread = new Thread(fileDemandExchangeHandler);
fileDemandExchangeHandlerThread.setName("FileDemandExchangeHandlerThread-" + subExchangeId);
fileDemandExchangeHandlerThread.start();
try {
fileDemandExchangeHandler.await();
} catch (InterruptedException e) {
logger.error(
"Got interrupted while waiting for fileDemandExchangeHandler "
+ subExchangeId
+ " to complete. Message: "
+ e.getMessage());
}
this.node.getObjectDataReplyHandler().removeResponseCallbackHandler(subExchangeId);
if (!fileDemandExchangeHandler.isCompleted()) {
logger.error(
"FileDemandExchangeHandler " + subExchangeId + " should be completed after wait.");
}
}
// start event aggregator
logger.info(
"Starting event aggregator on client ("
+ this.node.getPeerAddress().inetAddress().getHostName()
+ ":"
+ this.node.getPeerAddress().tcpPort()
+ "): Non-blocking background sync "
+ exchangeId);
this.eventAggregator.start();
logger.info(
"Reconciling local disk changes with merged object store (non-blocking background sync "
+ exchangeId
+ ")");
// create a temporary second object store to get changes made in the mean time of syncing
ITreeStorageAdapter objectStoreStorageManager =
this.objectStore.getObjectManager().getStorageAdapater();
TreePathElement pathElement = new TreePathElement("nonBlockingBackgroundSyncObjectStore");
if (objectStoreStorageManager.exists(StorageType.DIRECTORY, pathElement)) {
objectStoreStorageManager.delete(pathElement);
}
objectStoreStorageManager.persist(StorageType.DIRECTORY, pathElement, null);
// create the temporary object store in the .sync folder
ITreeStorageAdapter changeObjectStoreStorageManager =
new LocalStorageAdapter(
Paths.get(objectStoreStorageManager.getRootDir().getPath())
.resolve(pathElement.getPath()));
IObjectStore changeObjectStore =
new ObjectStore(
this.storageAdapter, "index.json", "object", changeObjectStoreStorageManager);
// build object store for differences in the mean time
List<String> ignoredPaths = new ArrayList<>();
Path origSyncFolder =
Paths.get(this.objectStore.getObjectManager().getStorageAdapater().getRootDir().getPath())
.getFileName();
ignoredPaths.add(origSyncFolder.toString());
changeObjectStore.sync(ignoredPaths);
// get differences between disk and merged object store
HashMap<ObjectStore.MergedObjectType, Set<String>> updatedOrDeletedPaths =
this.objectStore.mergeObjectStore(changeObjectStore);
// remove change object store again
changeObjectStoreStorageManager.delete(new TreePathElement("./"));
Set<String> deletedPathsInTheMeanTime =
updatedOrDeletedPaths.get(ObjectStore.MergedObjectType.DELETED);
Set<String> updatedPathsInTheMeanTime =
updatedOrDeletedPaths.get(ObjectStore.MergedObjectType.CHANGED);
logger.info(
"Found "
+ deletedPathsInTheMeanTime.size()
+ " paths which have been deleted in the mean time of syncing");
for (String deletedPath : deletedPathsInTheMeanTime) {
if (this.isIgnored(deletedPath)) {
logger.info("Ignore deletion of " + deletedPath + " since it matches an ignore pattern");
}
// publish a delete event to the SyncFileChangeListener
logger.trace("Creating delete event for " + deletedPath);
this.globalEventBus.publish(
new CreateBusEvent(
new DeleteEvent(
Paths.get(deletedPath),
Paths.get(deletedPath).getFileName().toString(),
null,
System.currentTimeMillis())));
}
logger.info(
"Found "
+ updatedPathsInTheMeanTime.size()
+ " paths which have changed in the mean time of syncing");
for (String updatedPath : updatedPathsInTheMeanTime) {
if (this.isIgnored(updatedPath)) {
logger.info("Ignore updating of " + updatedPath + " since it matches an ignore pattern");
}
// publish modify events to SyncFileChangeListener
logger.trace("Creating modify event for " + updatedPath);
PathObject updatedPathObject =
this.objectStore.getObjectManager().getObjectForPath(updatedPath);
this.globalEventBus.publish(
new CreateBusEvent(
new ModifyEvent(
Paths.get(updatedPath),
Paths.get(updatedPath).getFileName().toString(),
updatedPathObject
.getVersions()
.get(Math.max(0, updatedPathObject.getVersions().size() - 1))
.getHash(),
System.currentTimeMillis())));
}
logger.info("Completed non-blocking background sync " + exchangeId);
} catch (Exception e) {
logger.error("Got exception in NonBlockingBackgroundSyncer. Message: " + e.getMessage(), e);
}
}