@Override
public void run() {
try {
UUID exchangeId = UUID.randomUUID();
logger.info("Starting non blocking background syncer (Exchange: " + exchangeId + ")");
this.eventAggregator.stop();
FetchObjectStoreExchangeHandler fetchObjectStoreExchangeHandler =
new FetchObjectStoreExchangeHandler(this.node, this.nodeManager, exchangeId);
int nrOfRunningResponseCallbackExchanges =
this.node.getObjectDataReplyHandler().getResponseCallbackHandlers().size();
if (0 < nrOfRunningResponseCallbackExchanges
|| this.node.getObjectDataReplyHandler().areRequestCallbacksRunning()) {
// other callbacks are currently in progress
// -> postpone this run to next interval
logger.info(
"Skipping execution of background synchronisation since other exchanges are currently running");
return;
}
this.node
.getObjectDataReplyHandler()
.addResponseCallbackHandler(exchangeId, fetchObjectStoreExchangeHandler);
Thread fetchObjectStoreExchangeHandlerThread = new Thread(fetchObjectStoreExchangeHandler);
fetchObjectStoreExchangeHandlerThread.setName(
"FetchObjectStoreExchangeHandler-" + exchangeId);
fetchObjectStoreExchangeHandlerThread.start();
logger.info("Waiting for fetching of object stores to complete");
try {
fetchObjectStoreExchangeHandler.await();
} catch (InterruptedException e) {
logger.error("Got interrupted while waiting for fetching all object stores");
}
this.node.getObjectDataReplyHandler().removeResponseCallbackHandler(exchangeId);
if (!fetchObjectStoreExchangeHandler.isCompleted()) {
logger.error(
"FetchObjectStoreExchangeHandler should be completed after awaiting. Since we do not know about the other clients object store, we abort background sync for exchange "
+ exchangeId);
return;
}
FetchObjectStoreExchangeHandlerResult result = fetchObjectStoreExchangeHandler.getResult();
Map<ClientDevice, IObjectStore> objectStores = Zip.unzipObjectStore(this.objectStore, result);
// use a tree map for guaranteed ordering
Map<String, ClientDevice> deletedPaths = new TreeMap<>(new StringLengthComparator());
Map<String, ClientDevice> updatedPaths = new TreeMap<>(new StringLengthComparator());
Map<String, ClientDevice> conflictPaths = new TreeMap<>(new StringLengthComparator());
for (Map.Entry<ClientDevice, IObjectStore> entry : objectStores.entrySet()) {
HashMap<ObjectStore.MergedObjectType, Set<String>> outdatedOrDeletedPaths =
this.objectStore.mergeObjectStore(entry.getValue());
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.CHANGED)
.stream()
.filter(outDatedPath -> !this.isIgnored(outDatedPath))
.forEach(outDatedPath -> updatedPaths.put(outDatedPath, entry.getKey()));
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.DELETED)
.stream()
.filter(deletedPath -> !this.isIgnored(deletedPath))
.forEach(deletedPath -> deletedPaths.put(deletedPath, entry.getKey()));
outdatedOrDeletedPaths
.get(ObjectStore.MergedObjectType.CONFLICT)
.stream()
.filter(conflictPath -> !this.isIgnored(conflictPath))
.forEach(conflictPath -> conflictPaths.put(conflictPath, entry.getKey()));
entry.getValue().getObjectManager().getStorageAdapater().delete(new TreePathElement("./"));
this.objectStore
.getObjectManager()
.getStorageAdapater()
.delete(new TreePathElement(entry.getKey().getClientDeviceId().toString()));
}
// delete all removed files
logger.info("Removing all (" + deletedPaths.size() + ") deleted files");
for (Map.Entry<String, ClientDevice> entry : deletedPaths.entrySet()) {
logger.debug("Removing deleted path " + entry.getKey());
TreePathElement elementToDelete = new TreePathElement(entry.getKey());
// only delete the file on disk if it actually exists
if (this.storageAdapter.exists(StorageType.DIRECTORY, elementToDelete)
|| this.storageAdapter.exists(StorageType.FILE, elementToDelete)) {
this.storageAdapter.delete(elementToDelete);
}
}
logger.info("Creating all (" + conflictPaths.size() + ") conflict files");
for (Map.Entry<String, ClientDevice> entry : conflictPaths.entrySet()) {
logger.debug("Creating conflict file " + entry.getKey());
Path conflictFilePath =
ConflictHandler.createConflictFile(
this.globalEventBus,
this.node.getClientDeviceId().toString(),
this.objectStore,
this.storageAdapter,
new TreePathElement(entry.getKey()));
// we have to emit an ignore event here for the file syncer
// otherwise the final difference calculation will get an updated
// path (i.e. actually an create event) and will try to sync it to the other clients
if (null != conflictFilePath) {
this.globalEventBus.publish(
new IgnoreBusEvent(
new ModifyEvent(
conflictFilePath,
conflictFilePath.getFileName().toString(),
"weIgnoreTheHash",
System.currentTimeMillis())));
}
// now add this to the updated paths, so that we can fetch the original again
updatedPaths.put(entry.getKey(), entry.getValue());
}
// fetch all missing files
logger.info("Fetching all (" + updatedPaths.size() + ") missing files");
for (Map.Entry<String, ClientDevice> entry : updatedPaths.entrySet()) {
UUID subExchangeId = UUID.randomUUID();
logger.debug(
"Starting to fetch file "
+ entry.getKey()
+ " with subExchangeId "
+ subExchangeId
+ " (non blocking background sync "
+ exchangeId
+ ")");
// before updating, check the actual content hash on disk
// to prevent data loss during sync
PathObject mergedPathObject =
this.objectStore.getObjectManager().getObjectForPath(entry.getKey());
Version lastVersion =
mergedPathObject
.getVersions()
.get(Math.max(0, mergedPathObject.getVersions().size() - 1));
PathType pathType = mergedPathObject.getPathType();
StorageType storageType =
pathType.equals(PathType.DIRECTORY) ? StorageType.DIRECTORY : StorageType.FILE;
// only check version, if the file does exist on our disk,
// if not, we have to fetch it anyway
TreePathElement mergedTreeElement = new TreePathElement(mergedPathObject.getAbsolutePath());
if (this.storageAdapter.exists(storageType, mergedTreeElement)) {
this.objectStore.syncFile(mergedTreeElement);
PathObject modifiedPathObject =
this.objectStore.getObjectManager().getObjectForPath(entry.getKey());
Version modifiedLastVersion =
modifiedPathObject
.getVersions()
.get(Math.max(0, modifiedPathObject.getVersions().size() - 1));
if (!modifiedLastVersion.equals(lastVersion)) {
// we just changed the file on this client while syncing...
// therefore we use this state and do not request an outdated state from another client
logger.info(
"Detected file change while merging object store (from other client or end user)... Using our state");
continue;
}
}
// add owner, access type and sharers for object store to prevent overwriting when a file
// is fetched which does not exist yet
this.globalEventBus.publish(
new AddOwnerAndAccessTypeToObjectStoreBusEvent(
mergedPathObject.getOwner(), mergedPathObject.getAccessType(), entry.getKey()));
this.globalEventBus.publish(
new AddSharerToObjectStoreBusEvent(entry.getKey(), mergedPathObject.getSharers()));
FileDemandExchangeHandler fileDemandExchangeHandler =
new FileDemandExchangeHandler(
this.storageAdapter,
this.node,
this.nodeManager,
this.globalEventBus,
new NodeLocation(
entry.getValue().getUserName(),
entry.getValue().getClientDeviceId(),
entry.getValue().getPeerAddress()),
entry.getKey(),
subExchangeId);
this.node
.getObjectDataReplyHandler()
.addResponseCallbackHandler(subExchangeId, fileDemandExchangeHandler);
Thread fileDemandExchangeHandlerThread = new Thread(fileDemandExchangeHandler);
fileDemandExchangeHandlerThread.setName("FileDemandExchangeHandlerThread-" + subExchangeId);
fileDemandExchangeHandlerThread.start();
try {
fileDemandExchangeHandler.await();
} catch (InterruptedException e) {
logger.error(
"Got interrupted while waiting for fileDemandExchangeHandler "
+ subExchangeId
+ " to complete. Message: "
+ e.getMessage());
}
this.node.getObjectDataReplyHandler().removeResponseCallbackHandler(subExchangeId);
if (!fileDemandExchangeHandler.isCompleted()) {
logger.error(
"FileDemandExchangeHandler " + subExchangeId + " should be completed after wait.");
}
}
// start event aggregator
logger.info(
"Starting event aggregator on client ("
+ this.node.getPeerAddress().inetAddress().getHostName()
+ ":"
+ this.node.getPeerAddress().tcpPort()
+ "): Non-blocking background sync "
+ exchangeId);
this.eventAggregator.start();
logger.info(
"Reconciling local disk changes with merged object store (non-blocking background sync "
+ exchangeId
+ ")");
// create a temporary second object store to get changes made in the mean time of syncing
ITreeStorageAdapter objectStoreStorageManager =
this.objectStore.getObjectManager().getStorageAdapater();
TreePathElement pathElement = new TreePathElement("nonBlockingBackgroundSyncObjectStore");
if (objectStoreStorageManager.exists(StorageType.DIRECTORY, pathElement)) {
objectStoreStorageManager.delete(pathElement);
}
objectStoreStorageManager.persist(StorageType.DIRECTORY, pathElement, null);
// create the temporary object store in the .sync folder
ITreeStorageAdapter changeObjectStoreStorageManager =
new LocalStorageAdapter(
Paths.get(objectStoreStorageManager.getRootDir().getPath())
.resolve(pathElement.getPath()));
IObjectStore changeObjectStore =
new ObjectStore(
this.storageAdapter, "index.json", "object", changeObjectStoreStorageManager);
// build object store for differences in the mean time
List<String> ignoredPaths = new ArrayList<>();
Path origSyncFolder =
Paths.get(this.objectStore.getObjectManager().getStorageAdapater().getRootDir().getPath())
.getFileName();
ignoredPaths.add(origSyncFolder.toString());
changeObjectStore.sync(ignoredPaths);
// get differences between disk and merged object store
HashMap<ObjectStore.MergedObjectType, Set<String>> updatedOrDeletedPaths =
this.objectStore.mergeObjectStore(changeObjectStore);
// remove change object store again
changeObjectStoreStorageManager.delete(new TreePathElement("./"));
Set<String> deletedPathsInTheMeanTime =
updatedOrDeletedPaths.get(ObjectStore.MergedObjectType.DELETED);
Set<String> updatedPathsInTheMeanTime =
updatedOrDeletedPaths.get(ObjectStore.MergedObjectType.CHANGED);
logger.info(
"Found "
+ deletedPathsInTheMeanTime.size()
+ " paths which have been deleted in the mean time of syncing");
for (String deletedPath : deletedPathsInTheMeanTime) {
if (this.isIgnored(deletedPath)) {
logger.info("Ignore deletion of " + deletedPath + " since it matches an ignore pattern");
}
// publish a delete event to the SyncFileChangeListener
logger.trace("Creating delete event for " + deletedPath);
this.globalEventBus.publish(
new CreateBusEvent(
new DeleteEvent(
Paths.get(deletedPath),
Paths.get(deletedPath).getFileName().toString(),
null,
System.currentTimeMillis())));
}
logger.info(
"Found "
+ updatedPathsInTheMeanTime.size()
+ " paths which have changed in the mean time of syncing");
for (String updatedPath : updatedPathsInTheMeanTime) {
if (this.isIgnored(updatedPath)) {
logger.info("Ignore updating of " + updatedPath + " since it matches an ignore pattern");
}
// publish modify events to SyncFileChangeListener
logger.trace("Creating modify event for " + updatedPath);
PathObject updatedPathObject =
this.objectStore.getObjectManager().getObjectForPath(updatedPath);
this.globalEventBus.publish(
new CreateBusEvent(
new ModifyEvent(
Paths.get(updatedPath),
Paths.get(updatedPath).getFileName().toString(),
updatedPathObject
.getVersions()
.get(Math.max(0, updatedPathObject.getVersions().size() - 1))
.getHash(),
System.currentTimeMillis())));
}
logger.info("Completed non-blocking background sync " + exchangeId);
} catch (Exception e) {
logger.error("Got exception in NonBlockingBackgroundSyncer. Message: " + e.getMessage(), e);
}
}
