nextcloud-desktop/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator.swift
Claudio Cambra 35570a3160
Move remote sync functionality out of enumerator and into separate NextcloudSyncEngine class
Signed-off-by: Claudio Cambra <claudio.cambra@nextcloud.com>
2023-05-12 13:29:56 +08:00

423 lines
24 KiB
Swift

/*
* Copyright (C) 2022 by Claudio Cambra <claudio.cambra@nextcloud.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
import FileProvider
import NextcloudKit
import OSLog
class FileProviderEnumerator: NSObject, NSFileProviderEnumerator {
private let enumeratedItemIdentifier: NSFileProviderItemIdentifier
private var enumeratedItemMetadata: NextcloudItemMetadataTable?
private var enumeratingSystemIdentifier: Bool {
return FileProviderEnumerator.isSystemIdentifier(enumeratedItemIdentifier)
}
private let anchor = NSFileProviderSyncAnchor(Date().description.data(using: .utf8)!) // TODO: actually use this in NCKit and server requests
private static let maxItemsPerFileProviderPage = 100
let ncAccount: NextcloudAccount
let ncKit: NextcloudKit
var serverUrl: String = ""
var isInvalidated = false
let syncEngine = NextcloudSyncEngine()
private static func isSystemIdentifier(_ identifier: NSFileProviderItemIdentifier) -> Bool {
return identifier == .rootContainer ||
identifier == .trashContainer ||
identifier == .workingSet
}
init(enumeratedItemIdentifier: NSFileProviderItemIdentifier, ncAccount: NextcloudAccount, ncKit: NextcloudKit) {
self.enumeratedItemIdentifier = enumeratedItemIdentifier
self.ncAccount = ncAccount
self.ncKit = ncKit
if FileProviderEnumerator.isSystemIdentifier(enumeratedItemIdentifier) {
Logger.enumeration.debug("Providing enumerator for a system defined container: \(enumeratedItemIdentifier.rawValue, privacy: .public)")
self.serverUrl = ncAccount.davFilesUrl
} else {
Logger.enumeration.debug("Providing enumerator for item with identifier: \(enumeratedItemIdentifier.rawValue, privacy: .public)")
let dbManager = NextcloudFilesDatabaseManager.shared
enumeratedItemMetadata = dbManager.itemMetadataFromFileProviderItemIdentifier(enumeratedItemIdentifier)
if enumeratedItemMetadata != nil {
self.serverUrl = enumeratedItemMetadata!.serverUrl + "/" + enumeratedItemMetadata!.fileName
} else {
Logger.enumeration.error("Could not find itemMetadata for file with identifier: \(enumeratedItemIdentifier.rawValue, privacy: .public)")
}
}
Logger.enumeration.info("Set up enumerator for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash))")
super.init()
}
func invalidate() {
Logger.enumeration.debug("Enumerator is being invalidated for item with identifier: \(self.enumeratedItemIdentifier.rawValue, privacy: .public)")
syncEngine.invalidate()
self.isInvalidated = true
}
// MARK: - Protocol methods
func enumerateItems(for observer: NSFileProviderEnumerationObserver, startingAt page: NSFileProviderPage) {
Logger.enumeration.debug("Received enumerate items request for enumerator with user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash))")
/*
- inspect the page to determine whether this is an initial or a follow-up request (TODO)
If this is an enumerator for a directory, the root container or all directories:
- perform a server request to fetch directory contents
If this is an enumerator for the working set:
- perform a server request to update your local database
- fetch the working set from your local database
- inform the observer about the items returned by the server (possibly multiple times)
- inform the observer that you are finished with this page
*/
let dbManager = NextcloudFilesDatabaseManager.shared
// When enumerating items in the working set, we are expected to provide an account of the
// whole server and all its files.
if enumeratedItemIdentifier == .workingSet {
if page == NSFileProviderPage.initialPageSortedByDate as NSFileProviderPage ||
page == NSFileProviderPage.initialPageSortedByName as NSFileProviderPage {
// We enumerate items as we get the server data for two reasons:
// A) we avoid having a gigantic chunk of files to enumerate to the observer at the end
// B) we don't need to worry about resolving which files are truly deleted vs moved at the end
syncEngine.fullRecursiveScan(ncAccount: self.ncAccount, ncKit: self.ncKit, scanChangesOnly: false, singleFolderScanCompleteCompletionHandler: { metadatas, error in
guard error == nil else {
Logger.enumeration.error("There was an error during recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)")
observer.finishEnumeratingWithError(error!.toFileProviderError())
return;
}
guard let metadatas = metadatas else {
Logger.enumeration.warning("Received nil metadatas during recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)")
return
}
let items = FileProviderEnumerator.metadatasToFileProviderItems(metadatas, ncKit: self.ncKit)
observer.didEnumerate(items)
}) { metadatas, _, _, _, error in
if self.isInvalidated {
Logger.enumeration.info("Enumerator invalidated during working set item enumeration. For user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
observer.finishEnumeratingWithError(NSFileProviderError(.cannotSynchronize))
return
}
guard error == nil else {
Logger.enumeration.info("Finished recursive iteme enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)")
observer.finishEnumeratingWithError(error!.toFileProviderError())
return;
}
Logger.enumeration.info("Finished recursive item enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)). Enumerating items.")
observer.finishEnumerating(upTo: FileProviderEnumerator.fileProviderPageforNumPage(1))
}
return
} else {
Logger.enumeration.debug("Enumerating page \(page.rawValue) of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
// TODO!
observer.finishEnumerating(upTo: nil)
}
return
} else if enumeratedItemIdentifier == .trashContainer {
Logger.enumeration.debug("Enumerating trash set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash))")
// TODO!
observer.finishEnumerating(upTo: nil)
return
}
guard serverUrl != "" else {
Logger.enumeration.error("Enumerator has empty serverUrl -- can't enumerate that! For identifier: \(self.enumeratedItemIdentifier.rawValue, privacy: .public)")
observer.finishEnumeratingWithError(NSFileProviderError(.noSuchItem))
return
}
// TODO: Make better use of pagination and andle paging properly
if page == NSFileProviderPage.initialPageSortedByDate as NSFileProviderPage ||
page == NSFileProviderPage.initialPageSortedByName as NSFileProviderPage {
Logger.enumeration.debug("Enumerating initial page for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash))")
NextcloudSyncEngine.readServerUrl(serverUrl, ncAccount: ncAccount, ncKit: ncKit) { _, _, _, _, readError in
guard readError == nil else {
Logger.enumeration.error("Finishing enumeration for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) with error \(readError!.localizedDescription, privacy: .public)")
let nkReadError = NKError(error: readError!)
observer.finishEnumeratingWithError(nkReadError.toFileProviderError())
return
}
let ncKitAccount = self.ncAccount.ncKitAccount
// Return all now known metadatas
var metadatas: [NextcloudItemMetadataTable]
if self.enumeratingSystemIdentifier || (self.enumeratedItemMetadata != nil && self.enumeratedItemMetadata!.directory) {
metadatas = NextcloudFilesDatabaseManager.shared.itemMetadatas(account: ncKitAccount, serverUrl: self.serverUrl)
} else if (self.enumeratedItemMetadata != nil) {
guard let updatedEnumeratedItemMetadata = NextcloudFilesDatabaseManager.shared.itemMetadataFromOcId(self.enumeratedItemMetadata!.ocId) else {
Logger.enumeration.error("Could not finish enumeration for user: \(ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) as the enumerated item could not be fetched from database. \(self.enumeratedItemIdentifier.rawValue, privacy: .public)")
observer.finishEnumeratingWithError(NSFileProviderError(.noSuchItem))
return
}
metadatas = [updatedEnumeratedItemMetadata]
} else { // We need to have an enumeratedItemMetadata to have a non empty serverUrl
Logger.enumeration.error("Cannot finish enumeration for user: \(ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) as we do not have a valid server URL. NOTE: this error should not be possible and indicates something is going wrong before.")
observer.finishEnumeratingWithError(NSFileProviderError(.noSuchItem))
return
}
Logger.enumeration.info("Finished reading serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)). Processed \(metadatas.count) metadatas")
FileProviderEnumerator.completeEnumerationObserver(observer, ncKit: self.ncKit, numPage: 1, itemMetadatas: metadatas)
}
return;
}
let numPage = Int(String(data: page.rawValue, encoding: .utf8)!)!
Logger.enumeration.debug("Enumerating page \(numPage, privacy: .public) for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash))")
// TODO: Handle paging properly
// FileProviderEnumerator.completeObserver(observer, ncKit: ncKit, numPage: numPage, itemMetadatas: nil)
observer.finishEnumerating(upTo: nil)
}
func enumerateChanges(for observer: NSFileProviderChangeObserver, from anchor: NSFileProviderSyncAnchor) {
Logger.enumeration.debug("Received enumerate changes request for enumerator for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash))")
/*
- query the server for updates since the passed-in sync anchor (TODO)
If this is an enumerator for the working set:
- note the changes in your local database
- inform the observer about item deletions and updates (modifications + insertions)
- inform the observer when you have finished enumerating up to a subsequent sync anchor
*/
if enumeratedItemIdentifier == .workingSet {
Logger.enumeration.debug("Enumerating changes in working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
// Unlike when enumerating items we can't progressively enumerate items as we need to wait to resolve which items are truly deleted and which
// have just been moved elsewhere.
syncEngine.fullRecursiveScan(ncAccount: self.ncAccount,
ncKit: self.ncKit,
scanChangesOnly: true,
singleFolderScanCompleteCompletionHandler: { _, _ in }) { _, newMetadatas, updatedMetadatas, deletedMetadatas, error in
if self.isInvalidated {
Logger.enumeration.info("Enumerator invalidated during working set change scan. For user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
observer.finishEnumeratingWithError(NSFileProviderError(.cannotSynchronize))
return
}
guard error == nil else {
Logger.enumeration.info("Finished recursive change enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(error!.errorDescription, privacy: .public)")
observer.finishEnumeratingWithError(error!.toFileProviderError())
return
}
Logger.enumeration.info("Finished recursive change enumeration of working set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)). Enumerating items.")
FileProviderEnumerator.completeChangesObserver(observer,
anchor: anchor,
ncKit: self.ncKit,
newMetadatas: newMetadatas,
updatedMetadatas: updatedMetadatas,
deletedMetadatas: deletedMetadatas)
}
return
} else if enumeratedItemIdentifier == .trashContainer {
Logger.enumeration.debug("Enumerating changes in trash set for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
// TODO!
observer.finishEnumeratingChanges(upTo: anchor, moreComing: false)
return
}
Logger.enumeration.info("Enumerating changes for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash))")
// No matter what happens here we finish enumeration in some way, either from the error
// handling below or from the completeChangesObserver
NextcloudSyncEngine.readServerUrl(serverUrl, ncAccount: ncAccount, ncKit: ncKit, stopAtMatchingEtags: true) { _, newMetadatas, updatedMetadatas, deletedMetadatas, readError in
// If we get a 404 we might add more deleted metadatas
var currentDeletedMetadatas: [NextcloudItemMetadataTable] = []
if let notNilDeletedMetadatas = deletedMetadatas {
currentDeletedMetadatas = notNilDeletedMetadatas
}
guard readError == nil else {
Logger.enumeration.error("Finishing enumeration of changes for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash)) with serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) with error: \(readError!.localizedDescription, privacy: .public)")
let nkReadError = NKError(error: readError!)
let fpError = nkReadError.toFileProviderError()
if nkReadError.isNotFoundError {
Logger.enumeration.info("404 error means item no longer exists. Deleting metadata and reporting \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) as deletion without error")
guard let itemMetadata = self.enumeratedItemMetadata else {
Logger.enumeration.error("Invalid enumeratedItemMetadata, could not delete metadata nor report deletion")
observer.finishEnumeratingWithError(fpError)
return
}
let dbManager = NextcloudFilesDatabaseManager.shared
if itemMetadata.directory {
if let deletedDirectoryMetadatas = dbManager.deleteDirectoryAndSubdirectoriesMetadata(ocId: itemMetadata.ocId) {
currentDeletedMetadatas += deletedDirectoryMetadatas
} else {
Logger.enumeration.error("Something went wrong when recursively deleting directory not found.")
}
} else {
dbManager.deleteItemMetadata(ocId: itemMetadata.ocId)
}
FileProviderEnumerator.completeChangesObserver(observer, anchor: anchor, ncKit: self.ncKit, newMetadatas: nil, updatedMetadatas: nil, deletedMetadatas: [itemMetadata])
return
} else if nkReadError.isNoChangesError { // All is well, just no changed etags
Logger.enumeration.info("Error was to say no changed files -- not bad error. Finishing change enumeration.")
observer.finishEnumeratingChanges(upTo: anchor, moreComing: false)
return;
}
observer.finishEnumeratingWithError(fpError)
return
}
Logger.enumeration.info("Finished reading serverUrl: \(self.serverUrl, privacy: OSLogPrivacy.auto(mask: .hash)) for user: \(self.ncAccount.ncKitAccount, privacy: OSLogPrivacy.auto(mask: .hash))")
FileProviderEnumerator.completeChangesObserver(observer, anchor: anchor, ncKit: self.ncKit, newMetadatas: newMetadatas, updatedMetadatas: updatedMetadatas, deletedMetadatas: deletedMetadatas)
}
}
func currentSyncAnchor(completionHandler: @escaping (NSFileProviderSyncAnchor?) -> Void) {
completionHandler(anchor)
}
// MARK: - Helper methods
private static func metadatasToFileProviderItems(_ itemMetadatas: [NextcloudItemMetadataTable], ncKit: NextcloudKit) -> [NSFileProviderItem] {
var items: [NSFileProviderItem] = []
for itemMetadata in itemMetadatas {
if itemMetadata.e2eEncrypted {
Logger.enumeration.info("Skipping encrypted metadata in enumeration: \(itemMetadata.ocId) \(itemMetadata.fileName, privacy: OSLogPrivacy.auto(mask: .hash))")
continue
}
if let parentItemIdentifier = NextcloudFilesDatabaseManager.shared.parentItemIdentifierFromMetadata(itemMetadata) {
let item = FileProviderItem(metadata: itemMetadata, parentItemIdentifier: parentItemIdentifier, ncKit: ncKit)
Logger.enumeration.debug("Will enumerate item with ocId: \(itemMetadata.ocId) and name: \(itemMetadata.fileName, privacy: OSLogPrivacy.auto(mask: .hash))")
items.append(item)
} else {
Logger.enumeration.error("Could not get valid parentItemIdentifier for item with ocId: \(itemMetadata.ocId, privacy: .public) and name: \(itemMetadata.fileName, privacy: OSLogPrivacy.auto(mask: .hash)), skipping enumeration")
}
}
return items
}
private static func fileProviderPageforNumPage(_ numPage: Int) -> NSFileProviderPage {
return NSFileProviderPage("\(numPage)".data(using: .utf8)!)
}
private static func completeEnumerationObserver(_ observer: NSFileProviderEnumerationObserver, ncKit: NextcloudKit, numPage: Int, itemMetadatas: [NextcloudItemMetadataTable]) {
let items = FileProviderEnumerator.metadatasToFileProviderItems(itemMetadatas, ncKit: ncKit)
observer.didEnumerate(items)
Logger.enumeration.info("Did enumerate \(items.count) items")
// TODO: Handle paging properly
/*
if items.count == maxItemsPerFileProviderPage {
let nextPage = numPage + 1
let providerPage = NSFileProviderPage("\(nextPage)".data(using: .utf8)!)
observer.finishEnumerating(upTo: providerPage)
} else {
observer.finishEnumerating(upTo: nil)
}
*/
observer.finishEnumerating(upTo: fileProviderPageforNumPage(numPage))
}
private static func completeChangesObserver(_ observer: NSFileProviderChangeObserver, anchor: NSFileProviderSyncAnchor, ncKit: NextcloudKit, newMetadatas: [NextcloudItemMetadataTable]?, updatedMetadatas: [NextcloudItemMetadataTable]?, deletedMetadatas: [NextcloudItemMetadataTable]?) {
guard newMetadatas != nil || updatedMetadatas != nil || deletedMetadatas != nil else {
Logger.enumeration.error("Received invalid newMetadatas, updatedMetadatas or deletedMetadatas. Finished enumeration of changes with error.")
observer.finishEnumeratingWithError(NSFileProviderError(.noSuchItem))
return
}
// Observer does not care about new vs updated, so join
var allUpdatedMetadatas: [NextcloudItemMetadataTable] = []
var allDeletedMetadatas: [NextcloudItemMetadataTable] = []
if let newMetadatas = newMetadatas {
allUpdatedMetadatas += newMetadatas
}
if let updatedMetadatas = updatedMetadatas {
allUpdatedMetadatas += updatedMetadatas
}
if let deletedMetadatas = deletedMetadatas {
allDeletedMetadatas = deletedMetadatas
}
var allFpItemUpdates: [FileProviderItem] = []
var allFpItemDeletionsIdentifiers = Array(allDeletedMetadatas.map { NSFileProviderItemIdentifier($0.ocId) })
for updMetadata in allUpdatedMetadatas {
guard let parentItemIdentifier = NextcloudFilesDatabaseManager.shared.parentItemIdentifierFromMetadata(updMetadata) else {
Logger.enumeration.warning("Not enumerating change for metadata: \(updMetadata.ocId) \(updMetadata.fileName, privacy: OSLogPrivacy.auto(mask: .hash)) as could not get parent item metadata.")
continue
}
guard !updMetadata.e2eEncrypted else {
// Precaution, if all goes well in NKFile conversion then this should not happen
// TODO: Remove when E2EE supported
Logger.enumeration.info("Encrypted metadata in changes enumeration \(updMetadata.ocId) \(updMetadata.fileName, privacy: OSLogPrivacy.auto(mask: .hash)), adding to deletions")
allFpItemDeletionsIdentifiers.append(NSFileProviderItemIdentifier(updMetadata.ocId))
continue
}
let fpItem = FileProviderItem(metadata: updMetadata, parentItemIdentifier: parentItemIdentifier, ncKit: ncKit)
allFpItemUpdates.append(fpItem)
}
if !allFpItemUpdates.isEmpty {
observer.didUpdate(allFpItemUpdates)
}
if !allFpItemDeletionsIdentifiers.isEmpty {
observer.didDeleteItems(withIdentifiers: allFpItemDeletionsIdentifiers)
}
Logger.enumeration.info("Processed \(allUpdatedMetadatas.count) new or updated metadatas, \(allDeletedMetadatas.count) deleted metadatas.")
observer.finishEnumeratingChanges(upTo: anchor, moreComing: false)
}
}