nextcloud-desktop/shell_integration/MacOSX/NextcloudIntegration/FileProviderExt/FileProviderEnumerator+SyncEngine.swift

/*
 * Copyright (C) 2023 by Claudio Cambra <claudio.cambra@nextcloud.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
 * for more details.
 */

import FileProvider
import NextcloudKit
import OSLog

extension FileProviderEnumerator {
    func fullRecursiveScan(
        ncAccount: NextcloudAccount,
        ncKit: NextcloudKit,
        scanChangesOnly: Bool,
        completionHandler: @escaping (
            _ metadatas: [NextcloudItemMetadataTable],
            _ newMetadatas: [NextcloudItemMetadataTable],
            _ updatedMetadatas: [NextcloudItemMetadataTable],
            _ deletedMetadatas: [NextcloudItemMetadataTable],
            _ error: NKError?
        ) -> Void
    ) {
        let rootContainerDirectoryMetadata = NextcloudItemMetadataTable()
        rootContainerDirectoryMetadata.directory = true
        rootContainerDirectoryMetadata.ocId = NSFileProviderItemIdentifier.rootContainer.rawValue

        // Create a serial dispatch queue
        let dispatchQueue = DispatchQueue(
            label: "recursiveChangeEnumerationQueue", qos: .userInitiated)

        dispatchQueue.async {
            let results = self.scanRecursively(
                rootContainerDirectoryMetadata,
                ncAccount: ncAccount,
                ncKit: ncKit,
                scanChangesOnly: scanChangesOnly)

            // Run a check to ensure files deleted in one location are not updated in another (e.g. when moved)
            // The recursive scan provides us with updated/deleted metadatas only on a folder by folder basis;
            // so we need to check we are not simultaneously marking a moved file as deleted and updated
            var checkedDeletedMetadatas = results.deletedMetadatas

            for updatedMetadata in results.updatedMetadatas {
                guard
                    let matchingDeletedMetadataIdx = checkedDeletedMetadatas.firstIndex(where: {
                        $0.ocId == updatedMetadata.ocId
                    })
                else {
                    continue
                }

                checkedDeletedMetadatas.remove(at: matchingDeletedMetadataIdx)
            }

            DispatchQueue.main.async {
                completionHandler(
                    results.metadatas, results.newMetadatas, results.updatedMetadatas,
                    checkedDeletedMetadatas, results.error)
            }
        }
    }

    private func scanRecursively(
        _ directoryMetadata: NextcloudItemMetadataTable,
        ncAccount: NextcloudAccount,
        ncKit: NextcloudKit,
        scanChangesOnly: Bool
    ) -> (
        metadatas: [NextcloudItemMetadataTable],
        newMetadatas: [NextcloudItemMetadataTable],
        updatedMetadatas: [NextcloudItemMetadataTable],
        deletedMetadatas: [NextcloudItemMetadataTable],
        error: NKError?
    ) {
        if isInvalidated {
            return ([], [], [], [], nil)
        }

        assert(directoryMetadata.directory, "Can only recursively scan a directory.")

        // Will include results of recursive calls
        var allMetadatas: [NextcloudItemMetadataTable] = []
        var allNewMetadatas: [NextcloudItemMetadataTable] = []
        var allUpdatedMetadatas: [NextcloudItemMetadataTable] = []
        var allDeletedMetadatas: [NextcloudItemMetadataTable] = []

        let dbManager = NextcloudFilesDatabaseManager.shared
        let dispatchGroup = DispatchGroup()  // TODO: Maybe own thread?

        dispatchGroup.enter()

        var criticalError: NKError?
        let itemServerUrl =
            directoryMetadata.ocId == NSFileProviderItemIdentifier.rootContainer.rawValue
            ? ncAccount.davFilesUrl : directoryMetadata.serverUrl + "/" + directoryMetadata.fileName

        Logger.enumeration.debug("About to read: \(itemServerUrl, privacy: .public)")

        FileProviderEnumerator.readServerUrl(
            itemServerUrl, ncAccount: ncAccount, ncKit: ncKit, stopAtMatchingEtags: scanChangesOnly
        ) { metadatas, newMetadatas, updatedMetadatas, deletedMetadatas, readError in

            if readError != nil {
                let nkReadError = NKError(error: readError!)

                // Is the error is that we have found matching etags on this item, then ignore it
                // if we are doing a full rescan
                guard nkReadError.isNoChangesError, scanChangesOnly else {
                    Logger.enumeration.error(
                        "Finishing enumeration of changes at \(itemServerUrl, privacy: .public) with \(readError!.localizedDescription, privacy: .public)"
                    )

                    if nkReadError.isNotFoundError {
                        Logger.enumeration.info(
                            "404 error means item no longer exists. Deleting metadata and reporting as deletion without error"
                        )

                        if let deletedMetadatas =
                            dbManager.deleteDirectoryAndSubdirectoriesMetadata(
                                ocId: directoryMetadata.ocId)
                        {
                            allDeletedMetadatas += deletedMetadatas
                        } else {
                            Logger.enumeration.error(
                                "An error occurred while trying to delete directory and children not found in recursive scan"
                            )
                        }

                    } else if nkReadError.isNoChangesError {  // All is well, just no changed etags
                        Logger.enumeration.info(
                            "Error was to say no changed files -- not bad error. No need to check children."
                        )

                    } else if nkReadError.isUnauthenticatedError
                        || nkReadError.isCouldntConnectError
                    {
                        // If it is a critical error then stop, if not then continue
                        Logger.enumeration.error(
                            "Error will affect next enumerated items, so stopping enumeration.")
                        criticalError = nkReadError
                    }

                    dispatchGroup.leave()
                    return
                }
            }

            Logger.enumeration.info(
                "Finished reading serverUrl: \(itemServerUrl, privacy: .public) for user: \(ncAccount.ncKitAccount, privacy: .public)"
            )

            if let metadatas {
                allMetadatas += metadatas
            } else {
                Logger.enumeration.warning(
                    "WARNING: Nil metadatas received for reading of changes at \(itemServerUrl, privacy: .public) for user: \(ncAccount.ncKitAccount, privacy: .public)"
                )
            }

            if let newMetadatas {
                allNewMetadatas += newMetadatas
            } else {
                Logger.enumeration.warning(
                    "WARNING: Nil new metadatas received for reading of changes at \(itemServerUrl, privacy: .public) for user: \(ncAccount.ncKitAccount, privacy: .public)"
                )
            }

            if let updatedMetadatas {
                allUpdatedMetadatas += updatedMetadatas
            } else {
                Logger.enumeration.warning(
                    "WARNING: Nil updated metadatas received for reading of changes at \(itemServerUrl, privacy: .public) for user: \(ncAccount.ncKitAccount, privacy: .public)"
                )
            }

            if let deletedMetadatas {
                allDeletedMetadatas += deletedMetadatas
            } else {
                Logger.enumeration.warning(
                    "WARNING: Nil deleted metadatas received for reading of changes at \(itemServerUrl, privacy: .public) for user: \(ncAccount.ncKitAccount, privacy: .public)"
                )
            }

            dispatchGroup.leave()
        }

        dispatchGroup.wait()

        guard criticalError == nil else {
            return ([], [], [], [], error: criticalError)
        }

        var childDirectoriesToScan: [NextcloudItemMetadataTable] = []
        var candidateMetadatas: [NextcloudItemMetadataTable] =
            if scanChangesOnly {
                allUpdatedMetadatas + allNewMetadatas
            } else {
                allMetadatas
            }

        for candidateMetadata in candidateMetadatas {
            if candidateMetadata.directory {
                childDirectoriesToScan.append(candidateMetadata)
            }
        }

        if childDirectoriesToScan.isEmpty {
            return (
                metadatas: allMetadatas, newMetadatas: allNewMetadatas,
                updatedMetadatas: allUpdatedMetadatas, deletedMetadatas: allDeletedMetadatas, nil
            )
        }

        for childDirectory in childDirectoriesToScan {
            let childScanResult = scanRecursively(
                childDirectory,
                ncAccount: ncAccount,
                ncKit: ncKit,
                scanChangesOnly: scanChangesOnly)

            allMetadatas += childScanResult.metadatas
            allNewMetadatas += childScanResult.newMetadatas
            allUpdatedMetadatas += childScanResult.updatedMetadatas
            allDeletedMetadatas += childScanResult.deletedMetadatas
        }

        return (
            metadatas: allMetadatas, newMetadatas: allNewMetadatas,
            updatedMetadatas: allUpdatedMetadatas,
            deletedMetadatas: allDeletedMetadatas, nil
        )
    }

    static func handleDepth1ReadFileOrFolder(
        serverUrl: String,
        ncAccount: NextcloudAccount,
        files: [NKFile],
        error: NKError,
        completionHandler: @escaping (
            _ metadatas: [NextcloudItemMetadataTable]?,
            _ newMetadatas: [NextcloudItemMetadataTable]?,
            _ updatedMetadatas: [NextcloudItemMetadataTable]?,
            _ deletedMetadatas: [NextcloudItemMetadataTable]?,
            _ readError: Error?
        ) -> Void
    ) {
        guard error == .success else {
            Logger.enumeration.error(
                "1 depth readFileOrFolder of url: \(serverUrl, privacy: .public) did not complete successfully, received error: \(error.errorDescription, privacy: .public)"
            )
            completionHandler(nil, nil, nil, nil, error.error)
            return
        }

        Logger.enumeration.debug(
            "Starting async conversion of NKFiles for serverUrl: \(serverUrl, privacy: .public) for user: \(ncAccount.ncKitAccount, privacy: .public)"
        )

        let dbManager = NextcloudFilesDatabaseManager.shared

        DispatchQueue.global(qos: .userInitiated).async {
            NextcloudItemMetadataTable.metadatasFromDirectoryReadNKFiles(
                files, account: ncAccount.ncKitAccount
            ) { directoryMetadata, _, metadatas in

                // STORE DATA FOR CURRENTLY SCANNED DIRECTORY
                // We have now scanned this directory's contents, so update with etag in order to not check again if not needed
                // unless it's the root container
                if serverUrl != ncAccount.davFilesUrl {
                    dbManager.addItemMetadata(directoryMetadata)
                }

                // Don't update the etags for folders as we haven't checked their contents.
                // When we do a recursive check, if we update the etags now, we will think
                // that our local copies are up to date -- instead, leave them as the old.
                // They will get updated when they are the subject of a readServerUrl call.
                // (See above)
                let changedMetadatas = dbManager.updateItemMetadatas(
                    account: ncAccount.ncKitAccount, serverUrl: serverUrl,
                    updatedMetadatas: metadatas,
                    updateDirectoryEtags: false)

                DispatchQueue.main.async {
                    completionHandler(
                        metadatas, changedMetadatas.newMetadatas, changedMetadatas.updatedMetadatas,
                        changedMetadatas.deletedMetadatas, nil)
                }
            }
        }
    }

    static func readServerUrl(
        _ serverUrl: String,
        ncAccount: NextcloudAccount,
        ncKit: NextcloudKit,
        stopAtMatchingEtags: Bool = false,
        depth: String = "1",
        completionHandler: @escaping (
            _ metadatas: [NextcloudItemMetadataTable]?,
            _ newMetadatas: [NextcloudItemMetadataTable]?,
            _ updatedMetadatas: [NextcloudItemMetadataTable]?,
            _ deletedMetadatas: [NextcloudItemMetadataTable]?,
            _ readError: Error?
        ) -> Void
    ) {
        let dbManager = NextcloudFilesDatabaseManager.shared
        let ncKitAccount = ncAccount.ncKitAccount

        Logger.enumeration.debug(
            "Starting to read serverUrl: \(serverUrl, privacy: .public) for user: \(ncAccount.ncKitAccount, privacy: .public) at depth \(depth, privacy: .public). NCKit info: userId: \(ncKit.nkCommonInstance.user, privacy: .public), password is empty: \(ncKit.nkCommonInstance.password == "" ? "EMPTY PASSWORD" : "NOT EMPTY PASSWORD"), urlBase: \(ncKit.nkCommonInstance.urlBase, privacy: .public), ncVersion: \(ncKit.nkCommonInstance.nextcloudVersion, privacy: .public)"
        )

        ncKit.readFileOrFolder(serverUrlFileName: serverUrl, depth: depth, showHiddenFiles: true) {
            _, files, _, error in
            guard error == .success else {
                Logger.enumeration.error(
                    "\(depth, privacy: .public) depth readFileOrFolder of url: \(serverUrl, privacy: .public) did not complete successfully, received error: \(error.errorDescription, privacy: .public)"
                )
                completionHandler(nil, nil, nil, nil, error.error)
                return
            }

            guard let receivedFile = files.first else {
                Logger.enumeration.error(
                    "Received no items from readFileOrFolder of \(serverUrl, privacy: .public), not much we can do..."
                )
                completionHandler(nil, nil, nil, nil, error.error)
                return
            }

            guard receivedFile.directory else {
                Logger.enumeration.debug(
                    "Read item is a file. Converting NKfile for serverUrl: \(serverUrl, privacy: .public) for user: \(ncAccount.ncKitAccount, privacy: .public)"
                )
                let itemMetadata = NextcloudItemMetadataTable.fromNKFile(
                    receivedFile, account: ncKitAccount)
                dbManager.addItemMetadata(itemMetadata)  // TODO: Return some value when it is an update
                completionHandler([itemMetadata], nil, nil, nil, error.error)
                return
            }

            if stopAtMatchingEtags,
                let directoryMetadata = dbManager.directoryMetadata(
                    account: ncKitAccount, serverUrl: serverUrl)
            {
                let directoryEtag = directoryMetadata.etag

                guard directoryEtag == "" || directoryEtag != receivedFile.etag else {
                    Logger.enumeration.debug(
                        "Read server url called with flag to stop enumerating at matching etags. Returning and providing soft error."
                    )

                    let description =
                        "Fetched directory etag is same as that stored locally. Not fetching child items."
                    let nkError = NKError(
                        errorCode: NKError.noChangesErrorCode, errorDescription: description)

                    let metadatas = dbManager.itemMetadatas(
                        account: ncKitAccount, serverUrl: serverUrl)

                    completionHandler(metadatas, nil, nil, nil, nkError.error)
                    return
                }
            }

            if depth == "0" {
                if serverUrl != ncAccount.davFilesUrl {
                    let metadata = NextcloudItemMetadataTable.fromNKFile(
                        receivedFile, account: ncKitAccount)
                    let isNew = dbManager.itemMetadataFromOcId(metadata.ocId) == nil
                    let updatedMetadatas = isNew ? [] : [metadata]
                    let newMetadatas = isNew ? [metadata] : []

                    dbManager.addItemMetadata(metadata)

                    DispatchQueue.main.async {
                        completionHandler([metadata], newMetadatas, updatedMetadatas, nil, nil)
                    }
                }
            } else {
                handleDepth1ReadFileOrFolder(
                    serverUrl: serverUrl, ncAccount: ncAccount, files: files, error: error,
                    completionHandler: completionHandler)
            }
        }
    }
}