1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
|
/*
* SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
* SPDX-FileCopyrightText: 2014 ownCloud GmbH
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#pragma once
#include "networkjobs.h"
#include "syncoptions.h"
#include "syncfileitem.h"
#include "common/folderquota.h"
#include "common/remoteinfo.h"
#include <QObject>
#include <QElapsedTimer>
#include <QStringList>
#include <csync.h>
#include <QMap>
#include <QSet>
#include <QMutex>
#include <QWaitCondition>
#include <QRunnable>
#include <deque>
class ExcludedFiles;
namespace OCC {
namespace LocalDiscoveryEnums {
OCSYNC_EXPORT Q_NAMESPACE
enum class LocalDiscoveryStyle {
FilesystemOnly, //< read all local data from the filesystem
DatabaseAndFilesystem, //< read from the db, except for listed paths
};
Q_ENUM_NS(LocalDiscoveryStyle)
}
using OCC::LocalDiscoveryEnums::LocalDiscoveryStyle;
class Account;
class SyncJournalDb;
class ProcessDirectoryJob;
enum class ErrorCategory;
struct LocalInfo
{
/** FileName of the entry (this does not contains any directory or path, just the plain name */
QString name;
QString caseClashConflictingName;
time_t modtime = 0;
int64_t size = 0;
uint64_t inode = 0;
ItemType type = ItemTypeSkip;
bool isDirectory = false;
bool isHidden = false;
bool isVirtualFile = false;
bool isSymLink = false;
bool isMetadataMissing = false;
[[nodiscard]] bool isValid() const { return !name.isNull(); }
};
/**
* @brief Run list on a local directory and process the results for Discovery
*
* @ingroup libsync
*/
class DiscoverySingleLocalDirectoryJob : public QObject, public QRunnable
{
Q_OBJECT
public:
explicit DiscoverySingleLocalDirectoryJob(const AccountPtr &account, const QString &localPath, OCC::Vfs *vfs, QObject *parent = nullptr);
void run() override;
signals:
void finished(QVector<OCC::LocalInfo> result);
void finishedFatalError(QString errorString);
void finishedNonFatalError(QString errorString);
void itemDiscovered(OCC::SyncFileItemPtr item);
void childIgnored(bool b);
private slots:
private:
QString _localPath;
AccountPtr _account;
OCC::Vfs* _vfs;
public:
};
class FolderMetadata;
/**
* @brief Run a PROPFIND on a directory and process the results for Discovery
*
* @ingroup libsync
*/
class DiscoverySingleDirectoryJob : public QObject
{
Q_OBJECT
public:
explicit DiscoverySingleDirectoryJob(const AccountPtr &account,
const QString &path,
const QString &remoteRootFolderPath,
/* TODO for topLevelE2eeFolderPaths, from review: I still do not get why giving the whole QSet instead of just the parent of the folder we are in
sounds to me like it would be much more efficient to just have the e2ee parent folder that we are
inside*/
const QSet<QString> &topLevelE2eeFolderPaths,
SyncFileItem::EncryptionStatus parentEncryptionStatus,
QObject *parent = nullptr);
// Specify that this is the root and we need to check the data-fingerprint
void setIsRootPath() { _isRootPath = true; }
void start();
void abort();
[[nodiscard]] bool isFileDropDetected() const;
[[nodiscard]] bool encryptedMetadataNeedUpdate() const;
[[nodiscard]] SyncFileItem::EncryptionStatus currentEncryptionStatus() const;
[[nodiscard]] SyncFileItem::EncryptionStatus requiredEncryptionStatus() const;
// This is not actually a network job, it is just a job
signals:
void firstDirectoryPermissions(OCC::RemotePermissions);
void etag(const QByteArray &, const QDateTime &time);
void finished(const OCC::HttpResult<QVector<OCC::RemoteInfo>> &result);
void setfolderQuota(const FolderQuota &folderQuota);
private slots:
void directoryListingIteratedSlot(const QString &, const QMap<QString, QString> &);
void lsJobFinishedWithoutErrorSlot();
void lsJobFinishedWithErrorSlot(QNetworkReply *reply);
void fetchE2eMetadata();
void metadataReceived(const QJsonDocument &json, int statusCode);
void metadataError(const QByteArray& fileId, int httpReturnCode);
private:
[[nodiscard]] bool isE2eEncrypted() const { return _encryptionStatusCurrent != SyncFileItem::EncryptionStatus::NotEncrypted; }
QVector<RemoteInfo> _results;
QString _subPath;
QString _remoteRootFolderPath;
QByteArray _firstEtag;
QByteArray _fileId;
QByteArray _localFileId;
AccountPtr _account;
// The first result is for the directory itself and need to be ignored.
// This flag is true if it was already ignored.
bool _ignoredFirst = false;
// Set to true if this is the root path and we need to check the data-fingerprint
bool _isRootPath = false;
// If this directory is an external storage (The first item has 'M' in its permission)
bool _isExternalStorage = false;
// If this directory is e2ee
SyncFileItem::EncryptionStatus _encryptionStatusCurrent = SyncFileItem::EncryptionStatus::NotEncrypted;
bool _isFileDropDetected = false;
bool _encryptedMetadataNeedUpdate = false;
SyncFileItem::EncryptionStatus _encryptionStatusRequired = SyncFileItem::EncryptionStatus::NotEncrypted;
// If set, the discovery will finish with an error
int64_t _size = 0;
QString _error;
QPointer<LsColJob> _lsColJob;
// store top level E2EE folder paths as they are used later when discovering nested folders
QSet<QString> _topLevelE2eeFolderPaths;
public:
QByteArray _dataFingerprint;
FolderQuota _folderQuota;
};
class DiscoveryPhase : public QObject
{
Q_OBJECT
friend class ProcessDirectoryJob;
QPointer<ProcessDirectoryJob> _currentRootJob;
/** Maps the db-path of a deleted item to its SyncFileItem.
*
* If it turns out the item was renamed after all, the instruction
* can be changed. See findAndCancelDeletedJob(). Note that
* itemDiscovered() will already have been emitted for the item.
*/
QMap<QString, SyncFileItemPtr> _deletedItem;
QVector<QString> _directoryNamesToRestoreOnPropagation;
/** Maps the db-path of a deleted folder to its queued job.
*
* If a folder is deleted and must be recursed into, its job isn't
* executed immediately. Instead it's queued here and only run
* once the rest of the discovery has finished and we are certain
* that the folder wasn't just renamed. This avoids running the
* discovery on contents in the old location of renamed folders.
*
* See findAndCancelDeletedJob().
*/
QMap<QString, ProcessDirectoryJob *> _queuedDeletedDirectories;
// map source (original path) -> destinations (current server or local path)
QMap<QString, QString> _renamedItemsRemote;
QMap<QString, QString> _renamedItemsLocal;
// set of paths that should not be removed even though they are removed locally:
// there was a move to an invalid destination and now the source should be restored
//
// This applies recursively to subdirectories.
// All entries should have a trailing slash (even files), so lookup with
// lowerBound() is reliable.
//
// The value of this map doesn't matter.
QMap<QString, bool> _forbiddenDeletes;
/** Returns whether the db-path has been renamed locally or on the remote.
*
* Useful for avoiding processing of items that have already been claimed in
* a rename (would otherwise be discovered as deletions).
*/
[[nodiscard]] bool isRenamed(const QString &p) const;
int _currentlyActiveJobs = 0;
// both must contain a sorted list
QStringList _selectiveSyncBlackList;
QStringList _selectiveSyncWhiteList;
void scheduleMoreJobs();
[[nodiscard]] bool isInSelectiveSyncBlackList(const QString &path) const;
[[nodiscard]] bool activeFolderSizeLimit() const;
[[nodiscard]] bool notifyExistingFolderOverLimit() const;
void checkFolderSizeLimit(const QString &path,
const std::function<void(bool)> callback);
// Check if the new folder should be deselected or not.
// May be async. "Return" via the callback, true if the item is blacklisted
void checkSelectiveSyncNewFolder(const QString &path,
const RemotePermissions rp,
const std::function<void(bool)> callback);
void checkSelectiveSyncExistingFolder(const QString &path);
/** Given an original path, return the target path obtained when renaming is done.
*
* Note that it only considers parent directory renames. So if A/B got renamed to C/D,
* checking A/B/file would yield C/D/file, but checking A/B would yield A/B.
*/
[[nodiscard]] QString adjustRenamedPath(const QString &original, SyncFileItem::Direction) const;
/** If the db-path is scheduled for deletion, abort it.
*
* Check if there is already a job to delete that item:
* If that's not the case, return { false, QByteArray() }.
* If there is such a job, cancel that job and return true and the old etag.
*
* Used when having detected a rename: The rename source may have been
* discovered before and would have looked like a delete.
*
* See _deletedItem and _queuedDeletedDirectories.
*/
QPair<bool, QByteArray> findAndCancelDeletedJob(const QString &originalPath);
void enqueueDirectoryToDelete(const QString &path, ProcessDirectoryJob* const directoryJob);
bool recursiveCheckForDeletedParents(const QString &itemPath) const;
/// contains files/folder names that are requested to be deleted permanently
QSet<QString> _permanentDeletionRequests;
void markPermanentDeletionRequests();
public:
// input
QString _localDir; // absolute path to the local directory. ends with '/'
QString _remoteFolder; // remote folder, ends with '/'
SyncJournalDb *_statedb = nullptr;
AccountPtr _account;
SyncOptions _syncOptions;
ExcludedFiles *_excludes = nullptr;
QRegularExpression _invalidFilenameRx; // FIXME: maybe move in ExcludedFiles
QStringList _serverBlacklistedFiles; // The blacklist from the capabilities
QStringList _leadingAndTrailingSpacesFilesAllowed;
bool _shouldEnforceWindowsFileNameCompatibility = false;
bool _ignoreHiddenFiles = false;
std::function<bool(const QString &)> _shouldDiscoverLocaly;
void startJob(ProcessDirectoryJob *);
void setSelectiveSyncBlackList(const QStringList &list);
void setSelectiveSyncWhiteList(const QStringList &list);
// output
QByteArray _dataFingerprint;
bool _anotherSyncNeeded = false;
QHash<QString, long long> _filesNeedingScheduledSync;
QVector<QString> _filesUnscheduleSync;
QStringList _listExclusiveFiles;
QStringList _forbiddenFilenames;
QStringList _forbiddenBasenames;
QStringList _forbiddenExtensions;
QStringList _forbiddenChars;
bool _hasUploadErrorItems = false;
bool _hasDownloadRemovedItems = false;
bool _noCaseConflictRecordsInDb = false;
QSet<QString> _topLevelE2eeFolderPaths;
signals:
void fatalError(const QString &errorString, const OCC::ErrorCategory errorCategory);
void itemDiscovered(const OCC::SyncFileItemPtr &item);
void finished();
// A new folder was discovered and was not synced because of the confirmation feature
void newBigFolder(const QString &folder, bool isExternal);
void existingFolderNowBig(const QString &folder);
/** For excluded items that don't show up in itemDiscovered()
*
* The path is relative to the sync folder, similar to item->_file
*/
void silentlyExcluded(const QString &folderPath);
void addErrorToGui(const SyncFileItem::Status status, const QString &errorMessage, const QString &subject, const OCC::ErrorCategory category);
void remnantReadOnlyFolderDiscovered(const OCC::SyncFileItemPtr &item);
private slots:
void slotItemDiscovered(const OCC::SyncFileItemPtr &item);
};
/// Implementation of DiscoveryPhase::adjustRenamedPath
QString adjustRenamedPath(const QMap<QString, QString> &renamedItems, const QString &original);
}
|