File: discoveryphase.h

package info (click to toggle)
nextcloud-desktop 4.0.1-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 40,404 kB
  • sloc: cpp: 118,401; objc: 752; python: 606; sh: 395; ansic: 391; ruby: 174; makefile: 44; javascript: 32; xml: 6
file content (345 lines) | stat: -rw-r--r-- 12,324 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
/*
 * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
 * SPDX-FileCopyrightText: 2014 ownCloud GmbH
 * SPDX-License-Identifier: GPL-2.0-or-later
 */

#pragma once

#include "networkjobs.h"
#include "syncoptions.h"
#include "syncfileitem.h"

#include "common/folderquota.h"
#include "common/remoteinfo.h"

#include <QObject>
#include <QElapsedTimer>
#include <QStringList>
#include <csync.h>
#include <QMap>
#include <QSet>
#include <QMutex>
#include <QWaitCondition>
#include <QRunnable>
#include <deque>

class ExcludedFiles;

namespace OCC {

namespace LocalDiscoveryEnums {

OCSYNC_EXPORT Q_NAMESPACE

enum class LocalDiscoveryStyle {
    FilesystemOnly, //< read all local data from the filesystem
    DatabaseAndFilesystem, //< read from the db, except for listed paths
};

Q_ENUM_NS(LocalDiscoveryStyle)

}

using OCC::LocalDiscoveryEnums::LocalDiscoveryStyle;

class Account;
class SyncJournalDb;
class ProcessDirectoryJob;

enum class ErrorCategory;

struct LocalInfo
{
    /** FileName of the entry (this does not contains any directory or path, just the plain name */
    QString name;
    QString caseClashConflictingName;
    time_t modtime = 0;
    int64_t size = 0;
    uint64_t inode = 0;
    ItemType type = ItemTypeSkip;
    bool isDirectory = false;
    bool isHidden = false;
    bool isVirtualFile = false;
    bool isSymLink = false;
    bool isMetadataMissing = false;
    [[nodiscard]] bool isValid() const { return !name.isNull(); }
};

/**
 * @brief Run list on a local directory and process the results for Discovery
 *
 * @ingroup libsync
 */
class DiscoverySingleLocalDirectoryJob : public QObject, public QRunnable
{
    Q_OBJECT
public:
    explicit DiscoverySingleLocalDirectoryJob(const AccountPtr &account, const QString &localPath, OCC::Vfs *vfs, QObject *parent = nullptr);

    void run() override;
signals:
    void finished(QVector<OCC::LocalInfo> result);
    void finishedFatalError(QString errorString);
    void finishedNonFatalError(QString errorString);

    void itemDiscovered(OCC::SyncFileItemPtr item);
    void childIgnored(bool b);
private slots:
private:
    QString _localPath;
    AccountPtr _account;
    OCC::Vfs* _vfs;
public:
};

class FolderMetadata;

/**
 * @brief Run a PROPFIND on a directory and process the results for Discovery
 *
 * @ingroup libsync
 */
class DiscoverySingleDirectoryJob : public QObject
{
    Q_OBJECT
public:
    explicit DiscoverySingleDirectoryJob(const AccountPtr &account,
                                         const QString &path,
                                         const QString &remoteRootFolderPath,
        /* TODO for topLevelE2eeFolderPaths, from review: I still do not get why giving the whole QSet instead of just the parent of the folder we are in
        sounds to me like it would be much more efficient to just have the e2ee parent folder that we are
        inside*/
                                         const QSet<QString> &topLevelE2eeFolderPaths,
                                         SyncFileItem::EncryptionStatus parentEncryptionStatus,
                                         QObject *parent = nullptr);
    // Specify that this is the root and we need to check the data-fingerprint
    void setIsRootPath() { _isRootPath = true; }
    void start();
    void abort();
    [[nodiscard]] bool isFileDropDetected() const;
    [[nodiscard]] bool encryptedMetadataNeedUpdate() const;
    [[nodiscard]] SyncFileItem::EncryptionStatus currentEncryptionStatus() const;
    [[nodiscard]] SyncFileItem::EncryptionStatus requiredEncryptionStatus() const;

    // This is not actually a network job, it is just a job
signals:
    void firstDirectoryPermissions(OCC::RemotePermissions);
    void etag(const QByteArray &, const QDateTime &time);
    void finished(const OCC::HttpResult<QVector<OCC::RemoteInfo>> &result);
    void setfolderQuota(const FolderQuota &folderQuota);

private slots:
    void directoryListingIteratedSlot(const QString &, const QMap<QString, QString> &);
    void lsJobFinishedWithoutErrorSlot();
    void lsJobFinishedWithErrorSlot(QNetworkReply *reply);
    void fetchE2eMetadata();
    void metadataReceived(const QJsonDocument &json, int statusCode);
    void metadataError(const QByteArray& fileId, int httpReturnCode);

private:

    [[nodiscard]] bool isE2eEncrypted() const { return _encryptionStatusCurrent != SyncFileItem::EncryptionStatus::NotEncrypted; }

    QVector<RemoteInfo> _results;
    QString _subPath;
    QString _remoteRootFolderPath;
    QByteArray _firstEtag;
    QByteArray _fileId;
    QByteArray _localFileId;
    AccountPtr _account;
    // The first result is for the directory itself and need to be ignored.
    // This flag is true if it was already ignored.
    bool _ignoredFirst = false;
    // Set to true if this is the root path and we need to check the data-fingerprint
    bool _isRootPath = false;
    // If this directory is an external storage (The first item has 'M' in its permission)
    bool _isExternalStorage = false;
    // If this directory is e2ee
    SyncFileItem::EncryptionStatus _encryptionStatusCurrent = SyncFileItem::EncryptionStatus::NotEncrypted;
    bool _isFileDropDetected = false;
    bool _encryptedMetadataNeedUpdate = false;
    SyncFileItem::EncryptionStatus _encryptionStatusRequired = SyncFileItem::EncryptionStatus::NotEncrypted;

    // If set, the discovery will finish with an error
    int64_t _size = 0;
    QString _error;
    QPointer<LsColJob> _lsColJob;

    // store top level E2EE folder paths as they are used later when discovering nested folders
    QSet<QString> _topLevelE2eeFolderPaths;

public:
    QByteArray _dataFingerprint;
    FolderQuota _folderQuota;
};

class DiscoveryPhase : public QObject
{
    Q_OBJECT

    friend class ProcessDirectoryJob;

    QPointer<ProcessDirectoryJob> _currentRootJob;

    /** Maps the db-path of a deleted item to its SyncFileItem.
     *
     * If it turns out the item was renamed after all, the instruction
     * can be changed. See findAndCancelDeletedJob(). Note that
     * itemDiscovered() will already have been emitted for the item.
     */
    QMap<QString, SyncFileItemPtr> _deletedItem;

    QVector<QString> _directoryNamesToRestoreOnPropagation;

    /** Maps the db-path of a deleted folder to its queued job.
     *
     * If a folder is deleted and must be recursed into, its job isn't
     * executed immediately. Instead it's queued here and only run
     * once the rest of the discovery has finished and we are certain
     * that the folder wasn't just renamed. This avoids running the
     * discovery on contents in the old location of renamed folders.
     *
     * See findAndCancelDeletedJob().
     */
    QMap<QString, ProcessDirectoryJob *> _queuedDeletedDirectories;

    // map source (original path) -> destinations (current server or local path)
    QMap<QString, QString> _renamedItemsRemote;
    QMap<QString, QString> _renamedItemsLocal;

    // set of paths that should not be removed even though they are removed locally:
    // there was a move to an invalid destination and now the source should be restored
    //
    // This applies recursively to subdirectories.
    // All entries should have a trailing slash (even files), so lookup with
    // lowerBound() is reliable.
    //
    // The value of this map doesn't matter.
    QMap<QString, bool> _forbiddenDeletes;

    /** Returns whether the db-path has been renamed locally or on the remote.
     *
     * Useful for avoiding processing of items that have already been claimed in
     * a rename (would otherwise be discovered as deletions).
     */
    [[nodiscard]] bool isRenamed(const QString &p) const;

    int _currentlyActiveJobs = 0;

    // both must contain a sorted list
    QStringList _selectiveSyncBlackList;
    QStringList _selectiveSyncWhiteList;

    void scheduleMoreJobs();

    [[nodiscard]] bool isInSelectiveSyncBlackList(const QString &path) const;

    [[nodiscard]] bool activeFolderSizeLimit() const;
    [[nodiscard]] bool notifyExistingFolderOverLimit() const;

    void checkFolderSizeLimit(const QString &path,
			      const std::function<void(bool)> callback);

    // Check if the new folder should be deselected or not.
    // May be async. "Return" via the callback, true if the item is blacklisted
    void checkSelectiveSyncNewFolder(const QString &path,
                                     const RemotePermissions rp,
                                     const std::function<void(bool)> callback);

    void checkSelectiveSyncExistingFolder(const QString &path);

    /** Given an original path, return the target path obtained when renaming is done.
     *
     * Note that it only considers parent directory renames. So if A/B got renamed to C/D,
     * checking A/B/file would yield C/D/file, but checking A/B would yield A/B.
     */
    [[nodiscard]] QString adjustRenamedPath(const QString &original, SyncFileItem::Direction) const;

    /** If the db-path is scheduled for deletion, abort it.
     *
     * Check if there is already a job to delete that item:
     * If that's not the case, return { false, QByteArray() }.
     * If there is such a job, cancel that job and return true and the old etag.
     *
     * Used when having detected a rename: The rename source may have been
     * discovered before and would have looked like a delete.
     *
     * See _deletedItem and _queuedDeletedDirectories.
     */
    QPair<bool, QByteArray> findAndCancelDeletedJob(const QString &originalPath);

    void enqueueDirectoryToDelete(const QString &path, ProcessDirectoryJob* const directoryJob);

    bool recursiveCheckForDeletedParents(const QString &itemPath) const;

    /// contains files/folder names that are requested to be deleted permanently
    QSet<QString> _permanentDeletionRequests;

    void markPermanentDeletionRequests();

public:
    // input
    QString _localDir; // absolute path to the local directory. ends with '/'
    QString _remoteFolder; // remote folder, ends with '/'
    SyncJournalDb *_statedb = nullptr;
    AccountPtr _account;
    SyncOptions _syncOptions;
    ExcludedFiles *_excludes = nullptr;
    QRegularExpression _invalidFilenameRx; // FIXME: maybe move in ExcludedFiles
    QStringList _serverBlacklistedFiles; // The blacklist from the capabilities
    QStringList _leadingAndTrailingSpacesFilesAllowed;
    bool _shouldEnforceWindowsFileNameCompatibility = false;
    bool _ignoreHiddenFiles = false;
    std::function<bool(const QString &)> _shouldDiscoverLocaly;

    void startJob(ProcessDirectoryJob *);

    void setSelectiveSyncBlackList(const QStringList &list);
    void setSelectiveSyncWhiteList(const QStringList &list);

    // output
    QByteArray _dataFingerprint;
    bool _anotherSyncNeeded = false;
    QHash<QString, long long> _filesNeedingScheduledSync;
    QVector<QString> _filesUnscheduleSync;

    QStringList _listExclusiveFiles;

    QStringList _forbiddenFilenames;
    QStringList _forbiddenBasenames;
    QStringList _forbiddenExtensions;
    QStringList _forbiddenChars;

    bool _hasUploadErrorItems = false;
    bool _hasDownloadRemovedItems = false;

    bool _noCaseConflictRecordsInDb = false;

    QSet<QString> _topLevelE2eeFolderPaths;

signals:
    void fatalError(const QString &errorString, const OCC::ErrorCategory errorCategory);
    void itemDiscovered(const OCC::SyncFileItemPtr &item);
    void finished();

    // A new folder was discovered and was not synced because of the confirmation feature
    void newBigFolder(const QString &folder, bool isExternal);
    void existingFolderNowBig(const QString &folder);

    /** For excluded items that don't show up in itemDiscovered()
      *
      * The path is relative to the sync folder, similar to item->_file
      */
    void silentlyExcluded(const QString &folderPath);

    void addErrorToGui(const SyncFileItem::Status status, const QString &errorMessage, const QString &subject, const OCC::ErrorCategory category);

    void remnantReadOnlyFolderDiscovered(const OCC::SyncFileItemPtr &item);
private slots:
    void slotItemDiscovered(const OCC::SyncFileItemPtr &item);
};

/// Implementation of DiscoveryPhase::adjustRenamedPath
QString adjustRenamedPath(const QMap<QString, QString> &renamedItems, const QString &original);
}