File: resync.go

package info (click to toggle)
rclone 1.69.3%2Bdfsg-1
  • links: PTS, VCS
  • area: main
  • in suites: experimental
  • size: 45,716 kB
  • sloc: sh: 1,115; xml: 857; python: 754; javascript: 612; makefile: 299; ansic: 101; php: 74
file content (226 lines) | stat: -rw-r--r-- 6,862 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
package bisync

import (
	"context"
	"os"
	"path/filepath"

	"github.com/rclone/rclone/cmd/bisync/bilib"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/filter"
	"github.com/rclone/rclone/lib/terminal"
)

// for backward compatibility, --resync is now equivalent to --resync-mode path1
// and either flag is sufficient without the other.
func (b *bisyncRun) setResyncDefaults() {
	if b.opt.Resync && b.opt.ResyncMode == PreferNone {
		fs.Debugf(nil, Color(terminal.Dim, "defaulting to --resync-mode path1 as --resync is set")) //nolint:govet
		b.opt.ResyncMode = PreferPath1
	}
	if b.opt.ResyncMode != PreferNone {
		b.opt.Resync = true
		Opt.Resync = true // shouldn't be using this one, but set to be safe
	}

	// checks and warnings
	if (b.opt.ResyncMode == PreferNewer || b.opt.ResyncMode == PreferOlder) && (b.fs1.Precision() == fs.ModTimeNotSupported || b.fs2.Precision() == fs.ModTimeNotSupported) {
		fs.Logf(nil, Color(terminal.YellowFg, "WARNING: ignoring --resync-mode %s as at least one remote does not support modtimes."), b.opt.ResyncMode.String())
		b.opt.ResyncMode = PreferPath1
	} else if (b.opt.ResyncMode == PreferNewer || b.opt.ResyncMode == PreferOlder) && !b.opt.Compare.Modtime {
		fs.Logf(nil, Color(terminal.YellowFg, "WARNING: ignoring --resync-mode %s as --compare does not include modtime."), b.opt.ResyncMode.String())
		b.opt.ResyncMode = PreferPath1
	}
	if (b.opt.ResyncMode == PreferLarger || b.opt.ResyncMode == PreferSmaller) && !b.opt.Compare.Size {
		fs.Logf(nil, Color(terminal.YellowFg, "WARNING: ignoring --resync-mode %s as --compare does not include size."), b.opt.ResyncMode.String())
		b.opt.ResyncMode = PreferPath1
	}
}

// resync implements the --resync mode.
// It will generate path1 and path2 listings,
// copy any unique files to the opposite path,
// and resolve any differing files according to the --resync-mode.
func (b *bisyncRun) resync(octx, fctx context.Context) error {
	fs.Infof(nil, "Copying Path2 files to Path1")

	// Save blank filelists (will be filled from sync results)
	var ls1 = newFileList()
	var ls2 = newFileList()
	err = ls1.save(fctx, b.newListing1)
	if err != nil {
		b.handleErr(ls1, "error saving ls1 from resync", err, true, true)
		b.abort = true
	}
	err = ls2.save(fctx, b.newListing2)
	if err != nil {
		b.handleErr(ls2, "error saving ls2 from resync", err, true, true)
		b.abort = true
	}

	// Check access health on the Path1 and Path2 filesystems
	// enforce even though this is --resync
	if b.opt.CheckAccess {
		fs.Infof(nil, "Checking access health")

		filesNow1, filesNow2, err := b.findCheckFiles(fctx)
		if err != nil {
			b.critical = true
			b.retryable = true
			return err
		}

		ds1 := &deltaSet{
			checkFiles: bilib.Names{},
		}

		ds2 := &deltaSet{
			checkFiles: bilib.Names{},
		}

		for _, file := range filesNow1.list {
			if filepath.Base(file) == b.opt.CheckFilename {
				ds1.checkFiles.Add(file)
			}
		}

		for _, file := range filesNow2.list {
			if filepath.Base(file) == b.opt.CheckFilename {
				ds2.checkFiles.Add(file)
			}
		}

		err = b.checkAccess(ds1.checkFiles, ds2.checkFiles)
		if err != nil {
			b.critical = true
			b.retryable = true
			return err
		}
	}

	var results2to1 []Results
	var results1to2 []Results
	queues := queues{}

	b.indent("Path2", "Path1", "Resync is copying files to")
	ctxRun := b.opt.setDryRun(fctx)
	// fctx has our extra filters added!
	ctxSync, filterSync := filter.AddConfig(ctxRun)
	if filterSync.Opt.MinSize == -1 {
		fs.Debugf(nil, "filterSync.Opt.MinSize: %v", filterSync.Opt.MinSize)
	}
	b.resyncIs1to2 = false
	ctxSync = b.setResyncConfig(ctxSync)
	ctxSync = b.setBackupDir(ctxSync, 1)
	// 2 to 1
	if results2to1, err = b.resyncDir(ctxSync, b.fs2, b.fs1); err != nil {
		b.critical = true
		return err
	}

	b.indent("Path1", "Path2", "Resync is copying files to")
	b.resyncIs1to2 = true
	ctxSync = b.setResyncConfig(ctxSync)
	ctxSync = b.setBackupDir(ctxSync, 2)
	// 1 to 2
	if results1to2, err = b.resyncDir(ctxSync, b.fs1, b.fs2); err != nil {
		b.critical = true
		return err
	}

	fs.Infof(nil, "Resync updating listings")
	b.saveOldListings() // may not exist, as this is --resync
	b.replaceCurrentListings()

	resultsToQueue := func(results []Results) bilib.Names {
		names := bilib.Names{}
		for _, result := range results {
			if result.Name != "" &&
				(result.Flags != "d" || b.opt.CreateEmptySrcDirs) &&
				result.IsSrc && result.Src != "" &&
				(result.Winner.Err == nil || result.Flags == "d") {
				names.Add(result.Name)
			}
		}
		return names
	}

	// resync 2to1
	queues.copy2to1 = resultsToQueue(results2to1)
	if err = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false); err != nil {
		b.critical = true
		return err
	}

	// resync 1to2
	queues.copy1to2 = resultsToQueue(results1to2)
	if err = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true); err != nil {
		b.critical = true
		return err
	}

	if b.opt.CheckSync == CheckSyncTrue && !b.opt.DryRun {
		path1 := bilib.FsPath(b.fs1)
		path2 := bilib.FsPath(b.fs2)
		fs.Infof(nil, "Validating listings for Path1 %s vs Path2 %s", quotePath(path1), quotePath(path2))
		if err := b.checkSync(b.listing1, b.listing2); err != nil {
			b.critical = true
			return err
		}
	}

	if !b.opt.NoCleanup {
		_ = os.Remove(b.newListing1)
		_ = os.Remove(b.newListing2)
	}
	return nil
}

/*
	 --resync-mode implementation:
		PreferPath1: set ci.IgnoreExisting true, then false
		PreferPath2: set ci.IgnoreExisting false, then true
		PreferNewer: set ci.UpdateOlder in both directions
		PreferOlder: override EqualFn to implement custom logic
		PreferLarger: override EqualFn to implement custom logic
		PreferSmaller: override EqualFn to implement custom logic
*/
func (b *bisyncRun) setResyncConfig(ctx context.Context) context.Context {
	ci := fs.GetConfig(ctx)
	switch b.opt.ResyncMode {
	case PreferPath1:
		if !b.resyncIs1to2 { // 2to1 (remember 2to1 is first)
			ci.IgnoreExisting = true
		} else { // 1to2
			ci.IgnoreExisting = false
		}
	case PreferPath2:
		if !b.resyncIs1to2 { // 2to1 (remember 2to1 is first)
			ci.IgnoreExisting = false
		} else { // 1to2
			ci.IgnoreExisting = true
		}
	case PreferNewer:
		ci.UpdateOlder = true
	}
	// for older, larger, and smaller, we return it unchanged and handle it later
	return ctx
}

func (b *bisyncRun) resyncWhichIsWhich(src, dst fs.ObjectInfo) (path1, path2 fs.ObjectInfo) {
	if b.resyncIs1to2 {
		return src, dst
	}
	return dst, src
}

// equal in this context really means "don't transfer", so we should
// return true if the files are actually equal or if dest is winner,
// false if src is winner
// When can't determine, we end up running the normal Equal() to tie-break (due to our differ functions).
func (b *bisyncRun) resyncWinningPathToEqual(winningPath int) bool {
	if b.resyncIs1to2 {
		return winningPath != 1
	}
	return winningPath != 2
}