File: cmd.go

package info (click to toggle)
rclone 1.60.1%2Bdfsg-4
  • links: PTS, VCS
  • area: main
  • in suites: sid, trixie
  • size: 34,832 kB
  • sloc: sh: 957; xml: 857; python: 655; javascript: 612; makefile: 269; ansic: 101; php: 74
file content (226 lines) | stat: -rw-r--r-- 6,809 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
// Package bisync implements bisync
// Copyright (c) 2017-2020 Chris Nelson
package bisync

import (
	"context"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/rclone/rclone/cmd"
	"github.com/rclone/rclone/cmd/bisync/bilib"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/config"
	"github.com/rclone/rclone/fs/config/flags"
	"github.com/rclone/rclone/fs/filter"
	"github.com/rclone/rclone/fs/hash"

	"github.com/spf13/cobra"
)

// Options keep bisync options
type Options struct {
	Resync          bool
	CheckAccess     bool
	CheckFilename   string
	CheckSync       CheckSyncMode
	RemoveEmptyDirs bool
	MaxDelete       int // percentage from 0 to 100
	Force           bool
	FiltersFile     string
	Workdir         string
	DryRun          bool
	NoCleanup       bool
	SaveQueues      bool // save extra debugging files (test only flag)
}

// Default values
const (
	DefaultMaxDelete     int    = 50
	DefaultCheckFilename string = "RCLONE_TEST"
)

// DefaultWorkdir is default working directory
var DefaultWorkdir = filepath.Join(config.GetCacheDir(), "bisync")

// CheckSyncMode controls when to compare final listings
type CheckSyncMode int

// CheckSync modes
const (
	CheckSyncTrue  CheckSyncMode = iota // Compare final listings (default)
	CheckSyncFalse                      // Disable comparison of final listings
	CheckSyncOnly                       // Only compare listings from the last run, do not sync
)

func (x CheckSyncMode) String() string {
	switch x {
	case CheckSyncTrue:
		return "true"
	case CheckSyncFalse:
		return "false"
	case CheckSyncOnly:
		return "only"
	}
	return "unknown"
}

// Set a CheckSync mode from a string
func (x *CheckSyncMode) Set(s string) error {
	switch strings.ToLower(s) {
	case "true":
		*x = CheckSyncTrue
	case "false":
		*x = CheckSyncFalse
	case "only":
		*x = CheckSyncOnly
	default:
		return fmt.Errorf("unknown check-sync mode for bisync: %q", s)
	}
	return nil
}

// Type of the CheckSync value
func (x *CheckSyncMode) Type() string {
	return "string"
}

// Opt keeps command line options
var Opt Options

func init() {
	cmd.Root.AddCommand(commandDefinition)
	cmdFlags := commandDefinition.Flags()
	flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Path1 files may overwrite Path2 versions. Consider using --verbose or --dry-run first.")
	flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, makeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."))
	flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"))
	flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose")
	flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)")
	flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove empty directories at the final cleanup step.")
	flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file")
	flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"))
	flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)")
	flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).")
}

// bisync command definition
var commandDefinition = &cobra.Command{
	Use:   "bisync remote1:path1 remote2:path2",
	Short: shortHelp,
	Long:  longHelp,
	RunE: func(command *cobra.Command, args []string) error {
		cmd.CheckArgs(2, 2, command, args)
		fs1, file1, fs2, file2 := cmd.NewFsSrcDstFiles(args)
		if file1 != "" || file2 != "" {
			return errors.New("paths must be existing directories")
		}

		ctx := context.Background()
		opt := Opt
		opt.applyContext(ctx)

		if tzLocal {
			TZ = time.Local
		}

		commonHashes := fs1.Hashes().Overlap(fs2.Hashes())
		isDropbox1 := strings.HasPrefix(fs1.String(), "Dropbox")
		isDropbox2 := strings.HasPrefix(fs2.String(), "Dropbox")
		if commonHashes == hash.Set(0) && (isDropbox1 || isDropbox2) {
			ci := fs.GetConfig(ctx)
			if !ci.DryRun && !ci.RefreshTimes {
				fs.Debugf(nil, "Using flag --refresh-times is recommended")
			}
		}

		fs.Logf(nil, "bisync is EXPERIMENTAL. Don't use in production!")
		cmd.Run(false, true, command, func() error {
			err := Bisync(ctx, fs1, fs2, &opt)
			if err == ErrBisyncAborted {
				os.Exit(2)
			}
			return err
		})
		return nil
	},
}

func (opt *Options) applyContext(ctx context.Context) {
	maxDelete := DefaultMaxDelete
	ci := fs.GetConfig(ctx)
	if ci.MaxDelete >= 0 {
		maxDelete = int(ci.MaxDelete)
	}
	if maxDelete < 0 {
		maxDelete = 0
	}
	if maxDelete > 100 {
		maxDelete = 100
	}
	opt.MaxDelete = maxDelete
	// reset MaxDelete for fs/operations, bisync handles this parameter specially
	ci.MaxDelete = -1
	opt.DryRun = ci.DryRun
}

func (opt *Options) setDryRun(ctx context.Context) context.Context {
	ctxNew, ci := fs.AddConfig(ctx)
	ci.DryRun = opt.DryRun
	return ctxNew
}

func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
	filtersFile := opt.FiltersFile
	if filtersFile == "" {
		return ctx, nil
	}

	f, err := os.Open(filtersFile)
	if err != nil {
		return ctx, fmt.Errorf("specified filters file does not exist: %s", filtersFile)
	}

	fs.Infof(nil, "Using filters file %s", filtersFile)
	hasher := md5.New()
	if _, err := io.Copy(hasher, f); err != nil {
		_ = f.Close()
		return ctx, err
	}
	gotHash := hex.EncodeToString(hasher.Sum(nil))
	_ = f.Close()

	hashFile := filtersFile + ".md5"
	wantHash, err := ioutil.ReadFile(hashFile)
	if err != nil && !opt.Resync {
		return ctx, fmt.Errorf("filters file md5 hash not found (must run --resync): %s", filtersFile)
	}

	if gotHash != string(wantHash) && !opt.Resync {
		return ctx, fmt.Errorf("filters file has changed (must run --resync): %s", filtersFile)
	}

	if opt.Resync {
		fs.Infof(nil, "Storing filters file hash to %s", hashFile)
		if err := ioutil.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
			return ctx, err
		}
	}

	// Prepend our filter file first in the list
	filterOpt := filter.GetConfig(ctx).Opt
	filterOpt.FilterFrom = append([]string{filtersFile}, filterOpt.FilterFrom...)
	newFilter, err := filter.NewFilter(&filterOpt)
	if err != nil {
		return ctx, fmt.Errorf("invalid filters file: %s: %w", filtersFile, err)
	}

	return filter.ReplaceConfig(ctx, newFilter), nil
}