1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
|
// Package bisync implements bisync
// Copyright (c) 2017-2020 Chris Nelson
package bisync
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/cmd/bisync/bilib"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/hash"
"github.com/spf13/cobra"
)
// TestFunc allows mocking errors during tests
type TestFunc func()
// Options keep bisync options
type Options struct {
Resync bool // whether or not this is a resync
ResyncMode Prefer // which mode to use for resync
CheckAccess bool
CheckFilename string
CheckSync CheckSyncMode
CreateEmptySrcDirs bool
RemoveEmptyDirs bool
MaxDelete int // percentage from 0 to 100
Force bool
FiltersFile string
Workdir string
OrigBackupDir string
BackupDir1 string
BackupDir2 string
DryRun bool
NoCleanup bool
SaveQueues bool // save extra debugging files (test only flag)
IgnoreListingChecksum bool
Resilient bool
Recover bool
TestFn TestFunc // test-only option, for mocking errors
Compare CompareOpt
CompareFlag string
DebugName string
MaxLock time.Duration
ConflictResolve Prefer
ConflictLoser ConflictLoserAction
ConflictSuffixFlag string
ConflictSuffix1 string
ConflictSuffix2 string
}
// Default values
const (
DefaultMaxDelete int = 50
DefaultCheckFilename string = "RCLONE_TEST"
)
// DefaultWorkdir is default working directory
var DefaultWorkdir = filepath.Join(config.GetCacheDir(), "bisync")
// CheckSyncMode controls when to compare final listings
type CheckSyncMode int
// CheckSync modes
const (
CheckSyncTrue CheckSyncMode = iota // Compare final listings (default)
CheckSyncFalse // Disable comparison of final listings
CheckSyncOnly // Only compare listings from the last run, do not sync
)
func (x CheckSyncMode) String() string {
switch x {
case CheckSyncTrue:
return "true"
case CheckSyncFalse:
return "false"
case CheckSyncOnly:
return "only"
}
return "unknown"
}
// Set a CheckSync mode from a string
func (x *CheckSyncMode) Set(s string) error {
switch strings.ToLower(s) {
case "true":
*x = CheckSyncTrue
case "false":
*x = CheckSyncFalse
case "only":
*x = CheckSyncOnly
default:
return fmt.Errorf("unknown check-sync mode for bisync: %q", s)
}
return nil
}
// Type of the CheckSync value
func (x *CheckSyncMode) Type() string {
return "string"
}
// Opt keeps command line options
var Opt Options
func init() {
Opt.MaxLock = 0
cmd.Root.AddCommand(commandDefinition)
cmdFlags := commandDefinition.Flags()
// when adding new flags, remember to also update the rc params:
// cmd/bisync/rc.go cmd/bisync/help.go (not docs/content/rc.md)
// and the Command line syntax section of docs/content/bisync.md (it doesn't update automatically)
flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first.", "")
flags.FVarP(cmdFlags, &Opt.ResyncMode, "resync-mode", "", "During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.)", "")
flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, makeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "")
flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"), "")
flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "")
flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "")
flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "")
flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "")
flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file", "")
flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")
flags.StringVarP(cmdFlags, &Opt.BackupDir1, "backup-dir1", "", Opt.BackupDir1, "--backup-dir for Path1. Must be a non-overlapping path on the same remote.", "")
flags.StringVarP(cmdFlags, &Opt.BackupDir2, "backup-dir2", "", Opt.BackupDir2, "--backup-dir for Path2. Must be a non-overlapping path on the same remote.", "")
flags.StringVarP(cmdFlags, &Opt.DebugName, "debugname", "", Opt.DebugName, "Debug by tracking one file at various points throughout a bisync run (when -v or -vv)", "")
flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "")
flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "")
flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")
flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "")
flags.BoolVarP(cmdFlags, &Opt.Recover, "recover", "", Opt.Recover, "Automatically recover from interruptions without requiring --resync.", "")
flags.StringVarP(cmdFlags, &Opt.CompareFlag, "compare", "", Opt.CompareFlag, "Comma-separated list of bisync-specific compare options ex. 'size,modtime,checksum' (default: 'size,modtime')", "")
flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")
flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "")
flags.BoolVarP(cmdFlags, &Opt.Compare.DownloadHash, "download-hash", "", Opt.Compare.DownloadHash, "Compute hash by downloading when otherwise unavailable. (warning: may be slow and use lots of data!)", "")
flags.DurationVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", Opt.MaxLock, "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
flags.FVarP(cmdFlags, &Opt.ConflictResolve, "conflict-resolve", "", "Automatically resolve conflicts by preferring the version that is: "+ConflictResolveList+" (default: none)", "")
flags.FVarP(cmdFlags, &Opt.ConflictLoser, "conflict-loser", "", "Action to take on the loser of a sync conflict (when there is a winner) or on both files (when there is no winner): "+ConflictLoserList+" (default: num)", "")
flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "")
_ = cmdFlags.MarkHidden("debugname")
_ = cmdFlags.MarkHidden("localtime")
}
// bisync command definition
var commandDefinition = &cobra.Command{
Use: "bisync remote1:path1 remote2:path2",
Short: shortHelp,
Long: longHelp,
Annotations: map[string]string{
"versionIntroduced": "v1.58",
"groups": "Filter,Copy,Important",
"status": "Beta",
},
RunE: func(command *cobra.Command, args []string) error {
// NOTE: avoid putting too much handling here, as it won't apply to the rc.
// Generally it's best to put init-type stuff in Bisync() (operations.go)
cmd.CheckArgs(2, 2, command, args)
fs1, file1, fs2, file2 := cmd.NewFsSrcDstFiles(args)
if file1 != "" || file2 != "" {
return errors.New("paths must be existing directories")
}
ctx := context.Background()
opt := Opt
opt.applyContext(ctx)
if tzLocal {
TZ = time.Local
}
commonHashes := fs1.Hashes().Overlap(fs2.Hashes())
isDropbox1 := strings.HasPrefix(fs1.String(), "Dropbox")
isDropbox2 := strings.HasPrefix(fs2.String(), "Dropbox")
if commonHashes == hash.Set(0) && (isDropbox1 || isDropbox2) {
ci := fs.GetConfig(ctx)
if !ci.DryRun && !ci.RefreshTimes {
fs.Debugf(nil, "Using flag --refresh-times is recommended")
}
}
fs.Logf(nil, "bisync is IN BETA. Don't use in production!")
cmd.Run(false, true, command, func() error {
err := Bisync(ctx, fs1, fs2, &opt)
if err == ErrBisyncAborted {
return fserrors.FatalError(err)
}
return err
})
return nil
},
}
func (opt *Options) applyContext(ctx context.Context) {
maxDelete := DefaultMaxDelete
ci := fs.GetConfig(ctx)
if ci.MaxDelete >= 0 {
maxDelete = int(ci.MaxDelete)
}
if maxDelete < 0 {
maxDelete = 0
}
if maxDelete > 100 {
maxDelete = 100
}
opt.MaxDelete = maxDelete
// reset MaxDelete for fs/operations, bisync handles this parameter specially
ci.MaxDelete = -1
opt.DryRun = ci.DryRun
}
func (opt *Options) setDryRun(ctx context.Context) context.Context {
ctxNew, ci := fs.AddConfig(ctx)
ci.DryRun = opt.DryRun
return ctxNew
}
func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
filtersFile := opt.FiltersFile
if filtersFile == "" {
return ctx, nil
}
f, err := os.Open(filtersFile)
if err != nil {
return ctx, fmt.Errorf("specified filters file does not exist: %s", filtersFile)
}
fs.Infof(nil, "Using filters file %s", filtersFile)
hasher := md5.New()
if _, err := io.Copy(hasher, f); err != nil {
_ = f.Close()
return ctx, err
}
gotHash := hex.EncodeToString(hasher.Sum(nil))
_ = f.Close()
hashFile := filtersFile + ".md5"
wantHash, err := os.ReadFile(hashFile)
if err != nil && !opt.Resync {
return ctx, fmt.Errorf("filters file md5 hash not found (must run --resync): %s", filtersFile)
}
if gotHash != string(wantHash) && !opt.Resync {
return ctx, fmt.Errorf("filters file has changed (must run --resync): %s", filtersFile)
}
if opt.Resync {
if opt.DryRun {
fs.Infof(nil, "Skipped storing filters file hash to %s as --dry-run is set", hashFile)
} else {
fs.Infof(nil, "Storing filters file hash to %s", hashFile)
if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
return ctx, err
}
}
}
// Prepend our filter file first in the list
filterOpt := filter.GetConfig(ctx).Opt
filterOpt.FilterFrom = append([]string{filtersFile}, filterOpt.FilterFrom...)
newFilter, err := filter.NewFilter(&filterOpt)
if err != nil {
return ctx, fmt.Errorf("invalid filters file: %s: %w", filtersFile, err)
}
return filter.ReplaceConfig(ctx, newFilter), nil
}
|