1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
|
package gitannex
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"github.com/stretchr/testify/require"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/buildinfo"
)
// checkRcloneBinaryVersion runs whichever rclone is on the PATH and checks
// whether it reports a version that matches the test's expectations. Returns
// nil when the version is the expected version, otherwise returns an error.
func checkRcloneBinaryVersion(t *testing.T) error {
// versionInfo is a subset of information produced by "core/version".
type versionInfo struct {
Version string
IsGit bool
GoTags string
}
cmd := exec.Command("rclone", "rc", "--loopback", "core/version")
stdout, err := cmd.Output()
if err != nil {
return fmt.Errorf("failed to get rclone version: %w", err)
}
var parsed versionInfo
if err := json.Unmarshal(stdout, &parsed); err != nil {
return fmt.Errorf("failed to parse rclone version: %w", err)
}
if parsed.Version != fs.Version {
return fmt.Errorf("expected version %q, but got %q", fs.Version, parsed.Version)
}
if parsed.IsGit != strings.HasSuffix(fs.Version, "-DEV") {
return errors.New("expected rclone to be a dev build")
}
_, tagString := buildinfo.GetLinkingAndTags()
if parsed.GoTags != tagString {
// TODO: Skip the test when tags do not match.
t.Logf("expected tag string %q, but got %q. Not skipping!", tagString, parsed.GoTags)
}
return nil
}
// countFilesRecursively returns the number of files nested underneath `dir`. It
// counts files only and excludes directories.
func countFilesRecursively(t *testing.T, dir string) int {
remoteFiles, err := os.ReadDir(dir)
require.NoError(t, err)
var count int
for _, f := range remoteFiles {
if f.IsDir() {
subdir := filepath.Join(dir, f.Name())
count += countFilesRecursively(t, subdir)
} else {
count++
}
}
return count
}
func findFileWithContents(t *testing.T, dir string, wantContents []byte) bool {
remoteFiles, err := os.ReadDir(dir)
require.NoError(t, err)
for _, f := range remoteFiles {
fPath := filepath.Join(dir, f.Name())
if f.IsDir() {
if findFileWithContents(t, fPath, wantContents) {
return true
}
} else {
contents, err := os.ReadFile(fPath)
require.NoError(t, err)
if bytes.Equal(contents, wantContents) {
return true
}
}
}
return false
}
type e2eTestingContext struct {
t *testing.T
tempDir string
binDir string
homeDir string
configDir string
rcloneConfigDir string
ephemeralRepoDir string
}
// makeE2eTestingContext sets up a new e2eTestingContext rooted under
// `t.TempDir()`. It creates the skeleton directory structure shown below in the
// temp directory without creating any files.
//
// .
// |-- bin
// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
// |-- ephemeralRepo
// `-- user
// `-- .config
// `-- rclone
// `-- rclone.conf
func makeE2eTestingContext(t *testing.T) e2eTestingContext {
tempDir := t.TempDir()
binDir := filepath.Join(tempDir, "bin")
homeDir := filepath.Join(tempDir, "user")
configDir := filepath.Join(homeDir, ".config")
rcloneConfigDir := filepath.Join(configDir, "rclone")
ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")
for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
require.NoError(t, os.Mkdir(dir, 0700))
}
return e2eTestingContext{t, tempDir, binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir}
}
// Install the symlink that enables git-annex to invoke "rclone gitannex"
// without explicitly specifying the subcommand.
func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) {
rcloneBinaryPath, err := exec.LookPath("rclone")
require.NoError(t, err)
require.NoError(t, os.Symlink(
rcloneBinaryPath,
filepath.Join(e.binDir, "git-annex-remote-rclone-builtin")))
}
// Install a rclone.conf file in an appropriate location in the fake home
// directory. The config defines an rclone remote named "MyRcloneRemote" using
// the local backend.
func (e *e2eTestingContext) installRcloneConfig(t *testing.T) {
// Install the rclone.conf file that defines the remote.
rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf")
rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
}
// runInRepo runs the given command from within the ephemeral repo directory. To
// prevent accidental changes in the real home directory, it sets the HOME
// variable to a subdirectory of the temp directory. It also ensures that the
// git-annex-remote-rclone-builtin symlink will be found by extending the PATH.
func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) {
if testing.Verbose() {
t.Logf("Running %s %v\n", command, args)
}
cmd := exec.Command(command, args...)
cmd.Dir = e.ephemeralRepoDir
cmd.Env = []string{
"HOME=" + e.homeDir,
"PATH=" + os.Getenv("PATH") + ":" + e.binDir,
}
buf, err := cmd.CombinedOutput()
require.NoError(t, err, fmt.Sprintf("+ %s %v failed:\n%s\n", command, args, buf))
}
// createGitRepo creates an empty git repository in the ephemeral repo
// directory. It makes "global" config changes that are ultimately scoped to the
// calling test thanks to runInRepo() overriding the HOME environment variable.
func (e *e2eTestingContext) createGitRepo(t *testing.T) {
e.runInRepo(t, "git", "annex", "version")
e.runInRepo(t, "git", "config", "--global", "user.name", "User Name")
e.runInRepo(t, "git", "config", "--global", "user.email", "user@example.com")
e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main")
e.runInRepo(t, "git", "init")
e.runInRepo(t, "git", "annex", "init")
}
func skipE2eTestIfNecessary(t *testing.T) {
if testing.Short() {
t.Skip("Skipping due to short mode.")
}
// TODO: Support e2e tests on Windows. Need to evaluate the semantics of the
// HOME and PATH environment variables.
switch runtime.GOOS {
case "darwin",
"freebsd",
"linux",
"netbsd",
"openbsd",
"plan9",
"solaris":
default:
t.Skipf("GOOS %q is not supported.", runtime.GOOS)
}
if err := checkRcloneBinaryVersion(t); err != nil {
t.Skipf("Skipping due to rclone version: %s", err)
}
if _, err := exec.LookPath("git-annex"); err != nil {
t.Skipf("Skipping because git-annex was not found: %s", err)
}
}
// This end-to-end test runs `git annex testremote` in a temporary git repo.
// This test will be skipped unless the `rclone` binary on PATH reports the
// expected version.
//
// When run on CI, an rclone binary built from HEAD will be on the PATH. When
// running locally, you will likely need to ensure the current binary is on the
// PATH like so:
//
// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...
//
// In the future, this test will probably be extended to test a number of
// parameters like repo layouts, and runtime may suffer from a combinatorial
// explosion.
func TestEndToEnd(t *testing.T) {
skipE2eTestIfNecessary(t)
for _, mode := range allLayoutModes() {
mode := mode
t.Run(string(mode), func(t *testing.T) {
t.Parallel()
testingContext := makeE2eTestingContext(t)
testingContext.installRcloneGitannexSymlink(t)
testingContext.installRcloneConfig(t)
testingContext.createGitRepo(t)
testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote",
"type=external", "externaltype=rclone-builtin", "encryption=none",
"rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir,
"rclonelayout="+string(mode))
testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote")
})
}
}
// For each layout mode, migrate a single remote from git-annex-remote-rclone to
// git-annex-remote-rclone-builtin and run `git annex testremote`.
func TestEndToEndMigration(t *testing.T) {
skipE2eTestIfNecessary(t)
if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {
t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)
}
for _, mode := range allLayoutModes() {
mode := mode
t.Run(string(mode), func(t *testing.T) {
t.Parallel()
tc := makeE2eTestingContext(t)
tc.installRcloneGitannexSymlink(t)
tc.installRcloneConfig(t)
tc.createGitRepo(t)
remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")
require.NoError(t, os.Mkdir(remoteStorage, 0777))
tc.runInRepo(t,
"git", "annex", "initremote", "MigratedRemote",
"type=external", "externaltype=rclone", "encryption=none",
"target=MyRcloneRemote",
"rclone_layout="+string(mode),
"prefix="+remoteStorage,
)
fooFileContents := []byte{1, 2, 3, 4}
fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")
require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700))
tc.runInRepo(t, "git", "annex", "add", "foo")
tc.runInRepo(t, "git", "commit", "-m", "Add foo file")
// Git-annex objects are not writable, which prevents `testing` from
// cleaning up the temp directory. We can work around this by
// explicitly dropping any files we add to the annex.
t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })
tc.runInRepo(t, "git", "annex", "copy", "--to=MigratedRemote", "foo")
tc.runInRepo(t, "git", "annex", "fsck", "--from=MigratedRemote", "foo")
tc.runInRepo(t,
"git", "annex", "enableremote", "MigratedRemote",
"externaltype=rclone-builtin",
"rcloneremotename=MyRcloneRemote",
"rclonelayout="+string(mode),
"rcloneprefix="+remoteStorage,
)
tc.runInRepo(t, "git", "annex", "fsck", "--from=MigratedRemote", "foo")
tc.runInRepo(t, "git", "annex", "testremote", "MigratedRemote")
})
}
}
// For each layout mode, create two git-annex remotes with externaltype=rclone
// and externaltype=rclone-builtin respectively. Test that files copied to one
// remote are present on the other. Similarly, test that files deleted from one
// are removed on the other.
func TestEndToEndRepoLayoutCompat(t *testing.T) {
skipE2eTestIfNecessary(t)
if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {
t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)
}
for _, mode := range allLayoutModes() {
mode := mode
t.Run(string(mode), func(t *testing.T) {
t.Parallel()
tc := makeE2eTestingContext(t)
tc.installRcloneGitannexSymlink(t)
tc.installRcloneConfig(t)
tc.createGitRepo(t)
remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")
require.NoError(t, os.Mkdir(remoteStorage, 0777))
tc.runInRepo(t,
"git", "annex", "initremote", "Control",
"type=external", "externaltype=rclone", "encryption=none",
"target=MyRcloneRemote",
"rclone_layout="+string(mode),
"prefix="+remoteStorage)
tc.runInRepo(t,
"git", "annex", "initremote", "Experiment",
"type=external", "externaltype=rclone-builtin", "encryption=none",
"rcloneremotename=MyRcloneRemote",
"rclonelayout="+string(mode),
"rcloneprefix="+remoteStorage)
fooFileContents := []byte{1, 2, 3, 4}
fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")
require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700))
tc.runInRepo(t, "git", "annex", "add", "foo")
tc.runInRepo(t, "git", "commit", "-m", "Add foo file")
// Git-annex objects are not writable, which prevents `testing` from
// cleaning up the temp directory. We can work around this by
// explicitly dropping any files we add to the annex.
t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
// Copy the file to Control and verify it's present on Experiment.
tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
// Drop the file locally and verify we can copy it back from Experiment.
tc.runInRepo(t, "git", "annex", "drop", "--force", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
tc.runInRepo(t, "git", "annex", "copy", "--from=Experiment", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
// Drop the file from Experiment, copy it back to Experiment, and
// verify it's still present on Control.
tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo")
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo")
require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
// Drop the file from Control.
tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo")
require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
})
}
}
|