File: t-batch-transfer.sh

package info (click to toggle)
git-lfs 3.6.1-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 4,808 kB
  • sloc: sh: 21,256; makefile: 507; ruby: 417
file content (356 lines) | stat: -rwxr-xr-x 10,409 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
#!/usr/bin/env bash

. "$(dirname "$0")/testlib.sh"

begin_test "batch transfer"
(
  set -e

  # This initializes a new bare git repository in test/remote.
  # These remote repositories are global to every test, so keep the names
  # unique.
  reponame1="$(basename "$0" ".sh")"
  reponame2="CAPITALLETTERS"
  reponame=$reponame1$reponame2
  setup_remote_repo "$reponame"

  # Clone the repository from the test Git server.  This is empty, and will be
  # used to test a "git pull" below. The repo is cloned to $TRASHDIR/clone
  clone_repo "$reponame" clone

  # Clone the repository again to $TRASHDIR/repo. This will be used to commit
  # and push objects.
  clone_repo "$reponame" repo

  # This executes Git LFS from the local repo that was just cloned.
  git lfs track "*.dat" 2>&1 | tee track.log
  grep "Tracking \"\*.dat\"" track.log

  contents="a"
  contents_oid=$(calc_oid "$contents")

  printf "%s" "$contents" > a.dat
  git add a.dat
  git add .gitattributes
  git commit -m "add a.dat" 2>&1 | tee commit.log
  grep "main (root-commit)" commit.log
  grep "2 files changed" commit.log
  grep "create mode 100644 a.dat" commit.log
  grep "create mode 100644 .gitattributes" commit.log

  [ "a" = "$(cat a.dat)" ]

  # This is a small shell function that runs several git commands together.
  assert_pointer "main" "a.dat" "$contents_oid" 1

  refute_server_object "$reponame" "$contents_oid"

  # This pushes to the remote repository set up at the top of the test.
  git push origin main 2>&1 | tee push.log
  grep "Uploading LFS objects: 100% (1/1), 1 B" push.log
  grep "main -> main" push.log

  assert_server_object "$reponame" "$contents_oid"

  # change to the clone's working directory
  cd ../clone

  git pull origin main

  [ "a" = "$(cat a.dat)" ]

  assert_pointer "main" "a.dat" "$contents_oid" 1
)
end_test

begin_test "batch transfers occur in reverse order by size"
(
  set -e

  reponame="batch-order-test"
  setup_remote_repo "$reponame"
  clone_repo "$reponame" "$reponame"

  git lfs track "*.dat"
  git add .gitattributes
  git commit -m "initial commit"

  small_contents="small"
  small_oid="$(calc_oid "$small_contents")"
  printf "%s" "$small_contents" > small.dat

  bigger_contents="bigger"
  bigger_oid="$(calc_oid "$bigger_contents")"
  printf "%s" "$bigger_contents" > bigger.dat

  git add *.dat
  git commit -m "add small and large objects"

  GIT_CURL_VERBOSE=1 git push origin main 2>&1 | tee push.log

  batch="$(grep "{\"operation\":\"upload\"" push.log | head -1)"

  pos_small="$(substring_position "$batch" "$small_oid")"
  pos_large="$(substring_position "$batch" "$bigger_oid")"

  # Assert that the larger object shows up earlier in the batch than the
  # smaller object
  [ "$pos_large" -lt "$pos_small" ]
)
end_test

begin_test "batch transfers succeed with an empty hash algorithm"
(
  set -e

  reponame="batch-test-empty-algo"
  contents="batch-hash-algo-empty"
  setup_remote_repo "$reponame"
  clone_repo "$reponame" "$reponame"

  git lfs track "*.dat"
  printf "hi" > good.dat
  printf "%s" "$contents" > special.dat
  git add .gitattributes good.dat special.dat
  git commit -m "hi"

  git push origin main
  assert_server_object "$reponame" "$(calc_oid "$contents")"
)
end_test

begin_test "batch transfers fail with an unknown hash algorithm"
(
  set -e

  reponame="batch-test-invalid-algo"
  contents="batch-hash-algo-invalid"
  setup_remote_repo "$reponame"
  clone_repo "$reponame" "$reponame"

  git lfs track "*.dat"
  printf "hi" > good.dat
  printf "%s" "$contents" > special.dat
  git add .gitattributes good.dat special.dat
  git commit -m "hi"

  git push origin main 2>&1 | tee push.log
  grep 'unsupported hash algorithm' push.log
  refute_server_object "$reponame" "$(calc_oid "$contents")"
)
end_test

begin_test "batch transfers with ssh endpoint (git-lfs-authenticate)"
(
  set -e

  reponame="batch-ssh"
  setup_remote_repo "$reponame"
  clone_repo "$reponame" "$reponame"

  sshurl="${GITSERVER/http:\/\//ssh://git@}/$reponame"
  git config lfs.url "$sshurl"

  contents="test"
  git lfs track "*.dat"
  printf "%s" "$contents" > test.dat
  git add .gitattributes test.dat
  git commit -m "initial commit"

  GIT_TRACE=1 git push origin main >push.log 2>&1
  [ "1" -eq "$(grep -c "exec: lfs-ssh-echo.*git-lfs-authenticate /$reponame upload" push.log)" ]
  assert_server_object "$reponame" "$(calc_oid "$contents")"
)
end_test

assert_ssh_transfer_session_counts() {
  local log="$1"
  local msg="$2"
  local min="$3"
  local max="$4"

  local count="$(grep -c "$msg" "$log")"

  [ "$max" -ge "$count" ]
  [ "$min" -le "$count" ]
}

assert_ssh_transfer_sessions() {
  local log="$1"
  local direction="$2"
  local num_objs="$3"
  local objs_per_batch="$4"

  local min_expected_start=1
  local max_expected_start=$(( num_objs > objs_per_batch ? objs_per_batch : num_objs ))
  local min_expected_end=1
  local max_expected_end="$max_expected_start"

  local expected_ctrl=1

  # On upload we currently spawn one extra control socket SSH connection
  # to run locking commands and never shut it down cleanly, so our expected
  # start counts are higher than our expected termination counts.
  if [ "upload" = "$direction" ]; then
    (( ++expected_ctrl ))
    (( ++min_expected_start ))
    (( ++max_expected_start ))
  fi

  # Versions of Git prior to 2.11.0 invoke Git LFS via the "smudge" filter
  # rather than the "process" filter, so a separate Git LFS process runs for
  # each downloaded object and spawns its own control socket SSH connection.
  if [ "download" = "$direction" ]; then
    gitversion="$(git version | cut -d" " -f3)"
    set +e
    compare_version "$gitversion" '2.11.0'
    result=$?
    set -e
    if [ "$result" -eq "$VERSION_LOWER" ]; then
      min_expected_start="$num_objs"
      max_expected_start="$num_objs"
      min_expected_end="$num_objs"
      max_expected_end="$num_objs"
      expected_ctrl="$num_objs"
    fi
  fi

  local max_expected_nonctrl=$(( max_expected_start - expected_ctrl ))

  local lines="$(grep "exec: lfs-ssh-echo.*git-lfs-transfer .*${reponame}.git $direction" "$log")"
  local ctrl_count="$(printf '%s' "$lines" | grep -c -- '-oControlMaster=yes')"
  local nonctrl_count="$(printf '%s' "$lines" | grep -c -- '-oControlMaster=no')"

  [ "$expected_ctrl" -eq "$ctrl_count" ]
  [ "$max_expected_nonctrl" -ge "$nonctrl_count" ]

  assert_ssh_transfer_session_counts "$log" 'spawning pure SSH connection' \
    "$min_expected_start" "$max_expected_start"
  assert_ssh_transfer_session_counts "$log" 'pure SSH connection successful' \
    "$min_expected_start" "$max_expected_start"
  assert_ssh_transfer_session_counts "$log" 'terminating pure SSH connection' \
    "$min_expected_end" "$max_expected_end"
}

begin_test "batch transfers with ssh endpoint (git-lfs-transfer)"
(
  set -e

  setup_pure_ssh

  reponame="batch-ssh-transfer"
  setup_remote_repo "$reponame"
  clone_repo "$reponame" "$reponame"

  sshurl=$(ssh_remote "$reponame")
  git config lfs.url "$sshurl"

  contents="test"
  git lfs track "*.dat"
  printf "%s" "$contents" > test.dat
  git add .gitattributes test.dat
  git commit -m "initial commit"

  # On Windows we do not multiplex SSH connections by default, so we
  # enforce their use in order to match other platforms' connection counts.
  git config --global lfs.ssh.autoMultiplex true

  GIT_TRACE=1 git push origin main >push.log 2>&1
  assert_ssh_transfer_sessions 'push.log' 'upload' 1 8
  assert_remote_object "$reponame" "$(calc_oid "$contents")" "${#contents}"

  cd ..
  GIT_TRACE=1 git clone "$sshurl" "$reponame-2" 2>&1 | tee clone.log
  assert_ssh_transfer_sessions 'clone.log' 'download' 1 8

  cd "$reponame-2"
  git lfs fsck
)
end_test

begin_test "batch transfers with ssh endpoint and multiple objects (git-lfs-transfer)"
(
  set -e

  setup_pure_ssh

  reponame="batch-ssh-transfer-multiple"
  setup_remote_repo "$reponame"
  clone_repo "$reponame" "$reponame"

  contents1="test1"
  contents2="test2"
  contents3="test3"
  git lfs track "*.dat"
  printf "%s" "$contents1" >test1.dat
  printf "%s" "$contents2" >test2.dat
  printf "%s" "$contents3" >test3.dat
  git add .gitattributes test*.dat
  git commit -m "initial commit"

  sshurl=$(ssh_remote "$reponame")
  git config lfs.url "$sshurl"

  # On Windows we do not multiplex SSH connections by default, so we
  # enforce their use in order to match other platforms' connection counts.
  git config --global lfs.ssh.autoMultiplex true

  GIT_TRACE=1 git push origin main >push.log 2>&1
  assert_ssh_transfer_sessions 'push.log' 'upload' 3 8
  assert_remote_object "$reponame" "$(calc_oid "$contents1")" "${#contents1}"
  assert_remote_object "$reponame" "$(calc_oid "$contents2")" "${#contents2}"
  assert_remote_object "$reponame" "$(calc_oid "$contents3")" "${#contents3}"

  cd ..
  GIT_TRACE=1 git clone "$sshurl" "$reponame-2" 2>&1 | tee clone.log
  assert_ssh_transfer_sessions 'clone.log' 'download' 3 8

  cd "$reponame-2"
  git lfs fsck
)
end_test

begin_test "batch transfers with ssh endpoint and multiple objects and batches (git-lfs-transfer)"
(
  set -e

  setup_pure_ssh

  reponame="batch-ssh-transfer-multiple-batch"
  setup_remote_repo "$reponame"
  clone_repo "$reponame" "$reponame"

  contents1="test1"
  contents2="test2"
  contents3="test3"
  git lfs track "*.dat"
  printf "%s" "$contents1" >test1.dat
  printf "%s" "$contents2" >test2.dat
  printf "%s" "$contents3" >test3.dat
  git add .gitattributes test*.dat
  git commit -m "initial commit"

  sshurl=$(ssh_remote "$reponame")
  git config lfs.url "$sshurl"

  # On Windows we do not multiplex SSH connections by default, so we
  # enforce their use in order to match other platforms' connection counts.
  git config --global lfs.ssh.autoMultiplex true

  # Allow no more than two objects to be transferred in each batch.
  git config --global lfs.concurrentTransfers 2

  GIT_TRACE=1 git push origin main >push.log 2>&1
  assert_ssh_transfer_sessions 'push.log' 'upload' 3 2
  assert_remote_object "$reponame" "$(calc_oid "$contents1")" "${#contents1}"
  assert_remote_object "$reponame" "$(calc_oid "$contents2")" "${#contents2}"
  assert_remote_object "$reponame" "$(calc_oid "$contents3")" "${#contents3}"

  cd ..
  GIT_TRACE=1 git clone "$sshurl" "$reponame-2" 2>&1 | tee clone.log
  assert_ssh_transfer_sessions 'clone.log' 'download' 3 2

  cd "$reponame-2"
  git lfs fsck
)
end_test