1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
|
import os
import shutil
import stat
import pytest
from tests.testutils import cli, create_artifact_share, generate_junction
# Project directory
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"project",
)
# Assert that a given artifact is in the share
#
def assert_shared(cli, share, project, element_name):
# NOTE: 'test' here is the name of the project
# specified in the project.conf we are testing with.
#
cache_key = cli.get_element_key(project, element_name)
if not share.has_artifact('test', element_name, cache_key):
raise AssertionError("Artifact share at {} does not contain the expected element {}"
.format(share.repo, element_name))
# Assert that a given artifact is NOT in the share
#
def assert_not_shared(cli, share, project, element_name):
# NOTE: 'test' here is the name of the project
# specified in the project.conf we are testing with.
#
cache_key = cli.get_element_key(project, element_name)
if share.has_artifact('test', element_name, cache_key):
raise AssertionError("Artifact share at {} unexpectedly contains the element {}"
.format(share.repo, element_name))
# Tests that:
#
# * `bst build` pushes all build elements to configured 'push' cache
# * `bst pull --deps all` downloads everything from cache after local deletion
#
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull_all(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
# First build the target element and push to the remote.
cli.configure({
'artifacts': {'url': share.repo, 'push': True}
})
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Assert that everything is now cached in the remote.
all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
artifacts = os.path.join(cli.directory, 'artifacts')
shutil.rmtree(artifacts)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
assert cli.get_element_state(project, element_name) != 'cached'
# Now try bst pull
result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst'])
result.assert_success()
# And assert that it's again in the local cache, without having built
for element_name in all_elements:
assert cli.get_element_state(project, element_name) == 'cached'
# Tests that:
#
# * `bst build` pushes all build elements ONLY to configured 'push' cache
# * `bst pull` finds artifacts that are available only in the secondary cache
#
@pytest.mark.datafiles(DATA_DIR)
def test_pull_secondary_cache(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as share1,\
create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2:
# Build the target and push it to share2 only.
cli.configure({
'artifacts': [
{'url': share1.repo, 'push': False},
{'url': share2.repo, 'push': True},
]
})
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
assert_not_shared(cli, share1, project, 'target.bst')
assert_shared(cli, share2, project, 'target.bst')
# Delete the user's local artifact cache.
artifacts = os.path.join(cli.directory, 'artifacts')
shutil.rmtree(artifacts)
# Assert that the element is not cached anymore.
assert cli.get_element_state(project, 'target.bst') != 'cached'
# Now try bst pull
result = cli.run(project=project, args=['pull', 'target.bst'])
result.assert_success()
# And assert that it's again in the local cache, without having built,
# i.e. we found it in share2.
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Tests that:
#
# * `bst push --remote` pushes to the given remote, not one from the config
# * `bst pull --remote` pulls from the given remote
#
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull_specific_remote(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
with create_artifact_share(os.path.join(str(tmpdir), 'goodartifactshare')) as good_share,\
create_artifact_share(os.path.join(str(tmpdir), 'badartifactshare')) as bad_share:
# Build the target so we have it cached locally only.
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
state = cli.get_element_state(project, 'target.bst')
assert state == 'cached'
# Configure the default push location to be bad_share; we will assert that
# nothing actually gets pushed there.
cli.configure({
'artifacts': {'url': bad_share.repo, 'push': True},
})
# Now try `bst push` to the good_share.
result = cli.run(project=project, args=[
'push', 'target.bst', '--remote', good_share.repo
])
result.assert_success()
# Assert that all the artifacts are in the share we pushed
# to, and not the other.
assert_shared(cli, good_share, project, 'target.bst')
assert_not_shared(cli, bad_share, project, 'target.bst')
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the good_share.
#
artifacts = os.path.join(cli.directory, 'artifacts')
shutil.rmtree(artifacts)
result = cli.run(project=project, args=['pull', 'target.bst', '--remote',
good_share.repo])
result.assert_success()
# And assert that it's again in the local cache, without having built
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Tests that:
#
# * In non-strict mode, dependency changes don't block artifact reuse
#
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull_non_strict(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
workspace = os.path.join(str(tmpdir), 'workspace')
# First build the target element and push to the remote.
cli.configure({
'artifacts': {'url': share.repo, 'push': True},
'projects': {
'test': {'strict': False}
}
})
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Assert that everything is now cached in the remote.
all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
artifacts = os.path.join(cli.directory, 'artifacts')
shutil.rmtree(artifacts)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
assert cli.get_element_state(project, element_name) != 'cached'
# Add a file to force change in strict cache key of import-bin.bst
with open(os.path.join(str(project), 'files', 'bin-files', 'usr', 'bin', 'world'), 'w') as f:
f.write('world')
# Assert that the workspaced element requires a rebuild
assert cli.get_element_state(project, 'import-bin.bst') == 'buildable'
# Assert that the target is still waiting due to --no-strict
assert cli.get_element_state(project, 'target.bst') == 'waiting'
# Now try bst pull
result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst'])
result.assert_success()
# And assert that the target is again in the local cache, without having built
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Regression test for https://gitlab.com/BuildStream/buildstream/issues/202
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull_track_non_strict(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
# First build the target element and push to the remote.
cli.configure({
'artifacts': {'url': share.repo, 'push': True},
'projects': {
'test': {'strict': False}
}
})
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Assert that everything is now cached in the remote.
all_elements = {'target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'}
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
artifacts = os.path.join(cli.directory, 'artifacts')
shutil.rmtree(artifacts)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
assert cli.get_element_state(project, element_name) != 'cached'
# Now try bst build with tracking and pulling.
# Tracking will be skipped for target.bst as it doesn't have any sources.
# With the non-strict build plan target.bst immediately enters the pull queue.
# However, pulling has to be deferred until the dependencies have been
# tracked as the strict cache key needs to be calculated before querying
# the caches.
result = cli.run(project=project, args=['build', '--track-all', '--all', 'target.bst'])
result.assert_success()
assert set(result.get_pulled_elements()) == all_elements
@pytest.mark.datafiles(DATA_DIR)
def test_push_pull_cross_junction(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
subproject_path = os.path.join(project, 'files', 'sub-project')
junction_path = os.path.join(project, 'elements', 'junction.bst')
generate_junction(tmpdir, subproject_path, junction_path, store_ref=True)
# First build the target element and push to the remote.
cli.configure({
'artifacts': {'url': share.repo, 'push': True}
})
result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst'])
result.assert_success()
assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached'
cache_dir = os.path.join(project, 'cache', 'artifacts')
shutil.rmtree(cache_dir)
assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable'
# Now try bst pull
result = cli.run(project=project, args=['pull', 'junction.bst:import-etc.bst'])
result.assert_success()
# And assert that it's again in the local cache, without having built
assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached'
@pytest.mark.datafiles(DATA_DIR)
def test_pull_missing_blob(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
# First build the target element and push to the remote.
cli.configure({
'artifacts': {'url': share.repo, 'push': True}
})
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
assert cli.get_element_state(project, 'target.bst') == 'cached'
# Assert that everything is now cached in the remote.
all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
artifacts = os.path.join(cli.directory, 'artifacts')
shutil.rmtree(artifacts)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
assert cli.get_element_state(project, element_name) != 'cached'
# Now delete blobs in the remote without deleting the artifact ref.
# This simulates scenarios with concurrent artifact expiry.
remote_objdir = os.path.join(share.repodir, 'cas', 'objects')
shutil.rmtree(remote_objdir)
# Now try bst build
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
# Assert that no artifacts were pulled
assert len(result.get_pulled_elements()) == 0
@pytest.mark.datafiles(DATA_DIR)
def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
caplog.set_level(1)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
cli.configure({
'artifacts': {'url': share.repo}
})
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
assert not result.get_pulled_elements(), \
"No elements should have been pulled since the cache was empty"
assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr
assert "SKIPPED Pull" in result.stderr
@pytest.mark.datafiles(DATA_DIR)
def test_pull_access_rights(caplog, cli, tmpdir, datafiles):
project = str(datafiles)
checkout = os.path.join(str(tmpdir), 'checkout')
# Work-around datafiles not preserving mode
os.chmod(os.path.join(project, 'files/bin-files/usr/bin/hello'), 0o0755)
# We need a big file that does not go into a batch to test a different
# code path
os.makedirs(os.path.join(project, 'files/dev-files/usr/share'), exist_ok=True)
with open(os.path.join(project, 'files/dev-files/usr/share/big-file'), 'w') as f:
buf = ' ' * 4096
for _ in range(1024):
f.write(buf)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
cli.configure({
'artifacts': {'url': share.repo, 'push': True}
})
result = cli.run(project=project, args=['build', 'compose-all.bst'])
result.assert_success()
result = cli.run(project=project,
args=['checkout', '--hardlinks', '--no-integrate',
'compose-all.bst', checkout])
result.assert_success()
st = os.lstat(os.path.join(checkout, 'usr/include/pony.h'))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
st = os.lstat(os.path.join(checkout, 'usr/bin/hello'))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0755
st = os.lstat(os.path.join(checkout, 'usr/share/big-file'))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
shutil.rmtree(checkout)
artifacts = os.path.join(cli.directory, 'artifacts')
shutil.rmtree(artifacts)
result = cli.run(project=project, args=['pull', 'compose-all.bst'])
result.assert_success()
result = cli.run(project=project,
args=['checkout', '--hardlinks', '--no-integrate',
'compose-all.bst', checkout])
result.assert_success()
st = os.lstat(os.path.join(checkout, 'usr/include/pony.h'))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
st = os.lstat(os.path.join(checkout, 'usr/bin/hello'))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0755
st = os.lstat(os.path.join(checkout, 'usr/share/big-file'))
assert stat.S_ISREG(st.st_mode)
assert stat.S_IMODE(st.st_mode) == 0o0644
|
