File: test_File__download.py

package info (click to toggle)
python-briefcase 0.3.22-3
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 7,300 kB
  • sloc: python: 59,405; makefile: 57
file content (473 lines) | stat: -rw-r--r-- 16,697 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
import os
import platform
import shutil
import stat
from collections.abc import Iterable, Iterator
from pathlib import Path
from unittest import mock

import httpx
import pytest

from briefcase.exceptions import (
    BadNetworkResourceError,
    MissingNetworkResourceError,
    NetworkFailure,
)
from briefcase.integrations.base import ToolCache

TEMPORARY_DOWNLOAD_FILE_SUFFIX = ".download"


@pytest.fixture
def mock_tools(mock_tools) -> ToolCache:
    mock_tools.httpx = mock.MagicMock(spec_set=httpx)
    # Restore move so the temporary file can be moved after downloaded
    mock_tools.shutil.move = mock.MagicMock(wraps=shutil.move)
    return mock_tools


class _IteratorByteSteam(httpx.SyncByteStream):
    """Shim that satisfies ``httpx.Response`` ``stream`` parameter type.

    Cannot be replaced by any ``Iterable[bytes]`` because the base class requires
    an explicit finalization method ``close``."""

    def __init__(self, iterable: Iterable[bytes]) -> None:
        self.iterable = iterable

    def __iter__(self) -> Iterator[bytes]:
        return iter(self.iterable)


def _make_httpx_response(
    *,
    url: str,
    status_code: int,
    stream: list[bytes],
    method: str = "GET",
    headers: dict = {},
) -> httpx.Response:
    """Create a real ``httpx.Response`` with key methods wrapped by ``mock.Mock`` for spying.

    Wrapped methods:
        response.read
        response.iter_bytes
        response.headers.get
    """
    response = httpx.Response(
        request=httpx.Request(
            method=method,
            url=httpx.URL(url),
        ),
        status_code=status_code,
        headers=httpx.Headers(headers),
        # Always use ``stream`` rather than content because it's more flexible
        # even if the request is made non-streaming or the response is read with
        # ``response.read()``, httpx will still consume the ``stream`` response
        # content internally. This allows testing both the non-streaming and
        # streaming download paths without needing to complicate the response params
        stream=_IteratorByteSteam(stream),
    )

    response.read = mock.Mock(wraps=response.read)
    response.iter_bytes = mock.Mock(wraps=response.iter_bytes)
    response.headers.get = mock.Mock(wraps=response.headers.get)

    return response


@pytest.fixture
def file_perms() -> int:
    """The expected permissions for the downloaded file.

    Since umask can vary on different systems, it is updated to a known value and reset
    after the test finishes.

    On Windows, the umask seems to always be zero and chmod doesn't really do anything
    anyway.
    """
    if platform.system() == "Windows":
        yield 0o666
    else:
        orig_umask = os.umask(0o077)
        yield 0o600
        os.umask(orig_umask)


@pytest.mark.parametrize(
    "url,content_disposition",
    [
        # A `None` value for `content_disposition` means we skip the header.
        # Other values are passed through as HTTP header values.
        ("https://example.com/path/to/something.zip", None),
        # Ensure empty header is ignored.
        ("https://example.com/path/to/something.zip", ""),
        # Paradigmatic case for Content-Disposition: attachment.
        (
            "https://example.com/path/to/irrelevant.zip",
            "attachment; filename=something.zip",
        ),
        # Ensure extra parameters are ignored.
        (
            "https://example.com/path/to/irrelevant.zip",
            "attachment; filename=something.zip; ignored=okay",
        ),
        # Ensure garbage headers are ignored.
        ("https://example.com/path/to/something.zip", "garbage"),
        # Ensure we respect unusual quoting & case permitted by RFC 6266.
        (
            "https://example.com/path/to/irrelevant.zip",
            'ATTACHment; filename=    "something.zip"',
        ),
        # Ensure we use filename=, even if filename*= is provided. This makes us a
        # "legacy user agent" in the terms of RFC 6266, for our own simplicity.
        (
            "https://example.com/path/to/irrelevant.zip",
            'attachment; filename="something.zip"; filename*=utf-8'
            "''%e2%82%ac%20rates",
        ),
    ],
)
def test_new_download_oneshot(mock_tools, file_perms, url, content_disposition):
    """If no content-length is provided, ``File`` downloadds the file all at once."""
    response = _make_httpx_response(
        method="GET",
        url=url,
        status_code=200,
        headers=(
            {
                "content-disposition": content_disposition,
            }
            if content_disposition is not None
            else {}
        ),
        stream=[b"all content"],
    )
    mock_tools.httpx.stream.return_value.__enter__.return_value = response

    # Download the file
    filename = mock_tools.file.download(
        url="https://example.com/support?useful=Yes",
        download_path=mock_tools.base_path / "downloads",
    )

    # httpx.stream has been invoked, but content isn't iterated
    mock_tools.httpx.stream.assert_called_with(
        "GET",
        "https://example.com/support?useful=Yes",
        follow_redirects=True,
    )
    response.headers.get.assert_called_with("content-length")
    response.read.assert_called_once()

    # The filename is derived from the URL or header
    assert filename == mock_tools.base_path / "downloads/something.zip"

    # Temporary file was created in download dir and was renamed
    temp_filename = Path(mock_tools.shutil.move.call_args_list[0].args[0])
    assert temp_filename.parent == mock_tools.base_path / "downloads"
    assert temp_filename.name.startswith("something.zip.")
    assert temp_filename.name.endswith(TEMPORARY_DOWNLOAD_FILE_SUFFIX)
    mock_tools.shutil.move.assert_called_with(str(temp_filename), filename)

    # File permissions were set
    mock_tools.os.chmod.assert_called_with(filename, file_perms)
    assert stat.S_IMODE(os.stat(filename).st_mode) == file_perms

    # Attempt to delete Temporary file was made
    mock_tools.os.remove.assert_called_with(str(temp_filename))

    # File content is as expected
    with (mock_tools.base_path / "downloads/something.zip").open(encoding="utf-8") as f:
        assert f.read() == "all content"


def test_new_download_chunked(mock_tools, file_perms):
    """If a content-length is provided, ``File`` streams the response rather than downloading it all at once."""
    response = _make_httpx_response(
        method="GET",
        url="https://example.com/path/to/something.zip",
        status_code=200,
        headers={"content-length": "24"},
        stream=[
            b"chunk-1;",
            b"chunk-2;",
            b"chunk-3;",
        ],
    )
    mock_tools.httpx.stream.return_value.__enter__.return_value = response

    # Download the file
    filename = mock_tools.file.download(
        url="https://example.com/support?useful=Yes",
        download_path=mock_tools.base_path,
    )

    # httpx.stream has been invoked, and content is chunked.
    mock_tools.httpx.stream.assert_called_with(
        "GET",
        "https://example.com/support?useful=Yes",
        follow_redirects=True,
    )
    response.headers.get.assert_called_with("content-length")
    response.iter_bytes.assert_called_once_with(chunk_size=1048576)

    # The filename is derived from the URL
    assert filename == mock_tools.base_path / "something.zip"

    # Temporary file was created in download dir and was renamed
    temp_filename = Path(mock_tools.shutil.move.call_args_list[0].args[0])
    assert temp_filename.parent == mock_tools.base_path
    assert temp_filename.name.startswith("something.zip.")
    assert temp_filename.name.endswith(TEMPORARY_DOWNLOAD_FILE_SUFFIX)
    mock_tools.shutil.move.assert_called_with(str(temp_filename), filename)

    # File permissions were set
    mock_tools.os.chmod.assert_called_with(filename, file_perms)
    assert stat.S_IMODE(os.stat(filename).st_mode) == file_perms

    # Attempt to delete Temporary file was made
    mock_tools.os.remove.assert_called_with(str(temp_filename))

    # The downloaded file exists, and content is as expected
    assert filename.exists()
    with (mock_tools.base_path / "something.zip").open(encoding="utf-8") as f:
        assert f.read() == "chunk-1;chunk-2;chunk-3;"


def test_already_downloaded(mock_tools):
    """If the file already exists on disk, it isn't re-downloaded.

    The request is still made to derive the filename, but the content
    is never streamed."""

    # Create an existing file
    existing_file = mock_tools.base_path / "something.zip"
    with existing_file.open("w", encoding="utf-8") as f:
        f.write("existing content")

    url = "https://example.com/path/to/something.zip"

    response = _make_httpx_response(
        status_code=200,
        url=url,
        # Use content and a content-encoding that would cause a DecodeError
        # if ``File`` tried to read the response content.
        # Because the file already exists, ``File`` shouldn't try to read
        # the response, and the ``DecodingError`` won't occur.
        headers={"content-length": "100", "content-encoding": "gzip"},
        stream=[b"definitely not gzip content"],
    )
    mock_tools.httpx.stream.return_value.__enter__.return_value = response

    # Download the file
    filename = mock_tools.file.download(
        url=url,
        download_path=mock_tools.base_path,
    )

    # The GET request will have been made
    mock_tools.httpx.stream.assert_called_with(
        "GET",
        url,
        follow_redirects=True,
    )

    # The request's Content-Disposition header is consumed to
    # examine the filename; the request is abandoned before
    # any other headers are read.
    response.headers.get.assert_called_once_with("Content-Disposition")

    # but the file existed, so the method returns
    assert filename == existing_file
    assert filename.exists()

    # Temporary file was not created, moved, or deleted
    mock_tools.shutil.move.assert_not_called()
    mock_tools.os.chmod.assert_not_called()
    mock_tools.os.remove.assert_not_called()


def test_missing_resource(mock_tools):
    """MissingNetworkResourceError raises for 404 status code"""
    url = "https://example.com/something.zip?useful=Yes"
    response = _make_httpx_response(
        url=url,
        status_code=404,
        stream=[],
    )

    mock_tools.httpx.stream.return_value.__enter__.return_value = response

    # Download the file
    with pytest.raises(MissingNetworkResourceError):
        mock_tools.file.download(
            url=url,
            download_path=mock_tools.base_path,
        )

    # httpx.stream has been invoked, but nothing else.
    mock_tools.httpx.stream.assert_called_with(
        "GET",
        url,
        follow_redirects=True,
    )
    response.headers.get.assert_not_called()

    # The file doesn't exist as a result of the download failure
    assert not (mock_tools.base_path / "something.zip").exists()

    # Temporary file was not created, moved, or deleted
    mock_tools.shutil.move.assert_not_called()
    mock_tools.os.chmod.assert_not_called()
    mock_tools.os.remove.assert_not_called()


def test_bad_resource(mock_tools):
    """BadNetworkResourceError raises for non-200 status code"""
    url = "https://example.com/something.zip?useful=Yes"
    response = _make_httpx_response(
        status_code=500,
        url=url,
        stream=[],
    )

    mock_tools.httpx.stream.return_value.__enter__.return_value = response

    # Download the file
    with pytest.raises(BadNetworkResourceError):
        mock_tools.file.download(
            url=url,
            download_path=mock_tools.base_path,
        )

    # httpx.stream has been invoked, but nothing else.
    mock_tools.httpx.stream.assert_called_with(
        "GET",
        url,
        follow_redirects=True,
    )
    response.headers.get.assert_not_called()

    # The file doesn't exist as a result of the download failure
    assert not (mock_tools.base_path / "something.zip").exists()

    # Temporary file was not created, moved, or deleted
    mock_tools.shutil.move.assert_not_called()
    mock_tools.os.chmod.assert_not_called()
    mock_tools.os.remove.assert_not_called()


def test_iter_bytes_connection_error(mock_tools):
    """NetworkFailure raised if response.iter_bytes() errors
    and cleans up temporary files."""
    url = "https://example.com/something.zip?useful=Yes"
    response = _make_httpx_response(
        status_code=200,
        url=url,
        # Force a real DecodingError by setting the response encoding to
        # gzip with response content that is _not_ gzip
        headers={"content-length": "100", "content-encoding": "gzip"},
        stream=[b"definitely not gzip content"],
    )
    mock_tools.httpx.stream.return_value.__enter__.return_value = response

    # Download the file
    with pytest.raises(NetworkFailure, match="Unable to download something.zip"):
        mock_tools.file.download(
            url="https://example.com/something.zip?useful=Yes",
            download_path=mock_tools.base_path,
        )

    # httpx.stream has been invoked, but nothing else.
    mock_tools.httpx.stream.assert_called_with(
        "GET",
        "https://example.com/something.zip?useful=Yes",
        follow_redirects=True,
    )
    response.headers.get.assert_called_with("content-length")

    # The file doesn't exist as a result of the download failure
    assert not (mock_tools.base_path / "something.zip").exists()

    # Temporary file was not moved
    mock_tools.shutil.move.assert_not_called()
    mock_tools.os.chmod.assert_not_called()

    # Failure happens during response streaming, so the temporary file is create
    # but then also correctly cleaned up after the failure
    temp_filename = Path(mock_tools.os.remove.call_args_list[0].args[0])
    assert temp_filename.parent == mock_tools.base_path
    assert temp_filename.name.startswith("something.zip.")
    assert temp_filename.name.endswith(TEMPORARY_DOWNLOAD_FILE_SUFFIX)
    mock_tools.os.remove.assert_called_with(str(temp_filename))


def test_connection_error(mock_tools):
    """NetworkFailure raised if the connection fails."""
    # Use ftp scheme to force raising a real ProtocolError without needing to mock
    url = "ftp://example.com/something.zip"

    # Use the real httpx for this test instead of the MagicMock'd one from mock_tools
    # Keep using the fixture though, so that it still gets cleaned up after the test
    mock_tools.httpx = mock.Mock(wraps=httpx)

    # Failure leads to filename never being read, so the error message will use the full URL
    # rather than the filename
    with pytest.raises(NetworkFailure, match=f"Unable to download {url}"):
        mock_tools.file.download(
            url=url,
            download_path=mock_tools.base_path,
        )

    # httpx.stream has been invoked, but nothing else.
    mock_tools.httpx.stream.assert_called_with(
        "GET",
        url,
        follow_redirects=True,
    )

    # The file doesn't exist as a result of the download failure
    assert not (mock_tools.base_path / "something.zip").exists()

    # Temporary file was never created because the failure happens before a response
    # is every received
    assert not (mock_tools.base_path / "something.zip.download").exists()

    # Temporary file was not created, moved, or deleted
    mock_tools.shutil.move.assert_not_called()
    mock_tools.os.chmod.assert_not_called()
    mock_tools.os.remove.assert_not_called()


def test_redirect_connection_error(mock_tools):
    """NetworkFailure raises if the request leads to too many redirects."""
    mock_tools.httpx.stream.side_effect = [
        httpx.TooManyRedirects("Exceeded max redirects")
    ]

    # Download the file
    with pytest.raises(
        NetworkFailure,
        match=r"Unable to download https\:\/\/example.com\/something\.zip\?useful=Yes",
    ):
        mock_tools.file.download(
            url="https://example.com/something.zip?useful=Yes",
            download_path=mock_tools.base_path,
        )

    # httpx.stream has been invoked, but nothing else.
    mock_tools.httpx.stream.assert_called_with(
        "GET",
        "https://example.com/something.zip?useful=Yes",
        follow_redirects=True,
    )

    # The file doesn't exist as a result of the download failure
    assert not (mock_tools.base_path / "something.zip").exists()

    # Temporary file was not created, moved, or deleted
    mock_tools.shutil.move.assert_not_called()
    mock_tools.os.chmod.assert_not_called()
    mock_tools.os.remove.assert_not_called()