File: __init__.py

package info (click to toggle)
python-scrapy 2.13.3-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 5,664 kB
  • sloc: python: 52,028; xml: 199; makefile: 25; sh: 7
file content (74 lines) | stat: -rw-r--r-- 2,551 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import json
import os
import pstats
import shutil
import sys
import tempfile
from io import StringIO
from pathlib import Path
from subprocess import PIPE, Popen

from scrapy.utils.test import get_testenv


class TestCmdline:
    def setup_method(self):
        self.env = get_testenv()
        tests_path = Path(__file__).parent.parent
        self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
        self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"

    def _execute(self, *new_args, **kwargs):
        encoding = sys.stdout.encoding or "utf-8"
        args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
        proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
        comm = proc.communicate()[0].strip()
        return comm.decode(encoding)

    def test_default_settings(self):
        assert self._execute("settings", "--get", "TEST1") == "default"

    def test_override_settings_using_set_arg(self):
        assert (
            self._execute("settings", "--get", "TEST1", "-s", "TEST1=override")
            == "override"
        )

    def test_profiling(self):
        path = Path(tempfile.mkdtemp())
        filename = path / "res.prof"
        try:
            self._execute("version", "--profile", str(filename))
            assert filename.exists()
            out = StringIO()
            stats = pstats.Stats(str(filename), stream=out)
            stats.print_stats()
            out.seek(0)
            stats = out.read()
            assert str(Path("scrapy", "commands", "version.py")) in stats
            assert "tottime" in stats
        finally:
            shutil.rmtree(path)

    def test_override_dict_settings(self):
        EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"
        EXTENSIONS = {EXT_PATH: 200}
        settingsstr = self._execute(
            "settings",
            "--get",
            "EXTENSIONS",
            "-s",
            "EXTENSIONS=" + json.dumps(EXTENSIONS),
        )
        # XXX: There's gotta be a smarter way to do this...
        assert "..." not in settingsstr
        for char in ("'", "<", ">"):
            settingsstr = settingsstr.replace(char, '"')
        settingsdict = json.loads(settingsstr)
        assert set(settingsdict.keys()) == set(EXTENSIONS.keys())
        assert settingsdict[EXT_PATH] == 200

    def test_pathlib_path_as_feeds_key(self):
        assert self._execute("settings", "--get", "FEEDS") == json.dumps(
            {"items.csv": {"format": "csv", "fields": ["price", "name"]}}
        )