1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74
|
import json
import os
import pstats
import shutil
import sys
import tempfile
from io import StringIO
from pathlib import Path
from subprocess import PIPE, Popen
from scrapy.utils.test import get_testenv
class TestCmdline:
def setup_method(self):
self.env = get_testenv()
tests_path = Path(__file__).parent.parent
self.env["PYTHONPATH"] += os.pathsep + str(tests_path.parent)
self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
def _execute(self, *new_args, **kwargs):
encoding = sys.stdout.encoding or "utf-8"
args = (sys.executable, "-m", "scrapy.cmdline", *new_args)
proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
comm = proc.communicate()[0].strip()
return comm.decode(encoding)
def test_default_settings(self):
assert self._execute("settings", "--get", "TEST1") == "default"
def test_override_settings_using_set_arg(self):
assert (
self._execute("settings", "--get", "TEST1", "-s", "TEST1=override")
== "override"
)
def test_profiling(self):
path = Path(tempfile.mkdtemp())
filename = path / "res.prof"
try:
self._execute("version", "--profile", str(filename))
assert filename.exists()
out = StringIO()
stats = pstats.Stats(str(filename), stream=out)
stats.print_stats()
out.seek(0)
stats = out.read()
assert str(Path("scrapy", "commands", "version.py")) in stats
assert "tottime" in stats
finally:
shutil.rmtree(path)
def test_override_dict_settings(self):
EXT_PATH = "tests.test_cmdline.extensions.DummyExtension"
EXTENSIONS = {EXT_PATH: 200}
settingsstr = self._execute(
"settings",
"--get",
"EXTENSIONS",
"-s",
"EXTENSIONS=" + json.dumps(EXTENSIONS),
)
# XXX: There's gotta be a smarter way to do this...
assert "..." not in settingsstr
for char in ("'", "<", ">"):
settingsstr = settingsstr.replace(char, '"')
settingsdict = json.loads(settingsstr)
assert set(settingsdict.keys()) == set(EXTENSIONS.keys())
assert settingsdict[EXT_PATH] == 200
def test_pathlib_path_as_feeds_key(self):
assert self._execute("settings", "--get", "FEEDS") == json.dumps(
{"items.csv": {"format": "csv", "fields": ["price", "name"]}}
)
|