#! /usr/bin/python ## vim: fileencoding=utf-8 # Copyright (C) 2006 Adeodato Simó # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation, either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. from collections import namedtuple import email.utils import io import logging import os import os.path from pathlib import Path import pickle import re import shutil import subprocess import sys import tempfile import textwrap import warnings import pytest try: # skip tests that require apt_pkg if it is not available import apt_pkg _have_apt_pkg = True except (ImportError, AttributeError): _have_apt_pkg = False try: import charset_normalizer _have_charset_normalizer = True except ImportError: _have_charset_normalizer = False try: import chardet _have_chardet = True except ImportError: _have_chardet = False from debian import deb822 from debian.debian_support import Version from typing import ( Any, Callable, Dict, Generator, IO, List, Optional, Union, Text, Tuple, Type, TypeVar, ) # Only run tests that rely on the gpgv signature validation executable if # it is installed _gpgv_path = shutil.which('gpgv') or "" deb822.GPGV_EXECUTABLE = _gpgv_path # Deterministic tests are good; automatically skipping tests because optional # dependencies are not available is a way of accidentally missing problems. # Here, we control whether missing dependencies result in skipping tests # or if instead, missing dependencies cause test failures. # # FORBID_MISSING_APT_PKG: # any non-empty value for the environment variable FORBID_MISSING_APT_PKG # will mean that tests fail if apt_pkg (from python-apt) can't be found # # FORBID_MISSING_CHARSET_NORMALIZER: # any non-empty value for the environment variable # FORBID_MISSING_CHARSET_NORMALIZER will mean that tests fail if the # charset_normalizer module (from charset-normalizer) can't be found # # FORBID_MISSING_GPGV: # any non-empty value for the environment variable FORBID_MISSING_GPGV # will mean that tests fail if the gpgv program can't be found FORBID_MISSING_APT_PKG = os.environ.get("FORBID_MISSING_APT_PKG", None) FORBID_MISSING_CHARSET_NORMALIZER = os.environ.get("FORBID_MISSING_CHARSET_NORMALIZER", None) FORBID_MISSING_GPGV = os.environ.get("FORBID_MISSING_GPGV", None) UNPARSED_PACKAGE = '''\ Package: mutt Priority: standard Section: mail Installed-Size: 4471 Maintainer: Adeodato Simó Architecture: i386 Version: 1.5.12-1 Replaces: mutt-utf8 Provides: mail-reader, imap-client Depends: libc6 (>= 2.3.6-6), libdb4.4, libgnutls13 (>= 1.4.0-0), libidn11 (>= 0.5.18), libncursesw5 (>= 5.4-5), libsasl2 (>= 2.1.19.dfsg1), exim4 | mail-transport-agent Recommends: locales, mime-support Suggests: urlview, aspell | ispell, gnupg, mixmaster, openssl, ca-certificates Conflicts: mutt-utf8 Filename: pool/main/m/mutt/mutt_1.5.12-1_i386.deb Size: 1799444 MD5sum: d4a9e124beea99d8124c8e8543d22e9a SHA1: 5e3c295a921c287cf7cb3944f3efdcf18dd6701a SHA256: 02853602efe21d77cd88056a4e2a4350f298bcab3d895f5f9ae02aacad81442b Description: text-based mailreader supporting MIME, GPG, PGP and threading Mutt is a sophisticated text-based Mail User Agent. Some highlights: . * MIME support (including RFC1522 encoding/decoding of 8-bit message headers and UTF-8 support). * PGP/MIME support (RFC 2015). * Advanced IMAP client supporting SSL encryption and SASL authentication. * POP3 support. * Mailbox threading (both strict and non-strict). * Default keybindings are much like ELM. * Keybindings are configurable; Mush and PINE-like ones are provided as examples. * Handles MMDF, MH and Maildir in addition to regular mbox format. * Messages may be (indefinitely) postponed. * Colour support. * Highly configurable through easy but powerful rc file. Tag: interface::text-mode, made-of::lang:c, mail::imap, mail::pop, mail::user-agent, protocol::imap, protocol::ipv6, protocol::pop, protocol::ssl, role::sw:client, uitoolkit::ncurses, use::editing, works-with::mail Task: mail-server ''' PARSED_PACKAGE = deb822.Deb822Dict([ ('Package', 'mutt'), ('Priority', 'standard'), ('Section', 'mail'), ('Installed-Size', '4471'), ('Maintainer', 'Adeodato Simó '), ('Architecture', 'i386'), ('Version', '1.5.12-1'), ('Replaces', 'mutt-utf8'), ('Provides', 'mail-reader, imap-client'), ('Depends', 'libc6 (>= 2.3.6-6), libdb4.4, libgnutls13 (>= 1.4.0-0), libidn11 (>= 0.5.18), libncursesw5 (>= 5.4-5), libsasl2 (>= 2.1.19.dfsg1), exim4 | mail-transport-agent'), ('Recommends', 'locales, mime-support'), ('Suggests', 'urlview, aspell | ispell, gnupg, mixmaster, openssl, ca-certificates'), ('Conflicts', 'mutt-utf8'), ('Filename', 'pool/main/m/mutt/mutt_1.5.12-1_i386.deb'), ('Size', '1799444'), ('MD5sum', 'd4a9e124beea99d8124c8e8543d22e9a'), ('SHA1', '5e3c295a921c287cf7cb3944f3efdcf18dd6701a'), ('SHA256', '02853602efe21d77cd88056a4e2a4350f298bcab3d895f5f9ae02aacad81442b'), ('Description', '''text-based mailreader supporting MIME, GPG, PGP and threading Mutt is a sophisticated text-based Mail User Agent. Some highlights: . * MIME support (including RFC1522 encoding/decoding of 8-bit message headers and UTF-8 support). * PGP/MIME support (RFC 2015). * Advanced IMAP client supporting SSL encryption and SASL authentication. * POP3 support. * Mailbox threading (both strict and non-strict). * Default keybindings are much like ELM. * Keybindings are configurable; Mush and PINE-like ones are provided as examples. * Handles MMDF, MH and Maildir in addition to regular mbox format. * Messages may be (indefinitely) postponed. * Colour support. * Highly configurable through easy but powerful rc file.'''), ('Tag', 'interface::text-mode, made-of::lang:c, mail::imap, mail::pop, mail::user-agent, protocol::imap, protocol::ipv6, protocol::pop, protocol::ssl, role::sw:client, uitoolkit::ncurses, use::editing, works-with::mail'), ('Task', 'mail-server'), ]) GPG_SIGNED = [ '''\ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 %s -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.3 (GNU/Linux) Comment: Signed by Adeodato Simó iEYEARECAAYFAkRqYxkACgkQgyNlRdHEGIKccQCgnnUgfwYjQ7xd3zGGS2y5cXKt CcYAoOLYDF5G1h3oR1iDNyeCI6hRW03S =Um8T -----END PGP SIGNATURE----- ''', '''\ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 %s -----BEGIN PGP SIGNATURE----- iEYEARECAAYFAkRqYxkACgkQgyNlRdHEGIKccQCgnnUgfwYjQ7xd3zGGS2y5cXKt CcYAoOLYDF5G1h3oR1iDNyeCI6hRW03S =Um8T -----END PGP SIGNATURE----- ''', ] CHANGES_FILE = '''\ Format: 1.7 Date: Fri, 28 Dec 2007 17:08:48 +0100 Source: bzr-gtk Binary: bzr-gtk Architecture: source all Version: 0.93.0-2 Distribution: unstable Urgency: low Maintainer: Debian Bazaar Maintainers Changed-By: Chris Lamb Description: bzr-gtk - provides graphical interfaces to Bazaar (bzr) version control Closes: 440354 456438 Changes: bzr-gtk (0.93.0-2) unstable; urgency=low . [ Chris Lamb ] * Add patch for unclosed progress window. (Closes: #440354) Patch by Jean-François Fortin Tam * Fix broken icons in .desktop files (Closes: #456438). Files: 0fd797f4138a9d4fdeb8c30597d46bc9 1003 python optional bzr-gtk_0.93.0-2.dsc d9523676ae75c4ced299689456f252f4 3860 python optional bzr-gtk_0.93.0-2.diff.gz 8960459940314b21019dedd5519b47a5 168544 python optional bzr-gtk_0.93.0-2_all.deb ''' CHECKSUM_CHANGES_FILE = '''\ Format: 1.8 Date: Wed, 30 Apr 2008 23:58:24 -0600 Source: python-debian Binary: python-debian Architecture: source all Version: 0.1.10 Distribution: unstable Urgency: low Maintainer: Debian python-debian Maintainers Changed-By: John Wright Description: python-debian - Python modules to work with Debian-related data formats Closes: 473254 473259 Changes: python-debian (0.1.10) unstable; urgency=low . * debian_bundle/deb822.py, tests/test_deb822.py: - Do not cache _CaseInsensitiveString objects, since it causes case preservation issues under certain circumstances (Closes: #473254) - Add a test case * debian_bundle/deb822.py: - Add support for fixed-length subfields in multivalued fields. I updated the Release and PdiffIndex classes to use this. The default behavior for Release is that of apt-ftparchive, just because it's simpler. Changing the behavior to resemble dak requires simply setting the size_field_behavior attribute to 'dak'. (Ideally, deb822 would detect which behavior to use if given an actual Release file as input, but this is not implemented yet.) (Closes: #473259) - Add support for Checksums-{Sha1,Sha256} multivalued fields in Dsc and Changes classes * debian/control: - "python" --> "Python" in the Description field - Change the section to "python" Checksums-Sha1: d12d7c95563397ec37c0d877486367b409d849f5 1117 python-debian_0.1.10.dsc 19efe23f688fb7f2b20f33d563146330064ab1fa 109573 python-debian_0.1.10.tar.gz 22ff71048921a788ad9d90f9579c6667e6b3de3a 44260 python-debian_0.1.10_all.deb Checksums-Sha256: aae63dfb18190558af8e71118813dd6a11157c6fd92fdc9b5c3ac370daefe5e1 1117 python-debian_0.1.10.dsc d297c07395ffa0c4a35932b58e9c2be541e8a91a83ce762d82a8474c4fc96139 109573 python-debian_0.1.10.tar.gz 4c73727b6438d9ba60aeb5e314e2d8523f021da508405dc54317ad2b392834ee 44260 python-debian_0.1.10_all.deb Files: 469202dfd24d55a932af717c6377ee59 1117 python optional python-debian_0.1.10.dsc 4857552b0156fdd4fa99d21ec131d3d2 109573 python optional python-debian_0.1.10.tar.gz 81864d535c326c082de3763969c18be6 44260 python optional python-debian_0.1.10_all.deb ''' SIGNED_CHECKSUM_CHANGES_FILE = '''\ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 %s -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) iD8DBQFIGWQO0UIZh3p4ZWERAug/AJ93DWD9o+1VMgPDjWn/dsmPSgTWGQCeOfZi 6LAP26zP25GAeTlKwJQ17hs= =fwnP -----END PGP SIGNATURE----- ''' UNPARSED_PARAGRAPHS_WITH_COMMENTS = '''\ # Leading comments should be ignored. Source: foo Section: bar # An inline comment in the middle of a paragraph should be ignored. Priority: optional Homepage: http://www.debian.org/ Build-Depends: debhelper, # quux, (temporarily disabled by a comment character) python # Comments in the middle shouldn't result in extra blank paragraphs either. # Ditto. # A comment at the top of a paragraph should be ignored. Package: foo Architecture: any Description: An awesome package # This should still appear in the result. Blah, blah, blah. # So should this. # A comment at the end of a paragraph should be ignored. # Trailing comments shouldn't cause extra blank paragraphs. ''' PARSED_PARAGRAPHS_WITH_COMMENTS = [ deb822.Deb822Dict([ ('Source', 'foo'), ('Section', 'bar'), ('Priority', 'optional'), ('Homepage', 'http://www.debian.org/'), ('Build-Depends', 'debhelper,\n python'), ]), deb822.Deb822Dict([ ('Package', 'foo'), ('Architecture', 'any'), ('Description', 'An awesome package\n' ' # This should still appear in the result.\n' ' Blah, blah, blah. # So should this.'), ]), ] def find_test_file(filename): # type: (str) -> str """ find a test file that is located within the test suite """ return os.path.join(os.path.dirname(__file__), filename) KEYRING = os.path.abspath(find_test_file('test-keyring.gpg')) def open_utf8(filename, mode='r'): # type: (str, str) -> IO[Text] """Open a UTF-8 text file in text mode.""" return open(filename, mode=mode, encoding='UTF-8') class TestDeb822Dict: def make_dict(self): # type: () -> deb822.Deb822Dict d = deb822.Deb822Dict() d['TestKey'] = 1 d['another_key'] = 2 return d def test_case_insensitive_lookup(self) -> None: d = self.make_dict() assert 1 == d['testkey'] assert 2 == d['Another_keY'] def test_case_insensitive_assignment(self) -> None: d = self.make_dict() d['testkey'] = 3 assert 3 == d['TestKey'] assert 3 == d['testkey'] d.setdefault('foo', 4) assert 4 == d['Foo'] def test_case_preserved(self) -> None: d = self.make_dict() assert sorted(['another_key', 'TestKey']) == sorted(d.keys()) def test_order_preserved(self) -> None: d = self.make_dict() d['Third_key'] = 3 d['another_Key'] = 2.5 keys = ['TestKey', 'another_key', 'Third_key'] assert keys == list(d.keys()) assert list(zip(keys, d.values())) == list(d.items()) keys2 = [] for key in d: keys2.append(key) assert keys == keys2 def test_derived_dict_equality(self) -> None: d1 = self.make_dict() d2 = dict(d1) assert d1 == d2 def test_unicode_key_access(self) -> None: d = self.make_dict() assert 1 == d['testkey'] class TestDeb822: def assertWellParsed(self, deb822_, dict_): # type: (deb822.Deb822, deb822.Deb822Mapping) -> None """Check that the given Deb822 object has the very same keys and values as the given dict. """ assert deb822_.keys() == dict_.keys() for k, v in dict_.items(): assert v == deb822_[k] assert deb822_ == dict_ @staticmethod def gen_random_string(length=20): # type: (int) -> str from random import choice import string chars = string.ascii_letters + string.digits return ''.join([choice(chars) for i in range(length)]) def test_apt_pkg_installed(self) -> None: # If test suite is running in FORBID_MISSING_APT_PKG mode where # python-apt is mandatory, explicitly include a failing test to # highlight this problem. if FORBID_MISSING_APT_PKG and not _have_apt_pkg: pytest.fail("Required apt_pkg from python-apt is not installed (tests run in FORBID_MISSING_APT_PKG mode)") def test_charset_normalizer_installed(self) -> None: # If test suite is running in FORBID_MISSING_CHARSET_NORMALIZER mode # where charset-normalizer is mandatory, explicitly include a failing # test to highlight this problem. if FORBID_MISSING_CHARSET_NORMALIZER and not _have_charset_normalizer: pytest.fail("Required charset-normalizer is not installed (tests run in FORBID_MISSING_CHARSET_NORMALIZER mode)") def test_gpgv_installed(self) -> None: # If test suite is running in FORBID_MISSING_GPGV mode where # having gpgv is mandatory, explicitly include a failing test to # highlight this problem. if FORBID_MISSING_GPGV and not _gpgv_path: pytest.fail("Required gpgv executable is not installed (tests run in FORBID_MISSING_GPGV mode)") def test_parser(self) -> None: deb822_ = deb822.Deb822(UNPARSED_PACKAGE.splitlines()) self.assertWellParsed(deb822_, PARSED_PACKAGE) def test_pickling(self) -> None: # Ensure that the Deb822 objects can be serialised # See https://bugs.debian.org/975915 env = os.environ.copy() env['PYTHONHASHSEED'] = 'random' env['PYTHONPATH'] = os.pathsep.join(sys.path) args = [ sys.executable, '-c', '''\ from debian import deb822 import pickle d = deb822.Deb822("Field: value") with open("test_deb822.pickle", "wb") as fh: pickle.dump(d, fh) ''' ] try: # make the pickle in a different subprocess which will # seed the hashes differently subprocess.check_call(args, env=env) # now read it back with open("test_deb822.pickle", "rb") as fh: deb822_u = pickle.load(fh) finally: os.remove("test_deb822.pickle") # test that the field could do a round-trip assert deb822_u['Field'] == 'value' def test_parser_with_newlines(self) -> None: deb822_ = deb822.Deb822([ l+'\n' for l in UNPARSED_PACKAGE.splitlines()]) self.assertWellParsed(deb822_, PARSED_PACKAGE) def test_strip_initial_blanklines(self) -> None: deb822_ = deb822.Deb822(['\n'] * 3 + UNPARSED_PACKAGE.splitlines()) self.assertWellParsed(deb822_, PARSED_PACKAGE) def test_reorder(self) -> None: content = textwrap.dedent(""" Depends: bar Description: some-text Architecture: any Package: foo Recommends: baz """) paragraph = deb822.Deb822(content.splitlines()) # Verify the starting state assert list(paragraph.keys()) == \ ['Depends', 'Description', 'Architecture', 'Package', 'Recommends'] # no op paragraph.order_last('Recommends') assert list(paragraph.keys()) == \ ['Depends', 'Description', 'Architecture', 'Package', 'Recommends'] # no op paragraph.order_first('Depends') assert list(paragraph.keys()) == \ ['Depends', 'Description', 'Architecture', 'Package', 'Recommends'] paragraph.order_first('Package') assert list(paragraph.keys()) == \ ['Package', 'Depends', 'Description', 'Architecture', 'Recommends'] paragraph.order_last('Description') assert list(paragraph.keys()) == \ ['Package', 'Depends', 'Architecture', 'Recommends', 'Description'] paragraph.order_after('Recommends', 'Depends') assert list(paragraph.keys()) == \ ['Package', 'Depends', 'Recommends', 'Architecture', 'Description'] paragraph.order_before('Architecture', 'Depends') assert list(paragraph.keys()) == \ ['Package', 'Architecture', 'Depends', 'Recommends', 'Description'] with pytest.raises(ValueError): paragraph.order_after('Architecture', 'Architecture') with pytest.raises(ValueError): paragraph.order_before('Architecture', 'Architecture') with pytest.raises(KeyError): paragraph.order_before('Unknown-Field', 'Architecture') with pytest.raises(KeyError): paragraph.order_before('Architecture', 'Unknown-Field') def test_sort_fields(self) -> None: content = textwrap.dedent(""" Depends: bar Description: some-text Architecture: any Package: foo Recommends: baz """) paragraph = deb822.Deb822(content.splitlines()) # Initial state assert list(paragraph.keys()) == \ ['Depends', 'Description', 'Architecture', 'Package', 'Recommends'] # Sorting defaults to using name paragraph.sort_fields() assert list(paragraph.keys()) == \ ['Architecture', 'Depends', 'Description', 'Package', 'Recommends'] # Sorting using a key function order_table = { 'package': 1, 'architecture': 2, # Dependencies have the same order 'depends': 3, 'recommends': 3, # Desc definitely last 'description': 100, } paragraph.sort_fields(key=lambda x: order_table[x.lower()]) assert list(paragraph.keys()) == \ ['Package', 'Architecture', 'Depends', 'Recommends', 'Description'] def test_gpg_stripping(self) -> None: for string in GPG_SIGNED: unparsed_with_gpg = string % UNPARSED_PACKAGE deb822_ = deb822.Deb822(unparsed_with_gpg.splitlines()) self.assertWellParsed(deb822_, PARSED_PACKAGE) @pytest.mark.skipif(not _gpgv_path, reason="gpgv not installed") def test_gpg_info(self) -> None: unparsed_with_gpg = SIGNED_CHECKSUM_CHANGES_FILE % CHECKSUM_CHANGES_FILE deb822_from_str = deb822.Dsc(unparsed_with_gpg) result_from_str = deb822_from_str.get_gpg_info(keyrings=[KEYRING]) deb822_from_file = deb822.Dsc(io.StringIO(unparsed_with_gpg)) result_from_file = deb822_from_file.get_gpg_info(keyrings=[KEYRING]) deb822_from_lines = deb822.Dsc(unparsed_with_gpg.splitlines()) result_from_lines = deb822_from_lines.get_gpg_info(keyrings=[KEYRING]) valid = { 'GOODSIG': ['D14219877A786561', 'John Wright '], 'VALIDSIG': ['8FEFE900783CF175827C2F65D14219877A786561', '2008-05-01', '1209623566', '0', '3', '0', '17', '2', '01', '8FEFE900783CF175827C2F65D14219877A786561'], 'SIG_ID': ['j3UjSpdky92fcQISbm8W5PlwC/g', '2008-05-01', '1209623566'], } for result in result_from_str, result_from_file, result_from_lines: # The second part of the GOODSIG field could change if the primary # uid changes, so avoid checking that. Also, the first part of the # SIG_ID field has undergone at least one algorithm change in gpg, # so don't bother testing that either. assert set(result.keys()) == set(valid.keys()) assert result['GOODSIG'][0] == valid['GOODSIG'][0] assert result['VALIDSIG'] == valid['VALIDSIG'] assert result['SIG_ID'][1:] == valid['SIG_ID'][1:] @pytest.mark.skipif(not _gpgv_path, reason="gpgv not installed") def test_gpg_info2(self) -> None: with open(find_test_file('test_Dsc.badsig'), mode='rb') as f: dsc = deb822.Dsc(f) i = dsc.get_gpg_info(keyrings=[KEYRING]) assert i.valid() assert 'at' == dsc['Source'] def test_iter_paragraphs_array(self) -> None: text = (UNPARSED_PACKAGE + '\n\n\n' + UNPARSED_PACKAGE).splitlines() for d in deb822.Deb822.iter_paragraphs(text): self.assertWellParsed(d, PARSED_PACKAGE) def test_iter_paragraphs_file_io(self) -> None: text = io.StringIO(UNPARSED_PACKAGE + '\n\n\n' + UNPARSED_PACKAGE) for d in deb822.Deb822.iter_paragraphs(text, use_apt_pkg=False): self.assertWellParsed(d, PARSED_PACKAGE) @pytest.mark.skipif(not _have_apt_pkg, reason="apt_pkg is not available") def test_iter_paragraphs_file_io_apt_pkg(self) -> None: text = io.StringIO(UNPARSED_PACKAGE + '\n\n\n' + UNPARSED_PACKAGE) with pytest.warns(UserWarning): # The io.StringIO is not a real file so this will raise a warning for d in deb822.Deb822.iter_paragraphs(text, use_apt_pkg=True): self.assertWellParsed(d, PARSED_PACKAGE) def test_iter_paragraphs_file(self) -> None: text = io.StringIO() text.write(UNPARSED_PACKAGE) text.write('\n\n\n') text.write(UNPARSED_PACKAGE) with tempfile.NamedTemporaryFile() as fh: txt = text.getvalue().encode('UTF-8') fh.write(txt) fh.seek(0) for d in deb822.Deb822.iter_paragraphs(fh, use_apt_pkg=False): self.assertWellParsed(d, PARSED_PACKAGE) @pytest.mark.skipif(not _have_apt_pkg, reason="apt_pkg is not available") def test_iter_paragraphs_file_apt_pkg(self) -> None: text = io.StringIO() text.write(UNPARSED_PACKAGE) text.write('\n\n\n') text.write(UNPARSED_PACKAGE) with tempfile.NamedTemporaryFile() as fh: txt = text.getvalue().encode('UTF-8') fh.write(txt) fh.seek(0) for d in deb822.Deb822.iter_paragraphs(fh, use_apt_pkg=True): self.assertWellParsed(d, PARSED_PACKAGE) def test_iter_paragraphs_with_gpg(self) -> None: for string in GPG_SIGNED: string = string % UNPARSED_PACKAGE text = (string + '\n\n\n' + string).splitlines() count = 0 for d in deb822.Deb822.iter_paragraphs(text): self.assertWellParsed(d, PARSED_PACKAGE) count += 1 assert 2 == count def test_iter_paragraphs_bytes(self) -> None: text = (UNPARSED_PACKAGE + '\n\n\n' + UNPARSED_PACKAGE) binary = text.encode('utf-8') for d in deb822.Deb822.iter_paragraphs(binary): self.assertWellParsed(d, PARSED_PACKAGE) @pytest.mark.skipif(not _have_apt_pkg, reason="apt_pkg is not available") def test_iter_paragraphs_filename_apt_pkg(self, tmp_path: Path) -> None: self._test_iter_paragraphs_filename(tmp_path, True) def test_iter_paragraphs_filename_no_apt_pkg(self, tmp_path: Path) -> None: self._test_iter_paragraphs_filename(tmp_path, False) def _test_iter_paragraphs_filename(self, tmp_path: Path, use_apt_pkg: bool) -> None: text = (UNPARSED_PACKAGE + '\n\n\n' + UNPARSED_PACKAGE) tmp = tmp_path / "Packages" with open(tmp, "wt", encoding="UTF-8") as fh: fh.write(text) for d in deb822.Deb822.iter_paragraphs(tmp, use_apt_pkg=use_apt_pkg): self.assertWellParsed(d, PARSED_PACKAGE) assert len(list(deb822.Deb822.iter_paragraphs(tmp, use_apt_pkg=use_apt_pkg))) == 2 for d in deb822.Deb822.iter_paragraphs(str(tmp), use_apt_pkg=use_apt_pkg): self.assertWellParsed(d, PARSED_PACKAGE) assert len(list(deb822.Deb822.iter_paragraphs(str(tmp), use_apt_pkg=use_apt_pkg))) == 2 def _test_iter_paragraphs_count(self, filename, cmd, expected, *args, **kwargs): # type: (str, Callable[..., Any], int, *Any, **Any) -> None with open_utf8(filename) as fh: count = len(list(cmd(fh, *args, **kwargs))) assert expected == count, \ "Wrong number paragraphs were found: expected {expected}, got {count}".format( count=count, expected=expected, ) def _test_iter_paragraphs_with_extra_whitespace(self, tests): # type: (Callable[[str], None]) -> None """ Paragraphs splitting when stray whitespace is between From policy §5.1: The paragraphs are separated by empty lines. Parsers may accept lines consisting solely of spaces and tabs as paragraph separators, but control files should use empty lines. On the principle of "be strict in what you send; be generous in what you receive", deb822 should permit such extra whitespace between deb822 stanzas. See #715558 for further details. However, when dealing with Packages and Sources files, the behaviour of apt is to not split on whitespace-only lines, and so the internal parser must be able to avoid doing so when dealing with these data. See #913274 for further details. """ for extra_space in (" ", " ", "\t"): text = UNPARSED_PACKAGE + '%s\n' % extra_space + \ UNPARSED_PACKAGE fd, filename = tempfile.mkstemp() fp = os.fdopen(fd, 'wb') fp.write(text.encode('utf-8')) fp.close() try: tests(filename) finally: os.remove(filename) def test_iter_paragraphs_with_extra_whitespace_default(self) -> None: """ Paragraphs splitting with stray whitespace (default options) """ def tests(filename): # type: (str) -> None # apt_pkg not used, should split self._test_iter_paragraphs_count(filename, deb822.Deb822.iter_paragraphs, 2) self._test_iter_paragraphs_with_extra_whitespace(tests) def test_iter_paragraphs_with_extra_whitespace_no_apt_pkg(self) -> None: """ Paragraphs splitting with stray whitespace (without apt_pkg)""" def tests(filename): # type: (str) -> None # apt_pkg not used, should split self._test_iter_paragraphs_count(filename, deb822.Deb822.iter_paragraphs, 2, use_apt_pkg=False) # Specialised iter_paragraphs force use of apt_pkg and don't split self._test_iter_paragraphs_count(filename, deb822.Packages.iter_paragraphs, 1, use_apt_pkg=False) self._test_iter_paragraphs_count(filename, deb822.Sources.iter_paragraphs, 1, use_apt_pkg=False) # Explicitly set internal parser to not split strict = {'whitespace-separates-paragraphs': False} self._test_iter_paragraphs_count(filename, deb822.Packages.iter_paragraphs, 1, use_apt_pkg=False, strict=strict) self._test_iter_paragraphs_count(filename, deb822.Sources.iter_paragraphs, 1, use_apt_pkg=False, strict=strict) # Explicitly set internal parser to split strict = {'whitespace-separates-paragraphs': True} self._test_iter_paragraphs_count(filename, deb822.Packages.iter_paragraphs, 2, use_apt_pkg=False, strict=strict) self._test_iter_paragraphs_count(filename, deb822.Sources.iter_paragraphs, 2, use_apt_pkg=False, strict=strict) self._test_iter_paragraphs_with_extra_whitespace(tests) @pytest.mark.skipif(not _have_apt_pkg, reason="apt_pkg is not available") def test_iter_paragraphs_with_extra_whitespace_apt_pkg(self) -> None: """ Paragraphs splitting with stray whitespace (with apt_pkg) """ def tests(filename): # type: (str) -> None # apt_pkg used, should not split self._test_iter_paragraphs_count(filename, deb822.Deb822.iter_paragraphs, 1, use_apt_pkg=True) # Specialised iter_paragraphs force use of apt_pkg and don't split self._test_iter_paragraphs_count(filename, deb822.Packages.iter_paragraphs, 1, use_apt_pkg=True) self._test_iter_paragraphs_count(filename, deb822.Sources.iter_paragraphs, 1, use_apt_pkg=True) self._test_iter_paragraphs_with_extra_whitespace(tests) def _test_iter_paragraphs(self, filename, cls, **kwargs): # type: (str, Type[deb822.Deb822], **Any) -> None """Ensure iter_paragraphs consistency""" with open(filename, 'rb') as fh: packages_content = fh.read() # XXX: The way multivalued fields parsing works, we can't guarantee # that trailing whitespace is reproduced. packages_content = b"\n".join([line.rstrip() for line in packages_content.splitlines()] + [b'']) s = io.BytesIO() l = [] with warnings.catch_warnings(record=True) as warnings_record: with open_utf8(filename) as f: for p in cls.iter_paragraphs(f, **kwargs): p.dump(s) s.write(b"\n") l.append(p) if not _have_apt_pkg and kwargs.get("use_apt_pkg", True): assert len(warnings_record) == 1, "Expected warning not emitted from deb822" assert "apt_pkg" in str(warnings_record[0].message) assert "python3-apt" in str(warnings_record[0].message) assert s.getvalue() == packages_content if kwargs["shared_storage"] is False: # If shared_storage is False, data should be consistent across # iterations -- i.e. we can use "old" objects s = io.BytesIO() for p in l: p.dump(s) s.write(b"\n") assert s.getvalue() == packages_content def test_iter_paragraphs_apt_shared_storage_packages(self) -> None: self._test_iter_paragraphs(find_test_file("test_Packages"), deb822.Packages, use_apt_pkg=True, shared_storage=True) def test_iter_paragraphs_apt_no_shared_storage_packages(self) -> None: self._test_iter_paragraphs(find_test_file("test_Packages"), deb822.Packages, use_apt_pkg=True, shared_storage=False) def test_iter_paragraphs_no_apt_no_shared_storage_packages(self) -> None: self._test_iter_paragraphs(find_test_file("test_Packages"), deb822.Packages, use_apt_pkg=False, shared_storage=False) def test_iter_paragraphs_apt_shared_storage_sources(self) -> None: self._test_iter_paragraphs(find_test_file("test_Sources"), deb822.Sources, use_apt_pkg=True, shared_storage=True) def test_iter_paragraphs_apt_no_shared_storage_sources(self) -> None: self._test_iter_paragraphs(find_test_file("test_Sources"), deb822.Sources, use_apt_pkg=True, shared_storage=False) def test_iter_paragraphs_no_apt_no_shared_storage_sources(self) -> None: self._test_iter_paragraphs(find_test_file("test_Sources"), deb822.Sources, use_apt_pkg=False, shared_storage=False) def test_parser_empty_input(self) -> None: assert {} == deb822.Deb822([]) def test_iter_paragraphs_empty_input(self) -> None: generator = deb822.Deb822.iter_paragraphs([]) with pytest.raises(StopIteration): next(generator) def test_parser_limit_fields(self) -> None: wanted_fields = [ 'Package', 'MD5sum', 'Filename', 'Description' ] deb822_ = deb822.Deb822(UNPARSED_PACKAGE.splitlines(), wanted_fields) assert sorted(wanted_fields) == sorted(deb822_.keys()) for key in wanted_fields: assert PARSED_PACKAGE[key] == deb822_[key] def test_iter_paragraphs_limit_fields(self) -> None: wanted_fields = [ 'Package', 'MD5sum', 'Filename', 'Tag' ] for deb822_ in deb822.Deb822.iter_paragraphs( UNPARSED_PACKAGE.splitlines(), wanted_fields): assert sorted(wanted_fields) == sorted(deb822_.keys()) for key in wanted_fields: assert PARSED_PACKAGE[key] == deb822_[key] def test_dont_assume_trailing_newline(self) -> None: deb822a = deb822.Deb822(['Package: foo']) deb822b = deb822.Deb822(['Package: foo\n']) assert deb822a['Package'] == deb822b['Package'] deb822a = deb822.Deb822(['Description: foo\n', 'bar']) deb822b = deb822.Deb822(['Description: foo', 'bar\n']) assert deb822a['Description'] == deb822b['Description'] def test__delitem__(self) -> None: parsed = deb822.Deb822(UNPARSED_PACKAGE.splitlines()) deriv = deb822.Deb822(_parsed=parsed) dict_ = PARSED_PACKAGE.copy() for key in ('Package', 'MD5sum', 'Description'): del dict_[key] for d in (parsed, deriv): del d[key] d.keys() # ensure this does not raise error self.assertWellParsed(d, dict_) def test_policy_compliant_whitespace(self) -> None: string = ( 'Package: %(Package)s\n' 'Version :%(Version)s \n' 'Priority:%(Priority)s\t \n' 'Section \t :%(Section)s \n' 'Empty-Field: \t\t\t\n' 'Multiline-Field : a \n b\n c\n' ) % PARSED_PACKAGE deb822_ = deb822.Deb822(string.splitlines()) dict_ = PARSED_PACKAGE.copy() dict_['Empty-Field'] = '' dict_['Multiline-Field'] = 'a\n b\n c' # XXX should be 'a\nb\nc'? for k, v in deb822_.items(): assert dict_[k] == v def test_case_insensitive(self) -> None: # PARSED_PACKAGE is a deb822.Deb822Dict object, so we can test # it directly assert PARSED_PACKAGE['Architecture'] == PARSED_PACKAGE['architecture'] c_i_dict = deb822.Deb822Dict() test_string = self.gen_random_string() c_i_dict['Test-Key'] = test_string assert test_string == c_i_dict['test-key'] test_string_2 = self.gen_random_string() c_i_dict['TeSt-KeY'] = test_string_2 assert test_string_2 == c_i_dict['Test-Key'] deb822_ = deb822.Deb822(io.StringIO(UNPARSED_PACKAGE)) # deb822_.keys() will return non-normalized keys for k in deb822_: assert deb822_[k] == deb822_[k.lower()] def test_multiline_trailing_whitespace_after_colon(self) -> None: """Trailing whitespace after the field name on multiline fields If the field's value starts with a newline (e.g. on MD5Sum fields in Release files, or Files field in .dsc's, the dumped string should not have a trailing space after the colon. If the value does not start with a newline (e.g. the control file Description field), then there should be a space after the colon, as with non-multiline fields. """ # bad_re: match a line that starts with a "Field:", and ends in # whitespace bad_re = re.compile(r"^\S+:\s+$") for cls in deb822.Deb822, deb822.Changes: parsed = cls(CHANGES_FILE.splitlines()) for line in parsed.dump().splitlines(): assert bad_re.match(line) is None, \ "There should not be trailing whitespace " \ "after the colon in a multiline field " \ "starting with a newline" control_paragraph = """Package: python-debian Architecture: all Depends: ${python:Depends} Suggests: python-apt Provides: python-deb822 Conflicts: python-deb822 Replaces: python-deb822 Description: python modules to work with Debian-related data formats This package provides python modules that abstract many formats of Debian related files. Currently handled are: * Debtags information (debian_bundle.debtags module) * debian/changelog (debian_bundle.changelog module) * Packages files, pdiffs (debian_bundle.debian_support module) * Control files of single or multple RFC822-style paragraphs, e.g debian/control, .changes, .dsc, Packages, Sources, Release, etc. (debian_bundle.deb822 module) """ parsed_control = deb822.Deb822(control_paragraph.splitlines()) field_re = re.compile(r"^\S+:") field_with_space_re = re.compile(r"^\S+: ") dump = parsed_control.dump() assert dump is not None for line in dump.splitlines(): if field_re.match(line): assert field_with_space_re.match(line), \ "Multiline fields that do not start with " \ "newline should have a space between the " \ "colon and the beginning of the value" def test_blank_value(self) -> None: """Fields with blank values are parsable--so they should be dumpable""" d = deb822.Deb822() d['Foo'] = 'bar' d['Baz'] = '' d['Another-Key'] = 'another value' # Previous versions would raise an exception here -- this makes the # test fail and gives useful information, so I won't try to wrap around # it. dumped = d.dump() # May as well make sure the resulting string is what we want expected = "Foo: bar\nBaz:\nAnother-Key: another value\n" assert dumped == expected def test_copy(self) -> None: """The copy method of Deb822 should return another Deb822 object""" d = deb822.Deb822() d['Foo'] = 'bar' d['Bar'] = 'baz' d_copy = d.copy() assert isinstance(d_copy, deb822.Deb822) expected_dump = "Foo: bar\nBar: baz\n" assert d_copy.dump() == expected_dump def test_bug457929_multivalued_dump_works(self) -> None: """dump() was not working in multivalued classes, see #457929.""" changesobj = deb822.Changes(CHANGES_FILE.splitlines()) assert CHANGES_FILE == changesobj.dump() def test_bug487902_multivalued_checksums(self) -> None: """New multivalued field Checksums was not handled correctly, see #487902.""" changesobj = deb822.Changes(CHECKSUM_CHANGES_FILE.splitlines()) assert CHECKSUM_CHANGES_FILE == changesobj.dump() def test_case_preserved_in_input(self) -> None: """The field case in the output from dump() should be the same as the input, even if multiple Deb822 objects have been created using different case conventions. This is related to bug 473254 - the fix for this issue is probably the same as the fix for that bug. """ input1 = "Foo: bar\nBaz: bang\n" input2 = "foo: baz\nQux: thud\n" d1 = deb822.Deb822(input1.splitlines()) d2 = deb822.Deb822(input2.splitlines()) assert input1 == d1.dump() assert input2 == d2.dump() d3 = deb822.Deb822() if 'some-test-key' not in d3: d3['Some-Test-Key'] = 'some value' assert d3.dump() == "Some-Test-Key: some value\n" @pytest.mark.skipif(not _have_apt_pkg, reason="apt_pkg is not available") def test_unicode_values_apt_pkg(self) -> None: """Deb822 objects should contain only unicode values (Technically, they are allowed to contain any type of object, but when parsed from files, and when only string-type objects are added, the resulting object should have only unicode values.) """ objects = [] # type: List[deb822.Deb822] with open_utf8(find_test_file('test_Packages')) as f: objects.extend(deb822.Packages.iter_paragraphs(f)) with open_utf8(find_test_file('test_Sources')) as f: objects.extend(deb822.Deb822.iter_paragraphs(f)) with open(find_test_file('test_Sources.iso8859-1'), 'rb') as fb: objects.extend(deb822.Deb822.iter_paragraphs( fb, encoding="iso8859-1")) for d in objects: for value in d.values(): assert isinstance(value, str) # The same should be true for Sources and Changes except for their # _multivalued fields multi = [] # type: List[Union[deb822.Changes, deb822.Sources]] with open_utf8(find_test_file('test_Sources')) as f: multi.extend(deb822.Sources.iter_paragraphs(f)) for d in multi: for key, value in d.items(): if key.lower() not in d.__class__._multivalued_fields: assert isinstance(value, str) def test_unicode_values(self) -> None: """Deb822 objects should contain only unicode values (Technically, they are allowed to contain any type of object, but when parsed from files, and when only string-type objects are added, the resulting object should have only unicode values.) """ objects = [] objects.append(deb822.Deb822(UNPARSED_PACKAGE)) objects.append(deb822.Deb822(CHANGES_FILE)) with open_utf8(find_test_file('test_Packages')) as f: objects.extend(deb822.Deb822.iter_paragraphs(f)) with open_utf8(find_test_file('test_Sources')) as f: objects.extend(deb822.Deb822.iter_paragraphs(f)) for d in objects: for value in d.values(): assert isinstance(value, str) # The same should be true for Sources and Changes except for their # _multivalued fields multi = [] # type: List[Union[deb822.Changes, deb822.Sources]] multi.append(deb822.Changes(CHANGES_FILE)) multi.append(deb822.Changes(SIGNED_CHECKSUM_CHANGES_FILE % CHECKSUM_CHANGES_FILE)) for d in multi: for key, value in d.items(): if key.lower() not in d.__class__._multivalued_fields: assert isinstance(value, str) def test_encoding_integrity(self) -> None: with open_utf8(find_test_file('test_Sources')) as f: utf8 = list(deb822.Deb822.iter_paragraphs(f)) with open(find_test_file('test_Sources.iso8859-1'), 'rb') as fb: latin1 = list(deb822.Deb822.iter_paragraphs( fb, encoding='iso8859-1')) # dump() with no fd returns a unicode object - both should be identical assert len(utf8) == len(latin1) for i in range(len(utf8)): assert utf8[i].dump() == latin1[i].dump() # XXX: The way multiline fields parsing works, we can't guarantee # that trailing whitespace is reproduced. with open(find_test_file('test_Sources'), 'rb') as fb: utf8_contents = b"\n".join([line.rstrip() for line in fb] + [b'']) with open(find_test_file('test_Sources.iso8859-1'), 'rb') as fb: latin1_contents = b"\n".join([line.rstrip() for line in fb] + [b'']) utf8_to_latin1 = io.BytesIO() for d in utf8: d.dump(fd=utf8_to_latin1, encoding='iso8859-1') utf8_to_latin1.write(b"\n") latin1_to_utf8 = io.BytesIO() for d in latin1: d.dump(fd=latin1_to_utf8, encoding='utf-8') latin1_to_utf8.write(b"\n") assert utf8_contents == latin1_to_utf8.getvalue() assert latin1_contents == utf8_to_latin1.getvalue() @pytest.mark.skipif(not _have_charset_normalizer and not _have_chardet, reason="apt_pkg is not available") def test_mixed_encodings(self) -> None: """Test that we can handle a simple case of mixed encodings In general, this isn't guaranteed to work. It uses the chardet package, which tries to determine heuristically the encoding of the text given to it. But as far as I've seen, it's reliable for mixed latin1 and utf-8 in maintainer names in old Sources files... """ # Avoid spitting out the encoding warning during testing. warnings.filterwarnings(action='ignore', category=UnicodeWarning) filename = find_test_file('test_Sources.mixed_encoding') with open(filename, 'rb') as f1, open(filename, 'rb') as f2: for paragraphs in [ deb822.Sources.iter_paragraphs(f1), deb822.Sources.iter_paragraphs(f2, use_apt_pkg=False) ]: with warnings.catch_warnings(record=True) as warnings_record: p1 = next(paragraphs) assert p1['maintainer'] == \ 'Adeodato Sim\xf3 ' p2 = next(paragraphs) assert p2['uploaders'] == \ 'Frank K\xfcster ' if FORBID_MISSING_APT_PKG: assert not warnings_record, "Warnings emitted from deb822" def test_dump_text_mode(self) -> None: d = deb822.Deb822(CHANGES_FILE.splitlines()) buf = io.StringIO() d.dump(fd=buf, text_mode=True) assert CHANGES_FILE == buf.getvalue() def test_bug597249_colon_as_first_value_character(self) -> None: """Colon should be allowed as the first value character. See #597249. """ data = 'Foo: : bar' parsed = {'Foo': ': bar'} self.assertWellParsed(deb822.Deb822(data), parsed) @staticmethod def _dictset(d, key, value): # type: (Dict[str, Any], str, Any) -> None d[key] = value def test_field_value_ends_in_newline(self) -> None: """Field values are not allowed to end with newlines""" d = deb822.Deb822() with pytest.raises(ValueError): self._dictset(d, 'foo', 'bar\n') # type: ignore with pytest.raises(ValueError): self._dictset(d, 'foo', 'bar\nbaz\n') # type: ignore def test_field_value_contains_blank_line(self) -> None: """Field values are not allowed to contain blank lines""" d = deb822.Deb822() with pytest.raises(ValueError): self._dictset(d, 'foo', 'bar\n\nbaz') # type: ignore with pytest.raises(ValueError): self._dictset(d, 'foo', '\n\nbaz') # type: ignore def test_multivalued_field_contains_newline(self) -> None: # mypy does not handle metaprogramming for multivalued fields """Multivalued field components are not allowed to contain newlines""" d = deb822.Dsc() # We don't check at set time, since one could easily modify the list # without deb822 knowing. We instead check at get time. d['Files'] = [{'md5sum': 'deadbeef', 'size': '9605', 'name': 'bad\n'}] # type: ignore with pytest.raises(ValueError): d.get_as_string('files') def test_multivalued_field_varargs_missing(self) -> None: """Last field of multivalued fields remains unset.""" text = "Package-List:\n pkg deb section priority\n" d = deb822.Dsc(text) assert d["Package-List"] == [{"package": "pkg", "package-type": "deb", "section": "section", "priority": "priority"}] assert str(d) == text def test_multivalued_field_varargs_multiple(self) -> None: """Last field of multivalued fields should collect all remaining arguments.""" text = "Package-List:\n pkg deb section priority arch=all essential=yes\n" d = deb822.Dsc(text) assert d["Package-List"] == [{"package": "pkg", "package-type": "deb", "section": "section", "priority": "priority", "_other": "arch=all essential=yes"}] assert str(d) == text def _test_iter_paragraphs_comments(self, paragraphs): # type: (List[deb822.Deb822]) -> None assert len(paragraphs) == len(PARSED_PARAGRAPHS_WITH_COMMENTS) for i in range(len(paragraphs)): self.assertWellParsed(paragraphs[i], PARSED_PARAGRAPHS_WITH_COMMENTS[i]) @pytest.mark.skipif(not _have_apt_pkg, reason="apt_pkg is not available") def test_iter_paragraphs_comments_use_apt_pkg(self) -> None: """ apt_pkg does not support comments within multiline fields This test checks that a file with comments inside multiline fields generates an error from the apt_pkg parser. See also https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=750247#35 https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=807351 """ try: fd, filename = tempfile.mkstemp() fp = os.fdopen(fd, 'wb') fp.write(UNPARSED_PARAGRAPHS_WITH_COMMENTS.encode('utf-8')) fp.close() with open_utf8(filename) as fh: with pytest.raises(apt_pkg.Error): list(deb822.Deb822.iter_paragraphs(fh, use_apt_pkg=True)) finally: os.remove(filename) def test_iter_paragraphs_comments_native(self) -> None: paragraphs = list(deb822.Deb822.iter_paragraphs( UNPARSED_PARAGRAPHS_WITH_COMMENTS.splitlines(), use_apt_pkg=False)) self._test_iter_paragraphs_comments(paragraphs) def test_iter_paragraphs_string_comments_native(self) -> None: paragraphs = list(deb822.Deb822.iter_paragraphs( UNPARSED_PARAGRAPHS_WITH_COMMENTS, use_apt_pkg=False)) self._test_iter_paragraphs_comments(paragraphs) def test_explicit_source_field(self) -> None: """Test handling of explicit Source field in Packages """ # implicit source and version pkg = next(deb822.Packages.iter_paragraphs( UNPARSED_PACKAGE, use_apt_pkg=False, )) assert 'mutt' == pkg.source assert '1.5.12-1' == str(pkg.source_version) assert isinstance(pkg.source_version, Version) # explicit source, implicit version pkg = next(deb822.Packages.iter_paragraphs( UNPARSED_PACKAGE + "Source: mutt-dummy\n", use_apt_pkg=False, )) assert 'mutt-dummy' == pkg.source assert '1.5.12-1' == str(pkg.source_version) assert isinstance(pkg.source_version, Version) pkg = next(deb822.Packages.iter_paragraphs( UNPARSED_PACKAGE + "Source: mutt-dummy (1.2.3-1)\n", use_apt_pkg=False, )) assert 'mutt-dummy' == pkg.source assert '1.2.3-1' == str(pkg.source_version) assert isinstance(pkg.source_version, Version) def test_release(self) -> None: with open(find_test_file('test_Release')) as f: release = deb822.Release(f) assert release['Codename'] == 'sid' assert len(release['SHA1']) == 61 assert len(release['SHA256']) == 61 assert len(release['SHA512']) == 61 assert release['SHA512'][0]['size'] == '113433' def test_buildinfo(self) -> None: with open(find_test_file('test_BuildInfo')) as f: buildinfo = deb822.BuildInfo(f) assert buildinfo['Build-Origin'] == 'Debian' assert 'build-essential' in buildinfo['Installed-Build-Depends'] ibd = buildinfo.relations['installed-build-depends'] assert len(ibd) == 207 # type: ignore benv = buildinfo.get_environment() assert len(benv) == 3 assert benv['DEB_BUILD_OPTIONS'] == 'parallel=4' ch = buildinfo.get_changelog() assert ch is not None assert ch.version == '4.5.2-1+b1' assert ch.package == 'lxml' assert ch.urgency == 'low' assert 'Daemon' in ch.author del buildinfo['Binary-Only-Changes'] ch = buildinfo.get_changelog() assert ch is None def test_buildinfo_env_deserialise(self) -> None: data = r""" DEB_BUILD_OPTIONS="parallel=4" LANG="en_AU.UTF-8" LC_ALL="C UTF-8" LC_TIME="en_GB.UTF-8" LD_LIBRARY_PATH="/usr/lib/libeatmy\"data" SOURCE_DATE_EPOCH="1601784586" """ benv = dict(deb822.BuildInfo._env_deserialise(data)) assert len(benv) == 6 assert benv['DEB_BUILD_OPTIONS'] == 'parallel=4' assert '\n' in benv['LC_ALL'] assert '"' in benv['LD_LIBRARY_PATH'] assert benv['LD_LIBRARY_PATH'] == '/usr/lib/libeatmy"data' benv = dict(deb822.BuildInfo._env_deserialise("")) assert len(benv) == 0 def test_changes_binary_mode(self) -> None: """Trivial parse test for a signed file in binary mode""" with io.open(find_test_file('test_Changes'), 'rb') as f: changes = deb822.Changes(f) assert 'python-debian' == changes['Source'] def test_changes_text_mode(self) -> None: """Trivial parse test for a signed file in text mode""" with io.open(find_test_file('test_Changes'), 'r', encoding='utf-8') as f: changes = deb822.Changes(f) assert 'python-debian' == changes['Source'] def test_removals(self) -> None: with open(find_test_file('test_removals.822')) as f: removals = deb822.Removals.iter_paragraphs(f) r = next(removals) assert r['suite'] == 'unstable' assert r['date'] == u'Wed, 01 Jan 2014 17:03:54 +0000' # Date objects, timezones, cross-platform, portability nightmare... assert r.date.strftime('%S') == '54' assert len(r.binaries) == 1 assert r.binaries[0]['package'] == 'libzoom-ruby' assert r.binaries[0]['version'] == '0.4.1-5' assert r.binaries[0]['architectures'] == set(['all']) r = next(removals) assert len(r.binaries) == 3 r = next(removals) assert r['bug'] == '753912' assert r.bug == [753912] assert r.also_wnpp == [123456] r = next(removals) assert r.binaries[0]['architectures'] == \ set(['amd64', 'armel', 'armhf', 'hurd-i386', 'i386', 'kfreebsd-amd64', 'kfreebsd-i386', 'mips', 'mipsel', 'powerpc', 's390x', 'sparc']) class TestPkgRelations: def assertPkgDictEqual(self, expected, actual): # type: (deb822.Deb822Mapping, deb822.Deb822Mapping) -> None p1keys = sorted(expected.keys()) p2keys = sorted(actual.keys()) assert p1keys == p2keys, "Different fields present in packages" for k in p1keys: assert expected[k] == actual[k], "Different for field '%s'" % k @staticmethod def rel(dict_): # type: (deb822.Deb822MutableMapping) -> deb822.Deb822Mapping """Modify dict_ to ensure it contains all fields from parse_relations Accept a dict that partially describes a package relationship and add to it any missing keys. returns: modified dict_ """ if 'version' not in dict_: dict_['version'] = None if 'arch' not in dict_: dict_['arch'] = None if 'archqual' not in dict_: dict_['archqual'] = None if 'restrictions' not in dict_: dict_['restrictions'] = None return dict_ def test_packages(self) -> None: # make the syntax a bit more compact rel = TestPkgRelations.rel with warnings.catch_warnings(record=True) as warnings_record: f = open(find_test_file('test_Packages')) pkgs = deb822.Packages.iter_paragraphs(f) pkg1 = next(pkgs) rel1 = {'breaks': [], 'built-using': [], 'conflicts': [], 'depends': [ [rel({'name': 'file', 'archqual': 'i386'})], [rel({'name': 'libc6', 'version': ('>=', '2.7-1')})], [rel({'name': 'libpaper1'})], [rel({'name': 'psutils'})], ], 'enhances': [], 'pre-depends': [], 'provides': [], 'recommends': [ [rel({'name': 'bzip2'})], [rel({'name': 'lpr'}), rel({'name': 'rlpr'}), rel({'name': 'cupsys-client'})], [rel({'name': 'wdiff'})], ], 'replaces': [], 'suggests': [ [rel({'name': 'emacsen-common'})], [rel({'name': 'ghostscript'})], [rel({'name': 'graphicsmagick-imagemagick-compat'}), rel({'name': 'imagemagick'})], [rel({'name': 'groff'})], [rel({'name': 'gv'})], [rel({'name': 'html2ps'})], [rel({'name': 't1-cyrillic'})], [rel({'name': 'texlive-base-bin'})], ] } self.assertPkgDictEqual(rel1, pkg1.relations) pkg2 = next(pkgs) rel2 = {'breaks': [], 'built-using': [], 'conflicts': [], 'depends': [ [rel({'name': 'lrzsz'})], [rel({'name': 'openssh-client'}), rel({'name': 'telnet'}), rel({'name': 'telnet-ssl'})], [rel({'name': 'libc6', 'version': ('>=', '2.6.1-1')})], [rel({'name': 'libncurses5', 'version': ('>=', '5.6')})], [rel({'name': 'libreadline5', 'version': ('>=', '5.2')})], ], 'enhances': [], 'pre-depends': [], 'provides': [], 'recommends': [], 'replaces': [], 'suggests': [] } self.assertPkgDictEqual(rel2, pkg2.relations) pkg3 = next(pkgs) dep3 = [ [rel({'name': 'dcoprss', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'kdenetwork-kfile-plugins', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'kdict', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'kdnssd', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'kget', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'knewsticker', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'kopete', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'kpf', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'kppp', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'krdc', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'krfb', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'ksirc', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'kwifimanager', 'version': ('>=', '4:3.5.9-2')})], [rel({'name': 'librss1', 'version': ('>=', '4:3.5.9-2')})], ] assert dep3 == pkg3.relations['depends'] f.close() if FORBID_MISSING_APT_PKG: # Don't permit this test to succeed if warnings about apt_pkg # being missing were generated assert not warnings_record, "Warnings emitted from deb822" def test_pkgrelation_str(self, caplog): # type: (pytest.LogCaptureFixture) -> None bin_rels = [ 'file, libc6 (>= 2.7-1), libpaper1, psutils, ' 'perl:any, python:native' ] src_rels = [ 'apache2-src (>= 2.2.9), libaprutil1-dev, ' 'libcap-dev [!kfreebsd-i386 !kfreebsd-amd64 !hurd-i386], ' 'autoconf , ' 'debhelper (>> 5.0.0) ' ] for bin_rel in bin_rels: assert bin_rel == \ deb822.PkgRelation.str(deb822.PkgRelation.parse_relations( bin_rel)) for src_rel in src_rels: assert src_rel == \ deb822.PkgRelation.str(deb822.PkgRelation.parse_relations( \ src_rel)) with caplog.at_level(logging.WARNING): deb822.PkgRelation.parse_relations("foo bar") assert caplog.record_tuples == [( "debian.deb822", logging.WARNING, 'cannot parse package relationship ' '"foo bar", returning it raw' )] def test_sources(self) -> None: # make the syntax a bit more compact rel = TestPkgRelations.rel # Should not get warnings about missing python-apt from this code with warnings.catch_warnings(record=True) as warnings_record: f = open_utf8(find_test_file('test_Sources')) pkgs = deb822.Sources.iter_paragraphs(f) pkg1 = next(pkgs) rel1 = {'build-conflicts': [], 'build-conflicts-indep': [], 'build-conflicts-arch': [], 'build-depends': [ [rel({'name': 'apache2-src', 'version': ('>=', '2.2.9')})], [rel({'name': 'libaprutil1-dev'})], [rel({'arch': [(False, 'kfreebsd-i386'), (False, 'kfreebsd-amd64'), (False, 'hurd-i386')], 'name': 'libcap-dev'})], [rel({'name': 'autoconf'})], [rel({'name': 'debhelper', 'version': ('>>', '5.0.0')})], ], 'build-depends-indep': [], 'build-depends-arch': [], 'binary': [ [rel({'name': 'apache2-mpm-itk'})] ] } self.assertPkgDictEqual(rel1, pkg1.relations) pkg2 = next(pkgs) rel2 = {'build-conflicts': [], 'build-conflicts-indep': [], 'build-conflicts-arch': [], 'build-depends': [ [rel({'name': 'dpkg-dev', 'version': ('>=', '1.13.9')})], [rel({'name': 'autoconf', 'version': ('>=', '2.13')})], [rel({'name': 'bash'})], [rel({'name': 'bison', 'archqual': 'amd64'})], [rel({'name': 'flex'})], [rel({'name': 'gettext', 'archqual': 'any'})], [rel({'name': 'texinfo', 'restrictions': [ [(False, 'stage1')], [(False, 'stage2'), (False, 'cross')] ]})], [rel({'arch': [(True, 'hppa')], 'name': 'expect-tcl8.3', 'version': ('>=', '5.32.2'), 'restrictions': [[(False, 'stage1')]]})], [rel({'name': 'dejagnu', 'version': ('>=', '1.4.2-1.1'), 'arch': None})], [rel({'name': 'dpatch'})], [rel({'name': 'file'})], [rel({'name': 'bzip2', 'archqual': 'native'})], [rel({'name': 'lsb-release'})], ], 'build-depends-indep': [], 'build-depends-arch': [], 'binary': [ [rel({'name': 'binutils'})], [rel({'name': 'binutils-dev'})], [rel({'name': 'binutils-multiarch'})], [rel({'name': 'binutils-hppa64'})], [rel({'name': 'binutils-spu'})], [rel({'name': 'binutils-doc'})], [rel({'name': 'binutils-source'})], ] } self.assertPkgDictEqual(rel2, pkg2.relations) f.close() if FORBID_MISSING_APT_PKG: # Don't permit this test to succeed if warnings about apt_pkg # being missing were generated assert not warnings_record, "Warnings emitted from deb822" def test_restrictions_parse(self): # type: ignore """ test parsing of restriction formulas """ r = "foo " # relation 0, alternative 0, restrictions set 0, condition 0 terms = deb822.PkgRelation.parse_relations(r)[0][0]['restrictions'] assert terms is not None term = terms[0][0] assert term.enabled == True assert term[0] == True assert term.profile == 'cross' assert term[1] == 'cross' r = "foo " # relation 0, alternative 0, restrictions set 1, condition 0 terms = deb822.PkgRelation.parse_relations(r)[0][0]['restrictions'] assert terms is not None term = terms[1][0] assert term.enabled == False assert term[0] == False assert term.profile == 'stage2' assert term[1] == 'stage2' # relation 0, alternative 0, restrictions set 1, condition 1 terms = deb822.PkgRelation.parse_relations(r)[0][0]['restrictions'] assert terms is not None term = terms[1][1] assert term.enabled == False assert term[0] == False assert term.profile == 'cross' assert term[1] == 'cross' def test_multiarch_parse(self) -> None: """ test parsing of architecture qualifiers from multiarch Also ensure that the archqual part makes a round-trip, see https://bugs.debian.org/868249 """ r = "foo:native" # relation 0, alternative 0, arch qualifier rel = deb822.PkgRelation.parse_relations(r) term = rel[0][0]['archqual'] assert term == "native" assert deb822.PkgRelation.str(rel) == r class TestVersionAccessor: def test_get_version(self) -> None: # should not be available in most basic Deb822 p = deb822.Deb822(UNPARSED_PACKAGE.splitlines()) with pytest.raises(AttributeError): p.get_version() # type: ignore # should be available in Packages p = deb822.Packages(UNPARSED_PACKAGE.splitlines()) v = p.get_version() assert str(v) == '1.5.12-1' assert isinstance(v, Version) def test_set_version(self) -> None: # should not be available in most basic Deb822 p = deb822.Deb822(UNPARSED_PACKAGE.splitlines()) with pytest.raises(AttributeError): p.set_version() # type: ignore # should be available in Packages p = deb822.Packages(UNPARSED_PACKAGE.splitlines()) newver = '9.8.7-1' v = Version(newver) p.set_version(v) assert p['Version'] == newver assert isinstance(p['Version'], str) @pytest.mark.skipif(not _gpgv_path, reason="gpgv not installed") class TestGpgInfo: SampleData = namedtuple('SampleData', [ 'datastr', 'data', 'valid', ]) @pytest.fixture() def sampledata(self): # type: () -> Generator[TestGpgInfo.SampleData, None, None] datastr = SIGNED_CHECKSUM_CHANGES_FILE % CHECKSUM_CHANGES_FILE data = datastr.encode() valid = { 'GOODSIG': ['D14219877A786561', 'John Wright '], 'VALIDSIG': ['8FEFE900783CF175827C2F65D14219877A786561', '2008-05-01', '1209623566', '0', '3', '0', '17', '2', '01', '8FEFE900783CF175827C2F65D14219877A786561'], 'SIG_ID': ['j3UjSpdky92fcQISbm8W5PlwC/g', '2008-05-01', '1209623566'], } yield TestGpgInfo.SampleData( datastr, data, valid, ) def _validate_gpg_info(self, gpg_info, sampledata): # type: (deb822.GpgInfo, TestGpgInfo.SampleData) -> None # The second part of the GOODSIG field could change if the primary # uid changes, so avoid checking that. Also, the first part of the # SIG_ID field has undergone at least one algorithm change in gpg, # so don't bother testing that either. assert set(gpg_info.keys()) == set(sampledata.valid.keys()) assert gpg_info['GOODSIG'][0] == sampledata.valid['GOODSIG'][0] assert gpg_info['VALIDSIG'] == sampledata.valid['VALIDSIG'] assert gpg_info['SIG_ID'][1:] == sampledata.valid['SIG_ID'][1:] def test_from_sequence_string(self, sampledata): # type: (TestGpgInfo.SampleData) -> None gpg_info = deb822.GpgInfo.from_sequence(sampledata.data, keyrings=[KEYRING]) self._validate_gpg_info(gpg_info, sampledata) def test_from_sequence_newline_terminated(self, sampledata): # type: (TestGpgInfo.SampleData) -> None sequence = io.BytesIO(sampledata.data) gpg_info = deb822.GpgInfo.from_sequence(sequence, keyrings=[KEYRING]) self._validate_gpg_info(gpg_info, sampledata) def test_from_sequence_no_newlines(self, sampledata): # type: (TestGpgInfo.SampleData) -> None sequence = sampledata.data.splitlines() gpg_info = deb822.GpgInfo.from_sequence(sequence, keyrings=[KEYRING]) self._validate_gpg_info(gpg_info, sampledata) def test_from_file(self, sampledata): # type: (TestGpgInfo.SampleData) -> None fd, filename = tempfile.mkstemp() fp = os.fdopen(fd, 'wb') fp.write(sampledata.data) fp.close() try: gpg_info = deb822.GpgInfo.from_file(filename, keyrings=[KEYRING]) finally: os.remove(filename) self._validate_gpg_info(gpg_info, sampledata)