File: test_lexing.py

package info (click to toggle)
python-sybil 9.2.0-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 1,148 kB
  • sloc: python: 4,510; makefile: 90
file content (96 lines) | stat: -rw-r--r-- 3,213 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
import re

import pytest
from testfixtures import ShouldRaise, compare
from testfixtures.comparison import compare_text, compare_dict

from sybil import Lexeme
from sybil.parsers.abstract.lexers import BlockLexer, LexingException
from .helpers import lex, sample_path


def test_examples_from_parsing_tests():
    lexer = BlockLexer(start_pattern=re.compile('START'), end_pattern_template='END')
    path = sample_path('lexing-fail.txt')
    with ShouldRaise(
            LexingException(
                f"Could not find end of 'START', starting at line 1, column 1, "
                f"looking for 'END' in {path}:\n'\\nEDN\\n'"
            )
    ):
        lex('lexing-fail.txt', lexer)


def test_indented_block():
    lexer = BlockLexer(
        start_pattern=re.compile('(?P<prefix> *)START\n'),
        end_pattern_template=' {{{len_prefix}}}END'
    )
    region, = lex('lexing-indented-block.txt', lexer)
    compare(
        region.lexemes,
        expected={
            'source': Lexeme(
                "".join(
                    (
                        "\n",
                        "line 1\n",
                        "\n",
                        "  line 2\n",
                        "\n",
                    )
                ),
                offset=0,
                line_offset=0,
            ),
        }
    )


class TestLexemeStripping:

    @staticmethod
    def compare_lexeme(expected: Lexeme, actual: Lexeme):

        def _compare(x, y, context):
            if str(x) != str(y):
                return compare_text(x, y, context)
            return compare_dict(x.__dict__, y.__dict__, context)

        compare(expected=expected, actual=actual, comparers={Lexeme: _compare}, strict=True)

    @pytest.mark.parametrize("actual,message", [
        (Lexeme('bar', 10, 1), "'foo' (expected) != 'bar' (actual)"),
        (Lexeme('foo', 11, 1), "'offset': 10 (expected) != 11"),
        (Lexeme('foo', 10, 2), "'line_offset': 1 (expected) != 2 (actual)"),
        (Lexeme('foo', 12, 3), "line_offset': 1 (expected) != 3 (actual)\n"
                               "'offset': 10 (expected) != 12 (actual)"),
    ])
    def test_not_equal(self, actual: Lexeme, message: str):
        with ShouldRaise(AssertionError) as s:
            self.compare_lexeme(Lexeme('foo', 10, 1), actual)
        assert message in str(s.raised)

    def test_strip_no_newlines_present(self):
        self.compare_lexeme(
            actual=Lexeme('  foo  \n', 10, 1).strip_leading_newlines(),
            expected=Lexeme('  foo  \n', 10, 1)
        )

    def test_strip_one_newline_present(self):
        self.compare_lexeme(
            actual=Lexeme('\n  foo  \n', 10, 1).strip_leading_newlines(),
            expected=Lexeme('  foo  \n', 11, 2)
        )

    def test_strip_multiple_newlines_present(self):
        self.compare_lexeme(
            actual=Lexeme('\n\n\n  foo  \n', 13, 3).strip_leading_newlines(),
            expected=Lexeme('  foo  \n', 16, 6)
        )

    def test_strip_newlines_and_spaces_present(self):
        self.compare_lexeme(
            actual=Lexeme(' \n \n  foo  \n', 10, 1).strip_leading_newlines(),
            expected=Lexeme(' \n \n  foo  \n', 10, 1)
        )