1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
|
"""
Typograhic Number Theory tests
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
:copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import pytest
from pygments.lexers.tnt import TNTLexer
from pygments.token import Text, Operator, Keyword, Name, Number, \
Punctuation, Error
@pytest.fixture(autouse=True)
def lexer():
yield TNTLexer()
# whitespace
@pytest.mark.parametrize('text', (' a', ' \t0', '\n\n 3'))
def test_whitespace_positive_matches(lexer, text):
"""Test fragments that should be tokenized as whitespace text."""
assert lexer.whitespace(0, text) == len(text) - 1
assert lexer.whitespace(0, text, True) == len(text) - 1
assert lexer.cur[-1] == (0, Text, text[:-1])
@pytest.mark.parametrize('text', ('0 a=b premise', 'b=a symmetry'))
def test_whitespace_negative_matches(lexer, text):
"""Test statements that do not start with whitespace text."""
assert lexer.whitespace(0, text) == 0
with pytest.raises(AssertionError):
lexer.whitespace(0, text, True)
assert not lexer.cur
# terms that can go on either side of an = sign
@pytest.mark.parametrize('text', ('a ', "a' ", 'b ', "c' "))
def test_variable_positive_matches(lexer, text):
"""Test fragments that should be tokenized as variables."""
assert lexer.variable(0, text) == len(text) - 1
assert lexer.cur[-1] == (0, Name.Variable, text[:-1])
@pytest.mark.parametrize('text', ("' ", 'f ', "f' "))
def test_variable_negative_matches(lexer, text):
"""Test fragments that should **not** be tokenized as variables."""
with pytest.raises(AssertionError):
lexer.variable(0, text)
assert not lexer.cur
@pytest.mark.parametrize('text', ('0', 'S0', 'SSSSS0'))
def test_numeral_positive_matches(lexer, text):
"""Test fragments that should be tokenized as (unary) numerals."""
assert lexer.term(0, text) == len(text)
assert lexer.cur[-1] == (len(text) - 1, Number.Integer, text[-1])
if text != '0':
assert lexer.cur[-2] == (0, Number.Integer, text[:-1])
@pytest.mark.parametrize('text', (
'(a+b)', '(b.a)', '(c+d)'
))
def test_multiterm_positive_matches(lexer, text):
"""Test fragments that should be tokenized as a compound term."""
assert lexer.term(0, text) == len(text)
assert [t[1] for t in lexer.cur] == [
Punctuation, Name.Variable, Operator,
Name.Variable, Punctuation
]
@pytest.mark.parametrize('text', ('1', '=', 'A'))
def test_term_negative_matches(lexer, text):
"""Test fragments that should not be tokenized as terms at all."""
with pytest.raises(AssertionError):
lexer.term(0, text)
assert not lexer.cur
# full statements, minus rule
@pytest.mark.parametrize('text', ('~a=b ', '~~~~a=b '))
def test_negator_positive_matches(lexer, text):
"""Test statements that start with a negation."""
assert lexer.formula(0, text) == len(text) - 1
assert lexer.cur[0] == (0, Operator, text[:-4])
@pytest.mark.parametrize('text', ('Aa:a=b ', 'Eb:a=b '))
def test_quantifier_positive_matches(lexer, text):
"""Test statements that start with a quantifier."""
assert lexer.formula(0, text) == len(text) - 1
assert lexer.cur[0][1] == Keyword.Declaration
assert lexer.cur[1][1] == Name.Variable
assert lexer.cur[2] == (2, Punctuation, ':')
@pytest.mark.parametrize('text', ('Aaa=b', 'Eba=b'))
def test_quantifier_negative_matches(lexer, text):
"""Test quantifiers that are only partially valid."""
with pytest.raises(AssertionError):
lexer.formula(0, text)
# leftovers should still be valid
assert lexer.cur[0][1] == Keyword.Declaration
assert lexer.cur[1][1] == Name.Variable
@pytest.mark.parametrize('text', ('<a=b&b=a>', '<a=b|b=a>', '<a=b]b=a>'))
def test_compound_positive_matches(lexer, text):
"""Test statements that consist of multiple formulas compounded."""
assert lexer.formula(0, text) == len(text)
assert lexer.cur[0] == (0, Punctuation, '<')
assert lexer.cur[4][1] == Operator
assert lexer.cur[-1] == (len(text)-1, Punctuation, '>')
@pytest.mark.parametrize('text', ('<a=b/b=a>', '<a=b&b=a '))
def test_compound_negative_matches(lexer, text):
"""Test statements that look like compounds but are invalid."""
with pytest.raises(AssertionError):
lexer.formula(0, text)
assert lexer.cur[0] == (0, Punctuation, '<')
@pytest.mark.parametrize('text', ('a=b ', 'a=0 ', '0=b '))
def test_formula_postive_matches(lexer, text):
"""Test the normal singular formula."""
assert lexer.formula(0, text) == len(text) - 1
assert lexer.cur[0][2] == text[0]
assert lexer.cur[1] == (1, Operator, '=')
assert lexer.cur[2][2] == text[2]
@pytest.mark.parametrize('text', ('a/b', '0+0 '))
def test_formula_negative_matches(lexer, text):
"""Test anything but an equals sign."""
with pytest.raises(AssertionError):
lexer.formula(0, text)
# rules themselves
@pytest.mark.parametrize('text', (
'fantasy rule', 'carry over line 5', 'premise', 'joining',
'double-tilde', 'switcheroo', 'De Morgan', 'specification'
))
def test_rule_positive_matches(lexer, text):
"""Test some valid rules of TNT."""
assert lexer.rule(0, text) == len(text)
assert lexer.cur[0][:2] == (0, Keyword)
if text[-1].isdigit():
assert lexer.cur[1][1] == Number.Integer
@pytest.mark.parametrize('text', (
'fantasy', 'carry over', 'premse', 'unjoining',
'triple-tilde', 'switcheru', 'De-Morgan', 'despecification'
))
def test_rule_negative_matches(lexer, text):
"""Test some invalid rules of TNT."""
with pytest.raises(AssertionError):
lexer.rule(0, text)
# referrals
@pytest.mark.parametrize('text', ('(lines 1, 2, and 4)', '(line 3,5,6)',
'(lines 1, 6 and 0)'))
def test_lineno_positive_matches(lexer, text):
"""Test line referrals."""
assert lexer.lineno(0, text) == len(text)
assert lexer.cur[0] == (0, Punctuation, '(')
assert lexer.cur[1][:2] == (1, Text)
assert lexer.cur[2][1] == Number.Integer
assert lexer.cur[3] == (len(text)-1, Punctuation, ')')
@pytest.mark.parametrize('text', (
'(lines one, two, and four)1 ', # to avoid IndexError
'(lines 1 2 and 3)', '(lines 1 2 3)'
))
def test_lineno_negative_matches(lexer, text):
"""Test invalid line referrals."""
with pytest.raises(AssertionError):
lexer.lineno(0, text)
# worst-case: error text
@pytest.mark.parametrize('text', ('asdf', 'fdsa\nasdf', 'asdf\n '))
def test_error_till_line_end(lexer, text):
try:
nl = text.index('\n')
except ValueError:
nl = len(text)
try:
end = text.find(text.split(None, 2)[1])
except IndexError: # split failed
end = len(text)
assert lexer.error_till_line_end(0, text) == end
assert lexer.cur[0] == (0, Error, text[:nl])
# full statement, including rule (because this can't be tested any other way)
@pytest.mark.parametrize('text', ('[ push', '] pop'))
def test_fantasy_positive_matches(lexer, text):
"""Test statements that should be tokenized as push/pop statements."""
assert lexer.get_tokens_unprocessed(text)[0] == (0, Keyword, text[0])
# full text is already done by examplefiles, but here's some exceptions
@pytest.mark.parametrize('text', (
'0', 'a=b', 'premise',
'0 a=b premise', '1 b=a symmetry (line 0)'
))
def test_no_crashing(lexer, text):
"""Test incomplete text fragments that shouldn't crash the whole lexer."""
assert lexer.get_tokens(text)
|