1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
|
# Copyright © 2016-2021 Jakub Wilk <jwilk@jwilk.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the “Software”), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import io
from tests.tools import (
assert_equal,
assert_is,
assert_is_instance,
testcase,
)
import lib.parser as M
def t(s, exp):
if isinstance(exp, tuple):
exp = [exp]
file = io.StringIO(s)
result = list(M.parse_file(file))
assert_equal(len(result), len(exp))
for (loc, art, word), (xi, xart, xword) in zip(result, exp):
assert_is_instance(loc, M.Location)
assert_is(loc.file, file)
assert_equal(loc.lineno, xi)
assert_equal(xart, art)
assert_equal(xword, word)
@testcase
def test_mid_line():
t('Oh, yeah, an African swallow maybe,\nbut not a European swallow.\n', [
(1, 'an', 'African'),
(2, 'a', 'European'),
])
@testcase
def test_wrapped():
t('I thought we were an\nautonomous collective.', (2, 'an', 'autonomous'))
@testcase
def test_quotes():
t(
"a 'scratch'\n"
'a ‘scratch’\n'
'a "scratch"\n'
'a “scratch”\n',
[(i, 'a', 'scratch') for i in range(1, 5)]
)
@testcase
def test_underscore():
t('a European_swallow', (1, 'a', 'European'))
@testcase
def test_numbers():
t('an 8', (1, 'an', '8'))
t('an 8bit', (1, 'an', '8'))
t('an 8-bit', (1, 'an', '8'))
t('an 8 bit', (1, 'an', '8'))
t('a UTF16', (1, 'a', 'UTF'))
t('a UTF-16', (1, 'a', 'UTF'))
t('a UTF 16', (1, 'a', 'UTF'))
@testcase
def test_inner_apostrophe():
t("Esra'a Al Shafei", [])
t('Esra’a Al Shafei', [])
del testcase
# vim:ts=4 sts=4 sw=4 et
|
