1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
|
# Skipped because we have no idea where all those fixtures originate
import pytest
pytestmark = pytest.mark.skip
import pprint
import ruyaml
# Tokens mnemonic:
# directive: %
# document_start: ---
# document_end: ...
# alias: *
# anchor: &
# tag: !
# scalar _
# block_sequence_start: [[
# block_mapping_start: {{
# block_end: ]}
# flow_sequence_start: [
# flow_sequence_end: ]
# flow_mapping_start: {
# flow_mapping_end: }
# entry: ,
# key: ?
# value: :
_replaces = {
ruyaml.DirectiveToken: '%',
ruyaml.DocumentStartToken: '---',
ruyaml.DocumentEndToken: '...',
ruyaml.AliasToken: '*',
ruyaml.AnchorToken: '&',
ruyaml.TagToken: '!',
ruyaml.ScalarToken: '_',
ruyaml.BlockSequenceStartToken: '[[',
ruyaml.BlockMappingStartToken: '{{',
ruyaml.BlockEndToken: ']}',
ruyaml.FlowSequenceStartToken: '[',
ruyaml.FlowSequenceEndToken: ']',
ruyaml.FlowMappingStartToken: '{',
ruyaml.FlowMappingEndToken: '}',
ruyaml.BlockEntryToken: ',',
ruyaml.FlowEntryToken: ',',
ruyaml.KeyToken: '?',
ruyaml.ValueToken: ':',
}
def test_tokens(data_filename, tokens_filename, verbose=False):
tokens1 = []
with open(tokens_filename, 'r') as fp:
tokens2 = fp.read().split()
try:
yaml = ruyaml.YAML(typ='unsafe', pure=True)
with open(data_filename, 'rb') as fp1:
for token in yaml.scan(fp1):
if not isinstance(
token, (ruyaml.StreamStartToken, ruyaml.StreamEndToken)
):
tokens1.append(_replaces[token.__class__])
finally:
if verbose:
print('TOKENS1:', ' '.join(tokens1))
print('TOKENS2:', ' '.join(tokens2))
assert len(tokens1) == len(tokens2), (tokens1, tokens2)
for token1, token2 in zip(tokens1, tokens2):
assert token1 == token2, (token1, token2)
test_tokens.unittest = ['.data', '.tokens']
def test_scanner(data_filename, canonical_filename, verbose=False):
for filename in [data_filename, canonical_filename]:
tokens = []
try:
yaml = ruyaml.YAML(typ='unsafe', pure=False)
with open(filename, 'rb') as fp:
for token in yaml.scan(fp):
tokens.append(token.__class__.__name__)
finally:
if verbose:
pprint.pprint(tokens)
test_scanner.unittest = ['.data', '.canonical']
if __name__ == '__main__':
import test_appliance
test_appliance.run(globals())
|
