File: test_generator.rb

package info (click to toggle)
rexical 1.0.8-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 316 kB
  • sloc: ruby: 1,124; makefile: 8; ansic: 5
file content (307 lines) | stat: -rw-r--r-- 6,308 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
gem "minitest"
require 'minitest/autorun'
require 'tempfile'
require 'rexical'
require 'stringio'
require 'open3'

class TestGenerator < Minitest::Test
  def test_header_is_written_after_module
    rex = Rexical::Generator.new(
      "--independent" => true
    )
    rex.grammar_file = File.join File.dirname(__FILE__), 'assets', 'test.rex'
    rex.read_grammar
    rex.parse

    output = StringIO.new
    rex.write_scanner output

    comments = []
    output.string.split(/[\n]/).each do |line|
      comments << line.chomp if line =~ /^#/
    end

    assert_match 'DO NOT MODIFY', comments.join
    assert_equal '#--', comments.first
    assert_equal '#++', comments.last
  end

  def test_rubocop_security
    rex = Rexical::Generator.new(
      "--independent" => true
    )
    rex.grammar_file = File.join File.dirname(__FILE__), 'assets', 'test.rex'
    rex.read_grammar
    rex.parse

    output = Tempfile.new(["rex_output", ".rb"])
    begin
      rex.write_scanner output
      output.close

      #stdin, stdoe, wait_thr = Open3.popen2e "rubocop --only Security #{output.path}"
      #if ! wait_thr.value.success?
      #  fail stdoe.read
      #end
    ensure
      output.close
      output.unlink
    end
  end

  def test_read_non_existent_file
    rex = Rexical::Generator.new(nil)
    rex.grammar_file = 'non_existent_file'
    assert_raises Errno::ENOENT do
      rex.read_grammar
    end
  end

  def test_scanner_nests_classes
    source = parse_lexer %q{
module Foo
class Baz::Calculator < Bar
rule
  \d+       { [:NUMBER, text.to_i] }
  \s+       { [:S, text] }
end
end
    }

    assert_match 'Baz::Calculator < Bar', source
  end

  def test_scanner_inherits
    source = parse_lexer %q{
class Calculator < Bar
rule
  \d+       { [:NUMBER, text.to_i] }
  \s+       { [:S, text] }
end
    }

    assert_match 'Calculator < Bar', source
  end

  def test_scanner_inherits_many_levels
    source = parse_lexer %q{
class Calculator < Foo::Bar
rule
  \d+       { [:NUMBER, text.to_i] }
  \s+       { [:S, text] }
end
    }

    assert_match 'Calculator < Foo::Bar', source
  end

  def test_stateful_lexer
    m = build_lexer %q{
class Foo
rule
          \d      { @state = :digit; [:foo, text] }
  :digit  \w      { @state = nil; [:w, text] }
end
    }
    scanner = m::Foo.new
    scanner.scan_setup('1w1')
    assert_tokens [
      [:foo, '1'],
      [:w, 'w'],
      [:foo, '1']], scanner
  end

  def test_simple_scanner
    m = build_lexer %q{
class Calculator
rule
  \d+       { [:NUMBER, text.to_i] }
  \s+       { [:S, text] }
end
    }

    calc = m::Calculator.new
    calc.scan_setup('1 2 10')

    assert_tokens [[:NUMBER, 1],
                  [:S, ' '],
                  [:NUMBER, 2],
                  [:S, ' '],
                  [:NUMBER, 10]], calc
  end

  def test_simple_scanner_with_empty_action
    m = build_lexer %q{
class Calculator
rule
  \d+       { [:NUMBER, text.to_i] }
  \s+       # skips whitespaces
end
    }

    calc = m::Calculator.new
    calc.scan_setup('1 2 10')

    assert_tokens [[:NUMBER, 1],
                  [:NUMBER, 2],
                  [:NUMBER, 10]], calc
  end

  def test_parses_macros_with_escapes
    source = parse_lexer %q{
class Foo
macro
  w  [\ \t]+
rule
  {w}  { [:SPACE, text] }
end
    }

    assert source.index('@ss.scan(/[ \t]+/))')
  end

  def test_simple_scanner_with_macros
    m = build_lexer %q{
class Calculator
macro
  digit     \d+
rule
  {digit}       { [:NUMBER, text.to_i] }
  \s+       { [:S, text] }
end
    }

    calc = m::Calculator.new
    calc.scan_setup('1 2 10')

    assert_tokens [[:NUMBER, 1],
                  [:S, ' '],
                  [:NUMBER, 2],
                  [:S, ' '],
                  [:NUMBER, 10]], calc
  end

  def test_nested_macros
    source = parse_lexer %q{
class Calculator
macro
  nonascii  [^\0-\177]
  string    "{nonascii}*"
rule
  {string}       { [:STRING, text] }
end
    }

    assert_match '"[^\0-\177]*"', source
  end

  def test_more_nested_macros
    source = parse_lexer %q{
class Calculator
macro
  nonascii  [^\0-\177]
  sing      {nonascii}*
  string    "{sing}"
rule
  {string}       { [:STRING, text] }
end
    }

    assert_match '"[^\0-\177]*"', source
  end

  def test_changing_state_during_lexing
    lexer = build_lexer %q{
class Calculator
rule
       a       { self.state = :B  ; [:A, text] }
  :B   b       { self.state = nil ; [:B, text] }
end
    }

    calc1 = lexer::Calculator.new
    calc2 = lexer::Calculator.new
    calc1.scan_setup('aaaaa')
    calc2.scan_setup('ababa')

    # Doesn't lex all 'a's
    assert_raises(lexer::Calculator::ScanError) { tokens(calc1) }

    # Does lex alternating 'a's and 'b's
    calc2.scan_setup('ababa')

    assert_tokens [[:A, 'a'],
                   [:B, 'b'],
                   [:A, 'a'],
                   [:B, 'b'],
                   [:A, 'a']], calc2
  end

  def test_changing_state_is_possible_between_next_token_calls
    lexer = build_lexer %q{
class Calculator
rule
       a       { [:A, text] }
  :B   b       { [:B, text] }
end
    }

    calc = lexer::Calculator.new
    calc.scan_setup('ababa')

    assert_equal [:A, 'a'], calc.next_token
    calc.state = :B
    assert_equal [:B, 'b'], calc.next_token
    calc.state = nil
    assert_equal [:A, 'a'], calc.next_token
    calc.state = :B
    assert_equal [:B, 'b'], calc.next_token
    calc.state = nil
    assert_equal [:A, 'a'], calc.next_token
  end
  def test_match_eos
    lexer = build_lexer %q{
class Calculator
option
matcheos
rule
      a        { [:A, text] }
      $        { [:EOF, ""] }
:B    b        { [:B, text] }
     }
     calc = lexer::Calculator.new
     calc.scan_setup("a")
     assert_equal [:A, 'a'], calc.next_token
     assert_equal [:EOF, ""], calc.next_token
  end

  def parse_lexer(str)
    rex = Rexical::Generator.new("--independent" => true)
    out = StringIO.new

    rex.grammar_lines = StringScanner.new(str)
    rex.parse
    rex.write_scanner(out)

    out.string
  end

  def build_lexer(str)
    mod = Module.new
    mod.module_eval(parse_lexer(str))
    mod
  end

  def tokens(scanner)
    tokens = []
    while token = scanner.next_token
      tokens << token
    end
    tokens
  end

  def assert_tokens(expected, scanner)
    assert_equal expected, tokens(scanner)
  end
end