1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
|
require "spec"
require "csv"
class CSV::Lexer
def expect_cell(value, file = __FILE__, line = __LINE__)
token = next_token
token.kind.should eq(CSV::Token::Kind::Cell), file: file, line: line
token.value.should eq(value), file: file, line: line
end
def expect_eof(file = __FILE__, line = __LINE__)
next_token.kind.should eq(CSV::Token::Kind::Eof), file: file, line: line
end
def expect_newline(file = __FILE__, line = __LINE__)
next_token.kind.should eq(CSV::Token::Kind::Newline), file: file, line: line
end
end
describe CSV do
describe "lex" do
it "lexes two columns" do
lexer = CSV::Lexer.new("hello,world")
lexer.expect_cell "hello"
lexer.expect_cell "world"
lexer.expect_eof
end
it "lexes two columns with two rows" do
lexer = CSV::Lexer.new("hello,world\nfoo,bar")
lexer.expect_cell "hello"
lexer.expect_cell "world"
lexer.expect_newline
lexer.expect_cell "foo"
lexer.expect_cell "bar"
lexer.expect_eof
end
it "lexes two columns with two rows with \r\n" do
lexer = CSV::Lexer.new("hello,world\r\nfoo,bar")
lexer.expect_cell "hello"
lexer.expect_cell "world"
lexer.expect_newline
lexer.expect_cell "foo"
lexer.expect_cell "bar"
lexer.expect_eof
end
it "lexes two empty columns" do
lexer = CSV::Lexer.new(",")
lexer.expect_cell ""
lexer.expect_cell ""
lexer.expect_eof
end
it "lexes last empty column" do
lexer = CSV::Lexer.new("foo,")
lexer.expect_cell "foo"
lexer.expect_cell ""
lexer.expect_eof
end
it "lexes with empty columns" do
lexer = CSV::Lexer.new("foo,,bar")
lexer.expect_cell "foo"
lexer.expect_cell ""
lexer.expect_cell "bar"
lexer.expect_eof
end
it "lexes with whitespace" do
lexer = CSV::Lexer.new(" foo , bar ")
lexer.expect_cell " foo "
lexer.expect_cell " bar "
lexer.expect_eof
end
it "lexes two with quotes" do
lexer = CSV::Lexer.new(%("hello","world"))
lexer.expect_cell "hello"
lexer.expect_cell "world"
lexer.expect_eof
end
it "lexes two with inner quotes" do
lexer = CSV::Lexer.new(%("hel""lo","wor""ld"))
lexer.expect_cell %(hel"lo)
lexer.expect_cell %(wor"ld)
lexer.expect_eof
end
it "lexes with comma inside quote" do
lexer = CSV::Lexer.new(%("foo,bar"))
lexer.expect_cell "foo,bar"
lexer.expect_eof
end
it "lexes with newline inside quote" do
lexer = CSV::Lexer.new(%("foo\nbar"))
lexer.expect_cell "foo\nbar"
lexer.expect_eof
end
it "lexes newline followed by eof" do
lexer = CSV::Lexer.new("hello,world\n")
lexer.expect_cell "hello"
lexer.expect_cell "world"
lexer.expect_newline
lexer.expect_eof
end
it "lexes with a given separator" do
lexer = CSV::Lexer.new("hello;world\n", separator: ';')
lexer.expect_cell "hello"
lexer.expect_cell "world"
lexer.expect_newline
lexer.expect_eof
end
it "lexes with a given quote char" do
lexer = CSV::Lexer.new("'hello,world'\n", quote_char: '\'')
lexer.expect_cell "hello,world"
lexer.expect_newline
lexer.expect_eof
end
it "raises if single quote in the middle" do
expect_raises CSV::MalformedCSVError, "Unexpected quote at line 1, column 4" do
lexer = CSV::Lexer.new %(hel"lo)
lexer.next_token
end
end
it "raises if command, newline or end doesn't follow quote" do
expect_raises CSV::MalformedCSVError, "Expecting comma, newline or end, not 'a' at line 1, column 6" do
lexer = CSV::Lexer.new %("hel"a)
lexer.next_token
end
end
it "raises on unclosed quote" do
expect_raises CSV::MalformedCSVError, "Unclosed quote at line 1, column 5" do
lexer = CSV::Lexer.new %("foo)
lexer.next_token
end
end
it "doesn't consume char after \\n (#11172)" do
io = IO::Memory.new("a\nx")
lexer = CSV::Lexer.new(io)
lexer.expect_cell "a"
lexer.expect_newline
io.pos.should eq(2)
end
it "doesn't consume char after \\r (#11172)" do
io = IO::Memory.new("a\r\nx")
lexer = CSV::Lexer.new(io)
lexer.expect_cell "a"
io.pos.should eq(2)
lexer.expect_newline
lexer.expect_cell "x"
lexer.expect_eof
end
end
end
|
