1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
! Ripped and simplified from North sámi material: testing that lexer eats
! punctuation properly
Multichar_Symbols +PUNCT
LEXICON Root
,+PUNCT:, # ;
.+PUNCT:. # ;
..+PUNCT:.. # ;
...+PUNCT:... # ;
?+PUNCT:? # ;
…+PUNCT:… # ;
¿+PUNCT:¿ # ;
¶+PUNCT:¶ # ;
§+PUNCT:§ # ;
§§+PUNCT:§§ # ;
$+PUNCT:$ # ;
€+PUNCT:€ # ;
£+PUNCT:£ # ;
®+PUNCT:® # ;
©+PUNCT:© # ;
•+PUNCT:• # ; !U+2022 Bullet
●+PUNCT:● # ; !U+25CF Black circle
·+PUNCT:· # ; !U+0097 Middle dot
ּ+PUNCT:ּ # ; !U+05BC Hebrew (sic!) middle dot
·+PUNCT:· # ; !U+0387, Greek (sic!) middle dot, or rather semicolon, in Greek.
-+PUNCT:- # ; !hyphen
--+PUNCT:-- # ; !double hyphen
–+PUNCT:– # ; !endash
—+PUNCT:— # ; !emdash
―+PUNCT:― # ; !horizontal bar U+2015
=+PUNCT:= # ;
≈+PUNCT:≈ # ;
&+PUNCT:& # ;
'+PUNCT:' # ;
*+PUNCT:* # ;
++PUNCT:+ # ;
±+PUNCT:± # ;
`+PUNCT:` # ;
´+PUNCT:´ # ;
/+PUNCT:/ # ;
+/-+PUNCT:+/- # ;
~+PUNCT:~ # ;
«+PUNCT:« # ;
»+PUNCT:» # ;
“+PUNCT:“ # ;
”+PUNCT:” # ;
‘+PUNCT:‘ # ;
’+PUNCT:’ # ;
„+PUNCT:„ # ;
‹+PUNCT:‹ # ;
›+PUNCT:› # ;
(+PUNCT:( # ;
)+PUNCT:) # ;
[+PUNCT:[ # ;
]+PUNCT:] # ;
{+PUNCT:{ # ;
}+PUNCT:} # ;
! escape required
%"+PUNCT:%" # ;
%!+PUNCT:%! # ;
%<+PUNCT:%< # ;
%>+PUNCT:%> # ;
%%+PUNCT:%% # ;
%:+PUNCT:%: # ;
%;+PUNCT:%; # ;
%#+PUNCT:%# # ;
|
