1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
|
[RULE-ORDER]
GENITIVEPLACENAME WORD-WITHSUFFIX WORD-INFIX-COMPOUND QUOTE-SUFFIX
WORD-TOKEN ABBREVIATION-KNOWN NUMBER-ORDINAL DATE-REVERSE DATE
URL URL-WWW URL-DOMAIN E-MAIL
ABBREVIATION-COMPOUND COMPOUND-ABBREVIATION
QUOTE-COMPOUND WORD-COMPOUND NUMBER-COMPOUND
NUMBER-STRING STRING-NUMBER
WORD-PARPREFIX-PARSUFFIX WORD-PARPREFIX WORD-PARSUFFIX
ABBREVIATION INITIALS INITIAL SMILEY REVERSE-SMILEY
PUNCTUATION-MULTI
NUMBER-YEAR TIME FRACNUMBER NUMBER CURRENCY WORD PUNCTUATION UNKNOWN
# to do PREFIXES (is leeg nu) UNITS (uitgecommentarieerd in de c++ code)
[META-RULES]
SPLITTER=%
NUMBER-ORDINAL = \p{N}+-?(?: %ORDINALS% )(?:\Z|\P{Lu}|\P{Ll})$
ABBREVIATION-KNOWN = (?:\p{P}*)?(?:\A|[^\p{L}\.])((?:%ABBREVIATIONS%)\.)(?:\Z|\P{L})
WORD-TOKEN =(%TOKENS%)(?:\p{P}*)?$
WORD-WITHPREFIX = (?:\A|[^\p{Lu}\.]|[^\p{Ll}\.])(?: %ATTACHEDPREFIXES% )\p{L}+
WORD-WITHSUFFIX = ((?:\p{L}|\p{N}|-)+(?: %ATTACHEDSUFFIXES% ))(?:\Z)
WORD-INFIX-COMPOUND = ((?:\p{L}|\p{N}|-)+(?: %ATTACHEDSUFFIXES% )-(?:\p{L}+))$
PREFIX = (?:\A|[^\p{Lu}\.]|[^\p{Ll}\.])(%PREFIXES% )(\p{L}+)
SUFFIX = ((?:\p{L})+)( %SUFFIXES% )(?:\Z|\P{L})
[RULES]
%include url
%include e-mail
%include smiley
#Ex (oud)-studente(s)
WORD-PARPREFIX-PARSUFFIX=(?:\p{Ps}\p{L}+[\p{Pc}\p{Pd}]?\p{Pe}[\p{Pc}\p{Pd}]?)\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*(?:[\p{Pc}\p{Pd}]?\p{Ps}[\p{Pc}\p{Pd}]?\p{L}+\p{Pe})
#Ex: (oud)-studente, (on)zin,
WORD-PARPREFIX=(?:\p{Ps}\p{L}+[\p{Pc}\p{Pd}]?\p{Pe}[\p{Pc}\p{Pd}]?)\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*
#Ex: koning(in)
WORD-PARSUFFIX=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*(?:[\p{Pc}\p{Pd}]?\p{Ps}[\p{Pc}\p{Pd}]?\p{L}+\p{Pe})
#Keep dash/underscore connected parts (even if they are in parenthesis)
WORD-COMPOUND=[\p{L}\p{N}]+(?:[\p{Pc}\p{Pd}][\p{L}\p{N}]+)+
#Keep quote connected parts.
QUOTE-COMPOUND=[\p{L}\p{N}-]+(?:['`’‘]\p{L}{1})+(?:-[\p{L}\p{N}]+)+
#Abbreviations with multiple periods: a.u.b
ABBREVIATION=^(\p{L}{1,3}(?:\.\p{L}{1,3})+\.?)(?:\Z|[,:;])
# a compound of a word and an abbreviation with multiple periods: bouw-c.a.o.
COMPOUND-ABBREVIATION=^(?:\p{L}+-\p{L}{1,3}(?:\.\p{L}{1,3})+\.?)\Z
# a compound of an abbreviation with multiple periods and a word: C.B.S.-data
ABBREVIATION-COMPOUND=^(\p{L}{1,3}(?:\.\p{L}{1,3})+\.?-\p{L}+)
#Initials glued to a longer word: A.F.Zetterij -> A.F. Zetterij
INITIALS=(\p{L}(?:\.\p{L})+\.)\p{Lu}\p{L}{3,999}+
#retain initials
INITIAL=^(?:\p{Lt}|\p{Lu})\.$
#Homogeneous punctuation (ellipsis etc)
PUNCTUATION-MULTI=(?:\.|\-|[!\?]){2,}
#Date
DATE=\p{N}{1,2}-\p{N}{1,2}-\p{N}{2,4}
DATE-REVERSE=\p{N}{4}-\p{N}{1,2}-\p{N}{1,2}
FRACNUMBER=\p{N}+(?:/\p{N}+)+
#tel numberstring (23)3748748 or (+31)9393902
NUMBER-COMPOUND=\(\+{0,1}\p{N}{1,3}\)(?:\p{N}+)
#numberstring like 20jarige
NUMBER-STRING=^\p{N}+(?:\p{Pd}?)(?:\p{L}+)
#combinations like A50, a1
STRING-NUMBER=\p{L}+\p{Pd}?\p{N}+
#abbreviated year: '40
NUMBER-YEAR=(['`’‘]\p{N}{2})(?:\P{N}|\z)
#NUMBER-YEAR=('\p{N}{2})\P{N}
#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N}{1,2})?(am|pm|AM|PM)?
#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+
#'s-Gravenhage, 's-Hertogenbosch
GENITIVEPLACENAME=['`’‘][sS]-\p{Lu}\p{L}+
CURRENCY=\p{Sc}
WORD=[\p{L}\p{Mn}\-]+
PUNCTUATION=\p{P}
UNKNOWN=.
QUOTE-SUFFIX=(\p{Lu}(?:\p{L})+(?:s['`’‘]|x['`'‘]))
[PREFIXES]
[ATTACHEDSUFFIXES]
\['`’‘][sS]
\['`’‘][tT]
\['`’‘][nN]
\['`’‘][rR]
[ORDINALS]
e|E
de|DE
ste|STE
er|ER
[TOKENS]
\['`’‘][sS]
\['`’‘][kK]
\['`’‘][mM]
\['`’‘][nN]
\['`’‘][tT]
\['`’‘](?:ie|IE)
[UNITS]
km
m
cm
mm
g
kg
C
l
s
sec
min
gb
mb
kb
[CURRENCY]
eur|EUR
hfl
fl
f
[ABBREVIATIONS]
%include nl_afk
[FILTER]
%include ligatures
# replace all 'ticks'
#‘ '
#’ '
#` '
# also remove soft hyphen
\u00AD
[EOSMARKERS]
%include standard-eos
[QUOTES]
%include standard-quotes
%include exotic-quotes
|
