1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
|
"""
This is a Pure Python module to hyphenate text.
It is inspired by Ruby's Text::Hyphen, but currently reads standard *.dic files,
that must be installed separately.
In the future it's maybe nice if dictionaries could be distributed together with
this module, in a slightly prepared form, like in Ruby's Text::Hyphen.
Wilbert Berendsen, March 2008
info@wilbertberendsen.nl
License: LGPL. More info: http://python-hyphenator.googlecode.com/
"""
from __future__ import print_function
try:
chr = unichr
except NameError:
pass
import codecs
import re
__all__ = ["Hyphenator"]
# cache of per-file HyphenationDictionary objects
_hdcache = {}
# precompile some regular expressions
parse = re.compile(r'(\d?)(\D?)').findall
# Match ^^xx where xx is a two-digit hexadecimal value
_hex_re = re.compile(r'\^{2}([0-9a-f]{2})')
# replace the matched hex string with the corresponding unicode character
_hex_repl = lambda matchObj: chr(int(matchObj.group(1), 16))
def replace_hex(text):
"""Replaces ^^xx (where xx is a two-digit hexadecimal value) occurrences
by the corresponding unicode character.
"""
return _hex_re.sub(_hex_repl, text)
class ParsedAlternative(object):
"""Parse nonstandard hyphen pattern alternative.
when called with an odd value, the instance returns an integer with data
attribute (DataInt) about the current position in the pattern.
"""
def __init__(self, pat, alt):
alt = alt.split(',')
self.change = alt[0]
if len(alt) > 2:
self.index = int(alt[1])
self.cut = int(alt[2]) + 1
else:
self.index = 1
self.cut = len(re.sub(r'[\d\.]', '', pat)) + 1
if pat.startswith('.'):
self.index += 1
def __call__(self, val):
self.index -= 1
val = int(val)
if val & 1:
return DataInt(val, (self.change, self.index, self.cut))
else:
return val
class DataInt(int):
"""An integer with a data attribute.
Just an int some other data can be stuck to in a data attribute.
Instantiate with ref=other to use the data from the other DataInt.
"""
def __new__(cls, value, data=None, ref=None):
obj = int.__new__(cls, value)
if ref and type(ref) is DataInt:
obj.data = ref.data
else:
obj.data = data
return obj
class HyphenationDictionary(object):
"""Reads a hyph_*.dic file and stores the hyphenation patterns.
Parameters:
filename : filename of hyph_*.dic pattern file to read
"""
def __init__(self, filename):
self.patterns = {}
with open(filename, 'rb') as f:
# use correct encoding, specified in first line
for encoding in f.readline().split():
if encoding != b"charset":
try:
decoder = codecs.getreader(encoding.decode('ascii'))
break
except LookupError:
pass
else:
decoder = codecs.getreader('latin1')
for pat in decoder(f):
pat = pat.strip()
if not pat or pat[0] == '%':
continue
# replace ^^hh with the real character
pat = replace_hex(pat)
# read nonstandard hyphen alternatives
if '/' in pat:
pat, alt = pat.split('/', 1)
factory = ParsedAlternative(pat, alt)
else:
factory = int
tag, values = zip(*[(s, factory(i or "0"))
for i, s in parse(pat)])
# if only zeros, skip this pattern
if any(values):
# strip zeros and store start offset.
start, end = 0, len(values)
while not values[start]:
start += 1
while not values[end-1]:
end -= 1
self.patterns[''.join(tag)] = start, values[start:end]
self.cache = {}
self.maxlen = max(map(len, self.patterns))
def positions(self, word):
"""Returns a list of positions where the word can be hyphenated.
E.g. for the dutch word 'lettergrepen' this method returns
the list [3, 6, 9].
Each position is a 'data int' (DataInt) with a data attribute.
If the data attribute is not None, it contains a tuple with
information about nonstandard hyphenation at that point:
(change, index, cut)
change: is a string like 'ff=f', that describes how hyphenation
should take place.
index: where to substitute the change, counting from the current
point
cut: how many characters to remove while substituting the nonstandard
hyphenation
"""
word = word.lower()
try:
return self.cache[word]
except KeyError:
pass
prepWord = '.' + word + '.'
res = [0] * (len(prepWord) + 1)
for i in range(len(prepWord) - 1):
for j in range(i + 1, min(i + self.maxlen, len(prepWord)) + 1):
p = self.patterns.get(prepWord[i:j])
if p:
offset, values = p
s = slice(i + offset, i + offset + len(values))
res[s] = map(max, values, res[s])
positions = [DataInt(i - 1, ref=r) for i, r in enumerate(res) if r % 2]
self.cache[word] = positions
return positions
class Hyphenator(object):
"""Reads a hyph_*.dic file and stores the hyphenation patterns.
Provides methods to hyphenate strings in various ways.
Parameters:
-filename : filename of hyph_*.dic to read
-left: make the first syllable not shorter than this
-right: make the last syllable not shorter than this
-cache: if true (default), use a cached copy of the dic file, if possible
left and right may also later be changed:
h = Hyphenator(file)
h.left = 1
"""
def __init__(self, filename, left=2, right=2, cache=True):
self.left = left
self.right = right
if not cache or filename not in _hdcache:
_hdcache[filename] = HyphenationDictionary(filename)
self.hd = _hdcache[filename]
def positions(self, word):
"""Returns a list of positions where the word can be hyphenated.
See also HyphenationDictionary.positions. The points that are too far to
the left or right are removed.
"""
right = len(word) - self.right
return [i for i in self.hd.positions(word) if self.left <= i <= right]
def iterate(self, word):
"""Iterate over all hyphenation possibilities, the longest first."""
for p in reversed(self.positions(word)):
if p.data:
# get the nonstandard hyphenation data
change, index, cut = p.data
if word.isupper():
change = change.upper()
c1, c2 = change.split('=')
yield word[:p+index] + c1, c2 + word[p+index+cut:]
else:
yield word[:p], word[p:]
def wrap(self, word, width, hyphen='-'):
"""Returns the longest possible first part and the last part of the
hyphenated word.
The first part has the hyphen already attached. Returns None, if there
is no hyphenation point before width, or if the word could not be
hyphenated.
"""
width -= len(hyphen)
for w1, w2 in self.iterate(word):
if len(w1) <= width:
return w1 + hyphen, w2
def inserted(self, word, hyphen='-'):
"""Returns the word as a string with all the possible hyphens inserted.
E.g. for the dutch word 'lettergrepen' this method returns the string
'let-ter-gre-pen'. The hyphen string to use can be given as the second
parameter, that defaults to '-'.
"""
l = list(word)
for p in reversed(self.positions(word)):
if p.data:
# get the nonstandard hyphenation data
change, index, cut = p.data
if word.isupper():
change = change.upper()
l[p + index : p + index + cut] = change.replace('=', hyphen)
else:
l.insert(p, hyphen)
return ''.join(l)
__call__ = iterate
if __name__ == "__main__":
import sys
dict_file = sys.argv[1]
word = sys.argv[2]
if not isinstance(word, str):
import locale
word = word.decode(locale.getpreferredencoding())
h = Hyphenator(dict_file, left=1, right=1)
for i in h(word):
print(" \u2013 ".join(i))
|
