1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
|
# -*- coding: utf-8 -*-
# This work is licensed under the MIT License.
# To view a copy of this license, visit https://opensource.org/licenses/MIT
# Written by Abdullah Diab (mpcabd)
# Email: mpcabd@gmail.com
# Website: http://mpcabd.xyz
import re
from itertools import repeat
from .ligatures import LIGATURES
from .reshaper_config import auto_config
from .letters import (UNSHAPED, ISOLATED, TATWEEL, ZWJ, LETTERS_ARABIC,
LETTERS_ARABIC_V2, LETTERS_KURDISH, FINAL,
INITIAL, MEDIAL, connects_with_letters_before_and_after,
connects_with_letter_before, connects_with_letter_after)
HARAKAT_RE = re.compile(
'['
'\u0610-\u061a'
'\u064b-\u065f'
'\u0670'
'\u06d6-\u06dc'
'\u06df-\u06e8'
'\u06ea-\u06ed'
'\u08d4-\u08e1'
'\u08d4-\u08ed'
'\u08e3-\u08ff'
']',
re.UNICODE | re.X
)
class ArabicReshaper(object):
"""
A class for Arabic reshaper, it allows for fine-tune configuration over the
API.
If no configuration is passed to the constructor, the class will check for
an environment variable :envvar:`PYTHON_ARABIC_RESHAPER_CONFIGURATION_FILE`
, if the variable is available, the class will load the file pointed to by
the variable, and will read it as an ini file.
If the variable doesn't exist, the class will load with the default
configuration file :file:`default-config.ini`
Check these links for information on the configuration files format:
* Python 3: https://docs.python.org/3/library/configparser.html
See the default configuration file :file:`default-config.ini` for details
on how to configure your reshaper.
"""
def __init__(self, configuration=None, configuration_file=None):
super(ArabicReshaper, self).__init__()
self.configuration = auto_config(configuration, configuration_file)
self.language = self.configuration.get('language')
if self.language == 'ArabicV2':
self.letters = LETTERS_ARABIC_V2
elif self.language == 'Kurdish':
self.letters = LETTERS_KURDISH
else:
self.letters = LETTERS_ARABIC
@property
def _ligatures_re(self):
if not hasattr(self, '__ligatures_re'):
patterns = []
re_group_index_to_ligature_forms = {}
index = 0
FORMS = 1
MATCH = 0
for ligature_record in LIGATURES:
ligature, replacement = ligature_record
if not self.configuration.getboolean(ligature):
continue
re_group_index_to_ligature_forms[index] = replacement[FORMS]
patterns.append('({})'.format(replacement[MATCH]))
index += 1
self._re_group_index_to_ligature_forms = (
re_group_index_to_ligature_forms
)
self.__ligatures_re = re.compile('|'.join(patterns), re.UNICODE)
return self.__ligatures_re
def _get_ligature_forms_from_re_group_index(self, group_index):
if not hasattr(self, '_re_group_index_to_ligature_forms'):
return self._ligatures_re
return self._re_group_index_to_ligature_forms[group_index]
def reshape(self, text):
if not text:
return ''
output = []
LETTER = 0
FORM = 1
NOT_SUPPORTED = -1
delete_harakat = self.configuration.getboolean('delete_harakat')
delete_tatweel = self.configuration.getboolean('delete_tatweel')
support_zwj = self.configuration.getboolean('support_zwj')
shift_harakat_position = self.configuration.getboolean(
'shift_harakat_position'
)
use_unshaped_instead_of_isolated = self.configuration.getboolean(
'use_unshaped_instead_of_isolated'
)
positions_harakat = {}
isolated_form = (UNSHAPED
if use_unshaped_instead_of_isolated else ISOLATED)
for letter in text:
if HARAKAT_RE.match(letter):
if not delete_harakat:
position = len(output) - 1
if shift_harakat_position:
position -= 1
if position not in positions_harakat:
positions_harakat[position] = []
if shift_harakat_position:
positions_harakat[position].insert(0, letter)
else:
positions_harakat[position].append(letter)
elif letter == TATWEEL and delete_tatweel:
pass
elif letter == ZWJ and not support_zwj:
pass
elif letter not in self.letters:
output.append((letter, NOT_SUPPORTED))
elif not output: # first letter
output.append((letter, isolated_form))
else:
previous_letter = output[-1]
if previous_letter[FORM] == NOT_SUPPORTED:
output.append((letter, isolated_form))
elif not connects_with_letter_before(letter, self.letters):
output.append((letter, isolated_form))
elif not connects_with_letter_after(
previous_letter[LETTER], self.letters):
output.append((letter, isolated_form))
elif (previous_letter[FORM] == FINAL and not
connects_with_letters_before_and_after(
previous_letter[LETTER], self.letters
)):
output.append((letter, isolated_form))
elif previous_letter[FORM] == isolated_form:
output[-1] = (
previous_letter[LETTER],
INITIAL
)
output.append((letter, FINAL))
# Otherwise, we will change the previous letter to connect
# to the current letter
else:
output[-1] = (
previous_letter[LETTER],
MEDIAL
)
output.append((letter, FINAL))
# Remove ZWJ if it's the second to last item as it won't be useful
if support_zwj and len(output) > 1 and output[-2][LETTER] == ZWJ:
output.pop(len(output) - 2)
if support_zwj and output and output[-1][LETTER] == ZWJ:
output.pop()
if self.configuration.getboolean('support_ligatures'):
# Clean text from Harakat to be able to find ligatures
text = HARAKAT_RE.sub('', text)
# Clean text from Tatweel to find ligatures if delete_tatweel
if delete_tatweel:
text = text.replace(TATWEEL, '')
for match in re.finditer(self._ligatures_re, text):
group_index = next((
i for i, group in enumerate(match.groups()) if group
), -1)
forms = self._get_ligature_forms_from_re_group_index(
group_index
)
a, b = match.span()
a_form = output[a][FORM]
b_form = output[b - 1][FORM]
ligature_form = None
# +-----------+----------+---------+---------+----------+
# | a \ b | ISOLATED | INITIAL | MEDIAL | FINAL |
# +-----------+----------+---------+---------+----------+
# | ISOLATED | ISOLATED | INITIAL | INITIAL | ISOLATED |
# | INITIAL | ISOLATED | INITIAL | INITIAL | ISOLATED |
# | MEDIAL | FINAL | MEDIAL | MEDIAL | FINAL |
# | FINAL | FINAL | MEDIAL | MEDIAL | FINAL |
# +-----------+----------+---------+---------+----------+
if a_form in (isolated_form, INITIAL):
if b_form in (isolated_form, FINAL):
ligature_form = ISOLATED
else:
ligature_form = INITIAL
else:
if b_form in (isolated_form, FINAL):
ligature_form = FINAL
else:
ligature_form = MEDIAL
if not forms[ligature_form]:
continue
output[a] = (forms[ligature_form], NOT_SUPPORTED)
output[a+1:b] = repeat(('', NOT_SUPPORTED), b - 1 - a)
result = []
if not delete_harakat and -1 in positions_harakat:
result.extend(positions_harakat[-1])
for i, o in enumerate(output):
if o[LETTER]:
if o[FORM] == NOT_SUPPORTED or o[FORM] == UNSHAPED:
result.append(o[LETTER])
else:
result.append(self.letters[o[LETTER]][o[FORM]])
if not delete_harakat:
if i in positions_harakat:
result.extend(positions_harakat[i])
return ''.join(result)
default_reshaper = ArabicReshaper()
reshape = default_reshaper.reshape
|
