1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
# -*- coding: utf-8 -*-
#
# Copyright (c) 2009, 2010, 2013, 2021 kaliko <kaliko@azylum.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with this program.
# If not, see <http://www.gnu.org/licenses/>.
#
r"""
SimaStr
Special unicode() subclass to perform fuzzy match on specific strings with
known noise.
* SimaStr() object removes specific patterns from the string
* Diacritic are removed
* Equality test is done on lower-cased string
* Equality test is not an exact comparison, the levenshtein edition distance
between stripped and filtered strings is used
>>> from simastr import SimaStr
>>> art0 = SimaStr('The Desert Sessions & PJ Harvey')
>>> art1 = SimaStr('Desert Sessions And PJ Harvey')
>>> art0 == art1
>>> True
>>> art0 == 'Desert Sessions And PJ Harvey'
>>> True
>>> # diacritic filter + levenshtein example
>>> art0 = sima.lib.simastr.SimaStr('Hubert Félix Thiéphaine')
>>> art1 = sima.lib.simastr.SimaStr('Hubert-Felix Thiephaine')
>>> art0 == art1
>>> True
>>>
Current stripped word patterns (usually English followed by French and
Spanish alternatives)
leading (case-insensitive):
"the","le","la","les","el","los"
middle:
"[Aa]nd","&","[Nn]'?","[Ee]t"
trailing:
combination of "[- !?\.]+" "\(? ?[Ll]ive ?\)?"
Possibility to access to stripped string:
>>> art0 = SimaStr('The Desert Sessions & PJ Harvey')
>>> print (art0, art0.stripped)
>>> ('The Desert Sessions & PJ Harvey', 'Desert Sessions PJ Harvey')
TODO:
* Have a look to difflib.SequenceMatcher to find possible improvements
* Find a way to allow users patterns.
"""
__author__ = 'Jack Kaliko'
__version__ = '0.4'
# IMPORTS
import unicodedata
from re import compile as re_compile, U, I
from ..utils.leven import levenshtein_ratio
class SimaStr(str):
"""
Specific string object for artist names and song titles.
Here follows some class variables for regex to run on strings.
"""
diafilter = True
leven_ratio = 0.82
regexp_dict = {}
# Leading patterns: The Le Les
# case-insensitive matching for this RE
regexp_dict.update({'lead': '(the|l[ae][s]?|los|el)'})
# Middle patterns: And & Et N
regexp_dict.update({'mid': '(And|&|and|[Nn]\'?|et)'})
# Trailing patterns: ! ? live
# TODO: add "concert" key word
# add "Live at <somewhere>"
regexp_dict.update({'trail': r'([- !?\.]|\(? ?[Ll]ive ?\)?)'})
reg_lead = re_compile('^(?P<lead>%(lead)s )(?P<root0>.*)$' % regexp_dict, I | U)
reg_midl = re_compile('^(?P<root0>.*)(?P<mid> %(mid)s )(?P<root1>.*)' % regexp_dict, U)
reg_trail = re_compile('^(?P<root0>.*?)(?P<trail>%(trail)s+$)' % regexp_dict, U)
def __init__(self, fuzzstr):
"""
"""
super().__init__()
self.orig = str(fuzzstr)
self.stripped = str(fuzzstr.strip())
# fuzzy computation
self._get_root()
if self.__class__.diafilter:
self.remove_diacritics()
def __new__(cls, fuzzstr):
return super(SimaStr, cls).__new__(cls, fuzzstr)
def _get_root(self):
"""
Remove all patterns in string.
"""
sea = SimaStr.reg_lead.search(self.stripped)
if sea:
self.stripped = sea.group('root0')
sea = SimaStr.reg_midl.search(self.stripped)
if sea:
self.stripped = str().join([sea.group('root0'), ' ',
sea.group('root1')])
sea = SimaStr.reg_trail.search(self.stripped)
if sea:
self.stripped = sea.group('root0')
def remove_diacritics(self):
"""converting diacritics"""
self.stripped = ''.join(x for x in
unicodedata.normalize('NFKD', self.stripped)
if unicodedata.category(x) != 'Mn')
def __hash__(self):
return hash(self.stripped)
def __eq__(self, other):
if not isinstance(other, SimaStr):
other = SimaStr(other)
levenr = levenshtein_ratio(self.stripped.lower(),
other.stripped.lower())
if hash(self) == hash(other):
return True
return levenr >= self.__class__.leven_ratio
def __ne__(self, other):
if not isinstance(other, SimaStr):
return hash(self) != hash(SimaStr(other))
return hash(self) != hash(other)
# VIM MODLINE
# vim: ai ts=4 sw=4 sts=4 expandtab
|
