1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
#!/usr/bin/env python
# getismsi.py - script to donwload data from Wikipedia to build the database
#
# Copyright (C) 2011, 2013 Arthur de Jong
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA
from collections import defaultdict
import re
import urllib
# URLs that are downloaded
mcc_list_url = 'https://en.wikipedia.org/w/index.php?title=Mobile_country_code&action=raw'
cleanup_replacements = {
'Anguilla (United Kingdom)': 'Anguilla',
'Argentina|Argentine Republic': 'Argentina',
'Aruba (Kingdom of the Netherlands|Netherlands)': 'Aruba',
'Azerbaijan|Azerbaijani Republic': 'Azerbaijan',
'Bermuda (United Kingdom)': 'Bermuda',
'British Virgin Islands (United Kingdom)': 'British Virgin Islands',
'Brunei|Brunei Darussalam': 'Brunei',
'Cayman Islands': 'Cayman Islands (United Kingdom)',
'Cayman Islands (United Kingdom)': 'Cayman Islands (United Kingdom)',
'Czech Rep.': 'Czech Republic',
'Democratic People\'s Republic of Korea|Korea, North': 'North Korea',
'Denmark (Kingdom of Denmark)': 'Denmark',
'Faroe Islands (Kingdom of Denmark)': 'Faroe Islands (Denmark)',
'French Polynesia (France)': 'French Polynesia',
'Gabon|Gabonese Republic': 'Gabon',
'Georgia (country)|Georgia': 'Georgia',
'Gibraltar': 'Gibraltar (United Kingdom)',
'Gibraltar (United Kingdom)': 'Gibraltar (United Kingdom)',
'Greenland (Kingdom of Denmark)': 'Greenland (Denmark)',
'Guadeloupe': 'Guadeloupe (France)',
'Hong Kong (People\'s Republic of China|PRC)': 'Hong Kong (China)',
'Hong Kong (Special Administrative Region of People\'s Republic of China)': 'Hong Kong (China)',
'Korea (Rep. of)': 'South Korea',
'Kyrgyz Republic': 'Kyrgyzstan',
'Lao People\'s Democratic Republic|Laos': 'Laos',
'Macau (People\'s Republic of China)': 'Macau (China)',
'Macau (People\'s Republic of China|PRC)': 'Macau (China)',
'Martinique': 'Martinique (France)',
'Moldova (Republic of)': 'Moldova',
'Montenegro (Republic of)': 'Montenegro',
'Netherlands (Kingdom of the Netherlands)': 'Netherlands',
'Palestinian Authority': 'Palestinian territories',
'Palestinian territories|Palestine': 'Palestinian territories',
'People\'s Republic of China|China': 'China',
'Puerto Rico (United States)': 'Puerto Rico',
'Republic of Ireland|Ireland': 'Ireland',
'Republic of Korea|Korea, South': 'South Korea',
'Russian Federation': 'Russian Federation',
'Rwanda|Rwandese Republic': 'Rwanda',
'Serbia (Republic of)': 'Serbia',
'Somali Democratic Republic|Somalia': 'Somalia',
'Syrian Arab Republic': 'Syria',
'Syrian Arab Republic|Syria': 'Syria',
'Turks and Caicos Islands (United Kingdom)': 'Turks and Caicos Islands',
'United States': 'United States of America',
'United States Virgin Islands (United States)': 'United States Virgin Islands',
'Venezuela (Bolivarian Republic of)': 'Venezuela',
'Vietnam|Viet Nam': 'Vietnam',
}
remove_ref_re = re.compile(r'<ref>.*?</ref>')
remove_comment_re = re.compile(r'{{.*?}}')
remove_href_re = re.compile(ur'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+' +
ur'[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|' +
ur'(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|' +
ur'(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>' +
ur'?\xab\xbb\u201c\u201d\u2018\u2019]))')
def cleanup_value(val):
"""Remove unneeded markup from the value."""
# remove uninteresting things from value
val = remove_comment_re.sub('', val)
val = remove_ref_re.sub('', val)
val = remove_href_re.sub('', val)
val = val.replace('[', '').replace(']', '').replace('\'\'', '').strip()
val = val.split('|')[-1]
# replace value
val = val.replace('Unknown', '')
val = val.replace('United Kingdom|UK', 'United Kingdom')
val = val.replace('United States|US', 'United States')
val = val.replace('New Zealand|NZ', 'New Zealand').strip()
return cleanup_replacements.get(val, val)
def update_mncs(data, mcc, mnc, **kwargs):
"""Merge provided mnc information with the data that is already stored
in mccs."""
data[mcc][mnc].update(dict((k, cleanup_value(v)) for k, v in kwargs.items() if v))
def get_mncs_from_wikipedia(data):
"""Update the collection of Mobile Country Codes from Wikipedia.
This parses a Wikipedia page to extract the MCC and MNC, the first
part of any IMSI, and stores the results."""
mnc_country_re = re.compile(r'^[=]{2,4}\s+(?P<country>.*?)(\s+-\s+(?P<cc>[^\s]{2}))?\s+[=]{2,4}$')
mnc_line_re = re.compile(r'^\|\s*(?P<mcc>[0-9]+)' +
r'\s*\\\\\s*(?P<mnc>[0-9]+)' +
r'(\s*\\\\\s*(?P<brand>[^\\]*)' +
r'(\s*\\\\\s*(?P<operator>[^\\]*)' +
r'(\s*\\\\\s*(?P<status>[^\\]*)' +
r'(\s*\\\\\s*(?P<bands>[^\\]*)' +
r'(\s*\\\\\s*(?P<notes>[^\\]*)' +
r')?)?)?)?)?')
f = urllib.urlopen(mcc_list_url)
country = cc = ''
for line in f.readlines():
line = line.strip()
match = mnc_country_re.match(line)
if match:
country = match.group('country')
cc = (match.group('cc') or '').lower()
if '||' not in line:
continue
line = line.replace('||', '\\\\')
match = mnc_line_re.match(line)
if match:
mnc_list = str2range(match.group('mnc'))
for mnc in mnc_list:
update_mncs(data, match.group('mcc'), mnc,
country=country, cc=cc, brand=match.group('brand'),
operator=match.group('operator'),
status=match.group('status'),
bands=match.group('bands'))
def str2range(x):
result = []
for part in x.split(','):
if '-' in part:
a, b = part.split('-')
f = '%0' + str(len(b)) + 'd'
a, b = int(a), int(b)
for i in range(a, b + 1):
result.append(f % (i))
else:
a = part
result.append(part)
return result
if __name__ == '__main__':
# download/parse the information
data = defaultdict(lambda: defaultdict(dict))
get_mncs_from_wikipedia(data)
# print header
print '# generated from various sources'
print '# %s' % mcc_list_url
# build an ordered list of mccs
mcc_list = list(data.keys())
mcc_list.sort()
# go over mccs
for mcc in mcc_list:
print '%s' % mcc
# build an ordered list of mncs
mnc_list = data[mcc].keys()
mnc_list.sort()
for mnc in mnc_list:
info = data[mcc][mnc]
infokeys = info.keys()
infokeys.sort()
print ' %s%s' % (mnc, ''.join([' %s="%s"' % (k, info[k]) for k in infokeys]))
# try to get the length of mnc's
try:
l = len(mnc_list[0])
print ' %s-%s' % (l * '0', l * '9')
except IndexError:
pass # ignore
|
