1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
|
from requests.api import request
from oauthlib import oauth1
import json
import os
import re
from discogs_client.utils import backoff
from urllib.parse import parse_qsl
from typing import Union
class Fetcher:
"""
Base class for Fetchers, which wrap and normalize the APIs of various HTTP
libraries.
(It's a slightly leaky abstraction designed to make testing easier.)
"""
backoff_enabled = True
connect_timeout: Union[float, int, None] = None
read_timeout: Union[float, int, None] = None
def fetch(self, client, method, url, data=None, headers=None, json=True):
"""Fetch the given request
Parameters
----------
client : object
Instantiated discogs_client.client.Client object.
method : str
HTTP method.
url : str
API endpoint URL.
data : dict, optional
data to be sent in the request's body, by default None.
headers : dict, optional
HTTP headers, by default None.
json_format : bool, optional
If True, an object passed with the "data" arg will be converted into
a JSON string, by default True.
Returns
-------
content : bytes
as returned by Python "Requests"
status_code : int
as returned by Python "Requests"
Raises
------
NotImplementedError
Is raised if a child class doesn't implement a fetch method.
"""
raise NotImplementedError()
@backoff
def request(self, method, url, data, headers, params=None):
return request(
method=method, url=url, data=data,
headers=headers, params=params,
timeout=(self.connect_timeout, self.read_timeout)
)
class LoggingDelegator:
"""Wraps a fetcher and logs all requests."""
def __init__(self, fetcher):
self.fetcher = fetcher
self.requests = []
@property
def last_request(self):
return self.requests[-1] if self.requests else None
def fetch(self, client, method, url, data=None, headers=None, json=True):
"""Appends passed "fetcher" to a requests list and returns result of
fetcher.fetch method"""
self.requests.append((method, url, data, headers))
return self.fetcher.fetch(client, method, url, data, headers, json)
class RequestsFetcher(Fetcher):
"""Fetches via HTTP from the Discogs API (unauthenticated)"""
def fetch(self, client, method, url, data=None, headers=None, json=True):
"""
Parameters
----------
client : object
Unused in this subclass.
method : str
HTTP method.
url : str
API endpoint URL.
data : dict, optional
data to be sent in the request's body, by default None.
headers : dict, optional
HTTP headers, by default None.
json_format : bool, optional
Unused in this subclass, by default True.
Returns
-------
content : bytes
as returned by Python "Requests"
status_code : int
as returned by Python "Requests"
"""
resp = self.request(method, url, data=data, headers=headers)
self.rate_limit = resp.headers.get(
'X-Discogs-Ratelimit')
self.rate_limit_used = resp.headers.get(
'X-Discogs-Ratelimit-Used')
self.rate_limit_remaining = resp.headers.get(
'X-Discogs-Ratelimit-Remaining')
return resp.content, resp.status_code
class UserTokenRequestsFetcher(Fetcher):
"""Fetches via HTTP from the Discogs API using User-token authentication"""
def __init__(self, user_token):
self.user_token = user_token
def fetch(self, client, method, url, data=None, headers=None, json_format=True):
"""Fetch the given request on the user's behalf
Parameters
----------
client : object
Unused in this subclass.
method : str
HTTP method.
url : str
API endpoint URL.
data : dict, optional
data to be sent in the request's body, by default None.
headers : dict, optional
HTTP headers, by default None.
json_format : bool, optional
If True, an object passed with the "data" arg will be converted into
a JSON string, by default True.
Returns
-------
content : bytes
as returned by Python "Requests"
status_code : int
as returned by Python "Requests"
"""
data = json.dumps(data) if json_format and data else data
resp = self.request(
method, url, data=data, headers=headers, params={'token':self.user_token}
)
self.rate_limit = resp.headers.get(
'X-Discogs-Ratelimit')
self.rate_limit_used = resp.headers.get(
'X-Discogs-Ratelimit-Used')
self.rate_limit_remaining = resp.headers.get(
'X-Discogs-Ratelimit-Remaining')
return resp.content, resp.status_code
class OAuth2Fetcher(Fetcher):
"""Fetches via HTTP + OAuth 1.0a from the Discogs API."""
def __init__(self, consumer_key, consumer_secret, token=None, secret=None):
self.client = oauth1.Client(consumer_key, client_secret=consumer_secret)
self.store_token(token, secret)
def store_token_from_qs(self, query_string):
token_dict = dict(parse_qsl(query_string))
token = token_dict[b'oauth_token'].decode('utf-8')
secret = token_dict[b'oauth_token_secret'].decode('utf-8')
self.store_token(token, secret)
return token, secret
def forget_token(self):
self.store_token(None, None)
def store_token(self, token, secret):
self.client.resource_owner_key = token
self.client.resource_owner_secret = secret
def set_verifier(self, verifier):
self.client.verifier = verifier
def fetch(self, client, method, url, data=None, headers=None, json_format=True):
"""Fetch the given request on the user's behalf
Parameters
----------
client : object
Unused in this subclass.
method : str
HTTP method.
url : str
API endpoint URL.
data : dict, optional
Data to be sent in the request's body, by default None.
headers : dict, optional
HTTP headers, by default None.
json_format : bool, optional
If True, an object passed with the "data" arg will be converted into
a JSON string, by default True.
Returns
-------
content : bytes
as returned by Python "Requests"
status_code : int
as returned by Python "Requests"
"""
body = json.dumps(data) if json_format and data else data
uri, headers, body = self.client.sign(url, http_method=method,
body=body, headers=headers)
resp = self.request(method, url, data=body, headers=headers)
self.rate_limit = resp.headers.get(
'X-Discogs-Ratelimit')
self.rate_limit_used = resp.headers.get(
'X-Discogs-Ratelimit-Used')
self.rate_limit_remaining = resp.headers.get(
'X-Discogs-Ratelimit-Remaining')
return resp.content, resp.status_code
class FilesystemFetcher(Fetcher):
"""Fetches from a directory of files."""
default_response = json.dumps({'message': 'Resource not found.'}), 404
path_with_params = re.compile(r'(?P<dir>(\w+/)+)(?P<query>\w+)\?(?P<params>.*)')
def __init__(self, base_path):
self.base_path = base_path
def fetch(self, client, method, url, data=None, headers=None, json=True):
"""Fetch the given request
Parameters
----------
client : object
Instantiated discogs_client.client.Client object.
method : str
Unused in this subclass (this fetcher supports GET only).
url : str
API endpoint URL.
data : dict, optional
Unused in this subclass (this fetcher supports GET only).
headers : dict, optional
Unused in this subclass.
json_format : bool, optional
Returns
-------
content : bytes
status_code : int
"""
url = url.replace(client._base_url, '')
if json:
base_name = ''.join((url[1:], '.json'))
else:
base_name = url[1:]
path = os.path.join(self.base_path, base_name)
# The exact path might not exist, but check for files with different
# permutations of query parameters.
if not os.path.exists(path):
base_name = self.check_alternate_params(base_name, json)
path = os.path.join(self.base_path, base_name)
try:
path = path.replace('?', '_') # '?' is illegal in file names on Windows
with open(path, 'r') as f:
content = f.read().encode('utf8') # return bytes not unicode
return content, 200
except:
return self.default_response
def check_alternate_params(self, base_name, json):
"""
parse_qs() result is non-deterministic - a different file might be
requested, making the tests fail randomly, depending on the order of parameters in the query.
This fixes it by checking for matching file names with a different permutations of the parameters.
"""
match = self.path_with_params.match(base_name)
# No parameters in query - no match. Nothing to do.
if not match:
return base_name
ext = '.json' if json else ''
# The base name consists of one or more path elements (directories),
# a query (discogs.com endpoint), query parameters, and possibly an extension like 'json'.
# Extract these.
base_dir = os.path.join(self.base_path, match.group('dir'))
query = match.group('query') # we'll need this to only check relevant filenames
params_str = match.group('params')[:-len(ext)] # strip extension if any
params = set(params_str.split('&'))
# List files that match the same query, possibly with different parameters
filenames = [f for f in os.listdir(base_dir) if f.startswith(query)]
for f in filenames:
# Strip the query, the '?' sign (or its replacement) and the extension, if any
params2_str = f[len(query) + 1:-len(ext)]
params2 = set(params2_str.split('&'))
if params == params2:
return base_name.replace(params_str, params2_str)
# No matching alternatives found - revert to original.
return base_name
class MemoryFetcher(Fetcher):
"""Fetches from a dict of URL -> (content, status_code)."""
default_response = json.dumps({'message': 'Resource not found.'}), 404
def __init__(self, responses):
self.responses = responses
def fetch(self, client, method, url, data=None, headers=None, json=True):
"""Fetch the given request
Parameters
----------
client : object
Unused in this subclass.
method : str
Unused in this subclass (this fetcher supports GET only).
url : str
API endpoint URL.
data : dict, optional
Unused in this subclass (this fetcher supports GET only).
headers : dict, optional
Unused in this subclass.
json_format : bool, optional
Unused in this subclass
Returns
-------
content : bytes
status_code : int
"""
return self.responses.get(url, self.default_response)
|