1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
|
import os
import pytest
from boltons.dictutils import OMD
from boltons.iterutils import (first,
pairwise,
pairwise_iter,
windowed,
windowed_iter,
remap,
research,
default_enter,
default_exit,
get_path)
from boltons.namedutils import namedtuple
CUR_PATH = os.path.abspath(__file__)
isbool = lambda x: isinstance(x, bool)
isint = lambda x: isinstance(x, int)
odd = lambda x: isint(x) and x % 2 != 0
even = lambda x: isint(x) and x % 2 == 0
is_meaning_of_life = lambda x: x == 42
class TestFirst:
def test_empty_iterables(self):
"""
Empty iterables return None.
"""
s = set()
l = []
assert first(s) is None
assert first(l) is None
def test_default_value(self):
"""
Empty iterables + a default value return the default value.
"""
s = set()
l = []
assert first(s, default=42) == 42
assert first(l, default=3.14) == 3.14
l = [0, False, []]
assert first(l, default=3.14) == 3.14
def test_selection(self):
"""
Success cases with and without a key function.
"""
l = [(), 0, False, 3, []]
assert first(l, default=42) == 3
assert first(l, key=isint) == 0
assert first(l, key=isbool) is False
assert first(l, key=odd) == 3
assert first(l, key=even) == 0
assert first(l, key=is_meaning_of_life) is None
class TestRemap:
# TODO: test namedtuples and other immutable containers
def test_basic_clone(self):
orig = {"a": "b", "c": [1, 2]}
assert orig == remap(orig)
orig2 = [{1: 2}, {"a": "b", "c": [1, 2, {"cat": "dog"}]}]
assert orig2 == remap(orig2)
def test_empty(self):
assert [] == remap([])
assert {} == remap({})
assert set() == remap(set())
def test_unremappable(self):
obj = object()
with pytest.raises(TypeError):
remap(obj)
def test_basic_upper(self):
orig = {'a': 1, 'b': object(), 'c': {'d': set()}}
remapped = remap(orig, lambda p, k, v: (k.upper(), v))
assert orig['a'] == remapped['A']
assert orig['b'] == remapped['B']
assert orig['c']['d'] == remapped['C']['D']
def test_item_drop(self):
orig = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
even_items = remap(orig, lambda p, k, v: not (v % 2))
assert even_items == [0, 2, 4, 6, 8]
def test_noncallables(self):
with pytest.raises(TypeError):
remap([], visit='test')
with pytest.raises(TypeError):
remap([], enter='test')
with pytest.raises(TypeError):
remap([], exit='test')
def test_sub_selfref(self):
coll = [0, 1, 2, 3]
sub = []
sub.append(sub)
coll.append(sub)
with pytest.raises(RuntimeError):
# if equal, should recurse infinitely
assert coll == remap(coll)
def test_root_selfref(self):
selfref = [0, 1, 2, 3]
selfref.append(selfref)
with pytest.raises(RuntimeError):
assert selfref == remap(selfref)
selfref2 = {}
selfref2['self'] = selfref2
with pytest.raises(RuntimeError):
assert selfref2 == remap(selfref2)
def test_duperef(self):
val = ['hello']
duperef = [val, val]
remapped = remap(duperef)
assert remapped[0] is remapped[1]
assert remapped[0] is not duperef[0]
def test_namedtuple(self):
"""TODO: this fails right now because namedtuples' __new__ is
overridden to accept arguments. remap's default_enter tries
to create an empty namedtuple and gets a TypeError.
Could make it so that immutable types actually don't create a
blank new parent and instead use the old_parent as a
placeholder, creating a new one at exit-time from the value's
__class__ (how default_exit works now). But even then it would
have to *args in the values, as namedtuple constructors don't
take an iterable.
"""
Point = namedtuple('Point', 'x y')
point_map = {'origin': [Point(0, 0)]}
with pytest.raises(TypeError):
remapped = remap(point_map)
assert isinstance(remapped['origin'][0], Point)
def test_path(self):
path_map = {}
# test visit's path
target_str = 'test'
orig = [[[target_str]]]
ref_path = (0, 0, 0)
def visit(path, key, value):
if value is target_str:
path_map['target_str'] = path + (key,)
return key, value
remapped = remap(orig, visit=visit)
assert remapped == orig
assert path_map['target_str'] == ref_path
# test enter's path
target_obj = object()
orig = {'a': {'b': {'c': {'d': ['e', target_obj, 'f']}}}}
ref_path = ('a', 'b', 'c', 'd', 1)
def enter(path, key, value):
if value is target_obj:
path_map['target_obj'] = path + (key,)
return default_enter(path, key, value)
remapped = remap(orig, enter=enter)
assert remapped == orig
assert path_map['target_obj'] == ref_path
# test exit's path
target_set = frozenset([1, 7, 3, 8])
orig = [0, 1, 2, [3, 4, [5, target_set]]]
ref_path = (3, 2, 1)
def exit(path, key, old_parent, new_parent, new_items):
if old_parent is target_set:
path_map['target_set'] = path + (key,)
return default_exit(path, key, old_parent, new_parent, new_items)
remapped = remap(orig, exit=exit)
assert remapped == orig
assert path_map['target_set'] == ref_path
def test_reraise_visit(self):
root = {'A': 'b', 1: 2}
key_to_lower = lambda p, k, v: (k.lower(), v)
with pytest.raises(AttributeError):
remap(root, key_to_lower)
remapped = remap(root, key_to_lower, reraise_visit=False)
assert remapped['a'] == 'b'
assert remapped[1] == 2
def test_drop_nones(self):
orig = {'a': 1, 'b': None, 'c': [3, None, 4, None]}
ref = {'a': 1, 'c': [3, 4]}
drop_none = lambda p, k, v: v is not None
remapped = remap(orig, visit=drop_none)
assert remapped == ref
orig = [None] * 100
remapped = remap(orig, drop_none)
assert not remapped
def test_dict_to_omd(self):
def enter(path, key, value):
if isinstance(value, dict):
return OMD(), sorted(value.items())
return default_enter(path, key, value)
orig = [{'title': 'Wild Palms',
'ratings': {1: 1, 2: 3, 3: 5, 4: 6, 5: 3}},
{'title': 'Twin Peaks',
'ratings': {1: 3, 2: 2, 3: 8, 4: 12, 5: 15}}]
remapped = remap(orig, enter=enter)
assert remapped == orig
assert isinstance(remapped[0], OMD)
assert isinstance(remapped[0]['ratings'], OMD)
assert isinstance(remapped[1], OMD)
assert isinstance(remapped[1]['ratings'], OMD)
def test_sort_all_lists(self):
def exit(path, key, old_parent, new_parent, new_items):
# NB: in this case, I'd normally use *a, **kw
ret = default_exit(path, key, old_parent, new_parent, new_items)
if isinstance(ret, list):
ret.sort()
return ret
# NB: Airplane model numbers (Boeing and Airbus)
orig = [[[7, 0, 7],
[7, 2, 7],
[7, 7, 7],
[7, 3, 7]],
[[3, 8, 0],
[3, 2, 0],
[3, 1, 9],
[3, 5, 0]]]
ref = [[[0, 2, 3],
[0, 3, 5],
[0, 3, 8],
[1, 3, 9]],
[[0, 7, 7],
[2, 7, 7],
[3, 7, 7],
[7, 7, 7]]]
remapped = remap(orig, exit=exit)
assert remapped == ref
def test_collector_pattern(self):
all_interests = set()
def enter(path, key, value):
try:
all_interests.update(value['interests'])
except:
pass
return default_enter(path, key, value)
orig = [{'name': 'Kate',
'interests': ['theater', 'manga'],
'dads': [{'name': 'Chris',
'interests': ['biking', 'python']}]},
{'name': 'Avery',
'interests': ['museums', 'pears'],
'dads': [{'name': 'Kurt',
'interests': ['python', 'recursion']}]}]
ref = {'python', 'recursion', 'biking', 'museums',
'pears', 'theater', 'manga'}
remap(orig, enter=enter)
assert all_interests == ref
def test_add_length(self):
def exit(path, key, old_parent, new_parent, new_items):
ret = default_exit(path, key, old_parent, new_parent, new_items)
try:
ret['review_length'] = len(ret['review'])
except:
pass
return ret
orig = {'Star Trek':
{'TNG': {'stars': 10,
'review': "Episodic AND deep. <3 Data."},
'DS9': {'stars': 8.5,
'review': "Like TNG, but with a story and no Data."},
'ENT': {'stars': None,
'review': "Can't review what you can't watch."}},
'Babylon 5': {'stars': 6,
'review': "Sophomoric, like a bitter laugh."},
'Dr. Who': {'stars': None,
'review': "800 episodes is too many to review."}}
remapped = remap(orig, exit=exit)
assert (remapped['Star Trek']['TNG']['review_length']
< remapped['Star Trek']['DS9']['review_length'])
def test_prepop(self):
"""Demonstrating normalization and ID addition through prepopulating
the objects with an enter callback.
"""
base_obj = {'name': None,
'rank': None,
'id': 1}
def enter(path, key, value):
new_parent, new_items = default_enter(path, key, value)
try:
new_parent.update(base_obj)
base_obj['id'] += 1
except:
pass
return new_parent, new_items
orig = [{'name': 'Firefox', 'rank': 1},
{'name': 'Chrome', 'rank': 2},
{'name': 'IE'}]
ref = [{'name': 'Firefox', 'rank': 1, 'id': 1},
{'name': 'Chrome', 'rank': 2, 'id': 2},
{'name': 'IE', 'rank': None, 'id': 3}]
remapped = remap(orig, enter=enter)
assert remapped == ref
def test_remap_set(self):
# explicit test for sets to make sure #84 is covered
s = {1, 2, 3}
assert remap(s) == s
fs = frozenset([1, 2, 3])
assert remap(fs) == fs
def test_remap_file(self):
with open(CUR_PATH, 'rb') as f:
x = {'a': [1, 2, 3], 'f': [f]}
assert remap(x) == x
f.read()
assert remap(x) == x
f.close() # see #146
assert remap(x) == x
return
class TestGetPath:
def test_depth_one(self):
root = ['test']
assert get_path(root, (0,)) == 'test'
assert get_path(root, '0') == 'test'
root = {'key': 'value'}
assert get_path(root, ('key',)) == 'value'
assert get_path(root, 'key') == 'value'
def test_depth_two(self):
root = {'key': ['test']}
assert get_path(root, ('key', 0)) == 'test'
assert get_path(root, 'key.0') == 'test'
def test_research():
root = {}
with pytest.raises(TypeError):
research(root, query=None)
root = {'a': 'a'}
res = research(root, query=lambda p, k, v: v == 'a')
assert len(res) == 1
assert res[0] == (('a',), 'a')
def broken_query(p, k, v):
raise RuntimeError()
with pytest.raises(RuntimeError):
research(root, broken_query, reraise=True)
# empty results with default, reraise=False
assert research(root, broken_query) == []
def test_research_custom_enter():
# see #368
from types import SimpleNamespace as NS
root = NS(
a='a',
b='b',
c=NS(aa='aa') )
def query(path, key, value):
return value.startswith('a')
def custom_enter(path, key, value):
if isinstance(value, NS):
return [], value.__dict__.items()
return default_enter(path, key, value)
with pytest.raises(TypeError):
research(root, query)
assert research(root, query, enter=custom_enter) == [(('a',), 'a'), (('c', 'aa'), 'aa')]
def test_backoff_basic():
from boltons.iterutils import backoff
assert backoff(1, 16) == [1.0, 2.0, 4.0, 8.0, 16.0]
assert backoff(1, 1) == [1.0]
assert backoff(2, 15) == [2.0, 4.0, 8.0, 15.0]
def test_backoff_repeat():
from boltons.iterutils import backoff_iter
fives = []
for val in backoff_iter(5, 5, count='repeat'):
fives.append(val)
if len(fives) >= 1000:
break
assert fives == [5] * 1000
def test_backoff_zero_start():
from boltons.iterutils import backoff
assert backoff(0, 16) == [0.0, 1.0, 2.0, 4.0, 8.0, 16.0]
assert backoff(0, 15) == [0.0, 1.0, 2.0, 4.0, 8.0, 15.0]
slow_backoff = [round(x, 2) for x in backoff(0, 2.9, factor=1.2)]
assert slow_backoff == [0.0, 1.0, 1.2, 1.44, 1.73, 2.07, 2.49, 2.9]
def test_backoff_validation():
from boltons.iterutils import backoff
with pytest.raises(ValueError):
backoff(8, 2)
with pytest.raises(ValueError):
backoff(1, 0)
with pytest.raises(ValueError):
backoff(-1, 10)
with pytest.raises(ValueError):
backoff(2, 8, factor=0)
with pytest.raises(ValueError):
backoff(2, 8, jitter=20)
def test_backoff_jitter():
from boltons.iterutils import backoff
start, stop = 1, 256
unjittered = backoff(start, stop)
jittered = backoff(start, stop, jitter=True)
assert len(unjittered) == len(jittered)
assert [u >= j for u, j in zip(unjittered, jittered)]
neg_jittered = backoff(start, stop, jitter=-0.01)
assert len(unjittered) == len(neg_jittered)
assert [u <= j for u, j in zip(unjittered, neg_jittered)]
o_jittered = backoff(start, stop, jitter=-0.0)
assert len(unjittered) == len(o_jittered)
assert [u == j for u, j in zip(unjittered, o_jittered)]
nonconst_jittered = backoff(stop, stop, count=5, jitter=True)
assert len(nonconst_jittered) == 5
# no two should be equal realistically
assert len(set(nonconst_jittered)) == 5
def test_guiderator():
import string
from boltons.iterutils import GUIDerator
guid_iter = GUIDerator()
guid = next(guid_iter)
assert guid
assert len(guid) == guid_iter.size
assert all([c in string.hexdigits for c in guid])
guid2 = next(guid_iter)
assert guid != guid2
# custom size
guid_iter = GUIDerator(size=26)
assert len(next(guid_iter)) == 26
def test_seqguiderator():
import string
from boltons.iterutils import SequentialGUIDerator as GUIDerator
guid_iter = GUIDerator()
guid = next(guid_iter)
assert guid
assert len(guid) == guid_iter.size
assert all([c in string.hexdigits for c in guid])
guid2 = next(guid_iter)
assert guid != guid2
# custom size
for x in range(10000):
guid_iter = GUIDerator(size=26)
assert len(next(guid_iter)) == 26
def test_chunked_bytes():
# see #231
from boltons.iterutils import chunked
assert chunked(b'123', 2) in (['12', '3'], [b'12', b'3'])
def test_chunk_ranges():
from boltons.iterutils import chunk_ranges
assert list(chunk_ranges(input_offset=10, input_size=10, chunk_size=5)) == [(10, 15), (15, 20)]
assert list(chunk_ranges(input_offset=10, input_size=10, chunk_size=5, overlap_size=1)) == [(10, 15), (14, 19), (18, 20)]
assert list(chunk_ranges(input_offset=10, input_size=10, chunk_size=5, overlap_size=2)) == [(10, 15), (13, 18), (16, 20)]
assert list(chunk_ranges(input_offset=4, input_size=15, chunk_size=5, align=False)) == [(4, 9), (9, 14), (14, 19)]
assert list(chunk_ranges(input_offset=4, input_size=15, chunk_size=5, align=True)) == [(4, 5), (5, 10), (10, 15), (15, 19)]
assert list(chunk_ranges(input_offset=2, input_size=15, chunk_size=5, overlap_size=1, align=False)) == [(2, 7), (6, 11), (10, 15), (14, 17)]
assert list(chunk_ranges(input_offset=2, input_size=15, chunk_size=5, overlap_size=1, align=True)) == [(2, 5), (4, 9), (8, 13), (12, 17)]
assert list(chunk_ranges(input_offset=3, input_size=15, chunk_size=5, overlap_size=1, align=True)) == [(3, 5), (4, 9), (8, 13), (12, 17), (16, 18)]
assert list(chunk_ranges(input_offset=3, input_size=2, chunk_size=5, overlap_size=1, align=True)) == [(3, 5)]
def test_lstrip():
from boltons.iterutils import lstrip
assert lstrip([0,1,0,2,0,3,0],0) == [1,0,2,0,3,0]
assert lstrip([0,0,0,1,0,2,0,3,0],0) == [1,0,2,0,3,0]
assert lstrip([]) == []
def test_rstrip():
from boltons.iterutils import rstrip
assert rstrip([0,1,0,2,0,3,0],0) == [0,1,0,2,0,3]
assert rstrip([0,1,0,2,0,3,0,0,0],0) == [0,1,0,2,0,3]
assert rstrip([]) == []
def test_strip():
from boltons.iterutils import strip
assert strip([0,1,0,2,0,3,0],0) == [1,0,2,0,3]
assert strip([0,0,0,1,0,2,0,3,0,0,0],0) == [1,0,2,0,3]
assert strip([]) == []
def test_pairwise_filled():
assert pairwise(range(4)) == [(0, 1), (1, 2), (2, 3)]
assert pairwise(range(4), end=None) == [(0, 1), (1, 2), (2, 3), (3, None)]
assert pairwise([]) == []
assert pairwise([1], end=None) == [(1, None)]
assert list(pairwise_iter(range(4))) == [(0, 1), (1, 2), (2, 3)]
assert list(pairwise_iter(range(4), end=None)) == [(0, 1), (1, 2), (2, 3), (3, None)]
def test_windowed_filled():
assert windowed(range(4), 3) == [(0, 1, 2), (1, 2, 3)]
assert windowed(range(4), 3, fill=None) == [(0, 1, 2), (1, 2, 3), (2, 3, None), (3, None, None)]
assert windowed([], 3) == []
assert windowed([], 3, fill=None) == []
assert windowed([1, 2], 3, fill=None) == [(1, 2, None), (2, None, None)]
assert list(windowed_iter(range(4), 3)) == [(0, 1, 2), (1, 2, 3)]
assert list(windowed_iter(range(4), 3, fill=None)) == [(0, 1, 2), (1, 2, 3), (2, 3, None), (3, None, None)]
|