From c96fa091f38b3da81289f76f7f87521aca513e29 Mon Sep 17 00:00:00 2001 From: Stuart Archibald Origin: https://github.com/numba/numba/pull/8639 Date: Mon, 21 Nov 2022 13:56:00 +0000 Subject: [PATCH 01/20] Update string/bytes hash algs to siphash13 for Python 3.11 Python 3.11 uses siphash13 as its internal hashing algorithm, this impacts Numba in the case of string and byte types. This patch updates the hashing implementations to accommodate this change and adds new tests based on "golden" values obtained from the cPython test suite. --- numba/cpython/hashing.py | 193 +++++++++++++++++++++--------------- numba/tests/test_hashing.py | 88 +++++++++++++++- 2 files changed, 198 insertions(+), 83 deletions(-) --- a/numba/cpython/hashing.py +++ b/numba/cpython/hashing.py @@ -39,9 +39,7 @@ _PyHASH_IMAG = _PyHASH_MULTIPLIER _PyLong_SHIFT = sys.int_info.bits_per_digit _Py_HASH_CUTOFF = sys.hash_info.cutoff -_Py_hashfunc_name = ("siphash24" - if utils.PYVERSION >= (3, 11) - else sys.hash_info.algorithm) +_Py_hashfunc_name = sys.hash_info.algorithm # hash(obj) is implemented by calling obj.__hash__() @@ -509,7 +507,7 @@ # ------------------------------------------------------------------------------ -if _Py_hashfunc_name in ('siphash24', 'fnv'): +if _Py_hashfunc_name in ('siphash13', 'siphash24', 'fnv'): # Check for use of the FNV hashing alg, warn users that it's not implemented # and functionality relying of properties derived from hashing will be fine @@ -527,6 +525,12 @@ # This is a translation of CPython's siphash24 function: # https://github.com/python/cpython/blob/d1dd6be613381b996b9071443ef081de8e5f3aff/Python/pyhash.c#L287-L413 # noqa: E501 + # and also, since Py 3.11, a translation of CPython's siphash13 function: + # https://github.com/python/cpython/blob/9dda9020abcf0d51d59b283a89c58c8e1fb0f574/Python/pyhash.c#L376-L424 + # the only differences are in the use of SINGLE_ROUND in siphash13 vs. + # DOUBLE_ROUND in siphash24, and that siphash13 has an extra "ROUND" applied + # just before the final XORing of components to create the return value. + # /* ********************************************************************* # @@ -588,9 +592,7 @@ 'v1': types.uint64, 'v2': types.uint64, 'v3': types.uint64, }) - def _DOUBLE_ROUND(v0, v1, v2, v3): - v0, v1, v2, v3 = _HALF_ROUND(v0, v1, v2, v3, 13, 16) - v2, v1, v0, v3 = _HALF_ROUND(v2, v1, v0, v3, 17, 21) + def _SINGLE_ROUND(v0, v1, v2, v3): v0, v1, v2, v3 = _HALF_ROUND(v0, v1, v2, v3, 13, 16) v2, v1, v0, v3 = _HALF_ROUND(v2, v1, v0, v3, 17, 21) return v0, v1, v2, v3 @@ -598,80 +600,109 @@ @register_jitable(locals={'v0': types.uint64, 'v1': types.uint64, 'v2': types.uint64, - 'v3': types.uint64, - 'b': types.uint64, - 'mi': types.uint64, - 'tmp': types.Array(types.uint64, 1, 'C'), - 't': types.uint64, - 'mask': types.uint64, - 'jmp': types.uint64, - 'ohexefef': types.uint64}) - def _siphash24(k0, k1, src, src_sz): - b = types.uint64(src_sz) << 56 - v0 = k0 ^ types.uint64(0x736f6d6570736575) - v1 = k1 ^ types.uint64(0x646f72616e646f6d) - v2 = k0 ^ types.uint64(0x6c7967656e657261) - v3 = k1 ^ types.uint64(0x7465646279746573) - - idx = 0 - while (src_sz >= 8): - mi = grab_uint64_t(src, idx) - idx += 1 - src_sz -= 8 - v3 ^= mi - v0, v1, v2, v3 = _DOUBLE_ROUND(v0, v1, v2, v3) - v0 ^= mi - - # this is the switch fallthrough: - # https://github.com/python/cpython/blob/d1dd6be613381b996b9071443ef081de8e5f3aff/Python/pyhash.c#L390-L400 # noqa: E501 - t = types.uint64(0x0) - boffset = idx * 8 - ohexefef = types.uint64(0xff) - if src_sz >= 7: - jmp = (6 * 8) - mask = ~types.uint64(ohexefef << jmp) - t = (t & mask) | (types.uint64(grab_byte(src, boffset + 6)) - << jmp) - if src_sz >= 6: - jmp = (5 * 8) - mask = ~types.uint64(ohexefef << jmp) - t = (t & mask) | (types.uint64(grab_byte(src, boffset + 5)) - << jmp) - if src_sz >= 5: - jmp = (4 * 8) - mask = ~types.uint64(ohexefef << jmp) - t = (t & mask) | (types.uint64(grab_byte(src, boffset + 4)) - << jmp) - if src_sz >= 4: - t &= types.uint64(0xffffffff00000000) - for i in range(4): - jmp = i * 8 + 'v3': types.uint64, }) + def _DOUBLE_ROUND(v0, v1, v2, v3): + v0, v1, v2, v3 = _SINGLE_ROUND(v0, v1, v2, v3) + v0, v1, v2, v3 = _SINGLE_ROUND(v0, v1, v2, v3) + return v0, v1, v2, v3 + + def _gen_siphash(alg): + if alg == 'siphash13': + _ROUNDER = _SINGLE_ROUND + _EXTRA_ROUND = True + elif alg == 'siphash24': + _ROUNDER = _DOUBLE_ROUND + _EXTRA_ROUND = False + else: + assert 0, 'unreachable' + + @register_jitable(locals={'v0': types.uint64, + 'v1': types.uint64, + 'v2': types.uint64, + 'v3': types.uint64, + 'b': types.uint64, + 'mi': types.uint64, + 't': types.uint64, + 'mask': types.uint64, + 'jmp': types.uint64, + 'ohexefef': types.uint64}) + def _siphash(k0, k1, src, src_sz): + b = types.uint64(src_sz) << 56 + v0 = k0 ^ types.uint64(0x736f6d6570736575) + v1 = k1 ^ types.uint64(0x646f72616e646f6d) + v2 = k0 ^ types.uint64(0x6c7967656e657261) + v3 = k1 ^ types.uint64(0x7465646279746573) + + idx = 0 + while (src_sz >= 8): + mi = grab_uint64_t(src, idx) + idx += 1 + src_sz -= 8 + v3 ^= mi + v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3) + v0 ^= mi + + # this is the switch fallthrough: + # https://github.com/python/cpython/blob/d1dd6be613381b996b9071443ef081de8e5f3aff/Python/pyhash.c#L390-L400 # noqa: E501 + t = types.uint64(0x0) + boffset = idx * 8 + ohexefef = types.uint64(0xff) + if src_sz >= 7: + jmp = (6 * 8) + mask = ~types.uint64(ohexefef << jmp) + t = (t & mask) | (types.uint64(grab_byte(src, boffset + 6)) + << jmp) + if src_sz >= 6: + jmp = (5 * 8) + mask = ~types.uint64(ohexefef << jmp) + t = (t & mask) | (types.uint64(grab_byte(src, boffset + 5)) + << jmp) + if src_sz >= 5: + jmp = (4 * 8) + mask = ~types.uint64(ohexefef << jmp) + t = (t & mask) | (types.uint64(grab_byte(src, boffset + 4)) + << jmp) + if src_sz >= 4: + t &= types.uint64(0xffffffff00000000) + for i in range(4): + jmp = i * 8 + mask = ~types.uint64(ohexefef << jmp) + t = (t & mask) | (types.uint64(grab_byte(src, boffset + i)) + << jmp) + if src_sz >= 3: + jmp = (2 * 8) + mask = ~types.uint64(ohexefef << jmp) + t = (t & mask) | (types.uint64(grab_byte(src, boffset + 2)) + << jmp) + if src_sz >= 2: + jmp = (1 * 8) mask = ~types.uint64(ohexefef << jmp) - t = (t & mask) | (types.uint64(grab_byte(src, boffset + i)) - << jmp) - if src_sz >= 3: - jmp = (2 * 8) - mask = ~types.uint64(ohexefef << jmp) - t = (t & mask) | (types.uint64(grab_byte(src, boffset + 2)) - << jmp) - if src_sz >= 2: - jmp = (1 * 8) - mask = ~types.uint64(ohexefef << jmp) - t = (t & mask) | (types.uint64(grab_byte(src, boffset + 1)) - << jmp) - if src_sz >= 1: - mask = ~(ohexefef) - t = (t & mask) | (types.uint64(grab_byte(src, boffset + 0))) - - b |= t - v3 ^= b - v0, v1, v2, v3 = _DOUBLE_ROUND(v0, v1, v2, v3) - v0 ^= b - v2 ^= ohexefef - v0, v1, v2, v3 = _DOUBLE_ROUND(v0, v1, v2, v3) - v0, v1, v2, v3 = _DOUBLE_ROUND(v0, v1, v2, v3) - t = (v0 ^ v1) ^ (v2 ^ v3) - return t + t = (t & mask) | (types.uint64(grab_byte(src, boffset + 1)) + << jmp) + if src_sz >= 1: + mask = ~(ohexefef) + t = (t & mask) | (types.uint64(grab_byte(src, boffset + 0))) + + b |= t + v3 ^= b + v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3) + v0 ^= b + v2 ^= ohexefef + v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3) + v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3) + if _EXTRA_ROUND: + v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3) + t = (v0 ^ v1) ^ (v2 ^ v3) + return t + + return _siphash + + + _siphash13 = _gen_siphash('siphash13') + _siphash24 = _gen_siphash('siphash24') + + _siphasher = _siphash13 if _Py_hashfunc_name == 'siphash13' else _siphash24 + else: msg = "Unsupported hashing algorithm in use %s" % _Py_hashfunc_name raise ValueError(msg) @@ -732,7 +763,7 @@ _hash ^= _len _hash ^= _load_hashsecret('djbx33a_suffix') else: - tmp = _siphash24(types.uint64(_load_hashsecret('siphash_k0')), + tmp = _siphasher(types.uint64(_load_hashsecret('siphash_k0')), types.uint64(_load_hashsecret('siphash_k1')), val, _len) _hash = process_return(tmp) --- a/numba/tests/test_hashing.py +++ b/numba/tests/test_hashing.py @@ -5,6 +5,7 @@ import unittest +import os import sys import subprocess from collections import defaultdict @@ -12,11 +13,11 @@ import numpy as np -from numba import jit +from numba import jit, config from numba.core import types, utils import unittest from numba.tests.support import (TestCase, tag, CompilationCache, - skip_unless_py10_or_later) + skip_unless_py10_or_later, run_in_subprocess) from numba.cpython.unicode import compile_time_get_string_data from numba.cpython import hashing @@ -72,6 +73,89 @@ subprocess.check_call([sys.executable, '-c', dedent(work)]) +class TestHashAlgs(TestCase): + # This tests Numba hashing replication against cPython "gold", i.e. the + # actual hash values for given inputs, algs and PYTHONHASHSEEDs + # Test adapted from: + # https://github.com/python/cpython/blob/9dda9020abcf0d51d59b283a89c58c8e1fb0f574/Lib/test/test_hash.py#L197-L264 + # and + # https://github.com/python/cpython/blob/9dda9020abcf0d51d59b283a89c58c8e1fb0f574/Lib/test/test_hash.py#L174-L189 + + # 32bit little, 64bit little, 32bit big, 64bit big + known_hashes = { + 'djba33x': [ # only used for small strings + # seed 0, 'abc' + [193485960, 193485960, 193485960, 193485960], + # seed 42, 'abc' + [-678966196, 573763426263223372, -820489388, -4282905804826039665], + ], + 'siphash13': [ + # NOTE: PyUCS2 layout depends on endianness + # seed 0, 'abc' + [69611762, -4594863902769663758, 69611762, -4594863902769663758], + # seed 42, 'abc' + [-975800855, 3869580338025362921, -975800855, 3869580338025362921], + # seed 42, 'abcdefghijk' + [-595844228, 7764564197781545852, -595844228, 7764564197781545852], + # seed 0, 'äú∑ℇ' + [-1093288643, -2810468059467891395, -1041341092, 4925090034378237276], + # seed 42, 'äú∑ℇ' + [-585999602, -2845126246016066802, -817336969, -2219421378907968137], + ], + 'siphash24': [ + # NOTE: PyUCS2 layout depends on endianness + # seed 0, 'abc' + [1198583518, 4596069200710135518, 1198583518, 4596069200710135518], + # seed 42, 'abc' + [273876886, -4501618152524544106, 273876886, -4501618152524544106], + # seed 42, 'abcdefghijk' + [-1745215313, 4436719588892876975, -1745215313, 4436719588892876975], + # seed 0, 'äú∑ℇ' + [493570806, 5749986484189612790, -1006381564, -5915111450199468540], + # seed 42, 'äú∑ℇ' + [-1677110816, -2947981342227738144, -1860207793, -4296699217652516017], + ], + } + + def get_expected_hash(self, position, length): + if length < sys.hash_info.cutoff: + algorithm = "djba33x" + else: + algorithm = sys.hash_info.algorithm + IS_64BIT = not config.IS_32BITS + if sys.byteorder == 'little': + platform = 1 if IS_64BIT else 0 + else: + assert(sys.byteorder == 'big') + platform = 3 if IS_64BIT else 2 + return self.known_hashes[algorithm][position][platform] + + def get_hash_command(self, repr_): + return 'print(hash(eval(%a)))' % repr_ + + def get_hash(self, repr_, seed=None): + env = os.environ.copy() + if seed is not None: + env['PYTHONHASHSEED'] = str(seed) + else: + env.pop('PYTHONHASHSEED', None) + out, _ = run_in_subprocess(code=self.get_hash_command(repr_), + env=env) + stdout = out.decode().strip() + return int(stdout) + + def test_against_cpython_gold(self): + + args = (('abc', 0, 0), ('abc', 42, 1), ('abcdefghijk', 42, 2), + ('äú∑ℇ', 0, 3), ('äú∑ℇ', 42, 4),) + + for input_str, seed, position in args: + with self.subTest(input_str=input_str, seed=seed): + got = self.get_hash(repr(input_str), seed=seed) + expected = self.get_expected_hash(position, len(input_str)) + self.assertEqual(got, expected) + + class BaseTest(TestCase): def setUp(self):