From c96fa091f38b3da81289f76f7f87521aca513e29 Mon Sep 17 00:00:00 2001
From: Stuart Archibald <stuartarchibald@users.noreply.github.com>
Origin: https://github.com/numba/numba/pull/8639
Date: Mon, 21 Nov 2022 13:56:00 +0000
Subject: [PATCH 01/20] Update string/bytes hash algs to siphash13 for Python
 3.11

Python 3.11 uses siphash13 as its internal hashing algorithm,
this impacts Numba in the case of string and byte types. This
patch updates the hashing implementations to accommodate this
change and adds new tests based on "golden" values obtained from
the cPython test suite.
---
 numba/cpython/hashing.py    | 193 +++++++++++++++++++++---------------
 numba/tests/test_hashing.py |  88 +++++++++++++++-
 2 files changed, 198 insertions(+), 83 deletions(-)

--- a/numba/cpython/hashing.py
+++ b/numba/cpython/hashing.py
@@ -39,9 +39,7 @@
 _PyHASH_IMAG = _PyHASH_MULTIPLIER
 _PyLong_SHIFT = sys.int_info.bits_per_digit
 _Py_HASH_CUTOFF = sys.hash_info.cutoff
-_Py_hashfunc_name = ("siphash24"
-                     if utils.PYVERSION >= (3, 11)
-                     else sys.hash_info.algorithm)
+_Py_hashfunc_name = sys.hash_info.algorithm
 
 
 # hash(obj) is implemented by calling obj.__hash__()
@@ -509,7 +507,7 @@
 # ------------------------------------------------------------------------------
 
 
-if _Py_hashfunc_name in ('siphash24', 'fnv'):
+if _Py_hashfunc_name in ('siphash13', 'siphash24', 'fnv'):
 
     # Check for use of the FNV hashing alg, warn users that it's not implemented
     # and functionality relying of properties derived from hashing will be fine
@@ -527,6 +525,12 @@
 
     # This is a translation of CPython's siphash24 function:
     # https://github.com/python/cpython/blob/d1dd6be613381b996b9071443ef081de8e5f3aff/Python/pyhash.c#L287-L413    # noqa: E501
+    # and also, since Py 3.11, a translation of CPython's siphash13 function:
+    # https://github.com/python/cpython/blob/9dda9020abcf0d51d59b283a89c58c8e1fb0f574/Python/pyhash.c#L376-L424
+    # the only differences are in the use of SINGLE_ROUND in siphash13 vs.
+    # DOUBLE_ROUND in siphash24, and that siphash13 has an extra "ROUND" applied
+    # just before the final XORing of components to create the return value.
+
 
     # /* *********************************************************************
     # <MIT License>
@@ -588,9 +592,7 @@
                               'v1': types.uint64,
                               'v2': types.uint64,
                               'v3': types.uint64, })
-    def _DOUBLE_ROUND(v0, v1, v2, v3):
-        v0, v1, v2, v3 = _HALF_ROUND(v0, v1, v2, v3, 13, 16)
-        v2, v1, v0, v3 = _HALF_ROUND(v2, v1, v0, v3, 17, 21)
+    def _SINGLE_ROUND(v0, v1, v2, v3):
         v0, v1, v2, v3 = _HALF_ROUND(v0, v1, v2, v3, 13, 16)
         v2, v1, v0, v3 = _HALF_ROUND(v2, v1, v0, v3, 17, 21)
         return v0, v1, v2, v3
@@ -598,80 +600,109 @@
     @register_jitable(locals={'v0': types.uint64,
                               'v1': types.uint64,
                               'v2': types.uint64,
-                              'v3': types.uint64,
-                              'b': types.uint64,
-                              'mi': types.uint64,
-                              'tmp': types.Array(types.uint64, 1, 'C'),
-                              't': types.uint64,
-                              'mask': types.uint64,
-                              'jmp': types.uint64,
-                              'ohexefef': types.uint64})
-    def _siphash24(k0, k1, src, src_sz):
-        b = types.uint64(src_sz) << 56
-        v0 = k0 ^ types.uint64(0x736f6d6570736575)
-        v1 = k1 ^ types.uint64(0x646f72616e646f6d)
-        v2 = k0 ^ types.uint64(0x6c7967656e657261)
-        v3 = k1 ^ types.uint64(0x7465646279746573)
-
-        idx = 0
-        while (src_sz >= 8):
-            mi = grab_uint64_t(src, idx)
-            idx += 1
-            src_sz -= 8
-            v3 ^= mi
-            v0, v1, v2, v3 = _DOUBLE_ROUND(v0, v1, v2, v3)
-            v0 ^= mi
-
-        # this is the switch fallthrough:
-        # https://github.com/python/cpython/blob/d1dd6be613381b996b9071443ef081de8e5f3aff/Python/pyhash.c#L390-L400    # noqa: E501
-        t = types.uint64(0x0)
-        boffset = idx * 8
-        ohexefef = types.uint64(0xff)
-        if src_sz >= 7:
-            jmp = (6 * 8)
-            mask = ~types.uint64(ohexefef << jmp)
-            t = (t & mask) | (types.uint64(grab_byte(src, boffset + 6))
-                              << jmp)
-        if src_sz >= 6:
-            jmp = (5 * 8)
-            mask = ~types.uint64(ohexefef << jmp)
-            t = (t & mask) | (types.uint64(grab_byte(src, boffset + 5))
-                              << jmp)
-        if src_sz >= 5:
-            jmp = (4 * 8)
-            mask = ~types.uint64(ohexefef << jmp)
-            t = (t & mask) | (types.uint64(grab_byte(src, boffset + 4))
-                              << jmp)
-        if src_sz >= 4:
-            t &= types.uint64(0xffffffff00000000)
-            for i in range(4):
-                jmp = i * 8
+                              'v3': types.uint64, })
+    def _DOUBLE_ROUND(v0, v1, v2, v3):
+        v0, v1, v2, v3 = _SINGLE_ROUND(v0, v1, v2, v3)
+        v0, v1, v2, v3 = _SINGLE_ROUND(v0, v1, v2, v3)
+        return v0, v1, v2, v3
+
+    def _gen_siphash(alg):
+        if alg == 'siphash13':
+            _ROUNDER = _SINGLE_ROUND
+            _EXTRA_ROUND = True
+        elif alg == 'siphash24':
+            _ROUNDER = _DOUBLE_ROUND
+            _EXTRA_ROUND = False
+        else:
+            assert 0, 'unreachable'
+
+        @register_jitable(locals={'v0': types.uint64,
+                                  'v1': types.uint64,
+                                  'v2': types.uint64,
+                                  'v3': types.uint64,
+                                  'b': types.uint64,
+                                  'mi': types.uint64,
+                                  't': types.uint64,
+                                  'mask': types.uint64,
+                                  'jmp': types.uint64,
+                                  'ohexefef': types.uint64})
+        def _siphash(k0, k1, src, src_sz):
+            b = types.uint64(src_sz) << 56
+            v0 = k0 ^ types.uint64(0x736f6d6570736575)
+            v1 = k1 ^ types.uint64(0x646f72616e646f6d)
+            v2 = k0 ^ types.uint64(0x6c7967656e657261)
+            v3 = k1 ^ types.uint64(0x7465646279746573)
+
+            idx = 0
+            while (src_sz >= 8):
+                mi = grab_uint64_t(src, idx)
+                idx += 1
+                src_sz -= 8
+                v3 ^= mi
+                v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3)
+                v0 ^= mi
+
+            # this is the switch fallthrough:
+            # https://github.com/python/cpython/blob/d1dd6be613381b996b9071443ef081de8e5f3aff/Python/pyhash.c#L390-L400    # noqa: E501
+            t = types.uint64(0x0)
+            boffset = idx * 8
+            ohexefef = types.uint64(0xff)
+            if src_sz >= 7:
+                jmp = (6 * 8)
+                mask = ~types.uint64(ohexefef << jmp)
+                t = (t & mask) | (types.uint64(grab_byte(src, boffset + 6))
+                                << jmp)
+            if src_sz >= 6:
+                jmp = (5 * 8)
+                mask = ~types.uint64(ohexefef << jmp)
+                t = (t & mask) | (types.uint64(grab_byte(src, boffset + 5))
+                                << jmp)
+            if src_sz >= 5:
+                jmp = (4 * 8)
+                mask = ~types.uint64(ohexefef << jmp)
+                t = (t & mask) | (types.uint64(grab_byte(src, boffset + 4))
+                                << jmp)
+            if src_sz >= 4:
+                t &= types.uint64(0xffffffff00000000)
+                for i in range(4):
+                    jmp = i * 8
+                    mask = ~types.uint64(ohexefef << jmp)
+                    t = (t & mask) | (types.uint64(grab_byte(src, boffset + i))
+                                    << jmp)
+            if src_sz >= 3:
+                jmp = (2 * 8)
+                mask = ~types.uint64(ohexefef << jmp)
+                t = (t & mask) | (types.uint64(grab_byte(src, boffset + 2))
+                                << jmp)
+            if src_sz >= 2:
+                jmp = (1 * 8)
                 mask = ~types.uint64(ohexefef << jmp)
-                t = (t & mask) | (types.uint64(grab_byte(src, boffset + i))
-                                  << jmp)
-        if src_sz >= 3:
-            jmp = (2 * 8)
-            mask = ~types.uint64(ohexefef << jmp)
-            t = (t & mask) | (types.uint64(grab_byte(src, boffset + 2))
-                              << jmp)
-        if src_sz >= 2:
-            jmp = (1 * 8)
-            mask = ~types.uint64(ohexefef << jmp)
-            t = (t & mask) | (types.uint64(grab_byte(src, boffset + 1))
-                              << jmp)
-        if src_sz >= 1:
-            mask = ~(ohexefef)
-            t = (t & mask) | (types.uint64(grab_byte(src, boffset + 0)))
-
-        b |= t
-        v3 ^= b
-        v0, v1, v2, v3 = _DOUBLE_ROUND(v0, v1, v2, v3)
-        v0 ^= b
-        v2 ^= ohexefef
-        v0, v1, v2, v3 = _DOUBLE_ROUND(v0, v1, v2, v3)
-        v0, v1, v2, v3 = _DOUBLE_ROUND(v0, v1, v2, v3)
-        t = (v0 ^ v1) ^ (v2 ^ v3)
-        return t
+                t = (t & mask) | (types.uint64(grab_byte(src, boffset + 1))
+                                << jmp)
+            if src_sz >= 1:
+                mask = ~(ohexefef)
+                t = (t & mask) | (types.uint64(grab_byte(src, boffset + 0)))
+
+            b |= t
+            v3 ^= b
+            v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3)
+            v0 ^= b
+            v2 ^= ohexefef
+            v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3)
+            v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3)
+            if _EXTRA_ROUND:
+                v0, v1, v2, v3 = _ROUNDER(v0, v1, v2, v3)
+            t = (v0 ^ v1) ^ (v2 ^ v3)
+            return t
+
+        return _siphash
+
+
+    _siphash13 = _gen_siphash('siphash13')
+    _siphash24 = _gen_siphash('siphash24')
+
+    _siphasher = _siphash13 if  _Py_hashfunc_name == 'siphash13' else _siphash24
+
 else:
     msg = "Unsupported hashing algorithm in use %s" % _Py_hashfunc_name
     raise ValueError(msg)
@@ -732,7 +763,7 @@
         _hash ^= _len
         _hash ^= _load_hashsecret('djbx33a_suffix')
     else:
-        tmp = _siphash24(types.uint64(_load_hashsecret('siphash_k0')),
+        tmp = _siphasher(types.uint64(_load_hashsecret('siphash_k0')),
                          types.uint64(_load_hashsecret('siphash_k1')),
                          val, _len)
         _hash = process_return(tmp)
--- a/numba/tests/test_hashing.py
+++ b/numba/tests/test_hashing.py
@@ -5,6 +5,7 @@
 
 import unittest
 
+import os
 import sys
 import subprocess
 from collections import defaultdict
@@ -12,11 +13,11 @@
 
 import numpy as np
 
-from numba import jit
+from numba import jit, config
 from numba.core import types, utils
 import unittest
 from numba.tests.support import (TestCase, tag, CompilationCache,
-                                 skip_unless_py10_or_later)
+                                 skip_unless_py10_or_later, run_in_subprocess)
 
 from numba.cpython.unicode import compile_time_get_string_data
 from numba.cpython import hashing
@@ -72,6 +73,89 @@
         subprocess.check_call([sys.executable, '-c', dedent(work)])
 
 
+class TestHashAlgs(TestCase):
+    # This tests Numba hashing replication against cPython "gold", i.e. the
+    # actual hash values for given inputs, algs and PYTHONHASHSEEDs
+    # Test adapted from:
+    # https://github.com/python/cpython/blob/9dda9020abcf0d51d59b283a89c58c8e1fb0f574/Lib/test/test_hash.py#L197-L264
+    # and
+    # https://github.com/python/cpython/blob/9dda9020abcf0d51d59b283a89c58c8e1fb0f574/Lib/test/test_hash.py#L174-L189
+
+    # 32bit little, 64bit little, 32bit big, 64bit big
+    known_hashes = {
+        'djba33x': [ # only used for small strings
+            # seed 0, 'abc'
+            [193485960, 193485960,  193485960, 193485960],
+            # seed 42, 'abc'
+            [-678966196, 573763426263223372, -820489388, -4282905804826039665],
+            ],
+        'siphash13': [
+            # NOTE: PyUCS2 layout depends on endianness
+            # seed 0, 'abc'
+            [69611762, -4594863902769663758, 69611762, -4594863902769663758],
+            # seed 42, 'abc'
+            [-975800855, 3869580338025362921, -975800855, 3869580338025362921],
+            # seed 42, 'abcdefghijk'
+            [-595844228, 7764564197781545852, -595844228, 7764564197781545852],
+            # seed 0, 'äú∑ℇ'
+            [-1093288643, -2810468059467891395, -1041341092, 4925090034378237276],
+            # seed 42, 'äú∑ℇ'
+            [-585999602, -2845126246016066802, -817336969, -2219421378907968137],
+        ],
+        'siphash24': [
+            # NOTE: PyUCS2 layout depends on endianness
+            # seed 0, 'abc'
+            [1198583518, 4596069200710135518, 1198583518, 4596069200710135518],
+            # seed 42, 'abc'
+            [273876886, -4501618152524544106, 273876886, -4501618152524544106],
+            # seed 42, 'abcdefghijk'
+            [-1745215313, 4436719588892876975, -1745215313, 4436719588892876975],
+            # seed 0, 'äú∑ℇ'
+            [493570806, 5749986484189612790, -1006381564, -5915111450199468540],
+            # seed 42, 'äú∑ℇ'
+            [-1677110816, -2947981342227738144, -1860207793, -4296699217652516017],
+        ],
+    }
+
+    def get_expected_hash(self, position, length):
+        if length < sys.hash_info.cutoff:
+            algorithm = "djba33x"
+        else:
+            algorithm = sys.hash_info.algorithm
+        IS_64BIT = not config.IS_32BITS
+        if sys.byteorder == 'little':
+            platform = 1 if IS_64BIT else 0
+        else:
+            assert(sys.byteorder == 'big')
+            platform = 3 if IS_64BIT else 2
+        return self.known_hashes[algorithm][position][platform]
+
+    def get_hash_command(self, repr_):
+        return 'print(hash(eval(%a)))' % repr_
+
+    def get_hash(self, repr_, seed=None):
+        env = os.environ.copy()
+        if seed is not None:
+            env['PYTHONHASHSEED'] = str(seed)
+        else:
+            env.pop('PYTHONHASHSEED', None)
+        out, _ = run_in_subprocess(code=self.get_hash_command(repr_),
+                                   env=env)
+        stdout = out.decode().strip()
+        return int(stdout)
+
+    def test_against_cpython_gold(self):
+
+        args = (('abc', 0, 0), ('abc', 42, 1), ('abcdefghijk', 42, 2),
+                ('äú∑ℇ', 0, 3), ('äú∑ℇ', 42, 4),)
+
+        for input_str, seed, position in args:
+            with self.subTest(input_str=input_str, seed=seed):
+                got = self.get_hash(repr(input_str), seed=seed)
+                expected = self.get_expected_hash(position, len(input_str))
+                self.assertEqual(got, expected)
+
+
 class BaseTest(TestCase):
 
     def setUp(self):