1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
|
#pragma once
#include "../../CommonWindows.h"
#include "../../Asserts.h"
#include <atomic>
#if defined(DEATH_TARGET_WINDOWS)
# include "../../Environment.h"
#elif defined(DEATH_TARGET_APPLE)
# include <errno.h>
extern "C"
{
// Source: https://github.com/apple-oss-distributions/xnu/blob/xnu-8792.81.2/bsd/sys/ulock.h
// Modification: added __attribute((__weak__))
// Copyright (c) 2015 Apple Inc. All rights reserved.
__attribute((__weak__))
extern int __ulock_wait2(uint32_t operation, void* addr, uint64_t value, uint64_t timeout, uint64_t value2);
__attribute((__weak__))
extern int __ulock_wake(uint32_t operation, void* addr, uint64_t wake_value);
/* Operation bits [7, 0] contain the operation code. */
#define UL_COMPARE_AND_WAIT 1
#define UL_COMPARE_AND_WAIT_SHARED 3
#define UL_COMPARE_AND_WAIT64 5
#define UL_COMPARE_AND_WAIT64_SHARED 6
/* Operation bits [15, 8] contain the flags for __ulock_wake */
#define ULF_WAKE_ALL 0x00000100
#define ULF_WAKE_THREAD 0x00000200
#define ULF_WAKE_ALLOW_NON_OWNER 0x00000400
/* Operation bits [15, 8] contain the flags for __ulock_wake */
#define ULF_WAKE_ALL 0x00000100
#define ULF_WAKE_THREAD 0x00000200
#define ULF_WAKE_ALLOW_NON_OWNER 0x00000400
/* Operation bits [31, 24] contain the generic flags */
#define ULF_NO_ERRNO 0x01000000
}
#elif defined(__FreeBSD__) || defined(__DragonFly__)
// https://man.freebsd.org/cgi/man.cgi?query=_umtx_op
# include <sys/types.h>
# include <sys/umtx.h>
# include <errno.h>
# include <time.h>
#elif (defined(__linux__) || defined(__linux)) && !defined(__LSB_VERSION__) && !defined(DEATH_TARGET_EMSCRIPTEN)
# include <linux/futex.h>
# include <sys/syscall.h>
# include <errno.h>
# include <time.h>
# include <unistd.h>
#endif
namespace Death { namespace Threading { namespace Implementation {
//###==##====#=====--==~--~=~- --- -- - - - -
template <typename T>
struct RemoveAtomic {
using type = T;
};
template <typename T>
struct RemoveAtomic<std::atomic<T>> {
using type = T;
};
#if defined(DEATH_TARGET_WINDOWS)
using WaitOnAddressDelegate = decltype(::WaitOnAddress);
using WakeByAddressAllDelegate = decltype(::WakeByAddressAll);
using WakeByAddressSingleDelegate = decltype(::WakeByAddressSingle);
extern WaitOnAddressDelegate* _waitOnAddress;
extern WakeByAddressAllDelegate* _wakeByAddressAll;
extern WakeByAddressAllDelegate* _wakeByAddressSingle;
constexpr std::uint32_t Infinite = INFINITE;
void InitializeWaitOnAddress();
template<typename T>
inline bool WaitOnAddress(T& futex, typename RemoveAtomic<T>::type expectedValue, std::uint32_t timeoutMilliseconds)
{
BOOL waitResult = _waitOnAddress(&futex, &expectedValue, sizeof(T), timeoutMilliseconds);
DEATH_DEBUG_ASSERT(waitResult || ::GetLastError() == ERROR_TIMEOUT);
return !!waitResult;
}
template<typename T>
inline void WakeByAddressAll(T& futex)
{
_wakeByAddressAll(&futex);
}
template<typename T>
inline void WakeByAddressSingle(T& futex)
{
_wakeByAddressSingle(&futex);
}
inline bool IsWaitOnAddressSupported()
{
return Environment::IsWindows8();
}
#elif defined(DEATH_TARGET_APPLE)
constexpr std::uint32_t Infinite = ~0;
inline void InitializeWaitOnAddress()
{
}
template<typename T>
inline std::uint32_t GetBaseOperation(T&)
{
static_assert(sizeof(T) >= sizeof(std::uint32_t), "Can only operate on 32-bit or 64-bit variables");
std::uint32_t operation = ULF_NO_ERRNO;
if (sizeof(T) == sizeof(std::uint32_t)) {
operation |= UL_COMPARE_AND_WAIT;
} else {
operation |= UL_COMPARE_AND_WAIT64;
}
return operation;
}
template<typename T>
inline bool WaitOnAddress(T& futex, typename RemoveAtomic<T>::type expectedValue, std::uint32_t timeoutMilliseconds)
{
// Source code inspection shows __ulock_wait2 uses nanoseconds for timeout
std::uint64_t timeoutNanoseconds = (timeoutMilliseconds == Infinite ? UINT64_MAX : (timeoutMilliseconds * 1000000ull));
int r = __ulock_wait2(GetBaseOperation(futex), &futex, std::uint64_t(expectedValue), timeoutNanoseconds, 0);
return (r == 0 || r != -ETIMEDOUT);
}
template<typename T>
inline void WakeByAddressAll(T& futex)
{
__ulock_wake(GetBaseOperation(futex) | ULF_WAKE_ALL, &futex, 0);
}
template<typename T>
inline void WakeByAddressSingle(T& futex)
{
__ulock_wake(GetBaseOperation(futex), &futex, 0);
}
inline bool IsWaitOnAddressSupported()
{
return (__ulock_wake != nullptr && __ulock_wait2 != nullptr);
}
#elif defined(__FreeBSD__) || defined(__DragonFly__)
# define __DEATH_ALWAYS_USE_WAKEONADDRESS
constexpr std::uint32_t Infinite = ~0;
inline void InitializeWaitOnAddress()
{
}
template <typename T>
inline int WaitOnAddressInner(T& futex, typename RemoveAtomic<T>::type expectedValue, _umtx_time* tmp = nullptr)
{
// FreeBSD UMTX_OP_WAIT does not apply acquire or release memory barriers
int op = UMTX_OP_WAIT_UINT_PRIVATE;
if (sizeof(T) > sizeof(std::uint32_t)) {
op = UMTX_OP_WAIT; // No _PRIVATE version
}
// The timeout is passed in uaddr2, with its size in uaddr
void* uaddr = reinterpret_cast<void*>(tmp ? sizeof(*tmp) : 0);
void* uaddr2 = tmp;
return _umtx_op(&futex, op, (u_long)expectedValue, uaddr, uaddr2);
}
template<typename T>
inline bool WaitOnAddress(T& futex, typename RemoveAtomic<T>::type expectedValue, std::uint32_t timeoutMilliseconds)
{
if (timeoutMilliseconds == Infinite) {
int r = WaitOnAddressInner(futex, expectedValue);
return (r == 0);
} else {
struct _umtx_time tm = {};
clock_gettime(CLOCK_MONOTONIC, &tm._timeout);
tm._timeout.tv_sec += timeoutMilliseconds / 1000;
tm._timeout.tv_nsec += (timeoutMilliseconds % 1000) * 1000000;
if (tm._timeout.tv_nsec >= 1000000000) {
tm._timeout.tv_sec += tm._timeout.tv_nsec / 1000000000;
tm._timeout.tv_nsec %= 1000000000;
}
tm._flags = UMTX_ABSTIME;
tm._clockid = CLOCK_MONOTONIC;
int r = WaitOnAddressInner(futex, expectedValue, &tm);
return (r == 0 || errno != ETIMEDOUT);
}
}
template<typename T>
inline void WakeByAddressAll(T& futex)
{
_umtx_op(&futex, UMTX_OP_WAKE_PRIVATE, INT32_MAX, nullptr, nullptr);
}
template<typename T>
inline void WakeByAddressSingle(T& futex)
{
_umtx_op(&futex, UMTX_OP_WAKE_PRIVATE, 1, nullptr, nullptr);
}
inline constexpr bool IsWaitOnAddressSupported()
{
return true;
}
#elif (defined(__linux__) || defined(__linux)) && !defined(__LSB_VERSION__) && !defined(DEATH_TARGET_EMSCRIPTEN)
# define __DEATH_ALWAYS_USE_WAKEONADDRESS
constexpr std::uint32_t Infinite = ~0;
inline void InitializeWaitOnAddress()
{
}
inline long FutexOp(int* addr, int op, int val, std::uintptr_t val2 = 0, int* addr2 = nullptr, int val3 = 0) noexcept
{
// We use __NR_futex because some libcs (like Android's bionic) don't provide SYS_futex
return syscall(__NR_futex, addr, op | FUTEX_PRIVATE_FLAG, val, val2, addr2, val3);
}
template<typename T>
int* GetFutexAddress(T* ptr)
{
int* intPtr = reinterpret_cast<int*>(ptr);
# if defined(DEATH_TARGET_BIG_ENDIAN)
if (sizeof(T) > sizeof(int)) {
intPtr++; // We want a pointer to the least significant half
}
# endif
return intPtr;
}
template<typename T>
inline bool WaitOnAddress(T& futex, typename RemoveAtomic<T>::type expectedValue, std::uint32_t timeoutMilliseconds)
{
if (timeoutMilliseconds == Infinite) {
long r = FutexOp(GetFutexAddress(&futex), FUTEX_WAIT, (std::uintptr_t)expectedValue);
return (r == 0);
} else {
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
ts.tv_sec += timeoutMilliseconds / 1000;
ts.tv_nsec += (timeoutMilliseconds % 1000) * 1000000;
if (ts.tv_nsec >= 1000000000) {
ts.tv_sec += ts.tv_nsec / 1000000000;
ts.tv_nsec %= 1000000000;
}
long r = FutexOp(GetFutexAddress(&futex), FUTEX_WAIT_BITSET, (std::uintptr_t)expectedValue, (std::uintptr_t)&ts, nullptr, FUTEX_BITSET_MATCH_ANY);
return (r == 0 || errno != ETIMEDOUT);
}
}
template<typename T>
inline void WakeByAddressAll(T& futex)
{
FutexOp(GetFutexAddress(&futex), FUTEX_WAKE, INT32_MAX);
}
template<typename T>
inline void WakeByAddressSingle(T& futex)
{
FutexOp(GetFutexAddress(&futex), FUTEX_WAKE, 1);
}
inline constexpr bool IsWaitOnAddressSupported()
{
return true;
}
#else
constexpr std::uint32_t Infinite = ~0;
inline void InitializeWaitOnAddress()
{
}
template<typename T>
inline bool WaitOnAddress(T& futex, typename RemoveAtomic<T>::type expectedValue, std::uint32_t timeoutMilliseconds)
{
return false;
}
template<typename T>
inline void WakeByAddressAll(T& futex)
{
}
template<typename T>
inline void WakeByAddressSingle(T& futex)
{
}
inline constexpr bool IsWaitOnAddressSupported()
{
return false;
}
#endif
}}}
|