1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
#ifndef BPF_ATOMIC_H
#define BPF_ATOMIC_H
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include "bpf_experimental.h"
extern bool CONFIG_X86_64 __kconfig __weak;
/*
* __unqual_typeof(x) - Declare an unqualified scalar type, leaving
* non-scalar types unchanged,
*
* Prefer C11 _Generic for better compile-times and simpler code. Note: 'char'
* is not type-compatible with 'signed char', and we define a separate case.
*
* This is copied verbatim from kernel's include/linux/compiler_types.h, but
* with default expression (for pointers) changed from (x) to (typeof(x)0).
*
* This is because LLVM has a bug where for lvalue (x), it does not get rid of
* an extra address_space qualifier, but does in case of rvalue (typeof(x)0).
* Hence, for pointers, we need to create an rvalue expression to get the
* desired type. See https://github.com/llvm/llvm-project/issues/53400.
*/
#define __scalar_type_to_expr_cases(type) \
unsigned type : (unsigned type)0, signed type : (signed type)0
#define __unqual_typeof(x) \
typeof(_Generic((x), \
char: (char)0, \
__scalar_type_to_expr_cases(char), \
__scalar_type_to_expr_cases(short), \
__scalar_type_to_expr_cases(int), \
__scalar_type_to_expr_cases(long), \
__scalar_type_to_expr_cases(long long), \
default: (typeof(x))0))
/* No-op for BPF */
#define cpu_relax() ({})
#define READ_ONCE(x) (*(volatile typeof(x) *)&(x))
#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = (val))
#define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new)
#define try_cmpxchg(p, pold, new) \
({ \
__unqual_typeof(*(pold)) __o = *(pold); \
__unqual_typeof(*(p)) __r = cmpxchg(p, __o, new); \
if (__r != __o) \
*(pold) = __r; \
__r == __o; \
})
#define try_cmpxchg_relaxed(p, pold, new) try_cmpxchg(p, pold, new)
#define try_cmpxchg_acquire(p, pold, new) try_cmpxchg(p, pold, new)
#define smp_mb() \
({ \
volatile unsigned long __val; \
__sync_fetch_and_add(&__val, 0); \
})
#define smp_rmb() \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
else \
barrier(); \
})
#define smp_wmb() \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
else \
barrier(); \
})
/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */
#define smp_acquire__after_ctrl_dep() ({ smp_rmb(); })
#define smp_load_acquire(p) \
({ \
__unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \
if (!CONFIG_X86_64) \
smp_mb(); \
barrier(); \
__v; \
})
#define smp_store_release(p, val) \
({ \
if (!CONFIG_X86_64) \
smp_mb(); \
barrier(); \
WRITE_ONCE(*(p), val); \
})
#define smp_cond_load_relaxed_label(p, cond_expr, label) \
({ \
typeof(p) __ptr = (p); \
__unqual_typeof(*(p)) VAL; \
for (;;) { \
VAL = (__unqual_typeof(*(p)))READ_ONCE(*__ptr); \
if (cond_expr) \
break; \
cond_break_label(label); \
cpu_relax(); \
} \
(typeof(*(p)))VAL; \
})
#define smp_cond_load_acquire_label(p, cond_expr, label) \
({ \
__unqual_typeof(*p) __val = \
smp_cond_load_relaxed_label(p, cond_expr, label); \
smp_acquire__after_ctrl_dep(); \
(typeof(*(p)))__val; \
})
#define atomic_read(p) READ_ONCE((p)->counter)
#define atomic_cond_read_relaxed_label(p, cond_expr, label) \
smp_cond_load_relaxed_label(&(p)->counter, cond_expr, label)
#define atomic_cond_read_acquire_label(p, cond_expr, label) \
smp_cond_load_acquire_label(&(p)->counter, cond_expr, label)
#define atomic_try_cmpxchg_relaxed(p, pold, new) \
try_cmpxchg_relaxed(&(p)->counter, pold, new)
#define atomic_try_cmpxchg_acquire(p, pold, new) \
try_cmpxchg_acquire(&(p)->counter, pold, new)
#endif /* BPF_ATOMIC_H */
|
