1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_PREEMPT_H
#define __ASM_PREEMPT_H
#include <asm/current.h>
#include <linux/thread_info.h>
#include <asm/atomic_ops.h>
#include <asm/march.h>
#ifdef MARCH_HAS_Z196_FEATURES
/* We use the MSB mostly because its available */
#define PREEMPT_NEED_RESCHED 0x80000000
#define PREEMPT_ENABLED (0 + PREEMPT_NEED_RESCHED)
static __always_inline int preempt_count(void)
{
return READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED;
}
static __always_inline void preempt_count_set(int pc)
{
int old, new;
do {
old = READ_ONCE(get_lowcore()->preempt_count);
new = (old & PREEMPT_NEED_RESCHED) |
(pc & ~PREEMPT_NEED_RESCHED);
} while (__atomic_cmpxchg(&get_lowcore()->preempt_count,
old, new) != old);
}
static __always_inline void set_preempt_need_resched(void)
{
__atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
}
static __always_inline void clear_preempt_need_resched(void)
{
__atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
}
static __always_inline bool test_preempt_need_resched(void)
{
return !(READ_ONCE(get_lowcore()->preempt_count) & PREEMPT_NEED_RESCHED);
}
static __always_inline void __preempt_count_add(int val)
{
/*
* With some obscure config options and CONFIG_PROFILE_ALL_BRANCHES
* enabled, gcc 12 fails to handle __builtin_constant_p().
*/
if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES)) {
if (__builtin_constant_p(val) && (val >= -128) && (val <= 127)) {
__atomic_add_const(val, &get_lowcore()->preempt_count);
return;
}
}
__atomic_add(val, &get_lowcore()->preempt_count);
}
static __always_inline void __preempt_count_sub(int val)
{
__preempt_count_add(-val);
}
static __always_inline bool __preempt_count_dec_and_test(void)
{
return __atomic_add(-1, &get_lowcore()->preempt_count) == 1;
}
static __always_inline bool should_resched(int preempt_offset)
{
return unlikely(READ_ONCE(get_lowcore()->preempt_count) ==
preempt_offset);
}
#else /* MARCH_HAS_Z196_FEATURES */
#define PREEMPT_ENABLED (0)
static __always_inline int preempt_count(void)
{
return READ_ONCE(get_lowcore()->preempt_count);
}
static __always_inline void preempt_count_set(int pc)
{
get_lowcore()->preempt_count = pc;
}
static __always_inline void set_preempt_need_resched(void)
{
}
static __always_inline void clear_preempt_need_resched(void)
{
}
static __always_inline bool test_preempt_need_resched(void)
{
return false;
}
static __always_inline void __preempt_count_add(int val)
{
get_lowcore()->preempt_count += val;
}
static __always_inline void __preempt_count_sub(int val)
{
get_lowcore()->preempt_count -= val;
}
static __always_inline bool __preempt_count_dec_and_test(void)
{
return !--get_lowcore()->preempt_count && tif_need_resched();
}
static __always_inline bool should_resched(int preempt_offset)
{
return unlikely(preempt_count() == preempt_offset &&
tif_need_resched());
}
#endif /* MARCH_HAS_Z196_FEATURES */
#define init_task_preempt_count(p) do { } while (0)
/* Deferred to CPU bringup time */
#define init_idle_preempt_count(p, cpu) do { } while (0)
#ifdef CONFIG_PREEMPTION
extern void preempt_schedule(void);
#define __preempt_schedule() preempt_schedule()
extern void preempt_schedule_notrace(void);
#define __preempt_schedule_notrace() preempt_schedule_notrace()
#endif /* CONFIG_PREEMPTION */
#endif /* __ASM_PREEMPT_H */
|
