1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Date: Wed, 13 Jul 2016 18:22:23 +0200
Subject: [PATCH] timer: make the base lock raw
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.9/older/patches-4.9.20-rt16.tar.xz
The part where the base lock is held got more predictable / shorter after the
timer rework. One reason is the lack of re-cascading.
That means the lock can be made raw and held in IRQ context.
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
kernel/time/timer.c | 48 ++++++++++++++++++++++++------------------------
1 file changed, 24 insertions(+), 24 deletions(-)
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -193,7 +193,7 @@ EXPORT_SYMBOL(jiffies_64);
#endif
struct timer_base {
- spinlock_t lock;
+ raw_spinlock_t lock;
struct timer_list *running_timer;
unsigned long clk;
unsigned long next_expiry;
@@ -948,10 +948,10 @@ static struct timer_base *lock_timer_bas
if (!(tf & TIMER_MIGRATING)) {
base = get_timer_base(tf);
- spin_lock_irqsave(&base->lock, *flags);
+ raw_spin_lock_irqsave(&base->lock, *flags);
if (timer->flags == tf)
return base;
- spin_unlock_irqrestore(&base->lock, *flags);
+ raw_spin_unlock_irqrestore(&base->lock, *flags);
}
cpu_relax();
}
@@ -1023,9 +1023,9 @@ static inline int
/* See the comment in lock_timer_base() */
timer->flags |= TIMER_MIGRATING;
- spin_unlock(&base->lock);
+ raw_spin_unlock(&base->lock);
base = new_base;
- spin_lock(&base->lock);
+ raw_spin_lock(&base->lock);
WRITE_ONCE(timer->flags,
(timer->flags & ~TIMER_BASEMASK) | base->cpu);
}
@@ -1050,7 +1050,7 @@ static inline int
}
out_unlock:
- spin_unlock_irqrestore(&base->lock, flags);
+ raw_spin_unlock_irqrestore(&base->lock, flags);
return ret;
}
@@ -1144,16 +1144,16 @@ void add_timer_on(struct timer_list *tim
if (base != new_base) {
timer->flags |= TIMER_MIGRATING;
- spin_unlock(&base->lock);
+ raw_spin_unlock(&base->lock);
base = new_base;
- spin_lock(&base->lock);
+ raw_spin_lock(&base->lock);
WRITE_ONCE(timer->flags,
(timer->flags & ~TIMER_BASEMASK) | cpu);
}
debug_activate(timer, timer->expires);
internal_add_timer(base, timer);
- spin_unlock_irqrestore(&base->lock, flags);
+ raw_spin_unlock_irqrestore(&base->lock, flags);
}
EXPORT_SYMBOL_GPL(add_timer_on);
@@ -1180,7 +1180,7 @@ int del_timer(struct timer_list *timer)
if (timer_pending(timer)) {
base = lock_timer_base(timer, &flags);
ret = detach_if_pending(timer, base, true);
- spin_unlock_irqrestore(&base->lock, flags);
+ raw_spin_unlock_irqrestore(&base->lock, flags);
}
return ret;
@@ -1208,7 +1208,7 @@ int try_to_del_timer_sync(struct timer_l
timer_stats_timer_clear_start_info(timer);
ret = detach_if_pending(timer, base, true);
}
- spin_unlock_irqrestore(&base->lock, flags);
+ raw_spin_unlock_irqrestore(&base->lock, flags);
return ret;
}
@@ -1340,13 +1340,13 @@ static void expire_timers(struct timer_b
data = timer->data;
if (timer->flags & TIMER_IRQSAFE) {
- spin_unlock(&base->lock);
+ raw_spin_unlock(&base->lock);
call_timer_fn(timer, fn, data);
- spin_lock(&base->lock);
+ raw_spin_lock(&base->lock);
} else {
- spin_unlock_irq(&base->lock);
+ raw_spin_unlock_irq(&base->lock);
call_timer_fn(timer, fn, data);
- spin_lock_irq(&base->lock);
+ raw_spin_lock_irq(&base->lock);
}
}
}
@@ -1515,7 +1515,7 @@ u64 get_next_timer_interrupt(unsigned lo
if (cpu_is_offline(smp_processor_id()))
return expires;
- spin_lock(&base->lock);
+ raw_spin_lock(&base->lock);
nextevt = __next_timer_interrupt(base);
is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA);
base->next_expiry = nextevt;
@@ -1543,7 +1543,7 @@ u64 get_next_timer_interrupt(unsigned lo
if ((expires - basem) > TICK_NSEC)
base->is_idle = true;
}
- spin_unlock(&base->lock);
+ raw_spin_unlock(&base->lock);
return cmp_next_hrtimer_event(basem, expires);
}
@@ -1630,7 +1630,7 @@ static inline void __run_timers(struct t
if (!time_after_eq(jiffies, base->clk))
return;
- spin_lock_irq(&base->lock);
+ raw_spin_lock_irq(&base->lock);
while (time_after_eq(jiffies, base->clk)) {
@@ -1641,7 +1641,7 @@ static inline void __run_timers(struct t
expire_timers(base, heads + levels);
}
base->running_timer = NULL;
- spin_unlock_irq(&base->lock);
+ raw_spin_unlock_irq(&base->lock);
}
/*
@@ -1836,16 +1836,16 @@ int timers_dead_cpu(unsigned int cpu)
* The caller is globally serialized and nobody else
* takes two locks at once, deadlock is not possible.
*/
- spin_lock_irq(&new_base->lock);
- spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
+ raw_spin_lock_irq(&new_base->lock);
+ raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
BUG_ON(old_base->running_timer);
for (i = 0; i < WHEEL_SIZE; i++)
migrate_timer_list(new_base, old_base->vectors + i);
- spin_unlock(&old_base->lock);
- spin_unlock_irq(&new_base->lock);
+ raw_spin_unlock(&old_base->lock);
+ raw_spin_unlock_irq(&new_base->lock);
put_cpu_ptr(&timer_bases);
}
return 0;
@@ -1861,7 +1861,7 @@ static void __init init_timer_cpu(int cp
for (i = 0; i < NR_BASES; i++) {
base = per_cpu_ptr(&timer_bases[i], cpu);
base->cpu = cpu;
- spin_lock_init(&base->lock);
+ raw_spin_lock_init(&base->lock);
base->clk = jiffies;
}
}
|
