1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Date: Wed, 9 Apr 2014 11:58:17 +0200
Subject: percpu_ida: Use local locks
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.9/older/patches-4.9.20-rt16.tar.xz
the local_irq_save() + spin_lock() does not work that well on -RT
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
lib/percpu_ida.c | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
--- a/lib/percpu_ida.c
+++ b/lib/percpu_ida.c
@@ -26,6 +26,9 @@
#include <linux/string.h>
#include <linux/spinlock.h>
#include <linux/percpu_ida.h>
+#include <linux/locallock.h>
+
+static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock);
struct percpu_ida_cpu {
/*
@@ -148,13 +151,13 @@ int percpu_ida_alloc(struct percpu_ida *
unsigned long flags;
int tag;
- local_irq_save(flags);
+ local_lock_irqsave(irq_off_lock, flags);
tags = this_cpu_ptr(pool->tag_cpu);
/* Fastpath */
tag = alloc_local_tag(tags);
if (likely(tag >= 0)) {
- local_irq_restore(flags);
+ local_unlock_irqrestore(irq_off_lock, flags);
return tag;
}
@@ -173,6 +176,7 @@ int percpu_ida_alloc(struct percpu_ida *
if (!tags->nr_free)
alloc_global_tags(pool, tags);
+
if (!tags->nr_free)
steal_tags(pool, tags);
@@ -184,7 +188,7 @@ int percpu_ida_alloc(struct percpu_ida *
}
spin_unlock(&pool->lock);
- local_irq_restore(flags);
+ local_unlock_irqrestore(irq_off_lock, flags);
if (tag >= 0 || state == TASK_RUNNING)
break;
@@ -196,7 +200,7 @@ int percpu_ida_alloc(struct percpu_ida *
schedule();
- local_irq_save(flags);
+ local_lock_irqsave(irq_off_lock, flags);
tags = this_cpu_ptr(pool->tag_cpu);
}
if (state != TASK_RUNNING)
@@ -221,7 +225,7 @@ void percpu_ida_free(struct percpu_ida *
BUG_ON(tag >= pool->nr_tags);
- local_irq_save(flags);
+ local_lock_irqsave(irq_off_lock, flags);
tags = this_cpu_ptr(pool->tag_cpu);
spin_lock(&tags->lock);
@@ -253,7 +257,7 @@ void percpu_ida_free(struct percpu_ida *
spin_unlock(&pool->lock);
}
- local_irq_restore(flags);
+ local_unlock_irqrestore(irq_off_lock, flags);
}
EXPORT_SYMBOL_GPL(percpu_ida_free);
@@ -345,7 +349,7 @@ int percpu_ida_for_each_free(struct perc
struct percpu_ida_cpu *remote;
unsigned cpu, i, err = 0;
- local_irq_save(flags);
+ local_lock_irqsave(irq_off_lock, flags);
for_each_possible_cpu(cpu) {
remote = per_cpu_ptr(pool->tag_cpu, cpu);
spin_lock(&remote->lock);
@@ -367,7 +371,7 @@ int percpu_ida_for_each_free(struct perc
}
spin_unlock(&pool->lock);
out:
- local_irq_restore(flags);
+ local_unlock_irqrestore(irq_off_lock, flags);
return err;
}
EXPORT_SYMBOL_GPL(percpu_ida_for_each_free);
|
