Subject: Intrduce migrate_disable() + cpu_light()
From: Thomas Gleixner <tglx@linutronix.de>
Date: Fri, 17 Jun 2011 15:42:38 +0200
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.9/older/patches-4.9.20-rt16.tar.xz

Introduce migrate_disable(). The task can't be pushed to another CPU but can
be preempted.

From: Peter Zijlstra <a.p.zijlstra@chello.nl>:
|Make migrate_disable() be a preempt_disable() for !rt kernels. This
|allows generic code to use it but still enforces that these code
|sections stay relatively small.
|
|A preemptible migrate_disable() accessible for general use would allow
|people growing arbitrary per-cpu crap instead of clean these things
|up.

From: Steven Rostedt <rostedt@goodmis.org>
| The migrate_disable() can cause a bit of a overhead to the RT kernel,
| as changing the affinity is expensive to do at every lock encountered.
| As a running task can not migrate, the actual disabling of migration
| does not need to occur until the task is about to schedule out.
|
| In most cases, a task that disables migration will enable it before
| it schedules making this change improve performance tremendously.

On top of this build get/put_cpu_light(). It is similar to get_cpu():
it uses migrate_disable() instead of preempt_disable(). That means the user
remains on the same CPU but the function using it may be preempted and
invoked again from another caller on the same CPU.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpu.h     |    3 ++
 include/linux/preempt.h |    9 ++++++
 include/linux/sched.h   |   39 +++++++++++++++++++++-----
 include/linux/smp.h     |    3 ++
 kernel/sched/core.c     |   70 +++++++++++++++++++++++++++++++++++++++++++++++-
 kernel/sched/debug.c    |    7 ++++
 lib/smp_processor_id.c  |    5 ++-
 7 files changed, 125 insertions(+), 11 deletions(-)

--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -173,6 +173,9 @@ static inline void cpu_notifier_register
 #endif /* CONFIG_SMP */
 extern struct bus_type cpu_subsys;
 
+static inline void pin_current_cpu(void) { }
+static inline void unpin_current_cpu(void) { }
+
 #ifdef CONFIG_HOTPLUG_CPU
 /* Stop CPUs going up and down. */
 
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -257,11 +257,20 @@ do { \
 # define preempt_enable_rt()		preempt_enable()
 # define preempt_disable_nort()		barrier()
 # define preempt_enable_nort()		barrier()
+# ifdef CONFIG_SMP
+   extern void migrate_disable(void);
+   extern void migrate_enable(void);
+# else /* CONFIG_SMP */
+#  define migrate_disable()		barrier()
+#  define migrate_enable()		barrier()
+# endif /* CONFIG_SMP */
 #else
 # define preempt_disable_rt()		barrier()
 # define preempt_enable_rt()		barrier()
 # define preempt_disable_nort()		preempt_disable()
 # define preempt_enable_nort()		preempt_enable()
+# define migrate_disable()		preempt_disable()
+# define migrate_enable()		preempt_enable()
 #endif
 
 #ifdef CONFIG_PREEMPT_NOTIFIERS
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1520,6 +1520,12 @@ struct task_struct {
 #endif
 
 	unsigned int policy;
+#ifdef CONFIG_PREEMPT_RT_FULL
+	int migrate_disable;
+# ifdef CONFIG_SCHED_DEBUG
+	int migrate_disable_atomic;
+# endif
+#endif
 	int nr_cpus_allowed;
 	cpumask_t cpus_allowed;
 
@@ -1991,14 +1997,6 @@ static inline struct vm_struct *task_sta
 }
 #endif
 
-/* Future-safe accessor for struct task_struct's cpus_allowed. */
-#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed)
-
-static inline int tsk_nr_cpus_allowed(struct task_struct *p)
-{
-	return p->nr_cpus_allowed;
-}
-
 #define TNF_MIGRATED	0x01
 #define TNF_NO_GROUP	0x02
 #define TNF_SHARED	0x04
@@ -3516,6 +3514,31 @@ static inline void set_task_cpu(struct t
 
 #endif /* CONFIG_SMP */
 
+static inline int __migrate_disabled(struct task_struct *p)
+{
+#ifdef CONFIG_PREEMPT_RT_FULL
+	return p->migrate_disable;
+#else
+	return 0;
+#endif
+}
+
+/* Future-safe accessor for struct task_struct's cpus_allowed. */
+static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p)
+{
+	if (__migrate_disabled(p))
+		return cpumask_of(task_cpu(p));
+
+	return &p->cpus_allowed;
+}
+
+static inline int tsk_nr_cpus_allowed(struct task_struct *p)
+{
+	if (__migrate_disabled(p))
+		return 1;
+	return p->nr_cpus_allowed;
+}
+
 extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
 extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
 
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -197,6 +197,9 @@ static inline int get_boot_cpu_id(void)
 #define get_cpu()		({ preempt_disable(); smp_processor_id(); })
 #define put_cpu()		preempt_enable()
 
+#define get_cpu_light()		({ migrate_disable(); smp_processor_id(); })
+#define put_cpu_light()		migrate_enable()
+
 /*
  * Callback to arch code if there's nosmp or maxcpus=0 on the
  * boot command line:
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1100,6 +1100,11 @@ void do_set_cpus_allowed(struct task_str
 
 	lockdep_assert_held(&p->pi_lock);
 
+	if (__migrate_disabled(p)) {
+		cpumask_copy(&p->cpus_allowed, new_mask);
+		return;
+	}
+
 	queued = task_on_rq_queued(p);
 	running = task_current(rq, p);
 
@@ -1179,7 +1184,7 @@ static int __set_cpus_allowed_ptr(struct
 	}
 
 	/* Can the task run on the task's current CPU? If so, we're done */
-	if (cpumask_test_cpu(task_cpu(p), new_mask))
+	if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p))
 		goto out;
 
 	dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
@@ -3252,6 +3257,69 @@ static inline void schedule_debug(struct
 	schedstat_inc(this_rq()->sched_count);
 }
 
+#if defined(CONFIG_PREEMPT_RT_FULL) && defined(CONFIG_SMP)
+
+void migrate_disable(void)
+{
+	struct task_struct *p = current;
+
+	if (in_atomic()) {
+#ifdef CONFIG_SCHED_DEBUG
+		p->migrate_disable_atomic++;
+#endif
+		return;
+	}
+
+#ifdef CONFIG_SCHED_DEBUG
+	WARN_ON_ONCE(p->migrate_disable_atomic);
+#endif
+
+	if (p->migrate_disable) {
+		p->migrate_disable++;
+		return;
+	}
+
+	preempt_disable();
+	pin_current_cpu();
+	p->migrate_disable = 1;
+	preempt_enable();
+}
+EXPORT_SYMBOL(migrate_disable);
+
+void migrate_enable(void)
+{
+	struct task_struct *p = current;
+
+	if (in_atomic()) {
+#ifdef CONFIG_SCHED_DEBUG
+		p->migrate_disable_atomic--;
+#endif
+		return;
+	}
+
+#ifdef CONFIG_SCHED_DEBUG
+	WARN_ON_ONCE(p->migrate_disable_atomic);
+#endif
+	WARN_ON_ONCE(p->migrate_disable <= 0);
+
+	if (p->migrate_disable > 1) {
+		p->migrate_disable--;
+		return;
+	}
+
+	preempt_disable();
+	/*
+	 * Clearing migrate_disable causes tsk_cpus_allowed to
+	 * show the tasks original cpu affinity.
+	 */
+	p->migrate_disable = 0;
+
+	unpin_current_cpu();
+	preempt_enable();
+}
+EXPORT_SYMBOL(migrate_enable);
+#endif
+
 /*
  * Pick up the highest-prio task:
  */
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -558,6 +558,9 @@ void print_rt_rq(struct seq_file *m, int
 	P(rt_throttled);
 	PN(rt_time);
 	PN(rt_runtime);
+#ifdef CONFIG_SMP
+	P(rt_nr_migratory);
+#endif
 
 #undef PN
 #undef P
@@ -953,6 +956,10 @@ void proc_sched_show_task(struct task_st
 #endif
 	P(policy);
 	P(prio);
+#ifdef CONFIG_PREEMPT_RT_FULL
+	P(migrate_disable);
+#endif
+	P(nr_cpus_allowed);
 #undef PN_SCHEDSTAT
 #undef PN
 #undef __PN
--- a/lib/smp_processor_id.c
+++ b/lib/smp_processor_id.c
@@ -39,8 +39,9 @@ notrace static unsigned int check_preemp
 	if (!printk_ratelimit())
 		goto out_enable;
 
-	printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x] code: %s/%d\n",
-		what1, what2, preempt_count() - 1, current->comm, current->pid);
+	printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x %08x] code: %s/%d\n",
+		what1, what2, preempt_count() - 1, __migrate_disabled(current),
+		current->comm, current->pid);
 
 	print_symbol("caller is %s\n", (long)__builtin_return_address(0));
 	dump_stack();