Messages in this thread Patch in this message | | | From | "Paul E. McKenney" <> | Subject | [PATCH RFC tip/core/rcu 18/18] rcu: add primitives to check for RCU read-side critical sections | Date | Tue, 15 Dec 2009 15:02:41 -0800 |
| |
From: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Proposed for 2.6.34, not for inclusion.
Create rcu_read_lock_held(), rcu_read_lock_bh_held(), rcu_read_lock_sched_held(), and srcu_read_lock_held() primitives that return non-zero if there might be the corresponding type of RCU read-side critical section in effect at the time that they are invoked. If there is doubt, they report being in the critical section. They give exact answers if CONFIG_PROVE_LOCKING.
Also create rcu_dereference_check(), which takes a second boolean argument into which one puts rcu_read_lock_held() or similar. For example:
rcu_dereference_check(gp, rcu_read_lock_held() || lockdep_is_held(my_lock));
There will likely need to be a lockdep_might_be_held() to handle the case where debug_locks==0.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> --- include/linux/rcupdate.h | 85 ++++++++++++++++++++++++++++++++++++++++++++++ include/linux/rcutiny.h | 4 -- include/linux/rcutree.h | 1 - include/linux/srcu.h | 24 +++++++++++++ kernel/rcutorture.c | 12 +++++- kernel/rcutree_plugin.h | 22 ------------ lib/debug_locks.c | 1 + 7 files changed, 120 insertions(+), 29 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 723c564..84b891d 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -95,6 +95,70 @@ extern struct lockdep_map rcu_sched_lock_map; # define rcu_read_release_sched() \ lock_release(&rcu_sched_lock_map, 1, _THIS_IP_) +/** + * rcu_read_lock_held - might we be in RCU read-side critical section? + * + * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in + * an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, + * this assumes we are in an RCU read-side critical section unless it can + * prove otherwise. + */ +static inline int rcu_read_lock_held(void) +{ + if (debug_locks) + return lock_is_held(&rcu_lock_map); + return 1; +} + +/** + * rcu_read_lock_bh_held - might we be in RCU-bh read-side critical section? + * + * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in + * an RCU-bh read-side critical section. In absence of CONFIG_PROVE_LOCKING, + * this assumes we are in an RCU-bh read-side critical section unless it can + * prove otherwise. + */ +static inline int rcu_read_lock_bh_held(void) +{ + if (debug_locks) + return lock_is_held(&rcu_bh_lock_map); + return 1; +} + +/** + * rcu_read_lock_sched_held - might we be in RCU-sched read-side critical section? + * + * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in an + * RCU-sched read-side critical section. In absence of CONFIG_PROVE_LOCKING, + * this assumes we are in an RCU-sched read-side critical section unless it + * can prove otherwise. Note that disabling of preemption (including + * disabling irqs) counts as an RCU-sched read-side critical section. + */ +static inline int rcu_read_lock_sched_held(void) +{ + int lockdep_opinion = 0; + + if (debug_locks) + lockdep_opinion = lock_is_held(&rcu_sched_lock_map); + return lockdep_opinion || preempt_count() != 0; +} + +/** + * rcu_dereference_check - rcu_dereference with debug checking + * + * Do an rcu_dereference(), but check that the context is correct. + * For example, rcu_dereference_check(gp, rcu_read_lock_held()) to + * ensure that the rcu_dereference_check() executes within an RCU + * read-side critical section. It is also possible to check for + * locks being held, for example, by using lockdep_is_held(). + */ +#define rcu_dereference_check(p, c) \ + ({ \ + if (debug_locks) \ + WARN_ON_ONCE(!c); \ + rcu_dereference(p); \ + }) + #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ # define rcu_read_acquire() do { } while (0) @@ -104,6 +168,27 @@ extern struct lockdep_map rcu_sched_lock_map; # define rcu_read_acquire_sched() do { } while (0) # define rcu_read_release_sched() do { } while (0) +static inline int rcu_read_lock_held(void) +{ + return 1; +} + +static inline int rcu_read_lock_bh_held(void) +{ + return 1; +} + +static inline int rcu_read_lock_sched_held(void) +{ + return preempt_count() != 0; +} + +#define rcu_dereference_check(p, c) \ + ({ \ + (void)(c); \ + rcu_dereference(p); \ + }) + #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index c32b16d..b524590 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -29,10 +29,6 @@ void rcu_sched_qs(int cpu); void rcu_bh_qs(int cpu); -static inline int rcu_read_lock_held(void) -{ - return 0; -} #define __rcu_read_lock() preempt_disable() #define __rcu_read_unlock() preempt_enable() diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 8cd4ac1..564a025 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -37,7 +37,6 @@ extern void rcu_bh_qs(int cpu); extern int rcu_needs_cpu(int cpu); extern void rcu_scheduler_starting(void); extern int rcu_expedited_torture_stats(char *page); -extern int rcu_read_lock_held(void); #ifdef CONFIG_TREE_PREEMPT_RCU diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 5a07b90..9404d35 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -52,14 +52,38 @@ void synchronize_srcu_expedited(struct srcu_struct *sp); long srcu_batches_completed(struct srcu_struct *sp); #ifdef CONFIG_DEBUG_LOCK_ALLOC + extern struct lockdep_map srcu_lock_map; # define srcu_read_acquire() \ lock_acquire(&srcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) # define srcu_read_release() \ lock_release(&srcu_lock_map, 1, _THIS_IP_) + +/** + * srcu_read_lock_held - might we be in SRCU read-side critical section? + * + * If CONFIG_PROVE_LOCKING is selected and enabled, returns nonzero iff in + * an SRCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, + * this assumes we are in an SRCU read-side critical section unless it can + * prove otherwise. + */ +static inline int srcu_read_lock_held(void) +{ + if (debug_locks) + return lock_is_held(&srcu_lock_map); + return 1; +} + #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ + # define srcu_read_acquire() do { } while (0) # define srcu_read_release() do { } while (0) + +static inline int srcu_read_lock_held(void) +{ + return 1; +} + #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c index b4096d3..dc986f0 100644 --- a/kernel/rcutorture.c +++ b/kernel/rcutorture.c @@ -796,7 +796,11 @@ static void rcu_torture_timer(unsigned long unused) idx = cur_ops->readlock(); completed = cur_ops->completed(); - p = rcu_dereference(rcu_torture_current); + p = rcu_dereference_check(rcu_torture_current, + rcu_read_lock_held() || + rcu_read_lock_bh_held() || + rcu_read_lock_sched_held() || + srcu_read_lock_held()); if (p == NULL) { /* Leave because rcu_torture_writer is not yet underway */ cur_ops->readunlock(idx); @@ -853,7 +857,11 @@ rcu_torture_reader(void *arg) } idx = cur_ops->readlock(); completed = cur_ops->completed(); - p = rcu_dereference(rcu_torture_current); + p = rcu_dereference_check(rcu_torture_current, + rcu_read_lock_held() || + rcu_read_lock_bh_held() || + rcu_read_lock_sched_held() || + srcu_read_lock_held()); if (p == NULL) { /* Wait for rcu_torture_writer to get underway */ cur_ops->readunlock(idx); diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h index f6258ae..e77cdf3 100644 --- a/kernel/rcutree_plugin.h +++ b/kernel/rcutree_plugin.h @@ -310,18 +310,6 @@ void __rcu_read_unlock(void) } EXPORT_SYMBOL_GPL(__rcu_read_unlock); -/* - * Return 1 if the current task is provably within an RCU read-side - * critical section. The bit about checking a running task to see if - * it is blocked is a bit strange, but keep in mind that sleep and - * wakeup are not atomic operations. - */ -int rcu_read_lock_held(void) -{ - return ACCESS_ONCE(current->rcu_read_lock_nesting) != 0 || - (current->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED); -} - #ifdef CONFIG_RCU_CPU_STALL_DETECTOR /* @@ -773,16 +761,6 @@ static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags) #endif /* #ifdef CONFIG_HOTPLUG_CPU */ -/* - * Return 1 if the current task is provably within an RCU read-side - * critical section. But without preemptible RCU, we never can be - * sure, so always return 0. - */ -int rcu_read_lock_held(void) -{ - return 0; -} - #ifdef CONFIG_RCU_CPU_STALL_DETECTOR /* diff --git a/lib/debug_locks.c b/lib/debug_locks.c index bc3b117..5bf0020 100644 --- a/lib/debug_locks.c +++ b/lib/debug_locks.c @@ -23,6 +23,7 @@ * shut up after that. */ int debug_locks = 1; +EXPORT_SYMBOL_GPL(debug_locks); /* * The locking-testsuite uses <debug_locks_silent> to get a -- 1.5.2.5
| |