mirror of
https://github.com/fail0verflow/switch-linux.git
synced 2025-05-04 02:34:21 -04:00
The current mutex implementation has an atomic lock word and a non-atomic owner field. This disparity leads to a number of issues with the current mutex code as it means that we can have a locked mutex without an explicit owner (because the owner field has not been set, or already cleared). This leads to a number of weird corner cases, esp. between the optimistic spinning and debug code. Where the optimistic spinning code needs the owner field updated inside the lock region, the debug code is more relaxed because the whole lock is serialized by the wait_lock. Also, the spinning code itself has a few corner cases where we need to deal with a held lock without an owner field. Furthermore, it becomes even more of a problem when trying to fix starvation cases in the current code. We end up stacking special case on special case. To solve this rework the basic mutex implementation to be a single atomic word that contains the owner and uses the low bits for extra state. This matches how PI futexes and rt_mutex already work. By having the owner an integral part of the lock state a lot of the problems dissapear and we get a better option to deal with starvation cases, direct owner handoff. Changing the basic mutex does however invalidate all the arch specific mutex code; this patch leaves that unused in-place, a later patch will remove that. Tested-by: Jason Low <jason.low2@hpe.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Reviewed-by: Will Deacon <will.deacon@arm.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
192 lines
5.7 KiB
C
192 lines
5.7 KiB
C
/*
|
|
* Mutexes: blocking mutual exclusion locks
|
|
*
|
|
* started by Ingo Molnar:
|
|
*
|
|
* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
|
|
*
|
|
* This file contains the main data structure and API definitions.
|
|
*/
|
|
#ifndef __LINUX_MUTEX_H
|
|
#define __LINUX_MUTEX_H
|
|
|
|
#include <asm/current.h>
|
|
#include <linux/list.h>
|
|
#include <linux/spinlock_types.h>
|
|
#include <linux/linkage.h>
|
|
#include <linux/lockdep.h>
|
|
#include <linux/atomic.h>
|
|
#include <asm/processor.h>
|
|
#include <linux/osq_lock.h>
|
|
#include <linux/debug_locks.h>
|
|
|
|
/*
|
|
* Simple, straightforward mutexes with strict semantics:
|
|
*
|
|
* - only one task can hold the mutex at a time
|
|
* - only the owner can unlock the mutex
|
|
* - multiple unlocks are not permitted
|
|
* - recursive locking is not permitted
|
|
* - a mutex object must be initialized via the API
|
|
* - a mutex object must not be initialized via memset or copying
|
|
* - task may not exit with mutex held
|
|
* - memory areas where held locks reside must not be freed
|
|
* - held mutexes must not be reinitialized
|
|
* - mutexes may not be used in hardware or software interrupt
|
|
* contexts such as tasklets and timers
|
|
*
|
|
* These semantics are fully enforced when DEBUG_MUTEXES is
|
|
* enabled. Furthermore, besides enforcing the above rules, the mutex
|
|
* debugging code also implements a number of additional features
|
|
* that make lock debugging easier and faster:
|
|
*
|
|
* - uses symbolic names of mutexes, whenever they are printed in debug output
|
|
* - point-of-acquire tracking, symbolic lookup of function names
|
|
* - list of all locks held in the system, printout of them
|
|
* - owner tracking
|
|
* - detects self-recursing locks and prints out all relevant info
|
|
* - detects multi-task circular deadlocks and prints out all affected
|
|
* locks and tasks (and only those tasks)
|
|
*/
|
|
struct mutex {
|
|
atomic_long_t owner;
|
|
spinlock_t wait_lock;
|
|
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
|
|
struct optimistic_spin_queue osq; /* Spinner MCS lock */
|
|
#endif
|
|
struct list_head wait_list;
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
void *magic;
|
|
#endif
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
struct lockdep_map dep_map;
|
|
#endif
|
|
};
|
|
|
|
static inline struct task_struct *__mutex_owner(struct mutex *lock)
|
|
{
|
|
return (struct task_struct *)(atomic_long_read(&lock->owner) & ~0x03);
|
|
}
|
|
|
|
/*
|
|
* This is the control structure for tasks blocked on mutex,
|
|
* which resides on the blocked task's kernel stack:
|
|
*/
|
|
struct mutex_waiter {
|
|
struct list_head list;
|
|
struct task_struct *task;
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
void *magic;
|
|
#endif
|
|
};
|
|
|
|
#ifdef CONFIG_DEBUG_MUTEXES
|
|
|
|
#define __DEBUG_MUTEX_INITIALIZER(lockname) \
|
|
, .magic = &lockname
|
|
|
|
extern void mutex_destroy(struct mutex *lock);
|
|
|
|
#else
|
|
|
|
# define __DEBUG_MUTEX_INITIALIZER(lockname)
|
|
|
|
static inline void mutex_destroy(struct mutex *lock) {}
|
|
|
|
#endif
|
|
|
|
/**
|
|
* mutex_init - initialize the mutex
|
|
* @mutex: the mutex to be initialized
|
|
*
|
|
* Initialize the mutex to unlocked state.
|
|
*
|
|
* It is not allowed to initialize an already locked mutex.
|
|
*/
|
|
#define mutex_init(mutex) \
|
|
do { \
|
|
static struct lock_class_key __key; \
|
|
\
|
|
__mutex_init((mutex), #mutex, &__key); \
|
|
} while (0)
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \
|
|
, .dep_map = { .name = #lockname }
|
|
#else
|
|
# define __DEP_MAP_MUTEX_INITIALIZER(lockname)
|
|
#endif
|
|
|
|
#define __MUTEX_INITIALIZER(lockname) \
|
|
{ .owner = ATOMIC_LONG_INIT(0) \
|
|
, .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \
|
|
, .wait_list = LIST_HEAD_INIT(lockname.wait_list) \
|
|
__DEBUG_MUTEX_INITIALIZER(lockname) \
|
|
__DEP_MAP_MUTEX_INITIALIZER(lockname) }
|
|
|
|
#define DEFINE_MUTEX(mutexname) \
|
|
struct mutex mutexname = __MUTEX_INITIALIZER(mutexname)
|
|
|
|
extern void __mutex_init(struct mutex *lock, const char *name,
|
|
struct lock_class_key *key);
|
|
|
|
/**
|
|
* mutex_is_locked - is the mutex locked
|
|
* @lock: the mutex to be queried
|
|
*
|
|
* Returns 1 if the mutex is locked, 0 if unlocked.
|
|
*/
|
|
static inline int mutex_is_locked(struct mutex *lock)
|
|
{
|
|
/*
|
|
* XXX think about spin_is_locked
|
|
*/
|
|
return __mutex_owner(lock) != NULL;
|
|
}
|
|
|
|
/*
|
|
* See kernel/locking/mutex.c for detailed documentation of these APIs.
|
|
* Also see Documentation/locking/mutex-design.txt.
|
|
*/
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass);
|
|
extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock);
|
|
|
|
extern int __must_check mutex_lock_interruptible_nested(struct mutex *lock,
|
|
unsigned int subclass);
|
|
extern int __must_check mutex_lock_killable_nested(struct mutex *lock,
|
|
unsigned int subclass);
|
|
|
|
#define mutex_lock(lock) mutex_lock_nested(lock, 0)
|
|
#define mutex_lock_interruptible(lock) mutex_lock_interruptible_nested(lock, 0)
|
|
#define mutex_lock_killable(lock) mutex_lock_killable_nested(lock, 0)
|
|
|
|
#define mutex_lock_nest_lock(lock, nest_lock) \
|
|
do { \
|
|
typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \
|
|
_mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \
|
|
} while (0)
|
|
|
|
#else
|
|
extern void mutex_lock(struct mutex *lock);
|
|
extern int __must_check mutex_lock_interruptible(struct mutex *lock);
|
|
extern int __must_check mutex_lock_killable(struct mutex *lock);
|
|
|
|
# define mutex_lock_nested(lock, subclass) mutex_lock(lock)
|
|
# define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock)
|
|
# define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock)
|
|
# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock)
|
|
#endif
|
|
|
|
/*
|
|
* NOTE: mutex_trylock() follows the spin_trylock() convention,
|
|
* not the down_trylock() convention!
|
|
*
|
|
* Returns 1 if the mutex has been acquired successfully, and 0 on contention.
|
|
*/
|
|
extern int mutex_trylock(struct mutex *lock);
|
|
extern void mutex_unlock(struct mutex *lock);
|
|
|
|
extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);
|
|
|
|
#endif /* __LINUX_MUTEX_H */
|