Index: sys/kern/kern_mutex.c =================================================================== --- sys/kern/kern_mutex.c (revision 227657) +++ sys/kern/kern_mutex.c (working copy) @@ -274,7 +274,7 @@ _mtx_unlock_spin_flags(struct mtx *m, int opts, co * is already owned, it will recursively acquire the lock. */ int -_mtx_trylock(struct mtx *m, int opts, const char *file, int line) +mtx_trylock_flags_(struct mtx *m, int opts, const char *file, int line) { #ifdef LOCK_PROFILING uint64_t waittime = 0; @@ -540,7 +540,7 @@ _mtx_lock_spin(struct mtx *m, uintptr_t tid, int o #endif /* SMP */ void -_thread_lock_flags(struct thread *td, int opts, const char *file, int line) +thread_lock_flags_(struct thread *td, int opts, const char *file, int line) { struct mtx *m; uintptr_t tid; Index: sys/dev/ppbus/ppb_base.c =================================================================== --- sys/dev/ppbus/ppb_base.c (revision 227657) +++ sys/dev/ppbus/ppb_base.c (working copy) @@ -236,11 +236,9 @@ ppb_unlock(device_t bus) void _ppb_assert_locked(device_t bus, const char *file, int line) { -#ifdef INVARIANTS struct ppb_data *ppb = DEVTOSOFTC(bus); - _mtx_assert(ppb->ppc_lock, MA_OWNED, file, line); -#endif + mtx_assert_(ppb->ppc_lock, MA_OWNED, file, line); } void Index: sys/vm/vm_map.c =================================================================== --- sys/vm/vm_map.c (revision 227657) +++ sys/vm/vm_map.c (working copy) @@ -464,7 +464,7 @@ _vm_map_lock(vm_map_t map, const char *file, int l { if (map->system_map) - _mtx_lock_flags(&map->system_mtx, 0, file, line); + mtx_lock_flags_(&map->system_mtx, 0, file, line); else (void)_sx_xlock(&map->lock, 0, file, line); map->timestamp++; @@ -489,7 +489,7 @@ _vm_map_unlock(vm_map_t map, const char *file, int { if (map->system_map) - _mtx_unlock_flags(&map->system_mtx, 0, file, line); + mtx_unlock_flags_(&map->system_mtx, 0, file, line); else { _sx_xunlock(&map->lock, file, line); vm_map_process_deferred(); @@ -501,7 +501,7 @@ _vm_map_lock_read(vm_map_t map, const char *file, { if (map->system_map) - _mtx_lock_flags(&map->system_mtx, 0, file, line); + mtx_lock_flags_(&map->system_mtx, 0, file, line); else (void)_sx_slock(&map->lock, 0, file, line); } @@ -511,7 +511,7 @@ _vm_map_unlock_read(vm_map_t map, const char *file { if (map->system_map) - _mtx_unlock_flags(&map->system_mtx, 0, file, line); + mtx_unlock_flags_(&map->system_mtx, 0, file, line); else { _sx_sunlock(&map->lock, file, line); vm_map_process_deferred(); @@ -524,7 +524,7 @@ _vm_map_trylock(vm_map_t map, const char *file, in int error; error = map->system_map ? - !_mtx_trylock(&map->system_mtx, 0, file, line) : + !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : !_sx_try_xlock(&map->lock, file, line); if (error == 0) map->timestamp++; @@ -537,7 +537,7 @@ _vm_map_trylock_read(vm_map_t map, const char *fil int error; error = map->system_map ? - !_mtx_trylock(&map->system_mtx, 0, file, line) : + !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : !_sx_try_slock(&map->lock, file, line); return (error == 0); } @@ -558,9 +558,7 @@ _vm_map_lock_upgrade(vm_map_t map, const char *fil unsigned int last_timestamp; if (map->system_map) { -#ifdef INVARIANTS - _mtx_assert(&map->system_mtx, MA_OWNED, file, line); -#endif + mtx_assert_(&map->system_mtx, MA_OWNED, file, line); } else { if (!_sx_try_upgrade(&map->lock, file, line)) { last_timestamp = map->timestamp; @@ -586,9 +584,7 @@ _vm_map_lock_downgrade(vm_map_t map, const char *f { if (map->system_map) { -#ifdef INVARIANTS - _mtx_assert(&map->system_mtx, MA_OWNED, file, line); -#endif + mtx_assert_(&map->system_mtx, MA_OWNED, file, line); } else _sx_downgrade(&map->lock, file, line); } @@ -609,13 +605,14 @@ vm_map_locked(vm_map_t map) return (sx_xlocked(&map->lock)); } +/* XXX: INVARIANTS here is still necessary because of sx support. */ #ifdef INVARIANTS static void _vm_map_assert_locked(vm_map_t map, const char *file, int line) { if (map->system_map) - _mtx_assert(&map->system_mtx, MA_OWNED, file, line); + mtx_assert_(&map->system_mtx, MA_OWNED, file, line); else _sx_assert(&map->lock, SA_XLOCKED, file, line); } @@ -626,7 +623,7 @@ _vm_map_assert_locked_read(vm_map_t map, const cha { if (map->system_map) - _mtx_assert(&map->system_mtx, MA_OWNED, file, line); + mtx_assert_(&map->system_mtx, MA_OWNED, file, line); else _sx_assert(&map->lock, SA_SLOCKED, file, line); } @@ -661,7 +658,7 @@ _vm_map_unlock_and_wait(vm_map_t map, int timo, co mtx_lock(&map_sleep_mtx); if (map->system_map) - _mtx_unlock_flags(&map->system_mtx, 0, file, line); + mtx_unlock_flags_(&map->system_mtx, 0, file, line); else _sx_xunlock(&map->lock, file, line); return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", Index: sys/sys/mutex.h =================================================================== --- sys/sys/mutex.h (revision 227657) +++ sys/sys/mutex.h (working copy) @@ -81,6 +81,10 @@ * of the kernel via macros, thus allowing us to use the cpp LOCK_FILE * and LOCK_LINE. These functions should not be called directly by any * code using the API. Their macros cover their functionality. + * Functions with a `_' suffix are the entrypoint for the common + * KPI covering both compat shims and fast path case. These can be + * used by consumers willing to pass options, file and line + * informations, in an option-independent way. * * [See below for descriptions] * @@ -88,6 +92,8 @@ void mtx_init(struct mtx *m, const char *name, const char *type, int opts); void mtx_destroy(struct mtx *m); void mtx_sysinit(void *arg); +int mtx_trylock_flags_(struct mtx *m, int opts, const char *file, + int line); void mutex_init(void); void _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, int line); @@ -97,7 +103,6 @@ void _mtx_lock_spin(struct mtx *m, uintptr_t tid, const char *file, int line); #endif void _mtx_unlock_spin(struct mtx *m, int opts, const char *file, int line); -int _mtx_trylock(struct mtx *m, int opts, const char *file, int line); void _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line); void _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line); void _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, @@ -107,12 +112,12 @@ void _mtx_unlock_spin_flags(struct mtx *m, int opt #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) void _mtx_assert(const struct mtx *m, int what, const char *file, int line); #endif -void _thread_lock_flags(struct thread *, int, const char *, int); +void thread_lock_flags_(struct thread *, int, const char *, int); #define thread_lock(tdp) \ - _thread_lock_flags((tdp), 0, __FILE__, __LINE__) + thread_lock_flags_((tdp), 0, __FILE__, __LINE__) #define thread_lock_flags(tdp, opt) \ - _thread_lock_flags((tdp), (opt), __FILE__, __LINE__) + thread_lock_flags_((tdp), (opt), __FILE__, __LINE__) #define thread_unlock(tdp) \ mtx_unlock_spin((tdp)->td_lock) @@ -290,27 +295,48 @@ extern struct mtx_pool *mtxpool_sleep; #error LOCK_DEBUG not defined, include before #endif #if LOCK_DEBUG > 0 || defined(MUTEX_NOINLINE) -#define mtx_lock_flags(m, opts) \ - _mtx_lock_flags((m), (opts), LOCK_FILE, LOCK_LINE) -#define mtx_unlock_flags(m, opts) \ - _mtx_unlock_flags((m), (opts), LOCK_FILE, LOCK_LINE) -#define mtx_lock_spin_flags(m, opts) \ - _mtx_lock_spin_flags((m), (opts), LOCK_FILE, LOCK_LINE) -#define mtx_unlock_spin_flags(m, opts) \ - _mtx_unlock_spin_flags((m), (opts), LOCK_FILE, LOCK_LINE) +#define mtx_lock_flags_(m, opts, file, line) \ + _mtx_lock_flags((m), (opts), (file), (line)) +#define mtx_unlock_flags_(m, opts, file, line) \ + _mtx_unlock_flags((m), (opts), (file), (line)) +#define mtx_lock_spin_flags_(m, opts, file, line) \ + _mtx_lock_spin_flags((m), (opts), (file), (line)) +#define mtx_unlock_spin_flags_(m, opts, file, line) \ + _mtx_unlock_spin_flags((m), (opts), (file), (line)) #else /* LOCK_DEBUG == 0 && !MUTEX_NOINLINE */ +#define mtx_lock_flags_(m, opts, file, line) \ + __mtx_lock((m), curthread, (opts), (file), (line)) +#define mtx_unlock_flags_(m, opts, file, line) \ + __mtx_unlock((m), curthread, (opts), (file), (line)) +#define mtx_lock_spin_flags_(m, opts, file, line) \ + __mtx_lock_spin((m), curthread, (opts), (file), (line)) +#define mtx_unlock_spin_flags_(m, opts, file, line) \ + __mtx_unlock_spin((m)) +#endif /* LOCK_DEBUG > 0 || MUTEX_NOINLINE */ + +#ifdef INVARIANTS +#define mtx_assert_(m, what, file, line) \ + _mtx_assert((m), (what), (file), (line)) + +#define GIANT_REQUIRED mtx_assert_(&Giant, MA_OWNED, __FILE__, __LINE__) + +#else /* INVARIANTS */ +#define mtx_assert_(m, what) (void)0 +#define GIANT_REQUIRED +#endif /* INVARIANTS */ + #define mtx_lock_flags(m, opts) \ - __mtx_lock((m), curthread, (opts), LOCK_FILE, LOCK_LINE) + mtx_lock_flags_((m), (opts), LOCK_FILE, LOCK_LINE) #define mtx_unlock_flags(m, opts) \ - __mtx_unlock((m), curthread, (opts), LOCK_FILE, LOCK_LINE) + mtx_unlock_flags_((m), (opts), LOCK_FILE, LOCK_LINE) #define mtx_lock_spin_flags(m, opts) \ - __mtx_lock_spin((m), curthread, (opts), LOCK_FILE, LOCK_LINE) + mtx_lock_spin_flags_((m), (opts), LOCK_FILE, LOCK_LINE) #define mtx_unlock_spin_flags(m, opts) \ - __mtx_unlock_spin((m)) -#endif /* LOCK_DEBUG > 0 || MUTEX_NOINLINE */ - + mtx_unlock_spin_flags_((m), (opts), LOCK_FILE, LOCK_LINE) #define mtx_trylock_flags(m, opts) \ - _mtx_trylock((m), (opts), LOCK_FILE, LOCK_LINE) + mtx_trylock_flags_((m), (opts), LOCK_FILE, LOCK_LINE) +#define mtx_assert(m, what) \ + mtx_assert_((m), (what), __FILE__, __LINE__) #define mtx_sleep(chan, mtx, pri, wmesg, timo) \ _sleep((chan), &(mtx)->lock_object, (pri), (wmesg), (timo)) @@ -398,17 +424,6 @@ struct mtx_args { #define MA_NOTRECURSED LA_NOTRECURSED #endif -#ifdef INVARIANTS -#define mtx_assert(m, what) \ - _mtx_assert((m), (what), __FILE__, __LINE__) - -#define GIANT_REQUIRED mtx_assert(&Giant, MA_OWNED) - -#else /* INVARIANTS */ -#define mtx_assert(m, what) (void)0 -#define GIANT_REQUIRED -#endif /* INVARIANTS */ - /* * Common lock type names. */