--- sys/kern/kern_umtx.c.orig +++ sys/kern/kern_umtx.c @@ -4293,8 +4293,7 @@ #define USHM_OBJ_UMTX(o) \ ((struct umtx_shm_obj_list *)(&(o)->umtx_data)) -#define USHMF_REG_LINKED 0x0001 -#define USHMF_OBJ_LINKED 0x0002 +#define USHMF_LINKED 0x0001 struct umtx_shm_reg { TAILQ_ENTRY(umtx_shm_reg) ushm_reg_link; LIST_ENTRY(umtx_shm_reg) ushm_obj_link; @@ -4335,8 +4334,17 @@ static struct task umtx_shm_reg_delfree_task = TASK_INITIALIZER(0, umtx_shm_reg_delfree_tq, NULL); -static struct umtx_shm_reg * -umtx_shm_find_reg_locked(const struct umtx_key *key) +/* + * Returns 0 if a SHM with the passed key is found in the registry, in which + * case it is returned through 'oreg'. Otherwise, returns an error among ESRCH + * (no corresponding SHM; ESRCH was chosen for compatibility, ENOENT would have + * been preferable) or EOVERFLOW (there is a corresponding SHM, but reference + * count would overflow, so can't return it), in which case '*oreg' is left + * unchanged. + */ +static int +umtx_shm_find_reg_locked(const struct umtx_key *key, + struct umtx_shm_reg **const oreg) { struct umtx_shm_reg *reg; struct umtx_shm_reg_head *reg_head; @@ -4352,26 +4360,38 @@ reg->ushm_key.info.shared.offset == key->info.shared.offset) { KASSERT(reg->ushm_key.type == TYPE_SHM, ("TYPE_USHM")); - KASSERT(reg->ushm_refcnt > 0, + KASSERT(reg->ushm_refcnt != 0, ("reg %p refcnt 0 onlist", reg)); - KASSERT((reg->ushm_flags & USHMF_REG_LINKED) != 0, + KASSERT((reg->ushm_flags & USHMF_LINKED) != 0, ("reg %p not linked", reg)); + /* + * Don't let overflow happen, just deny a new reference + * (this is additional protection against some reference + * count leak, which is known not to be the case at the + * time of this writing). + */ + if (__predict_false(reg->ushm_refcnt == UINT_MAX)) + return (EOVERFLOW); reg->ushm_refcnt++; - return (reg); + *oreg = reg; + return (0); } } - return (NULL); + return (ESRCH); } -static struct umtx_shm_reg * -umtx_shm_find_reg(const struct umtx_key *key) +/* + * Calls umtx_shm_find_reg_unlocked() under the 'umtx_shm_lock'. + */ +static int +umtx_shm_find_reg(const struct umtx_key *key, struct umtx_shm_reg **const oreg) { - struct umtx_shm_reg *reg; + int error; mtx_lock(&umtx_shm_lock); - reg = umtx_shm_find_reg_locked(key); + error = umtx_shm_find_reg_locked(key, oreg); mtx_unlock(&umtx_shm_lock); - return (reg); + return (error); } static void @@ -4385,42 +4405,49 @@ } static bool -umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool force) +umtx_shm_unref_reg_locked(struct umtx_shm_reg *reg, bool linked_ref) { - bool res; - mtx_assert(&umtx_shm_lock, MA_OWNED); - KASSERT(reg->ushm_refcnt > 0, ("ushm_reg %p refcnt 0", reg)); - reg->ushm_refcnt--; - res = reg->ushm_refcnt == 0; - if (res || force) { - if ((reg->ushm_flags & USHMF_REG_LINKED) != 0) { - TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], - reg, ushm_reg_link); - reg->ushm_flags &= ~USHMF_REG_LINKED; - } - if ((reg->ushm_flags & USHMF_OBJ_LINKED) != 0) { - LIST_REMOVE(reg, ushm_obj_link); - reg->ushm_flags &= ~USHMF_OBJ_LINKED; - } + KASSERT(reg->ushm_refcnt != 0, ("ushm_reg %p refcnt 0", reg)); + + if (linked_ref) { + if ((reg->ushm_flags & USHMF_LINKED) == 0) + /* + * The reference tied to USHMF_LINKED has already been + * released concurrently. + */ + return (false); + + TAILQ_REMOVE(&umtx_shm_registry[reg->ushm_key.hash], reg, + ushm_reg_link); + LIST_REMOVE(reg, ushm_obj_link); + reg->ushm_flags &= ~USHMF_LINKED; } - return (res); + + reg->ushm_refcnt--; + return (reg->ushm_refcnt == 0); } static void -umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool force) +umtx_shm_unref_reg(struct umtx_shm_reg *reg, bool linked_ref) { vm_object_t object; bool dofree; - if (force) { + if (linked_ref) { + /* + * Note: This may be executed multiple times on the same + * shared-memory VM object in presence of concurrent callers + * because 'umtx_shm_lock' is not held all along in umtx_shm() + * and here. + */ object = reg->ushm_obj->shm_object; VM_OBJECT_WLOCK(object); vm_object_set_flag(object, OBJ_UMTXDEAD); VM_OBJECT_WUNLOCK(object); } mtx_lock(&umtx_shm_lock); - dofree = umtx_shm_unref_reg_locked(reg, force); + dofree = umtx_shm_unref_reg_locked(reg, linked_ref); mtx_unlock(&umtx_shm_lock); if (dofree) umtx_shm_free_reg(reg); @@ -4464,16 +4491,22 @@ struct ucred *cred; int error; - reg = umtx_shm_find_reg(key); - if (reg != NULL) { - *res = reg; - return (0); + error = umtx_shm_find_reg(key, res); + if (error != ESRCH) { + /* + * Either no error occured, and '*res' was filled, or EOVERFLOW + * was returned, indicating a reference count limit, and we + * won't create a duplicate registration. In both cases, we are + * done. + */ + return (error); } + /* No entry, we will create one. */ + cred = td->td_ucred; if (!chgumtxcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_UMTXP))) return (ENOMEM); reg = uma_zalloc(umtx_shm_reg_zone, M_WAITOK | M_ZERO); - reg->ushm_refcnt = 1; bcopy(key, ®->ushm_key, sizeof(*key)); reg->ushm_obj = shm_alloc(td->td_ucred, O_RDWR, false); reg->ushm_cred = crhold(cred); @@ -4483,18 +4516,32 @@ return (error); } mtx_lock(&umtx_shm_lock); - reg1 = umtx_shm_find_reg_locked(key); - if (reg1 != NULL) { + /* Re-lookup as 'umtx_shm_lock' has been temporarily released. */ + error = umtx_shm_find_reg_locked(key, ®1); + switch (error) { + case 0: mtx_unlock(&umtx_shm_lock); umtx_shm_free_reg(reg); *res = reg1; return (0); + case ESRCH: + break; + default: + mtx_unlock(&umtx_shm_lock); + umtx_shm_free_reg(reg); + return (error); } - reg->ushm_refcnt++; TAILQ_INSERT_TAIL(&umtx_shm_registry[key->hash], reg, ushm_reg_link); LIST_INSERT_HEAD(USHM_OBJ_UMTX(key->info.shared.object), reg, ushm_obj_link); - reg->ushm_flags = USHMF_REG_LINKED | USHMF_OBJ_LINKED; + reg->ushm_flags = USHMF_LINKED; + /* + * This is one reference for the registry and the list of shared + * mutexes referenced by the VM object containing the lock pointer, and + * another for the caller, which it will free after use. So, one of + * these is tied to the presence of USHMF_LINKED. + */ + reg->ushm_refcnt = 2; mtx_unlock(&umtx_shm_lock); *res = reg; return (0); @@ -4553,13 +4600,9 @@ if (error != 0) return (error); KASSERT(key.shared == 1, ("non-shared key")); - if ((flags & UMTX_SHM_CREAT) != 0) { - error = umtx_shm_create_reg(td, &key, ®); - } else { - reg = umtx_shm_find_reg(&key); - if (reg == NULL) - error = ESRCH; - } + error = (flags & UMTX_SHM_CREAT) != 0 ? + umtx_shm_create_reg(td, &key, ®) : + umtx_shm_find_reg(&key, ®); umtx_key_release(&key); if (error != 0) return (error);