--- sys/kern/uipc_socket.c.orig +++ sys/kern/uipc_socket.c @@ -345,7 +345,13 @@ static void so_splice_timeout(void *arg, int pending); static void so_splice_xfer(struct so_splice *s); -static int so_unsplice(struct socket *so, bool timeout); +enum so_unsplice_reason { + SO_UNSPLICE_SETSOCKOPT, + SO_UNSPLICE_TIMEOUT, + SO_UNSPLICE_EXCEPTION, +}; +static int so_unsplice(struct socket *so, struct so_splice *sp, + enum so_unsplice_reason why); static void splice_work_thread(void *ctx) @@ -653,7 +659,7 @@ sp->state = SPLICE_EXCEPTION; soref(so_src); mtx_unlock(&sp->mtx); - (void)so_unsplice(so_src, false); + (void)so_unsplice(so_src, sp, SO_UNSPLICE_EXCEPTION); sorele(so_src); } else { /* @@ -1638,7 +1644,7 @@ struct so_splice *sp; sp = arg; - (void)so_unsplice(sp->src, true); + (void)so_unsplice(sp->src, sp, SO_UNSPLICE_TIMEOUT); } /* @@ -1707,14 +1713,22 @@ error = EBUSY; if (error != 0) { SOCK_UNLOCK(so2); - so_unsplice(so, false); + mtx_lock(&sp->mtx); + sp->dst = NULL; + sp->state = SPLICE_EXCEPTION; + mtx_unlock(&sp->mtx); + (void)so_unsplice(so, sp, SO_UNSPLICE_EXCEPTION); return (error); } SOCK_SENDBUF_LOCK(so2); if (so2->so_snd.sb_tls_info != NULL) { SOCK_SENDBUF_UNLOCK(so2); SOCK_UNLOCK(so2); - so_unsplice(so, false); + mtx_lock(&sp->mtx); + sp->dst = NULL; + sp->state = SPLICE_EXCEPTION; + mtx_unlock(&sp->mtx); + (void)so_unsplice(so, sp, SO_UNSPLICE_EXCEPTION); return (EINVAL); } so2->so_snd.sb_flags |= SB_SPLICED; @@ -1740,11 +1754,12 @@ } static int -so_unsplice(struct socket *so, bool timeout) +so_unsplice(struct socket *so, struct so_splice *sp, + enum so_unsplice_reason why) { struct socket *so2; - struct so_splice *sp; - bool drain, so2rele; + int error; + bool drain; /* * First unset SB_SPLICED and hide the splice structure so that @@ -1762,38 +1777,66 @@ SOCK_UNLOCK(so); return (ENOTCONN); } - sp = so->so_splice; + if (sp == NULL) { + sp = so->so_splice; + } else if (sp != so->so_splice) { + SOCK_RECVBUF_UNLOCK(so); + SOCK_UNLOCK(so); + return (EINVAL); + } + error = 0; mtx_lock(&sp->mtx); - if (sp->state == SPLICE_INIT) { + switch (sp->state) { + case SPLICE_INIT: /* * A splice is in the middle of being set up. */ - mtx_unlock(&sp->mtx); + error = ENOTCONN; + break; + case SPLICE_EXCEPTION: + /* + * Make sure that unsplicing is serialized. + */ + if (why != SO_UNSPLICE_EXCEPTION) { + error = ENOTCONN; + break; + } + /* FALLTHROUGH */ + case SPLICE_IDLE: + sp->state = SPLICE_CLOSED; + break; + case SPLICE_QUEUED: + case SPLICE_RUNNING: + sp->state = SPLICE_CLOSING; + break; + default: + __assert_unreachable(); + } + mtx_unlock(&sp->mtx); + if (error != 0) { SOCK_RECVBUF_UNLOCK(so); SOCK_UNLOCK(so); - return (ENOTCONN); + return (error); } - mtx_unlock(&sp->mtx); so->so_rcv.sb_flags &= ~SB_SPLICED; so->so_splice = NULL; SOCK_RECVBUF_UNLOCK(so); SOCK_UNLOCK(so); so2 = sp->dst; - SOCK_LOCK(so2); - KASSERT(!SOLISTENING(so2), ("%s: so2 is listening", __func__)); - SOCK_SENDBUF_LOCK(so2); - KASSERT(sp->state == SPLICE_INIT || - (so2->so_snd.sb_flags & SB_SPLICED) != 0, - ("%s: so2 is not spliced", __func__)); - KASSERT(sp->state == SPLICE_INIT || - so2->so_splice_back == sp, - ("%s: so_splice_back != sp", __func__)); - so2->so_snd.sb_flags &= ~SB_SPLICED; - so2rele = so2->so_splice_back != NULL; - so2->so_splice_back = NULL; - SOCK_SENDBUF_UNLOCK(so2); - SOCK_UNLOCK(so2); + if (so2 != NULL) { + SOCK_LOCK(so2); + KASSERT(!SOLISTENING(so2), ("%s: so2 is listening", __func__)); + SOCK_SENDBUF_LOCK(so2); + KASSERT((so2->so_snd.sb_flags & SB_SPLICED) != 0, + ("%s: so2 is not spliced", __func__)); + KASSERT(so2->so_splice_back == sp, + ("%s: so_splice_back != sp", __func__)); + so2->so_snd.sb_flags &= ~SB_SPLICED; + so2->so_splice_back = NULL; + SOCK_SENDBUF_UNLOCK(so2); + SOCK_UNLOCK(so2); + } /* * No new work is being enqueued. The worker thread might be @@ -1801,22 +1844,11 @@ * finish before proceeding. */ mtx_lock(&sp->mtx); - switch (sp->state) { - case SPLICE_QUEUED: - case SPLICE_RUNNING: - sp->state = SPLICE_CLOSING; - while (sp->state == SPLICE_CLOSING) - msleep(sp, &sp->mtx, PSOCK, "unsplice", 0); - break; - case SPLICE_INIT: - case SPLICE_IDLE: - case SPLICE_EXCEPTION: - sp->state = SPLICE_CLOSED; - break; - default: - __assert_unreachable(); - } - if (!timeout) { + while (sp->state == SPLICE_CLOSING) + msleep(sp, &sp->mtx, PSOCK, "unsplice", 0); + KASSERT(sp->state == SPLICE_CLOSED, + ("%s: splice %p state %d", __func__, sp, sp->state)); + if (why != SO_UNSPLICE_TIMEOUT) { drain = taskqueue_cancel_timeout(taskqueue_thread, &sp->timeout, NULL) != 0; } else { @@ -1833,9 +1865,10 @@ sorwakeup(so); CURVNET_SET(so->so_vnet); sorele(so); - sowwakeup(so2); - if (so2rele) + if (so2 != NULL) { + sowwakeup(so2); sorele(so2); + } CURVNET_RESTORE(); so_splice_free(sp); return (0); @@ -4018,7 +4051,8 @@ error = so_splice(so, so2, &splice); fdrop(fp, sopt->sopt_td); } else { - error = so_unsplice(so, false); + error = so_unsplice(so, NULL, + SO_UNSPLICE_SETSOCKOPT); } break; }