Index: alpha/alpha/in_cksum.c =================================================================== RCS file: /ncvs/src/sys/alpha/alpha/in_cksum.c,v retrieving revision 1.2 diff -u -r1.2 in_cksum.c --- alpha/alpha/in_cksum.c 1999/08/28 00:38:27 1.2 +++ alpha/alpha/in_cksum.c 2000/03/26 18:28:24 @@ -69,6 +69,7 @@ sum = l_util.s[0] + l_util.s[1]; \ ADDCARRY(sum); \ } +#define INVERT sum == 0xffff ? sum : ~sum & 0xffff static const u_int32_t in_masks[] = { /*0 bytes*/ /*1 byte*/ /*2 bytes*/ /*3 bytes*/ @@ -173,6 +174,27 @@ return sum; } +u_short +in_addword(u_short a, u_short b) +{ + u_int64_t sum = a + b; + + ADDCARRY(sum); + return (sum); +} + +u_short +in_pseudo(u_int32_t a, u_int32_t b, u_int32_t c) +{ + u_int64_t sum; + union q_util q_util; + union l_util l_util; + + sum = (u_int64_t) a + b + c; + REDUCE16; + return (sum); +} + int in_cksum(m, len) register struct mbuf *m; @@ -202,6 +224,50 @@ } REDUCE16; return (~sum & 0xffff); +} + +u_short +in_cksum_skip(m, len, skip) + struct mbuf *m; + int len; + int skip; +{ + u_int64_t sum = 0; + int mlen = 0; + int clen = 0; + caddr_t addr; + union q_util q_util; + union l_util l_util; + + len -= skip; + for (; skip && m; m = m->m_next) { + if (m->m_len > skip) { + mlen = m->m_len - skip; + addr = mtod(m, caddr_t) + skip; + goto skip_start; + } else { + skip -= m->m_len; + } + } + + for (; m && len; m = m->m_next) { + if (m->m_len == 0) + continue; + mlen = m->m_len; + addr = mtod(m, caddr_t); +skip_start: + if (len < mlen) + mlen = len; + if ((clen ^ (long) addr) & 1) + sum += in_cksumdata(addr, mlen) << 8; + else + sum += in_cksumdata(addr, mlen); + + clen += mlen; + len -= mlen; + } + REDUCE16; + return (INVERT); } u_int in_cksum_hdr(ip) Index: alpha/include/in_cksum.h =================================================================== RCS file: /ncvs/src/sys/alpha/include/in_cksum.h,v retrieving revision 1.3 diff -u -r1.3 in_cksum.h --- alpha/include/in_cksum.h 1999/12/29 04:27:58 1.3 +++ alpha/include/in_cksum.h 2000/03/22 15:46:25 @@ -73,6 +73,9 @@ u_int in_cksum_hdr(const struct ip *ip); in_psum_t in_cksum_partial(in_psum_t psum, const u_short *w, int len); int in_cksum_finalize(in_psum_t psum); +u_short in_addword(u_short sum, u_short b); +u_short in_pseudo(u_int sum, u_int b, u_int c); +u_short in_cksum_skip(struct mbuf *m, int len, int skip); #endif #endif /* _MACHINE_IN_CKSUM_H_ */ Index: i386/i386/in_cksum.c =================================================================== RCS file: /ncvs/src/sys/i386/i386/in_cksum.c,v retrieving revision 1.17 diff -u -r1.17 in_cksum.c --- i386/i386/in_cksum.c 1999/12/20 12:11:34 1.17 +++ i386/i386/in_cksum.c 2000/03/22 15:44:26 @@ -57,6 +57,7 @@ #undef ADDCARRY #define ADDCARRY(x) if ((x) > 0xffff) (x) -= 0xffff #define REDUCE {sum = (sum & 0xffff) + (sum >> 16); ADDCARRY(sum);} +#define INVERT sum == 0xffff ? sum : ~sum & 0xffff /* * Thanks to gcc we don't have to guess @@ -244,6 +245,194 @@ } REDUCE; return (~sum & 0xffff); +} + +u_short +in_cksum_skip(m, len, skip) + struct mbuf *m; + int len; + int skip; +{ + register u_short *w; + register unsigned sum = 0; + register int mlen = 0; + int byte_swapped = 0; + union { char c[2]; u_short s; } su; + + len -= skip; + for (; skip && m; m = m->m_next) { + if (m->m_len > skip) { + mlen = m->m_len - skip; + w = (u_short *)(mtod(m, u_char *) + skip); + goto skip_start; + } else { + skip -= m->m_len; + } + } + + for (;m && len; m = m->m_next) { + if (m->m_len == 0) + continue; + w = mtod(m, u_short *); + if (mlen == -1) { + /* + * The first byte of this mbuf is the continuation + * of a word spanning between this mbuf and the + * last mbuf. + */ + + /* su.c[0] is already saved when scanning previous + * mbuf. sum was REDUCEd when we found mlen == -1 + */ + su.c[1] = *(u_char *)w; + sum += su.s; + w = (u_short *)((char *)w + 1); + mlen = m->m_len - 1; + len--; + } else + mlen = m->m_len; +skip_start: + if (len < mlen) + mlen = len; + len -= mlen; + /* + * Force to long boundary so we do longword aligned + * memory operations + */ + if (3 & (int) w) { + REDUCE; + if ((1 & (int) w) && (mlen > 0)) { + sum <<= 8; + su.c[0] = *(char *)w; + w = (u_short *)((char *)w + 1); + mlen--; + byte_swapped = 1; + } + if ((2 & (int) w) && (mlen >= 2)) { + sum += *w++; + mlen -= 2; + } + } + /* + * Advance to a 486 cache line boundary. + */ + if (4 & (int) w && mlen >= 4) { + ADD(0); + MOP; + w += 2; + mlen -= 4; + } + if (8 & (int) w && mlen >= 8) { + ADD(0); + ADDC(4); + MOP; + w += 4; + mlen -= 8; + } + /* + * Do as much of the checksum as possible 32 bits at at time. + * In fact, this loop is unrolled to make overhead from + * branches &c small. + */ + mlen -= 1; + while ((mlen -= 32) >= 0) { + u_char junk; + /* + * Add with carry 16 words and fold in the last + * carry by adding a 0 with carry. + * + * The early ADD(16) and the LOAD(32) are to load + * the next 2 cache lines in advance on 486's. The + * 486 has a penalty of 2 clock cycles for loading + * a cache line, plus whatever time the external + * memory takes to load the first word(s) addressed. + * These penalties are unavoidable. Subsequent + * accesses to a cache line being loaded (and to + * other external memory?) are delayed until the + * whole load finishes. These penalties are mostly + * avoided by not accessing external memory for + * 8 cycles after the ADD(16) and 12 cycles after + * the LOAD(32). The loop terminates when mlen + * is initially 33 (not 32) to guaranteed that + * the LOAD(32) is within bounds. + */ + ADD(16); + ADDC(0); + ADDC(4); + ADDC(8); + ADDC(12); + LOAD(32); + ADDC(20); + ADDC(24); + ADDC(28); + MOP; + w += 16; + } + mlen += 32 + 1; + if (mlen >= 32) { + ADD(16); + ADDC(0); + ADDC(4); + ADDC(8); + ADDC(12); + ADDC(20); + ADDC(24); + ADDC(28); + MOP; + w += 16; + mlen -= 32; + } + if (mlen >= 16) { + ADD(0); + ADDC(4); + ADDC(8); + ADDC(12); + MOP; + w += 8; + mlen -= 16; + } + if (mlen >= 8) { + ADD(0); + ADDC(4); + MOP; + w += 4; + mlen -= 8; + } + if (mlen == 0 && byte_swapped == 0) + continue; /* worth 1% maybe ?? */ + REDUCE; + while ((mlen -= 2) >= 0) { + sum += *w++; + } + if (byte_swapped) { + sum <<= 8; + byte_swapped = 0; + if (mlen == -1) { + su.c[1] = *(char *)w; + sum += su.s; + mlen = 0; + } else + mlen = -1; + } else if (mlen == -1) + /* + * This mbuf has odd number of bytes. + * There could be a word split betwen + * this mbuf and the next mbuf. + * Save the last byte (to prepend to next mbuf). + */ + su.c[0] = *(char *)w; + } + + if (len) + printf("cksum: out of data\n"); + if (mlen == -1) { + /* The last mbuf has odd # of bytes. Follow the + standard (the odd byte is shifted left by 8 bits) */ + su.c[1] = 0; + sum += su.s; + } + REDUCE; + return (INVERT); } /* Index: i386/include/in_cksum.h =================================================================== RCS file: /ncvs/src/sys/i386/include/in_cksum.h,v retrieving revision 1.7 diff -u -r1.7 in_cksum.h --- i386/include/in_cksum.h 1999/12/29 04:33:02 1.7 +++ i386/include/in_cksum.h 2000/03/22 15:45:12 @@ -81,6 +81,30 @@ ip->ip_sum = htons(__tmpsum + (__tmpsum >> 16)); } +static __inline u_short +in_addword(u_short sum, u_short b) +{ + + __asm("addw %2, %0" : "=r" (sum) : "0" (sum), "r" (b)); + __asm("adcw $0, %0" : "=r" (sum) : "0" (sum)); + + return (sum); +} + +static __inline u_short +in_pseudo(u_int sum, u_int b, u_int c) +{ + + __asm("addl %2, %0" : "=r" (sum) : "0" (sum), "r" (b)); + __asm("adcl %2, %0" : "=r" (sum) : "0" (sum), "r" (c)); + __asm("adcl $0, %0" : "=r" (sum) : "0" (sum)); + + sum = (sum & 0xffff) + (sum >> 16); + if (sum > 0xffff) + sum -= 0xffff; + return (sum); +} + #else u_int in_cksum_hdr __P((const struct ip *)); #define in_cksum_update(ip) \ @@ -94,6 +118,7 @@ typedef unsigned in_psum_t; #ifdef _KERNEL +u_short in_cksum_skip(struct mbuf *m, int len, int skip); in_psum_t in_cksum_partial(in_psum_t psum, const u_short *w, int len); int in_cksum_finalize(in_psum_t psum); #endif /* _KERNEL */ Index: net/if.h =================================================================== RCS file: /ncvs/src/sys/net/if.h,v retrieving revision 1.58 diff -u -r1.58 if.h --- net/if.h 1999/12/29 04:38:34 1.58 +++ net/if.h 2000/03/26 17:57:29 @@ -72,8 +72,8 @@ u_long ifi_omcasts; /* packets sent via multicast */ u_long ifi_iqdrops; /* dropped on input, this interface */ u_long ifi_noproto; /* destined for unsupported protocol */ - u_long ifi_recvtiming; /* usec spent receiving when timing */ - u_long ifi_xmittiming; /* usec spent xmitting when timing */ + u_long ifi_hwassist; /* HW offload capabilities */ + u_long ifi_unused; /* XXX was ifi_xmittiming */ struct timeval ifi_lastchange; /* time of last administrative change */ }; Index: net/if_var.h =================================================================== RCS file: /ncvs/src/sys/net/if_var.h,v retrieving revision 1.18 diff -u -r1.18 if_var.h --- net/if_var.h 1999/12/29 04:38:36 1.18 +++ net/if_var.h 2000/03/26 17:58:41 @@ -150,6 +150,7 @@ #define if_hdrlen if_data.ifi_hdrlen #define if_metric if_data.ifi_metric #define if_baudrate if_data.ifi_baudrate +#define if_hwassist if_data.ifi_hwassist #define if_ipackets if_data.ifi_ipackets #define if_ierrors if_data.ifi_ierrors #define if_opackets if_data.ifi_opackets Index: netinet/ip_input.c =================================================================== RCS file: /ncvs/src/sys/netinet/ip_input.c,v retrieving revision 1.130 diff -u -r1.130 ip_input.c --- netinet/ip_input.c 2000/02/23 20:11:57 1.130 +++ netinet/ip_input.c 2000/03/26 15:18:03 @@ -324,10 +324,14 @@ } ip = mtod(m, struct ip *); } - if (hlen == sizeof(struct ip)) { - sum = in_cksum_hdr(ip); + if (m->m_pkthdr.csum_flags & CSUM_IP_CHECKED) { + sum = !(m->m_pkthdr.csum_flags & CSUM_IP_VALID); } else { - sum = in_cksum(m, hlen); + if (hlen == sizeof(struct ip)) { + sum = in_cksum_hdr(ip); + } else { + sum = in_cksum(m, hlen); + } } if (sum) { ipstat.ips_badsum++; @@ -841,6 +845,9 @@ * our data already. If so, drop the data from the incoming * segment. If it provides all of our data, drop us, otherwise * stick new segment in the proper place. + * + * If some of the data is dropped from the the preceding + * segment, then it's checksum is invalidated. */ if (p) { i = GETIP(p)->ip_off + GETIP(p)->ip_len - ip->ip_off; @@ -848,6 +855,7 @@ if (i >= ip->ip_len) goto dropfrag; m_adj(m, i); + m->m_pkthdr.csum_flags = 0; ip->ip_off += i; ip->ip_len -= i; } @@ -870,6 +878,7 @@ GETIP(q)->ip_len -= i; GETIP(q)->ip_off += i; m_adj(q, i); + q->m_pkthdr.csum_flags = 0; break; } nq = q->m_nextpkt; @@ -927,6 +936,8 @@ nq = q->m_nextpkt; q->m_nextpkt = NULL; m_cat(m, q); + m->m_pkthdr.csum_flags &= q->m_pkthdr.csum_flags; + m->m_pkthdr.csum_data += q->m_pkthdr.csum_data; } #ifdef IPDIVERT Index: netinet/ip_output.c =================================================================== RCS file: /ncvs/src/sys/netinet/ip_output.c,v retrieving revision 1.99 diff -u -r1.99 ip_output.c --- netinet/ip_output.c 2000/03/09 14:57:15 1.99 +++ netinet/ip_output.c 2000/03/26 23:12:20 @@ -96,6 +96,7 @@ u_short ip_id; +static void in_delayed_cksum(struct mbuf *m); static struct mbuf *ip_insertoptions __P((struct mbuf *, struct mbuf *, int *)); static void ip_mloopback __P((struct ifnet *, struct mbuf *, struct sockaddr_in *, int)); @@ -132,7 +133,7 @@ int len, off, error = 0; struct sockaddr_in *dst; struct in_ifaddr *ia; - int isbroadcast; + int isbroadcast, sw_csum; #ifdef IPSEC struct route iproute; struct socket *so = NULL; @@ -692,6 +693,15 @@ state.ro = ro; state.dst = (struct sockaddr *)dst; + /* + * XXX + * delayed checksums are not currently compatible with IPsec + */ + if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA) { + in_delayed_cksum(m); + m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA; + } + error = ipsec4_output(&state, sp, flags); m = state.m; @@ -754,17 +764,29 @@ skip_ipsec: #endif /*IPSEC*/ + sw_csum = m->m_pkthdr.csum_flags | CSUM_IP; + m->m_pkthdr.csum_flags = sw_csum & ifp->if_hwassist; + sw_csum &= ~ifp->if_hwassist; + if (sw_csum & CSUM_DELAY_DATA) { + in_delayed_cksum(m); + sw_csum &= ~CSUM_DELAY_DATA; + } + /* - * If small enough for interface, can just send directly. + * If small enough for interface, or the interface will take + * care of the fragmentation for us, can just send directly. */ - if ((u_short)ip->ip_len <= ifp->if_mtu) { + if ((u_short)ip->ip_len <= ifp->if_mtu || + ifp->if_hwassist & CSUM_FRAGMENT) { ip->ip_len = htons((u_short)ip->ip_len); ip->ip_off = htons((u_short)ip->ip_off); ip->ip_sum = 0; - if (ip->ip_vhl == IP_VHL_BORING) { - ip->ip_sum = in_cksum_hdr(ip); - } else { - ip->ip_sum = in_cksum(m, hlen); + if (sw_csum & CSUM_DELAY_IP) { + if (ip->ip_vhl == IP_VHL_BORING) { + ip->ip_sum = in_cksum_hdr(ip); + } else { + ip->ip_sum = in_cksum(m, hlen); + } } error = (*ifp->if_output)(ifp, m, (struct sockaddr *)dst, ro->ro_rt); @@ -797,9 +819,20 @@ goto bad; } + /* + * if the interface will not calculate checksums on + * fragmented packets, then do it here. + */ + if (m->m_pkthdr.csum_flags & CSUM_DELAY_DATA && + (ifp->if_hwassist & CSUM_IP_FRAGS) == 0) { + in_delayed_cksum(m); + m->m_pkthdr.csum_flags &= ~CSUM_DELAY_DATA; + } + { int mhlen, firstlen = len; struct mbuf **mnext = &m->m_nextpkt; + int nfrags = 1; /* * Loop through length of segment after first fragment, @@ -814,7 +847,7 @@ ipstat.ips_odropped++; goto sendorfree; } - m->m_flags |= (m0->m_flags & M_MCAST); + m->m_flags |= (m0->m_flags & M_MCAST) | M_FRAG; m->m_data += max_linkhdr; mhip = mtod(m, struct ip *); *mhip = *ip; @@ -840,17 +873,27 @@ } m->m_pkthdr.len = mhlen + len; m->m_pkthdr.rcvif = (struct ifnet *)0; + m->m_pkthdr.csum_flags = m0->m_pkthdr.csum_flags; mhip->ip_off = htons((u_short)mhip->ip_off); mhip->ip_sum = 0; - if (mhip->ip_vhl == IP_VHL_BORING) { - mhip->ip_sum = in_cksum_hdr(mhip); - } else { - mhip->ip_sum = in_cksum(m, mhlen); + if (sw_csum & CSUM_DELAY_IP) { + if (mhip->ip_vhl == IP_VHL_BORING) { + mhip->ip_sum = in_cksum_hdr(mhip); + } else { + mhip->ip_sum = in_cksum(m, mhlen); + } } *mnext = m; mnext = &m->m_nextpkt; - ipstat.ips_ofragments++; + nfrags++; } + ipstat.ips_ofragments += nfrags; + + /* set first/last markers for fragment chain */ + m->m_flags |= M_LASTFRAG; + m0->m_flags |= M_FIRSTFRAG | M_FRAG; + m0->m_pkthdr.csum_data = nfrags; + /* * Update first fragment by trimming what's been copied out * and updating header, then send each fragment (in order). @@ -861,10 +904,12 @@ ip->ip_len = htons((u_short)m->m_pkthdr.len); ip->ip_off = htons((u_short)(ip->ip_off | IP_MF)); ip->ip_sum = 0; - if (ip->ip_vhl == IP_VHL_BORING) { - ip->ip_sum = in_cksum_hdr(ip); - } else { - ip->ip_sum = in_cksum(m, hlen); + if (sw_csum & CSUM_DELAY_IP) { + if (ip->ip_vhl == IP_VHL_BORING) { + ip->ip_sum = in_cksum_hdr(ip); + } else { + ip->ip_sum = in_cksum(m, hlen); + } } sendorfree: for (m = m0; m; m = m0) { @@ -896,6 +941,31 @@ bad: m_freem(m0); goto done; +} + +static void +in_delayed_cksum(struct mbuf *m) +{ + struct ip *ip; + u_short csum, offset; + + ip = mtod(m, struct ip *); + offset = IP_VHL_HL(ip->ip_vhl) << 2 ; + csum = in_cksum_skip(m, ip->ip_len, offset); + offset += m->m_pkthdr.csum_data; /* checksum offset */ + + if (offset + sizeof(u_short) > m->m_len) { + printf("delayed m_pullup, m->len: %d off: %d p: %d\n", + m->m_len, offset, ip->ip_p); + /* + * XXX + * this shouldn't happen, but if it does, the + * correct behavior may be to insert the checksum + * in the existing chain instead of rearranging it. + */ + m = m_pullup(m, offset + sizeof(u_short)); + } + *(u_short *)(m->m_data + offset) = csum; } /* Index: netinet/tcp_input.c =================================================================== RCS file: /ncvs/src/sys/netinet/tcp_input.c,v retrieving revision 1.107 diff -u -r1.107 tcp_input.c --- netinet/tcp_input.c 2000/03/11 11:20:52 1.107 +++ netinet/tcp_input.c 2000/03/22 18:40:02 @@ -100,6 +100,8 @@ #include #endif /*IPSEC*/ +#include + MALLOC_DEFINE(M_TSEGQ, "tseg_qent", "TCP segment queue entry"); static int tcprexmtthresh = 3; @@ -425,17 +427,27 @@ } ip = mtod(m, struct ip *); ipov = (struct ipovly *)ip; - - /* - * Checksum extended TCP header and data. - */ - tlen = ip->ip_len; - len = sizeof (struct ip) + tlen; - bzero(ipov->ih_x1, sizeof(ipov->ih_x1)); - ipov->ih_len = (u_short)tlen; - HTONS(ipov->ih_len); th = (struct tcphdr *)((caddr_t)ip + off0); - th->th_sum = in_cksum(m, len); + tlen = ip->ip_len; + + if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) { + if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) + th->th_sum = m->m_pkthdr.csum_data; + else + th->th_sum = in_pseudo(ip->ip_src.s_addr, + ip->ip_dst.s_addr, htonl(m->m_pkthdr.csum_data + + ip->ip_len + IPPROTO_TCP)); + th->th_sum ^= 0xffff; + } else { + /* + * Checksum extended TCP header and data. + */ + len = sizeof (struct ip) + tlen; + bzero(ipov->ih_x1, sizeof(ipov->ih_x1)); + ipov->ih_len = (u_short)tlen; + HTONS(ipov->ih_len); + th->th_sum = in_cksum(m, len); + } if (th->th_sum) { tcpstat.tcps_rcvbadsum++; goto drop; Index: netinet/tcp_output.c =================================================================== RCS file: /ncvs/src/sys/netinet/tcp_output.c,v retrieving revision 1.39 diff -u -r1.39 tcp_output.c --- netinet/tcp_output.c 2000/02/09 00:34:40 1.39 +++ netinet/tcp_output.c 2000/03/26 18:22:56 @@ -80,6 +80,8 @@ #include #endif /*IPSEC*/ +#include + #ifdef notyet extern struct mbuf *m_copypack(); #endif @@ -645,6 +647,7 @@ ip = mtod(m, struct ip *); ipov = (struct ipovly *)ip; th = (struct tcphdr *)(ip + 1); + /* this picks up the pseudo header (w/o the length) */ bcopy((caddr_t)tp->t_template->tt_ipgen, (caddr_t)ip, sizeof(struct ip)); bcopy((caddr_t)&tp->t_template->tt_t, (caddr_t)th, @@ -722,15 +725,15 @@ else #endif /* INET6 */ { + m->m_pkthdr.csum_flags = CSUM_TCP; + m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); if (len + optlen) - ipov->ih_len = htons((u_short)(sizeof (struct tcphdr) + - optlen + len)); - th->th_sum = in_cksum(m, (int)(hdrlen + len)); -#ifdef INET6 - /* Re-initialization for later version check */ - ip->ip_v = IPVERSION; - -#endif /* INET6 */ + th->th_sum = in_addword(th->th_sum, + htons((u_short)(optlen + len))); + + /* IP version must be set here for ipv4/ipv6 checking later */ + KASSERT(ip->ip_v == IPVERSION, + ("%s: IP version incorrect: %d", __FUNCTION__, ip->ip_v)); } /* Index: netinet/tcp_subr.c =================================================================== RCS file: /ncvs/src/sys/netinet/tcp_subr.c,v retrieving revision 1.73 diff -u -r1.73 tcp_subr.c --- netinet/tcp_subr.c 2000/02/28 21:18:21 1.73 +++ netinet/tcp_subr.c 2000/03/26 15:45:10 @@ -39,6 +39,7 @@ #include "opt_ipsec.h" #include "opt_tcpdebug.h" +#include #include #include #include @@ -93,6 +94,8 @@ #include #endif /*IPSEC*/ +#include + int tcp_mssdflt = TCP_MSS; SYSCTL_INT(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt, CTLFLAG_RW, &tcp_mssdflt , 0, "Default TCP Maximum Segment Size"); @@ -242,17 +245,19 @@ ip6->ip6_plen = sizeof(struct tcphdr); ip6->ip6_src = inp->in6p_laddr; ip6->ip6_dst = inp->in6p_faddr; + n->tt_t.th_sum = 0; } else #endif { - register struct ipovly *ipov; + struct ip *ip = (struct ip *)n->tt_ipgen; - ipov = (struct ipovly *)n->tt_ipgen; - bzero(ipov->ih_x1, sizeof(ipov->ih_x1)); - ipov->ih_pr = IPPROTO_TCP; - ipov->ih_len = htons(sizeof (struct tcpiphdr) - sizeof (struct ip)); - ipov->ih_src = inp->inp_laddr; - ipov->ih_dst = inp->inp_faddr; + bzero(ip, sizeof(struct ip)); /* XXX overkill? */ + ip->ip_vhl = IP_VHL_BORING; + ip->ip_p = IPPROTO_TCP; + ip->ip_src = inp->inp_laddr; + ip->ip_dst = inp->inp_faddr; + n->tt_t.th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, + htons(sizeof(struct tcphdr) + IPPROTO_TCP)); } n->tt_t.th_sport = inp->inp_lport; n->tt_t.th_dport = inp->inp_fport; @@ -262,7 +267,6 @@ n->tt_t.th_off = 5; n->tt_t.th_flags = 0; n->tt_t.th_win = 0; - n->tt_t.th_sum = 0; n->tt_t.th_urp = 0; return (n); } @@ -296,7 +300,6 @@ struct route *ro = 0; struct route sro; struct ip *ip; - struct ipovly *ipov; struct tcphdr *nth; #ifdef INET6 struct route_in6 *ro6 = 0; @@ -311,7 +314,6 @@ ip6 = ipgen; #endif /* INET6 */ ip = ipgen; - ipov = ipgen; if (tp) { if (!(flags & TH_RST)) { @@ -358,7 +360,6 @@ { bcopy((caddr_t)ip, mtod(m, caddr_t), sizeof(struct ip)); ip = mtod(m, struct ip *); - ipov = mtod(m, struct ipovly *); nth = (struct tcphdr *)(ip + 1); } bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr)); @@ -400,8 +401,9 @@ } else #endif { - ipov->ih_len = htons((u_short)(sizeof (struct tcphdr) + tlen)); tlen += sizeof (struct tcpiphdr); + ip->ip_len = tlen; + ip->ip_ttl = ip_defttl; } m->m_len = tlen; m->m_pkthdr.len = tlen; @@ -416,7 +418,6 @@ else nth->th_win = htons((u_short)win); nth->th_urp = 0; - nth->th_sum = 0; #ifdef INET6 if (isipv6) { nth->th_sum = in6_cksum(m, IPPROTO_TCP, @@ -429,14 +430,10 @@ } else #endif /* INET6 */ { - bzero(ipov->ih_x1, sizeof(ipov->ih_x1)); - nth->th_sum = in_cksum(m, tlen); -#ifdef INET6 - /* Re-initialization for later version check */ - ip->ip_vhl = IP_MAKE_VHL(IPVERSION, 0); -#endif /* INET6 */ - ip->ip_len = tlen; - ip->ip_ttl = ip_defttl; + nth->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr, + htons((u_short)(tlen - sizeof(struct ip) + ip->ip_p))); + m->m_pkthdr.csum_flags = CSUM_TCP; + m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); } #ifdef TCPDEBUG if (tp == NULL || (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) Index: netinet/udp_usrreq.c =================================================================== RCS file: /ncvs/src/sys/netinet/udp_usrreq.c,v retrieving revision 1.64 diff -u -r1.64 udp_usrreq.c --- netinet/udp_usrreq.c 1999/12/22 19:13:24 1.64 +++ netinet/udp_usrreq.c 2000/03/22 18:48:17 @@ -37,6 +37,7 @@ #include "opt_ipsec.h" #include "opt_inet6.h" +#include #include #include #include @@ -76,6 +77,8 @@ #include #endif /*IPSEC*/ +#include + /* * UDP protocol implementation. * Per RFC 768, August, 1980. @@ -208,9 +211,19 @@ * Checksum extended UDP header and data. */ if (uh->uh_sum) { - bzero(((struct ipovly *)ip)->ih_x1, 9); - ((struct ipovly *)ip)->ih_len = uh->uh_ulen; - uh->uh_sum = in_cksum(m, len + sizeof (struct ip)); + if (m->m_pkthdr.csum_flags & CSUM_DATA_VALID) { + if (m->m_pkthdr.csum_flags & CSUM_PSEUDO_HDR) + uh->uh_sum = m->m_pkthdr.csum_data; + else + uh->uh_sum = in_pseudo(ip->ip_src.s_addr, + ip->ip_dst.s_addr, htonl(ip->ip_len + + m->m_pkthdr.csum_data + IPPROTO_UDP)); + uh->uh_sum ^= 0xffff; + } else { + bzero(((struct ipovly *)ip)->ih_x1, 9); + ((struct ipovly *)ip)->ih_len = uh->uh_ulen; + uh->uh_sum = in_cksum(m, len + sizeof (struct ip)); + } if (uh->uh_sum) { udpstat.udps_badsum++; m_freem(m); @@ -679,22 +692,24 @@ * and addresses and length put into network format. */ ui = mtod(m, struct udpiphdr *); - bzero(ui->ui_x1, sizeof(ui->ui_x1)); + bzero(ui->ui_x1, sizeof(ui->ui_x1)); /* XXX still needed? */ ui->ui_pr = IPPROTO_UDP; - ui->ui_len = htons((u_short)len + sizeof (struct udphdr)); ui->ui_src = inp->inp_laddr; ui->ui_dst = inp->inp_faddr; ui->ui_sport = inp->inp_lport; ui->ui_dport = inp->inp_fport; - ui->ui_ulen = ui->ui_len; + ui->ui_ulen = htons((u_short)len + sizeof(struct udphdr)); /* - * Stuff checksum and output datagram. + * Set up checksum and output datagram. */ - ui->ui_sum = 0; if (udpcksum) { - if ((ui->ui_sum = in_cksum(m, sizeof (struct udpiphdr) + len)) == 0) - ui->ui_sum = 0xffff; + ui->ui_sum = in_pseudo(ui->ui_src.s_addr, ui->ui_dst.s_addr, + htons((u_short)len + sizeof(struct udphdr) + IPPROTO_UDP)); + m->m_pkthdr.csum_flags = CSUM_UDP; + m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); + } else { + ui->ui_sum = 0; } ((struct ip *)ui)->ip_len = sizeof (struct udpiphdr) + len; ((struct ip *)ui)->ip_ttl = inp->inp_ip_ttl; /* XXX */ Index: pci/if_ti.c =================================================================== RCS file: /ncvs/src/sys/pci/if_ti.c,v retrieving revision 1.25 diff -u -r1.25 if_ti.c --- pci/if_ti.c 2000/01/18 00:26:29 1.25 +++ pci/if_ti.c 2000/03/26 18:07:13 @@ -122,9 +122,7 @@ #include #include -#ifdef M_HWCKSUM -/*#define TI_CSUM_OFFLOAD*/ -#endif +#define TI_CSUM_FEATURES (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_IP_FRAGS) #if !defined(lint) static const char rcsid[] = @@ -792,11 +790,9 @@ r = &sc->ti_rdata->ti_rx_std_ring[i]; TI_HOSTADDR(r->ti_addr) = vtophys(mtod(m_new, caddr_t)); r->ti_type = TI_BDTYPE_RECV_BD; -#ifdef TI_CSUM_OFFLOAD - r->ti_flags = TI_BDFLAG_TCP_UDP_CKSUM|TI_BDFLAG_IP_CKSUM; -#else r->ti_flags = 0; -#endif + if (sc->arpcom.ac_if.if_hwassist) + r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM; r->ti_len = m_new->m_len; r->ti_idx = i; @@ -835,9 +831,8 @@ TI_HOSTADDR(r->ti_addr) = vtophys(mtod(m_new, caddr_t)); r->ti_type = TI_BDTYPE_RECV_BD; r->ti_flags = TI_BDFLAG_MINI_RING; -#ifdef TI_CSUM_OFFLOAD - r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM|TI_BDFLAG_IP_CKSUM; -#endif + if (sc->arpcom.ac_if.if_hwassist) + r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM; r->ti_len = m_new->m_len; r->ti_idx = i; @@ -896,9 +891,8 @@ TI_HOSTADDR(r->ti_addr) = vtophys(mtod(m_new, caddr_t)); r->ti_type = TI_BDTYPE_RECV_JUMBO_BD; r->ti_flags = TI_BDFLAG_JUMBO_RING; -#ifdef TI_CSUM_OFFLOAD - r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM|TI_BDFLAG_IP_CKSUM; -#endif + if (sc->arpcom.ac_if.if_hwassist) + r->ti_flags |= TI_BDFLAG_TCP_UDP_CKSUM | TI_BDFLAG_IP_CKSUM; r->ti_len = m_new->m_len; r->ti_idx = i; @@ -1206,6 +1200,8 @@ /* Initialize link to down state. */ sc->ti_linkstat = TI_EV_CODE_LINK_DOWN; + sc->arpcom.ac_if.if_hwassist = TI_CSUM_FEATURES; + /* Set endianness before we access any non-PCI registers. */ #if BYTE_ORDER == BIG_ENDIAN CSR_WRITE_4(sc, TI_MISC_HOST_CTL, @@ -1316,11 +1312,10 @@ * Only allow 1 DMA channel to be active at a time. * I don't think this is a good idea, but without it * the firmware racks up lots of nicDmaReadRingFull - * errors. + * errors. This is not compatible with hardware checksums. */ -#ifndef TI_CSUM_OFFLOAD - TI_SETBIT(sc, TI_GCR_OPMODE, TI_OPMODE_1_DMA_ACTIVE); -#endif + if (sc->arpcom.ac_if.if_hwassist == 0) + TI_SETBIT(sc, TI_GCR_OPMODE, TI_OPMODE_1_DMA_ACTIVE); /* Recommended settings from Tigon manual. */ CSR_WRITE_4(sc, TI_GCR_DMA_WRITECFG, TI_DMA_STATE_THRESH_8W); @@ -1399,9 +1394,9 @@ TI_HOSTADDR(rcb->ti_hostaddr) = vtophys(&sc->ti_rdata->ti_rx_std_ring); rcb->ti_max_len = TI_FRAMELEN; rcb->ti_flags = 0; -#ifdef TI_CSUM_OFFLOAD - rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM|TI_RCB_FLAG_IP_CKSUM; -#endif + if (sc->arpcom.ac_if.if_hwassist) + rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM | + TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM; #if NVLAN > 0 rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST; #endif @@ -1412,9 +1407,9 @@ vtophys(&sc->ti_rdata->ti_rx_jumbo_ring); rcb->ti_max_len = TI_JUMBO_FRAMELEN; rcb->ti_flags = 0; -#ifdef TI_CSUM_OFFLOAD - rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM|TI_RCB_FLAG_IP_CKSUM; -#endif + if (sc->arpcom.ac_if.if_hwassist) + rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM | + TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM; #if NVLAN > 0 rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST; #endif @@ -1432,9 +1427,9 @@ rcb->ti_flags = TI_RCB_FLAG_RING_DISABLED; else rcb->ti_flags = 0; -#ifdef TI_CSUM_OFFLOAD - rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM|TI_RCB_FLAG_IP_CKSUM; -#endif + if (sc->arpcom.ac_if.if_hwassist) + rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM | + TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM; #if NVLAN > 0 rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST; #endif @@ -1474,6 +1469,9 @@ #if NVLAN > 0 rcb->ti_flags |= TI_RCB_FLAG_VLAN_ASSIST; #endif + if (sc->arpcom.ac_if.if_hwassist) + rcb->ti_flags |= TI_RCB_FLAG_TCP_UDP_CKSUM | + TI_RCB_FLAG_IP_CKSUM | TI_RCB_FLAG_NO_PHDR_CKSUM; rcb->ti_max_len = TI_TX_RING_CNT; if (sc->ti_hwrev == TI_HWREV_TIGON) TI_HOSTADDR(rcb->ti_hostaddr) = TI_TX_RING_BASE; @@ -1791,9 +1789,6 @@ u_int16_t vlan_tag = 0; int have_tag = 0; #endif -#ifdef TI_CSUM_OFFLOAD - struct ip *ip; -#endif cur_rx = &sc->ti_rdata->ti_rx_return_ring[sc->ti_rx_saved_considx]; @@ -1876,12 +1871,13 @@ /* Remove header from mbuf and pass it on. */ m_adj(m, sizeof(struct ether_header)); -#ifdef TI_CSUM_OFFLOAD - ip = mtod(m, struct ip *); - if (!(cur_rx->ti_tcp_udp_cksum ^ 0xFFFF) && - !(ip->ip_off & htons(IP_MF | IP_OFFMASK | IP_RF))) - m->m_flags |= M_HWCKSUM; -#endif + if (ifp->if_hwassist) { + m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | + CSUM_DATA_VALID; + if ((cur_rx->ti_ip_cksum ^ 0xffff) == 0) + m->m_pkthdr.csum_flags |= CSUM_IP_VALID; + m->m_pkthdr.csum_data = cur_rx->ti_tcp_udp_cksum; + } #if NVLAN > 0 /* @@ -2025,6 +2021,7 @@ struct ti_tx_desc *f = NULL; struct mbuf *m; u_int32_t frag, cur, cnt = 0; + u_int16_t csum_flags = 0; #if NVLAN > 0 struct ifvlan *ifv = NULL; @@ -2037,6 +2034,16 @@ m = m_head; cur = frag = *txidx; + if (m_head->m_pkthdr.csum_flags) { + if (m_head->m_pkthdr.csum_flags & CSUM_IP) + csum_flags |= TI_BDFLAG_IP_CKSUM; + if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP)) + csum_flags |= TI_BDFLAG_TCP_UDP_CKSUM; + if (m_head->m_flags & M_LASTFRAG) + csum_flags |= TI_BDFLAG_IP_FRAG_END; + else if (m_head->m_flags & M_FRAG) + csum_flags |= TI_BDFLAG_IP_FRAG; + } /* * Start packing the mbufs in this chain into * the fragment pointers. Stop when we run out @@ -2064,7 +2071,7 @@ break; TI_HOSTADDR(f->ti_addr) = vtophys(mtod(m, vm_offset_t)); f->ti_len = m->m_len; - f->ti_flags = 0; + f->ti_flags = csum_flags; #if NVLAN > 0 if (ifv != NULL) { f->ti_flags |= TI_BDFLAG_VLAN_TAG; @@ -2123,6 +2130,24 @@ IF_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; + + /* + * XXX + * safety overkill. If this is a fragmented packet chain + * with delayed TCP/UDP checksums, then only encapsulate + * it if we have enough descriptors to handle the entire + * chain at once. + * (paranoia -- may not actually be needed) + */ + if (m_head->m_flags & M_FIRSTFRAG && + m_head->m_pkthdr.csum_flags & (CSUM_DELAY_DATA)) { + if ((TI_TX_RING_CNT - sc->ti_txcnt) < + m_head->m_pkthdr.csum_data + 16) { + IF_PREPEND(&ifp->if_snd, m_head); + ifp->if_flags |= IFF_OACTIVE; + break; + } + } /* * Pack the data into the transmit ring. If we Index: sys/mbuf.h =================================================================== RCS file: /ncvs/src/sys/sys/mbuf.h,v retrieving revision 1.45 diff -u -r1.45 mbuf.h --- sys/mbuf.h 2000/03/14 20:49:27 1.45 +++ sys/mbuf.h 2000/03/26 15:18:14 @@ -80,6 +80,9 @@ int len; /* total packet length */ /* variables for ip and tcp reassembly */ caddr_t header; /* pointer to packet header */ + /* variables for hardware checksum */ + int csum_flags; /* flags regarding checksum */ + int csum_data; /* data field used by csum routines */ }; /* description of external storage mapped into mbuf, valid if M_EXT set */ @@ -131,11 +134,28 @@ #define M_BCAST 0x0100 /* send/received as link-level broadcast */ #define M_MCAST 0x0200 /* send/received as link-level multicast */ #define M_FRAG 0x0400 /* packet is a fragment of a larger packet */ +#define M_FIRSTFRAG 0x0800 /* packet is first fragment */ +#define M_LASTFRAG 0x1000 /* packet is last fragment */ /* flags copied when copying m_pkthdr */ #define M_COPYFLAGS (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO1|M_PROTO2|M_PROTO3 | \ M_PROTO4|M_PROTO5|M_BCAST|M_MCAST|M_FRAG) +/* flags indicating hw checksum support and sw checksum requirements */ +#define CSUM_IP 0x0001 /* will csum IP */ +#define CSUM_TCP 0x0002 /* will csum TCP */ +#define CSUM_UDP 0x0004 /* will csum UDP */ +#define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */ +#define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */ + +#define CSUM_IP_CHECKED 0x0100 /* did csum IP */ +#define CSUM_IP_VALID 0x0200 /* ... the csum is valid */ +#define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ +#define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ + +#define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) +#define CSUM_DELAY_IP (CSUM_IP) /* XXX add ipv6 here too? */ + /* mbuf types */ #define MT_FREE 0 /* should be on free list */ #define MT_DATA 1 /* dynamic (data) allocation */ @@ -301,6 +321,7 @@ _mm->m_data = _mm->m_pktdat; \ _mm->m_flags = M_PKTHDR; \ _mm->m_pkthdr.rcvif = NULL; \ + _mm->m_pkthdr.csum_flags = 0; \ (m) = _mm; \ splx(_ms); \ } else { \