1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright (c) 2021, Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Intel Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 /*$FreeBSD$*/ 32 33 /** 34 * @file ice_common_txrx.h 35 * @brief common Tx/Rx utility functions 36 * 37 * Contains common utility functions for the Tx/Rx hot path. 38 * 39 * The functions do depend on the if_pkt_info_t structure. A suitable 40 * implementation of this structure must be provided if these functions are to 41 * be used without the iflib networking stack. 42 */ 43 44 #ifndef _ICE_COMMON_TXRX_H_ 45 #define _ICE_COMMON_TXRX_H_ 46 47 #include <netinet/udp.h> 48 #include <netinet/sctp.h> 49 50 /** 51 * ice_tso_detect_sparse - detect TSO packets with too many segments 52 * @pi: packet information 53 * 54 * Hardware only transmits packets with a maximum of 8 descriptors. For TSO 55 * packets, hardware needs to be able to build the split packets using 8 or 56 * fewer descriptors. Additionally, the header must be contained within at 57 * most 3 descriptors. 58 * 59 * To verify this, we walk the headers to find out how many descriptors the 60 * headers require (usually 1). Then we ensure that, for each TSO segment, its 61 * data plus the headers are contained within 8 or fewer descriptors. 62 */ 63 static inline int 64 ice_tso_detect_sparse(if_pkt_info_t pi) 65 { 66 int count, curseg, i, hlen, segsz, seglen, tsolen, hdrs, maxsegs; 67 bus_dma_segment_t *segs = pi->ipi_segs; 68 int nsegs = pi->ipi_nsegs; 69 70 curseg = hdrs = 0; 71 72 hlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 73 tsolen = pi->ipi_len - hlen; 74 75 /* First, count the number of descriptors for the header. 76 * Additionally, make sure it does not span more than 3 segments. 77 */ 78 i = 0; 79 curseg = segs[0].ds_len; 80 while (hlen > 0) { 81 hdrs++; 82 if (hdrs > ICE_MAX_TSO_HDR_SEGS) 83 return (1); 84 if (curseg == 0) { 85 i++; 86 if (__predict_false(i == nsegs)) 87 return (1); 88 89 curseg = segs[i].ds_len; 90 } 91 seglen = min(curseg, hlen); 92 curseg -= seglen; 93 hlen -= seglen; 94 } 95 96 maxsegs = ICE_MAX_TX_SEGS - hdrs; 97 98 /* We must count the headers, in order to verify that they take up 99 * 3 or fewer descriptors. However, we don't need to check the data 100 * if the total segments is small. 101 */ 102 if (nsegs <= maxsegs) 103 return (0); 104 105 count = 0; 106 107 /* Now check the data to make sure that each TSO segment is made up of 108 * no more than maxsegs descriptors. This ensures that hardware will 109 * be capable of performing TSO offload. 110 */ 111 while (tsolen > 0) { 112 segsz = pi->ipi_tso_segsz; 113 while (segsz > 0 && tsolen != 0) { 114 count++; 115 if (count > maxsegs) { 116 return (1); 117 } 118 if (curseg == 0) { 119 i++; 120 if (__predict_false(i == nsegs)) { 121 return (1); 122 } 123 curseg = segs[i].ds_len; 124 } 125 seglen = min(curseg, segsz); 126 segsz -= seglen; 127 curseg -= seglen; 128 tsolen -= seglen; 129 } 130 count = 0; 131 } 132 133 return (0); 134 } 135 136 /** 137 * ice_tso_setup - Setup a context descriptor to prepare for a TSO packet 138 * @txq: the Tx queue to use 139 * @pi: the packet info to prepare for 140 * 141 * Setup a context descriptor in preparation for sending a Tx packet that 142 * requires the TSO offload. Returns the index of the descriptor to use when 143 * encapsulating the Tx packet data into descriptors. 144 */ 145 static inline int 146 ice_tso_setup(struct ice_tx_queue *txq, if_pkt_info_t pi) 147 { 148 struct ice_tx_ctx_desc *txd; 149 u32 cmd, mss, type, tsolen; 150 int idx; 151 u64 type_cmd_tso_mss; 152 153 idx = pi->ipi_pidx; 154 txd = (struct ice_tx_ctx_desc *)&txq->tx_base[idx]; 155 tsolen = pi->ipi_len - (pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen); 156 157 type = ICE_TX_DESC_DTYPE_CTX; 158 cmd = ICE_TX_CTX_DESC_TSO; 159 /* TSO MSS must not be less than 64 */ 160 if (pi->ipi_tso_segsz < ICE_MIN_TSO_MSS) { 161 txq->stats.mss_too_small++; 162 pi->ipi_tso_segsz = ICE_MIN_TSO_MSS; 163 } 164 mss = pi->ipi_tso_segsz; 165 166 type_cmd_tso_mss = ((u64)type << ICE_TXD_CTX_QW1_DTYPE_S) | 167 ((u64)cmd << ICE_TXD_CTX_QW1_CMD_S) | 168 ((u64)tsolen << ICE_TXD_CTX_QW1_TSO_LEN_S) | 169 ((u64)mss << ICE_TXD_CTX_QW1_MSS_S); 170 txd->qw1 = htole64(type_cmd_tso_mss); 171 172 txd->tunneling_params = htole32(0); 173 txq->tso++; 174 175 return ((idx + 1) & (txq->desc_count-1)); 176 } 177 178 /** 179 * ice_tx_setup_offload - Setup register values for performing a Tx offload 180 * @txq: The Tx queue, used to track checksum offload stats 181 * @pi: the packet info to program for 182 * @cmd: the cmd register value to update 183 * @off: the off register value to update 184 * 185 * Based on the packet info provided, update the cmd and off values for 186 * enabling Tx offloads. This depends on the packet type and which offloads 187 * have been requested. 188 * 189 * We also track the total number of times that we've requested hardware 190 * offload a particular type of checksum for debugging purposes. 191 */ 192 static inline void 193 ice_tx_setup_offload(struct ice_tx_queue *txq, if_pkt_info_t pi, u32 *cmd, u32 *off) 194 { 195 u32 remaining_csum_flags = pi->ipi_csum_flags; 196 197 switch (pi->ipi_etype) { 198 #ifdef INET 199 case ETHERTYPE_IP: 200 if (pi->ipi_csum_flags & ICE_CSUM_IP) { 201 *cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM; 202 txq->stats.cso[ICE_CSO_STAT_TX_IP4]++; 203 remaining_csum_flags &= ~CSUM_IP; 204 } else 205 *cmd |= ICE_TX_DESC_CMD_IIPT_IPV4; 206 break; 207 #endif 208 #ifdef INET6 209 case ETHERTYPE_IPV6: 210 *cmd |= ICE_TX_DESC_CMD_IIPT_IPV6; 211 /* 212 * This indicates that the IIPT flag was set to the IPV6 value; 213 * there's no checksum for IPv6 packets. 214 */ 215 txq->stats.cso[ICE_CSO_STAT_TX_IP6]++; 216 break; 217 #endif 218 default: 219 txq->stats.cso[ICE_CSO_STAT_TX_L3_ERR]++; 220 break; 221 } 222 223 *off |= (pi->ipi_ehdrlen >> 1) << ICE_TX_DESC_LEN_MACLEN_S; 224 *off |= (pi->ipi_ip_hlen >> 2) << ICE_TX_DESC_LEN_IPLEN_S; 225 226 if (!(remaining_csum_flags & ~ICE_RX_CSUM_FLAGS)) 227 return; 228 229 switch (pi->ipi_ipproto) { 230 case IPPROTO_TCP: 231 if (pi->ipi_csum_flags & ICE_CSUM_TCP) { 232 *cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP; 233 *off |= (pi->ipi_tcp_hlen >> 2) << 234 ICE_TX_DESC_LEN_L4_LEN_S; 235 txq->stats.cso[ICE_CSO_STAT_TX_TCP]++; 236 } 237 break; 238 case IPPROTO_UDP: 239 if (pi->ipi_csum_flags & ICE_CSUM_UDP) { 240 *cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP; 241 *off |= (sizeof(struct udphdr) >> 2) << 242 ICE_TX_DESC_LEN_L4_LEN_S; 243 txq->stats.cso[ICE_CSO_STAT_TX_UDP]++; 244 } 245 break; 246 case IPPROTO_SCTP: 247 if (pi->ipi_csum_flags & ICE_CSUM_SCTP) { 248 *cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP; 249 *off |= (sizeof(struct sctphdr) >> 2) << 250 ICE_TX_DESC_LEN_L4_LEN_S; 251 txq->stats.cso[ICE_CSO_STAT_TX_SCTP]++; 252 } 253 break; 254 default: 255 txq->stats.cso[ICE_CSO_STAT_TX_L4_ERR]++; 256 break; 257 } 258 } 259 260 /** 261 * ice_rx_checksum - verify hardware checksum is valid or not 262 * @rxq: the Rx queue structure 263 * @flags: checksum flags to update 264 * @data: checksum data to update 265 * @status0: descriptor status data 266 * @ptype: packet type 267 * 268 * Determine whether the hardware indicated that the Rx checksum is valid. If 269 * so, update the checksum flags and data, informing the stack of the status 270 * of the checksum so that it does not spend time verifying it manually. 271 */ 272 static void 273 ice_rx_checksum(struct ice_rx_queue *rxq, uint32_t *flags, uint32_t *data, 274 u16 status0, u16 ptype) 275 { 276 const u16 l3_error = (BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S) | 277 BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)); 278 const u16 l4_error = (BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_L4E_S) | 279 BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S)); 280 const u16 xsum_errors = (l3_error | l4_error | 281 BIT(ICE_RX_FLEX_DESC_STATUS0_IPV6EXADD_S)); 282 struct ice_rx_ptype_decoded decoded; 283 bool is_ipv4, is_ipv6; 284 285 /* No L3 or L4 checksum was calculated */ 286 if (!(status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L3L4P_S))) { 287 return; 288 } 289 290 decoded = ice_decode_rx_desc_ptype(ptype); 291 *flags = 0; 292 293 if (!(decoded.known && decoded.outer_ip)) 294 return; 295 296 is_ipv4 = (decoded.outer_ip == ICE_RX_PTYPE_OUTER_IP) && 297 (decoded.outer_ip_ver == ICE_RX_PTYPE_OUTER_IPV4); 298 is_ipv6 = (decoded.outer_ip == ICE_RX_PTYPE_OUTER_IP) && 299 (decoded.outer_ip_ver == ICE_RX_PTYPE_OUTER_IPV6); 300 301 /* No checksum errors were reported */ 302 if (!(status0 & xsum_errors)) { 303 if (is_ipv4) 304 *flags |= CSUM_L3_CALC | CSUM_L3_VALID; 305 306 switch (decoded.inner_prot) { 307 case ICE_RX_PTYPE_INNER_PROT_TCP: 308 case ICE_RX_PTYPE_INNER_PROT_UDP: 309 case ICE_RX_PTYPE_INNER_PROT_SCTP: 310 *flags |= CSUM_L4_CALC | CSUM_L4_VALID; 311 *data |= htons(0xffff); 312 break; 313 default: 314 break; 315 } 316 317 return; 318 } 319 320 /* 321 * Certain IPv6 extension headers impact the validity of L4 checksums. 322 * If one of these headers exist, hardware will set the IPV6EXADD bit 323 * in the descriptor. If the bit is set then pretend like hardware 324 * didn't checksum this packet. 325 */ 326 if (is_ipv6 && (status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_IPV6EXADD_S))) { 327 rxq->stats.cso[ICE_CSO_STAT_RX_IP6_ERR]++; 328 return; 329 } 330 331 /* 332 * At this point, status0 must have at least one of the l3_error or 333 * l4_error bits set. 334 */ 335 336 if (status0 & l3_error) { 337 if (is_ipv4) { 338 rxq->stats.cso[ICE_CSO_STAT_RX_IP4_ERR]++; 339 *flags |= CSUM_L3_CALC; 340 } else { 341 /* Hardware indicated L3 error but this isn't IPv4? */ 342 rxq->stats.cso[ICE_CSO_STAT_RX_L3_ERR]++; 343 } 344 /* don't bother reporting L4 errors if we got an L3 error */ 345 return; 346 } else if (is_ipv4) { 347 *flags |= CSUM_L3_CALC | CSUM_L3_VALID; 348 } 349 350 if (status0 & l4_error) { 351 switch (decoded.inner_prot) { 352 case ICE_RX_PTYPE_INNER_PROT_TCP: 353 rxq->stats.cso[ICE_CSO_STAT_RX_TCP_ERR]++; 354 *flags |= CSUM_L4_CALC; 355 break; 356 case ICE_RX_PTYPE_INNER_PROT_UDP: 357 rxq->stats.cso[ICE_CSO_STAT_RX_UDP_ERR]++; 358 *flags |= CSUM_L4_CALC; 359 break; 360 case ICE_RX_PTYPE_INNER_PROT_SCTP: 361 rxq->stats.cso[ICE_CSO_STAT_RX_SCTP_ERR]++; 362 *flags |= CSUM_L4_CALC; 363 break; 364 default: 365 /* 366 * Hardware indicated L4 error, but this isn't one of 367 * the expected protocols. 368 */ 369 rxq->stats.cso[ICE_CSO_STAT_RX_L4_ERR]++; 370 } 371 } 372 } 373 374 /** 375 * ice_ptype_to_hash - Convert packet type to a hash value 376 * @ptype: the packet type to convert 377 * 378 * Given the packet type, convert to a suitable hashtype to report to the 379 * upper stack via the iri_rsstype value of the if_rxd_info_t structure. 380 * 381 * If the hash type is unknown we'll report M_HASHTYPE_OPAQUE. 382 */ 383 static inline int 384 ice_ptype_to_hash(u16 ptype) 385 { 386 struct ice_rx_ptype_decoded decoded; 387 388 if (ptype >= ARRAY_SIZE(ice_ptype_lkup)) 389 return M_HASHTYPE_OPAQUE; 390 391 decoded = ice_decode_rx_desc_ptype(ptype); 392 393 if (!decoded.known) 394 return M_HASHTYPE_OPAQUE; 395 396 if (decoded.outer_ip == ICE_RX_PTYPE_OUTER_L2) 397 return M_HASHTYPE_OPAQUE; 398 399 /* Note: anything that gets to this point is IP */ 400 if (decoded.outer_ip_ver == ICE_RX_PTYPE_OUTER_IPV6) { 401 switch (decoded.inner_prot) { 402 case ICE_RX_PTYPE_INNER_PROT_TCP: 403 return M_HASHTYPE_RSS_TCP_IPV6; 404 case ICE_RX_PTYPE_INNER_PROT_UDP: 405 return M_HASHTYPE_RSS_UDP_IPV6; 406 default: 407 return M_HASHTYPE_RSS_IPV6; 408 } 409 } 410 if (decoded.outer_ip_ver == ICE_RX_PTYPE_OUTER_IPV4) { 411 switch (decoded.inner_prot) { 412 case ICE_RX_PTYPE_INNER_PROT_TCP: 413 return M_HASHTYPE_RSS_TCP_IPV4; 414 case ICE_RX_PTYPE_INNER_PROT_UDP: 415 return M_HASHTYPE_RSS_UDP_IPV4; 416 default: 417 return M_HASHTYPE_RSS_IPV4; 418 } 419 } 420 421 /* We should never get here!! */ 422 return M_HASHTYPE_OPAQUE; 423 } 424 #endif 425