1bbb2f537SJohn Baldwin /*- 2bbb2f537SJohn Baldwin * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3bbb2f537SJohn Baldwin * 4bbb2f537SJohn Baldwin * Copyright (c) 2018-2019 Chelsio Communications, Inc. 5bbb2f537SJohn Baldwin * All rights reserved. 6bbb2f537SJohn Baldwin * Written by: John Baldwin <jhb@FreeBSD.org> 7bbb2f537SJohn Baldwin * 8bbb2f537SJohn Baldwin * Redistribution and use in source and binary forms, with or without 9bbb2f537SJohn Baldwin * modification, are permitted provided that the following conditions 10bbb2f537SJohn Baldwin * are met: 11bbb2f537SJohn Baldwin * 1. Redistributions of source code must retain the above copyright 12bbb2f537SJohn Baldwin * notice, this list of conditions and the following disclaimer. 13bbb2f537SJohn Baldwin * 2. Redistributions in binary form must reproduce the above copyright 14bbb2f537SJohn Baldwin * notice, this list of conditions and the following disclaimer in the 15bbb2f537SJohn Baldwin * documentation and/or other materials provided with the distribution. 16bbb2f537SJohn Baldwin * 17bbb2f537SJohn Baldwin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18bbb2f537SJohn Baldwin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19bbb2f537SJohn Baldwin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20bbb2f537SJohn Baldwin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21bbb2f537SJohn Baldwin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22bbb2f537SJohn Baldwin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23bbb2f537SJohn Baldwin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24bbb2f537SJohn Baldwin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25bbb2f537SJohn Baldwin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26bbb2f537SJohn Baldwin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27bbb2f537SJohn Baldwin * SUCH DAMAGE. 28bbb2f537SJohn Baldwin */ 29bbb2f537SJohn Baldwin 30bbb2f537SJohn Baldwin #include "opt_inet.h" 31bbb2f537SJohn Baldwin #include "opt_inet6.h" 32bbb2f537SJohn Baldwin #include "opt_kern_tls.h" 33bbb2f537SJohn Baldwin 34bbb2f537SJohn Baldwin #include <sys/cdefs.h> 35bbb2f537SJohn Baldwin __FBSDID("$FreeBSD$"); 36bbb2f537SJohn Baldwin 37bbb2f537SJohn Baldwin #include <sys/param.h> 38bbb2f537SJohn Baldwin #include <sys/ktr.h> 39bbb2f537SJohn Baldwin #include <sys/ktls.h> 40bbb2f537SJohn Baldwin #include <sys/sglist.h> 41bbb2f537SJohn Baldwin #include <sys/socket.h> 42bbb2f537SJohn Baldwin #include <sys/socketvar.h> 43bbb2f537SJohn Baldwin #include <sys/sockbuf.h> 44bbb2f537SJohn Baldwin #include <netinet/in.h> 45bbb2f537SJohn Baldwin #include <netinet/in_pcb.h> 46bbb2f537SJohn Baldwin #include <netinet/ip.h> 47bbb2f537SJohn Baldwin #include <netinet/ip6.h> 48bbb2f537SJohn Baldwin #include <netinet/tcp_var.h> 49bbb2f537SJohn Baldwin #include <opencrypto/cryptodev.h> 50bbb2f537SJohn Baldwin #include <opencrypto/xform.h> 51bbb2f537SJohn Baldwin 52bbb2f537SJohn Baldwin #include "common/common.h" 53bbb2f537SJohn Baldwin #include "common/t4_regs.h" 54bbb2f537SJohn Baldwin #include "common/t4_regs_values.h" 55bbb2f537SJohn Baldwin #include "common/t4_tcb.h" 56bbb2f537SJohn Baldwin #include "t4_l2t.h" 57bbb2f537SJohn Baldwin #include "t4_clip.h" 58bbb2f537SJohn Baldwin #include "t4_mp_ring.h" 59bbb2f537SJohn Baldwin #include "crypto/t4_crypto.h" 60bbb2f537SJohn Baldwin 61bbb2f537SJohn Baldwin #if defined(INET) || defined(INET6) 62bbb2f537SJohn Baldwin 63bbb2f537SJohn Baldwin #define TLS_HEADER_LENGTH 5 64bbb2f537SJohn Baldwin 65bbb2f537SJohn Baldwin struct tls_scmd { 66bbb2f537SJohn Baldwin __be32 seqno_numivs; 67bbb2f537SJohn Baldwin __be32 ivgen_hdrlen; 68bbb2f537SJohn Baldwin }; 69bbb2f537SJohn Baldwin 70bbb2f537SJohn Baldwin struct tlspcb { 71bbb2f537SJohn Baldwin struct m_snd_tag com; 72bbb2f537SJohn Baldwin struct vi_info *vi; /* virtual interface */ 73bbb2f537SJohn Baldwin struct adapter *sc; 74bbb2f537SJohn Baldwin struct l2t_entry *l2te; /* L2 table entry used by this connection */ 75ca457729SJohn Baldwin struct sge_txq *txq; 76bbb2f537SJohn Baldwin int tid; /* Connection identifier */ 77bbb2f537SJohn Baldwin 78bbb2f537SJohn Baldwin int tx_key_addr; 79bbb2f537SJohn Baldwin bool inline_key; 80bbb2f537SJohn Baldwin bool using_timestamps; 81bbb2f537SJohn Baldwin unsigned char enc_mode; 82bbb2f537SJohn Baldwin 83bbb2f537SJohn Baldwin struct tls_scmd scmd0; 84bbb2f537SJohn Baldwin struct tls_scmd scmd0_short; 85bbb2f537SJohn Baldwin 86bbb2f537SJohn Baldwin unsigned int tx_key_info_size; 87bbb2f537SJohn Baldwin 88bbb2f537SJohn Baldwin uint32_t prev_seq; 89bbb2f537SJohn Baldwin uint32_t prev_ack; 90bbb2f537SJohn Baldwin uint32_t prev_tsecr; 91bbb2f537SJohn Baldwin uint16_t prev_win; 92bbb2f537SJohn Baldwin uint16_t prev_mss; 93bbb2f537SJohn Baldwin 94bbb2f537SJohn Baldwin /* Only used outside of setup and teardown when using inline keys. */ 95bbb2f537SJohn Baldwin struct tls_keyctx keyctx; 96bbb2f537SJohn Baldwin 97bbb2f537SJohn Baldwin /* Fields only used during setup and teardown. */ 98bbb2f537SJohn Baldwin struct inpcb *inp; /* backpointer to host stack's PCB */ 99bbb2f537SJohn Baldwin struct sge_wrq *ctrlq; 100bbb2f537SJohn Baldwin struct clip_entry *ce; /* CLIP table entry used by this tid */ 101bbb2f537SJohn Baldwin 102bbb2f537SJohn Baldwin bool open_pending; 103bbb2f537SJohn Baldwin }; 104bbb2f537SJohn Baldwin 105bbb2f537SJohn Baldwin static void t6_tls_tag_free(struct m_snd_tag *mst); 106bbb2f537SJohn Baldwin static int ktls_setup_keys(struct tlspcb *tlsp, 107bbb2f537SJohn Baldwin const struct ktls_session *tls, struct sge_txq *txq); 108bbb2f537SJohn Baldwin 109bbb2f537SJohn Baldwin static const struct if_snd_tag_sw t6_tls_tag_sw = { 110bbb2f537SJohn Baldwin .snd_tag_free = t6_tls_tag_free, 111bbb2f537SJohn Baldwin .type = IF_SND_TAG_TYPE_TLS 112bbb2f537SJohn Baldwin }; 113bbb2f537SJohn Baldwin 114bbb2f537SJohn Baldwin static inline struct tlspcb * 115bbb2f537SJohn Baldwin mst_to_tls(struct m_snd_tag *t) 116bbb2f537SJohn Baldwin { 117bbb2f537SJohn Baldwin return (__containerof(t, struct tlspcb, com)); 118bbb2f537SJohn Baldwin } 119bbb2f537SJohn Baldwin 120bbb2f537SJohn Baldwin static struct tlspcb * 121bbb2f537SJohn Baldwin alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags) 122bbb2f537SJohn Baldwin { 123bbb2f537SJohn Baldwin struct port_info *pi = vi->pi; 124bbb2f537SJohn Baldwin struct adapter *sc = pi->adapter; 125bbb2f537SJohn Baldwin struct tlspcb *tlsp; 126bbb2f537SJohn Baldwin 127bbb2f537SJohn Baldwin tlsp = malloc(sizeof(*tlsp), M_CXGBE, M_ZERO | flags); 128bbb2f537SJohn Baldwin if (tlsp == NULL) 129bbb2f537SJohn Baldwin return (NULL); 130bbb2f537SJohn Baldwin 131bbb2f537SJohn Baldwin m_snd_tag_init(&tlsp->com, ifp, &t6_tls_tag_sw); 132bbb2f537SJohn Baldwin tlsp->vi = vi; 133bbb2f537SJohn Baldwin tlsp->sc = sc; 134bbb2f537SJohn Baldwin tlsp->ctrlq = &sc->sge.ctrlq[pi->port_id]; 135bbb2f537SJohn Baldwin tlsp->tid = -1; 136bbb2f537SJohn Baldwin tlsp->tx_key_addr = -1; 137bbb2f537SJohn Baldwin 138bbb2f537SJohn Baldwin return (tlsp); 139bbb2f537SJohn Baldwin } 140bbb2f537SJohn Baldwin 141bbb2f537SJohn Baldwin static int 142bbb2f537SJohn Baldwin ktls_act_open_cpl_size(bool isipv6) 143bbb2f537SJohn Baldwin { 144bbb2f537SJohn Baldwin 145bbb2f537SJohn Baldwin if (isipv6) 146bbb2f537SJohn Baldwin return (sizeof(struct cpl_t6_act_open_req6)); 147bbb2f537SJohn Baldwin else 148bbb2f537SJohn Baldwin return (sizeof(struct cpl_t6_act_open_req)); 149bbb2f537SJohn Baldwin } 150bbb2f537SJohn Baldwin 151bbb2f537SJohn Baldwin static void 152bbb2f537SJohn Baldwin mk_ktls_act_open_req(struct adapter *sc, struct vi_info *vi, struct inpcb *inp, 153bbb2f537SJohn Baldwin struct tlspcb *tlsp, int atid, void *dst) 154bbb2f537SJohn Baldwin { 155bbb2f537SJohn Baldwin struct tcpcb *tp = intotcpcb(inp); 156bbb2f537SJohn Baldwin struct cpl_t6_act_open_req *cpl6; 157bbb2f537SJohn Baldwin struct cpl_act_open_req *cpl; 158bbb2f537SJohn Baldwin uint64_t options; 159bbb2f537SJohn Baldwin int qid_atid; 160bbb2f537SJohn Baldwin 161bbb2f537SJohn Baldwin cpl6 = dst; 162bbb2f537SJohn Baldwin cpl = (struct cpl_act_open_req *)cpl6; 163bbb2f537SJohn Baldwin INIT_TP_WR(cpl6, 0); 164bbb2f537SJohn Baldwin qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | 165bbb2f537SJohn Baldwin V_TID_COOKIE(CPL_COOKIE_KERN_TLS); 166bbb2f537SJohn Baldwin OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 167bbb2f537SJohn Baldwin qid_atid)); 168bbb2f537SJohn Baldwin inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port, 169bbb2f537SJohn Baldwin &cpl->peer_ip, &cpl->peer_port); 170bbb2f537SJohn Baldwin 171bbb2f537SJohn Baldwin options = F_TCAM_BYPASS | V_ULP_MODE(ULP_MODE_NONE); 172bbb2f537SJohn Baldwin options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan); 173bbb2f537SJohn Baldwin options |= F_NON_OFFLOAD; 174bbb2f537SJohn Baldwin cpl->opt0 = htobe64(options); 175bbb2f537SJohn Baldwin 176bbb2f537SJohn Baldwin options = V_TX_QUEUE(sc->params.tp.tx_modq[vi->pi->tx_chan]); 177bbb2f537SJohn Baldwin if (tp->t_flags & TF_REQ_TSTMP) 178bbb2f537SJohn Baldwin options |= F_TSTAMPS_EN; 179bbb2f537SJohn Baldwin cpl->opt2 = htobe32(options); 180bbb2f537SJohn Baldwin } 181bbb2f537SJohn Baldwin 182bbb2f537SJohn Baldwin static void 183bbb2f537SJohn Baldwin mk_ktls_act_open_req6(struct adapter *sc, struct vi_info *vi, 184bbb2f537SJohn Baldwin struct inpcb *inp, struct tlspcb *tlsp, int atid, void *dst) 185bbb2f537SJohn Baldwin { 186bbb2f537SJohn Baldwin struct tcpcb *tp = intotcpcb(inp); 187bbb2f537SJohn Baldwin struct cpl_t6_act_open_req6 *cpl6; 188bbb2f537SJohn Baldwin struct cpl_act_open_req6 *cpl; 189bbb2f537SJohn Baldwin uint64_t options; 190bbb2f537SJohn Baldwin int qid_atid; 191bbb2f537SJohn Baldwin 192bbb2f537SJohn Baldwin cpl6 = dst; 193bbb2f537SJohn Baldwin cpl = (struct cpl_act_open_req6 *)cpl6; 194bbb2f537SJohn Baldwin INIT_TP_WR(cpl6, 0); 195bbb2f537SJohn Baldwin qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) | 196bbb2f537SJohn Baldwin V_TID_COOKIE(CPL_COOKIE_KERN_TLS); 197bbb2f537SJohn Baldwin OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 198bbb2f537SJohn Baldwin qid_atid)); 199bbb2f537SJohn Baldwin cpl->local_port = inp->inp_lport; 200bbb2f537SJohn Baldwin cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0]; 201bbb2f537SJohn Baldwin cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8]; 202bbb2f537SJohn Baldwin cpl->peer_port = inp->inp_fport; 203bbb2f537SJohn Baldwin cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0]; 204bbb2f537SJohn Baldwin cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8]; 205bbb2f537SJohn Baldwin 206bbb2f537SJohn Baldwin options = F_TCAM_BYPASS | V_ULP_MODE(ULP_MODE_NONE); 207bbb2f537SJohn Baldwin options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan); 208bbb2f537SJohn Baldwin options |= F_NON_OFFLOAD; 209bbb2f537SJohn Baldwin cpl->opt0 = htobe64(options); 210bbb2f537SJohn Baldwin 211bbb2f537SJohn Baldwin options = V_TX_QUEUE(sc->params.tp.tx_modq[vi->pi->tx_chan]); 212bbb2f537SJohn Baldwin if (tp->t_flags & TF_REQ_TSTMP) 213bbb2f537SJohn Baldwin options |= F_TSTAMPS_EN; 214bbb2f537SJohn Baldwin cpl->opt2 = htobe32(options); 215bbb2f537SJohn Baldwin } 216bbb2f537SJohn Baldwin 217bbb2f537SJohn Baldwin static int 218bbb2f537SJohn Baldwin send_ktls_act_open_req(struct adapter *sc, struct vi_info *vi, 219bbb2f537SJohn Baldwin struct inpcb *inp, struct tlspcb *tlsp, int atid) 220bbb2f537SJohn Baldwin { 221bbb2f537SJohn Baldwin struct wrqe *wr; 222bbb2f537SJohn Baldwin bool isipv6; 223bbb2f537SJohn Baldwin 224bbb2f537SJohn Baldwin isipv6 = (inp->inp_vflag & INP_IPV6) != 0; 225bbb2f537SJohn Baldwin if (isipv6) { 226bbb2f537SJohn Baldwin tlsp->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true); 227bbb2f537SJohn Baldwin if (tlsp->ce == NULL) 228bbb2f537SJohn Baldwin return (ENOENT); 229bbb2f537SJohn Baldwin } 230bbb2f537SJohn Baldwin 231bbb2f537SJohn Baldwin wr = alloc_wrqe(ktls_act_open_cpl_size(isipv6), tlsp->ctrlq); 232bbb2f537SJohn Baldwin if (wr == NULL) { 233bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: atid %d failed to alloc WR", __func__, 234bbb2f537SJohn Baldwin atid); 235bbb2f537SJohn Baldwin return (ENOMEM); 236bbb2f537SJohn Baldwin } 237bbb2f537SJohn Baldwin 238bbb2f537SJohn Baldwin if (isipv6) 239bbb2f537SJohn Baldwin mk_ktls_act_open_req6(sc, vi, inp, tlsp, atid, wrtod(wr)); 240bbb2f537SJohn Baldwin else 241bbb2f537SJohn Baldwin mk_ktls_act_open_req(sc, vi, inp, tlsp, atid, wrtod(wr)); 242bbb2f537SJohn Baldwin 243bbb2f537SJohn Baldwin tlsp->open_pending = true; 244bbb2f537SJohn Baldwin t4_wrq_tx(sc, wr); 245bbb2f537SJohn Baldwin return (0); 246bbb2f537SJohn Baldwin } 247bbb2f537SJohn Baldwin 248bbb2f537SJohn Baldwin static int 249bbb2f537SJohn Baldwin ktls_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss, 250bbb2f537SJohn Baldwin struct mbuf *m) 251bbb2f537SJohn Baldwin { 252bbb2f537SJohn Baldwin struct adapter *sc = iq->adapter; 253bbb2f537SJohn Baldwin const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1); 254bbb2f537SJohn Baldwin u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status))); 255bbb2f537SJohn Baldwin u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status)); 256bbb2f537SJohn Baldwin struct tlspcb *tlsp = lookup_atid(sc, atid); 257bbb2f537SJohn Baldwin struct inpcb *inp = tlsp->inp; 258bbb2f537SJohn Baldwin 259bbb2f537SJohn Baldwin CTR3(KTR_CXGBE, "%s: atid %d status %d", __func__, atid, status); 260bbb2f537SJohn Baldwin free_atid(sc, atid); 261bbb2f537SJohn Baldwin if (status == 0) 262bbb2f537SJohn Baldwin tlsp->tid = GET_TID(cpl); 263bbb2f537SJohn Baldwin 264bbb2f537SJohn Baldwin INP_WLOCK(inp); 265bbb2f537SJohn Baldwin tlsp->open_pending = false; 266bbb2f537SJohn Baldwin wakeup(tlsp); 267bbb2f537SJohn Baldwin INP_WUNLOCK(inp); 268bbb2f537SJohn Baldwin return (0); 269bbb2f537SJohn Baldwin } 270bbb2f537SJohn Baldwin 271bbb2f537SJohn Baldwin /* SET_TCB_FIELD sent as a ULP command looks like this */ 272bbb2f537SJohn Baldwin #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \ 273bbb2f537SJohn Baldwin sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core)) 274bbb2f537SJohn Baldwin 275bbb2f537SJohn Baldwin _Static_assert((LEN__SET_TCB_FIELD_ULP + sizeof(struct ulptx_idata)) % 16 == 0, 276bbb2f537SJohn Baldwin "CPL_SET_TCB_FIELD ULP command not 16-byte aligned"); 277bbb2f537SJohn Baldwin 278bbb2f537SJohn Baldwin static void 279bbb2f537SJohn Baldwin write_set_tcb_field_ulp(struct tlspcb *tlsp, void *dst, struct sge_txq *txq, 280bbb2f537SJohn Baldwin uint16_t word, uint64_t mask, uint64_t val) 281bbb2f537SJohn Baldwin { 282bbb2f537SJohn Baldwin struct ulp_txpkt *txpkt; 283bbb2f537SJohn Baldwin struct ulptx_idata *idata; 284bbb2f537SJohn Baldwin struct cpl_set_tcb_field_core *cpl; 285bbb2f537SJohn Baldwin 286bbb2f537SJohn Baldwin /* ULP_TXPKT */ 287bbb2f537SJohn Baldwin txpkt = dst; 288bbb2f537SJohn Baldwin txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | 289bbb2f537SJohn Baldwin V_ULP_TXPKT_DATAMODIFY(0) | 290bbb2f537SJohn Baldwin V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) | 291bbb2f537SJohn Baldwin V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1)); 292bbb2f537SJohn Baldwin txpkt->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16)); 293bbb2f537SJohn Baldwin 294bbb2f537SJohn Baldwin /* ULPTX_IDATA sub-command */ 295bbb2f537SJohn Baldwin idata = (struct ulptx_idata *)(txpkt + 1); 296bbb2f537SJohn Baldwin idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 297bbb2f537SJohn Baldwin idata->len = htobe32(sizeof(*cpl)); 298bbb2f537SJohn Baldwin 299bbb2f537SJohn Baldwin /* CPL_SET_TCB_FIELD */ 300bbb2f537SJohn Baldwin cpl = (struct cpl_set_tcb_field_core *)(idata + 1); 301bbb2f537SJohn Baldwin OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tlsp->tid)); 302bbb2f537SJohn Baldwin cpl->reply_ctrl = htobe16(F_NO_REPLY); 303bbb2f537SJohn Baldwin cpl->word_cookie = htobe16(V_WORD(word)); 304bbb2f537SJohn Baldwin cpl->mask = htobe64(mask); 305bbb2f537SJohn Baldwin cpl->val = htobe64(val); 306bbb2f537SJohn Baldwin 307bbb2f537SJohn Baldwin /* ULPTX_NOOP */ 308bbb2f537SJohn Baldwin idata = (struct ulptx_idata *)(cpl + 1); 309bbb2f537SJohn Baldwin idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 310bbb2f537SJohn Baldwin idata->len = htobe32(0); 311bbb2f537SJohn Baldwin } 312bbb2f537SJohn Baldwin 313bbb2f537SJohn Baldwin static int 314bbb2f537SJohn Baldwin ktls_set_tcb_fields(struct tlspcb *tlsp, struct tcpcb *tp, struct sge_txq *txq) 315bbb2f537SJohn Baldwin { 316bbb2f537SJohn Baldwin struct fw_ulptx_wr *wr; 317bbb2f537SJohn Baldwin struct mbuf *m; 318bbb2f537SJohn Baldwin char *dst; 319bbb2f537SJohn Baldwin void *items[1]; 320bbb2f537SJohn Baldwin int error, len; 321bbb2f537SJohn Baldwin 322bbb2f537SJohn Baldwin len = sizeof(*wr) + 3 * roundup2(LEN__SET_TCB_FIELD_ULP, 16); 323bbb2f537SJohn Baldwin if (tp->t_flags & TF_REQ_TSTMP) 324bbb2f537SJohn Baldwin len += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 325bbb2f537SJohn Baldwin m = alloc_wr_mbuf(len, M_NOWAIT); 326bbb2f537SJohn Baldwin if (m == NULL) { 327bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__, 328bbb2f537SJohn Baldwin tlsp->tid); 329bbb2f537SJohn Baldwin return (ENOMEM); 330bbb2f537SJohn Baldwin } 331bbb2f537SJohn Baldwin m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com); 332bbb2f537SJohn Baldwin m->m_pkthdr.csum_flags |= CSUM_SND_TAG; 333bbb2f537SJohn Baldwin 334bbb2f537SJohn Baldwin /* FW_ULPTX_WR */ 335bbb2f537SJohn Baldwin wr = mtod(m, void *); 336bbb2f537SJohn Baldwin wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); 337bbb2f537SJohn Baldwin wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | 338bbb2f537SJohn Baldwin V_FW_WR_LEN16(len / 16)); 339bbb2f537SJohn Baldwin wr->cookie = 0; 340bbb2f537SJohn Baldwin dst = (char *)(wr + 1); 341bbb2f537SJohn Baldwin 342bbb2f537SJohn Baldwin /* Clear TF_NON_OFFLOAD and set TF_CORE_BYPASS */ 343bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_T_FLAGS, 344bbb2f537SJohn Baldwin V_TCB_T_FLAGS(V_TF_CORE_BYPASS(1) | V_TF_NON_OFFLOAD(1)), 345bbb2f537SJohn Baldwin V_TCB_T_FLAGS(V_TF_CORE_BYPASS(1))); 346bbb2f537SJohn Baldwin dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 347bbb2f537SJohn Baldwin 348bbb2f537SJohn Baldwin /* Clear the SND_UNA_RAW, SND_NXT_RAW, and SND_MAX_RAW offsets. */ 349bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_SND_UNA_RAW, 350bbb2f537SJohn Baldwin V_TCB_SND_NXT_RAW(M_TCB_SND_NXT_RAW) | 351bbb2f537SJohn Baldwin V_TCB_SND_UNA_RAW(M_TCB_SND_UNA_RAW), 352bbb2f537SJohn Baldwin V_TCB_SND_NXT_RAW(0) | V_TCB_SND_UNA_RAW(0)); 353bbb2f537SJohn Baldwin dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 354bbb2f537SJohn Baldwin 355bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_SND_MAX_RAW, 356bbb2f537SJohn Baldwin V_TCB_SND_MAX_RAW(M_TCB_SND_MAX_RAW), V_TCB_SND_MAX_RAW(0)); 357bbb2f537SJohn Baldwin dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 358bbb2f537SJohn Baldwin 359bbb2f537SJohn Baldwin if (tp->t_flags & TF_REQ_TSTMP) { 360bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_TIMESTAMP_OFFSET, 361bbb2f537SJohn Baldwin V_TCB_TIMESTAMP_OFFSET(M_TCB_TIMESTAMP_OFFSET), 362bbb2f537SJohn Baldwin V_TCB_TIMESTAMP_OFFSET(tp->ts_offset >> 28)); 363bbb2f537SJohn Baldwin dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 364bbb2f537SJohn Baldwin } 365bbb2f537SJohn Baldwin 366bbb2f537SJohn Baldwin KASSERT(dst - (char *)wr == len, ("%s: length mismatch", __func__)); 367bbb2f537SJohn Baldwin 368bbb2f537SJohn Baldwin items[0] = m; 369bbb2f537SJohn Baldwin error = mp_ring_enqueue(txq->r, items, 1, 1); 370bbb2f537SJohn Baldwin if (error) 371bbb2f537SJohn Baldwin m_free(m); 372bbb2f537SJohn Baldwin return (error); 373bbb2f537SJohn Baldwin } 374bbb2f537SJohn Baldwin 375bbb2f537SJohn Baldwin int 376bbb2f537SJohn Baldwin t6_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, 377bbb2f537SJohn Baldwin struct m_snd_tag **pt) 378bbb2f537SJohn Baldwin { 379bbb2f537SJohn Baldwin const struct ktls_session *tls; 380bbb2f537SJohn Baldwin struct tlspcb *tlsp; 381bbb2f537SJohn Baldwin struct adapter *sc; 382bbb2f537SJohn Baldwin struct vi_info *vi; 383bbb2f537SJohn Baldwin struct inpcb *inp; 384bbb2f537SJohn Baldwin struct tcpcb *tp; 385bbb2f537SJohn Baldwin struct sge_txq *txq; 386bbb2f537SJohn Baldwin int atid, error, explicit_iv_size, keyid, mac_first; 387bbb2f537SJohn Baldwin 388bbb2f537SJohn Baldwin tls = params->tls.tls; 389bbb2f537SJohn Baldwin 390bbb2f537SJohn Baldwin /* Only TLS 1.1 and TLS 1.2 are currently supported. */ 391bbb2f537SJohn Baldwin if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE || 392bbb2f537SJohn Baldwin tls->params.tls_vminor < TLS_MINOR_VER_ONE || 393bbb2f537SJohn Baldwin tls->params.tls_vminor > TLS_MINOR_VER_TWO) 394bbb2f537SJohn Baldwin return (EPROTONOSUPPORT); 395bbb2f537SJohn Baldwin 396bbb2f537SJohn Baldwin /* Sanity check values in *tls. */ 397bbb2f537SJohn Baldwin switch (tls->params.cipher_algorithm) { 398bbb2f537SJohn Baldwin case CRYPTO_AES_CBC: 399bbb2f537SJohn Baldwin /* XXX: Explicitly ignore any provided IV. */ 400bbb2f537SJohn Baldwin switch (tls->params.cipher_key_len) { 401bbb2f537SJohn Baldwin case 128 / 8: 402bbb2f537SJohn Baldwin case 192 / 8: 403bbb2f537SJohn Baldwin case 256 / 8: 404bbb2f537SJohn Baldwin break; 405bbb2f537SJohn Baldwin default: 406bbb2f537SJohn Baldwin return (EINVAL); 407bbb2f537SJohn Baldwin } 408bbb2f537SJohn Baldwin switch (tls->params.auth_algorithm) { 409bbb2f537SJohn Baldwin case CRYPTO_SHA1_HMAC: 410bbb2f537SJohn Baldwin case CRYPTO_SHA2_256_HMAC: 411bbb2f537SJohn Baldwin case CRYPTO_SHA2_384_HMAC: 412bbb2f537SJohn Baldwin break; 413bbb2f537SJohn Baldwin default: 414bbb2f537SJohn Baldwin return (EPROTONOSUPPORT); 415bbb2f537SJohn Baldwin } 416bbb2f537SJohn Baldwin explicit_iv_size = AES_BLOCK_LEN; 417bbb2f537SJohn Baldwin mac_first = 1; 418bbb2f537SJohn Baldwin break; 419bbb2f537SJohn Baldwin case CRYPTO_AES_NIST_GCM_16: 420bbb2f537SJohn Baldwin if (tls->params.iv_len != SALT_SIZE) 421bbb2f537SJohn Baldwin return (EINVAL); 422bbb2f537SJohn Baldwin switch (tls->params.cipher_key_len) { 423bbb2f537SJohn Baldwin case 128 / 8: 424bbb2f537SJohn Baldwin case 192 / 8: 425bbb2f537SJohn Baldwin case 256 / 8: 426bbb2f537SJohn Baldwin break; 427bbb2f537SJohn Baldwin default: 428bbb2f537SJohn Baldwin return (EINVAL); 429bbb2f537SJohn Baldwin } 430bbb2f537SJohn Baldwin explicit_iv_size = 8; 431bbb2f537SJohn Baldwin mac_first = 0; 432bbb2f537SJohn Baldwin break; 433bbb2f537SJohn Baldwin default: 434bbb2f537SJohn Baldwin return (EPROTONOSUPPORT); 435bbb2f537SJohn Baldwin } 436bbb2f537SJohn Baldwin 437bbb2f537SJohn Baldwin vi = ifp->if_softc; 438bbb2f537SJohn Baldwin sc = vi->adapter; 439bbb2f537SJohn Baldwin 440bbb2f537SJohn Baldwin tlsp = alloc_tlspcb(ifp, vi, M_WAITOK); 441bbb2f537SJohn Baldwin 442bbb2f537SJohn Baldwin atid = alloc_atid(sc, tlsp); 443bbb2f537SJohn Baldwin if (atid < 0) { 444bbb2f537SJohn Baldwin error = ENOMEM; 445bbb2f537SJohn Baldwin goto failed; 446bbb2f537SJohn Baldwin } 447bbb2f537SJohn Baldwin 448bbb2f537SJohn Baldwin if (sc->tlst.inline_keys) 449bbb2f537SJohn Baldwin keyid = -1; 450bbb2f537SJohn Baldwin else 451bbb2f537SJohn Baldwin keyid = t4_alloc_tls_keyid(sc); 452bbb2f537SJohn Baldwin if (keyid < 0) { 453bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: atid %d using immediate key ctx", __func__, 454bbb2f537SJohn Baldwin atid); 455bbb2f537SJohn Baldwin tlsp->inline_key = true; 456bbb2f537SJohn Baldwin } else { 457bbb2f537SJohn Baldwin tlsp->tx_key_addr = keyid; 458bbb2f537SJohn Baldwin CTR3(KTR_CXGBE, "%s: atid %d allocated TX key addr %#x", 459bbb2f537SJohn Baldwin __func__, 460bbb2f537SJohn Baldwin atid, tlsp->tx_key_addr); 461bbb2f537SJohn Baldwin } 462bbb2f537SJohn Baldwin 463bbb2f537SJohn Baldwin inp = params->tls.inp; 464bbb2f537SJohn Baldwin INP_RLOCK(inp); 46553af6903SGleb Smirnoff if (inp->inp_flags & INP_DROPPED) { 466bbb2f537SJohn Baldwin INP_RUNLOCK(inp); 467bbb2f537SJohn Baldwin error = ECONNRESET; 468bbb2f537SJohn Baldwin goto failed; 469bbb2f537SJohn Baldwin } 470bbb2f537SJohn Baldwin tlsp->inp = inp; 471bbb2f537SJohn Baldwin 472e3966125SJohn Baldwin tp = intotcpcb(inp); 473bbb2f537SJohn Baldwin if (tp->t_flags & TF_REQ_TSTMP) { 474bbb2f537SJohn Baldwin tlsp->using_timestamps = true; 475bbb2f537SJohn Baldwin if ((tp->ts_offset & 0xfffffff) != 0) { 476bbb2f537SJohn Baldwin INP_RUNLOCK(inp); 477bbb2f537SJohn Baldwin error = EINVAL; 478bbb2f537SJohn Baldwin goto failed; 479bbb2f537SJohn Baldwin } 480bbb2f537SJohn Baldwin } else 481bbb2f537SJohn Baldwin tlsp->using_timestamps = false; 482bbb2f537SJohn Baldwin 483bbb2f537SJohn Baldwin error = send_ktls_act_open_req(sc, vi, inp, tlsp, atid); 484bbb2f537SJohn Baldwin if (error) { 485bbb2f537SJohn Baldwin INP_RUNLOCK(inp); 486bbb2f537SJohn Baldwin goto failed; 487bbb2f537SJohn Baldwin } 488bbb2f537SJohn Baldwin 489bbb2f537SJohn Baldwin /* Wait for reply to active open. */ 490bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: atid %d sent CPL_ACT_OPEN_REQ", __func__, 491bbb2f537SJohn Baldwin atid); 492bbb2f537SJohn Baldwin while (tlsp->open_pending) { 493bbb2f537SJohn Baldwin /* 494bbb2f537SJohn Baldwin * XXX: PCATCH? We would then have to discard the PCB 495bbb2f537SJohn Baldwin * when the completion CPL arrived. 496bbb2f537SJohn Baldwin */ 497bbb2f537SJohn Baldwin error = rw_sleep(tlsp, &inp->inp_lock, 0, "t6tlsop", 0); 498bbb2f537SJohn Baldwin } 499bbb2f537SJohn Baldwin 500bbb2f537SJohn Baldwin atid = -1; 501bbb2f537SJohn Baldwin if (tlsp->tid < 0) { 502bbb2f537SJohn Baldwin INP_RUNLOCK(inp); 503bbb2f537SJohn Baldwin error = ENOMEM; 504bbb2f537SJohn Baldwin goto failed; 505bbb2f537SJohn Baldwin } 506bbb2f537SJohn Baldwin 50753af6903SGleb Smirnoff if (inp->inp_flags & INP_DROPPED) { 508bbb2f537SJohn Baldwin INP_RUNLOCK(inp); 509bbb2f537SJohn Baldwin error = ECONNRESET; 510bbb2f537SJohn Baldwin goto failed; 511bbb2f537SJohn Baldwin } 512bbb2f537SJohn Baldwin 513bbb2f537SJohn Baldwin txq = &sc->sge.txq[vi->first_txq]; 514bbb2f537SJohn Baldwin if (inp->inp_flowtype != M_HASHTYPE_NONE) 515bbb2f537SJohn Baldwin txq += ((inp->inp_flowid % (vi->ntxq - vi->rsrv_noflowq)) + 516bbb2f537SJohn Baldwin vi->rsrv_noflowq); 517bbb2f537SJohn Baldwin tlsp->txq = txq; 518bbb2f537SJohn Baldwin 519bbb2f537SJohn Baldwin error = ktls_set_tcb_fields(tlsp, tp, txq); 520bbb2f537SJohn Baldwin INP_RUNLOCK(inp); 521bbb2f537SJohn Baldwin if (error) 522bbb2f537SJohn Baldwin goto failed; 523bbb2f537SJohn Baldwin 524bbb2f537SJohn Baldwin error = ktls_setup_keys(tlsp, tls, txq); 525bbb2f537SJohn Baldwin if (error) 526bbb2f537SJohn Baldwin goto failed; 527bbb2f537SJohn Baldwin 528bbb2f537SJohn Baldwin tlsp->enc_mode = t4_tls_cipher_mode(tls); 529bbb2f537SJohn Baldwin tlsp->tx_key_info_size = t4_tls_key_info_size(tls); 530bbb2f537SJohn Baldwin 531bbb2f537SJohn Baldwin /* The SCMD fields used when encrypting a full TLS record. */ 532bbb2f537SJohn Baldwin tlsp->scmd0.seqno_numivs = htobe32(V_SCMD_SEQ_NO_CTRL(3) | 533bbb2f537SJohn Baldwin V_SCMD_PROTO_VERSION(t4_tls_proto_ver(tls)) | 534bbb2f537SJohn Baldwin V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 535bbb2f537SJohn Baldwin V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) | 536bbb2f537SJohn Baldwin V_SCMD_CIPH_MODE(tlsp->enc_mode) | 537bbb2f537SJohn Baldwin V_SCMD_AUTH_MODE(t4_tls_auth_mode(tls)) | 538bbb2f537SJohn Baldwin V_SCMD_HMAC_CTRL(t4_tls_hmac_ctrl(tls)) | 539bbb2f537SJohn Baldwin V_SCMD_IV_SIZE(explicit_iv_size / 2) | V_SCMD_NUM_IVS(1)); 540bbb2f537SJohn Baldwin 541bbb2f537SJohn Baldwin tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | 542bbb2f537SJohn Baldwin V_SCMD_TLS_FRAG_ENABLE(0); 543bbb2f537SJohn Baldwin if (tlsp->inline_key) 544bbb2f537SJohn Baldwin tlsp->scmd0.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1); 545bbb2f537SJohn Baldwin tlsp->scmd0.ivgen_hdrlen = htobe32(tlsp->scmd0.ivgen_hdrlen); 546bbb2f537SJohn Baldwin 547bbb2f537SJohn Baldwin /* 548bbb2f537SJohn Baldwin * The SCMD fields used when encrypting a partial TLS record 549bbb2f537SJohn Baldwin * (no trailer and possibly a truncated payload). 550bbb2f537SJohn Baldwin */ 551bbb2f537SJohn Baldwin tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) | 552bbb2f537SJohn Baldwin V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) | 553bbb2f537SJohn Baldwin V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) | 554bbb2f537SJohn Baldwin V_SCMD_CIPH_AUTH_SEQ_CTRL((mac_first == 0)) | 555bbb2f537SJohn Baldwin V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) | 556bbb2f537SJohn Baldwin V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) | 557bbb2f537SJohn Baldwin V_SCMD_IV_SIZE(AES_BLOCK_LEN / 2) | V_SCMD_NUM_IVS(0); 558bbb2f537SJohn Baldwin if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) 559bbb2f537SJohn Baldwin tlsp->scmd0_short.seqno_numivs |= 560bbb2f537SJohn Baldwin V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CTR); 561bbb2f537SJohn Baldwin else 562bbb2f537SJohn Baldwin tlsp->scmd0_short.seqno_numivs |= 563bbb2f537SJohn Baldwin V_SCMD_CIPH_MODE(tlsp->enc_mode); 564bbb2f537SJohn Baldwin tlsp->scmd0_short.seqno_numivs = 565bbb2f537SJohn Baldwin htobe32(tlsp->scmd0_short.seqno_numivs); 566bbb2f537SJohn Baldwin 567bbb2f537SJohn Baldwin tlsp->scmd0_short.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) | 568bbb2f537SJohn Baldwin V_SCMD_TLS_FRAG_ENABLE(0) | 569bbb2f537SJohn Baldwin V_SCMD_AADIVDROP(1); 570bbb2f537SJohn Baldwin if (tlsp->inline_key) 571bbb2f537SJohn Baldwin tlsp->scmd0_short.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1); 572bbb2f537SJohn Baldwin 573bbb2f537SJohn Baldwin TXQ_LOCK(txq); 574bbb2f537SJohn Baldwin if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) 575bbb2f537SJohn Baldwin txq->kern_tls_gcm++; 576bbb2f537SJohn Baldwin else 577bbb2f537SJohn Baldwin txq->kern_tls_cbc++; 578bbb2f537SJohn Baldwin TXQ_UNLOCK(txq); 579bbb2f537SJohn Baldwin *pt = &tlsp->com; 580bbb2f537SJohn Baldwin return (0); 581bbb2f537SJohn Baldwin 582bbb2f537SJohn Baldwin failed: 583bbb2f537SJohn Baldwin if (atid >= 0) 584bbb2f537SJohn Baldwin free_atid(sc, atid); 585bbb2f537SJohn Baldwin m_snd_tag_rele(&tlsp->com); 586bbb2f537SJohn Baldwin return (error); 587bbb2f537SJohn Baldwin } 588bbb2f537SJohn Baldwin 589bbb2f537SJohn Baldwin static int 590bbb2f537SJohn Baldwin ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls, 591bbb2f537SJohn Baldwin struct sge_txq *txq) 592bbb2f537SJohn Baldwin { 593bbb2f537SJohn Baldwin struct tls_key_req *kwr; 594bbb2f537SJohn Baldwin struct tls_keyctx *kctx; 595bbb2f537SJohn Baldwin void *items[1]; 596bbb2f537SJohn Baldwin struct mbuf *m; 597bbb2f537SJohn Baldwin int error; 598bbb2f537SJohn Baldwin 599bbb2f537SJohn Baldwin /* 600bbb2f537SJohn Baldwin * Store the salt and keys in the key context. For 601bbb2f537SJohn Baldwin * connections with an inline key, this key context is passed 602bbb2f537SJohn Baldwin * as immediate data in each work request. For connections 603bbb2f537SJohn Baldwin * storing the key in DDR, a work request is used to store a 604bbb2f537SJohn Baldwin * copy of the key context in DDR. 605bbb2f537SJohn Baldwin */ 606bbb2f537SJohn Baldwin t4_tls_key_ctx(tls, KTLS_TX, &tlsp->keyctx); 607bbb2f537SJohn Baldwin if (tlsp->inline_key) 608bbb2f537SJohn Baldwin return (0); 609bbb2f537SJohn Baldwin 610bbb2f537SJohn Baldwin /* Populate key work request. */ 611bbb2f537SJohn Baldwin m = alloc_wr_mbuf(TLS_KEY_WR_SZ, M_NOWAIT); 612bbb2f537SJohn Baldwin if (m == NULL) { 613bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__, 614bbb2f537SJohn Baldwin tlsp->tid); 615bbb2f537SJohn Baldwin return (ENOMEM); 616bbb2f537SJohn Baldwin } 617bbb2f537SJohn Baldwin m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com); 618bbb2f537SJohn Baldwin m->m_pkthdr.csum_flags |= CSUM_SND_TAG; 619bbb2f537SJohn Baldwin kwr = mtod(m, void *); 620bbb2f537SJohn Baldwin memset(kwr, 0, TLS_KEY_WR_SZ); 621bbb2f537SJohn Baldwin 622bbb2f537SJohn Baldwin t4_write_tlskey_wr(tls, KTLS_TX, tlsp->tid, 0, tlsp->tx_key_addr, kwr); 623bbb2f537SJohn Baldwin kctx = (struct tls_keyctx *)(kwr + 1); 624bbb2f537SJohn Baldwin memcpy(kctx, &tlsp->keyctx, sizeof(*kctx)); 625bbb2f537SJohn Baldwin 626bbb2f537SJohn Baldwin /* 627bbb2f537SJohn Baldwin * Place the key work request in the transmit queue. It 628bbb2f537SJohn Baldwin * should be sent to the NIC before any TLS packets using this 629bbb2f537SJohn Baldwin * session. 630bbb2f537SJohn Baldwin */ 631bbb2f537SJohn Baldwin items[0] = m; 632bbb2f537SJohn Baldwin error = mp_ring_enqueue(txq->r, items, 1, 1); 633bbb2f537SJohn Baldwin if (error) 634bbb2f537SJohn Baldwin m_free(m); 635bbb2f537SJohn Baldwin else 636bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d sent key WR", __func__, tlsp->tid); 637bbb2f537SJohn Baldwin return (error); 638bbb2f537SJohn Baldwin } 639bbb2f537SJohn Baldwin 640bbb2f537SJohn Baldwin static u_int 641bbb2f537SJohn Baldwin ktls_base_wr_size(struct tlspcb *tlsp) 642bbb2f537SJohn Baldwin { 643bbb2f537SJohn Baldwin u_int wr_len; 644bbb2f537SJohn Baldwin 645bbb2f537SJohn Baldwin wr_len = sizeof(struct fw_ulptx_wr); // 16 646bbb2f537SJohn Baldwin wr_len += sizeof(struct ulp_txpkt); // 8 647bbb2f537SJohn Baldwin wr_len += sizeof(struct ulptx_idata); // 8 648bbb2f537SJohn Baldwin wr_len += sizeof(struct cpl_tx_sec_pdu);// 32 649bbb2f537SJohn Baldwin if (tlsp->inline_key) 650bbb2f537SJohn Baldwin wr_len += tlsp->tx_key_info_size; 651bbb2f537SJohn Baldwin else { 652bbb2f537SJohn Baldwin wr_len += sizeof(struct ulptx_sc_memrd);// 8 653bbb2f537SJohn Baldwin wr_len += sizeof(struct ulptx_idata); // 8 654bbb2f537SJohn Baldwin } 655bbb2f537SJohn Baldwin wr_len += sizeof(struct cpl_tx_data); // 16 656bbb2f537SJohn Baldwin return (wr_len); 657bbb2f537SJohn Baldwin } 658bbb2f537SJohn Baldwin 659bbb2f537SJohn Baldwin /* How many bytes of TCP payload to send for a given TLS record. */ 660bbb2f537SJohn Baldwin static u_int 661bbb2f537SJohn Baldwin ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls) 662bbb2f537SJohn Baldwin { 663bbb2f537SJohn Baldwin struct tls_record_layer *hdr; 664bbb2f537SJohn Baldwin u_int plen, mlen; 665bbb2f537SJohn Baldwin 666bbb2f537SJohn Baldwin M_ASSERTEXTPG(m_tls); 667bbb2f537SJohn Baldwin hdr = (void *)m_tls->m_epg_hdr; 668bbb2f537SJohn Baldwin plen = ntohs(hdr->tls_length); 669bbb2f537SJohn Baldwin 670bbb2f537SJohn Baldwin /* 671bbb2f537SJohn Baldwin * What range of the TLS record is the mbuf requesting to be 672bbb2f537SJohn Baldwin * sent. 673bbb2f537SJohn Baldwin */ 674bbb2f537SJohn Baldwin mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len; 675bbb2f537SJohn Baldwin 676bbb2f537SJohn Baldwin /* Always send complete records. */ 677bbb2f537SJohn Baldwin if (mlen == TLS_HEADER_LENGTH + plen) 678bbb2f537SJohn Baldwin return (mlen); 679bbb2f537SJohn Baldwin 680bbb2f537SJohn Baldwin /* 681bbb2f537SJohn Baldwin * If the host stack has asked to send part of the trailer, 682bbb2f537SJohn Baldwin * trim the length to avoid sending any of the trailer. There 683bbb2f537SJohn Baldwin * is no way to send a partial trailer currently. 684bbb2f537SJohn Baldwin */ 685bbb2f537SJohn Baldwin if (mlen > TLS_HEADER_LENGTH + plen - m_tls->m_epg_trllen) 686bbb2f537SJohn Baldwin mlen = TLS_HEADER_LENGTH + plen - m_tls->m_epg_trllen; 687bbb2f537SJohn Baldwin 688bbb2f537SJohn Baldwin 689bbb2f537SJohn Baldwin /* 690bbb2f537SJohn Baldwin * For AES-CBC adjust the ciphertext length for the block 691bbb2f537SJohn Baldwin * size. 692bbb2f537SJohn Baldwin */ 693bbb2f537SJohn Baldwin if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC && 694bbb2f537SJohn Baldwin mlen > TLS_HEADER_LENGTH) { 695bbb2f537SJohn Baldwin mlen = TLS_HEADER_LENGTH + rounddown(mlen - TLS_HEADER_LENGTH, 696bbb2f537SJohn Baldwin AES_BLOCK_LEN); 697bbb2f537SJohn Baldwin } 698bbb2f537SJohn Baldwin 699bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 700bbb2f537SJohn Baldwin CTR4(KTR_CXGBE, "%s: tid %d short TLS record (%u vs %u)", 701bbb2f537SJohn Baldwin __func__, tlsp->tid, mlen, TLS_HEADER_LENGTH + plen); 702bbb2f537SJohn Baldwin #endif 703bbb2f537SJohn Baldwin return (mlen); 704bbb2f537SJohn Baldwin } 705bbb2f537SJohn Baldwin 706bbb2f537SJohn Baldwin /* 707bbb2f537SJohn Baldwin * For a "short" TLS record, determine the offset into the TLS record 708bbb2f537SJohn Baldwin * payload to send. This offset does not include the TLS header, but 709bbb2f537SJohn Baldwin * a non-zero offset implies that a header will not be sent. 710bbb2f537SJohn Baldwin */ 711bbb2f537SJohn Baldwin static u_int 712bbb2f537SJohn Baldwin ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls) 713bbb2f537SJohn Baldwin { 714bbb2f537SJohn Baldwin struct tls_record_layer *hdr; 715bbb2f537SJohn Baldwin u_int offset, plen; 716bbb2f537SJohn Baldwin #ifdef INVARIANTS 717bbb2f537SJohn Baldwin u_int mlen; 718bbb2f537SJohn Baldwin #endif 719bbb2f537SJohn Baldwin 720bbb2f537SJohn Baldwin M_ASSERTEXTPG(m_tls); 721bbb2f537SJohn Baldwin hdr = (void *)m_tls->m_epg_hdr; 722bbb2f537SJohn Baldwin plen = ntohs(hdr->tls_length); 723bbb2f537SJohn Baldwin #ifdef INVARIANTS 724bbb2f537SJohn Baldwin mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len; 725bbb2f537SJohn Baldwin MPASS(mlen < TLS_HEADER_LENGTH + plen); 726bbb2f537SJohn Baldwin #endif 727bbb2f537SJohn Baldwin if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen) 728bbb2f537SJohn Baldwin return (0); 729bbb2f537SJohn Baldwin if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { 730bbb2f537SJohn Baldwin /* 731bbb2f537SJohn Baldwin * Always send something. This function is only called 732bbb2f537SJohn Baldwin * if we aren't sending the tag at all, but if the 733bbb2f537SJohn Baldwin * request starts in the tag then we are in an odd 734bbb2f537SJohn Baldwin * state where would effectively send nothing. Cap 735bbb2f537SJohn Baldwin * the offset at the last byte of the record payload 736bbb2f537SJohn Baldwin * to send the last cipher block. 737bbb2f537SJohn Baldwin */ 738bbb2f537SJohn Baldwin offset = min(mtod(m_tls, vm_offset_t) - m_tls->m_epg_hdrlen, 739bbb2f537SJohn Baldwin (plen - TLS_HEADER_LENGTH - m_tls->m_epg_trllen) - 1); 740bbb2f537SJohn Baldwin return (rounddown(offset, AES_BLOCK_LEN)); 741bbb2f537SJohn Baldwin } 742bbb2f537SJohn Baldwin return (0); 743bbb2f537SJohn Baldwin } 744bbb2f537SJohn Baldwin 745bbb2f537SJohn Baldwin static u_int 746bbb2f537SJohn Baldwin ktls_sgl_size(u_int nsegs) 747bbb2f537SJohn Baldwin { 748bbb2f537SJohn Baldwin u_int wr_len; 749bbb2f537SJohn Baldwin 750bbb2f537SJohn Baldwin /* First segment is part of ulptx_sgl. */ 751bbb2f537SJohn Baldwin nsegs--; 752bbb2f537SJohn Baldwin 753bbb2f537SJohn Baldwin wr_len = sizeof(struct ulptx_sgl); 754bbb2f537SJohn Baldwin wr_len += 8 * ((3 * nsegs) / 2 + (nsegs & 1)); 755bbb2f537SJohn Baldwin return (wr_len); 756bbb2f537SJohn Baldwin } 757bbb2f537SJohn Baldwin 758bbb2f537SJohn Baldwin static int 759bbb2f537SJohn Baldwin ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls, 760bbb2f537SJohn Baldwin int *nsegsp) 761bbb2f537SJohn Baldwin { 762bbb2f537SJohn Baldwin struct tls_record_layer *hdr; 763bbb2f537SJohn Baldwin u_int imm_len, offset, plen, wr_len, tlen; 764bbb2f537SJohn Baldwin 765bbb2f537SJohn Baldwin M_ASSERTEXTPG(m_tls); 766bbb2f537SJohn Baldwin 767bbb2f537SJohn Baldwin /* 768bbb2f537SJohn Baldwin * Determine the size of the TLS record payload to send 769bbb2f537SJohn Baldwin * excluding header and trailer. 770bbb2f537SJohn Baldwin */ 771bbb2f537SJohn Baldwin tlen = ktls_tcp_payload_length(tlsp, m_tls); 772bbb2f537SJohn Baldwin if (tlen <= m_tls->m_epg_hdrlen) { 773bbb2f537SJohn Baldwin /* 774bbb2f537SJohn Baldwin * For requests that only want to send the TLS header, 775bbb2f537SJohn Baldwin * send a tunnelled packet as immediate data. 776bbb2f537SJohn Baldwin */ 777bbb2f537SJohn Baldwin wr_len = sizeof(struct fw_eth_tx_pkt_wr) + 778bbb2f537SJohn Baldwin sizeof(struct cpl_tx_pkt_core) + 779bbb2f537SJohn Baldwin roundup2(m->m_len + m_tls->m_len, 16); 780bbb2f537SJohn Baldwin if (wr_len > SGE_MAX_WR_LEN) { 781bbb2f537SJohn Baldwin CTR3(KTR_CXGBE, 782bbb2f537SJohn Baldwin "%s: tid %d TLS header-only packet too long (len %d)", 783bbb2f537SJohn Baldwin __func__, tlsp->tid, m->m_len + m_tls->m_len); 784bbb2f537SJohn Baldwin } 785bbb2f537SJohn Baldwin 786bbb2f537SJohn Baldwin /* This should always be the last TLS record in a chain. */ 787bbb2f537SJohn Baldwin MPASS(m_tls->m_next == NULL); 788ca457729SJohn Baldwin *nsegsp = 0; 789bbb2f537SJohn Baldwin return (wr_len); 790bbb2f537SJohn Baldwin } 791bbb2f537SJohn Baldwin 792bbb2f537SJohn Baldwin hdr = (void *)m_tls->m_epg_hdr; 793bbb2f537SJohn Baldwin plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_epg_trllen; 794bbb2f537SJohn Baldwin if (tlen < plen) { 795bbb2f537SJohn Baldwin plen = tlen; 796bbb2f537SJohn Baldwin offset = ktls_payload_offset(tlsp, m_tls); 797bbb2f537SJohn Baldwin } else 798bbb2f537SJohn Baldwin offset = 0; 799bbb2f537SJohn Baldwin 800bbb2f537SJohn Baldwin /* Calculate the size of the work request. */ 801bbb2f537SJohn Baldwin wr_len = ktls_base_wr_size(tlsp); 802bbb2f537SJohn Baldwin 803bbb2f537SJohn Baldwin /* 804bbb2f537SJohn Baldwin * Full records and short records with an offset of 0 include 805bbb2f537SJohn Baldwin * the TLS header as immediate data. Short records include a 806bbb2f537SJohn Baldwin * raw AES IV as immediate data. 807bbb2f537SJohn Baldwin */ 808bbb2f537SJohn Baldwin imm_len = 0; 809bbb2f537SJohn Baldwin if (offset == 0) 810bbb2f537SJohn Baldwin imm_len += m_tls->m_epg_hdrlen; 811bbb2f537SJohn Baldwin if (plen == tlen) 812bbb2f537SJohn Baldwin imm_len += AES_BLOCK_LEN; 813bbb2f537SJohn Baldwin wr_len += roundup2(imm_len, 16); 814bbb2f537SJohn Baldwin 815bbb2f537SJohn Baldwin /* TLS record payload via DSGL. */ 816bbb2f537SJohn Baldwin *nsegsp = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + offset, 817bbb2f537SJohn Baldwin plen - (m_tls->m_epg_hdrlen + offset)); 818bbb2f537SJohn Baldwin wr_len += ktls_sgl_size(*nsegsp); 819bbb2f537SJohn Baldwin 820bbb2f537SJohn Baldwin wr_len = roundup2(wr_len, 16); 821bbb2f537SJohn Baldwin return (wr_len); 822bbb2f537SJohn Baldwin } 823bbb2f537SJohn Baldwin 824bbb2f537SJohn Baldwin /* 825bbb2f537SJohn Baldwin * See if we have any TCP options requiring a dedicated options-only 826bbb2f537SJohn Baldwin * packet. 827bbb2f537SJohn Baldwin */ 828bbb2f537SJohn Baldwin static int 829bbb2f537SJohn Baldwin ktls_has_tcp_options(struct tcphdr *tcp) 830bbb2f537SJohn Baldwin { 831bbb2f537SJohn Baldwin u_char *cp; 832bbb2f537SJohn Baldwin int cnt, opt, optlen; 833bbb2f537SJohn Baldwin 834bbb2f537SJohn Baldwin cp = (u_char *)(tcp + 1); 835bbb2f537SJohn Baldwin cnt = tcp->th_off * 4 - sizeof(struct tcphdr); 836bbb2f537SJohn Baldwin for (; cnt > 0; cnt -= optlen, cp += optlen) { 837bbb2f537SJohn Baldwin opt = cp[0]; 838bbb2f537SJohn Baldwin if (opt == TCPOPT_EOL) 839bbb2f537SJohn Baldwin break; 840bbb2f537SJohn Baldwin if (opt == TCPOPT_NOP) 841bbb2f537SJohn Baldwin optlen = 1; 842bbb2f537SJohn Baldwin else { 843bbb2f537SJohn Baldwin if (cnt < 2) 844bbb2f537SJohn Baldwin break; 845bbb2f537SJohn Baldwin optlen = cp[1]; 846bbb2f537SJohn Baldwin if (optlen < 2 || optlen > cnt) 847bbb2f537SJohn Baldwin break; 848bbb2f537SJohn Baldwin } 849bbb2f537SJohn Baldwin switch (opt) { 850bbb2f537SJohn Baldwin case TCPOPT_NOP: 851bbb2f537SJohn Baldwin case TCPOPT_TIMESTAMP: 852bbb2f537SJohn Baldwin break; 853bbb2f537SJohn Baldwin default: 854bbb2f537SJohn Baldwin return (1); 855bbb2f537SJohn Baldwin } 856bbb2f537SJohn Baldwin } 857bbb2f537SJohn Baldwin return (0); 858bbb2f537SJohn Baldwin } 859bbb2f537SJohn Baldwin 860bbb2f537SJohn Baldwin /* 861bbb2f537SJohn Baldwin * Find the TCP timestamp option. 862bbb2f537SJohn Baldwin */ 863bbb2f537SJohn Baldwin static void * 864bbb2f537SJohn Baldwin ktls_find_tcp_timestamps(struct tcphdr *tcp) 865bbb2f537SJohn Baldwin { 866bbb2f537SJohn Baldwin u_char *cp; 867bbb2f537SJohn Baldwin int cnt, opt, optlen; 868bbb2f537SJohn Baldwin 869bbb2f537SJohn Baldwin cp = (u_char *)(tcp + 1); 870bbb2f537SJohn Baldwin cnt = tcp->th_off * 4 - sizeof(struct tcphdr); 871bbb2f537SJohn Baldwin for (; cnt > 0; cnt -= optlen, cp += optlen) { 872bbb2f537SJohn Baldwin opt = cp[0]; 873bbb2f537SJohn Baldwin if (opt == TCPOPT_EOL) 874bbb2f537SJohn Baldwin break; 875bbb2f537SJohn Baldwin if (opt == TCPOPT_NOP) 876bbb2f537SJohn Baldwin optlen = 1; 877bbb2f537SJohn Baldwin else { 878bbb2f537SJohn Baldwin if (cnt < 2) 879bbb2f537SJohn Baldwin break; 880bbb2f537SJohn Baldwin optlen = cp[1]; 881bbb2f537SJohn Baldwin if (optlen < 2 || optlen > cnt) 882bbb2f537SJohn Baldwin break; 883bbb2f537SJohn Baldwin } 884bbb2f537SJohn Baldwin if (opt == TCPOPT_TIMESTAMP && optlen == TCPOLEN_TIMESTAMP) 885bbb2f537SJohn Baldwin return (cp + 2); 886bbb2f537SJohn Baldwin } 887bbb2f537SJohn Baldwin return (NULL); 888bbb2f537SJohn Baldwin } 889bbb2f537SJohn Baldwin 890bbb2f537SJohn Baldwin int 891ca457729SJohn Baldwin t6_ktls_parse_pkt(struct mbuf *m) 892bbb2f537SJohn Baldwin { 893bbb2f537SJohn Baldwin struct tlspcb *tlsp; 894bbb2f537SJohn Baldwin struct ether_header *eh; 895bbb2f537SJohn Baldwin struct ip *ip; 896bbb2f537SJohn Baldwin struct ip6_hdr *ip6; 897bbb2f537SJohn Baldwin struct tcphdr *tcp; 898bbb2f537SJohn Baldwin struct mbuf *m_tls; 899ca457729SJohn Baldwin void *items[1]; 900bbb2f537SJohn Baldwin int nsegs; 901bbb2f537SJohn Baldwin u_int wr_len, tot_len; 902bbb2f537SJohn Baldwin 903bbb2f537SJohn Baldwin /* 904bbb2f537SJohn Baldwin * Locate headers in initial mbuf. 905bbb2f537SJohn Baldwin * 906bbb2f537SJohn Baldwin * XXX: This assumes all of the headers are in the initial mbuf. 907bbb2f537SJohn Baldwin * Could perhaps use m_advance() like parse_pkt() if that turns 908bbb2f537SJohn Baldwin * out to not be true. 909bbb2f537SJohn Baldwin */ 910bbb2f537SJohn Baldwin M_ASSERTPKTHDR(m); 911bbb2f537SJohn Baldwin MPASS(m->m_pkthdr.snd_tag != NULL); 912bbb2f537SJohn Baldwin tlsp = mst_to_tls(m->m_pkthdr.snd_tag); 913bbb2f537SJohn Baldwin 914bbb2f537SJohn Baldwin if (m->m_len <= sizeof(*eh) + sizeof(*ip)) { 915bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d header mbuf too short", __func__, 916bbb2f537SJohn Baldwin tlsp->tid); 917bbb2f537SJohn Baldwin return (EINVAL); 918bbb2f537SJohn Baldwin } 919bbb2f537SJohn Baldwin eh = mtod(m, struct ether_header *); 920bbb2f537SJohn Baldwin if (ntohs(eh->ether_type) != ETHERTYPE_IP && 921bbb2f537SJohn Baldwin ntohs(eh->ether_type) != ETHERTYPE_IPV6) { 922bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d mbuf not ETHERTYPE_IP{,V6}", 923bbb2f537SJohn Baldwin __func__, tlsp->tid); 924bbb2f537SJohn Baldwin return (EINVAL); 925bbb2f537SJohn Baldwin } 926bbb2f537SJohn Baldwin m->m_pkthdr.l2hlen = sizeof(*eh); 927bbb2f537SJohn Baldwin 928bbb2f537SJohn Baldwin /* XXX: Reject unsupported IP options? */ 929bbb2f537SJohn Baldwin if (ntohs(eh->ether_type) == ETHERTYPE_IP) { 930bbb2f537SJohn Baldwin ip = (struct ip *)(eh + 1); 931bbb2f537SJohn Baldwin if (ip->ip_p != IPPROTO_TCP) { 932bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d mbuf not IPPROTO_TCP", 933bbb2f537SJohn Baldwin __func__, tlsp->tid); 934bbb2f537SJohn Baldwin return (EINVAL); 935bbb2f537SJohn Baldwin } 936bbb2f537SJohn Baldwin m->m_pkthdr.l3hlen = ip->ip_hl * 4; 937bbb2f537SJohn Baldwin } else { 938bbb2f537SJohn Baldwin ip6 = (struct ip6_hdr *)(eh + 1); 939bbb2f537SJohn Baldwin if (ip6->ip6_nxt != IPPROTO_TCP) { 940bbb2f537SJohn Baldwin CTR3(KTR_CXGBE, "%s: tid %d mbuf not IPPROTO_TCP (%u)", 941bbb2f537SJohn Baldwin __func__, tlsp->tid, ip6->ip6_nxt); 942bbb2f537SJohn Baldwin return (EINVAL); 943bbb2f537SJohn Baldwin } 944bbb2f537SJohn Baldwin m->m_pkthdr.l3hlen = sizeof(struct ip6_hdr); 945bbb2f537SJohn Baldwin } 946bbb2f537SJohn Baldwin if (m->m_len < m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + 947bbb2f537SJohn Baldwin sizeof(*tcp)) { 948bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d header mbuf too short (2)", 949bbb2f537SJohn Baldwin __func__, tlsp->tid); 950bbb2f537SJohn Baldwin return (EINVAL); 951bbb2f537SJohn Baldwin } 952bbb2f537SJohn Baldwin tcp = (struct tcphdr *)((char *)(eh + 1) + m->m_pkthdr.l3hlen); 953bbb2f537SJohn Baldwin m->m_pkthdr.l4hlen = tcp->th_off * 4; 954bbb2f537SJohn Baldwin 955bbb2f537SJohn Baldwin /* Bail if there is TCP payload before the TLS record. */ 956bbb2f537SJohn Baldwin if (m->m_len != m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + 957bbb2f537SJohn Baldwin m->m_pkthdr.l4hlen) { 958bbb2f537SJohn Baldwin CTR6(KTR_CXGBE, 959bbb2f537SJohn Baldwin "%s: tid %d header mbuf bad length (%d + %d + %d != %d)", 960bbb2f537SJohn Baldwin __func__, tlsp->tid, m->m_pkthdr.l2hlen, 961bbb2f537SJohn Baldwin m->m_pkthdr.l3hlen, m->m_pkthdr.l4hlen, m->m_len); 962bbb2f537SJohn Baldwin return (EINVAL); 963bbb2f537SJohn Baldwin } 964bbb2f537SJohn Baldwin 965bbb2f537SJohn Baldwin /* Assume all headers are in 'm' for now. */ 966bbb2f537SJohn Baldwin MPASS(m->m_next != NULL); 967bbb2f537SJohn Baldwin MPASS(m->m_next->m_flags & M_EXTPG); 968bbb2f537SJohn Baldwin 969bbb2f537SJohn Baldwin tot_len = 0; 970bbb2f537SJohn Baldwin 971bbb2f537SJohn Baldwin /* 972bbb2f537SJohn Baldwin * Each of the remaining mbufs in the chain should reference a 973bbb2f537SJohn Baldwin * TLS record. 974bbb2f537SJohn Baldwin */ 975bbb2f537SJohn Baldwin for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { 976bbb2f537SJohn Baldwin MPASS(m_tls->m_flags & M_EXTPG); 977bbb2f537SJohn Baldwin 978bbb2f537SJohn Baldwin wr_len = ktls_wr_len(tlsp, m, m_tls, &nsegs); 979bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 980bbb2f537SJohn Baldwin CTR4(KTR_CXGBE, "%s: tid %d wr_len %d nsegs %d", __func__, 981bbb2f537SJohn Baldwin tlsp->tid, wr_len, nsegs); 982bbb2f537SJohn Baldwin #endif 983bbb2f537SJohn Baldwin if (wr_len > SGE_MAX_WR_LEN || nsegs > TX_SGL_SEGS) 984bbb2f537SJohn Baldwin return (EFBIG); 985bbb2f537SJohn Baldwin tot_len += roundup2(wr_len, EQ_ESIZE); 986bbb2f537SJohn Baldwin 987bbb2f537SJohn Baldwin /* 988bbb2f537SJohn Baldwin * Store 'nsegs' for the first TLS record in the 989bbb2f537SJohn Baldwin * header mbuf's metadata. 990bbb2f537SJohn Baldwin */ 991ca457729SJohn Baldwin if (m_tls == m->m_next) 992ca457729SJohn Baldwin set_mbuf_nsegs(m, nsegs); 993bbb2f537SJohn Baldwin } 994bbb2f537SJohn Baldwin 995bbb2f537SJohn Baldwin MPASS(tot_len != 0); 996bbb2f537SJohn Baldwin 997bbb2f537SJohn Baldwin /* 998bbb2f537SJohn Baldwin * See if we have any TCP options or a FIN requiring a 999bbb2f537SJohn Baldwin * dedicated packet. 1000bbb2f537SJohn Baldwin */ 1001bbb2f537SJohn Baldwin if ((tcp->th_flags & TH_FIN) != 0 || ktls_has_tcp_options(tcp)) { 1002bbb2f537SJohn Baldwin wr_len = sizeof(struct fw_eth_tx_pkt_wr) + 1003bbb2f537SJohn Baldwin sizeof(struct cpl_tx_pkt_core) + roundup2(m->m_len, 16); 1004bbb2f537SJohn Baldwin if (wr_len > SGE_MAX_WR_LEN) { 1005bbb2f537SJohn Baldwin CTR3(KTR_CXGBE, 1006bbb2f537SJohn Baldwin "%s: tid %d options-only packet too long (len %d)", 1007bbb2f537SJohn Baldwin __func__, tlsp->tid, m->m_len); 1008bbb2f537SJohn Baldwin return (EINVAL); 1009bbb2f537SJohn Baldwin } 1010bbb2f537SJohn Baldwin tot_len += roundup2(wr_len, EQ_ESIZE); 1011bbb2f537SJohn Baldwin } 1012bbb2f537SJohn Baldwin 1013bbb2f537SJohn Baldwin /* Include room for a TP work request to program an L2T entry. */ 1014bbb2f537SJohn Baldwin tot_len += EQ_ESIZE; 1015bbb2f537SJohn Baldwin 1016bbb2f537SJohn Baldwin /* 1017bbb2f537SJohn Baldwin * Include room for a ULPTX work request including up to 5 1018bbb2f537SJohn Baldwin * CPL_SET_TCB_FIELD commands before the first TLS work 1019bbb2f537SJohn Baldwin * request. 1020bbb2f537SJohn Baldwin */ 1021bbb2f537SJohn Baldwin wr_len = sizeof(struct fw_ulptx_wr) + 1022bbb2f537SJohn Baldwin 5 * roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1023bbb2f537SJohn Baldwin 1024bbb2f537SJohn Baldwin /* 1025bbb2f537SJohn Baldwin * If timestamps are present, reserve 1 more command for 1026bbb2f537SJohn Baldwin * setting the echoed timestamp. 1027bbb2f537SJohn Baldwin */ 1028bbb2f537SJohn Baldwin if (tlsp->using_timestamps) 1029bbb2f537SJohn Baldwin wr_len += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1030bbb2f537SJohn Baldwin 1031bbb2f537SJohn Baldwin tot_len += roundup2(wr_len, EQ_ESIZE); 1032bbb2f537SJohn Baldwin 1033ca457729SJohn Baldwin set_mbuf_len16(m, tot_len / 16); 1034bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 1035bbb2f537SJohn Baldwin CTR4(KTR_CXGBE, "%s: tid %d len16 %d nsegs %d", __func__, 1036ca457729SJohn Baldwin tlsp->tid, mbuf_len16(m), mbuf_nsegs(m)); 1037bbb2f537SJohn Baldwin #endif 1038ca457729SJohn Baldwin items[0] = m; 1039ca457729SJohn Baldwin return (mp_ring_enqueue(tlsp->txq->r, items, 1, 256)); 1040bbb2f537SJohn Baldwin } 1041bbb2f537SJohn Baldwin 1042bbb2f537SJohn Baldwin /* 1043bbb2f537SJohn Baldwin * If the SGL ends on an address that is not 16 byte aligned, this function will 1044bbb2f537SJohn Baldwin * add a 0 filled flit at the end. 1045bbb2f537SJohn Baldwin */ 1046bbb2f537SJohn Baldwin static void 1047bbb2f537SJohn Baldwin write_gl_to_buf(struct sglist *gl, caddr_t to) 1048bbb2f537SJohn Baldwin { 1049bbb2f537SJohn Baldwin struct sglist_seg *seg; 1050bbb2f537SJohn Baldwin __be64 *flitp; 1051bbb2f537SJohn Baldwin struct ulptx_sgl *usgl; 1052bbb2f537SJohn Baldwin int i, nflits, nsegs; 1053bbb2f537SJohn Baldwin 1054bbb2f537SJohn Baldwin KASSERT(((uintptr_t)to & 0xf) == 0, 1055bbb2f537SJohn Baldwin ("%s: SGL must start at a 16 byte boundary: %p", __func__, to)); 1056bbb2f537SJohn Baldwin 1057bbb2f537SJohn Baldwin nsegs = gl->sg_nseg; 1058bbb2f537SJohn Baldwin MPASS(nsegs > 0); 1059bbb2f537SJohn Baldwin 1060bbb2f537SJohn Baldwin nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2; 1061bbb2f537SJohn Baldwin flitp = (__be64 *)to; 1062bbb2f537SJohn Baldwin seg = &gl->sg_segs[0]; 1063bbb2f537SJohn Baldwin usgl = (void *)flitp; 1064bbb2f537SJohn Baldwin 1065bbb2f537SJohn Baldwin usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | 1066bbb2f537SJohn Baldwin V_ULPTX_NSGE(nsegs)); 1067bbb2f537SJohn Baldwin usgl->len0 = htobe32(seg->ss_len); 1068bbb2f537SJohn Baldwin usgl->addr0 = htobe64(seg->ss_paddr); 1069bbb2f537SJohn Baldwin seg++; 1070bbb2f537SJohn Baldwin 1071bbb2f537SJohn Baldwin for (i = 0; i < nsegs - 1; i++, seg++) { 1072bbb2f537SJohn Baldwin usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len); 1073bbb2f537SJohn Baldwin usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr); 1074bbb2f537SJohn Baldwin } 1075bbb2f537SJohn Baldwin if (i & 1) 1076bbb2f537SJohn Baldwin usgl->sge[i / 2].len[1] = htobe32(0); 1077bbb2f537SJohn Baldwin flitp += nflits; 1078bbb2f537SJohn Baldwin 1079bbb2f537SJohn Baldwin if (nflits & 1) { 1080bbb2f537SJohn Baldwin MPASS(((uintptr_t)flitp) & 0xf); 1081bbb2f537SJohn Baldwin *flitp++ = 0; 1082bbb2f537SJohn Baldwin } 1083bbb2f537SJohn Baldwin 1084bbb2f537SJohn Baldwin MPASS((((uintptr_t)flitp) & 0xf) == 0); 1085bbb2f537SJohn Baldwin } 1086bbb2f537SJohn Baldwin 1087bbb2f537SJohn Baldwin static inline void 1088bbb2f537SJohn Baldwin copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 1089bbb2f537SJohn Baldwin { 1090bbb2f537SJohn Baldwin 1091bbb2f537SJohn Baldwin MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]); 1092bbb2f537SJohn Baldwin MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]); 1093bbb2f537SJohn Baldwin 1094bbb2f537SJohn Baldwin if (__predict_true((uintptr_t)(*to) + len <= 1095bbb2f537SJohn Baldwin (uintptr_t)&eq->desc[eq->sidx])) { 1096bbb2f537SJohn Baldwin bcopy(from, *to, len); 1097bbb2f537SJohn Baldwin (*to) += len; 1098bbb2f537SJohn Baldwin if ((uintptr_t)(*to) == (uintptr_t)&eq->desc[eq->sidx]) 1099bbb2f537SJohn Baldwin (*to) = (caddr_t)eq->desc; 1100bbb2f537SJohn Baldwin } else { 1101bbb2f537SJohn Baldwin int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to); 1102bbb2f537SJohn Baldwin 1103bbb2f537SJohn Baldwin bcopy(from, *to, portion); 1104bbb2f537SJohn Baldwin from += portion; 1105bbb2f537SJohn Baldwin portion = len - portion; /* remaining */ 1106bbb2f537SJohn Baldwin bcopy(from, (void *)eq->desc, portion); 1107bbb2f537SJohn Baldwin (*to) = (caddr_t)eq->desc + portion; 1108bbb2f537SJohn Baldwin } 1109bbb2f537SJohn Baldwin } 1110bbb2f537SJohn Baldwin 1111bbb2f537SJohn Baldwin static int 1112bbb2f537SJohn Baldwin ktls_write_tcp_options(struct sge_txq *txq, void *dst, struct mbuf *m, 1113bbb2f537SJohn Baldwin u_int available, u_int pidx) 1114bbb2f537SJohn Baldwin { 1115bbb2f537SJohn Baldwin struct tx_sdesc *txsd; 1116bbb2f537SJohn Baldwin struct fw_eth_tx_pkt_wr *wr; 1117bbb2f537SJohn Baldwin struct cpl_tx_pkt_core *cpl; 1118bbb2f537SJohn Baldwin uint32_t ctrl; 1119bbb2f537SJohn Baldwin uint64_t ctrl1; 1120bbb2f537SJohn Baldwin int len16, ndesc, pktlen; 1121bbb2f537SJohn Baldwin struct ether_header *eh; 1122bbb2f537SJohn Baldwin struct ip *ip, newip; 1123bbb2f537SJohn Baldwin struct ip6_hdr *ip6, newip6; 1124bbb2f537SJohn Baldwin struct tcphdr *tcp, newtcp; 1125bbb2f537SJohn Baldwin caddr_t out; 1126bbb2f537SJohn Baldwin 1127bbb2f537SJohn Baldwin TXQ_LOCK_ASSERT_OWNED(txq); 1128bbb2f537SJohn Baldwin M_ASSERTPKTHDR(m); 1129bbb2f537SJohn Baldwin 1130bbb2f537SJohn Baldwin wr = dst; 1131bbb2f537SJohn Baldwin pktlen = m->m_len; 1132bbb2f537SJohn Baldwin ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen; 1133bbb2f537SJohn Baldwin len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16); 1134bbb2f537SJohn Baldwin ndesc = tx_len16_to_desc(len16); 1135bbb2f537SJohn Baldwin MPASS(ndesc <= available); 1136bbb2f537SJohn Baldwin 1137bbb2f537SJohn Baldwin /* Firmware work request header */ 1138bbb2f537SJohn Baldwin wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 1139bbb2f537SJohn Baldwin V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 1140bbb2f537SJohn Baldwin 1141bbb2f537SJohn Baldwin ctrl = V_FW_WR_LEN16(len16); 1142bbb2f537SJohn Baldwin wr->equiq_to_len16 = htobe32(ctrl); 1143bbb2f537SJohn Baldwin wr->r3 = 0; 1144bbb2f537SJohn Baldwin 1145bbb2f537SJohn Baldwin cpl = (void *)(wr + 1); 1146bbb2f537SJohn Baldwin 1147bbb2f537SJohn Baldwin /* CPL header */ 1148bbb2f537SJohn Baldwin cpl->ctrl0 = txq->cpl_ctrl0; 1149bbb2f537SJohn Baldwin cpl->pack = 0; 1150bbb2f537SJohn Baldwin cpl->len = htobe16(pktlen); 1151bbb2f537SJohn Baldwin 1152bbb2f537SJohn Baldwin out = (void *)(cpl + 1); 1153bbb2f537SJohn Baldwin 1154bbb2f537SJohn Baldwin /* Copy over Ethernet header. */ 1155bbb2f537SJohn Baldwin eh = mtod(m, struct ether_header *); 1156bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); 1157bbb2f537SJohn Baldwin 1158bbb2f537SJohn Baldwin /* Fixup length in IP header and copy out. */ 1159bbb2f537SJohn Baldwin if (ntohs(eh->ether_type) == ETHERTYPE_IP) { 1160bbb2f537SJohn Baldwin ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); 1161bbb2f537SJohn Baldwin newip = *ip; 1162bbb2f537SJohn Baldwin newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); 1163bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); 1164bbb2f537SJohn Baldwin if (m->m_pkthdr.l3hlen > sizeof(*ip)) 1165bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, 1166bbb2f537SJohn Baldwin m->m_pkthdr.l3hlen - sizeof(*ip)); 1167bbb2f537SJohn Baldwin ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP) | 1168bbb2f537SJohn Baldwin V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | 1169bbb2f537SJohn Baldwin V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); 1170bbb2f537SJohn Baldwin } else { 1171bbb2f537SJohn Baldwin ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); 1172bbb2f537SJohn Baldwin newip6 = *ip6; 1173bbb2f537SJohn Baldwin newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); 1174bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); 1175bbb2f537SJohn Baldwin MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); 1176bbb2f537SJohn Baldwin ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP6) | 1177bbb2f537SJohn Baldwin V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | 1178bbb2f537SJohn Baldwin V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); 1179bbb2f537SJohn Baldwin } 1180bbb2f537SJohn Baldwin cpl->ctrl1 = htobe64(ctrl1); 1181bbb2f537SJohn Baldwin txq->txcsum++; 1182bbb2f537SJohn Baldwin 1183bbb2f537SJohn Baldwin /* Clear PUSH and FIN in the TCP header if present. */ 1184bbb2f537SJohn Baldwin tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); 1185bbb2f537SJohn Baldwin newtcp = *tcp; 1186bbb2f537SJohn Baldwin newtcp.th_flags &= ~(TH_PUSH | TH_FIN); 1187bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); 1188bbb2f537SJohn Baldwin 1189bbb2f537SJohn Baldwin /* Copy rest of packet. */ 1190bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, pktlen - 1191bbb2f537SJohn Baldwin (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); 1192bbb2f537SJohn Baldwin txq->imm_wrs++; 1193bbb2f537SJohn Baldwin 1194bbb2f537SJohn Baldwin txq->txpkt_wrs++; 1195bbb2f537SJohn Baldwin 1196bbb2f537SJohn Baldwin txq->kern_tls_options++; 1197bbb2f537SJohn Baldwin 1198bbb2f537SJohn Baldwin txsd = &txq->sdesc[pidx]; 1199bbb2f537SJohn Baldwin txsd->m = NULL; 1200bbb2f537SJohn Baldwin txsd->desc_used = ndesc; 1201bbb2f537SJohn Baldwin 1202bbb2f537SJohn Baldwin return (ndesc); 1203bbb2f537SJohn Baldwin } 1204bbb2f537SJohn Baldwin 1205bbb2f537SJohn Baldwin static int 1206bbb2f537SJohn Baldwin ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m, 1207bbb2f537SJohn Baldwin struct mbuf *m_tls, u_int available, tcp_seq tcp_seqno, u_int pidx) 1208bbb2f537SJohn Baldwin { 1209bbb2f537SJohn Baldwin struct tx_sdesc *txsd; 1210bbb2f537SJohn Baldwin struct fw_eth_tx_pkt_wr *wr; 1211bbb2f537SJohn Baldwin struct cpl_tx_pkt_core *cpl; 1212bbb2f537SJohn Baldwin uint32_t ctrl; 1213bbb2f537SJohn Baldwin uint64_t ctrl1; 1214bbb2f537SJohn Baldwin int len16, ndesc, pktlen; 1215bbb2f537SJohn Baldwin struct ether_header *eh; 1216bbb2f537SJohn Baldwin struct ip *ip, newip; 1217bbb2f537SJohn Baldwin struct ip6_hdr *ip6, newip6; 1218bbb2f537SJohn Baldwin struct tcphdr *tcp, newtcp; 1219bbb2f537SJohn Baldwin caddr_t out; 1220bbb2f537SJohn Baldwin 1221bbb2f537SJohn Baldwin TXQ_LOCK_ASSERT_OWNED(txq); 1222bbb2f537SJohn Baldwin M_ASSERTPKTHDR(m); 1223bbb2f537SJohn Baldwin 1224bbb2f537SJohn Baldwin /* Locate the template TLS header. */ 1225bbb2f537SJohn Baldwin M_ASSERTEXTPG(m_tls); 1226bbb2f537SJohn Baldwin 1227bbb2f537SJohn Baldwin /* This should always be the last TLS record in a chain. */ 1228bbb2f537SJohn Baldwin MPASS(m_tls->m_next == NULL); 1229bbb2f537SJohn Baldwin 1230bbb2f537SJohn Baldwin wr = dst; 1231bbb2f537SJohn Baldwin pktlen = m->m_len + m_tls->m_len; 1232bbb2f537SJohn Baldwin ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen; 1233bbb2f537SJohn Baldwin len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16); 1234bbb2f537SJohn Baldwin ndesc = tx_len16_to_desc(len16); 1235bbb2f537SJohn Baldwin MPASS(ndesc <= available); 1236bbb2f537SJohn Baldwin 1237bbb2f537SJohn Baldwin /* Firmware work request header */ 1238bbb2f537SJohn Baldwin wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 1239bbb2f537SJohn Baldwin V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 1240bbb2f537SJohn Baldwin 1241bbb2f537SJohn Baldwin ctrl = V_FW_WR_LEN16(len16); 1242bbb2f537SJohn Baldwin wr->equiq_to_len16 = htobe32(ctrl); 1243bbb2f537SJohn Baldwin wr->r3 = 0; 1244bbb2f537SJohn Baldwin 1245bbb2f537SJohn Baldwin cpl = (void *)(wr + 1); 1246bbb2f537SJohn Baldwin 1247bbb2f537SJohn Baldwin /* CPL header */ 1248bbb2f537SJohn Baldwin cpl->ctrl0 = txq->cpl_ctrl0; 1249bbb2f537SJohn Baldwin cpl->pack = 0; 1250bbb2f537SJohn Baldwin cpl->len = htobe16(pktlen); 1251bbb2f537SJohn Baldwin 1252bbb2f537SJohn Baldwin out = (void *)(cpl + 1); 1253bbb2f537SJohn Baldwin 1254bbb2f537SJohn Baldwin /* Copy over Ethernet header. */ 1255bbb2f537SJohn Baldwin eh = mtod(m, struct ether_header *); 1256bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); 1257bbb2f537SJohn Baldwin 1258bbb2f537SJohn Baldwin /* Fixup length in IP header and copy out. */ 1259bbb2f537SJohn Baldwin if (ntohs(eh->ether_type) == ETHERTYPE_IP) { 1260bbb2f537SJohn Baldwin ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); 1261bbb2f537SJohn Baldwin newip = *ip; 1262bbb2f537SJohn Baldwin newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); 1263bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); 1264bbb2f537SJohn Baldwin if (m->m_pkthdr.l3hlen > sizeof(*ip)) 1265bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, 1266bbb2f537SJohn Baldwin m->m_pkthdr.l3hlen - sizeof(*ip)); 1267bbb2f537SJohn Baldwin ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP) | 1268bbb2f537SJohn Baldwin V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | 1269bbb2f537SJohn Baldwin V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); 1270bbb2f537SJohn Baldwin } else { 1271bbb2f537SJohn Baldwin ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); 1272bbb2f537SJohn Baldwin newip6 = *ip6; 1273bbb2f537SJohn Baldwin newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); 1274bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); 1275bbb2f537SJohn Baldwin MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); 1276bbb2f537SJohn Baldwin ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP6) | 1277bbb2f537SJohn Baldwin V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | 1278bbb2f537SJohn Baldwin V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); 1279bbb2f537SJohn Baldwin } 1280bbb2f537SJohn Baldwin cpl->ctrl1 = htobe64(ctrl1); 1281bbb2f537SJohn Baldwin txq->txcsum++; 1282bbb2f537SJohn Baldwin 1283bbb2f537SJohn Baldwin /* Set sequence number in TCP header. */ 1284bbb2f537SJohn Baldwin tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); 1285bbb2f537SJohn Baldwin newtcp = *tcp; 1286bbb2f537SJohn Baldwin newtcp.th_seq = htonl(tcp_seqno + mtod(m_tls, vm_offset_t)); 1287bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); 1288bbb2f537SJohn Baldwin 1289bbb2f537SJohn Baldwin /* Copy rest of TCP header. */ 1290bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len - 1291bbb2f537SJohn Baldwin (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); 1292bbb2f537SJohn Baldwin 1293bbb2f537SJohn Baldwin /* Copy the subset of the TLS header requested. */ 1294bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr + 1295bbb2f537SJohn Baldwin mtod(m_tls, vm_offset_t), &out, m_tls->m_len); 1296bbb2f537SJohn Baldwin txq->imm_wrs++; 1297bbb2f537SJohn Baldwin 1298bbb2f537SJohn Baldwin txq->txpkt_wrs++; 1299bbb2f537SJohn Baldwin 1300bbb2f537SJohn Baldwin txq->kern_tls_header++; 1301bbb2f537SJohn Baldwin 1302bbb2f537SJohn Baldwin txsd = &txq->sdesc[pidx]; 1303bbb2f537SJohn Baldwin txsd->m = m; 1304bbb2f537SJohn Baldwin txsd->desc_used = ndesc; 1305bbb2f537SJohn Baldwin 1306bbb2f537SJohn Baldwin return (ndesc); 1307bbb2f537SJohn Baldwin } 1308bbb2f537SJohn Baldwin 1309bbb2f537SJohn Baldwin _Static_assert(sizeof(struct cpl_set_tcb_field) <= EQ_ESIZE, 1310bbb2f537SJohn Baldwin "CPL_SET_TCB_FIELD must be smaller than a single TX descriptor"); 1311bbb2f537SJohn Baldwin _Static_assert(W_TCB_SND_UNA_RAW == W_TCB_SND_NXT_RAW, 1312bbb2f537SJohn Baldwin "SND_NXT_RAW and SND_UNA_RAW are in different words"); 1313bbb2f537SJohn Baldwin 1314bbb2f537SJohn Baldwin static int 1315ca457729SJohn Baldwin ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq, void *dst, 1316ca457729SJohn Baldwin struct mbuf *m, struct tcphdr *tcp, struct mbuf *m_tls, u_int available, 1317ca457729SJohn Baldwin tcp_seq tcp_seqno, uint32_t *tsopt, u_int pidx, bool set_l2t_idx) 1318bbb2f537SJohn Baldwin { 1319bbb2f537SJohn Baldwin struct sge_eq *eq = &txq->eq; 1320bbb2f537SJohn Baldwin struct tx_sdesc *txsd; 1321bbb2f537SJohn Baldwin struct fw_ulptx_wr *wr; 1322bbb2f537SJohn Baldwin struct ulp_txpkt *txpkt; 1323bbb2f537SJohn Baldwin struct ulptx_sc_memrd *memrd; 1324bbb2f537SJohn Baldwin struct ulptx_idata *idata; 1325bbb2f537SJohn Baldwin struct cpl_tx_sec_pdu *sec_pdu; 1326bbb2f537SJohn Baldwin struct cpl_tx_data *tx_data; 1327bbb2f537SJohn Baldwin struct tls_record_layer *hdr; 1328bbb2f537SJohn Baldwin char *iv, *out; 1329bbb2f537SJohn Baldwin u_int aad_start, aad_stop; 1330bbb2f537SJohn Baldwin u_int auth_start, auth_stop, auth_insert; 1331bbb2f537SJohn Baldwin u_int cipher_start, cipher_stop, iv_offset; 1332bbb2f537SJohn Baldwin u_int imm_len, mss, ndesc, offset, plen, tlen, twr_len, wr_len; 1333ca457729SJohn Baldwin u_int fields, nsegs, tx_max_offset, tx_max; 1334bbb2f537SJohn Baldwin bool first_wr, last_wr, using_scratch; 1335bbb2f537SJohn Baldwin 1336bbb2f537SJohn Baldwin ndesc = 0; 1337bbb2f537SJohn Baldwin MPASS(tlsp->txq == txq); 1338bbb2f537SJohn Baldwin 1339bbb2f537SJohn Baldwin first_wr = (tlsp->prev_seq == 0 && tlsp->prev_ack == 0 && 1340bbb2f537SJohn Baldwin tlsp->prev_win == 0); 1341bbb2f537SJohn Baldwin 1342bbb2f537SJohn Baldwin /* 1343bbb2f537SJohn Baldwin * Use the per-txq scratch pad if near the end of the ring to 1344bbb2f537SJohn Baldwin * simplify handling of wrap-around. This uses a simple but 1345bbb2f537SJohn Baldwin * not quite perfect test of using the scratch buffer if we 1346bbb2f537SJohn Baldwin * can't fit a maximal work request in without wrapping. 1347bbb2f537SJohn Baldwin */ 1348bbb2f537SJohn Baldwin using_scratch = (eq->sidx - pidx < SGE_MAX_WR_LEN / EQ_ESIZE); 1349bbb2f537SJohn Baldwin 1350bbb2f537SJohn Baldwin /* Locate the TLS header. */ 1351bbb2f537SJohn Baldwin M_ASSERTEXTPG(m_tls); 1352bbb2f537SJohn Baldwin hdr = (void *)m_tls->m_epg_hdr; 1353bbb2f537SJohn Baldwin plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_epg_trllen; 1354bbb2f537SJohn Baldwin 1355bbb2f537SJohn Baldwin /* Determine how much of the TLS record to send. */ 1356bbb2f537SJohn Baldwin tlen = ktls_tcp_payload_length(tlsp, m_tls); 1357bbb2f537SJohn Baldwin if (tlen <= m_tls->m_epg_hdrlen) { 1358bbb2f537SJohn Baldwin /* 1359bbb2f537SJohn Baldwin * For requests that only want to send the TLS header, 1360bbb2f537SJohn Baldwin * send a tunnelled packet as immediate data. 1361bbb2f537SJohn Baldwin */ 1362bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 1363bbb2f537SJohn Baldwin CTR3(KTR_CXGBE, "%s: tid %d header-only TLS record %u", 1364bbb2f537SJohn Baldwin __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno); 1365bbb2f537SJohn Baldwin #endif 1366bbb2f537SJohn Baldwin return (ktls_write_tunnel_packet(txq, dst, m, m_tls, available, 1367bbb2f537SJohn Baldwin tcp_seqno, pidx)); 1368bbb2f537SJohn Baldwin } 1369bbb2f537SJohn Baldwin if (tlen < plen) { 1370bbb2f537SJohn Baldwin plen = tlen; 1371bbb2f537SJohn Baldwin offset = ktls_payload_offset(tlsp, m_tls); 1372bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 1373bbb2f537SJohn Baldwin CTR4(KTR_CXGBE, "%s: tid %d short TLS record %u with offset %u", 1374bbb2f537SJohn Baldwin __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno, offset); 1375bbb2f537SJohn Baldwin #endif 1376bbb2f537SJohn Baldwin if (m_tls->m_next == NULL && (tcp->th_flags & TH_FIN) != 0) { 1377bbb2f537SJohn Baldwin txq->kern_tls_fin_short++; 1378bbb2f537SJohn Baldwin #ifdef INVARIANTS 1379bbb2f537SJohn Baldwin panic("%s: FIN on short TLS record", __func__); 1380bbb2f537SJohn Baldwin #endif 1381bbb2f537SJohn Baldwin } 1382bbb2f537SJohn Baldwin } else 1383bbb2f537SJohn Baldwin offset = 0; 1384bbb2f537SJohn Baldwin 1385bbb2f537SJohn Baldwin /* 1386bbb2f537SJohn Baldwin * This is the last work request for a given TLS mbuf chain if 1387bbb2f537SJohn Baldwin * it is the last mbuf in the chain and FIN is not set. If 1388bbb2f537SJohn Baldwin * FIN is set, then ktls_write_tcp_fin() will write out the 1389bbb2f537SJohn Baldwin * last work request. 1390bbb2f537SJohn Baldwin */ 1391bbb2f537SJohn Baldwin last_wr = m_tls->m_next == NULL && (tcp->th_flags & TH_FIN) == 0; 1392bbb2f537SJohn Baldwin 1393bbb2f537SJohn Baldwin /* 1394bbb2f537SJohn Baldwin * The host stack may ask us to not send part of the start of 1395bbb2f537SJohn Baldwin * a TLS record. (For example, the stack might have 1396bbb2f537SJohn Baldwin * previously sent a "short" TLS record and might later send 1397bbb2f537SJohn Baldwin * down an mbuf that requests to send the remainder of the TLS 1398bbb2f537SJohn Baldwin * record.) The crypto engine must process a TLS record from 1399bbb2f537SJohn Baldwin * the beginning if computing a GCM tag or HMAC, so we always 1400bbb2f537SJohn Baldwin * send the TLS record from the beginning as input to the 1401bbb2f537SJohn Baldwin * crypto engine and via CPL_TX_DATA to TP. However, TP will 1402bbb2f537SJohn Baldwin * drop individual packets after they have been chopped up 1403bbb2f537SJohn Baldwin * into MSS-sized chunks if the entire sequence range of those 1404bbb2f537SJohn Baldwin * packets is less than SND_UNA. SND_UNA is computed as 1405bbb2f537SJohn Baldwin * TX_MAX - SND_UNA_RAW. Thus, use the offset stored in 1406bbb2f537SJohn Baldwin * m_data to set TX_MAX to the first byte in the TCP sequence 1407bbb2f537SJohn Baldwin * space the host actually wants us to send and set 1408bbb2f537SJohn Baldwin * SND_UNA_RAW to 0. 1409bbb2f537SJohn Baldwin * 1410bbb2f537SJohn Baldwin * If the host sends us back to back requests that span the 1411bbb2f537SJohn Baldwin * trailer of a single TLS record (first request ends "in" the 1412bbb2f537SJohn Baldwin * trailer and second request starts at the next byte but 1413bbb2f537SJohn Baldwin * still "in" the trailer), the initial bytes of the trailer 1414bbb2f537SJohn Baldwin * that the first request drops will not be retransmitted. If 1415bbb2f537SJohn Baldwin * the host uses the same requests when retransmitting the 1416bbb2f537SJohn Baldwin * connection will hang. To handle this, always transmit the 1417bbb2f537SJohn Baldwin * full trailer for a request that begins "in" the trailer 1418bbb2f537SJohn Baldwin * (the second request in the example above). This should 1419bbb2f537SJohn Baldwin * also help to avoid retransmits for the common case. 1420bbb2f537SJohn Baldwin * 1421bbb2f537SJohn Baldwin * A similar condition exists when using CBC for back to back 1422bbb2f537SJohn Baldwin * requests that span a single AES block. The first request 1423bbb2f537SJohn Baldwin * will be truncated to end at the end of the previous AES 1424bbb2f537SJohn Baldwin * block. To handle this, always begin transmission at the 1425bbb2f537SJohn Baldwin * start of the current AES block. 1426bbb2f537SJohn Baldwin */ 1427bbb2f537SJohn Baldwin tx_max_offset = mtod(m_tls, vm_offset_t); 1428bbb2f537SJohn Baldwin if (tx_max_offset > TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - 1429bbb2f537SJohn Baldwin m_tls->m_epg_trllen) { 1430bbb2f537SJohn Baldwin /* Always send the full trailer. */ 1431bbb2f537SJohn Baldwin tx_max_offset = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - 1432bbb2f537SJohn Baldwin m_tls->m_epg_trllen; 1433bbb2f537SJohn Baldwin } 1434bbb2f537SJohn Baldwin if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC && 1435bbb2f537SJohn Baldwin tx_max_offset > TLS_HEADER_LENGTH) { 1436bbb2f537SJohn Baldwin /* Always send all of the first AES block. */ 1437bbb2f537SJohn Baldwin tx_max_offset = TLS_HEADER_LENGTH + 1438bbb2f537SJohn Baldwin rounddown(tx_max_offset - TLS_HEADER_LENGTH, 1439bbb2f537SJohn Baldwin AES_BLOCK_LEN); 1440bbb2f537SJohn Baldwin } 1441bbb2f537SJohn Baldwin tx_max = tcp_seqno + tx_max_offset; 1442bbb2f537SJohn Baldwin 1443bbb2f537SJohn Baldwin /* 1444bbb2f537SJohn Baldwin * Update TCB fields. Reserve space for the FW_ULPTX_WR header 1445bbb2f537SJohn Baldwin * but don't populate it until we know how many field updates 1446bbb2f537SJohn Baldwin * are required. 1447bbb2f537SJohn Baldwin */ 1448bbb2f537SJohn Baldwin if (using_scratch) 1449bbb2f537SJohn Baldwin wr = (void *)txq->ss; 1450bbb2f537SJohn Baldwin else 1451bbb2f537SJohn Baldwin wr = dst; 1452bbb2f537SJohn Baldwin out = (void *)(wr + 1); 1453bbb2f537SJohn Baldwin fields = 0; 1454bbb2f537SJohn Baldwin if (set_l2t_idx) { 1455ca457729SJohn Baldwin KASSERT(m->m_next == m_tls, 1456bbb2f537SJohn Baldwin ("trying to set L2T_IX for subsequent TLS WR")); 1457bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 1458bbb2f537SJohn Baldwin CTR3(KTR_CXGBE, "%s: tid %d set L2T_IX to %d", __func__, 1459bbb2f537SJohn Baldwin tlsp->tid, tlsp->l2te->idx); 1460bbb2f537SJohn Baldwin #endif 1461bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_L2T_IX, 1462bbb2f537SJohn Baldwin V_TCB_L2T_IX(M_TCB_L2T_IX), V_TCB_L2T_IX(tlsp->l2te->idx)); 1463bbb2f537SJohn Baldwin out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1464bbb2f537SJohn Baldwin fields++; 1465bbb2f537SJohn Baldwin } 1466bbb2f537SJohn Baldwin if (tsopt != NULL && tlsp->prev_tsecr != ntohl(tsopt[1])) { 1467ca457729SJohn Baldwin KASSERT(m->m_next == m_tls, 1468bbb2f537SJohn Baldwin ("trying to set T_RTSEQ_RECENT for subsequent TLS WR")); 1469bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 1470bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d wrote updated T_RTSEQ_RECENT", 1471bbb2f537SJohn Baldwin __func__, tlsp->tid); 1472bbb2f537SJohn Baldwin #endif 1473bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_T_RTSEQ_RECENT, 1474bbb2f537SJohn Baldwin V_TCB_T_RTSEQ_RECENT(M_TCB_T_RTSEQ_RECENT), 1475bbb2f537SJohn Baldwin V_TCB_T_RTSEQ_RECENT(ntohl(tsopt[1]))); 1476bbb2f537SJohn Baldwin out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1477bbb2f537SJohn Baldwin fields++; 1478bbb2f537SJohn Baldwin 1479bbb2f537SJohn Baldwin tlsp->prev_tsecr = ntohl(tsopt[1]); 1480bbb2f537SJohn Baldwin } 1481bbb2f537SJohn Baldwin 1482bbb2f537SJohn Baldwin if (first_wr || tlsp->prev_seq != tx_max) { 1483ca457729SJohn Baldwin KASSERT(m->m_next == m_tls, 1484bbb2f537SJohn Baldwin ("trying to set TX_MAX for subsequent TLS WR")); 1485bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 1486bbb2f537SJohn Baldwin CTR4(KTR_CXGBE, 1487bbb2f537SJohn Baldwin "%s: tid %d setting TX_MAX to %u (tcp_seqno %u)", 1488bbb2f537SJohn Baldwin __func__, tlsp->tid, tx_max, tcp_seqno); 1489bbb2f537SJohn Baldwin #endif 1490bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_TX_MAX, 1491bbb2f537SJohn Baldwin V_TCB_TX_MAX(M_TCB_TX_MAX), V_TCB_TX_MAX(tx_max)); 1492bbb2f537SJohn Baldwin out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1493bbb2f537SJohn Baldwin fields++; 1494bbb2f537SJohn Baldwin } 1495bbb2f537SJohn Baldwin 1496bbb2f537SJohn Baldwin /* 1497bbb2f537SJohn Baldwin * If there is data to drop at the beginning of this TLS 1498bbb2f537SJohn Baldwin * record or if this is a retransmit, 1499bbb2f537SJohn Baldwin * reset SND_UNA_RAW to 0 so that SND_UNA == TX_MAX. 1500bbb2f537SJohn Baldwin */ 1501bbb2f537SJohn Baldwin if (tlsp->prev_seq != tx_max || mtod(m_tls, vm_offset_t) != 0) { 1502ca457729SJohn Baldwin KASSERT(m->m_next == m_tls, 1503bbb2f537SJohn Baldwin ("trying to clear SND_UNA_RAW for subsequent TLS WR")); 1504bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 1505bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d clearing SND_UNA_RAW", __func__, 1506bbb2f537SJohn Baldwin tlsp->tid); 1507bbb2f537SJohn Baldwin #endif 1508bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_SND_UNA_RAW, 1509bbb2f537SJohn Baldwin V_TCB_SND_UNA_RAW(M_TCB_SND_UNA_RAW), 1510bbb2f537SJohn Baldwin V_TCB_SND_UNA_RAW(0)); 1511bbb2f537SJohn Baldwin out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1512bbb2f537SJohn Baldwin fields++; 1513bbb2f537SJohn Baldwin } 1514bbb2f537SJohn Baldwin 1515bbb2f537SJohn Baldwin /* 1516bbb2f537SJohn Baldwin * Store the expected sequence number of the next byte after 1517bbb2f537SJohn Baldwin * this record. 1518bbb2f537SJohn Baldwin */ 1519bbb2f537SJohn Baldwin tlsp->prev_seq = tcp_seqno + tlen; 1520bbb2f537SJohn Baldwin 1521bbb2f537SJohn Baldwin if (first_wr || tlsp->prev_ack != ntohl(tcp->th_ack)) { 1522ca457729SJohn Baldwin KASSERT(m->m_next == m_tls, 1523bbb2f537SJohn Baldwin ("trying to set RCV_NXT for subsequent TLS WR")); 1524bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_NXT, 1525bbb2f537SJohn Baldwin V_TCB_RCV_NXT(M_TCB_RCV_NXT), 1526bbb2f537SJohn Baldwin V_TCB_RCV_NXT(ntohl(tcp->th_ack))); 1527bbb2f537SJohn Baldwin out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1528bbb2f537SJohn Baldwin fields++; 1529bbb2f537SJohn Baldwin 1530bbb2f537SJohn Baldwin tlsp->prev_ack = ntohl(tcp->th_ack); 1531bbb2f537SJohn Baldwin } 1532bbb2f537SJohn Baldwin 1533bbb2f537SJohn Baldwin if (first_wr || tlsp->prev_win != ntohs(tcp->th_win)) { 1534ca457729SJohn Baldwin KASSERT(m->m_next == m_tls, 1535bbb2f537SJohn Baldwin ("trying to set RCV_WND for subsequent TLS WR")); 1536bbb2f537SJohn Baldwin write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_WND, 1537bbb2f537SJohn Baldwin V_TCB_RCV_WND(M_TCB_RCV_WND), 1538bbb2f537SJohn Baldwin V_TCB_RCV_WND(ntohs(tcp->th_win))); 1539bbb2f537SJohn Baldwin out += roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1540bbb2f537SJohn Baldwin fields++; 1541bbb2f537SJohn Baldwin 1542bbb2f537SJohn Baldwin tlsp->prev_win = ntohs(tcp->th_win); 1543bbb2f537SJohn Baldwin } 1544bbb2f537SJohn Baldwin 1545ca457729SJohn Baldwin /* Use cached value for first record in chain. */ 1546ca457729SJohn Baldwin if (m->m_next == m_tls) 1547ca457729SJohn Baldwin nsegs = mbuf_nsegs(m); 1548ca457729SJohn Baldwin else 1549bbb2f537SJohn Baldwin nsegs = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + 1550bbb2f537SJohn Baldwin offset, plen - (m_tls->m_epg_hdrlen + offset)); 1551bbb2f537SJohn Baldwin 1552bbb2f537SJohn Baldwin /* Calculate the size of the TLS work request. */ 1553bbb2f537SJohn Baldwin twr_len = ktls_base_wr_size(tlsp); 1554bbb2f537SJohn Baldwin 1555bbb2f537SJohn Baldwin imm_len = 0; 1556bbb2f537SJohn Baldwin if (offset == 0) 1557bbb2f537SJohn Baldwin imm_len += m_tls->m_epg_hdrlen; 1558bbb2f537SJohn Baldwin if (plen == tlen) 1559bbb2f537SJohn Baldwin imm_len += AES_BLOCK_LEN; 1560bbb2f537SJohn Baldwin twr_len += roundup2(imm_len, 16); 1561bbb2f537SJohn Baldwin twr_len += ktls_sgl_size(nsegs); 1562bbb2f537SJohn Baldwin 1563bbb2f537SJohn Baldwin /* 1564bbb2f537SJohn Baldwin * If any field updates were required, determine if they can 1565bbb2f537SJohn Baldwin * be included in the TLS work request. If not, use the 1566bbb2f537SJohn Baldwin * FW_ULPTX_WR work request header at 'wr' as a dedicated work 1567bbb2f537SJohn Baldwin * request for the field updates and start a new work request 1568bbb2f537SJohn Baldwin * for the TLS work request afterward. 1569bbb2f537SJohn Baldwin */ 1570bbb2f537SJohn Baldwin if (fields != 0) { 1571bbb2f537SJohn Baldwin wr_len = fields * roundup2(LEN__SET_TCB_FIELD_ULP, 16); 1572bbb2f537SJohn Baldwin if (twr_len + wr_len <= SGE_MAX_WR_LEN && 1573bbb2f537SJohn Baldwin tlsp->sc->tlst.combo_wrs) { 1574bbb2f537SJohn Baldwin wr_len += twr_len; 1575bbb2f537SJohn Baldwin txpkt = (void *)out; 1576bbb2f537SJohn Baldwin } else { 1577bbb2f537SJohn Baldwin wr_len += sizeof(*wr); 1578bbb2f537SJohn Baldwin wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); 1579bbb2f537SJohn Baldwin wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | 1580bbb2f537SJohn Baldwin V_FW_WR_LEN16(wr_len / 16)); 1581bbb2f537SJohn Baldwin wr->cookie = 0; 1582bbb2f537SJohn Baldwin 1583bbb2f537SJohn Baldwin /* 1584bbb2f537SJohn Baldwin * If we were using scratch space, copy the 1585bbb2f537SJohn Baldwin * field updates work request to the ring. 1586bbb2f537SJohn Baldwin */ 1587bbb2f537SJohn Baldwin if (using_scratch) { 1588bbb2f537SJohn Baldwin out = dst; 1589bbb2f537SJohn Baldwin copy_to_txd(eq, txq->ss, &out, wr_len); 1590bbb2f537SJohn Baldwin } 1591bbb2f537SJohn Baldwin 1592bbb2f537SJohn Baldwin ndesc = howmany(wr_len, EQ_ESIZE); 1593bbb2f537SJohn Baldwin MPASS(ndesc <= available); 1594bbb2f537SJohn Baldwin 1595bbb2f537SJohn Baldwin txq->raw_wrs++; 1596bbb2f537SJohn Baldwin txsd = &txq->sdesc[pidx]; 1597bbb2f537SJohn Baldwin txsd->m = NULL; 1598bbb2f537SJohn Baldwin txsd->desc_used = ndesc; 1599bbb2f537SJohn Baldwin IDXINCR(pidx, ndesc, eq->sidx); 1600bbb2f537SJohn Baldwin dst = &eq->desc[pidx]; 1601bbb2f537SJohn Baldwin 1602bbb2f537SJohn Baldwin /* 1603bbb2f537SJohn Baldwin * Determine if we should use scratch space 1604bbb2f537SJohn Baldwin * for the TLS work request based on the 1605bbb2f537SJohn Baldwin * available space after advancing pidx for 1606bbb2f537SJohn Baldwin * the field updates work request. 1607bbb2f537SJohn Baldwin */ 1608bbb2f537SJohn Baldwin wr_len = twr_len; 1609bbb2f537SJohn Baldwin using_scratch = (eq->sidx - pidx < 1610bbb2f537SJohn Baldwin howmany(wr_len, EQ_ESIZE)); 1611bbb2f537SJohn Baldwin if (using_scratch) 1612bbb2f537SJohn Baldwin wr = (void *)txq->ss; 1613bbb2f537SJohn Baldwin else 1614bbb2f537SJohn Baldwin wr = dst; 1615bbb2f537SJohn Baldwin txpkt = (void *)(wr + 1); 1616bbb2f537SJohn Baldwin } 1617bbb2f537SJohn Baldwin } else { 1618bbb2f537SJohn Baldwin wr_len = twr_len; 1619bbb2f537SJohn Baldwin txpkt = (void *)out; 1620bbb2f537SJohn Baldwin } 1621bbb2f537SJohn Baldwin 1622bbb2f537SJohn Baldwin wr_len = roundup2(wr_len, 16); 1623bbb2f537SJohn Baldwin MPASS(ndesc + howmany(wr_len, EQ_ESIZE) <= available); 1624bbb2f537SJohn Baldwin 1625bbb2f537SJohn Baldwin /* FW_ULPTX_WR */ 1626bbb2f537SJohn Baldwin wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR)); 1627bbb2f537SJohn Baldwin wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA | 1628bbb2f537SJohn Baldwin V_FW_WR_LEN16(wr_len / 16)); 1629bbb2f537SJohn Baldwin wr->cookie = 0; 1630bbb2f537SJohn Baldwin 1631bbb2f537SJohn Baldwin /* ULP_TXPKT */ 1632bbb2f537SJohn Baldwin txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) | 1633bbb2f537SJohn Baldwin V_ULP_TXPKT_DATAMODIFY(0) | 1634bbb2f537SJohn Baldwin V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) | 1635bbb2f537SJohn Baldwin V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1)); 1636bbb2f537SJohn Baldwin txpkt->len = htobe32(howmany(twr_len - sizeof(*wr), 16)); 1637bbb2f537SJohn Baldwin 1638bbb2f537SJohn Baldwin /* ULPTX_IDATA sub-command */ 1639bbb2f537SJohn Baldwin idata = (void *)(txpkt + 1); 1640bbb2f537SJohn Baldwin idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | 1641bbb2f537SJohn Baldwin V_ULP_TX_SC_MORE(1)); 1642bbb2f537SJohn Baldwin idata->len = sizeof(struct cpl_tx_sec_pdu); 1643bbb2f537SJohn Baldwin 1644bbb2f537SJohn Baldwin /* 1645bbb2f537SJohn Baldwin * The key context, CPL_TX_DATA, and immediate data are part 1646bbb2f537SJohn Baldwin * of this ULPTX_IDATA when using an inline key. When reading 1647bbb2f537SJohn Baldwin * the key from memory, the CPL_TX_DATA and immediate data are 1648bbb2f537SJohn Baldwin * part of a separate ULPTX_IDATA. 1649bbb2f537SJohn Baldwin */ 1650bbb2f537SJohn Baldwin if (tlsp->inline_key) 1651bbb2f537SJohn Baldwin idata->len += tlsp->tx_key_info_size + 1652bbb2f537SJohn Baldwin sizeof(struct cpl_tx_data) + imm_len; 1653bbb2f537SJohn Baldwin idata->len = htobe32(idata->len); 1654bbb2f537SJohn Baldwin 1655bbb2f537SJohn Baldwin /* CPL_TX_SEC_PDU */ 1656bbb2f537SJohn Baldwin sec_pdu = (void *)(idata + 1); 1657bbb2f537SJohn Baldwin 1658bbb2f537SJohn Baldwin /* 1659bbb2f537SJohn Baldwin * For short records, AAD is counted as header data in SCMD0, 1660bbb2f537SJohn Baldwin * the IV is next followed by a cipher region for the payload. 1661bbb2f537SJohn Baldwin */ 1662bbb2f537SJohn Baldwin if (plen == tlen) { 1663bbb2f537SJohn Baldwin aad_start = 0; 1664bbb2f537SJohn Baldwin aad_stop = 0; 1665bbb2f537SJohn Baldwin iv_offset = 1; 1666bbb2f537SJohn Baldwin auth_start = 0; 1667bbb2f537SJohn Baldwin auth_stop = 0; 1668bbb2f537SJohn Baldwin auth_insert = 0; 1669bbb2f537SJohn Baldwin cipher_start = AES_BLOCK_LEN + 1; 1670bbb2f537SJohn Baldwin cipher_stop = 0; 1671bbb2f537SJohn Baldwin 1672bbb2f537SJohn Baldwin sec_pdu->pldlen = htobe32(16 + plen - 1673bbb2f537SJohn Baldwin (m_tls->m_epg_hdrlen + offset)); 1674bbb2f537SJohn Baldwin 1675bbb2f537SJohn Baldwin /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 1676bbb2f537SJohn Baldwin sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs; 1677bbb2f537SJohn Baldwin sec_pdu->ivgen_hdrlen = htobe32( 1678bbb2f537SJohn Baldwin tlsp->scmd0_short.ivgen_hdrlen | 1679bbb2f537SJohn Baldwin V_SCMD_HDR_LEN(offset == 0 ? m_tls->m_epg_hdrlen : 0)); 1680bbb2f537SJohn Baldwin 1681bbb2f537SJohn Baldwin txq->kern_tls_short++; 1682bbb2f537SJohn Baldwin } else { 1683bbb2f537SJohn Baldwin /* 1684bbb2f537SJohn Baldwin * AAD is TLS header. IV is after AAD. The cipher region 1685bbb2f537SJohn Baldwin * starts after the IV. See comments in ccr_authenc() and 1686bbb2f537SJohn Baldwin * ccr_gmac() in t4_crypto.c regarding cipher and auth 1687bbb2f537SJohn Baldwin * start/stop values. 1688bbb2f537SJohn Baldwin */ 1689bbb2f537SJohn Baldwin aad_start = 1; 1690bbb2f537SJohn Baldwin aad_stop = TLS_HEADER_LENGTH; 1691bbb2f537SJohn Baldwin iv_offset = TLS_HEADER_LENGTH + 1; 1692bbb2f537SJohn Baldwin cipher_start = m_tls->m_epg_hdrlen + 1; 1693bbb2f537SJohn Baldwin if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { 1694bbb2f537SJohn Baldwin cipher_stop = 0; 1695bbb2f537SJohn Baldwin auth_start = cipher_start; 1696bbb2f537SJohn Baldwin auth_stop = 0; 1697bbb2f537SJohn Baldwin auth_insert = 0; 1698bbb2f537SJohn Baldwin } else { 1699bbb2f537SJohn Baldwin cipher_stop = 0; 1700bbb2f537SJohn Baldwin auth_start = cipher_start; 1701bbb2f537SJohn Baldwin auth_stop = 0; 1702bbb2f537SJohn Baldwin auth_insert = 0; 1703bbb2f537SJohn Baldwin } 1704bbb2f537SJohn Baldwin 1705bbb2f537SJohn Baldwin sec_pdu->pldlen = htobe32(plen); 1706bbb2f537SJohn Baldwin 1707bbb2f537SJohn Baldwin /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ 1708bbb2f537SJohn Baldwin sec_pdu->seqno_numivs = tlsp->scmd0.seqno_numivs; 1709bbb2f537SJohn Baldwin sec_pdu->ivgen_hdrlen = tlsp->scmd0.ivgen_hdrlen; 1710bbb2f537SJohn Baldwin 1711bbb2f537SJohn Baldwin if (mtod(m_tls, vm_offset_t) == 0) 1712bbb2f537SJohn Baldwin txq->kern_tls_full++; 1713bbb2f537SJohn Baldwin else 1714bbb2f537SJohn Baldwin txq->kern_tls_partial++; 1715bbb2f537SJohn Baldwin } 1716bbb2f537SJohn Baldwin sec_pdu->op_ivinsrtofst = htobe32( 1717bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) | 1718bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) | 1719bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_IVINSRTOFST(iv_offset)); 1720bbb2f537SJohn Baldwin sec_pdu->aadstart_cipherstop_hi = htobe32( 1721bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_AADSTART(aad_start) | 1722bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) | 1723bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) | 1724bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4)); 1725bbb2f537SJohn Baldwin sec_pdu->cipherstop_lo_authinsert = htobe32( 1726bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) | 1727bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) | 1728bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) | 1729bbb2f537SJohn Baldwin V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert)); 1730bbb2f537SJohn Baldwin 1731bbb2f537SJohn Baldwin sec_pdu->scmd1 = htobe64(m_tls->m_epg_seqno); 1732bbb2f537SJohn Baldwin 1733bbb2f537SJohn Baldwin /* Key context */ 1734bbb2f537SJohn Baldwin out = (void *)(sec_pdu + 1); 1735bbb2f537SJohn Baldwin if (tlsp->inline_key) { 1736bbb2f537SJohn Baldwin memcpy(out, &tlsp->keyctx, tlsp->tx_key_info_size); 1737bbb2f537SJohn Baldwin out += tlsp->tx_key_info_size; 1738bbb2f537SJohn Baldwin } else { 1739bbb2f537SJohn Baldwin /* ULPTX_SC_MEMRD to read key context. */ 1740bbb2f537SJohn Baldwin memrd = (void *)out; 1741bbb2f537SJohn Baldwin memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) | 1742bbb2f537SJohn Baldwin V_ULP_TX_SC_MORE(1) | 1743bbb2f537SJohn Baldwin V_ULPTX_LEN16(tlsp->tx_key_info_size >> 4)); 1744bbb2f537SJohn Baldwin memrd->addr = htobe32(tlsp->tx_key_addr >> 5); 1745bbb2f537SJohn Baldwin 1746bbb2f537SJohn Baldwin /* ULPTX_IDATA for CPL_TX_DATA and TLS header. */ 1747bbb2f537SJohn Baldwin idata = (void *)(memrd + 1); 1748bbb2f537SJohn Baldwin idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) | 1749bbb2f537SJohn Baldwin V_ULP_TX_SC_MORE(1)); 1750bbb2f537SJohn Baldwin idata->len = htobe32(sizeof(struct cpl_tx_data) + imm_len); 1751bbb2f537SJohn Baldwin 1752bbb2f537SJohn Baldwin out = (void *)(idata + 1); 1753bbb2f537SJohn Baldwin } 1754bbb2f537SJohn Baldwin 1755bbb2f537SJohn Baldwin /* CPL_TX_DATA */ 1756bbb2f537SJohn Baldwin tx_data = (void *)out; 1757bbb2f537SJohn Baldwin OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tlsp->tid)); 1758bbb2f537SJohn Baldwin if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1759bbb2f537SJohn Baldwin mss = m->m_pkthdr.tso_segsz; 1760bbb2f537SJohn Baldwin tlsp->prev_mss = mss; 1761bbb2f537SJohn Baldwin } else if (tlsp->prev_mss != 0) 1762bbb2f537SJohn Baldwin mss = tlsp->prev_mss; 1763bbb2f537SJohn Baldwin else 1764bbb2f537SJohn Baldwin mss = tlsp->vi->ifp->if_mtu - 1765bbb2f537SJohn Baldwin (m->m_pkthdr.l3hlen + m->m_pkthdr.l4hlen); 1766bbb2f537SJohn Baldwin if (offset == 0) { 1767bbb2f537SJohn Baldwin tx_data->len = htobe32(V_TX_DATA_MSS(mss) | V_TX_LENGTH(tlen)); 1768bbb2f537SJohn Baldwin tx_data->rsvd = htobe32(tcp_seqno); 1769bbb2f537SJohn Baldwin } else { 1770bbb2f537SJohn Baldwin tx_data->len = htobe32(V_TX_DATA_MSS(mss) | 1771bbb2f537SJohn Baldwin V_TX_LENGTH(tlen - (m_tls->m_epg_hdrlen + offset))); 1772bbb2f537SJohn Baldwin tx_data->rsvd = htobe32(tcp_seqno + m_tls->m_epg_hdrlen + offset); 1773bbb2f537SJohn Baldwin } 1774bbb2f537SJohn Baldwin tx_data->flags = htobe32(F_TX_BYPASS); 1775bbb2f537SJohn Baldwin if (last_wr && tcp->th_flags & TH_PUSH) 1776bbb2f537SJohn Baldwin tx_data->flags |= htobe32(F_TX_PUSH | F_TX_SHOVE); 1777bbb2f537SJohn Baldwin 1778bbb2f537SJohn Baldwin /* Populate the TLS header */ 1779bbb2f537SJohn Baldwin out = (void *)(tx_data + 1); 1780bbb2f537SJohn Baldwin if (offset == 0) { 1781bbb2f537SJohn Baldwin memcpy(out, m_tls->m_epg_hdr, m_tls->m_epg_hdrlen); 1782bbb2f537SJohn Baldwin out += m_tls->m_epg_hdrlen; 1783bbb2f537SJohn Baldwin } 1784bbb2f537SJohn Baldwin 1785bbb2f537SJohn Baldwin /* AES IV for a short record. */ 1786bbb2f537SJohn Baldwin if (plen == tlen) { 1787bbb2f537SJohn Baldwin iv = out; 1788bbb2f537SJohn Baldwin if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) { 1789bbb2f537SJohn Baldwin memcpy(iv, tlsp->keyctx.u.txhdr.txsalt, SALT_SIZE); 1790bbb2f537SJohn Baldwin memcpy(iv + 4, hdr + 1, 8); 1791bbb2f537SJohn Baldwin *(uint32_t *)(iv + 12) = htobe32(2 + 1792bbb2f537SJohn Baldwin offset / AES_BLOCK_LEN); 1793bbb2f537SJohn Baldwin } else 1794bbb2f537SJohn Baldwin memcpy(iv, hdr + 1, AES_BLOCK_LEN); 1795bbb2f537SJohn Baldwin out += AES_BLOCK_LEN; 1796bbb2f537SJohn Baldwin } 1797bbb2f537SJohn Baldwin 1798bbb2f537SJohn Baldwin if (imm_len % 16 != 0) { 1799bbb2f537SJohn Baldwin /* Zero pad to an 8-byte boundary. */ 1800bbb2f537SJohn Baldwin memset(out, 0, 8 - (imm_len % 8)); 1801bbb2f537SJohn Baldwin out += 8 - (imm_len % 8); 1802bbb2f537SJohn Baldwin 1803bbb2f537SJohn Baldwin /* 1804bbb2f537SJohn Baldwin * Insert a ULP_TX_SC_NOOP if needed so the SGL is 1805bbb2f537SJohn Baldwin * 16-byte aligned. 1806bbb2f537SJohn Baldwin */ 1807bbb2f537SJohn Baldwin if (imm_len % 16 <= 8) { 1808bbb2f537SJohn Baldwin idata = (void *)out; 1809bbb2f537SJohn Baldwin idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP)); 1810bbb2f537SJohn Baldwin idata->len = htobe32(0); 1811bbb2f537SJohn Baldwin out = (void *)(idata + 1); 1812bbb2f537SJohn Baldwin } 1813bbb2f537SJohn Baldwin } 1814bbb2f537SJohn Baldwin 1815bbb2f537SJohn Baldwin /* SGL for record payload */ 1816bbb2f537SJohn Baldwin sglist_reset(txq->gl); 1817bbb2f537SJohn Baldwin if (sglist_append_mbuf_epg(txq->gl, m_tls, m_tls->m_epg_hdrlen + offset, 1818bbb2f537SJohn Baldwin plen - (m_tls->m_epg_hdrlen + offset)) != 0) { 1819bbb2f537SJohn Baldwin #ifdef INVARIANTS 1820bbb2f537SJohn Baldwin panic("%s: failed to append sglist", __func__); 1821bbb2f537SJohn Baldwin #endif 1822bbb2f537SJohn Baldwin } 1823bbb2f537SJohn Baldwin write_gl_to_buf(txq->gl, out); 1824bbb2f537SJohn Baldwin 1825bbb2f537SJohn Baldwin if (using_scratch) { 1826bbb2f537SJohn Baldwin out = dst; 1827bbb2f537SJohn Baldwin copy_to_txd(eq, txq->ss, &out, wr_len); 1828bbb2f537SJohn Baldwin } 1829bbb2f537SJohn Baldwin 1830bbb2f537SJohn Baldwin ndesc += howmany(wr_len, EQ_ESIZE); 1831bbb2f537SJohn Baldwin MPASS(ndesc <= available); 1832bbb2f537SJohn Baldwin 1833bbb2f537SJohn Baldwin txq->kern_tls_records++; 1834bbb2f537SJohn Baldwin txq->kern_tls_octets += tlen - mtod(m_tls, vm_offset_t); 1835bbb2f537SJohn Baldwin if (mtod(m_tls, vm_offset_t) != 0) { 1836bbb2f537SJohn Baldwin if (offset == 0) 1837bbb2f537SJohn Baldwin txq->kern_tls_waste += mtod(m_tls, vm_offset_t); 1838bbb2f537SJohn Baldwin else 1839bbb2f537SJohn Baldwin txq->kern_tls_waste += mtod(m_tls, vm_offset_t) - 1840bbb2f537SJohn Baldwin (m_tls->m_epg_hdrlen + offset); 1841bbb2f537SJohn Baldwin } 1842bbb2f537SJohn Baldwin 1843bbb2f537SJohn Baldwin txsd = &txq->sdesc[pidx]; 1844bbb2f537SJohn Baldwin if (last_wr) 1845bbb2f537SJohn Baldwin txsd->m = m; 1846bbb2f537SJohn Baldwin else 1847bbb2f537SJohn Baldwin txsd->m = NULL; 1848bbb2f537SJohn Baldwin txsd->desc_used = howmany(wr_len, EQ_ESIZE); 1849bbb2f537SJohn Baldwin 1850bbb2f537SJohn Baldwin return (ndesc); 1851bbb2f537SJohn Baldwin } 1852bbb2f537SJohn Baldwin 1853bbb2f537SJohn Baldwin static int 1854bbb2f537SJohn Baldwin ktls_write_tcp_fin(struct sge_txq *txq, void *dst, struct mbuf *m, 1855bbb2f537SJohn Baldwin u_int available, tcp_seq tcp_seqno, u_int pidx) 1856bbb2f537SJohn Baldwin { 1857bbb2f537SJohn Baldwin struct tx_sdesc *txsd; 1858bbb2f537SJohn Baldwin struct fw_eth_tx_pkt_wr *wr; 1859bbb2f537SJohn Baldwin struct cpl_tx_pkt_core *cpl; 1860bbb2f537SJohn Baldwin uint32_t ctrl; 1861bbb2f537SJohn Baldwin uint64_t ctrl1; 1862bbb2f537SJohn Baldwin int len16, ndesc, pktlen; 1863bbb2f537SJohn Baldwin struct ether_header *eh; 1864bbb2f537SJohn Baldwin struct ip *ip, newip; 1865bbb2f537SJohn Baldwin struct ip6_hdr *ip6, newip6; 1866bbb2f537SJohn Baldwin struct tcphdr *tcp, newtcp; 1867bbb2f537SJohn Baldwin caddr_t out; 1868bbb2f537SJohn Baldwin 1869bbb2f537SJohn Baldwin TXQ_LOCK_ASSERT_OWNED(txq); 1870bbb2f537SJohn Baldwin M_ASSERTPKTHDR(m); 1871bbb2f537SJohn Baldwin 1872bbb2f537SJohn Baldwin wr = dst; 1873bbb2f537SJohn Baldwin pktlen = m->m_len; 1874bbb2f537SJohn Baldwin ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen; 1875bbb2f537SJohn Baldwin len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16); 1876bbb2f537SJohn Baldwin ndesc = tx_len16_to_desc(len16); 1877bbb2f537SJohn Baldwin MPASS(ndesc <= available); 1878bbb2f537SJohn Baldwin 1879bbb2f537SJohn Baldwin /* Firmware work request header */ 1880bbb2f537SJohn Baldwin wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 1881bbb2f537SJohn Baldwin V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl)); 1882bbb2f537SJohn Baldwin 1883bbb2f537SJohn Baldwin ctrl = V_FW_WR_LEN16(len16); 1884bbb2f537SJohn Baldwin wr->equiq_to_len16 = htobe32(ctrl); 1885bbb2f537SJohn Baldwin wr->r3 = 0; 1886bbb2f537SJohn Baldwin 1887bbb2f537SJohn Baldwin cpl = (void *)(wr + 1); 1888bbb2f537SJohn Baldwin 1889bbb2f537SJohn Baldwin /* CPL header */ 1890bbb2f537SJohn Baldwin cpl->ctrl0 = txq->cpl_ctrl0; 1891bbb2f537SJohn Baldwin cpl->pack = 0; 1892bbb2f537SJohn Baldwin cpl->len = htobe16(pktlen); 1893bbb2f537SJohn Baldwin 1894bbb2f537SJohn Baldwin out = (void *)(cpl + 1); 1895bbb2f537SJohn Baldwin 1896bbb2f537SJohn Baldwin /* Copy over Ethernet header. */ 1897bbb2f537SJohn Baldwin eh = mtod(m, struct ether_header *); 1898bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen); 1899bbb2f537SJohn Baldwin 1900bbb2f537SJohn Baldwin /* Fixup length in IP header and copy out. */ 1901bbb2f537SJohn Baldwin if (ntohs(eh->ether_type) == ETHERTYPE_IP) { 1902bbb2f537SJohn Baldwin ip = (void *)((char *)eh + m->m_pkthdr.l2hlen); 1903bbb2f537SJohn Baldwin newip = *ip; 1904bbb2f537SJohn Baldwin newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen); 1905bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip)); 1906bbb2f537SJohn Baldwin if (m->m_pkthdr.l3hlen > sizeof(*ip)) 1907bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out, 1908bbb2f537SJohn Baldwin m->m_pkthdr.l3hlen - sizeof(*ip)); 1909bbb2f537SJohn Baldwin ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP) | 1910bbb2f537SJohn Baldwin V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | 1911bbb2f537SJohn Baldwin V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); 1912bbb2f537SJohn Baldwin } else { 1913bbb2f537SJohn Baldwin ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen); 1914bbb2f537SJohn Baldwin newip6 = *ip6; 1915bbb2f537SJohn Baldwin newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen); 1916bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6)); 1917bbb2f537SJohn Baldwin MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6)); 1918bbb2f537SJohn Baldwin ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP6) | 1919bbb2f537SJohn Baldwin V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) | 1920bbb2f537SJohn Baldwin V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen); 1921bbb2f537SJohn Baldwin } 1922bbb2f537SJohn Baldwin cpl->ctrl1 = htobe64(ctrl1); 1923bbb2f537SJohn Baldwin txq->txcsum++; 1924bbb2f537SJohn Baldwin 1925bbb2f537SJohn Baldwin /* Set sequence number in TCP header. */ 1926bbb2f537SJohn Baldwin tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen); 1927bbb2f537SJohn Baldwin newtcp = *tcp; 1928bbb2f537SJohn Baldwin newtcp.th_seq = htonl(tcp_seqno); 1929bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp)); 1930bbb2f537SJohn Baldwin 1931bbb2f537SJohn Baldwin /* Copy rest of packet. */ 1932bbb2f537SJohn Baldwin copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len - 1933bbb2f537SJohn Baldwin (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp))); 1934bbb2f537SJohn Baldwin txq->imm_wrs++; 1935bbb2f537SJohn Baldwin 1936bbb2f537SJohn Baldwin txq->txpkt_wrs++; 1937bbb2f537SJohn Baldwin 1938bbb2f537SJohn Baldwin txq->kern_tls_fin++; 1939bbb2f537SJohn Baldwin 1940bbb2f537SJohn Baldwin txsd = &txq->sdesc[pidx]; 1941bbb2f537SJohn Baldwin txsd->m = m; 1942bbb2f537SJohn Baldwin txsd->desc_used = ndesc; 1943bbb2f537SJohn Baldwin 1944bbb2f537SJohn Baldwin return (ndesc); 1945bbb2f537SJohn Baldwin } 1946bbb2f537SJohn Baldwin 1947bbb2f537SJohn Baldwin int 1948ca457729SJohn Baldwin t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, 1949bbb2f537SJohn Baldwin u_int available) 1950bbb2f537SJohn Baldwin { 1951bbb2f537SJohn Baldwin struct sge_eq *eq = &txq->eq; 1952bbb2f537SJohn Baldwin struct tx_sdesc *txsd; 1953bbb2f537SJohn Baldwin struct tlspcb *tlsp; 1954bbb2f537SJohn Baldwin struct tcphdr *tcp; 1955bbb2f537SJohn Baldwin struct mbuf *m_tls; 1956bbb2f537SJohn Baldwin struct ether_header *eh; 1957bbb2f537SJohn Baldwin tcp_seq tcp_seqno; 1958bbb2f537SJohn Baldwin u_int ndesc, pidx, totdesc; 1959bbb2f537SJohn Baldwin uint16_t vlan_tag; 1960bbb2f537SJohn Baldwin bool has_fin, set_l2t_idx; 1961bbb2f537SJohn Baldwin void *tsopt; 1962bbb2f537SJohn Baldwin 1963bbb2f537SJohn Baldwin M_ASSERTPKTHDR(m); 1964bbb2f537SJohn Baldwin MPASS(m->m_pkthdr.snd_tag != NULL); 1965bbb2f537SJohn Baldwin tlsp = mst_to_tls(m->m_pkthdr.snd_tag); 1966bbb2f537SJohn Baldwin 1967bbb2f537SJohn Baldwin totdesc = 0; 1968bbb2f537SJohn Baldwin eh = mtod(m, struct ether_header *); 1969bbb2f537SJohn Baldwin tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen + 1970bbb2f537SJohn Baldwin m->m_pkthdr.l3hlen); 1971bbb2f537SJohn Baldwin pidx = eq->pidx; 1972bbb2f537SJohn Baldwin has_fin = (tcp->th_flags & TH_FIN) != 0; 1973bbb2f537SJohn Baldwin 1974bbb2f537SJohn Baldwin /* 1975bbb2f537SJohn Baldwin * If this TLS record has a FIN, then we will send any 1976bbb2f537SJohn Baldwin * requested options as part of the FIN packet. 1977bbb2f537SJohn Baldwin */ 1978bbb2f537SJohn Baldwin if (!has_fin && ktls_has_tcp_options(tcp)) { 1979bbb2f537SJohn Baldwin ndesc = ktls_write_tcp_options(txq, dst, m, available, pidx); 1980bbb2f537SJohn Baldwin totdesc += ndesc; 1981bbb2f537SJohn Baldwin IDXINCR(pidx, ndesc, eq->sidx); 1982bbb2f537SJohn Baldwin dst = &eq->desc[pidx]; 1983bbb2f537SJohn Baldwin #ifdef VERBOSE_TRACES 1984bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d wrote TCP options packet", __func__, 1985bbb2f537SJohn Baldwin tlsp->tid); 1986bbb2f537SJohn Baldwin #endif 1987bbb2f537SJohn Baldwin } 1988bbb2f537SJohn Baldwin 1989bbb2f537SJohn Baldwin /* 1990bbb2f537SJohn Baldwin * Allocate a new L2T entry if necessary. This may write out 1991bbb2f537SJohn Baldwin * a work request to the txq. 1992bbb2f537SJohn Baldwin */ 1993bbb2f537SJohn Baldwin if (m->m_flags & M_VLANTAG) 1994bbb2f537SJohn Baldwin vlan_tag = m->m_pkthdr.ether_vtag; 1995bbb2f537SJohn Baldwin else 1996bbb2f537SJohn Baldwin vlan_tag = 0xfff; 1997bbb2f537SJohn Baldwin set_l2t_idx = false; 1998bbb2f537SJohn Baldwin if (tlsp->l2te == NULL || tlsp->l2te->vlan != vlan_tag || 1999bbb2f537SJohn Baldwin memcmp(tlsp->l2te->dmac, eh->ether_dhost, ETHER_ADDR_LEN) != 0) { 2000bbb2f537SJohn Baldwin set_l2t_idx = true; 2001bbb2f537SJohn Baldwin if (tlsp->l2te) 2002bbb2f537SJohn Baldwin t4_l2t_release(tlsp->l2te); 2003bbb2f537SJohn Baldwin tlsp->l2te = t4_l2t_alloc_tls(tlsp->sc, txq, dst, &ndesc, 2004bbb2f537SJohn Baldwin vlan_tag, tlsp->vi->pi->lport, eh->ether_dhost); 2005bbb2f537SJohn Baldwin if (tlsp->l2te == NULL) 2006bbb2f537SJohn Baldwin CXGBE_UNIMPLEMENTED("failed to allocate TLS L2TE"); 2007bbb2f537SJohn Baldwin if (ndesc != 0) { 2008bbb2f537SJohn Baldwin MPASS(ndesc <= available - totdesc); 2009bbb2f537SJohn Baldwin 2010bbb2f537SJohn Baldwin txq->raw_wrs++; 2011bbb2f537SJohn Baldwin txsd = &txq->sdesc[pidx]; 2012bbb2f537SJohn Baldwin txsd->m = NULL; 2013bbb2f537SJohn Baldwin txsd->desc_used = ndesc; 2014bbb2f537SJohn Baldwin totdesc += ndesc; 2015bbb2f537SJohn Baldwin IDXINCR(pidx, ndesc, eq->sidx); 2016bbb2f537SJohn Baldwin dst = &eq->desc[pidx]; 2017bbb2f537SJohn Baldwin } 2018bbb2f537SJohn Baldwin } 2019bbb2f537SJohn Baldwin 2020bbb2f537SJohn Baldwin /* 2021bbb2f537SJohn Baldwin * Iterate over each TLS record constructing a work request 2022bbb2f537SJohn Baldwin * for that record. 2023bbb2f537SJohn Baldwin */ 2024bbb2f537SJohn Baldwin for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) { 2025bbb2f537SJohn Baldwin MPASS(m_tls->m_flags & M_EXTPG); 2026bbb2f537SJohn Baldwin 2027bbb2f537SJohn Baldwin /* 2028bbb2f537SJohn Baldwin * Determine the initial TCP sequence number for this 2029bbb2f537SJohn Baldwin * record. 2030bbb2f537SJohn Baldwin */ 2031bbb2f537SJohn Baldwin tsopt = NULL; 2032bbb2f537SJohn Baldwin if (m_tls == m->m_next) { 2033bbb2f537SJohn Baldwin tcp_seqno = ntohl(tcp->th_seq) - 2034bbb2f537SJohn Baldwin mtod(m_tls, vm_offset_t); 2035bbb2f537SJohn Baldwin if (tlsp->using_timestamps) 2036bbb2f537SJohn Baldwin tsopt = ktls_find_tcp_timestamps(tcp); 2037bbb2f537SJohn Baldwin } else { 2038bbb2f537SJohn Baldwin MPASS(mtod(m_tls, vm_offset_t) == 0); 2039bbb2f537SJohn Baldwin tcp_seqno = tlsp->prev_seq; 2040bbb2f537SJohn Baldwin } 2041bbb2f537SJohn Baldwin 2042bbb2f537SJohn Baldwin ndesc = ktls_write_tls_wr(tlsp, txq, dst, m, tcp, m_tls, 2043ca457729SJohn Baldwin available - totdesc, tcp_seqno, tsopt, pidx, set_l2t_idx); 2044bbb2f537SJohn Baldwin totdesc += ndesc; 2045bbb2f537SJohn Baldwin IDXINCR(pidx, ndesc, eq->sidx); 2046bbb2f537SJohn Baldwin dst = &eq->desc[pidx]; 2047bbb2f537SJohn Baldwin 2048bbb2f537SJohn Baldwin /* Only need to set the L2T index once. */ 2049bbb2f537SJohn Baldwin set_l2t_idx = false; 2050bbb2f537SJohn Baldwin } 2051bbb2f537SJohn Baldwin 2052bbb2f537SJohn Baldwin if (has_fin) { 2053bbb2f537SJohn Baldwin /* 2054bbb2f537SJohn Baldwin * If the TCP header for this chain has FIN sent, then 2055bbb2f537SJohn Baldwin * explicitly send a packet that has FIN set. This 2056bbb2f537SJohn Baldwin * will also have PUSH set if requested. This assumes 2057bbb2f537SJohn Baldwin * we sent at least one TLS record work request and 2058bbb2f537SJohn Baldwin * uses the TCP sequence number after that reqeust as 2059bbb2f537SJohn Baldwin * the sequence number for the FIN packet. 2060bbb2f537SJohn Baldwin */ 2061bbb2f537SJohn Baldwin ndesc = ktls_write_tcp_fin(txq, dst, m, available, 2062bbb2f537SJohn Baldwin tlsp->prev_seq, pidx); 2063bbb2f537SJohn Baldwin totdesc += ndesc; 2064bbb2f537SJohn Baldwin } 2065bbb2f537SJohn Baldwin 2066bbb2f537SJohn Baldwin MPASS(totdesc <= available); 2067bbb2f537SJohn Baldwin return (totdesc); 2068bbb2f537SJohn Baldwin } 2069bbb2f537SJohn Baldwin 2070bbb2f537SJohn Baldwin static void 2071bbb2f537SJohn Baldwin t6_tls_tag_free(struct m_snd_tag *mst) 2072bbb2f537SJohn Baldwin { 2073bbb2f537SJohn Baldwin struct adapter *sc; 2074bbb2f537SJohn Baldwin struct tlspcb *tlsp; 2075bbb2f537SJohn Baldwin 2076bbb2f537SJohn Baldwin tlsp = mst_to_tls(mst); 2077bbb2f537SJohn Baldwin sc = tlsp->sc; 2078bbb2f537SJohn Baldwin 2079bbb2f537SJohn Baldwin CTR2(KTR_CXGBE, "%s: tid %d", __func__, tlsp->tid); 2080bbb2f537SJohn Baldwin 2081bbb2f537SJohn Baldwin if (tlsp->l2te) 2082bbb2f537SJohn Baldwin t4_l2t_release(tlsp->l2te); 2083bbb2f537SJohn Baldwin if (tlsp->tid >= 0) 2084bbb2f537SJohn Baldwin release_tid(sc, tlsp->tid, tlsp->ctrlq); 2085bbb2f537SJohn Baldwin if (tlsp->ce) 2086bbb2f537SJohn Baldwin t4_release_clip_entry(sc, tlsp->ce); 2087bbb2f537SJohn Baldwin if (tlsp->tx_key_addr >= 0) 2088bbb2f537SJohn Baldwin t4_free_tls_keyid(sc, tlsp->tx_key_addr); 2089bbb2f537SJohn Baldwin 2090bbb2f537SJohn Baldwin zfree(tlsp, M_CXGBE); 2091bbb2f537SJohn Baldwin } 2092bbb2f537SJohn Baldwin 2093bbb2f537SJohn Baldwin void 2094bbb2f537SJohn Baldwin t6_ktls_modload(void) 2095bbb2f537SJohn Baldwin { 2096bbb2f537SJohn Baldwin 2097bbb2f537SJohn Baldwin t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, ktls_act_open_rpl, 2098bbb2f537SJohn Baldwin CPL_COOKIE_KERN_TLS); 2099bbb2f537SJohn Baldwin } 2100bbb2f537SJohn Baldwin 2101bbb2f537SJohn Baldwin void 2102bbb2f537SJohn Baldwin t6_ktls_modunload(void) 2103bbb2f537SJohn Baldwin { 2104bbb2f537SJohn Baldwin 2105bbb2f537SJohn Baldwin t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, NULL, 2106bbb2f537SJohn Baldwin CPL_COOKIE_KERN_TLS); 2107bbb2f537SJohn Baldwin } 2108bbb2f537SJohn Baldwin 2109bbb2f537SJohn Baldwin #else 2110bbb2f537SJohn Baldwin 2111bbb2f537SJohn Baldwin int 2112bbb2f537SJohn Baldwin t6_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, 2113bbb2f537SJohn Baldwin struct m_snd_tag **pt) 2114bbb2f537SJohn Baldwin { 2115bbb2f537SJohn Baldwin return (ENXIO); 2116bbb2f537SJohn Baldwin } 2117bbb2f537SJohn Baldwin 2118bbb2f537SJohn Baldwin int 2119ca457729SJohn Baldwin t6_ktls_parse_pkt(struct mbuf *m) 2120bbb2f537SJohn Baldwin { 2121bbb2f537SJohn Baldwin return (EINVAL); 2122bbb2f537SJohn Baldwin } 2123bbb2f537SJohn Baldwin 2124bbb2f537SJohn Baldwin int 2125ca457729SJohn Baldwin t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, 2126bbb2f537SJohn Baldwin u_int available) 2127bbb2f537SJohn Baldwin { 2128bbb2f537SJohn Baldwin panic("can't happen"); 2129bbb2f537SJohn Baldwin } 2130bbb2f537SJohn Baldwin 2131bbb2f537SJohn Baldwin void 2132bbb2f537SJohn Baldwin t6_ktls_modload(void) 2133bbb2f537SJohn Baldwin { 2134bbb2f537SJohn Baldwin } 2135bbb2f537SJohn Baldwin 2136bbb2f537SJohn Baldwin void 2137bbb2f537SJohn Baldwin t6_ktls_modunload(void) 2138bbb2f537SJohn Baldwin { 2139bbb2f537SJohn Baldwin } 2140bbb2f537SJohn Baldwin 2141bbb2f537SJohn Baldwin #endif 2142