1 /*- 2 * Copyright (c) 2020-2021, Mellanox Technologies, Ltd. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include "opt_inet.h" 27 #include "opt_inet6.h" 28 29 #include <dev/mlx5/mlx5_en/en.h> 30 31 #include <dev/mlx5/mlx5_core/fs_core.h> 32 #include <dev/mlx5/mlx5_core/fs_tcp.h> 33 #include <dev/mlx5/device.h> 34 35 #include <sys/domain.h> 36 37 #include <netinet/in_pcb.h> 38 39 #if defined(INET) || defined(INET6) 40 static void 41 accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct inpcb *inp) 42 { 43 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 44 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); 45 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 46 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 47 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 48 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 49 &inp->inp_faddr, 4); 50 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 51 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 52 &inp->inp_laddr, 4); 53 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 54 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 55 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 56 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 57 } 58 #endif 59 60 #ifdef INET6 61 static void 62 accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct inpcb *inp) 63 { 64 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 65 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); 66 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 67 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 68 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 69 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 70 &inp->in6p_faddr, 16); 71 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 72 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 73 &inp->in6p_laddr, 16); 74 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 75 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 76 0xff, 16); 77 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 78 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 79 0xff, 16); 80 } 81 #endif 82 83 void 84 mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *rule) 85 { 86 mlx5_del_flow_rule(rule); 87 } 88 89 struct mlx5_flow_rule * 90 mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv, 91 struct inpcb *inp, uint32_t tirn, uint32_t flow_tag, 92 uint16_t vlan_id) 93 { 94 struct mlx5_flow_destination dest = {}; 95 struct mlx5e_flow_table *ft = NULL; 96 #if defined(INET) || defined(INET6) 97 struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp; 98 #endif 99 struct mlx5_flow_rule *flow; 100 struct mlx5_flow_spec *spec; 101 102 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 103 if (!spec) 104 return (ERR_PTR(-ENOMEM)); 105 106 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 107 108 INP_RLOCK(inp); 109 /* Set VLAN ID to match, if any. */ 110 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); 111 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid); 112 if (vlan_id != MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN) { 113 MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag); 114 MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vlan_id); 115 } 116 117 /* Set TCP port numbers. */ 118 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 119 outer_headers.tcp_dport); 120 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 121 outer_headers.tcp_sport); 122 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, 123 ntohs(inp->inp_lport)); 124 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, 125 ntohs(inp->inp_fport)); 126 127 /* Set IP addresses. */ 128 switch (INP_SOCKAF(inp->inp_socket)) { 129 #ifdef INET 130 case AF_INET: 131 accel_fs_tcp_set_ipv4_flow(spec, inp); 132 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP]; 133 break; 134 #endif 135 #ifdef INET6 136 case AF_INET6: 137 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 && 138 IN6_IS_ADDR_V4MAPPED(&inp->in6p_faddr)) { 139 accel_fs_tcp_set_ipv4_flow(spec, inp); 140 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP]; 141 } else { 142 accel_fs_tcp_set_ipv6_flow(spec, inp); 143 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV6_TCP]; 144 } 145 break; 146 #endif 147 default: 148 break; 149 } 150 INP_RUNLOCK(inp); 151 152 if (!ft) { 153 flow = ERR_PTR(-EINVAL); 154 goto out; 155 } 156 157 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 158 dest.tir_num = tirn; 159 160 flow = mlx5_add_flow_rule(ft->t, spec->match_criteria_enable, 161 spec->match_criteria, 162 spec->match_value, 163 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, 164 flow_tag, 165 &dest); 166 out: 167 kvfree(spec); 168 return (flow); 169 } 170 171 static int 172 accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type) 173 { 174 static u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)]; 175 static u32 match_value[MLX5_ST_SZ_DW(fte_match_param)]; 176 struct mlx5_flow_destination dest = {}; 177 struct mlx5e_accel_fs_tcp *fs_tcp; 178 struct mlx5_flow_rule *rule; 179 180 fs_tcp = &priv->fts.accel_tcp; 181 182 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 183 184 /* 185 * Traffic not matched by flow table rules should be forwarded 186 * to the next flow table in order to not be dropped by the 187 * default action. Refer to the diagram in 188 * mlx5_en_flow_table.c for more information about the order 189 * of flow tables. 190 */ 191 dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ? 192 priv->fts.vlan.t : fs_tcp->tables[type + 1].t; 193 194 rule = mlx5_add_flow_rule(fs_tcp->tables[type].t, 0, match_criteria, match_value, 195 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, MLX5_FS_DEFAULT_FLOW_TAG, &dest); 196 if (IS_ERR(rule)) 197 return (PTR_ERR(rule)); 198 199 fs_tcp->default_rules[type] = rule; 200 return (0); 201 } 202 203 #define MLX5E_ACCEL_FS_TCP_NUM_GROUPS (2) 204 #define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE (BIT(16) - 1) 205 #define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE (BIT(0)) 206 #define MLX5E_ACCEL_FS_TCP_TABLE_SIZE (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\ 207 MLX5E_ACCEL_FS_TCP_GROUP2_SIZE) 208 static int 209 accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, int type) 210 { 211 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 212 void *outer_headers_c; 213 int ix = 0; 214 u32 *in; 215 int err; 216 u8 *mc; 217 218 ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); 219 in = kvzalloc(inlen, GFP_KERNEL); 220 if (!in || !ft->g) { 221 kfree(ft->g); 222 kvfree(in); 223 return (-ENOMEM); 224 } 225 226 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 227 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 228 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 229 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version); 230 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, cvlan_tag); 231 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, first_vid); 232 233 switch (type) { 234 case MLX5E_ACCEL_FS_IPV4_TCP: 235 case MLX5E_ACCEL_FS_IPV6_TCP: 236 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); 237 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); 238 break; 239 default: 240 err = -EINVAL; 241 goto out; 242 } 243 244 switch (type) { 245 case MLX5E_ACCEL_FS_IPV4_TCP: 246 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 247 src_ipv4_src_ipv6.ipv4_layout.ipv4); 248 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 249 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 250 break; 251 case MLX5E_ACCEL_FS_IPV6_TCP: 252 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 253 src_ipv4_src_ipv6.ipv6_layout.ipv6), 254 0xff, 16); 255 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 256 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 257 0xff, 16); 258 break; 259 default: 260 err = -EINVAL; 261 goto out; 262 } 263 264 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 265 MLX5_SET_CFG(in, start_flow_index, ix); 266 ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE; 267 MLX5_SET_CFG(in, end_flow_index, ix - 1); 268 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 269 if (IS_ERR(ft->g[ft->num_groups])) 270 goto err; 271 ft->num_groups++; 272 273 /* Default Flow Group */ 274 memset(in, 0, inlen); 275 MLX5_SET_CFG(in, start_flow_index, ix); 276 ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE; 277 MLX5_SET_CFG(in, end_flow_index, ix - 1); 278 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 279 if (IS_ERR(ft->g[ft->num_groups])) 280 goto err; 281 ft->num_groups++; 282 283 kvfree(in); 284 return (0); 285 286 err: 287 err = PTR_ERR(ft->g[ft->num_groups]); 288 ft->g[ft->num_groups] = NULL; 289 out: 290 kvfree(in); 291 292 return (err); 293 } 294 295 static void 296 accel_fs_tcp_destroy_groups(struct mlx5e_flow_table *ft) 297 { 298 int i; 299 300 for (i = ft->num_groups - 1; i >= 0; i--) { 301 if (!IS_ERR_OR_NULL(ft->g[i])) 302 mlx5_destroy_flow_group(ft->g[i]); 303 ft->g[i] = NULL; 304 } 305 ft->num_groups = 0; 306 } 307 308 static int 309 accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type) 310 { 311 struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type]; 312 int err; 313 314 ft->num_groups = 0; 315 ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, 0, "tcp", 316 MLX5E_ACCEL_FS_TCP_TABLE_SIZE); 317 if (IS_ERR(ft->t)) { 318 err = PTR_ERR(ft->t); 319 ft->t = NULL; 320 return (err); 321 } 322 323 err = accel_fs_tcp_create_groups(ft, type); 324 if (err) 325 goto err_destroy_flow_table; 326 327 return (0); 328 329 err_destroy_flow_table: 330 mlx5_destroy_flow_table(ft->t); 331 ft->t = NULL; 332 return (err); 333 } 334 335 static void 336 accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) 337 { 338 struct mlx5e_accel_fs_tcp *fs_tcp; 339 struct mlx5e_flow_table *ft; 340 341 fs_tcp = &priv->fts.accel_tcp; 342 ft = fs_tcp->tables + i; 343 344 mlx5_del_flow_rule(fs_tcp->default_rules[i]); 345 346 accel_fs_tcp_destroy_groups(ft); 347 kfree(ft->g); 348 ft->g = NULL; 349 mlx5_destroy_flow_table(ft->t); 350 ft->t = NULL; 351 } 352 353 void 354 mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) 355 { 356 int i; 357 358 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) 359 return; 360 361 for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) 362 accel_fs_tcp_destroy_table(priv, i); 363 } 364 365 int 366 mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) 367 { 368 int i, err; 369 370 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) 371 return (0); 372 373 /* Setup namespace pointer. */ 374 priv->fts.accel_tcp.ns = mlx5_get_flow_namespace( 375 priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS); 376 377 /* 378 * Create flow tables first, because the priority level is 379 * assigned at allocation time. 380 */ 381 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { 382 err = accel_fs_tcp_create_table(priv, i); 383 if (err) 384 goto err_destroy_tables; 385 } 386 387 /* Create default rules last. */ 388 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { 389 err = accel_fs_tcp_add_default_rule(priv, i); 390 if (err) 391 goto err_destroy_rules; 392 } 393 return (0); 394 395 err_destroy_rules: 396 while (i--) 397 mlx5_del_flow_rule(priv->fts.accel_tcp.default_rules[i]); 398 i = MLX5E_ACCEL_FS_TCP_NUM_TYPES; 399 400 err_destroy_tables: 401 while (i--) 402 accel_fs_tcp_destroy_table(priv, i); 403 return (err); 404 } 405