1 /*-
2 * Copyright (c) 2020-2021, Mellanox Technologies, Ltd.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_inet.h"
27 #include "opt_inet6.h"
28
29 #include <dev/mlx5/mlx5_en/en.h>
30
31 #include <dev/mlx5/mlx5_core/fs_core.h>
32 #include <dev/mlx5/mlx5_core/fs_tcp.h>
33 #include <dev/mlx5/device.h>
34
35 #include <sys/domain.h>
36
37 #include <netinet/in_pcb.h>
38
39 #if defined(INET) || defined(INET6)
40 static void
accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec * spec,struct inpcb * inp)41 accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct inpcb *inp)
42 {
43 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
44 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
45 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
46 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
47 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
48 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
49 &inp->inp_faddr, 4);
50 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
51 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
52 &inp->inp_laddr, 4);
53 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
54 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
55 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
56 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
57 }
58 #endif
59
60 #ifdef INET6
61 static void
accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec * spec,struct inpcb * inp)62 accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct inpcb *inp)
63 {
64 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
65 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
66 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
67 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
68 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
69 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
70 &inp->in6p_faddr, 16);
71 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
72 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
73 &inp->in6p_laddr, 16);
74 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
75 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
76 0xff, 16);
77 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
78 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
79 0xff, 16);
80 }
81 #endif
82
83 void
mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule * rule)84 mlx5e_accel_fs_del_inpcb(struct mlx5_flow_rule *rule)
85 {
86 mlx5_del_flow_rule(&rule);
87 }
88
89 struct mlx5_flow_rule *
mlx5e_accel_fs_add_inpcb(struct mlx5e_priv * priv,struct inpcb * inp,uint32_t tirn,uint32_t flow_tag,uint16_t vlan_id)90 mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv,
91 struct inpcb *inp, uint32_t tirn, uint32_t flow_tag,
92 uint16_t vlan_id)
93 {
94 struct mlx5_flow_destination dest = {};
95 struct mlx5e_flow_table *ft = NULL;
96 #if defined(INET) || defined(INET6)
97 struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp;
98 #endif
99 struct mlx5_flow_rule *flow;
100 struct mlx5_flow_spec *spec;
101 struct mlx5_flow_act flow_act = {
102 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
103 .flow_tag = flow_tag,
104 };
105
106 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
107 if (!spec)
108 return (ERR_PTR(-ENOMEM));
109
110 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
111
112 INP_RLOCK(inp);
113 /* Set VLAN ID to match, if any. */
114 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
115 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid);
116 if (vlan_id != MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN) {
117 MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);
118 MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vlan_id);
119 }
120
121 /* Set TCP port numbers. */
122 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
123 outer_headers.tcp_dport);
124 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
125 outer_headers.tcp_sport);
126 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
127 ntohs(inp->inp_lport));
128 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
129 ntohs(inp->inp_fport));
130
131 /* Set IP addresses. */
132 switch (INP_SOCKAF(inp->inp_socket)) {
133 #ifdef INET
134 case AF_INET:
135 accel_fs_tcp_set_ipv4_flow(spec, inp);
136 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP];
137 break;
138 #endif
139 #ifdef INET6
140 case AF_INET6:
141 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
142 IN6_IS_ADDR_V4MAPPED(&inp->in6p_faddr)) {
143 accel_fs_tcp_set_ipv4_flow(spec, inp);
144 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP];
145 } else {
146 accel_fs_tcp_set_ipv6_flow(spec, inp);
147 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV6_TCP];
148 }
149 break;
150 #endif
151 default:
152 break;
153 }
154 INP_RUNLOCK(inp);
155
156 if (!ft) {
157 flow = ERR_PTR(-EINVAL);
158 goto out;
159 }
160
161 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
162 dest.tir_num = tirn;
163
164 flow = mlx5_add_flow_rule(ft->t, spec->match_criteria_enable,
165 spec->match_criteria,
166 spec->match_value,
167 MLX5_FLOW_RULE_FWD_ACTION_DEST,
168 &flow_act,
169 &dest);
170 out:
171 kvfree(spec);
172 return (flow);
173 }
174
175 static int
accel_fs_tcp_add_default_rule(struct mlx5e_priv * priv,int type)176 accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type)
177 {
178 static u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
179 static u32 match_value[MLX5_ST_SZ_DW(fte_match_param)];
180 struct mlx5_flow_destination dest = {};
181 struct mlx5e_accel_fs_tcp *fs_tcp;
182 struct mlx5_flow_rule *rule;
183 struct mlx5_flow_act flow_act = {
184 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
185 .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
186 };
187
188 fs_tcp = &priv->fts.accel_tcp;
189
190 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
191
192 /*
193 * Traffic not matched by flow table rules should be forwarded
194 * to the next flow table in order to not be dropped by the
195 * default action. Refer to the diagram in
196 * mlx5_en_flow_table.c for more information about the order
197 * of flow tables.
198 */
199 dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ?
200 priv->fts.vlan.t : fs_tcp->tables[type + 1].t;
201
202 rule = mlx5_add_flow_rule(fs_tcp->tables[type].t, 0, match_criteria, match_value,
203 MLX5_FLOW_RULE_FWD_ACTION_DEST, &flow_act, &dest);
204 if (IS_ERR(rule))
205 return (PTR_ERR(rule));
206
207 fs_tcp->default_rules[type] = rule;
208 return (0);
209 }
210
211 #define MLX5E_ACCEL_FS_TCP_NUM_GROUPS (2)
212 #define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE (BIT(16) - 1)
213 #define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE (BIT(0))
214 #define MLX5E_ACCEL_FS_TCP_TABLE_SIZE (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\
215 MLX5E_ACCEL_FS_TCP_GROUP2_SIZE)
216 static int
accel_fs_tcp_create_groups(struct mlx5e_flow_table * ft,int type)217 accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, int type)
218 {
219 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
220 void *outer_headers_c;
221 int ix = 0;
222 u32 *in;
223 int err;
224 u8 *mc;
225
226 ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
227 in = kvzalloc(inlen, GFP_KERNEL);
228 if (!in || !ft->g) {
229 kfree(ft->g);
230 kvfree(in);
231 return (-ENOMEM);
232 }
233
234 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
235 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
236 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
237 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
238 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, cvlan_tag);
239 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, first_vid);
240
241 switch (type) {
242 case MLX5E_ACCEL_FS_IPV4_TCP:
243 case MLX5E_ACCEL_FS_IPV6_TCP:
244 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
245 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
246 break;
247 default:
248 err = -EINVAL;
249 goto out;
250 }
251
252 switch (type) {
253 case MLX5E_ACCEL_FS_IPV4_TCP:
254 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
255 src_ipv4_src_ipv6.ipv4_layout.ipv4);
256 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
257 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
258 break;
259 case MLX5E_ACCEL_FS_IPV6_TCP:
260 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
261 src_ipv4_src_ipv6.ipv6_layout.ipv6),
262 0xff, 16);
263 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
264 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
265 0xff, 16);
266 break;
267 default:
268 err = -EINVAL;
269 goto out;
270 }
271
272 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
273 MLX5_SET_CFG(in, start_flow_index, ix);
274 ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE;
275 MLX5_SET_CFG(in, end_flow_index, ix - 1);
276 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
277 if (IS_ERR(ft->g[ft->num_groups]))
278 goto err;
279 ft->num_groups++;
280
281 /* Default Flow Group */
282 memset(in, 0, inlen);
283 MLX5_SET_CFG(in, start_flow_index, ix);
284 ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE;
285 MLX5_SET_CFG(in, end_flow_index, ix - 1);
286 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
287 if (IS_ERR(ft->g[ft->num_groups]))
288 goto err;
289 ft->num_groups++;
290
291 kvfree(in);
292 return (0);
293
294 err:
295 err = PTR_ERR(ft->g[ft->num_groups]);
296 ft->g[ft->num_groups] = NULL;
297 out:
298 kvfree(in);
299
300 return (err);
301 }
302
303 static void
accel_fs_tcp_destroy_groups(struct mlx5e_flow_table * ft)304 accel_fs_tcp_destroy_groups(struct mlx5e_flow_table *ft)
305 {
306 int i;
307
308 for (i = ft->num_groups - 1; i >= 0; i--) {
309 if (!IS_ERR_OR_NULL(ft->g[i]))
310 mlx5_destroy_flow_group(ft->g[i]);
311 ft->g[i] = NULL;
312 }
313 ft->num_groups = 0;
314 }
315
316 static int
accel_fs_tcp_create_table(struct mlx5e_priv * priv,int type)317 accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type)
318 {
319 struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type];
320 int err;
321
322 ft->num_groups = 0;
323 ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, 0, "tcp",
324 MLX5E_ACCEL_FS_TCP_TABLE_SIZE);
325 if (IS_ERR(ft->t)) {
326 err = PTR_ERR(ft->t);
327 ft->t = NULL;
328 return (err);
329 }
330
331 err = accel_fs_tcp_create_groups(ft, type);
332 if (err)
333 goto err_destroy_flow_table;
334
335 return (0);
336
337 err_destroy_flow_table:
338 mlx5_destroy_flow_table(ft->t);
339 ft->t = NULL;
340 return (err);
341 }
342
343 static void
accel_fs_tcp_destroy_table(struct mlx5e_priv * priv,int i)344 accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i)
345 {
346 struct mlx5e_accel_fs_tcp *fs_tcp;
347 struct mlx5e_flow_table *ft;
348
349 fs_tcp = &priv->fts.accel_tcp;
350 ft = fs_tcp->tables + i;
351
352 accel_fs_tcp_destroy_groups(ft);
353 kfree(ft->g);
354 ft->g = NULL;
355 mlx5_destroy_flow_table(ft->t);
356 ft->t = NULL;
357 }
358
359 void
mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv * priv)360 mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv)
361 {
362 int i;
363
364 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
365 return;
366
367 for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
368 mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]);
369 accel_fs_tcp_destroy_table(priv, i);
370 }
371 }
372
373 int
mlx5e_accel_fs_tcp_create(struct mlx5e_priv * priv)374 mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv)
375 {
376 int i, err;
377
378 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
379 return (0);
380
381 /* Setup namespace pointer. */
382 priv->fts.accel_tcp.ns = mlx5_get_flow_namespace(
383 priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS);
384
385 /*
386 * Create flow tables first, because the priority level is
387 * assigned at allocation time.
388 */
389 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
390 err = accel_fs_tcp_create_table(priv, i);
391 if (err)
392 goto err_destroy_tables;
393 }
394
395 /* Create default rules last. */
396 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
397 err = accel_fs_tcp_add_default_rule(priv, i);
398 if (err)
399 goto err_destroy_rules;
400 }
401 return (0);
402
403 err_destroy_rules:
404 while (i--)
405 mlx5_del_flow_rule(&priv->fts.accel_tcp.default_rules[i]);
406 i = MLX5E_ACCEL_FS_TCP_NUM_TYPES;
407
408 err_destroy_tables:
409 while (i--)
410 accel_fs_tcp_destroy_table(priv, i);
411 return (err);
412 }
413