1 /*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/etherdevice.h>
34 #include <linux/idr.h>
35 #include <linux/mlx5/driver.h>
36 #include <linux/mlx5/mlx5_ifc.h>
37 #include <linux/mlx5/vport.h>
38 #include <linux/mlx5/fs.h>
39 #include "mlx5_core.h"
40 #include "eswitch.h"
41 #include "esw/indir_table.h"
42 #include "esw/acl/ofld.h"
43 #include "rdma.h"
44 #include "en.h"
45 #include "fs_core.h"
46 #include "lib/mlx5.h"
47 #include "lib/devcom.h"
48 #include "lib/eq.h"
49 #include "lib/fs_chains.h"
50 #include "en_tc.h"
51 #include "en/mapping.h"
52 #include "devlink.h"
53 #include "lag/lag.h"
54 #include "en/tc/post_meter.h"
55
56 #define mlx5_esw_for_each_rep(esw, i, rep) \
57 xa_for_each(&((esw)->offloads.vport_reps), i, rep)
58
59 /* There are two match-all miss flows, one for unicast dst mac and
60 * one for multicast.
61 */
62 #define MLX5_ESW_MISS_FLOWS (2)
63 #define UPLINK_REP_INDEX 0
64
65 #define MLX5_ESW_VPORT_TBL_SIZE 128
66 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
67
68 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
69
70 #define MLX5_ESW_MAX_CTRL_EQS 4
71
72 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
73 .max_fte = MLX5_ESW_VPORT_TBL_SIZE,
74 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
75 .flags = 0,
76 };
77
mlx5_eswitch_get_rep(struct mlx5_eswitch * esw,u16 vport_num)78 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
79 u16 vport_num)
80 {
81 return xa_load(&esw->offloads.vport_reps, vport_num);
82 }
83
84 static void
mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr)85 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
86 struct mlx5_flow_spec *spec,
87 struct mlx5_esw_flow_attr *attr)
88 {
89 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep)
90 return;
91
92 if (attr->int_port) {
93 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port);
94
95 return;
96 }
97
98 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ?
99 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
100 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
101 }
102
103 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
104 * are not needed as well in the following process. So clear them all for simplicity.
105 */
106 void
mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec)107 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
108 {
109 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
110 void *misc2;
111
112 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
113 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
114
115 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
116 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
117
118 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
119 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
120 }
121 }
122
123 static void
mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr,struct mlx5_eswitch * src_esw,u16 vport)124 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
125 struct mlx5_flow_spec *spec,
126 struct mlx5_flow_attr *attr,
127 struct mlx5_eswitch *src_esw,
128 u16 vport)
129 {
130 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
131 u32 metadata;
132 void *misc2;
133 void *misc;
134
135 /* Use metadata matching because vport is not represented by single
136 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
137 */
138 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
139 if (mlx5_esw_indir_table_decap_vport(attr))
140 vport = mlx5_esw_indir_table_decap_vport(attr);
141
142 if (!attr->chain && esw_attr && esw_attr->int_port)
143 metadata =
144 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port);
145 else
146 metadata =
147 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport);
148
149 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
150 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata);
151
152 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
153 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
154 mlx5_eswitch_get_vport_metadata_mask());
155
156 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
157 } else {
158 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
159 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
160
161 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
162 MLX5_SET(fte_match_set_misc, misc,
163 source_eswitch_owner_vhca_id,
164 MLX5_CAP_GEN(src_esw->dev, vhca_id));
165
166 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
167 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
168 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
169 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
170 source_eswitch_owner_vhca_id);
171
172 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
173 }
174 }
175
176 static int
esw_setup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)177 esw_setup_decap_indir(struct mlx5_eswitch *esw,
178 struct mlx5_flow_attr *attr)
179 {
180 struct mlx5_flow_table *ft;
181
182 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
183 return -EOPNOTSUPP;
184
185 ft = mlx5_esw_indir_table_get(esw, attr,
186 mlx5_esw_indir_table_decap_vport(attr), true);
187 return PTR_ERR_OR_ZERO(ft);
188 }
189
190 static void
esw_cleanup_decap_indir(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)191 esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
192 struct mlx5_flow_attr *attr)
193 {
194 if (mlx5_esw_indir_table_decap_vport(attr))
195 mlx5_esw_indir_table_put(esw,
196 mlx5_esw_indir_table_decap_vport(attr),
197 true);
198 }
199
200 static int
esw_setup_mtu_dest(struct mlx5_flow_destination * dest,struct mlx5e_meter_attr * meter,int i)201 esw_setup_mtu_dest(struct mlx5_flow_destination *dest,
202 struct mlx5e_meter_attr *meter,
203 int i)
204 {
205 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE;
206 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN;
207 dest[i].range.min = 0;
208 dest[i].range.max = meter->params.mtu;
209 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter);
210 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter);
211
212 return 0;
213 }
214
215 static int
esw_setup_sampler_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,u32 sampler_id,int i)216 esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
217 struct mlx5_flow_act *flow_act,
218 u32 sampler_id,
219 int i)
220 {
221 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
222 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
223 dest[i].sampler_id = sampler_id;
224
225 return 0;
226 }
227
228 static int
esw_setup_ft_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int i)229 esw_setup_ft_dest(struct mlx5_flow_destination *dest,
230 struct mlx5_flow_act *flow_act,
231 struct mlx5_eswitch *esw,
232 struct mlx5_flow_attr *attr,
233 int i)
234 {
235 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
236 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
237 dest[i].ft = attr->dest_ft;
238
239 if (mlx5_esw_indir_table_decap_vport(attr))
240 return esw_setup_decap_indir(esw, attr);
241 return 0;
242 }
243
244 static void
esw_setup_accept_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,int i)245 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
246 struct mlx5_fs_chains *chains, int i)
247 {
248 if (mlx5_chains_ignore_flow_level_supported(chains))
249 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
250 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
251 dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
252 }
253
254 static void
esw_setup_slow_path_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,int i)255 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
256 struct mlx5_eswitch *esw, int i)
257 {
258 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level))
259 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
260 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
261 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw);
262 }
263
264 static int
esw_setup_chain_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level,int i)265 esw_setup_chain_dest(struct mlx5_flow_destination *dest,
266 struct mlx5_flow_act *flow_act,
267 struct mlx5_fs_chains *chains,
268 u32 chain, u32 prio, u32 level,
269 int i)
270 {
271 struct mlx5_flow_table *ft;
272
273 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
274 ft = mlx5_chains_get_table(chains, chain, prio, level);
275 if (IS_ERR(ft))
276 return PTR_ERR(ft);
277
278 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
279 dest[i].ft = ft;
280 return 0;
281 }
282
esw_put_dest_tables_loop(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int from,int to)283 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
284 int from, int to)
285 {
286 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
287 struct mlx5_fs_chains *chains = esw_chains(esw);
288 int i;
289
290 for (i = from; i < to; i++)
291 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
292 mlx5_chains_put_table(chains, 0, 1, 0);
293 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport,
294 esw_attr->dests[i].mdev))
295 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false);
296 }
297
298 static bool
esw_is_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)299 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
300 {
301 int i;
302
303 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
304 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
305 return true;
306 return false;
307 }
308
309 static int
esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains,struct mlx5_flow_attr * attr,int * i)310 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
311 struct mlx5_flow_act *flow_act,
312 struct mlx5_eswitch *esw,
313 struct mlx5_fs_chains *chains,
314 struct mlx5_flow_attr *attr,
315 int *i)
316 {
317 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
318 int err;
319
320 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
321 return -EOPNOTSUPP;
322
323 /* flow steering cannot handle more than one dest with the same ft
324 * in a single flow
325 */
326 if (esw_attr->out_count - esw_attr->split_count > 1)
327 return -EOPNOTSUPP;
328
329 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
330 if (err)
331 return err;
332
333 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) {
334 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
335 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat;
336 }
337 (*i)++;
338
339 return 0;
340 }
341
esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)342 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
343 struct mlx5_flow_attr *attr)
344 {
345 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
346
347 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
348 }
349
350 static bool
esw_is_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)351 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
352 {
353 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
354 bool result = false;
355 int i;
356
357 /* Indirect table is supported only for flows with in_port uplink
358 * and the destination is vport on the same eswitch as the uplink,
359 * return false in case at least one of destinations doesn't meet
360 * this criteria.
361 */
362 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) {
363 if (esw_attr->dests[i].vport_valid &&
364 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport,
365 esw_attr->dests[i].mdev)) {
366 result = true;
367 } else {
368 result = false;
369 break;
370 }
371 }
372 return result;
373 }
374
375 static int
esw_setup_indir_table(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,int * i)376 esw_setup_indir_table(struct mlx5_flow_destination *dest,
377 struct mlx5_flow_act *flow_act,
378 struct mlx5_eswitch *esw,
379 struct mlx5_flow_attr *attr,
380 int *i)
381 {
382 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
383 int j, err;
384
385 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE))
386 return -EOPNOTSUPP;
387
388 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
389 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
390 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
391
392 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr,
393 esw_attr->dests[j].vport, false);
394 if (IS_ERR(dest[*i].ft)) {
395 err = PTR_ERR(dest[*i].ft);
396 goto err_indir_tbl_get;
397 }
398 }
399
400 if (mlx5_esw_indir_table_decap_vport(attr)) {
401 err = esw_setup_decap_indir(esw, attr);
402 if (err)
403 goto err_indir_tbl_get;
404 }
405
406 return 0;
407
408 err_indir_tbl_get:
409 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
410 return err;
411 }
412
esw_cleanup_indir_table(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)413 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
414 {
415 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
416
417 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
418 esw_cleanup_decap_indir(esw, attr);
419 }
420
421 static void
esw_cleanup_chain_dest(struct mlx5_fs_chains * chains,u32 chain,u32 prio,u32 level)422 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
423 {
424 mlx5_chains_put_table(chains, chain, prio, level);
425 }
426
esw_same_vhca_id(struct mlx5_core_dev * mdev1,struct mlx5_core_dev * mdev2)427 static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2)
428 {
429 return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id);
430 }
431
esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx)432 static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw,
433 struct mlx5_esw_flow_attr *esw_attr,
434 int attr_idx)
435 {
436 if (esw->offloads.ft_ipsec_tx_pol &&
437 esw_attr->dests[attr_idx].vport_valid &&
438 esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK &&
439 /* To be aligned with software, encryption is needed only for tunnel device */
440 (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) &&
441 esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport &&
442 esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev))
443 return true;
444
445 return false;
446 }
447
esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr)448 static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw,
449 struct mlx5_esw_flow_attr *esw_attr)
450 {
451 int i;
452
453 if (!esw->offloads.ft_ipsec_tx_pol)
454 return true;
455
456 for (i = 0; i < esw_attr->split_count; i++)
457 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i))
458 return false;
459
460 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
461 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) &&
462 (esw_attr->out_count - esw_attr->split_count > 1))
463 return false;
464
465 return true;
466 }
467
468 static void
esw_setup_dest_fwd_vport(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)469 esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
470 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
471 int attr_idx, int dest_idx, bool pkt_reformat)
472 {
473 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
474 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport;
475 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
476 dest[dest_idx].vport.vhca_id =
477 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
478 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
479 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK &&
480 mlx5_lag_is_mpesw(esw->dev))
481 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK;
482 }
483 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
484 if (pkt_reformat) {
485 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
486 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
487 }
488 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
489 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
490 }
491 }
492
493 static void
esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)494 esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
495 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
496 int attr_idx, int dest_idx, bool pkt_reformat)
497 {
498 dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol;
499 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
500 if (pkt_reformat &&
501 esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) {
502 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
503 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
504 }
505 }
506
507 static void
esw_setup_vport_dest(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int attr_idx,int dest_idx,bool pkt_reformat)508 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
509 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
510 int attr_idx, int dest_idx, bool pkt_reformat)
511 {
512 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
513 esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr,
514 attr_idx, dest_idx, pkt_reformat);
515 else
516 esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr,
517 attr_idx, dest_idx, pkt_reformat);
518 }
519
520 static int
esw_setup_vport_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_esw_flow_attr * esw_attr,int i)521 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
522 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
523 int i)
524 {
525 int j;
526
527 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
528 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
529 return i;
530 }
531
532 static bool
esw_src_port_rewrite_supported(struct mlx5_eswitch * esw)533 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
534 {
535 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
536 mlx5_eswitch_vport_match_metadata_enabled(esw) &&
537 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
538 }
539
540 static bool
esw_dests_to_int_external(struct mlx5_flow_destination * dests,int max_dest)541 esw_dests_to_int_external(struct mlx5_flow_destination *dests, int max_dest)
542 {
543 bool internal_dest = false, external_dest = false;
544 int i;
545
546 for (i = 0; i < max_dest; i++) {
547 if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT &&
548 dests[i].type != MLX5_FLOW_DESTINATION_TYPE_UPLINK)
549 continue;
550
551 /* Uplink dest is external, but considered as internal
552 * if there is reformat because firmware uses LB+hairpin to support it.
553 */
554 if (dests[i].vport.num == MLX5_VPORT_UPLINK &&
555 !(dests[i].vport.flags & MLX5_FLOW_DEST_VPORT_REFORMAT_ID))
556 external_dest = true;
557 else
558 internal_dest = true;
559
560 if (internal_dest && external_dest)
561 return true;
562 }
563
564 return false;
565 }
566
567 static int
esw_setup_dests(struct mlx5_flow_destination * dest,struct mlx5_flow_act * flow_act,struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_spec * spec,int * i)568 esw_setup_dests(struct mlx5_flow_destination *dest,
569 struct mlx5_flow_act *flow_act,
570 struct mlx5_eswitch *esw,
571 struct mlx5_flow_attr *attr,
572 struct mlx5_flow_spec *spec,
573 int *i)
574 {
575 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
576 struct mlx5_fs_chains *chains = esw_chains(esw);
577 int err = 0;
578
579 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
580 esw_src_port_rewrite_supported(esw))
581 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE;
582
583 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
584 esw_setup_slow_path_dest(dest, flow_act, esw, *i);
585 (*i)++;
586 goto out;
587 }
588
589 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) {
590 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i);
591 (*i)++;
592 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
593 esw_setup_accept_dest(dest, flow_act, chains, *i);
594 (*i)++;
595 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) {
596 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i);
597 (*i)++;
598 } else if (esw_is_indir_table(esw, attr)) {
599 err = esw_setup_indir_table(dest, flow_act, esw, attr, i);
600 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
601 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
602 } else {
603 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
604
605 if (attr->dest_ft) {
606 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i);
607 (*i)++;
608 } else if (attr->dest_chain) {
609 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
610 1, 0, *i);
611 (*i)++;
612 }
613 }
614
615 out:
616 return err;
617 }
618
619 static void
esw_cleanup_dests(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr)620 esw_cleanup_dests(struct mlx5_eswitch *esw,
621 struct mlx5_flow_attr *attr)
622 {
623 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
624 struct mlx5_fs_chains *chains = esw_chains(esw);
625
626 if (attr->dest_ft) {
627 esw_cleanup_decap_indir(esw, attr);
628 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
629 if (attr->dest_chain)
630 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
631 else if (esw_is_indir_table(esw, attr))
632 esw_cleanup_indir_table(esw, attr);
633 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
634 esw_cleanup_chain_src_port_rewrite(esw, attr);
635 }
636 }
637
638 static void
esw_setup_meter(struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act)639 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act)
640 {
641 struct mlx5e_flow_meter_handle *meter;
642
643 meter = attr->meter_attr.meter;
644 flow_act->exe_aso.type = attr->exe_aso_type;
645 flow_act->exe_aso.object_id = meter->obj_id;
646 flow_act->exe_aso.flow_meter.meter_idx = meter->idx;
647 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN;
648 /* use metadata reg 5 for packet color */
649 flow_act->exe_aso.return_reg_id = 5;
650 }
651
652 struct mlx5_flow_handle *
mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)653 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
654 struct mlx5_flow_spec *spec,
655 struct mlx5_flow_attr *attr)
656 {
657 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
658 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
659 struct mlx5_fs_chains *chains = esw_chains(esw);
660 bool split = !!(esw_attr->split_count);
661 struct mlx5_vport_tbl_attr fwd_attr;
662 struct mlx5_flow_destination *dest;
663 struct mlx5_flow_handle *rule;
664 struct mlx5_flow_table *fdb;
665 int i = 0;
666
667 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
668 return ERR_PTR(-EOPNOTSUPP);
669
670 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
671 return ERR_PTR(-EOPNOTSUPP);
672
673 if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr))
674 return ERR_PTR(-EOPNOTSUPP);
675
676 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
677 if (!dest)
678 return ERR_PTR(-ENOMEM);
679
680 flow_act.action = attr->action;
681
682 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
683 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
684 flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
685 flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
686 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
687 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
688 flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
689 flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
690 }
691 }
692
693 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
694
695 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
696 int err;
697
698 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
699 if (err) {
700 rule = ERR_PTR(err);
701 goto err_create_goto_table;
702 }
703
704 /* Header rewrite with combined wire+loopback in FDB is not allowed */
705 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) &&
706 esw_dests_to_int_external(dest, i)) {
707 esw_warn(esw->dev,
708 "FDB: Header rewrite with forwarding to both internal and external dests is not allowed\n");
709 rule = ERR_PTR(-EINVAL);
710 goto err_esw_get;
711 }
712 }
713
714 if (esw_attr->decap_pkt_reformat)
715 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
716
717 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
718 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
719 dest[i].counter_id = mlx5_fc_id(attr->counter);
720 i++;
721 }
722
723 if (attr->outer_match_level != MLX5_MATCH_NONE)
724 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
725 if (attr->inner_match_level != MLX5_MATCH_NONE)
726 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
727
728 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
729 flow_act.modify_hdr = attr->modify_hdr;
730
731 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
732 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)
733 esw_setup_meter(attr, &flow_act);
734
735 if (split) {
736 fwd_attr.chain = attr->chain;
737 fwd_attr.prio = attr->prio;
738 fwd_attr.vport = esw_attr->in_rep->vport;
739 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
740
741 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
742 } else {
743 if (attr->chain || attr->prio)
744 fdb = mlx5_chains_get_table(chains, attr->chain,
745 attr->prio, 0);
746 else
747 fdb = attr->ft;
748
749 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT))
750 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
751 esw_attr->in_mdev->priv.eswitch,
752 esw_attr->in_rep->vport);
753 }
754 if (IS_ERR(fdb)) {
755 rule = ERR_CAST(fdb);
756 goto err_esw_get;
757 }
758
759 if (!i) {
760 kfree(dest);
761 dest = NULL;
762 }
763
764 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
765 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
766 &flow_act, dest, i);
767 else
768 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
769 if (IS_ERR(rule))
770 goto err_add_rule;
771 else
772 atomic64_inc(&esw->offloads.num_flows);
773
774 kfree(dest);
775 return rule;
776
777 err_add_rule:
778 if (split)
779 mlx5_esw_vporttbl_put(esw, &fwd_attr);
780 else if (attr->chain || attr->prio)
781 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
782 err_esw_get:
783 esw_cleanup_dests(esw, attr);
784 err_create_goto_table:
785 kfree(dest);
786 return rule;
787 }
788
789 struct mlx5_flow_handle *
mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * attr)790 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
791 struct mlx5_flow_spec *spec,
792 struct mlx5_flow_attr *attr)
793 {
794 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
795 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
796 struct mlx5_fs_chains *chains = esw_chains(esw);
797 struct mlx5_vport_tbl_attr fwd_attr;
798 struct mlx5_flow_destination *dest;
799 struct mlx5_flow_table *fast_fdb;
800 struct mlx5_flow_table *fwd_fdb;
801 struct mlx5_flow_handle *rule;
802 int i, err = 0;
803
804 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL);
805 if (!dest)
806 return ERR_PTR(-ENOMEM);
807
808 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
809 if (IS_ERR(fast_fdb)) {
810 rule = ERR_CAST(fast_fdb);
811 goto err_get_fast;
812 }
813
814 fwd_attr.chain = attr->chain;
815 fwd_attr.prio = attr->prio;
816 fwd_attr.vport = esw_attr->in_rep->vport;
817 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
818 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
819 if (IS_ERR(fwd_fdb)) {
820 rule = ERR_CAST(fwd_fdb);
821 goto err_get_fwd;
822 }
823
824 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
825 for (i = 0; i < esw_attr->split_count; i++) {
826 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
827 /* Source port rewrite (forward to ovs internal port or statck device) isn't
828 * supported in the rule of split action.
829 */
830 err = -EOPNOTSUPP;
831 else
832 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
833
834 if (err) {
835 rule = ERR_PTR(err);
836 goto err_chain_src_rewrite;
837 }
838 }
839 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
840 dest[i].ft = fwd_fdb;
841 i++;
842
843 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
844 esw_attr->in_mdev->priv.eswitch,
845 esw_attr->in_rep->vport);
846
847 if (attr->outer_match_level != MLX5_MATCH_NONE)
848 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
849
850 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
851 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
852
853 if (IS_ERR(rule)) {
854 i = esw_attr->split_count;
855 goto err_chain_src_rewrite;
856 }
857
858 atomic64_inc(&esw->offloads.num_flows);
859
860 kfree(dest);
861 return rule;
862 err_chain_src_rewrite:
863 mlx5_esw_vporttbl_put(esw, &fwd_attr);
864 err_get_fwd:
865 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
866 err_get_fast:
867 kfree(dest);
868 return rule;
869 }
870
871 static void
__mlx5_eswitch_del_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr,bool fwd_rule)872 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
873 struct mlx5_flow_handle *rule,
874 struct mlx5_flow_attr *attr,
875 bool fwd_rule)
876 {
877 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
878 struct mlx5_fs_chains *chains = esw_chains(esw);
879 bool split = (esw_attr->split_count > 0);
880 struct mlx5_vport_tbl_attr fwd_attr;
881 int i;
882
883 mlx5_del_flow_rules(rule);
884
885 if (!mlx5e_tc_attr_flags_skip(attr->flags)) {
886 /* unref the term table */
887 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
888 if (esw_attr->dests[i].termtbl)
889 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
890 }
891 }
892
893 atomic64_dec(&esw->offloads.num_flows);
894
895 if (fwd_rule || split) {
896 fwd_attr.chain = attr->chain;
897 fwd_attr.prio = attr->prio;
898 fwd_attr.vport = esw_attr->in_rep->vport;
899 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
900 }
901
902 if (fwd_rule) {
903 mlx5_esw_vporttbl_put(esw, &fwd_attr);
904 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
905 } else {
906 if (split)
907 mlx5_esw_vporttbl_put(esw, &fwd_attr);
908 else if (attr->chain || attr->prio)
909 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
910 esw_cleanup_dests(esw, attr);
911 }
912 }
913
914 void
mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)915 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
916 struct mlx5_flow_handle *rule,
917 struct mlx5_flow_attr *attr)
918 {
919 __mlx5_eswitch_del_rule(esw, rule, attr, false);
920 }
921
922 void
mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_flow_attr * attr)923 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
924 struct mlx5_flow_handle *rule,
925 struct mlx5_flow_attr *attr)
926 {
927 __mlx5_eswitch_del_rule(esw, rule, attr, true);
928 }
929
930 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch * on_esw,struct mlx5_eswitch * from_esw,struct mlx5_eswitch_rep * rep,u32 sqn)931 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
932 struct mlx5_eswitch *from_esw,
933 struct mlx5_eswitch_rep *rep,
934 u32 sqn)
935 {
936 struct mlx5_flow_act flow_act = {0};
937 struct mlx5_flow_destination dest = {};
938 struct mlx5_flow_handle *flow_rule;
939 struct mlx5_flow_spec *spec;
940 void *misc;
941 u16 vport;
942
943 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
944 if (!spec) {
945 flow_rule = ERR_PTR(-ENOMEM);
946 goto out;
947 }
948
949 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
950 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
951
952 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
953 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
954
955 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
956
957 /* source vport is the esw manager */
958 vport = from_esw->manager_vport;
959
960 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) {
961 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
962 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
963 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport));
964
965 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
966 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
967 mlx5_eswitch_get_vport_metadata_mask());
968
969 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
970 } else {
971 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
972 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
973
974 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
975 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
976 MLX5_CAP_GEN(from_esw->dev, vhca_id));
977
978 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
979 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
980
981 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
982 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
983 source_eswitch_owner_vhca_id);
984
985 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
986 }
987
988 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
989 dest.vport.num = rep->vport;
990 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
991 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
992 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
993
994 if (rep->vport == MLX5_VPORT_UPLINK &&
995 on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) {
996 dest.ft = on_esw->offloads.ft_ipsec_tx_pol;
997 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL;
998 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
999 } else {
1000 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1001 dest.vport.num = rep->vport;
1002 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
1003 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1004 }
1005
1006 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) &&
1007 rep->vport == MLX5_VPORT_UPLINK)
1008 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
1009
1010 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw),
1011 spec, &flow_act, &dest, 1);
1012 if (IS_ERR(flow_rule))
1013 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n",
1014 PTR_ERR(flow_rule));
1015 out:
1016 kvfree(spec);
1017 return flow_rule;
1018 }
1019 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
1020
mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle * rule)1021 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
1022 {
1023 mlx5_del_flow_rules(rule);
1024 }
1025
mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle * rule)1026 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
1027 {
1028 if (rule)
1029 mlx5_del_flow_rules(rule);
1030 }
1031
1032 struct mlx5_flow_handle *
mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch * esw,u16 vport_num)1033 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
1034 {
1035 struct mlx5_flow_destination dest = {};
1036 struct mlx5_flow_act flow_act = {0};
1037 struct mlx5_flow_handle *flow_rule;
1038 struct mlx5_flow_spec *spec;
1039
1040 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1041 if (!spec)
1042 return ERR_PTR(-ENOMEM);
1043
1044 MLX5_SET(fte_match_param, spec->match_criteria,
1045 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1046 MLX5_SET(fte_match_param, spec->match_criteria,
1047 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1048 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1049 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1050
1051 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1052 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1053 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1054
1055 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1056 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1057 dest.vport.num = vport_num;
1058
1059 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1060 spec, &flow_act, &dest, 1);
1061 if (IS_ERR(flow_rule))
1062 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n",
1063 vport_num, PTR_ERR(flow_rule));
1064
1065 kvfree(spec);
1066 return flow_rule;
1067 }
1068
mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch * esw)1069 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1070 {
1071 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1072 MLX5_FDB_TO_VPORT_REG_C_1;
1073 }
1074
esw_set_passing_vport_metadata(struct mlx5_eswitch * esw,bool enable)1075 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
1076 {
1077 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
1078 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1079 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
1080 u8 curr, wanted;
1081 int err;
1082
1083 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1084 !mlx5_eswitch_vport_match_metadata_enabled(esw))
1085 return 0;
1086
1087 MLX5_SET(query_esw_vport_context_in, in, opcode,
1088 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1089 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
1090 if (err)
1091 return err;
1092
1093 curr = MLX5_GET(query_esw_vport_context_out, out,
1094 esw_vport_context.fdb_to_vport_reg_c_id);
1095 wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1096 if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1097 wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
1098
1099 if (enable)
1100 curr |= wanted;
1101 else
1102 curr &= ~wanted;
1103
1104 MLX5_SET(modify_esw_vport_context_in, min,
1105 esw_vport_context.fdb_to_vport_reg_c_id, curr);
1106 MLX5_SET(modify_esw_vport_context_in, min,
1107 field_select.fdb_to_vport_reg_c_id, 1);
1108
1109 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
1110 if (!err) {
1111 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1112 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1113 else
1114 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1115 }
1116
1117 return err;
1118 }
1119
peer_miss_rules_setup(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev,struct mlx5_flow_spec * spec,struct mlx5_flow_destination * dest)1120 static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1121 struct mlx5_core_dev *peer_dev,
1122 struct mlx5_flow_spec *spec,
1123 struct mlx5_flow_destination *dest)
1124 {
1125 void *misc;
1126
1127 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1128 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1129 misc_parameters_2);
1130 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1131 mlx5_eswitch_get_vport_metadata_mask());
1132
1133 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1134 } else {
1135 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1136 misc_parameters);
1137
1138 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1139 MLX5_CAP_GEN(peer_dev, vhca_id));
1140
1141 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1142
1143 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1144 misc_parameters);
1145 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1146 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1147 source_eswitch_owner_vhca_id);
1148 }
1149
1150 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1151 dest->vport.num = peer_dev->priv.eswitch->manager_vport;
1152 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
1153 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
1154 }
1155
esw_set_peer_miss_rule_source_port(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,struct mlx5_flow_spec * spec,u16 vport)1156 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1157 struct mlx5_eswitch *peer_esw,
1158 struct mlx5_flow_spec *spec,
1159 u16 vport)
1160 {
1161 void *misc;
1162
1163 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1164 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1165 misc_parameters_2);
1166 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1167 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1168 vport));
1169 } else {
1170 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1171 misc_parameters);
1172 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1173 }
1174 }
1175
esw_add_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1176 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1177 struct mlx5_core_dev *peer_dev)
1178 {
1179 struct mlx5_flow_destination dest = {};
1180 struct mlx5_flow_act flow_act = {0};
1181 struct mlx5_flow_handle **flows;
1182 /* total vports is the same for both e-switches */
1183 int nvports = esw->total_vports;
1184 struct mlx5_flow_handle *flow;
1185 struct mlx5_flow_spec *spec;
1186 struct mlx5_vport *vport;
1187 int err, pfindex;
1188 unsigned long i;
1189 void *misc;
1190
1191 if (!MLX5_VPORT_MANAGER(esw->dev) && !mlx5_core_is_ecpf_esw_manager(esw->dev))
1192 return 0;
1193
1194 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1195 if (!spec)
1196 return -ENOMEM;
1197
1198 peer_miss_rules_setup(esw, peer_dev, spec, &dest);
1199
1200 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL);
1201 if (!flows) {
1202 err = -ENOMEM;
1203 goto alloc_flows_err;
1204 }
1205
1206 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1207 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1208 misc_parameters);
1209
1210 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1211 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1212 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1213 spec, MLX5_VPORT_PF);
1214
1215 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1216 spec, &flow_act, &dest, 1);
1217 if (IS_ERR(flow)) {
1218 err = PTR_ERR(flow);
1219 goto add_pf_flow_err;
1220 }
1221 flows[vport->index] = flow;
1222 }
1223
1224 if (mlx5_ecpf_vport_exists(esw->dev)) {
1225 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1226 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1227 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1228 spec, &flow_act, &dest, 1);
1229 if (IS_ERR(flow)) {
1230 err = PTR_ERR(flow);
1231 goto add_ecpf_flow_err;
1232 }
1233 flows[vport->index] = flow;
1234 }
1235
1236 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1237 esw_set_peer_miss_rule_source_port(esw,
1238 peer_dev->priv.eswitch,
1239 spec, vport->vport);
1240
1241 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1242 spec, &flow_act, &dest, 1);
1243 if (IS_ERR(flow)) {
1244 err = PTR_ERR(flow);
1245 goto add_vf_flow_err;
1246 }
1247 flows[vport->index] = flow;
1248 }
1249
1250 if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1251 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1252 if (i >= mlx5_core_max_ec_vfs(peer_dev))
1253 break;
1254 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1255 spec, vport->vport);
1256 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1257 spec, &flow_act, &dest, 1);
1258 if (IS_ERR(flow)) {
1259 err = PTR_ERR(flow);
1260 goto add_ec_vf_flow_err;
1261 }
1262 flows[vport->index] = flow;
1263 }
1264 }
1265
1266 pfindex = mlx5_get_dev_index(peer_dev);
1267 if (pfindex >= MLX5_MAX_PORTS) {
1268 esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n",
1269 pfindex, MLX5_MAX_PORTS);
1270 err = -EINVAL;
1271 goto add_ec_vf_flow_err;
1272 }
1273 esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows;
1274
1275 kvfree(spec);
1276 return 0;
1277
1278 add_ec_vf_flow_err:
1279 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1280 if (!flows[vport->index])
1281 continue;
1282 mlx5_del_flow_rules(flows[vport->index]);
1283 }
1284 add_vf_flow_err:
1285 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1286 if (!flows[vport->index])
1287 continue;
1288 mlx5_del_flow_rules(flows[vport->index]);
1289 }
1290 if (mlx5_ecpf_vport_exists(esw->dev)) {
1291 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1292 mlx5_del_flow_rules(flows[vport->index]);
1293 }
1294 add_ecpf_flow_err:
1295 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1296 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1297 mlx5_del_flow_rules(flows[vport->index]);
1298 }
1299 add_pf_flow_err:
1300 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
1301 kvfree(flows);
1302 alloc_flows_err:
1303 kvfree(spec);
1304 return err;
1305 }
1306
esw_del_fdb_peer_miss_rules(struct mlx5_eswitch * esw,struct mlx5_core_dev * peer_dev)1307 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1308 struct mlx5_core_dev *peer_dev)
1309 {
1310 u16 peer_index = mlx5_get_dev_index(peer_dev);
1311 struct mlx5_flow_handle **flows;
1312 struct mlx5_vport *vport;
1313 unsigned long i;
1314
1315 flows = esw->fdb_table.offloads.peer_miss_rules[peer_index];
1316 if (!flows)
1317 return;
1318
1319 if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1320 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) {
1321 /* The flow for a particular vport could be NULL if the other ECPF
1322 * has fewer or no VFs enabled
1323 */
1324 if (!flows[vport->index])
1325 continue;
1326 mlx5_del_flow_rules(flows[vport->index]);
1327 }
1328 }
1329
1330 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev))
1331 mlx5_del_flow_rules(flows[vport->index]);
1332
1333 if (mlx5_ecpf_vport_exists(esw->dev)) {
1334 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1335 mlx5_del_flow_rules(flows[vport->index]);
1336 }
1337
1338 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1339 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1340 mlx5_del_flow_rules(flows[vport->index]);
1341 }
1342
1343 kvfree(flows);
1344 esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL;
1345 }
1346
esw_add_fdb_miss_rule(struct mlx5_eswitch * esw)1347 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1348 {
1349 struct mlx5_flow_act flow_act = {0};
1350 struct mlx5_flow_destination dest = {};
1351 struct mlx5_flow_handle *flow_rule = NULL;
1352 struct mlx5_flow_spec *spec;
1353 void *headers_c;
1354 void *headers_v;
1355 int err = 0;
1356 u8 *dmac_c;
1357 u8 *dmac_v;
1358
1359 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1360 if (!spec) {
1361 err = -ENOMEM;
1362 goto out;
1363 }
1364
1365 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1366 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1367 outer_headers);
1368 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1369 outer_headers.dmac_47_16);
1370 dmac_c[0] = 0x01;
1371
1372 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1373 dest.vport.num = esw->manager_vport;
1374 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1375
1376 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1377 spec, &flow_act, &dest, 1);
1378 if (IS_ERR(flow_rule)) {
1379 err = PTR_ERR(flow_rule);
1380 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
1381 goto out;
1382 }
1383
1384 esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1385
1386 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1387 outer_headers);
1388 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1389 outer_headers.dmac_47_16);
1390 dmac_v[0] = 0x01;
1391 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
1392 spec, &flow_act, &dest, 1);
1393 if (IS_ERR(flow_rule)) {
1394 err = PTR_ERR(flow_rule);
1395 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1396 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1397 goto out;
1398 }
1399
1400 esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1401
1402 out:
1403 kvfree(spec);
1404 return err;
1405 }
1406
1407 struct mlx5_flow_handle *
esw_add_restore_rule(struct mlx5_eswitch * esw,u32 tag)1408 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1409 {
1410 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1411 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1412 struct mlx5_flow_context *flow_context;
1413 struct mlx5_flow_handle *flow_rule;
1414 struct mlx5_flow_destination dest;
1415 struct mlx5_flow_spec *spec;
1416 void *misc;
1417
1418 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1419 return ERR_PTR(-EOPNOTSUPP);
1420
1421 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1422 if (!spec)
1423 return ERR_PTR(-ENOMEM);
1424
1425 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1426 misc_parameters_2);
1427 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1428 ESW_REG_C0_USER_DATA_METADATA_MASK);
1429 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1430 misc_parameters_2);
1431 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1432 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1433 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1434 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1435 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
1436
1437 flow_context = &spec->flow_context;
1438 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1439 flow_context->flow_tag = tag;
1440 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1441 dest.ft = esw->offloads.ft_offloads;
1442
1443 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1444 kvfree(spec);
1445
1446 if (IS_ERR(flow_rule))
1447 esw_warn(esw->dev,
1448 "Failed to create restore rule for tag: %d, err(%d)\n",
1449 tag, (int)PTR_ERR(flow_rule));
1450
1451 return flow_rule;
1452 }
1453
1454 #define MAX_PF_SQ 256
1455 #define MAX_SQ_NVPORTS 32
1456
1457 void
mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch * esw,u32 * flow_group_in,int match_params)1458 mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1459 u32 *flow_group_in,
1460 int match_params)
1461 {
1462 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1463 flow_group_in,
1464 match_criteria);
1465
1466 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1467 MLX5_SET(create_flow_group_in, flow_group_in,
1468 match_criteria_enable,
1469 MLX5_MATCH_MISC_PARAMETERS_2 | match_params);
1470
1471 MLX5_SET(fte_match_param, match_criteria,
1472 misc_parameters_2.metadata_reg_c_0,
1473 mlx5_eswitch_get_vport_metadata_mask());
1474 } else {
1475 MLX5_SET(create_flow_group_in, flow_group_in,
1476 match_criteria_enable,
1477 MLX5_MATCH_MISC_PARAMETERS | match_params);
1478
1479 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1480 misc_parameters.source_port);
1481 }
1482 }
1483
1484 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
esw_vport_tbl_put(struct mlx5_eswitch * esw)1485 static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
1486 {
1487 struct mlx5_vport_tbl_attr attr;
1488 struct mlx5_vport *vport;
1489 unsigned long i;
1490
1491 attr.chain = 0;
1492 attr.prio = 1;
1493 mlx5_esw_for_each_vport(esw, i, vport) {
1494 attr.vport = vport->vport;
1495 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1496 mlx5_esw_vporttbl_put(esw, &attr);
1497 }
1498 }
1499
esw_vport_tbl_get(struct mlx5_eswitch * esw)1500 static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
1501 {
1502 struct mlx5_vport_tbl_attr attr;
1503 struct mlx5_flow_table *fdb;
1504 struct mlx5_vport *vport;
1505 unsigned long i;
1506
1507 attr.chain = 0;
1508 attr.prio = 1;
1509 mlx5_esw_for_each_vport(esw, i, vport) {
1510 attr.vport = vport->vport;
1511 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
1512 fdb = mlx5_esw_vporttbl_get(esw, &attr);
1513 if (IS_ERR(fdb))
1514 goto out;
1515 }
1516 return 0;
1517
1518 out:
1519 esw_vport_tbl_put(esw);
1520 return PTR_ERR(fdb);
1521 }
1522
1523 #define fdb_modify_header_fwd_to_table_supported(esw) \
1524 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
esw_init_chains_offload_flags(struct mlx5_eswitch * esw,u32 * flags)1525 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1526 {
1527 struct mlx5_core_dev *dev = esw->dev;
1528
1529 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1530 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1531
1532 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1533 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1534 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1535 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1536 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1537 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1538 esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1539 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1540 /* Disabled when ttl workaround is needed, e.g
1541 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1542 */
1543 esw_warn(dev,
1544 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1545 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1546 } else {
1547 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1548 esw_info(dev, "Supported tc chains and prios offload\n");
1549 }
1550
1551 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1552 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1553 }
1554
1555 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1556 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1557 {
1558 struct mlx5_core_dev *dev = esw->dev;
1559 struct mlx5_flow_table *nf_ft, *ft;
1560 struct mlx5_chains_attr attr = {};
1561 struct mlx5_fs_chains *chains;
1562 int err;
1563
1564 esw_init_chains_offload_flags(esw, &attr.flags);
1565 attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1566 attr.max_grp_num = esw->params.large_group_num;
1567 attr.default_ft = miss_fdb;
1568 attr.mapping = esw->offloads.reg_c0_obj_pool;
1569
1570 chains = mlx5_chains_create(dev, &attr);
1571 if (IS_ERR(chains)) {
1572 err = PTR_ERR(chains);
1573 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1574 return err;
1575 }
1576 mlx5_chains_print_info(chains);
1577
1578 esw->fdb_table.offloads.esw_chains_priv = chains;
1579
1580 /* Create tc_end_ft which is the always created ft chain */
1581 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1582 1, 0);
1583 if (IS_ERR(nf_ft)) {
1584 err = PTR_ERR(nf_ft);
1585 goto nf_ft_err;
1586 }
1587
1588 /* Always open the root for fast path */
1589 ft = mlx5_chains_get_table(chains, 0, 1, 0);
1590 if (IS_ERR(ft)) {
1591 err = PTR_ERR(ft);
1592 goto level_0_err;
1593 }
1594
1595 /* Open level 1 for split fdb rules now if prios isn't supported */
1596 if (!mlx5_chains_prios_supported(chains)) {
1597 err = esw_vport_tbl_get(esw);
1598 if (err)
1599 goto level_1_err;
1600 }
1601
1602 mlx5_chains_set_end_ft(chains, nf_ft);
1603
1604 return 0;
1605
1606 level_1_err:
1607 mlx5_chains_put_table(chains, 0, 1, 0);
1608 level_0_err:
1609 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1610 nf_ft_err:
1611 mlx5_chains_destroy(chains);
1612 esw->fdb_table.offloads.esw_chains_priv = NULL;
1613
1614 return err;
1615 }
1616
1617 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1618 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1619 {
1620 if (!mlx5_chains_prios_supported(chains))
1621 esw_vport_tbl_put(esw);
1622 mlx5_chains_put_table(chains, 0, 1, 0);
1623 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1624 mlx5_chains_destroy(chains);
1625 }
1626
1627 #else /* CONFIG_MLX5_CLS_ACT */
1628
1629 static int
esw_chains_create(struct mlx5_eswitch * esw,struct mlx5_flow_table * miss_fdb)1630 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1631 { return 0; }
1632
1633 static void
esw_chains_destroy(struct mlx5_eswitch * esw,struct mlx5_fs_chains * chains)1634 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1635 {}
1636
1637 #endif
1638
1639 static int
esw_create_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1640 esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
1641 struct mlx5_flow_table *fdb,
1642 u32 *flow_group_in,
1643 int *ix)
1644 {
1645 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1646 struct mlx5_flow_group *g;
1647 void *match_criteria;
1648 int count, err = 0;
1649
1650 memset(flow_group_in, 0, inlen);
1651
1652 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS);
1653
1654 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1655 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1656
1657 if (!mlx5_eswitch_vport_match_metadata_enabled(esw) &&
1658 MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1659 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1660 misc_parameters.source_eswitch_owner_vhca_id);
1661 MLX5_SET(create_flow_group_in, flow_group_in,
1662 source_eswitch_owner_vhca_id_valid, 1);
1663 }
1664
1665 /* See comment at table_size calculation */
1666 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
1667 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1668 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
1669 *ix += count;
1670
1671 g = mlx5_create_flow_group(fdb, flow_group_in);
1672 if (IS_ERR(g)) {
1673 err = PTR_ERR(g);
1674 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1675 goto out;
1676 }
1677 esw->fdb_table.offloads.send_to_vport_grp = g;
1678
1679 out:
1680 return err;
1681 }
1682
1683 static int
esw_create_meta_send_to_vport_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1684 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
1685 struct mlx5_flow_table *fdb,
1686 u32 *flow_group_in,
1687 int *ix)
1688 {
1689 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1690 struct mlx5_flow_group *g;
1691 void *match_criteria;
1692 int err = 0;
1693
1694 if (!esw_src_port_rewrite_supported(esw))
1695 return 0;
1696
1697 memset(flow_group_in, 0, inlen);
1698
1699 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1700 MLX5_MATCH_MISC_PARAMETERS_2);
1701
1702 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1703
1704 MLX5_SET(fte_match_param, match_criteria,
1705 misc_parameters_2.metadata_reg_c_0,
1706 mlx5_eswitch_get_vport_metadata_mask());
1707 MLX5_SET(fte_match_param, match_criteria,
1708 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1709
1710 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1711 MLX5_SET(create_flow_group_in, flow_group_in,
1712 end_flow_index, *ix + esw->total_vports - 1);
1713 *ix += esw->total_vports;
1714
1715 g = mlx5_create_flow_group(fdb, flow_group_in);
1716 if (IS_ERR(g)) {
1717 err = PTR_ERR(g);
1718 esw_warn(esw->dev,
1719 "Failed to create send-to-vport meta flow group err(%d)\n", err);
1720 goto send_vport_meta_err;
1721 }
1722 esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1723
1724 return 0;
1725
1726 send_vport_meta_err:
1727 return err;
1728 }
1729
1730 static int
esw_create_peer_esw_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1731 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
1732 struct mlx5_flow_table *fdb,
1733 u32 *flow_group_in,
1734 int *ix)
1735 {
1736 int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1);
1737 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1738 struct mlx5_flow_group *g;
1739 void *match_criteria;
1740 int err = 0;
1741
1742 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
1743 return 0;
1744
1745 memset(flow_group_in, 0, inlen);
1746
1747 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0);
1748
1749 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1750 match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1751 flow_group_in,
1752 match_criteria);
1753
1754 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1755 misc_parameters.source_eswitch_owner_vhca_id);
1756
1757 MLX5_SET(create_flow_group_in, flow_group_in,
1758 source_eswitch_owner_vhca_id_valid, 1);
1759 }
1760
1761 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1762 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1763 *ix + max_peer_ports);
1764 *ix += max_peer_ports + 1;
1765
1766 g = mlx5_create_flow_group(fdb, flow_group_in);
1767 if (IS_ERR(g)) {
1768 err = PTR_ERR(g);
1769 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
1770 goto out;
1771 }
1772 esw->fdb_table.offloads.peer_miss_grp = g;
1773
1774 out:
1775 return err;
1776 }
1777
1778 static int
esw_create_miss_group(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,u32 * flow_group_in,int * ix)1779 esw_create_miss_group(struct mlx5_eswitch *esw,
1780 struct mlx5_flow_table *fdb,
1781 u32 *flow_group_in,
1782 int *ix)
1783 {
1784 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1785 struct mlx5_flow_group *g;
1786 void *match_criteria;
1787 int err = 0;
1788 u8 *dmac;
1789
1790 memset(flow_group_in, 0, inlen);
1791
1792 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1793 MLX5_MATCH_OUTER_HEADERS);
1794 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1795 match_criteria);
1796 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1797 outer_headers.dmac_47_16);
1798 dmac[0] = 0x01;
1799
1800 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
1801 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1802 *ix + MLX5_ESW_MISS_FLOWS);
1803
1804 g = mlx5_create_flow_group(fdb, flow_group_in);
1805 if (IS_ERR(g)) {
1806 err = PTR_ERR(g);
1807 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
1808 goto miss_err;
1809 }
1810 esw->fdb_table.offloads.miss_grp = g;
1811
1812 err = esw_add_fdb_miss_rule(esw);
1813 if (err)
1814 goto miss_rule_err;
1815
1816 return 0;
1817
1818 miss_rule_err:
1819 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1820 miss_err:
1821 return err;
1822 }
1823
esw_create_offloads_fdb_tables(struct mlx5_eswitch * esw)1824 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1825 {
1826 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1827 struct mlx5_flow_table_attr ft_attr = {};
1828 struct mlx5_core_dev *dev = esw->dev;
1829 struct mlx5_flow_namespace *root_ns;
1830 struct mlx5_flow_table *fdb = NULL;
1831 int table_size, ix = 0, err = 0;
1832 u32 flags = 0, *flow_group_in;
1833
1834 esw_debug(esw->dev, "Create offloads FDB Tables\n");
1835
1836 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1837 if (!flow_group_in)
1838 return -ENOMEM;
1839
1840 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1841 if (!root_ns) {
1842 esw_warn(dev, "Failed to get FDB flow namespace\n");
1843 err = -EOPNOTSUPP;
1844 goto ns_err;
1845 }
1846 esw->fdb_table.offloads.ns = root_ns;
1847 err = mlx5_flow_namespace_set_mode(root_ns,
1848 esw->dev->priv.steering->mode);
1849 if (err) {
1850 esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1851 goto ns_err;
1852 }
1853
1854 /* To be strictly correct:
1855 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1856 * should be:
1857 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1858 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1859 * but as the peer device might not be in switchdev mode it's not
1860 * possible. We use the fact that by default FW sets max vfs and max sfs
1861 * to the same value on both devices. If it needs to be changed in the future note
1862 * the peer miss group should also be created based on the number of
1863 * total vports of the peer (currently is also uses esw->total_vports).
1864 */
1865 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
1866 esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS;
1867
1868 /* create the slow path fdb with encap set, so further table instances
1869 * can be created at run time while VFs are probed if the FW allows that.
1870 */
1871 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1872 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1873 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1874
1875 ft_attr.flags = flags;
1876 ft_attr.max_fte = table_size;
1877 ft_attr.prio = FDB_SLOW_PATH;
1878
1879 fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1880 if (IS_ERR(fdb)) {
1881 err = PTR_ERR(fdb);
1882 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1883 goto slow_fdb_err;
1884 }
1885 esw->fdb_table.offloads.slow_fdb = fdb;
1886
1887 /* Create empty TC-miss managed table. This allows plugging in following
1888 * priorities without directly exposing their level 0 table to
1889 * eswitch_offloads and passing it as miss_fdb to following call to
1890 * esw_chains_create().
1891 */
1892 memset(&ft_attr, 0, sizeof(ft_attr));
1893 ft_attr.prio = FDB_TC_MISS;
1894 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr);
1895 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) {
1896 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table);
1897 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1898 goto tc_miss_table_err;
1899 }
1900
1901 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table);
1902 if (err) {
1903 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
1904 goto fdb_chains_err;
1905 }
1906
1907 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1908 if (err)
1909 goto send_vport_err;
1910
1911 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix);
1912 if (err)
1913 goto send_vport_meta_err;
1914
1915 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix);
1916 if (err)
1917 goto peer_miss_err;
1918
1919 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix);
1920 if (err)
1921 goto miss_err;
1922
1923 kvfree(flow_group_in);
1924 return 0;
1925
1926 miss_err:
1927 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1928 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1929 peer_miss_err:
1930 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1931 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1932 send_vport_meta_err:
1933 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1934 send_vport_err:
1935 esw_chains_destroy(esw, esw_chains(esw));
1936 fdb_chains_err:
1937 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1938 tc_miss_table_err:
1939 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw));
1940 slow_fdb_err:
1941 /* Holds true only as long as DMFS is the default */
1942 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
1943 ns_err:
1944 kvfree(flow_group_in);
1945 return err;
1946 }
1947
esw_destroy_offloads_fdb_tables(struct mlx5_eswitch * esw)1948 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
1949 {
1950 if (!mlx5_eswitch_get_slow_fdb(esw))
1951 return;
1952
1953 esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
1954 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
1955 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1956 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1957 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1958 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1959 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1960 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
1961 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1962
1963 esw_chains_destroy(esw, esw_chains(esw));
1964
1965 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1966 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw));
1967 /* Holds true only as long as DMFS is the default */
1968 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1969 MLX5_FLOW_STEERING_MODE_DMFS);
1970 atomic64_set(&esw->user_count, 0);
1971 }
1972
esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch * esw)1973 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
1974 {
1975 int nvports;
1976
1977 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1978 if (mlx5e_tc_int_port_supported(esw))
1979 nvports += MLX5E_TC_MAX_INT_PORT_NUM;
1980
1981 return nvports;
1982 }
1983
esw_create_offloads_table(struct mlx5_eswitch * esw)1984 static int esw_create_offloads_table(struct mlx5_eswitch *esw)
1985 {
1986 struct mlx5_flow_table_attr ft_attr = {};
1987 struct mlx5_core_dev *dev = esw->dev;
1988 struct mlx5_flow_table *ft_offloads;
1989 struct mlx5_flow_namespace *ns;
1990 int err = 0;
1991
1992 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1993 if (!ns) {
1994 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
1995 return -EOPNOTSUPP;
1996 }
1997
1998 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
1999 MLX5_ESW_FT_OFFLOADS_DROP_RULE;
2000 ft_attr.prio = 1;
2001
2002 ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
2003 if (IS_ERR(ft_offloads)) {
2004 err = PTR_ERR(ft_offloads);
2005 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
2006 return err;
2007 }
2008
2009 esw->offloads.ft_offloads = ft_offloads;
2010 return 0;
2011 }
2012
esw_destroy_offloads_table(struct mlx5_eswitch * esw)2013 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
2014 {
2015 struct mlx5_esw_offload *offloads = &esw->offloads;
2016
2017 mlx5_destroy_flow_table(offloads->ft_offloads);
2018 }
2019
esw_create_vport_rx_group(struct mlx5_eswitch * esw)2020 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
2021 {
2022 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2023 struct mlx5_flow_group *g;
2024 u32 *flow_group_in;
2025 int nvports;
2026 int err = 0;
2027
2028 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
2029 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2030 if (!flow_group_in)
2031 return -ENOMEM;
2032
2033 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0);
2034
2035 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2036 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
2037
2038 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2039
2040 if (IS_ERR(g)) {
2041 err = PTR_ERR(g);
2042 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
2043 goto out;
2044 }
2045
2046 esw->offloads.vport_rx_group = g;
2047 out:
2048 kvfree(flow_group_in);
2049 return err;
2050 }
2051
esw_destroy_vport_rx_group(struct mlx5_eswitch * esw)2052 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
2053 {
2054 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
2055 }
2056
esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch * esw)2057 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
2058 {
2059 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
2060 * for the drop rule, which is placed at the end of the table.
2061 * So return the total of vport and int_port as rule index.
2062 */
2063 return esw_get_nr_ft_offloads_steering_src_ports(esw);
2064 }
2065
esw_create_vport_rx_drop_group(struct mlx5_eswitch * esw)2066 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
2067 {
2068 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2069 struct mlx5_flow_group *g;
2070 u32 *flow_group_in;
2071 int flow_index;
2072 int err = 0;
2073
2074 flow_index = esw_create_vport_rx_drop_rule_index(esw);
2075
2076 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2077 if (!flow_group_in)
2078 return -ENOMEM;
2079
2080 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
2081 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
2082
2083 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
2084
2085 if (IS_ERR(g)) {
2086 err = PTR_ERR(g);
2087 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
2088 goto out;
2089 }
2090
2091 esw->offloads.vport_rx_drop_group = g;
2092 out:
2093 kvfree(flow_group_in);
2094 return err;
2095 }
2096
esw_destroy_vport_rx_drop_group(struct mlx5_eswitch * esw)2097 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
2098 {
2099 if (esw->offloads.vport_rx_drop_group)
2100 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group);
2101 }
2102
2103 void
mlx5_esw_set_spec_source_port(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_spec * spec)2104 mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw,
2105 u16 vport,
2106 struct mlx5_flow_spec *spec)
2107 {
2108 void *misc;
2109
2110 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
2111 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
2112 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2113 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
2114
2115 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
2116 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2117 mlx5_eswitch_get_vport_metadata_mask());
2118
2119 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
2120 } else {
2121 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
2122 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
2123
2124 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2125 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2126
2127 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2128 }
2129 }
2130
2131 struct mlx5_flow_handle *
mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch * esw,u16 vport,struct mlx5_flow_destination * dest)2132 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
2133 struct mlx5_flow_destination *dest)
2134 {
2135 struct mlx5_flow_act flow_act = {0};
2136 struct mlx5_flow_handle *flow_rule;
2137 struct mlx5_flow_spec *spec;
2138
2139 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2140 if (!spec) {
2141 flow_rule = ERR_PTR(-ENOMEM);
2142 goto out;
2143 }
2144
2145 mlx5_esw_set_spec_source_port(esw, vport, spec);
2146
2147 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2148 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
2149 &flow_act, dest, 1);
2150 if (IS_ERR(flow_rule)) {
2151 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
2152 goto out;
2153 }
2154
2155 out:
2156 kvfree(spec);
2157 return flow_rule;
2158 }
2159
esw_create_vport_rx_drop_rule(struct mlx5_eswitch * esw)2160 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2161 {
2162 struct mlx5_flow_act flow_act = {};
2163 struct mlx5_flow_handle *flow_rule;
2164
2165 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2166 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL,
2167 &flow_act, NULL, 0);
2168 if (IS_ERR(flow_rule)) {
2169 esw_warn(esw->dev,
2170 "fs offloads: Failed to add vport rx drop rule err %ld\n",
2171 PTR_ERR(flow_rule));
2172 return PTR_ERR(flow_rule);
2173 }
2174
2175 esw->offloads.vport_rx_drop_rule = flow_rule;
2176
2177 return 0;
2178 }
2179
esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch * esw)2180 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
2181 {
2182 if (esw->offloads.vport_rx_drop_rule)
2183 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule);
2184 }
2185
mlx5_eswitch_inline_mode_get(struct mlx5_eswitch * esw,u8 * mode)2186 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
2187 {
2188 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
2189 struct mlx5_core_dev *dev = esw->dev;
2190 struct mlx5_vport *vport;
2191 unsigned long i;
2192
2193 if (!MLX5_CAP_GEN(dev, vport_group_manager))
2194 return -EOPNOTSUPP;
2195
2196 if (!mlx5_esw_is_fdb_created(esw))
2197 return -EOPNOTSUPP;
2198
2199 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
2200 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
2201 mlx5_mode = MLX5_INLINE_MODE_NONE;
2202 goto out;
2203 case MLX5_CAP_INLINE_MODE_L2:
2204 mlx5_mode = MLX5_INLINE_MODE_L2;
2205 goto out;
2206 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
2207 goto query_vports;
2208 }
2209
2210 query_vports:
2211 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
2212 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
2213 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode);
2214 if (prev_mlx5_mode != mlx5_mode)
2215 return -EINVAL;
2216 prev_mlx5_mode = mlx5_mode;
2217 }
2218
2219 out:
2220 *mode = mlx5_mode;
2221 return 0;
2222 }
2223
esw_destroy_restore_table(struct mlx5_eswitch * esw)2224 static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2225 {
2226 struct mlx5_esw_offload *offloads = &esw->offloads;
2227
2228 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2229 return;
2230
2231 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
2232 mlx5_destroy_flow_group(offloads->restore_group);
2233 mlx5_destroy_flow_table(offloads->ft_offloads_restore);
2234 }
2235
esw_create_restore_table(struct mlx5_eswitch * esw)2236 static int esw_create_restore_table(struct mlx5_eswitch *esw)
2237 {
2238 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
2239 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2240 struct mlx5_flow_table_attr ft_attr = {};
2241 struct mlx5_core_dev *dev = esw->dev;
2242 struct mlx5_flow_namespace *ns;
2243 struct mlx5_modify_hdr *mod_hdr;
2244 void *match_criteria, *misc;
2245 struct mlx5_flow_table *ft;
2246 struct mlx5_flow_group *g;
2247 u32 *flow_group_in;
2248 int err = 0;
2249
2250 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2251 return 0;
2252
2253 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2254 if (!ns) {
2255 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2256 return -EOPNOTSUPP;
2257 }
2258
2259 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2260 if (!flow_group_in) {
2261 err = -ENOMEM;
2262 goto out_free;
2263 }
2264
2265 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
2266 ft = mlx5_create_flow_table(ns, &ft_attr);
2267 if (IS_ERR(ft)) {
2268 err = PTR_ERR(ft);
2269 esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2270 err);
2271 goto out_free;
2272 }
2273
2274 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2275 match_criteria);
2276 misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2277 misc_parameters_2);
2278
2279 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
2280 ESW_REG_C0_USER_DATA_METADATA_MASK);
2281 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2282 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2283 ft_attr.max_fte - 1);
2284 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2285 MLX5_MATCH_MISC_PARAMETERS_2);
2286 g = mlx5_create_flow_group(ft, flow_group_in);
2287 if (IS_ERR(g)) {
2288 err = PTR_ERR(g);
2289 esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2290 err);
2291 goto err_group;
2292 }
2293
2294 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2295 MLX5_SET(copy_action_in, modact, src_field,
2296 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2297 MLX5_SET(copy_action_in, modact, dst_field,
2298 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2299 mod_hdr = mlx5_modify_header_alloc(esw->dev,
2300 MLX5_FLOW_NAMESPACE_KERNEL, 1,
2301 modact);
2302 if (IS_ERR(mod_hdr)) {
2303 err = PTR_ERR(mod_hdr);
2304 esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2305 err);
2306 goto err_mod_hdr;
2307 }
2308
2309 esw->offloads.ft_offloads_restore = ft;
2310 esw->offloads.restore_group = g;
2311 esw->offloads.restore_copy_hdr_id = mod_hdr;
2312
2313 kvfree(flow_group_in);
2314
2315 return 0;
2316
2317 err_mod_hdr:
2318 mlx5_destroy_flow_group(g);
2319 err_group:
2320 mlx5_destroy_flow_table(ft);
2321 out_free:
2322 kvfree(flow_group_in);
2323
2324 return err;
2325 }
2326
esw_offloads_start(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)2327 static int esw_offloads_start(struct mlx5_eswitch *esw,
2328 struct netlink_ext_ack *extack)
2329 {
2330 int err;
2331
2332 esw->mode = MLX5_ESWITCH_OFFLOADS;
2333 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs);
2334 if (err) {
2335 NL_SET_ERR_MSG_MOD(extack,
2336 "Failed setting eswitch to offloads");
2337 esw->mode = MLX5_ESWITCH_LEGACY;
2338 mlx5_rescan_drivers(esw->dev);
2339 return err;
2340 }
2341 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2342 if (mlx5_eswitch_inline_mode_get(esw,
2343 &esw->offloads.inline_mode)) {
2344 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
2345 NL_SET_ERR_MSG_MOD(extack,
2346 "Inline mode is different between vports");
2347 }
2348 }
2349 return 0;
2350 }
2351
mlx5_esw_offloads_rep_init(struct mlx5_eswitch * esw,const struct mlx5_vport * vport)2352 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport)
2353 {
2354 struct mlx5_eswitch_rep *rep;
2355 int rep_type;
2356 int err;
2357
2358 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
2359 if (!rep)
2360 return -ENOMEM;
2361
2362 rep->vport = vport->vport;
2363 rep->vport_index = vport->index;
2364 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2365 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
2366
2367 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL);
2368 if (err)
2369 goto insert_err;
2370
2371 return 0;
2372
2373 insert_err:
2374 kfree(rep);
2375 return err;
2376 }
2377
mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep)2378 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2379 struct mlx5_eswitch_rep *rep)
2380 {
2381 xa_erase(&esw->offloads.vport_reps, rep->vport);
2382 kfree(rep);
2383 }
2384
esw_offloads_cleanup_reps(struct mlx5_eswitch * esw)2385 static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2386 {
2387 struct mlx5_eswitch_rep *rep;
2388 unsigned long i;
2389
2390 mlx5_esw_for_each_rep(esw, i, rep)
2391 mlx5_esw_offloads_rep_cleanup(esw, rep);
2392 xa_destroy(&esw->offloads.vport_reps);
2393 }
2394
esw_offloads_init_reps(struct mlx5_eswitch * esw)2395 static int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2396 {
2397 struct mlx5_vport *vport;
2398 unsigned long i;
2399 int err;
2400
2401 xa_init(&esw->offloads.vport_reps);
2402
2403 mlx5_esw_for_each_vport(esw, i, vport) {
2404 err = mlx5_esw_offloads_rep_init(esw, vport);
2405 if (err)
2406 goto err;
2407 }
2408 return 0;
2409
2410 err:
2411 esw_offloads_cleanup_reps(esw);
2412 return err;
2413 }
2414
esw_port_metadata_set(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx,struct netlink_ext_ack * extack)2415 static int esw_port_metadata_set(struct devlink *devlink, u32 id,
2416 struct devlink_param_gset_ctx *ctx,
2417 struct netlink_ext_ack *extack)
2418 {
2419 struct mlx5_core_dev *dev = devlink_priv(devlink);
2420 struct mlx5_eswitch *esw = dev->priv.eswitch;
2421 int err = 0;
2422
2423 down_write(&esw->mode_lock);
2424 if (mlx5_esw_is_fdb_created(esw)) {
2425 err = -EBUSY;
2426 goto done;
2427 }
2428 if (!mlx5_esw_vport_match_metadata_supported(esw)) {
2429 err = -EOPNOTSUPP;
2430 goto done;
2431 }
2432 if (ctx->val.vbool)
2433 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2434 else
2435 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
2436 done:
2437 up_write(&esw->mode_lock);
2438 return err;
2439 }
2440
esw_port_metadata_get(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx)2441 static int esw_port_metadata_get(struct devlink *devlink, u32 id,
2442 struct devlink_param_gset_ctx *ctx)
2443 {
2444 struct mlx5_core_dev *dev = devlink_priv(devlink);
2445
2446 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch);
2447 return 0;
2448 }
2449
esw_port_metadata_validate(struct devlink * devlink,u32 id,union devlink_param_value val,struct netlink_ext_ack * extack)2450 static int esw_port_metadata_validate(struct devlink *devlink, u32 id,
2451 union devlink_param_value val,
2452 struct netlink_ext_ack *extack)
2453 {
2454 struct mlx5_core_dev *dev = devlink_priv(devlink);
2455 u8 esw_mode;
2456
2457 esw_mode = mlx5_eswitch_mode(dev);
2458 if (esw_mode == MLX5_ESWITCH_OFFLOADS) {
2459 NL_SET_ERR_MSG_MOD(extack,
2460 "E-Switch must either disabled or non switchdev mode");
2461 return -EBUSY;
2462 }
2463 return 0;
2464 }
2465
2466 static const struct devlink_param esw_devlink_params[] = {
2467 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA,
2468 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL,
2469 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2470 esw_port_metadata_get,
2471 esw_port_metadata_set,
2472 esw_port_metadata_validate),
2473 };
2474
esw_offloads_init(struct mlx5_eswitch * esw)2475 int esw_offloads_init(struct mlx5_eswitch *esw)
2476 {
2477 int err;
2478
2479 err = esw_offloads_init_reps(esw);
2480 if (err)
2481 return err;
2482
2483 if (MLX5_ESWITCH_MANAGER(esw->dev) &&
2484 mlx5_esw_vport_match_metadata_supported(esw))
2485 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2486
2487 err = devl_params_register(priv_to_devlink(esw->dev),
2488 esw_devlink_params,
2489 ARRAY_SIZE(esw_devlink_params));
2490 if (err)
2491 goto err_params;
2492
2493 return 0;
2494
2495 err_params:
2496 esw_offloads_cleanup_reps(esw);
2497 return err;
2498 }
2499
esw_offloads_cleanup(struct mlx5_eswitch * esw)2500 void esw_offloads_cleanup(struct mlx5_eswitch *esw)
2501 {
2502 devl_params_unregister(priv_to_devlink(esw->dev),
2503 esw_devlink_params,
2504 ARRAY_SIZE(esw_devlink_params));
2505 esw_offloads_cleanup_reps(esw);
2506 }
2507
__esw_offloads_load_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2508 static int __esw_offloads_load_rep(struct mlx5_eswitch *esw,
2509 struct mlx5_eswitch_rep *rep, u8 rep_type)
2510 {
2511 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2512 REP_REGISTERED, REP_LOADED) == REP_REGISTERED)
2513 return esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2514
2515 return 0;
2516 }
2517
__esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_eswitch_rep * rep,u8 rep_type)2518 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2519 struct mlx5_eswitch_rep *rep, u8 rep_type)
2520 {
2521 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2522 REP_LOADED, REP_REGISTERED) == REP_LOADED)
2523 esw->offloads.rep_ops[rep_type]->unload(rep);
2524 }
2525
__unload_reps_all_vport(struct mlx5_eswitch * esw,u8 rep_type)2526 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
2527 {
2528 struct mlx5_eswitch_rep *rep;
2529 unsigned long i;
2530
2531 mlx5_esw_for_each_rep(esw, i, rep)
2532 __esw_offloads_unload_rep(esw, rep, rep_type);
2533 }
2534
mlx5_esw_offloads_rep_load(struct mlx5_eswitch * esw,u16 vport_num)2535 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
2536 {
2537 struct mlx5_eswitch_rep *rep;
2538 int rep_type;
2539 int err;
2540
2541 rep = mlx5_eswitch_get_rep(esw, vport_num);
2542 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2543 err = __esw_offloads_load_rep(esw, rep, rep_type);
2544 if (err)
2545 goto err_reps;
2546 }
2547
2548 return 0;
2549
2550 err_reps:
2551 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2552 for (--rep_type; rep_type >= 0; rep_type--)
2553 __esw_offloads_unload_rep(esw, rep, rep_type);
2554 return err;
2555 }
2556
mlx5_esw_offloads_rep_unload(struct mlx5_eswitch * esw,u16 vport_num)2557 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
2558 {
2559 struct mlx5_eswitch_rep *rep;
2560 int rep_type;
2561
2562 rep = mlx5_eswitch_get_rep(esw, vport_num);
2563 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2564 __esw_offloads_unload_rep(esw, rep, rep_type);
2565 }
2566
mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2567 int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2568 {
2569 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2570 return 0;
2571
2572 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport);
2573 }
2574
mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2575 void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2576 {
2577 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2578 return;
2579
2580 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport);
2581 }
2582
mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport,struct mlx5_devlink_port * dl_port,u32 controller,u32 sfnum)2583 int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
2584 struct mlx5_devlink_port *dl_port,
2585 u32 controller, u32 sfnum)
2586 {
2587 return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum);
2588 }
2589
mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2590 void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2591 {
2592 mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport);
2593 }
2594
mlx5_esw_offloads_load_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2595 int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2596 {
2597 int err;
2598
2599 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2600 return 0;
2601
2602 err = mlx5_esw_offloads_devlink_port_register(esw, vport);
2603 if (err)
2604 return err;
2605
2606 err = mlx5_esw_offloads_rep_load(esw, vport->vport);
2607 if (err)
2608 goto load_err;
2609 return err;
2610
2611 load_err:
2612 mlx5_esw_offloads_devlink_port_unregister(esw, vport);
2613 return err;
2614 }
2615
mlx5_esw_offloads_unload_rep(struct mlx5_eswitch * esw,struct mlx5_vport * vport)2616 void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
2617 {
2618 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2619 return;
2620
2621 mlx5_esw_offloads_rep_unload(esw, vport->vport);
2622
2623 mlx5_esw_offloads_devlink_port_unregister(esw, vport);
2624 }
2625
esw_set_slave_root_fdb(struct mlx5_core_dev * master,struct mlx5_core_dev * slave)2626 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2627 struct mlx5_core_dev *slave)
2628 {
2629 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2630 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2631 struct mlx5_flow_root_namespace *root;
2632 struct mlx5_flow_namespace *ns;
2633 int err;
2634
2635 MLX5_SET(set_flow_table_root_in, in, opcode,
2636 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2637 MLX5_SET(set_flow_table_root_in, in, table_type,
2638 FS_FT_FDB);
2639
2640 if (master) {
2641 ns = mlx5_get_flow_namespace(master,
2642 MLX5_FLOW_NAMESPACE_FDB);
2643 root = find_root(&ns->node);
2644 mutex_lock(&root->chain_lock);
2645 MLX5_SET(set_flow_table_root_in, in,
2646 table_eswitch_owner_vhca_id_valid, 1);
2647 MLX5_SET(set_flow_table_root_in, in,
2648 table_eswitch_owner_vhca_id,
2649 MLX5_CAP_GEN(master, vhca_id));
2650 MLX5_SET(set_flow_table_root_in, in, table_id,
2651 root->root_ft->id);
2652 } else {
2653 ns = mlx5_get_flow_namespace(slave,
2654 MLX5_FLOW_NAMESPACE_FDB);
2655 root = find_root(&ns->node);
2656 mutex_lock(&root->chain_lock);
2657 MLX5_SET(set_flow_table_root_in, in, table_id,
2658 root->root_ft->id);
2659 }
2660
2661 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2662 mutex_unlock(&root->chain_lock);
2663
2664 return err;
2665 }
2666
__esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,struct mlx5_vport * vport,struct mlx5_flow_table * acl)2667 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2668 struct mlx5_core_dev *slave,
2669 struct mlx5_vport *vport,
2670 struct mlx5_flow_table *acl)
2671 {
2672 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2673 struct mlx5_flow_handle *flow_rule = NULL;
2674 struct mlx5_flow_destination dest = {};
2675 struct mlx5_flow_act flow_act = {};
2676 struct mlx5_flow_spec *spec;
2677 int err = 0;
2678 void *misc;
2679
2680 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2681 if (!spec)
2682 return -ENOMEM;
2683
2684 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2685 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2686 misc_parameters);
2687 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2688 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index);
2689
2690 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2691 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2692 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2693 source_eswitch_owner_vhca_id);
2694
2695 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2696 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2697 dest.vport.num = slave->priv.eswitch->manager_vport;
2698 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2699 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2700
2701 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act,
2702 &dest, 1);
2703 if (IS_ERR(flow_rule)) {
2704 err = PTR_ERR(flow_rule);
2705 } else {
2706 err = xa_insert(&vport->egress.offloads.bounce_rules,
2707 slave_index, flow_rule, GFP_KERNEL);
2708 if (err)
2709 mlx5_del_flow_rules(flow_rule);
2710 }
2711
2712 kvfree(spec);
2713 return err;
2714 }
2715
esw_master_egress_create_resources(struct mlx5_eswitch * esw,struct mlx5_flow_namespace * egress_ns,struct mlx5_vport * vport,size_t count)2716 static int esw_master_egress_create_resources(struct mlx5_eswitch *esw,
2717 struct mlx5_flow_namespace *egress_ns,
2718 struct mlx5_vport *vport, size_t count)
2719 {
2720 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2721 struct mlx5_flow_table_attr ft_attr = {
2722 .max_fte = count, .prio = 0, .level = 0,
2723 };
2724 struct mlx5_flow_table *acl;
2725 struct mlx5_flow_group *g;
2726 void *match_criteria;
2727 u32 *flow_group_in;
2728 int err;
2729
2730 if (vport->egress.acl)
2731 return 0;
2732
2733 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2734 if (!flow_group_in)
2735 return -ENOMEM;
2736
2737 if (vport->vport || mlx5_core_is_ecpf(esw->dev))
2738 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT;
2739
2740 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport);
2741 if (IS_ERR(acl)) {
2742 err = PTR_ERR(acl);
2743 goto out;
2744 }
2745
2746 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2747 match_criteria);
2748 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2749 misc_parameters.source_port);
2750 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2751 misc_parameters.source_eswitch_owner_vhca_id);
2752 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2753 MLX5_MATCH_MISC_PARAMETERS);
2754
2755 MLX5_SET(create_flow_group_in, flow_group_in,
2756 source_eswitch_owner_vhca_id_valid, 1);
2757 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2758 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count);
2759
2760 g = mlx5_create_flow_group(acl, flow_group_in);
2761 if (IS_ERR(g)) {
2762 err = PTR_ERR(g);
2763 goto err_group;
2764 }
2765
2766 vport->egress.acl = acl;
2767 vport->egress.offloads.bounce_grp = g;
2768 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB;
2769 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC);
2770
2771 kvfree(flow_group_in);
2772
2773 return 0;
2774
2775 err_group:
2776 mlx5_destroy_flow_table(acl);
2777 out:
2778 kvfree(flow_group_in);
2779 return err;
2780 }
2781
esw_master_egress_destroy_resources(struct mlx5_vport * vport)2782 static void esw_master_egress_destroy_resources(struct mlx5_vport *vport)
2783 {
2784 if (!xa_empty(&vport->egress.offloads.bounce_rules))
2785 return;
2786 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp);
2787 vport->egress.offloads.bounce_grp = NULL;
2788 mlx5_destroy_flow_table(vport->egress.acl);
2789 vport->egress.acl = NULL;
2790 }
2791
esw_set_master_egress_rule(struct mlx5_core_dev * master,struct mlx5_core_dev * slave,size_t count)2792 static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2793 struct mlx5_core_dev *slave, size_t count)
2794 {
2795 struct mlx5_eswitch *esw = master->priv.eswitch;
2796 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id);
2797 struct mlx5_flow_namespace *egress_ns;
2798 struct mlx5_vport *vport;
2799 int err;
2800
2801 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
2802 if (IS_ERR(vport))
2803 return PTR_ERR(vport);
2804
2805 egress_ns = mlx5_get_flow_vport_acl_namespace(master,
2806 MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2807 vport->index);
2808 if (!egress_ns)
2809 return -EINVAL;
2810
2811 if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB)
2812 return 0;
2813
2814 err = esw_master_egress_create_resources(esw, egress_ns, vport, count);
2815 if (err)
2816 return err;
2817
2818 if (xa_load(&vport->egress.offloads.bounce_rules, slave_index))
2819 return -EINVAL;
2820
2821 err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl);
2822 if (err)
2823 goto err_rule;
2824
2825 return 0;
2826
2827 err_rule:
2828 esw_master_egress_destroy_resources(vport);
2829 return err;
2830 }
2831
esw_unset_master_egress_rule(struct mlx5_core_dev * dev,struct mlx5_core_dev * slave_dev)2832 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev,
2833 struct mlx5_core_dev *slave_dev)
2834 {
2835 struct mlx5_vport *vport;
2836
2837 vport = mlx5_eswitch_get_vport(dev->priv.eswitch,
2838 dev->priv.eswitch->manager_vport);
2839
2840 esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id));
2841
2842 if (xa_empty(&vport->egress.offloads.bounce_rules)) {
2843 esw_acl_egress_ofld_cleanup(vport);
2844 xa_destroy(&vport->egress.offloads.bounce_rules);
2845 }
2846 }
2847
mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw,int max_slaves)2848 int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw,
2849 struct mlx5_eswitch *slave_esw, int max_slaves)
2850 {
2851 int err;
2852
2853 err = esw_set_slave_root_fdb(master_esw->dev,
2854 slave_esw->dev);
2855 if (err)
2856 return err;
2857
2858 err = esw_set_master_egress_rule(master_esw->dev,
2859 slave_esw->dev, max_slaves);
2860 if (err)
2861 goto err_acl;
2862
2863 return err;
2864
2865 err_acl:
2866 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2867 return err;
2868 }
2869
mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch * master_esw,struct mlx5_eswitch * slave_esw)2870 void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw,
2871 struct mlx5_eswitch *slave_esw)
2872 {
2873 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2874 esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev);
2875 }
2876
2877 #define ESW_OFFLOADS_DEVCOM_PAIR (0)
2878 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2879
mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2880 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw,
2881 struct mlx5_eswitch *peer_esw)
2882 {
2883 const struct mlx5_eswitch_rep_ops *ops;
2884 struct mlx5_eswitch_rep *rep;
2885 unsigned long i;
2886 u8 rep_type;
2887
2888 mlx5_esw_for_each_rep(esw, i, rep) {
2889 rep_type = NUM_REP_TYPES;
2890 while (rep_type--) {
2891 ops = esw->offloads.rep_ops[rep_type];
2892 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2893 ops->event)
2894 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw);
2895 }
2896 }
2897 }
2898
mlx5_esw_offloads_unpair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2899 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw,
2900 struct mlx5_eswitch *peer_esw)
2901 {
2902 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
2903 mlx5e_tc_clean_fdb_peer_flows(esw);
2904 #endif
2905 mlx5_esw_offloads_rep_event_unpair(esw, peer_esw);
2906 esw_del_fdb_peer_miss_rules(esw, peer_esw->dev);
2907 }
2908
mlx5_esw_offloads_pair(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw)2909 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2910 struct mlx5_eswitch *peer_esw)
2911 {
2912 const struct mlx5_eswitch_rep_ops *ops;
2913 struct mlx5_eswitch_rep *rep;
2914 unsigned long i;
2915 u8 rep_type;
2916 int err;
2917
2918 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
2919 if (err)
2920 return err;
2921
2922 mlx5_esw_for_each_rep(esw, i, rep) {
2923 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2924 ops = esw->offloads.rep_ops[rep_type];
2925 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2926 ops->event) {
2927 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
2928 if (err)
2929 goto err_out;
2930 }
2931 }
2932 }
2933
2934 return 0;
2935
2936 err_out:
2937 mlx5_esw_offloads_unpair(esw, peer_esw);
2938 return err;
2939 }
2940
mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch * esw,struct mlx5_eswitch * peer_esw,bool pair)2941 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2942 struct mlx5_eswitch *peer_esw,
2943 bool pair)
2944 {
2945 u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
2946 u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
2947 struct mlx5_flow_root_namespace *peer_ns;
2948 struct mlx5_flow_root_namespace *ns;
2949 int err;
2950
2951 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2952 ns = esw->dev->priv.steering->fdb_root_ns;
2953
2954 if (pair) {
2955 err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id);
2956 if (err)
2957 return err;
2958
2959 err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id);
2960 if (err) {
2961 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
2962 return err;
2963 }
2964 } else {
2965 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id);
2966 mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id);
2967 }
2968
2969 return 0;
2970 }
2971
mlx5_esw_offloads_devcom_event(int event,void * my_data,void * event_data)2972 static int mlx5_esw_offloads_devcom_event(int event,
2973 void *my_data,
2974 void *event_data)
2975 {
2976 struct mlx5_eswitch *esw = my_data;
2977 struct mlx5_eswitch *peer_esw = event_data;
2978 u16 esw_i, peer_esw_i;
2979 bool esw_paired;
2980 int err;
2981
2982 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
2983 esw_i = MLX5_CAP_GEN(esw->dev, vhca_id);
2984 esw_paired = !!xa_load(&esw->paired, peer_esw_i);
2985
2986 switch (event) {
2987 case ESW_OFFLOADS_DEVCOM_PAIR:
2988 if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2989 mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
2990 break;
2991
2992 if (esw_paired)
2993 break;
2994
2995 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
2996 if (err)
2997 goto err_out;
2998
2999 err = mlx5_esw_offloads_pair(esw, peer_esw);
3000 if (err)
3001 goto err_peer;
3002
3003 err = mlx5_esw_offloads_pair(peer_esw, esw);
3004 if (err)
3005 goto err_pair;
3006
3007 err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL);
3008 if (err)
3009 goto err_xa;
3010
3011 err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL);
3012 if (err)
3013 goto err_peer_xa;
3014
3015 esw->num_peers++;
3016 peer_esw->num_peers++;
3017 mlx5_devcom_comp_set_ready(esw->devcom, true);
3018 break;
3019
3020 case ESW_OFFLOADS_DEVCOM_UNPAIR:
3021 if (!esw_paired)
3022 break;
3023
3024 peer_esw->num_peers--;
3025 esw->num_peers--;
3026 if (!esw->num_peers && !peer_esw->num_peers)
3027 mlx5_devcom_comp_set_ready(esw->devcom, false);
3028 xa_erase(&peer_esw->paired, esw_i);
3029 xa_erase(&esw->paired, peer_esw_i);
3030 mlx5_esw_offloads_unpair(peer_esw, esw);
3031 mlx5_esw_offloads_unpair(esw, peer_esw);
3032 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
3033 break;
3034 }
3035
3036 return 0;
3037
3038 err_peer_xa:
3039 xa_erase(&esw->paired, peer_esw_i);
3040 err_xa:
3041 mlx5_esw_offloads_unpair(peer_esw, esw);
3042 err_pair:
3043 mlx5_esw_offloads_unpair(esw, peer_esw);
3044 err_peer:
3045 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
3046 err_out:
3047 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
3048 event, err);
3049 return err;
3050 }
3051
mlx5_esw_offloads_devcom_init(struct mlx5_eswitch * esw,u64 key)3052 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key)
3053 {
3054 int i;
3055
3056 for (i = 0; i < MLX5_MAX_PORTS; i++)
3057 INIT_LIST_HEAD(&esw->offloads.peer_flows[i]);
3058 mutex_init(&esw->offloads.peer_mutex);
3059
3060 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
3061 return;
3062
3063 if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) &&
3064 !mlx5_lag_is_supported(esw->dev))
3065 return;
3066
3067 xa_init(&esw->paired);
3068 esw->num_peers = 0;
3069 esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc,
3070 MLX5_DEVCOM_ESW_OFFLOADS,
3071 key,
3072 mlx5_esw_offloads_devcom_event,
3073 esw);
3074 if (IS_ERR(esw->devcom))
3075 return;
3076
3077 mlx5_devcom_send_event(esw->devcom,
3078 ESW_OFFLOADS_DEVCOM_PAIR,
3079 ESW_OFFLOADS_DEVCOM_UNPAIR,
3080 esw);
3081 }
3082
mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch * esw)3083 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
3084 {
3085 if (IS_ERR_OR_NULL(esw->devcom))
3086 return;
3087
3088 mlx5_devcom_send_event(esw->devcom,
3089 ESW_OFFLOADS_DEVCOM_UNPAIR,
3090 ESW_OFFLOADS_DEVCOM_UNPAIR,
3091 esw);
3092
3093 mlx5_devcom_unregister_component(esw->devcom);
3094 xa_destroy(&esw->paired);
3095 esw->devcom = NULL;
3096 }
3097
mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch * esw)3098 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw)
3099 {
3100 return mlx5_devcom_comp_is_ready(esw->devcom);
3101 }
3102
mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch * esw)3103 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
3104 {
3105 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
3106 return false;
3107
3108 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
3109 MLX5_FDB_TO_VPORT_REG_C_0))
3110 return false;
3111
3112 return true;
3113 }
3114
3115 #define MLX5_ESW_METADATA_RSVD_UPLINK 1
3116
3117 /* Share the same metadata for uplink's. This is fine because:
3118 * (a) In shared FDB mode (LAG) both uplink's are treated the
3119 * same and tagged with the same metadata.
3120 * (b) In non shared FDB mode, packets from physical port0
3121 * cannot hit eswitch of PF1 and vice versa.
3122 */
mlx5_esw_match_metadata_reserved(struct mlx5_eswitch * esw)3123 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw)
3124 {
3125 return MLX5_ESW_METADATA_RSVD_UPLINK;
3126 }
3127
mlx5_esw_match_metadata_alloc(struct mlx5_eswitch * esw)3128 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
3129 {
3130 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
3131 /* Reserve 0xf for internal port offload */
3132 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2;
3133 u32 pf_num;
3134 int id;
3135
3136 /* Only 4 bits of pf_num */
3137 pf_num = mlx5_get_dev_index(esw->dev);
3138 if (pf_num > max_pf_num)
3139 return 0;
3140
3141 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
3142 /* Use only non-zero vport_id (2-4095) for all PF's */
3143 id = ida_alloc_range(&esw->offloads.vport_metadata_ida,
3144 MLX5_ESW_METADATA_RSVD_UPLINK + 1,
3145 vport_end_ida, GFP_KERNEL);
3146 if (id < 0)
3147 return 0;
3148 id = (pf_num << ESW_VPORT_BITS) | id;
3149 return id;
3150 }
3151
mlx5_esw_match_metadata_free(struct mlx5_eswitch * esw,u32 metadata)3152 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
3153 {
3154 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
3155
3156 /* Metadata contains only 12 bits of actual ida id */
3157 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
3158 }
3159
esw_offloads_vport_metadata_setup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3160 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
3161 struct mlx5_vport *vport)
3162 {
3163 if (vport->vport == MLX5_VPORT_UPLINK)
3164 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw);
3165 else
3166 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
3167
3168 vport->metadata = vport->default_metadata;
3169 return vport->metadata ? 0 : -ENOSPC;
3170 }
3171
esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3172 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
3173 struct mlx5_vport *vport)
3174 {
3175 if (!vport->default_metadata)
3176 return;
3177
3178 if (vport->vport == MLX5_VPORT_UPLINK)
3179 return;
3180
3181 WARN_ON(vport->metadata != vport->default_metadata);
3182 mlx5_esw_match_metadata_free(esw, vport->default_metadata);
3183 }
3184
esw_offloads_metadata_uninit(struct mlx5_eswitch * esw)3185 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
3186 {
3187 struct mlx5_vport *vport;
3188 unsigned long i;
3189
3190 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3191 return;
3192
3193 mlx5_esw_for_each_vport(esw, i, vport)
3194 esw_offloads_vport_metadata_cleanup(esw, vport);
3195 }
3196
esw_offloads_metadata_init(struct mlx5_eswitch * esw)3197 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
3198 {
3199 struct mlx5_vport *vport;
3200 unsigned long i;
3201 int err;
3202
3203 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
3204 return 0;
3205
3206 mlx5_esw_for_each_vport(esw, i, vport) {
3207 err = esw_offloads_vport_metadata_setup(esw, vport);
3208 if (err)
3209 goto metadata_err;
3210 }
3211
3212 return 0;
3213
3214 metadata_err:
3215 esw_offloads_metadata_uninit(esw);
3216 return err;
3217 }
3218
3219 int
esw_vport_create_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3220 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
3221 struct mlx5_vport *vport)
3222 {
3223 int err;
3224
3225 err = esw_acl_ingress_ofld_setup(esw, vport);
3226 if (err)
3227 return err;
3228
3229 err = esw_acl_egress_ofld_setup(esw, vport);
3230 if (err)
3231 goto egress_err;
3232
3233 return 0;
3234
3235 egress_err:
3236 esw_acl_ingress_ofld_cleanup(esw, vport);
3237 return err;
3238 }
3239
3240 void
esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch * esw,struct mlx5_vport * vport)3241 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
3242 struct mlx5_vport *vport)
3243 {
3244 esw_acl_egress_ofld_cleanup(vport);
3245 esw_acl_ingress_ofld_cleanup(esw, vport);
3246 }
3247
esw_create_offloads_acl_tables(struct mlx5_eswitch * esw)3248 static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw)
3249 {
3250 struct mlx5_vport *uplink, *manager;
3251 int ret;
3252
3253 uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3254 if (IS_ERR(uplink))
3255 return PTR_ERR(uplink);
3256
3257 ret = esw_vport_create_offloads_acl_tables(esw, uplink);
3258 if (ret)
3259 return ret;
3260
3261 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3262 if (IS_ERR(manager)) {
3263 ret = PTR_ERR(manager);
3264 goto err_manager;
3265 }
3266
3267 ret = esw_vport_create_offloads_acl_tables(esw, manager);
3268 if (ret)
3269 goto err_manager;
3270
3271 return 0;
3272
3273 err_manager:
3274 esw_vport_destroy_offloads_acl_tables(esw, uplink);
3275 return ret;
3276 }
3277
esw_destroy_offloads_acl_tables(struct mlx5_eswitch * esw)3278 static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw)
3279 {
3280 struct mlx5_vport *vport;
3281
3282 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
3283 if (!IS_ERR(vport))
3284 esw_vport_destroy_offloads_acl_tables(esw, vport);
3285
3286 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
3287 if (!IS_ERR(vport))
3288 esw_vport_destroy_offloads_acl_tables(esw, vport);
3289 }
3290
mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch * esw)3291 int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw)
3292 {
3293 struct mlx5_eswitch_rep *rep;
3294 unsigned long i;
3295 int ret;
3296
3297 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
3298 return 0;
3299
3300 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
3301 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
3302 return 0;
3303
3304 ret = __esw_offloads_load_rep(esw, rep, REP_IB);
3305 if (ret)
3306 return ret;
3307
3308 mlx5_esw_for_each_rep(esw, i, rep) {
3309 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
3310 __esw_offloads_load_rep(esw, rep, REP_IB);
3311 }
3312
3313 return 0;
3314 }
3315
esw_offloads_steering_init(struct mlx5_eswitch * esw)3316 static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
3317 {
3318 struct mlx5_esw_indir_table *indir;
3319 int err;
3320
3321 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
3322 mutex_init(&esw->fdb_table.offloads.vports.lock);
3323 hash_init(esw->fdb_table.offloads.vports.table);
3324 atomic64_set(&esw->user_count, 0);
3325
3326 indir = mlx5_esw_indir_table_init();
3327 if (IS_ERR(indir)) {
3328 err = PTR_ERR(indir);
3329 goto create_indir_err;
3330 }
3331 esw->fdb_table.offloads.indir = indir;
3332
3333 err = esw_create_offloads_acl_tables(esw);
3334 if (err)
3335 goto create_acl_err;
3336
3337 err = esw_create_offloads_table(esw);
3338 if (err)
3339 goto create_offloads_err;
3340
3341 err = esw_create_restore_table(esw);
3342 if (err)
3343 goto create_restore_err;
3344
3345 err = esw_create_offloads_fdb_tables(esw);
3346 if (err)
3347 goto create_fdb_err;
3348
3349 err = esw_create_vport_rx_group(esw);
3350 if (err)
3351 goto create_fg_err;
3352
3353 err = esw_create_vport_rx_drop_group(esw);
3354 if (err)
3355 goto create_rx_drop_fg_err;
3356
3357 err = esw_create_vport_rx_drop_rule(esw);
3358 if (err)
3359 goto create_rx_drop_rule_err;
3360
3361 return 0;
3362
3363 create_rx_drop_rule_err:
3364 esw_destroy_vport_rx_drop_group(esw);
3365 create_rx_drop_fg_err:
3366 esw_destroy_vport_rx_group(esw);
3367 create_fg_err:
3368 esw_destroy_offloads_fdb_tables(esw);
3369 create_fdb_err:
3370 esw_destroy_restore_table(esw);
3371 create_restore_err:
3372 esw_destroy_offloads_table(esw);
3373 create_offloads_err:
3374 esw_destroy_offloads_acl_tables(esw);
3375 create_acl_err:
3376 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3377 create_indir_err:
3378 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3379 return err;
3380 }
3381
esw_offloads_steering_cleanup(struct mlx5_eswitch * esw)3382 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3383 {
3384 esw_destroy_vport_rx_drop_rule(esw);
3385 esw_destroy_vport_rx_drop_group(esw);
3386 esw_destroy_vport_rx_group(esw);
3387 esw_destroy_offloads_fdb_tables(esw);
3388 esw_destroy_restore_table(esw);
3389 esw_destroy_offloads_table(esw);
3390 esw_destroy_offloads_acl_tables(esw);
3391 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3392 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
3393 }
3394
3395 static void
esw_vfs_changed_event_handler(struct mlx5_eswitch * esw,const u32 * out)3396 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
3397 {
3398 struct devlink *devlink;
3399 bool host_pf_disabled;
3400 u16 new_num_vfs;
3401
3402 new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3403 host_params_context.host_num_of_vfs);
3404 host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3405 host_params_context.host_pf_disabled);
3406
3407 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3408 return;
3409
3410 devlink = priv_to_devlink(esw->dev);
3411 devl_lock(devlink);
3412 /* Number of VFs can only change from "0 to x" or "x to 0". */
3413 if (esw->esw_funcs.num_vfs > 0) {
3414 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
3415 } else {
3416 int err;
3417
3418 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
3419 MLX5_VPORT_UC_ADDR_CHANGE);
3420 if (err) {
3421 devl_unlock(devlink);
3422 return;
3423 }
3424 }
3425 esw->esw_funcs.num_vfs = new_num_vfs;
3426 devl_unlock(devlink);
3427 }
3428
esw_functions_changed_event_handler(struct work_struct * work)3429 static void esw_functions_changed_event_handler(struct work_struct *work)
3430 {
3431 struct mlx5_host_work *host_work;
3432 struct mlx5_eswitch *esw;
3433 const u32 *out;
3434
3435 host_work = container_of(work, struct mlx5_host_work, work);
3436 esw = host_work->esw;
3437
3438 out = mlx5_esw_query_functions(esw->dev);
3439 if (IS_ERR(out))
3440 goto out;
3441
3442 esw_vfs_changed_event_handler(esw, out);
3443 kvfree(out);
3444 out:
3445 kfree(host_work);
3446 }
3447
mlx5_esw_funcs_changed_handler(struct notifier_block * nb,unsigned long type,void * data)3448 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
3449 {
3450 struct mlx5_esw_functions *esw_funcs;
3451 struct mlx5_host_work *host_work;
3452 struct mlx5_eswitch *esw;
3453
3454 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
3455 if (!host_work)
3456 return NOTIFY_DONE;
3457
3458 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3459 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
3460
3461 host_work->esw = esw;
3462
3463 INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
3464 queue_work(esw->work_queue, &host_work->work);
3465
3466 return NOTIFY_OK;
3467 }
3468
mlx5_esw_host_number_init(struct mlx5_eswitch * esw)3469 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3470 {
3471 const u32 *query_host_out;
3472
3473 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3474 return 0;
3475
3476 query_host_out = mlx5_esw_query_functions(esw->dev);
3477 if (IS_ERR(query_host_out))
3478 return PTR_ERR(query_host_out);
3479
3480 /* Mark non local controller with non zero controller number. */
3481 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3482 host_params_context.host_number);
3483 kvfree(query_host_out);
3484 return 0;
3485 }
3486
mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch * esw,u32 controller)3487 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3488 {
3489 /* Local controller is always valid */
3490 if (controller == 0)
3491 return true;
3492
3493 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3494 return false;
3495
3496 /* External host number starts with zero in device */
3497 return (controller == esw->offloads.host_number + 1);
3498 }
3499
esw_offloads_enable(struct mlx5_eswitch * esw)3500 int esw_offloads_enable(struct mlx5_eswitch *esw)
3501 {
3502 struct mapping_ctx *reg_c0_obj_pool;
3503 struct mlx5_vport *vport;
3504 unsigned long i;
3505 u64 mapping_id;
3506 int err;
3507
3508 mutex_init(&esw->offloads.termtbl_mutex);
3509 mlx5_rdma_enable_roce(esw->dev);
3510
3511 err = mlx5_esw_host_number_init(esw);
3512 if (err)
3513 goto err_metadata;
3514
3515 err = esw_offloads_metadata_init(esw);
3516 if (err)
3517 goto err_metadata;
3518
3519 err = esw_set_passing_vport_metadata(esw, true);
3520 if (err)
3521 goto err_vport_metadata;
3522
3523 mapping_id = mlx5_query_nic_system_image_guid(esw->dev);
3524
3525 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN,
3526 sizeof(struct mlx5_mapped_obj),
3527 ESW_REG_C0_USER_DATA_METADATA_MASK,
3528 true);
3529
3530 if (IS_ERR(reg_c0_obj_pool)) {
3531 err = PTR_ERR(reg_c0_obj_pool);
3532 goto err_pool;
3533 }
3534 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3535
3536 err = esw_offloads_steering_init(esw);
3537 if (err)
3538 goto err_steering_init;
3539
3540 /* Representor will control the vport link state */
3541 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3542 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3543 if (mlx5_core_ec_sriov_enabled(esw->dev))
3544 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs)
3545 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3546
3547 /* Uplink vport rep must load first. */
3548 err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
3549 if (err)
3550 goto err_uplink;
3551
3552 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
3553 if (err)
3554 goto err_vports;
3555
3556 return 0;
3557
3558 err_vports:
3559 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK);
3560 err_uplink:
3561 esw_offloads_steering_cleanup(esw);
3562 err_steering_init:
3563 mapping_destroy(reg_c0_obj_pool);
3564 err_pool:
3565 esw_set_passing_vport_metadata(esw, false);
3566 err_vport_metadata:
3567 esw_offloads_metadata_uninit(esw);
3568 err_metadata:
3569 mlx5_rdma_disable_roce(esw->dev);
3570 mutex_destroy(&esw->offloads.termtbl_mutex);
3571 return err;
3572 }
3573
esw_offloads_stop(struct mlx5_eswitch * esw,struct netlink_ext_ack * extack)3574 static int esw_offloads_stop(struct mlx5_eswitch *esw,
3575 struct netlink_ext_ack *extack)
3576 {
3577 int err;
3578
3579 esw->mode = MLX5_ESWITCH_LEGACY;
3580
3581 /* If changing from switchdev to legacy mode without sriov enabled,
3582 * no need to create legacy fdb.
3583 */
3584 if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev))
3585 return 0;
3586
3587 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS);
3588 if (err)
3589 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
3590
3591 return err;
3592 }
3593
esw_offloads_disable(struct mlx5_eswitch * esw)3594 void esw_offloads_disable(struct mlx5_eswitch *esw)
3595 {
3596 mlx5_eswitch_disable_pf_vf_vports(esw);
3597 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK);
3598 esw_set_passing_vport_metadata(esw, false);
3599 esw_offloads_steering_cleanup(esw);
3600 mapping_destroy(esw->offloads.reg_c0_obj_pool);
3601 esw_offloads_metadata_uninit(esw);
3602 mlx5_rdma_disable_roce(esw->dev);
3603 mutex_destroy(&esw->offloads.termtbl_mutex);
3604 }
3605
esw_mode_from_devlink(u16 mode,u16 * mlx5_mode)3606 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
3607 {
3608 switch (mode) {
3609 case DEVLINK_ESWITCH_MODE_LEGACY:
3610 *mlx5_mode = MLX5_ESWITCH_LEGACY;
3611 break;
3612 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3613 *mlx5_mode = MLX5_ESWITCH_OFFLOADS;
3614 break;
3615 default:
3616 return -EINVAL;
3617 }
3618
3619 return 0;
3620 }
3621
esw_mode_to_devlink(u16 mlx5_mode,u16 * mode)3622 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
3623 {
3624 switch (mlx5_mode) {
3625 case MLX5_ESWITCH_LEGACY:
3626 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
3627 break;
3628 case MLX5_ESWITCH_OFFLOADS:
3629 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3630 break;
3631 default:
3632 return -EINVAL;
3633 }
3634
3635 return 0;
3636 }
3637
esw_inline_mode_from_devlink(u8 mode,u8 * mlx5_mode)3638 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3639 {
3640 switch (mode) {
3641 case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3642 *mlx5_mode = MLX5_INLINE_MODE_NONE;
3643 break;
3644 case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3645 *mlx5_mode = MLX5_INLINE_MODE_L2;
3646 break;
3647 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3648 *mlx5_mode = MLX5_INLINE_MODE_IP;
3649 break;
3650 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3651 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3652 break;
3653 default:
3654 return -EINVAL;
3655 }
3656
3657 return 0;
3658 }
3659
esw_inline_mode_to_devlink(u8 mlx5_mode,u8 * mode)3660 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3661 {
3662 switch (mlx5_mode) {
3663 case MLX5_INLINE_MODE_NONE:
3664 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3665 break;
3666 case MLX5_INLINE_MODE_L2:
3667 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3668 break;
3669 case MLX5_INLINE_MODE_IP:
3670 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3671 break;
3672 case MLX5_INLINE_MODE_TCP_UDP:
3673 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3674 break;
3675 default:
3676 return -EINVAL;
3677 }
3678
3679 return 0;
3680 }
3681
mlx5_eswitch_block_mode(struct mlx5_core_dev * dev)3682 int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev)
3683 {
3684 struct mlx5_eswitch *esw = dev->priv.eswitch;
3685 int err;
3686
3687 if (!mlx5_esw_allowed(esw))
3688 return 0;
3689
3690 /* Take TC into account */
3691 err = mlx5_esw_try_lock(esw);
3692 if (err < 0)
3693 return err;
3694
3695 esw->offloads.num_block_mode++;
3696 mlx5_esw_unlock(esw);
3697 return 0;
3698 }
3699
mlx5_eswitch_unblock_mode(struct mlx5_core_dev * dev)3700 void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev)
3701 {
3702 struct mlx5_eswitch *esw = dev->priv.eswitch;
3703
3704 if (!mlx5_esw_allowed(esw))
3705 return;
3706
3707 down_write(&esw->mode_lock);
3708 esw->offloads.num_block_mode--;
3709 up_write(&esw->mode_lock);
3710 }
3711
mlx5_devlink_eswitch_mode_set(struct devlink * devlink,u16 mode,struct netlink_ext_ack * extack)3712 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
3713 struct netlink_ext_ack *extack)
3714 {
3715 u16 cur_mlx5_mode, mlx5_mode = 0;
3716 struct mlx5_eswitch *esw;
3717 int err = 0;
3718
3719 esw = mlx5_devlink_eswitch_get(devlink);
3720 if (IS_ERR(esw))
3721 return PTR_ERR(esw);
3722
3723 if (esw_mode_from_devlink(mode, &mlx5_mode))
3724 return -EINVAL;
3725
3726 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && mlx5_get_sd(esw->dev)) {
3727 NL_SET_ERR_MSG_MOD(extack,
3728 "Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured.");
3729 return -EPERM;
3730 }
3731
3732 mlx5_lag_disable_change(esw->dev);
3733 err = mlx5_esw_try_lock(esw);
3734 if (err < 0) {
3735 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
3736 goto enable_lag;
3737 }
3738 cur_mlx5_mode = err;
3739 err = 0;
3740
3741 if (cur_mlx5_mode == mlx5_mode)
3742 goto unlock;
3743
3744 if (esw->offloads.num_block_mode) {
3745 NL_SET_ERR_MSG_MOD(extack,
3746 "Can't change eswitch mode when IPsec SA and/or policies are configured");
3747 err = -EOPNOTSUPP;
3748 goto unlock;
3749 }
3750
3751 esw->eswitch_operation_in_progress = true;
3752 up_write(&esw->mode_lock);
3753
3754 mlx5_eswitch_disable_locked(esw);
3755 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
3756 if (mlx5_devlink_trap_get_num_active(esw->dev)) {
3757 NL_SET_ERR_MSG_MOD(extack,
3758 "Can't change mode while devlink traps are active");
3759 err = -EOPNOTSUPP;
3760 goto skip;
3761 }
3762 err = esw_offloads_start(esw, extack);
3763 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
3764 err = esw_offloads_stop(esw, extack);
3765 mlx5_rescan_drivers(esw->dev);
3766 } else {
3767 err = -EINVAL;
3768 }
3769
3770 skip:
3771 down_write(&esw->mode_lock);
3772 esw->eswitch_operation_in_progress = false;
3773 unlock:
3774 mlx5_esw_unlock(esw);
3775 enable_lag:
3776 mlx5_lag_enable_change(esw->dev);
3777 return err;
3778 }
3779
mlx5_devlink_eswitch_mode_get(struct devlink * devlink,u16 * mode)3780 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3781 {
3782 struct mlx5_eswitch *esw;
3783
3784 esw = mlx5_devlink_eswitch_get(devlink);
3785 if (IS_ERR(esw))
3786 return PTR_ERR(esw);
3787
3788 return esw_mode_to_devlink(esw->mode, mode);
3789 }
3790
mlx5_esw_vports_inline_set(struct mlx5_eswitch * esw,u8 mlx5_mode,struct netlink_ext_ack * extack)3791 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
3792 struct netlink_ext_ack *extack)
3793 {
3794 struct mlx5_core_dev *dev = esw->dev;
3795 struct mlx5_vport *vport;
3796 u16 err_vport_num = 0;
3797 unsigned long i;
3798 int err = 0;
3799
3800 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3801 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3802 if (err) {
3803 err_vport_num = vport->vport;
3804 NL_SET_ERR_MSG_MOD(extack,
3805 "Failed to set min inline on vport");
3806 goto revert_inline_mode;
3807 }
3808 }
3809 if (mlx5_core_ec_sriov_enabled(esw->dev)) {
3810 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
3811 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3812 if (err) {
3813 err_vport_num = vport->vport;
3814 NL_SET_ERR_MSG_MOD(extack,
3815 "Failed to set min inline on vport");
3816 goto revert_ec_vf_inline_mode;
3817 }
3818 }
3819 }
3820 return 0;
3821
3822 revert_ec_vf_inline_mode:
3823 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
3824 if (vport->vport == err_vport_num)
3825 break;
3826 mlx5_modify_nic_vport_min_inline(dev,
3827 vport->vport,
3828 esw->offloads.inline_mode);
3829 }
3830 revert_inline_mode:
3831 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3832 if (vport->vport == err_vport_num)
3833 break;
3834 mlx5_modify_nic_vport_min_inline(dev,
3835 vport->vport,
3836 esw->offloads.inline_mode);
3837 }
3838 return err;
3839 }
3840
mlx5_devlink_eswitch_inline_mode_set(struct devlink * devlink,u8 mode,struct netlink_ext_ack * extack)3841 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
3842 struct netlink_ext_ack *extack)
3843 {
3844 struct mlx5_core_dev *dev = devlink_priv(devlink);
3845 struct mlx5_eswitch *esw;
3846 u8 mlx5_mode;
3847 int err;
3848
3849 esw = mlx5_devlink_eswitch_get(devlink);
3850 if (IS_ERR(esw))
3851 return PTR_ERR(esw);
3852
3853 down_write(&esw->mode_lock);
3854
3855 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
3856 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
3857 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
3858 err = 0;
3859 goto out;
3860 }
3861
3862 fallthrough;
3863 case MLX5_CAP_INLINE_MODE_L2:
3864 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
3865 err = -EOPNOTSUPP;
3866 goto out;
3867 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3868 break;
3869 }
3870
3871 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3872 NL_SET_ERR_MSG_MOD(extack,
3873 "Can't set inline mode when flows are configured");
3874 err = -EOPNOTSUPP;
3875 goto out;
3876 }
3877
3878 err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
3879 if (err)
3880 goto out;
3881
3882 esw->eswitch_operation_in_progress = true;
3883 up_write(&esw->mode_lock);
3884
3885 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
3886 if (!err)
3887 esw->offloads.inline_mode = mlx5_mode;
3888
3889 down_write(&esw->mode_lock);
3890 esw->eswitch_operation_in_progress = false;
3891 up_write(&esw->mode_lock);
3892 return 0;
3893
3894 out:
3895 up_write(&esw->mode_lock);
3896 return err;
3897 }
3898
mlx5_devlink_eswitch_inline_mode_get(struct devlink * devlink,u8 * mode)3899 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
3900 {
3901 struct mlx5_eswitch *esw;
3902
3903 esw = mlx5_devlink_eswitch_get(devlink);
3904 if (IS_ERR(esw))
3905 return PTR_ERR(esw);
3906
3907 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
3908 }
3909
mlx5_eswitch_block_encap(struct mlx5_core_dev * dev)3910 bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev)
3911 {
3912 struct mlx5_eswitch *esw = dev->priv.eswitch;
3913
3914 if (!mlx5_esw_allowed(esw))
3915 return true;
3916
3917 down_write(&esw->mode_lock);
3918 if (esw->mode != MLX5_ESWITCH_LEGACY &&
3919 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
3920 up_write(&esw->mode_lock);
3921 return false;
3922 }
3923
3924 esw->offloads.num_block_encap++;
3925 up_write(&esw->mode_lock);
3926 return true;
3927 }
3928
mlx5_eswitch_unblock_encap(struct mlx5_core_dev * dev)3929 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev)
3930 {
3931 struct mlx5_eswitch *esw = dev->priv.eswitch;
3932
3933 if (!mlx5_esw_allowed(esw))
3934 return;
3935
3936 down_write(&esw->mode_lock);
3937 esw->offloads.num_block_encap--;
3938 up_write(&esw->mode_lock);
3939 }
3940
mlx5_devlink_eswitch_encap_mode_set(struct devlink * devlink,enum devlink_eswitch_encap_mode encap,struct netlink_ext_ack * extack)3941 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
3942 enum devlink_eswitch_encap_mode encap,
3943 struct netlink_ext_ack *extack)
3944 {
3945 struct mlx5_core_dev *dev = devlink_priv(devlink);
3946 struct mlx5_eswitch *esw;
3947 int err = 0;
3948
3949 esw = mlx5_devlink_eswitch_get(devlink);
3950 if (IS_ERR(esw))
3951 return PTR_ERR(esw);
3952
3953 down_write(&esw->mode_lock);
3954
3955 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
3956 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
3957 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
3958 err = -EOPNOTSUPP;
3959 goto unlock;
3960 }
3961
3962 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
3963 err = -EOPNOTSUPP;
3964 goto unlock;
3965 }
3966
3967 if (esw->mode == MLX5_ESWITCH_LEGACY) {
3968 esw->offloads.encap = encap;
3969 goto unlock;
3970 }
3971
3972 if (esw->offloads.encap == encap)
3973 goto unlock;
3974
3975 if (atomic64_read(&esw->offloads.num_flows) > 0) {
3976 NL_SET_ERR_MSG_MOD(extack,
3977 "Can't set encapsulation when flows are configured");
3978 err = -EOPNOTSUPP;
3979 goto unlock;
3980 }
3981
3982 if (esw->offloads.num_block_encap) {
3983 NL_SET_ERR_MSG_MOD(extack,
3984 "Can't set encapsulation when IPsec SA and/or policies are configured");
3985 err = -EOPNOTSUPP;
3986 goto unlock;
3987 }
3988
3989 esw->eswitch_operation_in_progress = true;
3990 up_write(&esw->mode_lock);
3991
3992 esw_destroy_offloads_fdb_tables(esw);
3993
3994 esw->offloads.encap = encap;
3995
3996 err = esw_create_offloads_fdb_tables(esw);
3997
3998 if (err) {
3999 NL_SET_ERR_MSG_MOD(extack,
4000 "Failed re-creating fast FDB table");
4001 esw->offloads.encap = !encap;
4002 (void)esw_create_offloads_fdb_tables(esw);
4003 }
4004
4005 down_write(&esw->mode_lock);
4006 esw->eswitch_operation_in_progress = false;
4007
4008 unlock:
4009 up_write(&esw->mode_lock);
4010 return err;
4011 }
4012
mlx5_devlink_eswitch_encap_mode_get(struct devlink * devlink,enum devlink_eswitch_encap_mode * encap)4013 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
4014 enum devlink_eswitch_encap_mode *encap)
4015 {
4016 struct mlx5_eswitch *esw;
4017
4018 esw = mlx5_devlink_eswitch_get(devlink);
4019 if (IS_ERR(esw))
4020 return PTR_ERR(esw);
4021
4022 *encap = esw->offloads.encap;
4023 return 0;
4024 }
4025
4026 static bool
mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch * esw,u16 vport_num)4027 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
4028 {
4029 /* Currently, only ECPF based device has representor for host PF. */
4030 if (vport_num == MLX5_VPORT_PF &&
4031 !mlx5_core_is_ecpf_esw_manager(esw->dev))
4032 return false;
4033
4034 if (vport_num == MLX5_VPORT_ECPF &&
4035 !mlx5_ecpf_vport_exists(esw->dev))
4036 return false;
4037
4038 return true;
4039 }
4040
mlx5_eswitch_register_vport_reps(struct mlx5_eswitch * esw,const struct mlx5_eswitch_rep_ops * ops,u8 rep_type)4041 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
4042 const struct mlx5_eswitch_rep_ops *ops,
4043 u8 rep_type)
4044 {
4045 struct mlx5_eswitch_rep_data *rep_data;
4046 struct mlx5_eswitch_rep *rep;
4047 unsigned long i;
4048
4049 esw->offloads.rep_ops[rep_type] = ops;
4050 mlx5_esw_for_each_rep(esw, i, rep) {
4051 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
4052 rep->esw = esw;
4053 rep_data = &rep->rep_data[rep_type];
4054 atomic_set(&rep_data->state, REP_REGISTERED);
4055 }
4056 }
4057 }
4058 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
4059
mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch * esw,u8 rep_type)4060 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
4061 {
4062 struct mlx5_eswitch_rep *rep;
4063 unsigned long i;
4064
4065 if (esw->mode == MLX5_ESWITCH_OFFLOADS)
4066 __unload_reps_all_vport(esw, rep_type);
4067
4068 mlx5_esw_for_each_rep(esw, i, rep)
4069 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
4070 }
4071 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
4072
mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch * esw,u8 rep_type)4073 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
4074 {
4075 struct mlx5_eswitch_rep *rep;
4076
4077 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
4078 return rep->rep_data[rep_type].priv;
4079 }
4080
mlx5_eswitch_get_proto_dev(struct mlx5_eswitch * esw,u16 vport,u8 rep_type)4081 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
4082 u16 vport,
4083 u8 rep_type)
4084 {
4085 struct mlx5_eswitch_rep *rep;
4086
4087 rep = mlx5_eswitch_get_rep(esw, vport);
4088
4089 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
4090 esw->offloads.rep_ops[rep_type]->get_proto_dev)
4091 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
4092 return NULL;
4093 }
4094 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
4095
mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch * esw,u8 rep_type)4096 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
4097 {
4098 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
4099 }
4100 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
4101
mlx5_eswitch_vport_rep(struct mlx5_eswitch * esw,u16 vport)4102 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
4103 u16 vport)
4104 {
4105 return mlx5_eswitch_get_rep(esw, vport);
4106 }
4107 EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
4108
mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch * esw)4109 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
4110 {
4111 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
4112 }
4113 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
4114
mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch * esw)4115 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
4116 {
4117 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
4118 }
4119 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
4120
mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch * esw,u16 vport_num)4121 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
4122 u16 vport_num)
4123 {
4124 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4125
4126 if (WARN_ON_ONCE(IS_ERR(vport)))
4127 return 0;
4128
4129 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
4130 }
4131 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
4132
mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch * esw,u16 vport_num,u16 * vhca_id)4133 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
4134 {
4135 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4136 void *query_ctx;
4137 void *hca_caps;
4138 int err;
4139
4140 *vhca_id = 0;
4141
4142 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4143 if (!query_ctx)
4144 return -ENOMEM;
4145
4146 err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx);
4147 if (err)
4148 goto out_free;
4149
4150 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4151 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
4152
4153 out_free:
4154 kfree(query_ctx);
4155 return err;
4156 }
4157
mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch * esw,u16 vport_num)4158 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
4159 {
4160 u16 *old_entry, *vhca_map_entry, vhca_id;
4161 int err;
4162
4163 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
4164 if (err) {
4165 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
4166 vport_num, err);
4167 return err;
4168 }
4169
4170 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
4171 if (!vhca_map_entry)
4172 return -ENOMEM;
4173
4174 *vhca_map_entry = vport_num;
4175 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
4176 if (xa_is_err(old_entry)) {
4177 kfree(vhca_map_entry);
4178 return xa_err(old_entry);
4179 }
4180 kfree(old_entry);
4181 return 0;
4182 }
4183
mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch * esw,u16 vport_num)4184 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
4185 {
4186 u16 *vhca_map_entry, vhca_id;
4187 int err;
4188
4189 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
4190 if (err)
4191 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
4192 vport_num, err);
4193
4194 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id);
4195 kfree(vhca_map_entry);
4196 }
4197
mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch * esw,u16 vhca_id,u16 * vport_num)4198 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
4199 {
4200 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
4201
4202 if (!res)
4203 return -ENOENT;
4204
4205 *vport_num = *res;
4206 return 0;
4207 }
4208
mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch * esw,u16 vport_num)4209 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
4210 u16 vport_num)
4211 {
4212 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
4213
4214 if (WARN_ON_ONCE(IS_ERR(vport)))
4215 return 0;
4216
4217 return vport->metadata;
4218 }
4219 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
4220
mlx5_devlink_port_fn_hw_addr_get(struct devlink_port * port,u8 * hw_addr,int * hw_addr_len,struct netlink_ext_ack * extack)4221 int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port,
4222 u8 *hw_addr, int *hw_addr_len,
4223 struct netlink_ext_ack *extack)
4224 {
4225 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4226 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4227
4228 mutex_lock(&esw->state_lock);
4229 ether_addr_copy(hw_addr, vport->info.mac);
4230 *hw_addr_len = ETH_ALEN;
4231 mutex_unlock(&esw->state_lock);
4232 return 0;
4233 }
4234
mlx5_devlink_port_fn_hw_addr_set(struct devlink_port * port,const u8 * hw_addr,int hw_addr_len,struct netlink_ext_ack * extack)4235 int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port,
4236 const u8 *hw_addr, int hw_addr_len,
4237 struct netlink_ext_ack *extack)
4238 {
4239 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4240 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4241
4242 return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr);
4243 }
4244
mlx5_devlink_port_fn_migratable_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4245 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled,
4246 struct netlink_ext_ack *extack)
4247 {
4248 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4249 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4250
4251 if (!MLX5_CAP_GEN(esw->dev, migration)) {
4252 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4253 return -EOPNOTSUPP;
4254 }
4255
4256 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4257 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4258 return -EOPNOTSUPP;
4259 }
4260
4261 mutex_lock(&esw->state_lock);
4262 *is_enabled = vport->info.mig_enabled;
4263 mutex_unlock(&esw->state_lock);
4264 return 0;
4265 }
4266
mlx5_devlink_port_fn_migratable_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4267 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable,
4268 struct netlink_ext_ack *extack)
4269 {
4270 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4271 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4272 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4273 void *query_ctx;
4274 void *hca_caps;
4275 int err;
4276
4277 if (!MLX5_CAP_GEN(esw->dev, migration)) {
4278 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration");
4279 return -EOPNOTSUPP;
4280 }
4281
4282 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4283 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4284 return -EOPNOTSUPP;
4285 }
4286
4287 mutex_lock(&esw->state_lock);
4288
4289 if (vport->info.mig_enabled == enable) {
4290 err = 0;
4291 goto out;
4292 }
4293
4294 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4295 if (!query_ctx) {
4296 err = -ENOMEM;
4297 goto out;
4298 }
4299
4300 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx,
4301 MLX5_CAP_GENERAL_2);
4302 if (err) {
4303 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4304 goto out_free;
4305 }
4306
4307 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4308 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable);
4309
4310 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport,
4311 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2);
4312 if (err) {
4313 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap");
4314 goto out_free;
4315 }
4316
4317 vport->info.mig_enabled = enable;
4318
4319 out_free:
4320 kfree(query_ctx);
4321 out:
4322 mutex_unlock(&esw->state_lock);
4323 return err;
4324 }
4325
mlx5_devlink_port_fn_roce_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4326 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled,
4327 struct netlink_ext_ack *extack)
4328 {
4329 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4330 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4331
4332 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4333 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4334 return -EOPNOTSUPP;
4335 }
4336
4337 mutex_lock(&esw->state_lock);
4338 *is_enabled = vport->info.roce_enabled;
4339 mutex_unlock(&esw->state_lock);
4340 return 0;
4341 }
4342
mlx5_devlink_port_fn_roce_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4343 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable,
4344 struct netlink_ext_ack *extack)
4345 {
4346 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4347 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4348 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4349 u16 vport_num = vport->vport;
4350 void *query_ctx;
4351 void *hca_caps;
4352 int err;
4353
4354 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4355 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
4356 return -EOPNOTSUPP;
4357 }
4358
4359 mutex_lock(&esw->state_lock);
4360
4361 if (vport->info.roce_enabled == enable) {
4362 err = 0;
4363 goto out;
4364 }
4365
4366 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4367 if (!query_ctx) {
4368 err = -ENOMEM;
4369 goto out;
4370 }
4371
4372 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx,
4373 MLX5_CAP_GENERAL);
4374 if (err) {
4375 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4376 goto out_free;
4377 }
4378
4379 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4380 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable);
4381
4382 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num,
4383 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
4384 if (err) {
4385 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap");
4386 goto out_free;
4387 }
4388
4389 vport->info.roce_enabled = enable;
4390
4391 out_free:
4392 kfree(query_ctx);
4393 out:
4394 mutex_unlock(&esw->state_lock);
4395 return err;
4396 }
4397
4398 int
mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch * esw,struct mlx5_flow_handle * rule,struct mlx5_esw_flow_attr * esw_attr,int attr_idx)4399 mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule,
4400 struct mlx5_esw_flow_attr *esw_attr, int attr_idx)
4401 {
4402 struct mlx5_flow_destination new_dest = {};
4403 struct mlx5_flow_destination old_dest = {};
4404
4405 if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx))
4406 return 0;
4407
4408 esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false);
4409 esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false);
4410
4411 return mlx5_modify_rule_destination(rule, &new_dest, &old_dest);
4412 }
4413
4414 #ifdef CONFIG_XFRM_OFFLOAD
mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4415 int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled,
4416 struct netlink_ext_ack *extack)
4417 {
4418 struct mlx5_eswitch *esw;
4419 struct mlx5_vport *vport;
4420 int err = 0;
4421
4422 esw = mlx5_devlink_eswitch_get(port->devlink);
4423 if (IS_ERR(esw))
4424 return PTR_ERR(esw);
4425
4426 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) {
4427 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto");
4428 return -EOPNOTSUPP;
4429 }
4430
4431 vport = mlx5_devlink_port_vport_get(port);
4432
4433 mutex_lock(&esw->state_lock);
4434 if (!vport->enabled) {
4435 err = -EOPNOTSUPP;
4436 goto unlock;
4437 }
4438
4439 *is_enabled = vport->info.ipsec_crypto_enabled;
4440 unlock:
4441 mutex_unlock(&esw->state_lock);
4442 return err;
4443 }
4444
mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4445 int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable,
4446 struct netlink_ext_ack *extack)
4447 {
4448 struct mlx5_eswitch *esw;
4449 struct mlx5_vport *vport;
4450 u16 vport_num;
4451 int err;
4452
4453 esw = mlx5_devlink_eswitch_get(port->devlink);
4454 if (IS_ERR(esw))
4455 return PTR_ERR(esw);
4456
4457 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4458 err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num);
4459 if (err) {
4460 NL_SET_ERR_MSG_MOD(extack,
4461 "Device doesn't support IPsec crypto");
4462 return err;
4463 }
4464
4465 vport = mlx5_devlink_port_vport_get(port);
4466
4467 mutex_lock(&esw->state_lock);
4468 if (!vport->enabled) {
4469 err = -EOPNOTSUPP;
4470 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4471 goto unlock;
4472 }
4473
4474 if (vport->info.ipsec_crypto_enabled == enable)
4475 goto unlock;
4476
4477 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4478 err = -EBUSY;
4479 goto unlock;
4480 }
4481
4482 err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable);
4483 if (err) {
4484 NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto");
4485 goto unlock;
4486 }
4487
4488 vport->info.ipsec_crypto_enabled = enable;
4489 if (enable)
4490 esw->enabled_ipsec_vf_count++;
4491 else
4492 esw->enabled_ipsec_vf_count--;
4493 unlock:
4494 mutex_unlock(&esw->state_lock);
4495 return err;
4496 }
4497
mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port * port,bool * is_enabled,struct netlink_ext_ack * extack)4498 int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled,
4499 struct netlink_ext_ack *extack)
4500 {
4501 struct mlx5_eswitch *esw;
4502 struct mlx5_vport *vport;
4503 int err = 0;
4504
4505 esw = mlx5_devlink_eswitch_get(port->devlink);
4506 if (IS_ERR(esw))
4507 return PTR_ERR(esw);
4508
4509 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) {
4510 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet");
4511 return -EOPNOTSUPP;
4512 }
4513
4514 vport = mlx5_devlink_port_vport_get(port);
4515
4516 mutex_lock(&esw->state_lock);
4517 if (!vport->enabled) {
4518 err = -EOPNOTSUPP;
4519 goto unlock;
4520 }
4521
4522 *is_enabled = vport->info.ipsec_packet_enabled;
4523 unlock:
4524 mutex_unlock(&esw->state_lock);
4525 return err;
4526 }
4527
mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port * port,bool enable,struct netlink_ext_ack * extack)4528 int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port,
4529 bool enable,
4530 struct netlink_ext_ack *extack)
4531 {
4532 struct mlx5_eswitch *esw;
4533 struct mlx5_vport *vport;
4534 u16 vport_num;
4535 int err;
4536
4537 esw = mlx5_devlink_eswitch_get(port->devlink);
4538 if (IS_ERR(esw))
4539 return PTR_ERR(esw);
4540
4541 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index);
4542 err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num);
4543 if (err) {
4544 NL_SET_ERR_MSG_MOD(extack,
4545 "Device doesn't support IPsec packet mode");
4546 return err;
4547 }
4548
4549 vport = mlx5_devlink_port_vport_get(port);
4550 mutex_lock(&esw->state_lock);
4551 if (!vport->enabled) {
4552 err = -EOPNOTSUPP;
4553 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled");
4554 goto unlock;
4555 }
4556
4557 if (vport->info.ipsec_packet_enabled == enable)
4558 goto unlock;
4559
4560 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) {
4561 err = -EBUSY;
4562 goto unlock;
4563 }
4564
4565 err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable);
4566 if (err) {
4567 NL_SET_ERR_MSG_MOD(extack,
4568 "Failed to set IPsec packet mode");
4569 goto unlock;
4570 }
4571
4572 vport->info.ipsec_packet_enabled = enable;
4573 if (enable)
4574 esw->enabled_ipsec_vf_count++;
4575 else
4576 esw->enabled_ipsec_vf_count--;
4577 unlock:
4578 mutex_unlock(&esw->state_lock);
4579 return err;
4580 }
4581 #endif /* CONFIG_XFRM_OFFLOAD */
4582
4583 int
mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port * port,u32 * max_io_eqs,struct netlink_ext_ack * extack)4584 mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs,
4585 struct netlink_ext_ack *extack)
4586 {
4587 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4588 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4589 u16 vport_num = vport->vport;
4590 struct mlx5_eswitch *esw;
4591 void *query_ctx;
4592 void *hca_caps;
4593 u32 max_eqs;
4594 int err;
4595
4596 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4597 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4598 NL_SET_ERR_MSG_MOD(extack,
4599 "Device doesn't support VHCA management");
4600 return -EOPNOTSUPP;
4601 }
4602
4603 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4604 if (!query_ctx)
4605 return -ENOMEM;
4606
4607 mutex_lock(&esw->state_lock);
4608 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx,
4609 MLX5_CAP_GENERAL);
4610 if (err) {
4611 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4612 goto out;
4613 }
4614
4615 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4616 max_eqs = MLX5_GET(cmd_hca_cap, hca_caps, max_num_eqs);
4617 if (max_eqs < MLX5_ESW_MAX_CTRL_EQS)
4618 *max_io_eqs = 0;
4619 else
4620 *max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS;
4621 out:
4622 mutex_unlock(&esw->state_lock);
4623 kfree(query_ctx);
4624 return err;
4625 }
4626
4627 int
mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port * port,u32 max_io_eqs,struct netlink_ext_ack * extack)4628 mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, u32 max_io_eqs,
4629 struct netlink_ext_ack *extack)
4630 {
4631 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
4632 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
4633 u16 vport_num = vport->vport;
4634 struct mlx5_eswitch *esw;
4635 void *query_ctx;
4636 void *hca_caps;
4637 u16 max_eqs;
4638 int err;
4639
4640 esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
4641 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
4642 NL_SET_ERR_MSG_MOD(extack,
4643 "Device doesn't support VHCA management");
4644 return -EOPNOTSUPP;
4645 }
4646
4647 if (check_add_overflow(max_io_eqs, MLX5_ESW_MAX_CTRL_EQS, &max_eqs)) {
4648 NL_SET_ERR_MSG_MOD(extack, "Supplied value out of range");
4649 return -EINVAL;
4650 }
4651
4652 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
4653 if (!query_ctx)
4654 return -ENOMEM;
4655
4656 mutex_lock(&esw->state_lock);
4657 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx,
4658 MLX5_CAP_GENERAL);
4659 if (err) {
4660 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps");
4661 goto out;
4662 }
4663
4664 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
4665 MLX5_SET(cmd_hca_cap, hca_caps, max_num_eqs, max_eqs);
4666
4667 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num,
4668 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
4669 if (err)
4670 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA caps");
4671
4672 out:
4673 mutex_unlock(&esw->state_lock);
4674 kfree(query_ctx);
4675 return err;
4676 }
4677