1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2019 Mellanox Technologies.
3
4 #include <linux/mlx5/fs.h>
5 #include "eswitch.h"
6 #include "en_tc.h"
7 #include "fs_core.h"
8
9 struct mlx5_termtbl_handle {
10 struct hlist_node termtbl_hlist;
11
12 struct mlx5_flow_table *termtbl;
13 struct mlx5_flow_act flow_act;
14 struct mlx5_flow_destination dest;
15
16 struct mlx5_flow_handle *rule;
17 int ref_count;
18 };
19
20 static u32
mlx5_eswitch_termtbl_hash(struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest)21 mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act,
22 struct mlx5_flow_destination *dest)
23 {
24 u32 hash;
25
26 hash = jhash_1word(flow_act->action, 0);
27 hash = jhash((const void *)&flow_act->vlan,
28 sizeof(flow_act->vlan), hash);
29 hash = jhash((const void *)&dest->vport.num,
30 sizeof(dest->vport.num), hash);
31 hash = jhash((const void *)&dest->vport.vhca_id,
32 sizeof(dest->vport.num), hash);
33 if (dest->vport.pkt_reformat)
34 hash = jhash(dest->vport.pkt_reformat,
35 sizeof(*dest->vport.pkt_reformat),
36 hash);
37 return hash;
38 }
39
40 static int
mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act * flow_act1,struct mlx5_flow_destination * dest1,struct mlx5_flow_act * flow_act2,struct mlx5_flow_destination * dest2)41 mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1,
42 struct mlx5_flow_destination *dest1,
43 struct mlx5_flow_act *flow_act2,
44 struct mlx5_flow_destination *dest2)
45 {
46 int ret;
47
48 ret = flow_act1->action != flow_act2->action ||
49 dest1->vport.num != dest2->vport.num ||
50 dest1->vport.vhca_id != dest2->vport.vhca_id ||
51 memcmp(&flow_act1->vlan, &flow_act2->vlan,
52 sizeof(flow_act1->vlan));
53 if (ret)
54 return ret;
55
56 return dest1->vport.pkt_reformat && dest2->vport.pkt_reformat ?
57 memcmp(dest1->vport.pkt_reformat, dest2->vport.pkt_reformat,
58 sizeof(*dest1->vport.pkt_reformat)) : 0;
59 }
60
61 static int
mlx5_eswitch_termtbl_create(struct mlx5_core_dev * dev,struct mlx5_termtbl_handle * tt,struct mlx5_flow_act * flow_act)62 mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev,
63 struct mlx5_termtbl_handle *tt,
64 struct mlx5_flow_act *flow_act)
65 {
66 struct mlx5_flow_table_attr ft_attr = {};
67 struct mlx5_flow_namespace *root_ns;
68 int err;
69
70 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
71 if (!root_ns) {
72 esw_warn(dev, "Failed to get FDB flow namespace\n");
73 return -EOPNOTSUPP;
74 }
75
76 /* As this is the terminating action then the termination table is the
77 * same prio as the slow path
78 */
79 ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION |
80 MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
81 ft_attr.prio = FDB_SLOW_PATH;
82 ft_attr.max_fte = 1;
83 ft_attr.autogroup.max_num_groups = 1;
84 tt->termtbl = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr);
85 if (IS_ERR(tt->termtbl)) {
86 esw_warn(dev, "Failed to create termination table (error %d)\n",
87 IS_ERR(tt->termtbl));
88 return -EOPNOTSUPP;
89 }
90
91 tt->rule = mlx5_add_flow_rules(tt->termtbl, NULL, flow_act,
92 &tt->dest, 1);
93 if (IS_ERR(tt->rule)) {
94 esw_warn(dev, "Failed to create termination table rule (error %d)\n",
95 IS_ERR(tt->rule));
96 goto add_flow_err;
97 }
98 return 0;
99
100 add_flow_err:
101 err = mlx5_destroy_flow_table(tt->termtbl);
102 if (err)
103 esw_warn(dev, "Failed to destroy termination table\n");
104
105 return -EOPNOTSUPP;
106 }
107
108 static struct mlx5_termtbl_handle *
mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch * esw,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest,struct mlx5_esw_flow_attr * attr)109 mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch *esw,
110 struct mlx5_flow_act *flow_act,
111 struct mlx5_flow_destination *dest,
112 struct mlx5_esw_flow_attr *attr)
113 {
114 struct mlx5_termtbl_handle *tt;
115 bool found = false;
116 u32 hash_key;
117 int err;
118
119 mutex_lock(&esw->offloads.termtbl_mutex);
120 hash_key = mlx5_eswitch_termtbl_hash(flow_act, dest);
121 hash_for_each_possible(esw->offloads.termtbl_tbl, tt,
122 termtbl_hlist, hash_key) {
123 if (!mlx5_eswitch_termtbl_cmp(&tt->flow_act, &tt->dest,
124 flow_act, dest)) {
125 found = true;
126 break;
127 }
128 }
129 if (found)
130 goto tt_add_ref;
131
132 tt = kzalloc(sizeof(*tt), GFP_KERNEL);
133 if (!tt) {
134 err = -ENOMEM;
135 goto tt_create_err;
136 }
137
138 tt->dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
139 tt->dest.vport.num = dest->vport.num;
140 tt->dest.vport.vhca_id = dest->vport.vhca_id;
141 tt->dest.vport.flags = dest->vport.flags;
142 memcpy(&tt->flow_act, flow_act, sizeof(*flow_act));
143
144 err = mlx5_eswitch_termtbl_create(esw->dev, tt, flow_act);
145 if (err)
146 goto tt_create_err;
147
148 hash_add(esw->offloads.termtbl_tbl, &tt->termtbl_hlist, hash_key);
149 tt_add_ref:
150 tt->ref_count++;
151 mutex_unlock(&esw->offloads.termtbl_mutex);
152 return tt;
153 tt_create_err:
154 kfree(tt);
155 mutex_unlock(&esw->offloads.termtbl_mutex);
156 return ERR_PTR(err);
157 }
158
159 void
mlx5_eswitch_termtbl_put(struct mlx5_eswitch * esw,struct mlx5_termtbl_handle * tt)160 mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw,
161 struct mlx5_termtbl_handle *tt)
162 {
163 mutex_lock(&esw->offloads.termtbl_mutex);
164 if (--tt->ref_count == 0)
165 hash_del(&tt->termtbl_hlist);
166 mutex_unlock(&esw->offloads.termtbl_mutex);
167
168 if (!tt->ref_count) {
169 mlx5_del_flow_rules(tt->rule);
170 mlx5_destroy_flow_table(tt->termtbl);
171 kfree(tt);
172 }
173 }
174
mlx5_eswitch_termtbl_is_encap_reformat(struct mlx5_pkt_reformat * rt)175 static bool mlx5_eswitch_termtbl_is_encap_reformat(struct mlx5_pkt_reformat *rt)
176 {
177 switch (rt->reformat_type) {
178 case MLX5_REFORMAT_TYPE_L2_TO_VXLAN:
179 case MLX5_REFORMAT_TYPE_L2_TO_NVGRE:
180 case MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
181 case MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
182 return true;
183 default:
184 return false;
185 }
186 }
187
188 static void
mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act * src,struct mlx5_flow_act * dst)189 mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src,
190 struct mlx5_flow_act *dst)
191 {
192 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
193 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
194 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
195 memcpy(&dst->vlan[0], &src->vlan[0], sizeof(src->vlan[0]));
196 memset(&src->vlan[0], 0, sizeof(src->vlan[0]));
197
198 if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
199 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
200 dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2;
201 memcpy(&dst->vlan[1], &src->vlan[1], sizeof(src->vlan[1]));
202 memset(&src->vlan[1], 0, sizeof(src->vlan[1]));
203 }
204 }
205
206 if (src->action & MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT &&
207 mlx5_eswitch_termtbl_is_encap_reformat(src->pkt_reformat)) {
208 src->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
209 dst->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
210 dst->pkt_reformat = src->pkt_reformat;
211 src->pkt_reformat = NULL;
212 }
213 }
214
mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch * esw,const struct mlx5_flow_spec * spec)215 static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw,
216 const struct mlx5_flow_spec *spec)
217 {
218 u16 port_mask, port_value;
219
220 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
221 return spec->flow_context.flow_source ==
222 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
223
224 port_mask = MLX5_GET(fte_match_param, spec->match_criteria,
225 misc_parameters.source_port);
226 port_value = MLX5_GET(fte_match_param, spec->match_value,
227 misc_parameters.source_port);
228 return (port_mask & port_value) == MLX5_VPORT_UPLINK;
229 }
230
231 bool
mlx5_eswitch_termtbl_required(struct mlx5_eswitch * esw,struct mlx5_flow_attr * attr,struct mlx5_flow_act * flow_act,struct mlx5_flow_spec * spec)232 mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw,
233 struct mlx5_flow_attr *attr,
234 struct mlx5_flow_act *flow_act,
235 struct mlx5_flow_spec *spec)
236 {
237 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
238 int i;
239
240 if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) ||
241 attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH ||
242 !mlx5_eswitch_offload_is_uplink_port(esw, spec))
243 return false;
244
245 /* push vlan on RX */
246 if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH)
247 return true;
248
249 /* hairpin */
250 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
251 if (esw_attr->dests[i].rep->vport == MLX5_VPORT_UPLINK)
252 return true;
253
254 return false;
255 }
256
257 struct mlx5_flow_handle *
mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch * esw,struct mlx5_flow_table * fdb,struct mlx5_flow_spec * spec,struct mlx5_esw_flow_attr * attr,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest,int num_dest)258 mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw,
259 struct mlx5_flow_table *fdb,
260 struct mlx5_flow_spec *spec,
261 struct mlx5_esw_flow_attr *attr,
262 struct mlx5_flow_act *flow_act,
263 struct mlx5_flow_destination *dest,
264 int num_dest)
265 {
266 struct mlx5_flow_act term_tbl_act = {};
267 struct mlx5_flow_handle *rule = NULL;
268 bool term_table_created = false;
269 int num_vport_dests = 0;
270 int i, curr_dest;
271
272 mlx5_eswitch_termtbl_actions_move(flow_act, &term_tbl_act);
273 term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
274
275 for (i = 0; i < num_dest; i++) {
276 struct mlx5_termtbl_handle *tt;
277
278 /* only vport destinations can be terminated */
279 if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT)
280 continue;
281
282 /* get the terminating table for the action list */
283 tt = mlx5_eswitch_termtbl_get_create(esw, &term_tbl_act,
284 &dest[i], attr);
285 if (IS_ERR(tt)) {
286 esw_warn(esw->dev, "Failed to get termination table (error %d)\n",
287 IS_ERR(tt));
288 goto revert_changes;
289 }
290 attr->dests[num_vport_dests].termtbl = tt;
291 num_vport_dests++;
292
293 /* link the destination with the termination table */
294 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
295 dest[i].ft = tt->termtbl;
296 term_table_created = true;
297 }
298
299 /* at least one destination should reference a termination table */
300 if (!term_table_created)
301 goto revert_changes;
302
303 /* create the FTE */
304 rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
305 if (IS_ERR(rule))
306 goto revert_changes;
307
308 goto out;
309
310 revert_changes:
311 /* revert the changes that were made to the original flow_act
312 * and fall-back to the original rule actions
313 */
314 mlx5_eswitch_termtbl_actions_move(&term_tbl_act, flow_act);
315
316 for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) {
317 struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl;
318
319 /* search for the destination associated with the
320 * current term table
321 */
322 for (i = 0; i < num_dest; i++) {
323 if (dest[i].ft != tt->termtbl)
324 continue;
325
326 memset(&dest[i], 0, sizeof(dest[i]));
327 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
328 dest[i].vport.num = tt->dest.vport.num;
329 dest[i].vport.vhca_id = tt->dest.vport.vhca_id;
330 mlx5_eswitch_termtbl_put(esw, tt);
331 break;
332 }
333 }
334 rule = mlx5_add_flow_rules(fdb, spec, flow_act, dest, num_dest);
335 out:
336 return rule;
337 }
338