1 /*-
2 * Copyright (c) 2013-2021, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_rss.h"
27 #include "opt_ratelimit.h"
28
29 #include <linux/module.h>
30 #include <dev/mlx5/driver.h>
31 #include <dev/mlx5/mlx5_core/mlx5_core.h>
32 #include <dev/mlx5/mlx5_core/fs_core.h>
33 #include <linux/string.h>
34 #include <linux/compiler.h>
35
36 #define INIT_TREE_NODE_ARRAY_SIZE(...) (sizeof((struct init_tree_node[]){__VA_ARGS__}) /\
37 sizeof(struct init_tree_node))
38
39 #define ADD_PRIO(name_val, flags_val, min_level_val, max_ft_val, caps_val, \
40 ...) {.type = FS_TYPE_PRIO,\
41 .name = name_val,\
42 .min_ft_level = min_level_val,\
43 .flags = flags_val,\
44 .max_ft = max_ft_val,\
45 .caps = caps_val,\
46 .children = (struct init_tree_node[]) {__VA_ARGS__},\
47 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
48 }
49
50 #define ADD_FT_PRIO(name_val, flags_val, max_ft_val, ...)\
51 ADD_PRIO(name_val, flags_val, 0, max_ft_val, {},\
52 __VA_ARGS__)\
53
54 #define ADD_NS(name_val, ...) {.type = FS_TYPE_NAMESPACE,\
55 .name = name_val,\
56 .children = (struct init_tree_node[]) {__VA_ARGS__},\
57 .ar_size = INIT_TREE_NODE_ARRAY_SIZE(__VA_ARGS__) \
58 }
59
60 #define INIT_CAPS_ARRAY_SIZE(...) (sizeof((long[]){__VA_ARGS__}) /\
61 sizeof(long))
62
63 #define FS_CAP(cap) (__mlx5_bit_off(flow_table_nic_cap, cap))
64
65 #define FS_REQUIRED_CAPS(...) {.arr_sz = INIT_CAPS_ARRAY_SIZE(__VA_ARGS__), \
66 .caps = (long[]) {__VA_ARGS__}}
67
68 /* Flowtable sizes: */
69 #define BYPASS_MAX_FT 5
70 #define BYPASS_PRIO_MAX_FT 1
71 #define OFFLOADS_MAX_FT 2
72 #define KERNEL_MAX_FT 5
73 #define LEFTOVER_MAX_FT 1
74
75 /* Flowtable levels: */
76 #define OFFLOADS_MIN_LEVEL 3
77 #define KERNEL_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)
78 #define LEFTOVER_MIN_LEVEL (KERNEL_MIN_LEVEL + 1)
79 #define BYPASS_MIN_LEVEL (MLX5_NUM_BYPASS_FTS + LEFTOVER_MIN_LEVEL)
80
81 struct node_caps {
82 size_t arr_sz;
83 long *caps;
84 };
85
86 struct init_tree_node {
87 enum fs_type type;
88 const char *name;
89 struct init_tree_node *children;
90 int ar_size;
91 struct node_caps caps;
92 u8 flags;
93 int min_ft_level;
94 int prio;
95 int max_ft;
96 } root_fs = {
97 .type = FS_TYPE_NAMESPACE,
98 .name = "root",
99 .ar_size = 4,
100 .children = (struct init_tree_node[]) {
101 ADD_PRIO("by_pass_prio", 0, BYPASS_MIN_LEVEL, 0,
102 FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en),
103 FS_CAP(flow_table_properties_nic_receive.modify_root)),
104 ADD_NS("by_pass_ns",
105 ADD_FT_PRIO("prio0", 0,
106 BYPASS_PRIO_MAX_FT),
107 ADD_FT_PRIO("prio1", 0,
108 BYPASS_PRIO_MAX_FT),
109 ADD_FT_PRIO("prio2", 0,
110 BYPASS_PRIO_MAX_FT),
111 ADD_FT_PRIO("prio3", 0,
112 BYPASS_PRIO_MAX_FT),
113 ADD_FT_PRIO("prio4", 0,
114 BYPASS_PRIO_MAX_FT),
115 ADD_FT_PRIO("prio5", 0,
116 BYPASS_PRIO_MAX_FT),
117 ADD_FT_PRIO("prio6", 0,
118 BYPASS_PRIO_MAX_FT),
119 ADD_FT_PRIO("prio7", 0,
120 BYPASS_PRIO_MAX_FT),
121 ADD_FT_PRIO("prio-mcast", 0,
122 BYPASS_PRIO_MAX_FT))),
123 ADD_PRIO("offloads_prio", 0, OFFLOADS_MIN_LEVEL, 0, {},
124 ADD_NS("offloads_ns",
125 ADD_FT_PRIO("prio_offloads-0", 0,
126 OFFLOADS_MAX_FT))),
127 ADD_PRIO("kernel_prio", 0, KERNEL_MIN_LEVEL, 0, {},
128 ADD_NS("kernel_ns",
129 ADD_FT_PRIO("prio_kernel-0", 0,
130 KERNEL_MAX_FT))),
131 ADD_PRIO("leftovers_prio", MLX5_CORE_FS_PRIO_SHARED,
132 LEFTOVER_MIN_LEVEL, 0,
133 FS_REQUIRED_CAPS(FS_CAP(flow_table_properties_nic_receive.flow_modify_en),
134 FS_CAP(flow_table_properties_nic_receive.modify_root)),
135 ADD_NS("leftover_ns",
136 ADD_FT_PRIO("leftovers_prio-0",
137 MLX5_CORE_FS_PRIO_SHARED,
138 LEFTOVER_MAX_FT)))
139 }
140 };
141
142 /* Tree creation functions */
143
find_root(struct fs_base * node)144 static struct mlx5_flow_root_namespace *find_root(struct fs_base *node)
145 {
146 struct fs_base *parent;
147
148 /* Make sure we only read it once while we go up the tree */
149 while ((parent = node->parent))
150 node = parent;
151
152 if (node->type != FS_TYPE_NAMESPACE) {
153 return NULL;
154 }
155
156 return container_of(container_of(node,
157 struct mlx5_flow_namespace,
158 base),
159 struct mlx5_flow_root_namespace,
160 ns);
161 }
162
fs_get_dev(struct fs_base * node)163 static inline struct mlx5_core_dev *fs_get_dev(struct fs_base *node)
164 {
165 struct mlx5_flow_root_namespace *root = find_root(node);
166
167 if (root)
168 return root->dev;
169 return NULL;
170 }
171
fs_init_node(struct fs_base * node,unsigned int refcount)172 static void fs_init_node(struct fs_base *node,
173 unsigned int refcount)
174 {
175 kref_init(&node->refcount);
176 atomic_set(&node->users_refcount, refcount);
177 init_completion(&node->complete);
178 INIT_LIST_HEAD(&node->list);
179 mutex_init(&node->lock);
180 }
181
_fs_add_node(struct fs_base * node,const char * name,struct fs_base * parent)182 static void _fs_add_node(struct fs_base *node,
183 const char *name,
184 struct fs_base *parent)
185 {
186 if (parent)
187 atomic_inc(&parent->users_refcount);
188 node->name = kstrdup_const(name, GFP_KERNEL);
189 node->parent = parent;
190 }
191
fs_add_node(struct fs_base * node,struct fs_base * parent,const char * name,unsigned int refcount)192 static void fs_add_node(struct fs_base *node,
193 struct fs_base *parent, const char *name,
194 unsigned int refcount)
195 {
196 fs_init_node(node, refcount);
197 _fs_add_node(node, name, parent);
198 }
199
200 static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref),
201 bool parent_locked);
202
203 static void fs_del_dst(struct mlx5_flow_rule *dst);
204 static void _fs_del_ft(struct mlx5_flow_table *ft);
205 static void fs_del_fg(struct mlx5_flow_group *fg);
206 static void fs_del_fte(struct fs_fte *fte);
207
cmd_remove_node(struct fs_base * base)208 static void cmd_remove_node(struct fs_base *base)
209 {
210 switch (base->type) {
211 case FS_TYPE_FLOW_DEST:
212 fs_del_dst(container_of(base, struct mlx5_flow_rule, base));
213 break;
214 case FS_TYPE_FLOW_TABLE:
215 _fs_del_ft(container_of(base, struct mlx5_flow_table, base));
216 break;
217 case FS_TYPE_FLOW_GROUP:
218 fs_del_fg(container_of(base, struct mlx5_flow_group, base));
219 break;
220 case FS_TYPE_FLOW_ENTRY:
221 fs_del_fte(container_of(base, struct fs_fte, base));
222 break;
223 default:
224 break;
225 }
226 }
227
__fs_remove_node(struct kref * kref)228 static void __fs_remove_node(struct kref *kref)
229 {
230 struct fs_base *node = container_of(kref, struct fs_base, refcount);
231
232 if (node->parent) {
233 if (node->type == FS_TYPE_FLOW_DEST)
234 mutex_lock(&node->parent->parent->lock);
235 mutex_lock(&node->parent->lock);
236 }
237 mutex_lock(&node->lock);
238 cmd_remove_node(node);
239 mutex_unlock(&node->lock);
240 complete(&node->complete);
241 if (node->parent) {
242 mutex_unlock(&node->parent->lock);
243 if (node->type == FS_TYPE_FLOW_DEST)
244 mutex_unlock(&node->parent->parent->lock);
245 _fs_put(node->parent, _fs_remove_node, false);
246 }
247 }
248
_fs_remove_node(struct kref * kref)249 void _fs_remove_node(struct kref *kref)
250 {
251 struct fs_base *node = container_of(kref, struct fs_base, refcount);
252
253 __fs_remove_node(kref);
254 kfree_const(node->name);
255 kfree(node);
256 }
257
fs_get(struct fs_base * node)258 static void fs_get(struct fs_base *node)
259 {
260 atomic_inc(&node->users_refcount);
261 }
262
_fs_put(struct fs_base * node,void (* kref_cb)(struct kref * kref),bool parent_locked)263 static void _fs_put(struct fs_base *node, void (*kref_cb)(struct kref *kref),
264 bool parent_locked)
265 {
266 struct fs_base *parent_node = node->parent;
267
268 if (parent_node && !parent_locked)
269 mutex_lock(&parent_node->lock);
270 if (atomic_dec_and_test(&node->users_refcount)) {
271 if (parent_node) {
272 /*remove from parent's list*/
273 list_del_init(&node->list);
274 mutex_unlock(&parent_node->lock);
275 }
276 kref_put(&node->refcount, kref_cb);
277 if (parent_node && parent_locked)
278 mutex_lock(&parent_node->lock);
279 } else if (parent_node && !parent_locked) {
280 mutex_unlock(&parent_node->lock);
281 }
282 }
283
fs_put(struct fs_base * node)284 static void fs_put(struct fs_base *node)
285 {
286 _fs_put(node, __fs_remove_node, false);
287 }
288
fs_put_parent_locked(struct fs_base * node)289 static void fs_put_parent_locked(struct fs_base *node)
290 {
291 _fs_put(node, __fs_remove_node, true);
292 }
293
fs_remove_node(struct fs_base * node)294 static void fs_remove_node(struct fs_base *node)
295 {
296 fs_put(node);
297 wait_for_completion(&node->complete);
298 kfree_const(node->name);
299 kfree(node);
300 }
301
fs_remove_node_parent_locked(struct fs_base * node)302 static void fs_remove_node_parent_locked(struct fs_base *node)
303 {
304 fs_put_parent_locked(node);
305 wait_for_completion(&node->complete);
306 kfree_const(node->name);
307 kfree(node);
308 }
309
fs_alloc_fte(u32 sw_action,struct mlx5_flow_act * flow_act,u32 * match_value,unsigned int index)310 static struct fs_fte *fs_alloc_fte(u32 sw_action,
311 struct mlx5_flow_act *flow_act,
312 u32 *match_value,
313 unsigned int index)
314 {
315 struct fs_fte *fte;
316
317
318 fte = kzalloc(sizeof(*fte), GFP_KERNEL);
319 if (!fte)
320 return ERR_PTR(-ENOMEM);
321
322 memcpy(fte->val, match_value, sizeof(fte->val));
323 fte->base.type = FS_TYPE_FLOW_ENTRY;
324 fte->dests_size = 0;
325 fte->index = index;
326 INIT_LIST_HEAD(&fte->dests);
327 fte->flow_act = *flow_act;
328 fte->sw_action = sw_action;
329
330 return fte;
331 }
332
alloc_star_ft_entry(struct mlx5_flow_table * ft,struct mlx5_flow_group * fg,u32 * match_value,unsigned int index)333 static struct fs_fte *alloc_star_ft_entry(struct mlx5_flow_table *ft,
334 struct mlx5_flow_group *fg,
335 u32 *match_value,
336 unsigned int index)
337 {
338 int err;
339 struct fs_fte *fte;
340 struct mlx5_flow_rule *dst;
341 struct mlx5_flow_act flow_act = {
342 .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
343 .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
344 };
345
346 if (fg->num_ftes == fg->max_ftes)
347 return ERR_PTR(-ENOSPC);
348
349 fte = fs_alloc_fte(MLX5_FLOW_RULE_FWD_ACTION_DEST,
350 &flow_act, match_value, index);
351 if (IS_ERR(fte))
352 return fte;
353
354 /*create dst*/
355 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
356 if (!dst) {
357 err = -ENOMEM;
358 goto free_fte;
359 }
360
361 fte->base.parent = &fg->base;
362 fte->dests_size = 1;
363 dst->dest_attr.type = MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE;
364 dst->base.parent = &fte->base;
365 list_add(&dst->base.list, &fte->dests);
366 /* assumed that the callee creates the star rules sorted by index */
367 list_add_tail(&fte->base.list, &fg->ftes);
368 fg->num_ftes++;
369
370 return fte;
371
372 free_fte:
373 kfree(fte);
374 return ERR_PTR(err);
375 }
376
377 /* assume that fte can't be changed */
free_star_fte_entry(struct fs_fte * fte)378 static void free_star_fte_entry(struct fs_fte *fte)
379 {
380 struct mlx5_flow_group *fg;
381 struct mlx5_flow_rule *dst, *temp;
382
383 fs_get_parent(fg, fte);
384
385 list_for_each_entry_safe(dst, temp, &fte->dests, base.list) {
386 fte->dests_size--;
387 list_del(&dst->base.list);
388 kfree(dst);
389 }
390
391 list_del(&fte->base.list);
392 fg->num_ftes--;
393 kfree(fte);
394 }
395
fs_alloc_fg(u32 * create_fg_in)396 static struct mlx5_flow_group *fs_alloc_fg(u32 *create_fg_in)
397 {
398 struct mlx5_flow_group *fg;
399 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
400 create_fg_in, match_criteria);
401 u8 match_criteria_enable = MLX5_GET(create_flow_group_in,
402 create_fg_in,
403 match_criteria_enable);
404 fg = kzalloc(sizeof(*fg), GFP_KERNEL);
405 if (!fg)
406 return ERR_PTR(-ENOMEM);
407
408 INIT_LIST_HEAD(&fg->ftes);
409 fg->mask.match_criteria_enable = match_criteria_enable;
410 memcpy(&fg->mask.match_criteria, match_criteria,
411 sizeof(fg->mask.match_criteria));
412 fg->base.type = FS_TYPE_FLOW_GROUP;
413 fg->start_index = MLX5_GET(create_flow_group_in, create_fg_in,
414 start_flow_index);
415 fg->max_ftes = MLX5_GET(create_flow_group_in, create_fg_in,
416 end_flow_index) - fg->start_index + 1;
417 return fg;
418 }
419
420 static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio);
421 static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr,
422 struct fs_prio *prio);
423
424 /* assumed src_ft and dst_ft can't be freed */
fs_set_star_rule(struct mlx5_core_dev * dev,struct mlx5_flow_table * src_ft,struct mlx5_flow_table * dst_ft)425 static int fs_set_star_rule(struct mlx5_core_dev *dev,
426 struct mlx5_flow_table *src_ft,
427 struct mlx5_flow_table *dst_ft)
428 {
429 struct mlx5_flow_rule *src_dst;
430 struct fs_fte *src_fte;
431 int err = 0;
432 u32 *match_value;
433 int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
434
435 src_dst = list_first_entry(&src_ft->star_rule.fte->dests,
436 struct mlx5_flow_rule, base.list);
437 match_value = mlx5_vzalloc(match_len);
438 if (!match_value) {
439 mlx5_core_warn(dev, "failed to allocate inbox\n");
440 return -ENOMEM;
441 }
442 /*Create match context*/
443
444 fs_get_parent(src_fte, src_dst);
445
446 src_dst->dest_attr.ft = dst_ft;
447 if (dst_ft) {
448 err = mlx5_cmd_fs_set_fte(dev,
449 src_ft->vport,
450 &src_fte->status,
451 match_value, src_ft->type,
452 src_ft->id, src_fte->index,
453 src_ft->star_rule.fg->id,
454 &src_fte->flow_act,
455 src_fte->sw_action,
456 src_fte->dests_size,
457 &src_fte->dests);
458 if (err)
459 goto free;
460
461 fs_get(&dst_ft->base);
462 } else {
463 mlx5_cmd_fs_delete_fte(dev,
464 src_ft->vport,
465 &src_fte->status,
466 src_ft->type, src_ft->id,
467 src_fte->index);
468 }
469
470 free:
471 kvfree(match_value);
472 return err;
473 }
474
connect_prev_fts(struct fs_prio * locked_prio,struct fs_prio * prev_prio,struct mlx5_flow_table * next_ft)475 static int connect_prev_fts(struct fs_prio *locked_prio,
476 struct fs_prio *prev_prio,
477 struct mlx5_flow_table *next_ft)
478 {
479 struct mlx5_flow_table *iter;
480 int err = 0;
481 struct mlx5_core_dev *dev = fs_get_dev(&prev_prio->base);
482
483 if (!dev)
484 return -ENODEV;
485
486 mutex_lock(&prev_prio->base.lock);
487 fs_for_each_ft(iter, prev_prio) {
488 struct mlx5_flow_rule *src_dst =
489 list_first_entry(&iter->star_rule.fte->dests,
490 struct mlx5_flow_rule, base.list);
491 struct mlx5_flow_table *prev_ft = src_dst->dest_attr.ft;
492
493 if (prev_ft == next_ft)
494 continue;
495
496 err = fs_set_star_rule(dev, iter, next_ft);
497 if (err) {
498 mlx5_core_warn(dev,
499 "mlx5: flow steering can't connect prev and next\n");
500 goto unlock;
501 } else {
502 /* Assume ft's prio is locked */
503 if (prev_ft) {
504 struct fs_prio *prio;
505
506 fs_get_parent(prio, prev_ft);
507 if (prio == locked_prio)
508 fs_put_parent_locked(&prev_ft->base);
509 else
510 fs_put(&prev_ft->base);
511 }
512 }
513 }
514
515 unlock:
516 mutex_unlock(&prev_prio->base.lock);
517 return 0;
518 }
519
create_star_rule(struct mlx5_flow_table * ft,struct fs_prio * prio)520 static int create_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio)
521 {
522 struct mlx5_flow_group *fg;
523 int err;
524 u32 *fg_in;
525 u32 *match_value;
526 struct mlx5_flow_table *next_ft;
527 struct mlx5_flow_table *prev_ft;
528 struct mlx5_flow_root_namespace *root = find_root(&prio->base);
529 int fg_inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
530 int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
531
532 fg_in = mlx5_vzalloc(fg_inlen);
533 if (!fg_in) {
534 mlx5_core_warn(root->dev, "failed to allocate inbox\n");
535 return -ENOMEM;
536 }
537
538 match_value = mlx5_vzalloc(match_len);
539 if (!match_value) {
540 mlx5_core_warn(root->dev, "failed to allocate inbox\n");
541 kvfree(fg_in);
542 return -ENOMEM;
543 }
544
545 MLX5_SET(create_flow_group_in, fg_in, start_flow_index, ft->max_fte);
546 MLX5_SET(create_flow_group_in, fg_in, end_flow_index, ft->max_fte);
547 fg = fs_alloc_fg(fg_in);
548 if (IS_ERR(fg)) {
549 err = PTR_ERR(fg);
550 goto out;
551 }
552 ft->star_rule.fg = fg;
553 err = mlx5_cmd_fs_create_fg(fs_get_dev(&prio->base),
554 fg_in, ft->vport, ft->type,
555 ft->id,
556 &fg->id);
557 if (err)
558 goto free_fg;
559
560 ft->star_rule.fte = alloc_star_ft_entry(ft, fg,
561 match_value,
562 ft->max_fte);
563 if (IS_ERR(ft->star_rule.fte))
564 goto free_star_rule;
565
566 mutex_lock(&root->fs_chain_lock);
567 next_ft = find_next_ft(prio);
568 err = fs_set_star_rule(root->dev, ft, next_ft);
569 if (err) {
570 mutex_unlock(&root->fs_chain_lock);
571 goto free_star_rule;
572 }
573 if (next_ft) {
574 struct fs_prio *parent;
575
576 fs_get_parent(parent, next_ft);
577 fs_put(&next_ft->base);
578 }
579 prev_ft = find_prev_ft(ft, prio);
580 if (prev_ft) {
581 struct fs_prio *prev_parent;
582
583 fs_get_parent(prev_parent, prev_ft);
584
585 err = connect_prev_fts(NULL, prev_parent, ft);
586 if (err) {
587 mutex_unlock(&root->fs_chain_lock);
588 goto destroy_chained_star_rule;
589 }
590 fs_put(&prev_ft->base);
591 }
592 mutex_unlock(&root->fs_chain_lock);
593 kvfree(fg_in);
594 kvfree(match_value);
595
596 return 0;
597
598 destroy_chained_star_rule:
599 fs_set_star_rule(fs_get_dev(&prio->base), ft, NULL);
600 if (next_ft)
601 fs_put(&next_ft->base);
602 free_star_rule:
603 free_star_fte_entry(ft->star_rule.fte);
604 mlx5_cmd_fs_destroy_fg(fs_get_dev(&ft->base), ft->vport,
605 ft->type, ft->id,
606 fg->id);
607 free_fg:
608 kfree(fg);
609 out:
610 kvfree(fg_in);
611 kvfree(match_value);
612 return err;
613 }
614
destroy_star_rule(struct mlx5_flow_table * ft,struct fs_prio * prio)615 static void destroy_star_rule(struct mlx5_flow_table *ft, struct fs_prio *prio)
616 {
617 int err;
618 struct mlx5_flow_root_namespace *root;
619 struct mlx5_core_dev *dev = fs_get_dev(&prio->base);
620 struct mlx5_flow_table *prev_ft, *next_ft;
621 struct fs_prio *prev_prio;
622
623 WARN_ON(!dev);
624
625 root = find_root(&prio->base);
626 if (!root)
627 mlx5_core_err(dev,
628 "flow steering failed to find root of priority %s",
629 prio->base.name);
630
631 /* In order to ensure atomic deletion, first update
632 * prev ft to point on the next ft.
633 */
634 mutex_lock(&root->fs_chain_lock);
635 prev_ft = find_prev_ft(ft, prio);
636 next_ft = find_next_ft(prio);
637 if (prev_ft) {
638 fs_get_parent(prev_prio, prev_ft);
639 /*Prev is connected to ft, only if ft is the first(last) in the prio*/
640 err = connect_prev_fts(prio, prev_prio, next_ft);
641 if (err)
642 mlx5_core_warn(root->dev,
643 "flow steering can't connect prev and next of flow table\n");
644 fs_put(&prev_ft->base);
645 }
646
647 err = fs_set_star_rule(root->dev, ft, NULL);
648 /*One put is for fs_get in find next ft*/
649 if (next_ft) {
650 fs_put(&next_ft->base);
651 if (!err)
652 fs_put(&next_ft->base);
653 }
654
655 mutex_unlock(&root->fs_chain_lock);
656 err = mlx5_cmd_fs_destroy_fg(dev, ft->vport, ft->type, ft->id,
657 ft->star_rule.fg->id);
658 if (err)
659 mlx5_core_warn(dev,
660 "flow steering can't destroy star entry group(index:%d) of ft:%s\n", ft->star_rule.fg->start_index,
661 ft->base.name);
662 free_star_fte_entry(ft->star_rule.fte);
663
664 kfree(ft->star_rule.fg);
665 ft->star_rule.fg = NULL;
666 }
667
find_prio(struct mlx5_flow_namespace * ns,unsigned int prio)668 static struct fs_prio *find_prio(struct mlx5_flow_namespace *ns,
669 unsigned int prio)
670 {
671 struct fs_prio *iter_prio;
672
673 fs_for_each_prio(iter_prio, ns) {
674 if (iter_prio->prio == prio)
675 return iter_prio;
676 }
677
678 return NULL;
679 }
680
681 static unsigned int _alloc_new_level(struct fs_prio *prio,
682 struct mlx5_flow_namespace *match);
683
__alloc_new_level(struct mlx5_flow_namespace * ns,struct fs_prio * prio)684 static unsigned int __alloc_new_level(struct mlx5_flow_namespace *ns,
685 struct fs_prio *prio)
686 {
687 unsigned int level = 0;
688 struct fs_prio *p;
689
690 if (!ns)
691 return 0;
692
693 mutex_lock(&ns->base.lock);
694 fs_for_each_prio(p, ns) {
695 if (p != prio)
696 level += p->max_ft;
697 else
698 break;
699 }
700 mutex_unlock(&ns->base.lock);
701
702 fs_get_parent(prio, ns);
703 if (prio)
704 WARN_ON(prio->base.type != FS_TYPE_PRIO);
705
706 return level + _alloc_new_level(prio, ns);
707 }
708
709 /* Called under lock of priority, hence locking all upper objects */
_alloc_new_level(struct fs_prio * prio,struct mlx5_flow_namespace * match)710 static unsigned int _alloc_new_level(struct fs_prio *prio,
711 struct mlx5_flow_namespace *match)
712 {
713 struct mlx5_flow_namespace *ns;
714 struct fs_base *it;
715 unsigned int level = 0;
716
717 if (!prio)
718 return 0;
719
720 mutex_lock(&prio->base.lock);
721 fs_for_each_ns_or_ft_reverse(it, prio) {
722 if (it->type == FS_TYPE_NAMESPACE) {
723 struct fs_prio *p;
724
725 fs_get_obj(ns, it);
726
727 if (match != ns) {
728 mutex_lock(&ns->base.lock);
729 fs_for_each_prio(p, ns)
730 level += p->max_ft;
731 mutex_unlock(&ns->base.lock);
732 } else {
733 break;
734 }
735 } else {
736 struct mlx5_flow_table *ft;
737
738 fs_get_obj(ft, it);
739 mutex_unlock(&prio->base.lock);
740 return level + ft->level + 1;
741 }
742 }
743
744 fs_get_parent(ns, prio);
745 mutex_unlock(&prio->base.lock);
746 return __alloc_new_level(ns, prio) + level;
747 }
748
alloc_new_level(struct fs_prio * prio)749 static unsigned int alloc_new_level(struct fs_prio *prio)
750 {
751 return _alloc_new_level(prio, NULL);
752 }
753
update_root_ft_create(struct mlx5_flow_root_namespace * root,struct mlx5_flow_table * ft)754 static int update_root_ft_create(struct mlx5_flow_root_namespace *root,
755 struct mlx5_flow_table *ft)
756 {
757 int err = 0;
758 int min_level = INT_MAX;
759
760 if (root->root_ft)
761 min_level = root->root_ft->level;
762
763 if (ft->level < min_level)
764 err = mlx5_cmd_update_root_ft(root->dev, ft->type,
765 ft->id);
766 else
767 return err;
768
769 if (err)
770 mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n",
771 ft->id);
772 else
773 root->root_ft = ft;
774
775 return err;
776 }
777
_create_ft_common(struct mlx5_flow_namespace * ns,u16 vport,struct fs_prio * fs_prio,int max_fte,const char * name)778 static struct mlx5_flow_table *_create_ft_common(struct mlx5_flow_namespace *ns,
779 u16 vport,
780 struct fs_prio *fs_prio,
781 int max_fte,
782 const char *name)
783 {
784 struct mlx5_flow_table *ft;
785 int err;
786 int log_table_sz;
787 int ft_size;
788 char gen_name[20];
789 struct mlx5_flow_root_namespace *root = find_root(&ns->base);
790 struct mlx5_core_dev *dev = fs_get_dev(&ns->base);
791
792 if (!root) {
793 mlx5_core_err(dev,
794 "flow steering failed to find root of namespace %s",
795 ns->base.name);
796 return ERR_PTR(-ENODEV);
797 }
798
799 if (fs_prio->num_ft == fs_prio->max_ft)
800 return ERR_PTR(-ENOSPC);
801
802 ft = kzalloc(sizeof(*ft), GFP_KERNEL);
803 if (!ft)
804 return ERR_PTR(-ENOMEM);
805
806 fs_init_node(&ft->base, 1);
807 INIT_LIST_HEAD(&ft->fgs);
808
809 /* Temporarily WA until we expose the level set in the API */
810 if (root->table_type == FS_FT_ESW_EGRESS_ACL ||
811 root->table_type == FS_FT_ESW_INGRESS_ACL)
812 ft->level = 0;
813 else
814 ft->level = alloc_new_level(fs_prio);
815
816 ft->base.type = FS_TYPE_FLOW_TABLE;
817 ft->vport = vport;
818 ft->type = root->table_type;
819 /*Two entries are reserved for star rules*/
820 ft_size = roundup_pow_of_two(max_fte + 2);
821 /*User isn't aware to those rules*/
822 ft->max_fte = ft_size - 2;
823 log_table_sz = ilog2(ft_size);
824
825 if (name == NULL || name[0] == '\0') {
826 snprintf(gen_name, sizeof(gen_name), "flow_table_%u", ft->id);
827 name = gen_name;
828 }
829
830 err = mlx5_cmd_fs_create_ft(root->dev, ft->vport, ft->type,
831 ft->level, log_table_sz, name, &ft->id);
832 if (err)
833 goto free_ft;
834
835 err = create_star_rule(ft, fs_prio);
836 if (err)
837 goto del_ft;
838
839 if ((root->table_type == FS_FT_NIC_RX) && MLX5_CAP_FLOWTABLE(root->dev,
840 flow_table_properties_nic_receive.modify_root)) {
841 err = update_root_ft_create(root, ft);
842 if (err)
843 goto destroy_star_rule;
844 }
845
846 _fs_add_node(&ft->base, name, &fs_prio->base);
847
848 list_add_tail(&ft->base.list, &fs_prio->objs);
849 fs_prio->num_ft++;
850
851 return ft;
852
853 destroy_star_rule:
854 destroy_star_rule(ft, fs_prio);
855 del_ft:
856 mlx5_cmd_fs_destroy_ft(root->dev, ft->vport, ft->type, ft->id);
857 free_ft:
858 kfree(ft);
859 return ERR_PTR(err);
860 }
861
create_ft_common(struct mlx5_flow_namespace * ns,u16 vport,unsigned int prio,int max_fte,const char * name)862 static struct mlx5_flow_table *create_ft_common(struct mlx5_flow_namespace *ns,
863 u16 vport,
864 unsigned int prio,
865 int max_fte,
866 const char *name)
867 {
868 struct fs_prio *fs_prio = NULL;
869 fs_prio = find_prio(ns, prio);
870 if (!fs_prio)
871 return ERR_PTR(-EINVAL);
872
873 return _create_ft_common(ns, vport, fs_prio, max_fte, name);
874 }
875
876
877 static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns,
878 struct list_head *start);
879
880 static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio,
881 struct list_head *start);
882
mlx5_create_autogrouped_shared_flow_table(struct fs_prio * fs_prio)883 static struct mlx5_flow_table *mlx5_create_autogrouped_shared_flow_table(struct fs_prio *fs_prio)
884 {
885 struct mlx5_flow_table *ft;
886
887 ft = find_first_ft_in_prio(fs_prio, &fs_prio->objs);
888 if (ft) {
889 ft->shared_refcount++;
890 return ft;
891 }
892
893 return NULL;
894 }
895
mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace * ns,int prio,const char * name,int num_flow_table_entries,int max_num_groups,int num_reserved_entries)896 struct mlx5_flow_table *mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
897 int prio,
898 const char *name,
899 int num_flow_table_entries,
900 int max_num_groups,
901 int num_reserved_entries)
902 {
903 struct mlx5_flow_table *ft = NULL;
904 struct fs_prio *fs_prio;
905 bool is_shared_prio;
906
907 if (max_num_groups > (num_flow_table_entries - num_reserved_entries))
908 return ERR_PTR(-EINVAL);
909 if (num_reserved_entries > num_flow_table_entries)
910 return ERR_PTR(-EINVAL);
911
912 fs_prio = find_prio(ns, prio);
913 if (!fs_prio)
914 return ERR_PTR(-EINVAL);
915
916 is_shared_prio = fs_prio->flags & MLX5_CORE_FS_PRIO_SHARED;
917 if (is_shared_prio) {
918 mutex_lock(&fs_prio->shared_lock);
919 ft = mlx5_create_autogrouped_shared_flow_table(fs_prio);
920 }
921
922 if (ft)
923 goto return_ft;
924
925 ft = create_ft_common(ns, 0, prio, num_flow_table_entries,
926 name);
927 if (IS_ERR(ft))
928 goto return_ft;
929
930 ft->autogroup.active = true;
931 ft->autogroup.max_types = max_num_groups;
932 ft->autogroup.max_fte = num_flow_table_entries - num_reserved_entries;
933 /* We save place for flow groups in addition to max types */
934 ft->autogroup.group_size = ft->autogroup.max_fte / (max_num_groups + 1);
935
936 if (is_shared_prio)
937 ft->shared_refcount = 1;
938
939 return_ft:
940 if (is_shared_prio)
941 mutex_unlock(&fs_prio->shared_lock);
942 return ft;
943 }
944 EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
945
mlx5_create_vport_flow_table(struct mlx5_flow_namespace * ns,u16 vport,int prio,const char * name,int num_flow_table_entries)946 struct mlx5_flow_table *mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns,
947 u16 vport,
948 int prio,
949 const char *name,
950 int num_flow_table_entries)
951 {
952 return create_ft_common(ns, vport, prio, num_flow_table_entries, name);
953 }
954 EXPORT_SYMBOL(mlx5_create_vport_flow_table);
955
mlx5_create_flow_table(struct mlx5_flow_namespace * ns,int prio,const char * name,int num_flow_table_entries)956 struct mlx5_flow_table *mlx5_create_flow_table(struct mlx5_flow_namespace *ns,
957 int prio,
958 const char *name,
959 int num_flow_table_entries)
960 {
961 return create_ft_common(ns, 0, prio, num_flow_table_entries, name);
962 }
963 EXPORT_SYMBOL(mlx5_create_flow_table);
964
_fs_del_ft(struct mlx5_flow_table * ft)965 static void _fs_del_ft(struct mlx5_flow_table *ft)
966 {
967 int err;
968 struct mlx5_core_dev *dev = fs_get_dev(&ft->base);
969 struct fs_prio *prio;
970
971 err = mlx5_cmd_fs_destroy_ft(dev, ft->vport, ft->type, ft->id);
972 if (err)
973 mlx5_core_warn(dev, "flow steering can't destroy ft %s\n",
974 ft->base.name);
975
976 fs_get_parent(prio, ft);
977 prio->num_ft--;
978 }
979
update_root_ft_destroy(struct mlx5_flow_root_namespace * root,struct mlx5_flow_table * ft)980 static int update_root_ft_destroy(struct mlx5_flow_root_namespace *root,
981 struct mlx5_flow_table *ft)
982 {
983 int err = 0;
984 struct fs_prio *prio;
985 struct mlx5_flow_table *next_ft = NULL;
986 struct mlx5_flow_table *put_ft = NULL;
987
988 if (root->root_ft != ft)
989 return 0;
990
991 fs_get_parent(prio, ft);
992 /*Assuming objs containis only flow tables and
993 * flow tables are sorted by level.
994 */
995 if (!list_is_last(&ft->base.list, &prio->objs)) {
996 next_ft = list_next_entry(ft, base.list);
997 } else {
998 next_ft = find_next_ft(prio);
999 put_ft = next_ft;
1000 }
1001
1002 if (next_ft) {
1003 err = mlx5_cmd_update_root_ft(root->dev, next_ft->type,
1004 next_ft->id);
1005 if (err)
1006 mlx5_core_warn(root->dev, "Update root flow table of id=%u failed\n",
1007 ft->id);
1008 }
1009 if (!err)
1010 root->root_ft = next_ft;
1011
1012 if (put_ft)
1013 fs_put(&put_ft->base);
1014
1015 return err;
1016 }
1017
1018 /*Objects in the same prio are destroyed in the reverse order they were createrd*/
mlx5_destroy_flow_table(struct mlx5_flow_table * ft)1019 int mlx5_destroy_flow_table(struct mlx5_flow_table *ft)
1020 {
1021 int err = 0;
1022 struct fs_prio *prio;
1023 struct mlx5_flow_root_namespace *root;
1024 bool is_shared_prio;
1025 struct mlx5_core_dev *dev;
1026
1027 fs_get_parent(prio, ft);
1028 root = find_root(&prio->base);
1029 dev = fs_get_dev(&prio->base);
1030
1031 if (!root) {
1032 mlx5_core_err(dev,
1033 "flow steering failed to find root of priority %s",
1034 prio->base.name);
1035 return -ENODEV;
1036 }
1037
1038 is_shared_prio = prio->flags & MLX5_CORE_FS_PRIO_SHARED;
1039 if (is_shared_prio) {
1040 mutex_lock(&prio->shared_lock);
1041 if (ft->shared_refcount > 1) {
1042 --ft->shared_refcount;
1043 fs_put(&ft->base);
1044 mutex_unlock(&prio->shared_lock);
1045 return 0;
1046 }
1047 }
1048
1049 mutex_lock(&prio->base.lock);
1050 mutex_lock(&ft->base.lock);
1051
1052 err = update_root_ft_destroy(root, ft);
1053 if (err)
1054 goto unlock_ft;
1055
1056 /* delete two last entries */
1057 destroy_star_rule(ft, prio);
1058
1059 mutex_unlock(&ft->base.lock);
1060 fs_remove_node_parent_locked(&ft->base);
1061 mutex_unlock(&prio->base.lock);
1062 if (is_shared_prio)
1063 mutex_unlock(&prio->shared_lock);
1064
1065 return err;
1066
1067 unlock_ft:
1068 mutex_unlock(&ft->base.lock);
1069 mutex_unlock(&prio->base.lock);
1070 if (is_shared_prio)
1071 mutex_unlock(&prio->shared_lock);
1072
1073 return err;
1074 }
1075 EXPORT_SYMBOL(mlx5_destroy_flow_table);
1076
fs_create_fg(struct mlx5_core_dev * dev,struct mlx5_flow_table * ft,struct list_head * prev,u32 * fg_in,int refcount)1077 static struct mlx5_flow_group *fs_create_fg(struct mlx5_core_dev *dev,
1078 struct mlx5_flow_table *ft,
1079 struct list_head *prev,
1080 u32 *fg_in,
1081 int refcount)
1082 {
1083 struct mlx5_flow_group *fg;
1084 unsigned int group_size;
1085 int err;
1086 char name[20];
1087
1088 fg = fs_alloc_fg(fg_in);
1089 if (IS_ERR(fg))
1090 return fg;
1091
1092 group_size = MLX5_GET(create_flow_group_in, fg_in, end_flow_index) -
1093 MLX5_GET(create_flow_group_in, fg_in, start_flow_index) + 1;
1094 err = mlx5_cmd_fs_create_fg(dev, fg_in,
1095 ft->vport, ft->type, ft->id,
1096 &fg->id);
1097 if (err)
1098 goto free_fg;
1099
1100 mutex_lock(&ft->base.lock);
1101
1102 if (ft->autogroup.active && group_size == ft->autogroup.group_size)
1103 ft->autogroup.num_types++;
1104
1105 snprintf(name, sizeof(name), "group_%u", fg->id);
1106 /*Add node to tree*/
1107 fs_add_node(&fg->base, &ft->base, name, refcount);
1108 /*Add node to group list*/
1109 list_add(&fg->base.list, prev);
1110 mutex_unlock(&ft->base.lock);
1111
1112 return fg;
1113
1114 free_fg:
1115 kfree(fg);
1116 return ERR_PTR(err);
1117 }
1118
mlx5_create_flow_group(struct mlx5_flow_table * ft,u32 * in)1119 struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft,
1120 u32 *in)
1121 {
1122 struct mlx5_flow_group *fg;
1123 struct mlx5_core_dev *dev = fs_get_dev(&ft->base);
1124 unsigned int start_index;
1125
1126 start_index = MLX5_GET(create_flow_group_in, in, start_flow_index);
1127 if (!dev)
1128 return ERR_PTR(-ENODEV);
1129
1130 if (ft->autogroup.active && start_index < ft->autogroup.max_fte)
1131 return ERR_PTR(-EPERM);
1132
1133 fg = fs_create_fg(dev, ft, ft->fgs.prev, in, 1);
1134
1135 return fg;
1136 }
1137 EXPORT_SYMBOL(mlx5_create_flow_group);
1138
1139 /*Group is destoyed when all the rules in the group were removed*/
fs_del_fg(struct mlx5_flow_group * fg)1140 static void fs_del_fg(struct mlx5_flow_group *fg)
1141 {
1142 struct mlx5_flow_table *parent_ft;
1143 struct mlx5_core_dev *dev;
1144
1145 fs_get_parent(parent_ft, fg);
1146 dev = fs_get_dev(&parent_ft->base);
1147 WARN_ON(!dev);
1148
1149 if (parent_ft->autogroup.active &&
1150 fg->max_ftes == parent_ft->autogroup.group_size &&
1151 fg->start_index < parent_ft->autogroup.max_fte)
1152 parent_ft->autogroup.num_types--;
1153
1154 if (mlx5_cmd_fs_destroy_fg(dev, parent_ft->vport,
1155 parent_ft->type,
1156 parent_ft->id, fg->id))
1157 mlx5_core_warn(dev, "flow steering can't destroy fg\n");
1158 }
1159
mlx5_destroy_flow_group(struct mlx5_flow_group * fg)1160 void mlx5_destroy_flow_group(struct mlx5_flow_group *fg)
1161 {
1162 fs_remove_node(&fg->base);
1163 }
1164 EXPORT_SYMBOL(mlx5_destroy_flow_group);
1165
_fs_match_exact_val(void * mask,void * val1,void * val2,size_t size)1166 static bool _fs_match_exact_val(void *mask, void *val1, void *val2, size_t size)
1167 {
1168 unsigned int i;
1169
1170 /* TODO: optimize by comparing 64bits when possible */
1171 for (i = 0; i < size; i++, mask++, val1++, val2++)
1172 if ((*((u8 *)val1) & (*(u8 *)mask)) !=
1173 ((*(u8 *)val2) & (*(u8 *)mask)))
1174 return false;
1175
1176 return true;
1177 }
1178
fs_match_exact_val(struct mlx5_core_fs_mask * mask,void * val1,void * val2)1179 bool fs_match_exact_val(struct mlx5_core_fs_mask *mask,
1180 void *val1, void *val2)
1181 {
1182 if (mask->match_criteria_enable &
1183 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS) {
1184 void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
1185 val1, outer_headers);
1186 void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
1187 val2, outer_headers);
1188 void *fte_mask = MLX5_ADDR_OF(fte_match_param,
1189 mask->match_criteria, outer_headers);
1190
1191 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
1192 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
1193 return false;
1194 }
1195
1196 if (mask->match_criteria_enable &
1197 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS) {
1198 void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
1199 val1, misc_parameters);
1200 void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
1201 val2, misc_parameters);
1202 void *fte_mask = MLX5_ADDR_OF(fte_match_param,
1203 mask->match_criteria, misc_parameters);
1204
1205 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
1206 MLX5_ST_SZ_BYTES(fte_match_set_misc)))
1207 return false;
1208 }
1209 if (mask->match_criteria_enable &
1210 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS) {
1211 void *fte_match1 = MLX5_ADDR_OF(fte_match_param,
1212 val1, inner_headers);
1213 void *fte_match2 = MLX5_ADDR_OF(fte_match_param,
1214 val2, inner_headers);
1215 void *fte_mask = MLX5_ADDR_OF(fte_match_param,
1216 mask->match_criteria, inner_headers);
1217
1218 if (!_fs_match_exact_val(fte_mask, fte_match1, fte_match2,
1219 MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4)))
1220 return false;
1221 }
1222 return true;
1223 }
1224
fs_match_exact_mask(u8 match_criteria_enable1,u8 match_criteria_enable2,void * mask1,void * mask2)1225 bool fs_match_exact_mask(u8 match_criteria_enable1,
1226 u8 match_criteria_enable2,
1227 void *mask1, void *mask2)
1228 {
1229 return match_criteria_enable1 == match_criteria_enable2 &&
1230 !memcmp(mask1, mask2, MLX5_ST_SZ_BYTES(fte_match_param));
1231 }
1232
1233 static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns,
1234 struct list_head *start);
1235
_find_first_ft_in_prio_reverse(struct fs_prio * prio,struct list_head * start)1236 static struct mlx5_flow_table *_find_first_ft_in_prio_reverse(struct fs_prio *prio,
1237 struct list_head *start)
1238 {
1239 struct fs_base *it = container_of(start, struct fs_base, list);
1240
1241 if (!prio)
1242 return NULL;
1243
1244 fs_for_each_ns_or_ft_continue_reverse(it, prio) {
1245 struct mlx5_flow_namespace *ns;
1246 struct mlx5_flow_table *ft;
1247
1248 if (it->type == FS_TYPE_FLOW_TABLE) {
1249 fs_get_obj(ft, it);
1250 fs_get(&ft->base);
1251 return ft;
1252 }
1253
1254 fs_get_obj(ns, it);
1255 WARN_ON(ns->base.type != FS_TYPE_NAMESPACE);
1256
1257 ft = find_first_ft_in_ns_reverse(ns, &ns->prios);
1258 if (ft)
1259 return ft;
1260 }
1261
1262 return NULL;
1263 }
1264
find_first_ft_in_prio_reverse(struct fs_prio * prio,struct list_head * start)1265 static struct mlx5_flow_table *find_first_ft_in_prio_reverse(struct fs_prio *prio,
1266 struct list_head *start)
1267 {
1268 struct mlx5_flow_table *ft;
1269
1270 if (!prio)
1271 return NULL;
1272
1273 mutex_lock(&prio->base.lock);
1274 ft = _find_first_ft_in_prio_reverse(prio, start);
1275 mutex_unlock(&prio->base.lock);
1276
1277 return ft;
1278 }
1279
find_first_ft_in_ns_reverse(struct mlx5_flow_namespace * ns,struct list_head * start)1280 static struct mlx5_flow_table *find_first_ft_in_ns_reverse(struct mlx5_flow_namespace *ns,
1281 struct list_head *start)
1282 {
1283 struct fs_prio *prio;
1284
1285 if (!ns)
1286 return NULL;
1287
1288 fs_get_obj(prio, container_of(start, struct fs_base, list));
1289 mutex_lock(&ns->base.lock);
1290 fs_for_each_prio_continue_reverse(prio, ns) {
1291 struct mlx5_flow_table *ft;
1292
1293 ft = find_first_ft_in_prio_reverse(prio, &prio->objs);
1294 if (ft) {
1295 mutex_unlock(&ns->base.lock);
1296 return ft;
1297 }
1298 }
1299 mutex_unlock(&ns->base.lock);
1300
1301 return NULL;
1302 }
1303
1304 /* Returned a held ft, assumed curr is protected, assumed curr's parent is
1305 * locked
1306 */
find_prev_ft(struct mlx5_flow_table * curr,struct fs_prio * prio)1307 static struct mlx5_flow_table *find_prev_ft(struct mlx5_flow_table *curr,
1308 struct fs_prio *prio)
1309 {
1310 struct mlx5_flow_table *ft = NULL;
1311 struct fs_base *curr_base;
1312
1313 if (!curr)
1314 return NULL;
1315
1316 /* prio has either namespace or flow-tables, but not both */
1317 if (!list_empty(&prio->objs) &&
1318 list_first_entry(&prio->objs, struct mlx5_flow_table, base.list) !=
1319 curr)
1320 return NULL;
1321
1322 while (!ft && prio) {
1323 struct mlx5_flow_namespace *ns;
1324
1325 fs_get_parent(ns, prio);
1326 ft = find_first_ft_in_ns_reverse(ns, &prio->base.list);
1327 curr_base = &ns->base;
1328 fs_get_parent(prio, ns);
1329
1330 if (prio && !ft)
1331 ft = find_first_ft_in_prio_reverse(prio,
1332 &curr_base->list);
1333 }
1334 return ft;
1335 }
1336
_find_first_ft_in_prio(struct fs_prio * prio,struct list_head * start)1337 static struct mlx5_flow_table *_find_first_ft_in_prio(struct fs_prio *prio,
1338 struct list_head *start)
1339 {
1340 struct fs_base *it = container_of(start, struct fs_base, list);
1341
1342 if (!prio)
1343 return NULL;
1344
1345 fs_for_each_ns_or_ft_continue(it, prio) {
1346 struct mlx5_flow_namespace *ns;
1347 struct mlx5_flow_table *ft;
1348
1349 if (it->type == FS_TYPE_FLOW_TABLE) {
1350 fs_get_obj(ft, it);
1351 fs_get(&ft->base);
1352 return ft;
1353 }
1354
1355 fs_get_obj(ns, it);
1356 WARN_ON(ns->base.type != FS_TYPE_NAMESPACE);
1357
1358 ft = find_first_ft_in_ns(ns, &ns->prios);
1359 if (ft)
1360 return ft;
1361 }
1362
1363 return NULL;
1364 }
1365
find_first_ft_in_prio(struct fs_prio * prio,struct list_head * start)1366 static struct mlx5_flow_table *find_first_ft_in_prio(struct fs_prio *prio,
1367 struct list_head *start)
1368 {
1369 struct mlx5_flow_table *ft;
1370
1371 if (!prio)
1372 return NULL;
1373
1374 mutex_lock(&prio->base.lock);
1375 ft = _find_first_ft_in_prio(prio, start);
1376 mutex_unlock(&prio->base.lock);
1377
1378 return ft;
1379 }
1380
find_first_ft_in_ns(struct mlx5_flow_namespace * ns,struct list_head * start)1381 static struct mlx5_flow_table *find_first_ft_in_ns(struct mlx5_flow_namespace *ns,
1382 struct list_head *start)
1383 {
1384 struct fs_prio *prio;
1385
1386 if (!ns)
1387 return NULL;
1388
1389 fs_get_obj(prio, container_of(start, struct fs_base, list));
1390 mutex_lock(&ns->base.lock);
1391 fs_for_each_prio_continue(prio, ns) {
1392 struct mlx5_flow_table *ft;
1393
1394 ft = find_first_ft_in_prio(prio, &prio->objs);
1395 if (ft) {
1396 mutex_unlock(&ns->base.lock);
1397 return ft;
1398 }
1399 }
1400 mutex_unlock(&ns->base.lock);
1401
1402 return NULL;
1403 }
1404
1405 /* returned a held ft, assumed curr is protected, assumed curr's parent is
1406 * locked
1407 */
find_next_ft(struct fs_prio * prio)1408 static struct mlx5_flow_table *find_next_ft(struct fs_prio *prio)
1409 {
1410 struct mlx5_flow_table *ft = NULL;
1411 struct fs_base *curr_base;
1412
1413 while (!ft && prio) {
1414 struct mlx5_flow_namespace *ns;
1415
1416 fs_get_parent(ns, prio);
1417 ft = find_first_ft_in_ns(ns, &prio->base.list);
1418 curr_base = &ns->base;
1419 fs_get_parent(prio, ns);
1420
1421 if (!ft && prio)
1422 ft = _find_first_ft_in_prio(prio, &curr_base->list);
1423 }
1424 return ft;
1425 }
1426
1427
1428 /* called under ft mutex lock */
create_autogroup(struct mlx5_flow_table * ft,u8 match_criteria_enable,u32 * match_criteria)1429 static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft,
1430 u8 match_criteria_enable,
1431 u32 *match_criteria)
1432 {
1433 unsigned int group_size;
1434 unsigned int candidate_index = 0;
1435 struct mlx5_flow_group *g;
1436 struct mlx5_flow_group *ret;
1437 struct list_head *prev = &ft->fgs;
1438 struct mlx5_core_dev *dev;
1439 u32 *in;
1440 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1441 void *match_criteria_addr;
1442 u32 max_fte = ft->autogroup.max_fte;
1443
1444 if (!ft->autogroup.active)
1445 return ERR_PTR(-ENOENT);
1446
1447 dev = fs_get_dev(&ft->base);
1448 if (!dev)
1449 return ERR_PTR(-ENODEV);
1450
1451 in = mlx5_vzalloc(inlen);
1452 if (!in) {
1453 mlx5_core_warn(dev, "failed to allocate inbox\n");
1454 return ERR_PTR(-ENOMEM);
1455 }
1456
1457
1458 if (ft->autogroup.num_types < ft->autogroup.max_types)
1459 group_size = ft->autogroup.group_size;
1460 else
1461 group_size = 1;
1462
1463 if (group_size == 0) {
1464 mlx5_core_warn(dev,
1465 "flow steering can't create group size of 0\n");
1466 ret = ERR_PTR(-EINVAL);
1467 goto out;
1468 }
1469
1470 /* sorted by start_index */
1471 fs_for_each_fg(g, ft) {
1472 if (candidate_index + group_size > g->start_index)
1473 candidate_index = g->start_index + g->max_ftes;
1474 else
1475 break;
1476 prev = &g->base.list;
1477 }
1478
1479 if (candidate_index + group_size > max_fte) {
1480 ret = ERR_PTR(-ENOSPC);
1481 goto out;
1482 }
1483
1484 MLX5_SET(create_flow_group_in, in, match_criteria_enable,
1485 match_criteria_enable);
1486 MLX5_SET(create_flow_group_in, in, start_flow_index, candidate_index);
1487 MLX5_SET(create_flow_group_in, in, end_flow_index, candidate_index +
1488 group_size - 1);
1489 match_criteria_addr = MLX5_ADDR_OF(create_flow_group_in,
1490 in, match_criteria);
1491 memcpy(match_criteria_addr, match_criteria,
1492 MLX5_ST_SZ_BYTES(fte_match_param));
1493
1494 ret = fs_create_fg(dev, ft, prev, in, 0);
1495 out:
1496 kvfree(in);
1497 return ret;
1498 }
1499
get_ns_with_notifiers(struct fs_base * node)1500 static struct mlx5_flow_namespace *get_ns_with_notifiers(struct fs_base *node)
1501 {
1502 struct mlx5_flow_namespace *ns = NULL;
1503
1504 while (node && (node->type != FS_TYPE_NAMESPACE ||
1505 list_empty(&container_of(node, struct
1506 mlx5_flow_namespace,
1507 base)->list_notifiers)))
1508 node = node->parent;
1509
1510 if (node)
1511 fs_get_obj(ns, node);
1512
1513 return ns;
1514 }
1515
1516
1517 /*Assumption- fte is locked*/
call_to_add_rule_notifiers(struct mlx5_flow_rule * dst,struct fs_fte * fte)1518 static void call_to_add_rule_notifiers(struct mlx5_flow_rule *dst,
1519 struct fs_fte *fte)
1520 {
1521 struct mlx5_flow_namespace *ns;
1522 struct mlx5_flow_handler *iter_handler;
1523 struct fs_client_priv_data *iter_client;
1524 void *data;
1525 bool is_new_rule = list_first_entry(&fte->dests,
1526 struct mlx5_flow_rule,
1527 base.list) == dst;
1528 int err;
1529
1530 ns = get_ns_with_notifiers(&fte->base);
1531 if (!ns)
1532 return;
1533
1534 down_read(&ns->notifiers_rw_sem);
1535 list_for_each_entry(iter_handler, &ns->list_notifiers,
1536 list) {
1537 if (iter_handler->add_dst_cb) {
1538 data = NULL;
1539 mutex_lock(&dst->clients_lock);
1540 list_for_each_entry(
1541 iter_client, &dst->clients_data, list) {
1542 if (iter_client->fs_handler == iter_handler) {
1543 data = iter_client->client_dst_data;
1544 break;
1545 }
1546 }
1547 mutex_unlock(&dst->clients_lock);
1548 err = iter_handler->add_dst_cb(dst,
1549 is_new_rule,
1550 data,
1551 iter_handler->client_context);
1552 if (err)
1553 break;
1554 }
1555 }
1556 up_read(&ns->notifiers_rw_sem);
1557 }
1558
call_to_del_rule_notifiers(struct mlx5_flow_rule * dst,struct fs_fte * fte)1559 static void call_to_del_rule_notifiers(struct mlx5_flow_rule *dst,
1560 struct fs_fte *fte)
1561 {
1562 struct mlx5_flow_namespace *ns;
1563 struct mlx5_flow_handler *iter_handler;
1564 struct fs_client_priv_data *iter_client;
1565 void *data;
1566 bool ctx_changed = (fte->dests_size == 0);
1567
1568 ns = get_ns_with_notifiers(&fte->base);
1569 if (!ns)
1570 return;
1571 down_read(&ns->notifiers_rw_sem);
1572 list_for_each_entry(iter_handler, &ns->list_notifiers,
1573 list) {
1574 data = NULL;
1575 mutex_lock(&dst->clients_lock);
1576 list_for_each_entry(iter_client, &dst->clients_data, list) {
1577 if (iter_client->fs_handler == iter_handler) {
1578 data = iter_client->client_dst_data;
1579 break;
1580 }
1581 }
1582 mutex_unlock(&dst->clients_lock);
1583 if (iter_handler->del_dst_cb) {
1584 iter_handler->del_dst_cb(dst, ctx_changed, data,
1585 iter_handler->client_context);
1586 }
1587 }
1588 up_read(&ns->notifiers_rw_sem);
1589 }
1590
1591 /* fte should not be deleted while calling this function */
_fs_add_dst_fte(struct fs_fte * fte,struct mlx5_flow_group * fg,struct mlx5_flow_destination * dest)1592 static struct mlx5_flow_rule *_fs_add_dst_fte(struct fs_fte *fte,
1593 struct mlx5_flow_group *fg,
1594 struct mlx5_flow_destination *dest)
1595 {
1596 struct mlx5_flow_table *ft;
1597 struct mlx5_flow_rule *dst;
1598 int err;
1599
1600 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1601 if (!dst)
1602 return ERR_PTR(-ENOMEM);
1603
1604 memcpy(&dst->dest_attr, dest, sizeof(*dest));
1605 dst->base.type = FS_TYPE_FLOW_DEST;
1606 INIT_LIST_HEAD(&dst->clients_data);
1607 mutex_init(&dst->clients_lock);
1608 fs_get_parent(ft, fg);
1609 /*Add dest to dests list- added as first element after the head*/
1610 list_add_tail(&dst->base.list, &fte->dests);
1611 fte->dests_size++;
1612 err = mlx5_cmd_fs_set_fte(fs_get_dev(&ft->base),
1613 ft->vport,
1614 &fte->status,
1615 fte->val, ft->type,
1616 ft->id, fte->index, fg->id, &fte->flow_act,
1617 fte->sw_action, fte->dests_size, &fte->dests);
1618 if (err)
1619 goto free_dst;
1620
1621 list_del(&dst->base.list);
1622
1623 return dst;
1624
1625 free_dst:
1626 list_del(&dst->base.list);
1627 kfree(dst);
1628 fte->dests_size--;
1629 return ERR_PTR(err);
1630 }
1631
get_dest_name(struct mlx5_flow_destination * dest)1632 static char *get_dest_name(struct mlx5_flow_destination *dest)
1633 {
1634 char *name = kzalloc(sizeof(char) * 20, GFP_KERNEL);
1635
1636 switch (dest->type) {
1637 case MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE:
1638 snprintf(name, 20, "dest_%s_%u", "flow_table",
1639 dest->ft->id);
1640 return name;
1641 case MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT:
1642 snprintf(name, 20, "dest_%s_%u", "vport",
1643 dest->vport_num);
1644 return name;
1645 case MLX5_FLOW_CONTEXT_DEST_TYPE_TIR:
1646 snprintf(name, 20, "dest_%s_%u", "tir", dest->tir_num);
1647 return name;
1648 default:
1649 kfree(name);
1650 return NULL;
1651 }
1652 }
1653
1654 /* assumed fg is locked */
fs_get_free_fg_index(struct mlx5_flow_group * fg,struct list_head ** prev)1655 static unsigned int fs_get_free_fg_index(struct mlx5_flow_group *fg,
1656 struct list_head **prev)
1657 {
1658 struct fs_fte *fte;
1659 unsigned int start = fg->start_index;
1660
1661 if (prev)
1662 *prev = &fg->ftes;
1663
1664 /* assumed list is sorted by index */
1665 fs_for_each_fte(fte, fg) {
1666 if (fte->index != start)
1667 return start;
1668 start++;
1669 if (prev)
1670 *prev = &fte->base.list;
1671 }
1672
1673 return start;
1674 }
1675
1676
fs_create_fte(struct mlx5_flow_group * fg,u32 * match_value,u32 sw_action,struct mlx5_flow_act * flow_act,struct list_head ** prev)1677 static struct fs_fte *fs_create_fte(struct mlx5_flow_group *fg,
1678 u32 *match_value,
1679 u32 sw_action,
1680 struct mlx5_flow_act *flow_act,
1681 struct list_head **prev)
1682 {
1683 struct fs_fte *fte;
1684 int index = 0;
1685
1686 index = fs_get_free_fg_index(fg, prev);
1687 fte = fs_alloc_fte(sw_action, flow_act, match_value, index);
1688 if (IS_ERR(fte))
1689 return fte;
1690
1691 return fte;
1692 }
1693
add_rule_to_tree(struct mlx5_flow_rule * rule,struct fs_fte * fte)1694 static void add_rule_to_tree(struct mlx5_flow_rule *rule,
1695 struct fs_fte *fte)
1696 {
1697 char *dest_name;
1698
1699 dest_name = get_dest_name(&rule->dest_attr);
1700 fs_add_node(&rule->base, &fte->base, dest_name, 1);
1701 /* re-add to list, since fs_add_node reset our list */
1702 list_add_tail(&rule->base.list, &fte->dests);
1703 kfree(dest_name);
1704 call_to_add_rule_notifiers(rule, fte);
1705 }
1706
fs_del_dst(struct mlx5_flow_rule * dst)1707 static void fs_del_dst(struct mlx5_flow_rule *dst)
1708 {
1709 struct mlx5_flow_table *ft;
1710 struct mlx5_flow_group *fg;
1711 struct fs_fte *fte;
1712 u32 *match_value;
1713 struct mlx5_core_dev *dev = fs_get_dev(&dst->base);
1714 int match_len = MLX5_ST_SZ_BYTES(fte_match_param);
1715 int err;
1716
1717 WARN_ON(!dev);
1718
1719 match_value = mlx5_vzalloc(match_len);
1720 if (!match_value) {
1721 mlx5_core_warn(dev, "failed to allocate inbox\n");
1722 return;
1723 }
1724
1725 fs_get_parent(fte, dst);
1726 fs_get_parent(fg, fte);
1727 sx_assert(&fg->base.lock.sx, SX_XLOCKED);
1728 memcpy(match_value, fte->val, sizeof(fte->val));
1729 /* ft can't be changed as fg is locked */
1730 fs_get_parent(ft, fg);
1731 list_del(&dst->base.list);
1732 fte->dests_size--;
1733 if (fte->dests_size) {
1734 err = mlx5_cmd_fs_set_fte(dev, ft->vport,
1735 &fte->status, match_value, ft->type,
1736 ft->id, fte->index, fg->id,
1737 &fte->flow_act, fte->sw_action,
1738 fte->dests_size, &fte->dests);
1739 if (err) {
1740 mlx5_core_warn(dev, "%s can't delete dst %s\n",
1741 __func__, dst->base.name);
1742 goto err;
1743 }
1744 }
1745 call_to_del_rule_notifiers(dst, fte);
1746 err:
1747 kvfree(match_value);
1748 }
1749
fs_del_fte(struct fs_fte * fte)1750 static void fs_del_fte(struct fs_fte *fte)
1751 {
1752 struct mlx5_flow_table *ft;
1753 struct mlx5_flow_group *fg;
1754 int err;
1755 struct mlx5_core_dev *dev;
1756
1757 fs_get_parent(fg, fte);
1758 fs_get_parent(ft, fg);
1759
1760 dev = fs_get_dev(&ft->base);
1761 WARN_ON(!dev);
1762
1763 err = mlx5_cmd_fs_delete_fte(dev, ft->vport, &fte->status,
1764 ft->type, ft->id, fte->index);
1765 if (err)
1766 mlx5_core_warn(dev, "flow steering can't delete fte %s\n",
1767 fte->base.name);
1768
1769 fg->num_ftes--;
1770 }
1771
check_conflicting_actions(const struct mlx5_flow_act * act1,const struct mlx5_flow_act * act2)1772 static bool check_conflicting_actions(const struct mlx5_flow_act *act1,
1773 const struct mlx5_flow_act *act2)
1774 {
1775 u32 action1 = act1->actions;
1776 u32 action2 = act2->actions;
1777 u32 xored_actions;
1778
1779 xored_actions = action1 ^ action2;
1780
1781 if (xored_actions & (MLX5_FLOW_ACT_ACTIONS_FLOW_TAG))
1782 return true;
1783
1784 if (action1 & MLX5_FLOW_ACT_ACTIONS_FLOW_TAG &&
1785 act1->flow_tag != act2->flow_tag)
1786 return true;
1787
1788 /* Can even have complex actions in merged rules */
1789 if (action1 & MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR)
1790 return true;
1791
1792 if (action1 & MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT)
1793 return true;
1794
1795 if (action1 & MLX5_FLOW_ACT_ACTIONS_COUNT)
1796 return true;
1797
1798 return false;
1799 }
1800
1801 /* assuming parent fg is locked */
1802 /* Add dst algorithm */
fs_add_dst_fg(struct mlx5_flow_group * fg,u32 * match_value,u32 sw_action,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest)1803 static struct mlx5_flow_rule *fs_add_dst_fg(struct mlx5_flow_group *fg,
1804 u32 *match_value,
1805 u32 sw_action,
1806 struct mlx5_flow_act *flow_act,
1807 struct mlx5_flow_destination *dest)
1808 {
1809 struct fs_fte *fte;
1810 struct mlx5_flow_rule *dst;
1811 struct mlx5_flow_table *ft;
1812 struct list_head *prev;
1813 char fte_name[20];
1814
1815 mutex_lock(&fg->base.lock);
1816 if (flow_act->flags & MLX5_FLOW_ACT_NO_APPEND)
1817 goto insert_fte;
1818
1819 fs_for_each_fte(fte, fg) {
1820 /* TODO: Check of size against PRM max size */
1821 mutex_lock(&fte->base.lock);
1822 if (fs_match_exact_val(&fg->mask, match_value, &fte->val) &&
1823 sw_action == fte->sw_action &&
1824 !check_conflicting_actions(flow_act, &fte->flow_act)) {
1825 dst = _fs_add_dst_fte(fte, fg, dest);
1826 mutex_unlock(&fte->base.lock);
1827 if (IS_ERR(dst))
1828 goto unlock_fg;
1829 goto add_rule;
1830 }
1831 mutex_unlock(&fte->base.lock);
1832 }
1833
1834 insert_fte:
1835 fs_get_parent(ft, fg);
1836 if (fg->num_ftes == fg->max_ftes) {
1837 dst = ERR_PTR(-ENOSPC);
1838 goto unlock_fg;
1839 }
1840
1841 fte = fs_create_fte(fg, match_value, sw_action, flow_act, &prev);
1842 if (IS_ERR(fte)) {
1843 dst = (void *)fte;
1844 goto unlock_fg;
1845 }
1846 dst = _fs_add_dst_fte(fte, fg, dest);
1847 if (IS_ERR(dst)) {
1848 kfree(fte);
1849 goto unlock_fg;
1850 }
1851
1852 fg->num_ftes++;
1853
1854 snprintf(fte_name, sizeof(fte_name), "fte%u", fte->index);
1855 /* Add node to tree */
1856 fs_add_node(&fte->base, &fg->base, fte_name, 0);
1857 list_add(&fte->base.list, prev);
1858 add_rule:
1859 add_rule_to_tree(dst, fte);
1860 unlock_fg:
1861 mutex_unlock(&fg->base.lock);
1862 return dst;
1863 }
1864
fs_add_dst_ft(struct mlx5_flow_table * ft,u8 match_criteria_enable,u32 * match_criteria,u32 * match_value,u32 sw_action,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest)1865 static struct mlx5_flow_rule *fs_add_dst_ft(struct mlx5_flow_table *ft,
1866 u8 match_criteria_enable,
1867 u32 *match_criteria,
1868 u32 *match_value,
1869 u32 sw_action,
1870 struct mlx5_flow_act *flow_act,
1871 struct mlx5_flow_destination *dest)
1872 {
1873 /*? where dst_entry is allocated*/
1874 struct mlx5_flow_group *g;
1875 struct mlx5_flow_rule *dst;
1876
1877 fs_get(&ft->base);
1878 mutex_lock(&ft->base.lock);
1879 fs_for_each_fg(g, ft)
1880 if (fs_match_exact_mask(g->mask.match_criteria_enable,
1881 match_criteria_enable,
1882 g->mask.match_criteria,
1883 match_criteria)) {
1884 mutex_unlock(&ft->base.lock);
1885
1886 dst = fs_add_dst_fg(g, match_value, sw_action, flow_act, dest);
1887 if (PTR_ERR(dst) && PTR_ERR(dst) != -ENOSPC)
1888 goto unlock;
1889 }
1890 mutex_unlock(&ft->base.lock);
1891
1892 g = create_autogroup(ft, match_criteria_enable, match_criteria);
1893 if (IS_ERR(g)) {
1894 dst = (void *)g;
1895 goto unlock;
1896 }
1897
1898 dst = fs_add_dst_fg(g, match_value,
1899 sw_action, flow_act, dest);
1900 if (IS_ERR(dst)) {
1901 /* Remove assumes refcount > 0 and autogroup creates a group
1902 * with a refcount = 0.
1903 */
1904 fs_get(&g->base);
1905 fs_remove_node(&g->base);
1906 goto unlock;
1907 }
1908
1909 unlock:
1910 fs_put(&ft->base);
1911 return dst;
1912 }
1913
1914 struct mlx5_flow_rule *
mlx5_add_flow_rule(struct mlx5_flow_table * ft,u8 match_criteria_enable,u32 * match_criteria,u32 * match_value,u32 sw_action,struct mlx5_flow_act * flow_act,struct mlx5_flow_destination * dest)1915 mlx5_add_flow_rule(struct mlx5_flow_table *ft,
1916 u8 match_criteria_enable,
1917 u32 *match_criteria,
1918 u32 *match_value,
1919 u32 sw_action,
1920 struct mlx5_flow_act *flow_act,
1921 struct mlx5_flow_destination *dest)
1922 {
1923 struct mlx5_flow_rule *dst;
1924 struct mlx5_flow_namespace *ns;
1925
1926 ns = get_ns_with_notifiers(&ft->base);
1927 if (ns)
1928 down_read(&ns->dests_rw_sem);
1929 dst = fs_add_dst_ft(ft, match_criteria_enable, match_criteria,
1930 match_value, sw_action, flow_act, dest);
1931 if (ns)
1932 up_read(&ns->dests_rw_sem);
1933
1934 return dst;
1935
1936
1937 }
1938 EXPORT_SYMBOL(mlx5_add_flow_rule);
1939
mlx5_del_flow_rule(struct mlx5_flow_rule ** pp)1940 void mlx5_del_flow_rule(struct mlx5_flow_rule **pp)
1941 {
1942 struct mlx5_flow_namespace *ns;
1943 struct mlx5_flow_rule *dst;
1944
1945 dst = *pp;
1946 *pp = NULL;
1947
1948 if (IS_ERR_OR_NULL(dst))
1949 return;
1950 ns = get_ns_with_notifiers(&dst->base);
1951 if (ns)
1952 down_read(&ns->dests_rw_sem);
1953 fs_remove_node(&dst->base);
1954 if (ns)
1955 up_read(&ns->dests_rw_sem);
1956 }
1957 EXPORT_SYMBOL(mlx5_del_flow_rule);
1958
1959 #define MLX5_CORE_FS_ROOT_NS_NAME "root"
1960 #define MLX5_CORE_FS_ESW_EGRESS_ACL "esw_egress_root"
1961 #define MLX5_CORE_FS_ESW_INGRESS_ACL "esw_ingress_root"
1962 #define MLX5_CORE_FS_FDB_ROOT_NS_NAME "fdb_root"
1963 #define MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME "sniffer_rx_root"
1964 #define MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME "sniffer_tx_root"
1965 #define MLX5_CORE_FS_PRIO_MAX_FT 4
1966 #define MLX5_CORE_FS_PRIO_MAX_NS 1
1967
fs_create_prio(struct mlx5_flow_namespace * ns,unsigned prio,int max_ft,const char * name,u8 flags)1968 static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns,
1969 unsigned prio, int max_ft,
1970 const char *name, u8 flags)
1971 {
1972 struct fs_prio *fs_prio;
1973
1974 fs_prio = kzalloc(sizeof(*fs_prio), GFP_KERNEL);
1975 if (!fs_prio)
1976 return ERR_PTR(-ENOMEM);
1977
1978 fs_prio->base.type = FS_TYPE_PRIO;
1979 fs_add_node(&fs_prio->base, &ns->base, name, 1);
1980 fs_prio->max_ft = max_ft;
1981 fs_prio->max_ns = MLX5_CORE_FS_PRIO_MAX_NS;
1982 fs_prio->prio = prio;
1983 fs_prio->flags = flags;
1984 list_add_tail(&fs_prio->base.list, &ns->prios);
1985 INIT_LIST_HEAD(&fs_prio->objs);
1986 mutex_init(&fs_prio->shared_lock);
1987
1988 return fs_prio;
1989 }
1990
cleanup_root_ns(struct mlx5_core_dev * dev)1991 static void cleanup_root_ns(struct mlx5_core_dev *dev)
1992 {
1993 struct mlx5_flow_root_namespace *root_ns = dev->root_ns;
1994 struct fs_prio *iter_prio;
1995
1996 if (!root_ns)
1997 return;
1998
1999 /* stage 1 */
2000 fs_for_each_prio(iter_prio, &root_ns->ns) {
2001 struct mlx5_flow_namespace *iter_ns;
2002
2003 fs_for_each_ns(iter_ns, iter_prio) {
2004 while (!list_empty(&iter_ns->prios)) {
2005 struct fs_base *iter_prio2 =
2006 list_first_entry(&iter_ns->prios,
2007 struct fs_base,
2008 list);
2009
2010 fs_remove_node(iter_prio2);
2011 }
2012 }
2013 }
2014
2015 /* stage 2 */
2016 fs_for_each_prio(iter_prio, &root_ns->ns) {
2017 while (!list_empty(&iter_prio->objs)) {
2018 struct fs_base *iter_ns =
2019 list_first_entry(&iter_prio->objs,
2020 struct fs_base,
2021 list);
2022
2023 fs_remove_node(iter_ns);
2024 }
2025 }
2026 /* stage 3 */
2027 while (!list_empty(&root_ns->ns.prios)) {
2028 struct fs_base *iter_prio =
2029 list_first_entry(&root_ns->ns.prios,
2030 struct fs_base,
2031 list);
2032
2033 fs_remove_node(iter_prio);
2034 }
2035
2036 fs_remove_node(&root_ns->ns.base);
2037 dev->root_ns = NULL;
2038 }
2039
cleanup_single_prio_root_ns(struct mlx5_core_dev * dev,struct mlx5_flow_root_namespace * root_ns)2040 static void cleanup_single_prio_root_ns(struct mlx5_core_dev *dev,
2041 struct mlx5_flow_root_namespace *root_ns)
2042 {
2043 struct fs_base *prio;
2044
2045 if (!root_ns)
2046 return;
2047
2048 if (!list_empty(&root_ns->ns.prios)) {
2049 prio = list_first_entry(&root_ns->ns.prios,
2050 struct fs_base,
2051 list);
2052 fs_remove_node(prio);
2053 }
2054 fs_remove_node(&root_ns->ns.base);
2055 root_ns = NULL;
2056 }
2057
mlx5_cleanup_fs(struct mlx5_core_dev * dev)2058 void mlx5_cleanup_fs(struct mlx5_core_dev *dev)
2059 {
2060 mlx5_cleanup_fc_stats(dev);
2061 cleanup_root_ns(dev);
2062 cleanup_single_prio_root_ns(dev, dev->sniffer_rx_root_ns);
2063 cleanup_single_prio_root_ns(dev, dev->sniffer_tx_root_ns);
2064 cleanup_single_prio_root_ns(dev, dev->fdb_root_ns);
2065 cleanup_single_prio_root_ns(dev, dev->esw_egress_root_ns);
2066 cleanup_single_prio_root_ns(dev, dev->esw_ingress_root_ns);
2067 }
2068
fs_init_namespace(struct mlx5_flow_namespace * ns)2069 static struct mlx5_flow_namespace *fs_init_namespace(struct mlx5_flow_namespace
2070 *ns)
2071 {
2072 ns->base.type = FS_TYPE_NAMESPACE;
2073 init_rwsem(&ns->dests_rw_sem);
2074 init_rwsem(&ns->notifiers_rw_sem);
2075 INIT_LIST_HEAD(&ns->prios);
2076 INIT_LIST_HEAD(&ns->list_notifiers);
2077
2078 return ns;
2079 }
2080
create_root_ns(struct mlx5_core_dev * dev,enum fs_ft_type table_type,char * name)2081 static struct mlx5_flow_root_namespace *create_root_ns(struct mlx5_core_dev *dev,
2082 enum fs_ft_type
2083 table_type,
2084 char *name)
2085 {
2086 struct mlx5_flow_root_namespace *root_ns;
2087 struct mlx5_flow_namespace *ns;
2088
2089 /* create the root namespace */
2090 root_ns = mlx5_vzalloc(sizeof(*root_ns));
2091 if (!root_ns)
2092 goto err;
2093
2094 root_ns->dev = dev;
2095 root_ns->table_type = table_type;
2096 mutex_init(&root_ns->fs_chain_lock);
2097
2098 ns = &root_ns->ns;
2099 fs_init_namespace(ns);
2100 fs_add_node(&ns->base, NULL, name, 1);
2101
2102 return root_ns;
2103 err:
2104 return NULL;
2105 }
2106
init_fdb_root_ns(struct mlx5_core_dev * dev)2107 static int init_fdb_root_ns(struct mlx5_core_dev *dev)
2108 {
2109 struct fs_prio *prio;
2110
2111 dev->fdb_root_ns = create_root_ns(dev, FS_FT_FDB,
2112 MLX5_CORE_FS_FDB_ROOT_NS_NAME);
2113 if (!dev->fdb_root_ns)
2114 return -ENOMEM;
2115
2116 /* create 1 prio*/
2117 prio = fs_create_prio(&dev->fdb_root_ns->ns, 0, 1, "fdb_prio", 0);
2118 if (IS_ERR(prio))
2119 return PTR_ERR(prio);
2120 else
2121 return 0;
2122 }
2123
2124 #define MAX_VPORTS 128
2125
init_egress_acl_root_ns(struct mlx5_core_dev * dev)2126 static int init_egress_acl_root_ns(struct mlx5_core_dev *dev)
2127 {
2128 struct fs_prio *prio;
2129
2130 dev->esw_egress_root_ns = create_root_ns(dev, FS_FT_ESW_EGRESS_ACL,
2131 MLX5_CORE_FS_ESW_EGRESS_ACL);
2132 if (!dev->esw_egress_root_ns)
2133 return -ENOMEM;
2134
2135 /* create 1 prio*/
2136 prio = fs_create_prio(&dev->esw_egress_root_ns->ns, 0, MAX_VPORTS,
2137 "esw_egress_prio", 0);
2138 if (IS_ERR(prio))
2139 return PTR_ERR(prio);
2140 else
2141 return 0;
2142 }
2143
init_ingress_acl_root_ns(struct mlx5_core_dev * dev)2144 static int init_ingress_acl_root_ns(struct mlx5_core_dev *dev)
2145 {
2146 struct fs_prio *prio;
2147
2148 dev->esw_ingress_root_ns = create_root_ns(dev, FS_FT_ESW_INGRESS_ACL,
2149 MLX5_CORE_FS_ESW_INGRESS_ACL);
2150 if (!dev->esw_ingress_root_ns)
2151 return -ENOMEM;
2152
2153 /* create 1 prio*/
2154 prio = fs_create_prio(&dev->esw_ingress_root_ns->ns, 0, MAX_VPORTS,
2155 "esw_ingress_prio", 0);
2156 if (IS_ERR(prio))
2157 return PTR_ERR(prio);
2158 else
2159 return 0;
2160 }
2161
init_sniffer_rx_root_ns(struct mlx5_core_dev * dev)2162 static int init_sniffer_rx_root_ns(struct mlx5_core_dev *dev)
2163 {
2164 struct fs_prio *prio;
2165
2166 dev->sniffer_rx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_RX,
2167 MLX5_CORE_FS_SNIFFER_RX_ROOT_NS_NAME);
2168 if (!dev->sniffer_rx_root_ns)
2169 return -ENOMEM;
2170
2171 /* create 1 prio*/
2172 prio = fs_create_prio(&dev->sniffer_rx_root_ns->ns, 0, 1,
2173 "sniffer_prio", 0);
2174 if (IS_ERR(prio))
2175 return PTR_ERR(prio);
2176 else
2177 return 0;
2178 }
2179
2180
init_sniffer_tx_root_ns(struct mlx5_core_dev * dev)2181 static int init_sniffer_tx_root_ns(struct mlx5_core_dev *dev)
2182 {
2183 struct fs_prio *prio;
2184
2185 dev->sniffer_tx_root_ns = create_root_ns(dev, FS_FT_SNIFFER_TX,
2186 MLX5_CORE_FS_SNIFFER_TX_ROOT_NS_NAME);
2187 if (!dev->sniffer_tx_root_ns)
2188 return -ENOMEM;
2189
2190 /* create 1 prio*/
2191 prio = fs_create_prio(&dev->sniffer_tx_root_ns->ns, 0, 1,
2192 "sniffer_prio", 0);
2193 if (IS_ERR(prio))
2194 return PTR_ERR(prio);
2195 else
2196 return 0;
2197 }
2198
fs_create_namespace(struct fs_prio * prio,const char * name)2199 static struct mlx5_flow_namespace *fs_create_namespace(struct fs_prio *prio,
2200 const char *name)
2201 {
2202 struct mlx5_flow_namespace *ns;
2203
2204 ns = kzalloc(sizeof(*ns), GFP_KERNEL);
2205 if (!ns)
2206 return ERR_PTR(-ENOMEM);
2207
2208 fs_init_namespace(ns);
2209 fs_add_node(&ns->base, &prio->base, name, 1);
2210 list_add_tail(&ns->base.list, &prio->objs);
2211
2212 return ns;
2213 }
2214
2215 #define FLOW_TABLE_BIT_SZ 1
2216 #define GET_FLOW_TABLE_CAP(dev, offset) \
2217 ((be32_to_cpu(*((__be32 *)(dev->hca_caps_cur[MLX5_CAP_FLOW_TABLE]) + \
2218 offset / 32)) >> \
2219 (32 - FLOW_TABLE_BIT_SZ - (offset & 0x1f))) & FLOW_TABLE_BIT_SZ)
2220
has_required_caps(struct mlx5_core_dev * dev,struct node_caps * caps)2221 static bool has_required_caps(struct mlx5_core_dev *dev, struct node_caps *caps)
2222 {
2223 int i;
2224
2225 for (i = 0; i < caps->arr_sz; i++) {
2226 if (!GET_FLOW_TABLE_CAP(dev, caps->caps[i]))
2227 return false;
2228 }
2229 return true;
2230 }
2231
_init_root_tree(struct mlx5_core_dev * dev,int max_ft_level,struct init_tree_node * node,struct fs_base * base_parent,struct init_tree_node * tree_parent)2232 static int _init_root_tree(struct mlx5_core_dev *dev, int max_ft_level,
2233 struct init_tree_node *node, struct fs_base *base_parent,
2234 struct init_tree_node *tree_parent)
2235 {
2236 struct mlx5_flow_namespace *fs_ns;
2237 struct fs_prio *fs_prio;
2238 int priority;
2239 struct fs_base *base;
2240 int i;
2241 int err = 0;
2242
2243 if (node->type == FS_TYPE_PRIO) {
2244 if ((node->min_ft_level > max_ft_level) ||
2245 !has_required_caps(dev, &node->caps))
2246 goto out;
2247
2248 fs_get_obj(fs_ns, base_parent);
2249 priority = node - tree_parent->children;
2250 fs_prio = fs_create_prio(fs_ns, priority,
2251 node->max_ft,
2252 node->name, node->flags);
2253 if (IS_ERR(fs_prio)) {
2254 err = PTR_ERR(fs_prio);
2255 goto out;
2256 }
2257 base = &fs_prio->base;
2258 } else if (node->type == FS_TYPE_NAMESPACE) {
2259 fs_get_obj(fs_prio, base_parent);
2260 fs_ns = fs_create_namespace(fs_prio, node->name);
2261 if (IS_ERR(fs_ns)) {
2262 err = PTR_ERR(fs_ns);
2263 goto out;
2264 }
2265 base = &fs_ns->base;
2266 } else {
2267 return -EINVAL;
2268 }
2269 for (i = 0; i < node->ar_size; i++) {
2270 err = _init_root_tree(dev, max_ft_level, &node->children[i], base,
2271 node);
2272 if (err)
2273 break;
2274 }
2275 out:
2276 return err;
2277 }
2278
init_root_tree(struct mlx5_core_dev * dev,int max_ft_level,struct init_tree_node * node,struct fs_base * parent)2279 static int init_root_tree(struct mlx5_core_dev *dev, int max_ft_level,
2280 struct init_tree_node *node, struct fs_base *parent)
2281 {
2282 int i;
2283 struct mlx5_flow_namespace *fs_ns;
2284 int err = 0;
2285
2286 fs_get_obj(fs_ns, parent);
2287 for (i = 0; i < node->ar_size; i++) {
2288 err = _init_root_tree(dev, max_ft_level,
2289 &node->children[i], &fs_ns->base, node);
2290 if (err)
2291 break;
2292 }
2293 return err;
2294 }
2295
2296 static int sum_max_ft_in_prio(struct fs_prio *prio);
sum_max_ft_in_ns(struct mlx5_flow_namespace * ns)2297 static int sum_max_ft_in_ns(struct mlx5_flow_namespace *ns)
2298 {
2299 struct fs_prio *prio;
2300 int sum = 0;
2301
2302 fs_for_each_prio(prio, ns) {
2303 sum += sum_max_ft_in_prio(prio);
2304 }
2305 return sum;
2306 }
2307
sum_max_ft_in_prio(struct fs_prio * prio)2308 static int sum_max_ft_in_prio(struct fs_prio *prio)
2309 {
2310 int sum = 0;
2311 struct fs_base *it;
2312 struct mlx5_flow_namespace *ns;
2313
2314 if (prio->max_ft)
2315 return prio->max_ft;
2316
2317 fs_for_each_ns_or_ft(it, prio) {
2318 if (it->type == FS_TYPE_FLOW_TABLE)
2319 continue;
2320
2321 fs_get_obj(ns, it);
2322 sum += sum_max_ft_in_ns(ns);
2323 }
2324 prio->max_ft = sum;
2325 return sum;
2326 }
2327
set_max_ft(struct mlx5_flow_namespace * ns)2328 static void set_max_ft(struct mlx5_flow_namespace *ns)
2329 {
2330 struct fs_prio *prio;
2331
2332 if (!ns)
2333 return;
2334
2335 fs_for_each_prio(prio, ns)
2336 sum_max_ft_in_prio(prio);
2337 }
2338
init_root_ns(struct mlx5_core_dev * dev)2339 static int init_root_ns(struct mlx5_core_dev *dev)
2340 {
2341 int max_ft_level = MLX5_CAP_FLOWTABLE(dev,
2342 flow_table_properties_nic_receive.
2343 max_ft_level);
2344
2345 dev->root_ns = create_root_ns(dev, FS_FT_NIC_RX,
2346 MLX5_CORE_FS_ROOT_NS_NAME);
2347 if (IS_ERR_OR_NULL(dev->root_ns))
2348 goto err;
2349
2350
2351 if (init_root_tree(dev, max_ft_level, &root_fs, &dev->root_ns->ns.base))
2352 goto err;
2353
2354 set_max_ft(&dev->root_ns->ns);
2355
2356 return 0;
2357 err:
2358 return -ENOMEM;
2359 }
2360
mlx5_get_match_criteria_enable(struct mlx5_flow_rule * rule)2361 u8 mlx5_get_match_criteria_enable(struct mlx5_flow_rule *rule)
2362 {
2363 struct fs_base *pbase;
2364 struct mlx5_flow_group *fg;
2365
2366 pbase = rule->base.parent;
2367 WARN_ON(!pbase);
2368 pbase = pbase->parent;
2369 WARN_ON(!pbase);
2370
2371 fs_get_obj(fg, pbase);
2372 return fg->mask.match_criteria_enable;
2373 }
2374
mlx5_get_match_value(u32 * match_value,struct mlx5_flow_rule * rule)2375 void mlx5_get_match_value(u32 *match_value,
2376 struct mlx5_flow_rule *rule)
2377 {
2378 struct fs_base *pbase;
2379 struct fs_fte *fte;
2380
2381 pbase = rule->base.parent;
2382 WARN_ON(!pbase);
2383 fs_get_obj(fte, pbase);
2384
2385 memcpy(match_value, fte->val, sizeof(fte->val));
2386 }
2387
mlx5_get_match_criteria(u32 * match_criteria,struct mlx5_flow_rule * rule)2388 void mlx5_get_match_criteria(u32 *match_criteria,
2389 struct mlx5_flow_rule *rule)
2390 {
2391 struct fs_base *pbase;
2392 struct mlx5_flow_group *fg;
2393
2394 pbase = rule->base.parent;
2395 WARN_ON(!pbase);
2396 pbase = pbase->parent;
2397 WARN_ON(!pbase);
2398
2399 fs_get_obj(fg, pbase);
2400 memcpy(match_criteria, &fg->mask.match_criteria,
2401 sizeof(fg->mask.match_criteria));
2402 }
2403
mlx5_init_fs(struct mlx5_core_dev * dev)2404 int mlx5_init_fs(struct mlx5_core_dev *dev)
2405 {
2406 int err;
2407
2408 if (MLX5_CAP_GEN(dev, nic_flow_table)) {
2409 err = init_root_ns(dev);
2410 if (err)
2411 goto err;
2412 }
2413
2414 err = init_fdb_root_ns(dev);
2415 if (err)
2416 goto err;
2417
2418 err = init_egress_acl_root_ns(dev);
2419 if (err)
2420 goto err;
2421
2422 err = init_ingress_acl_root_ns(dev);
2423 if (err)
2424 goto err;
2425
2426 err = init_sniffer_tx_root_ns(dev);
2427 if (err)
2428 goto err;
2429
2430 err = init_sniffer_rx_root_ns(dev);
2431 if (err)
2432 goto err;
2433
2434 err = mlx5_init_fc_stats(dev);
2435 if (err)
2436 goto err;
2437
2438 return 0;
2439 err:
2440 mlx5_cleanup_fs(dev);
2441 return err;
2442 }
2443
mlx5_get_flow_namespace(struct mlx5_core_dev * dev,enum mlx5_flow_namespace_type type)2444 struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
2445 enum mlx5_flow_namespace_type type)
2446 {
2447 struct mlx5_flow_root_namespace *root_ns = dev->root_ns;
2448 int prio;
2449 static struct fs_prio *fs_prio;
2450 struct mlx5_flow_namespace *ns;
2451
2452 switch (type) {
2453 case MLX5_FLOW_NAMESPACE_BYPASS:
2454 prio = 0;
2455 break;
2456 case MLX5_FLOW_NAMESPACE_OFFLOADS:
2457 prio = 1;
2458 break;
2459 case MLX5_FLOW_NAMESPACE_KERNEL:
2460 prio = 2;
2461 break;
2462 case MLX5_FLOW_NAMESPACE_LEFTOVERS:
2463 prio = 3;
2464 break;
2465 case MLX5_FLOW_NAMESPACE_FDB:
2466 if (dev->fdb_root_ns)
2467 return &dev->fdb_root_ns->ns;
2468 else
2469 return NULL;
2470 case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
2471 if (dev->esw_egress_root_ns)
2472 return &dev->esw_egress_root_ns->ns;
2473 else
2474 return NULL;
2475 case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
2476 if (dev->esw_ingress_root_ns)
2477 return &dev->esw_ingress_root_ns->ns;
2478 else
2479 return NULL;
2480 case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
2481 if (dev->sniffer_rx_root_ns)
2482 return &dev->sniffer_rx_root_ns->ns;
2483 else
2484 return NULL;
2485 case MLX5_FLOW_NAMESPACE_SNIFFER_TX:
2486 if (dev->sniffer_tx_root_ns)
2487 return &dev->sniffer_tx_root_ns->ns;
2488 else
2489 return NULL;
2490 default:
2491 return NULL;
2492 }
2493
2494 if (!root_ns)
2495 return NULL;
2496
2497 fs_prio = find_prio(&root_ns->ns, prio);
2498 if (!fs_prio)
2499 return NULL;
2500
2501 ns = list_first_entry(&fs_prio->objs,
2502 typeof(*ns),
2503 base.list);
2504
2505 return ns;
2506 }
2507 EXPORT_SYMBOL(mlx5_get_flow_namespace);
2508
2509
mlx5_set_rule_private_data(struct mlx5_flow_rule * rule,struct mlx5_flow_handler * fs_handler,void * client_data)2510 int mlx5_set_rule_private_data(struct mlx5_flow_rule *rule,
2511 struct mlx5_flow_handler *fs_handler,
2512 void *client_data)
2513 {
2514 struct fs_client_priv_data *priv_data;
2515
2516 mutex_lock(&rule->clients_lock);
2517 /*Check that hanlder isn't exists in the list already*/
2518 list_for_each_entry(priv_data, &rule->clients_data, list) {
2519 if (priv_data->fs_handler == fs_handler) {
2520 priv_data->client_dst_data = client_data;
2521 goto unlock;
2522 }
2523 }
2524 priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL);
2525 if (!priv_data) {
2526 mutex_unlock(&rule->clients_lock);
2527 return -ENOMEM;
2528 }
2529
2530 priv_data->client_dst_data = client_data;
2531 priv_data->fs_handler = fs_handler;
2532 list_add(&priv_data->list, &rule->clients_data);
2533
2534 unlock:
2535 mutex_unlock(&rule->clients_lock);
2536
2537 return 0;
2538 }
2539
remove_from_clients(struct mlx5_flow_rule * rule,bool ctx_changed,void * client_data,void * context)2540 static int remove_from_clients(struct mlx5_flow_rule *rule,
2541 bool ctx_changed,
2542 void *client_data,
2543 void *context)
2544 {
2545 struct fs_client_priv_data *iter_client;
2546 struct fs_client_priv_data *temp_client;
2547 struct mlx5_flow_handler *handler = (struct
2548 mlx5_flow_handler*)context;
2549
2550 mutex_lock(&rule->clients_lock);
2551 list_for_each_entry_safe(iter_client, temp_client,
2552 &rule->clients_data, list) {
2553 if (iter_client->fs_handler == handler) {
2554 list_del(&iter_client->list);
2555 kfree(iter_client);
2556 break;
2557 }
2558 }
2559 mutex_unlock(&rule->clients_lock);
2560
2561 return 0;
2562 }
2563
mlx5_register_rule_notifier(struct mlx5_core_dev * dev,enum mlx5_flow_namespace_type ns_type,rule_event_fn add_cb,rule_event_fn del_cb,void * context)2564 struct mlx5_flow_handler *mlx5_register_rule_notifier(struct mlx5_core_dev *dev,
2565 enum mlx5_flow_namespace_type ns_type,
2566 rule_event_fn add_cb,
2567 rule_event_fn del_cb,
2568 void *context)
2569 {
2570 struct mlx5_flow_namespace *ns;
2571 struct mlx5_flow_handler *handler;
2572
2573 ns = mlx5_get_flow_namespace(dev, ns_type);
2574 if (!ns)
2575 return ERR_PTR(-EINVAL);
2576
2577 handler = kzalloc(sizeof(*handler), GFP_KERNEL);
2578 if (!handler)
2579 return ERR_PTR(-ENOMEM);
2580
2581 handler->add_dst_cb = add_cb;
2582 handler->del_dst_cb = del_cb;
2583 handler->client_context = context;
2584 handler->ns = ns;
2585 down_write(&ns->notifiers_rw_sem);
2586 list_add_tail(&handler->list, &ns->list_notifiers);
2587 up_write(&ns->notifiers_rw_sem);
2588
2589 return handler;
2590 }
2591
2592 static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns,
2593 rule_event_fn add_rule_cb,
2594 void *context);
2595
mlx5_unregister_rule_notifier(struct mlx5_flow_handler * handler)2596 void mlx5_unregister_rule_notifier(struct mlx5_flow_handler *handler)
2597 {
2598 struct mlx5_flow_namespace *ns = handler->ns;
2599
2600 /*Remove from dst's clients*/
2601 down_write(&ns->dests_rw_sem);
2602 down_write(&ns->notifiers_rw_sem);
2603 iterate_rules_in_ns(ns, remove_from_clients, handler);
2604 list_del(&handler->list);
2605 up_write(&ns->notifiers_rw_sem);
2606 up_write(&ns->dests_rw_sem);
2607 kfree(handler);
2608 }
2609
iterate_rules_in_ft(struct mlx5_flow_table * ft,rule_event_fn add_rule_cb,void * context)2610 static void iterate_rules_in_ft(struct mlx5_flow_table *ft,
2611 rule_event_fn add_rule_cb,
2612 void *context)
2613 {
2614 struct mlx5_flow_group *iter_fg;
2615 struct fs_fte *iter_fte;
2616 struct mlx5_flow_rule *iter_rule;
2617 int err = 0;
2618 bool is_new_rule;
2619
2620 mutex_lock(&ft->base.lock);
2621 fs_for_each_fg(iter_fg, ft) {
2622 mutex_lock(&iter_fg->base.lock);
2623 fs_for_each_fte(iter_fte, iter_fg) {
2624 mutex_lock(&iter_fte->base.lock);
2625 is_new_rule = true;
2626 fs_for_each_dst(iter_rule, iter_fte) {
2627 fs_get(&iter_rule->base);
2628 err = add_rule_cb(iter_rule,
2629 is_new_rule,
2630 NULL,
2631 context);
2632 fs_put_parent_locked(&iter_rule->base);
2633 if (err)
2634 break;
2635 is_new_rule = false;
2636 }
2637 mutex_unlock(&iter_fte->base.lock);
2638 if (err)
2639 break;
2640 }
2641 mutex_unlock(&iter_fg->base.lock);
2642 if (err)
2643 break;
2644 }
2645 mutex_unlock(&ft->base.lock);
2646 }
2647
iterate_rules_in_prio(struct fs_prio * prio,rule_event_fn add_rule_cb,void * context)2648 static void iterate_rules_in_prio(struct fs_prio *prio,
2649 rule_event_fn add_rule_cb,
2650 void *context)
2651 {
2652 struct fs_base *it;
2653
2654 mutex_lock(&prio->base.lock);
2655 fs_for_each_ns_or_ft(it, prio) {
2656 if (it->type == FS_TYPE_FLOW_TABLE) {
2657 struct mlx5_flow_table *ft;
2658
2659 fs_get_obj(ft, it);
2660 iterate_rules_in_ft(ft, add_rule_cb, context);
2661 } else {
2662 struct mlx5_flow_namespace *ns;
2663
2664 fs_get_obj(ns, it);
2665 iterate_rules_in_ns(ns, add_rule_cb, context);
2666 }
2667 }
2668 mutex_unlock(&prio->base.lock);
2669 }
2670
iterate_rules_in_ns(struct mlx5_flow_namespace * ns,rule_event_fn add_rule_cb,void * context)2671 static void iterate_rules_in_ns(struct mlx5_flow_namespace *ns,
2672 rule_event_fn add_rule_cb,
2673 void *context)
2674 {
2675 struct fs_prio *iter_prio;
2676
2677 mutex_lock(&ns->base.lock);
2678 fs_for_each_prio(iter_prio, ns) {
2679 iterate_rules_in_prio(iter_prio, add_rule_cb, context);
2680 }
2681 mutex_unlock(&ns->base.lock);
2682 }
2683
mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace * ns,rule_event_fn add_rule_cb,void * context)2684 void mlx5_flow_iterate_existing_rules(struct mlx5_flow_namespace *ns,
2685 rule_event_fn add_rule_cb,
2686 void *context)
2687 {
2688 down_write(&ns->dests_rw_sem);
2689 down_read(&ns->notifiers_rw_sem);
2690 iterate_rules_in_ns(ns, add_rule_cb, context);
2691 up_read(&ns->notifiers_rw_sem);
2692 up_write(&ns->dests_rw_sem);
2693 }
2694
2695
mlx5_del_flow_rules_list(struct mlx5_flow_rules_list * rules_list)2696 void mlx5_del_flow_rules_list(struct mlx5_flow_rules_list *rules_list)
2697 {
2698 struct mlx5_flow_rule_node *iter_node;
2699 struct mlx5_flow_rule_node *temp_node;
2700
2701 list_for_each_entry_safe(iter_node, temp_node, &rules_list->head, list) {
2702 list_del(&iter_node->list);
2703 kfree(iter_node);
2704 }
2705
2706 kfree(rules_list);
2707 }
2708
2709 #define ROCEV1_ETHERTYPE 0x8915
set_rocev1_rules(struct list_head * rules_list)2710 static int set_rocev1_rules(struct list_head *rules_list)
2711 {
2712 struct mlx5_flow_rule_node *rocev1_rule;
2713
2714 rocev1_rule = kzalloc(sizeof(*rocev1_rule), GFP_KERNEL);
2715 if (!rocev1_rule)
2716 return -ENOMEM;
2717
2718 rocev1_rule->match_criteria_enable =
2719 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
2720 MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_criteria, ethertype,
2721 0xffff);
2722 MLX5_SET(fte_match_set_lyr_2_4, rocev1_rule->match_value, ethertype,
2723 ROCEV1_ETHERTYPE);
2724
2725 list_add_tail(&rocev1_rule->list, rules_list);
2726
2727 return 0;
2728 }
2729
2730 #define ROCEV2_UDP_PORT 4791
set_rocev2_rules(struct list_head * rules_list)2731 static int set_rocev2_rules(struct list_head *rules_list)
2732 {
2733 struct mlx5_flow_rule_node *ipv4_rule;
2734 struct mlx5_flow_rule_node *ipv6_rule;
2735
2736 ipv4_rule = kzalloc(sizeof(*ipv4_rule), GFP_KERNEL);
2737 if (!ipv4_rule)
2738 return -ENOMEM;
2739
2740 ipv6_rule = kzalloc(sizeof(*ipv6_rule), GFP_KERNEL);
2741 if (!ipv6_rule) {
2742 kfree(ipv4_rule);
2743 return -ENOMEM;
2744 }
2745
2746 ipv4_rule->match_criteria_enable =
2747 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
2748 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ethertype,
2749 0xffff);
2750 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ethertype,
2751 0x0800);
2752 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, ip_protocol,
2753 0xff);
2754 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, ip_protocol,
2755 IPPROTO_UDP);
2756 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_criteria, udp_dport,
2757 0xffff);
2758 MLX5_SET(fte_match_set_lyr_2_4, ipv4_rule->match_value, udp_dport,
2759 ROCEV2_UDP_PORT);
2760
2761 ipv6_rule->match_criteria_enable =
2762 1 << MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS;
2763 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ethertype,
2764 0xffff);
2765 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ethertype,
2766 0x86dd);
2767 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, ip_protocol,
2768 0xff);
2769 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, ip_protocol,
2770 IPPROTO_UDP);
2771 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_criteria, udp_dport,
2772 0xffff);
2773 MLX5_SET(fte_match_set_lyr_2_4, ipv6_rule->match_value, udp_dport,
2774 ROCEV2_UDP_PORT);
2775
2776 list_add_tail(&ipv4_rule->list, rules_list);
2777 list_add_tail(&ipv6_rule->list, rules_list);
2778
2779 return 0;
2780 }
2781
2782
get_roce_flow_rules(u8 roce_mode)2783 struct mlx5_flow_rules_list *get_roce_flow_rules(u8 roce_mode)
2784 {
2785 int err = 0;
2786 struct mlx5_flow_rules_list *rules_list =
2787 kzalloc(sizeof(*rules_list), GFP_KERNEL);
2788
2789 if (!rules_list)
2790 return NULL;
2791
2792 INIT_LIST_HEAD(&rules_list->head);
2793
2794 if (roce_mode & MLX5_ROCE_VERSION_1_CAP) {
2795 err = set_rocev1_rules(&rules_list->head);
2796 if (err)
2797 goto free_list;
2798 }
2799 if (roce_mode & MLX5_ROCE_VERSION_2_CAP)
2800 err = set_rocev2_rules(&rules_list->head);
2801 if (err)
2802 goto free_list;
2803
2804 return rules_list;
2805
2806 free_list:
2807 mlx5_del_flow_rules_list(rules_list);
2808 return NULL;
2809 }
2810
mlx5_modify_header_alloc(struct mlx5_core_dev * dev,enum mlx5_flow_namespace_type ns_type,u8 num_actions,void * modify_actions)2811 struct mlx5_modify_hdr *mlx5_modify_header_alloc(struct mlx5_core_dev *dev,
2812 enum mlx5_flow_namespace_type ns_type,
2813 u8 num_actions,
2814 void *modify_actions)
2815 {
2816 struct mlx5_modify_hdr *modify_hdr;
2817 int err;
2818
2819 modify_hdr = kzalloc(sizeof(*modify_hdr), GFP_KERNEL);
2820 if (!modify_hdr)
2821 return ERR_PTR(-ENOMEM);
2822
2823 modify_hdr->ns_type = ns_type;
2824 err = mlx5_cmd_modify_header_alloc(dev, ns_type, num_actions,
2825 modify_actions, modify_hdr);
2826 if (err) {
2827 kfree(modify_hdr);
2828 return ERR_PTR(err);
2829 }
2830
2831 return modify_hdr;
2832 }
2833 EXPORT_SYMBOL(mlx5_modify_header_alloc);
2834
mlx5_modify_header_dealloc(struct mlx5_core_dev * dev,struct mlx5_modify_hdr * modify_hdr)2835 void mlx5_modify_header_dealloc(struct mlx5_core_dev *dev,
2836 struct mlx5_modify_hdr *modify_hdr)
2837 {
2838 mlx5_cmd_modify_header_dealloc(dev, modify_hdr);
2839 kfree(modify_hdr);
2840 }
2841 EXPORT_SYMBOL(mlx5_modify_header_dealloc);
2842
mlx5_packet_reformat_alloc(struct mlx5_core_dev * dev,struct mlx5_pkt_reformat_params * params,enum mlx5_flow_namespace_type ns_type)2843 struct mlx5_pkt_reformat *mlx5_packet_reformat_alloc(struct mlx5_core_dev *dev,
2844 struct mlx5_pkt_reformat_params *params,
2845 enum mlx5_flow_namespace_type ns_type)
2846 {
2847 struct mlx5_pkt_reformat *pkt_reformat;
2848 int err;
2849
2850 pkt_reformat = kzalloc(sizeof(*pkt_reformat), GFP_KERNEL);
2851 if (!pkt_reformat)
2852 return ERR_PTR(-ENOMEM);
2853
2854 pkt_reformat->ns_type = ns_type;
2855 pkt_reformat->reformat_type = params->type;
2856 err = mlx5_cmd_packet_reformat_alloc(dev, params, ns_type,
2857 pkt_reformat);
2858 if (err) {
2859 kfree(pkt_reformat);
2860 return ERR_PTR(err);
2861 }
2862
2863 return pkt_reformat;
2864 }
2865 EXPORT_SYMBOL(mlx5_packet_reformat_alloc);
2866
mlx5_packet_reformat_dealloc(struct mlx5_core_dev * dev,struct mlx5_pkt_reformat * pkt_reformat)2867 void mlx5_packet_reformat_dealloc(struct mlx5_core_dev *dev,
2868 struct mlx5_pkt_reformat *pkt_reformat)
2869 {
2870 mlx5_cmd_packet_reformat_dealloc(dev, pkt_reformat);
2871 kfree(pkt_reformat);
2872 }
2873 EXPORT_SYMBOL(mlx5_packet_reformat_dealloc);
2874
2875