1*6bf9d8f6SLeon Romanovsky /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2b4e64397SDennis Dalessandro /*
3fe314195SDennis Dalessandro * Copyright(c) 2016 Intel Corporation.
4b4e64397SDennis Dalessandro */
5b4e64397SDennis Dalessandro
6*6bf9d8f6SLeon Romanovsky #ifndef DEF_RDMAVT_INCMR_H
7*6bf9d8f6SLeon Romanovsky #define DEF_RDMAVT_INCMR_H
8*6bf9d8f6SLeon Romanovsky
9b4e64397SDennis Dalessandro /*
10b4e64397SDennis Dalessandro * For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once
11b4e64397SDennis Dalessandro * drivers no longer need access to the MR directly.
12b4e64397SDennis Dalessandro */
13338adfddSSebastian Sanchez #include <linux/percpu-refcount.h>
14b4e64397SDennis Dalessandro
15b4e64397SDennis Dalessandro /*
16b4e64397SDennis Dalessandro * A segment is a linear region of low physical memory.
17b4e64397SDennis Dalessandro * Used by the verbs layer.
18b4e64397SDennis Dalessandro */
19b4e64397SDennis Dalessandro struct rvt_seg {
20b4e64397SDennis Dalessandro void *vaddr;
21b4e64397SDennis Dalessandro size_t length;
22b4e64397SDennis Dalessandro };
23b4e64397SDennis Dalessandro
24b4e64397SDennis Dalessandro /* The number of rvt_segs that fit in a page. */
25b4e64397SDennis Dalessandro #define RVT_SEGSZ (PAGE_SIZE / sizeof(struct rvt_seg))
26b4e64397SDennis Dalessandro
27b4e64397SDennis Dalessandro struct rvt_segarray {
28b4e64397SDennis Dalessandro struct rvt_seg segs[RVT_SEGSZ];
29b4e64397SDennis Dalessandro };
30b4e64397SDennis Dalessandro
31b4e64397SDennis Dalessandro struct rvt_mregion {
32b4e64397SDennis Dalessandro struct ib_pd *pd; /* shares refcnt of ibmr.pd */
33b4e64397SDennis Dalessandro u64 user_base; /* User's address for this region */
34b4e64397SDennis Dalessandro u64 iova; /* IB start address of this region */
35b4e64397SDennis Dalessandro size_t length;
36b4e64397SDennis Dalessandro u32 lkey;
37b4e64397SDennis Dalessandro u32 offset; /* offset (bytes) to start of region */
38b4e64397SDennis Dalessandro int access_flags;
39b4e64397SDennis Dalessandro u32 max_segs; /* number of rvt_segs in all the arrays */
40b4e64397SDennis Dalessandro u32 mapsz; /* size of the map array */
41338adfddSSebastian Sanchez atomic_t lkey_invalid; /* true if current lkey is invalid */
42b4e64397SDennis Dalessandro u8 page_shift; /* 0 - non unform/non powerof2 sizes */
43b4e64397SDennis Dalessandro u8 lkey_published; /* in global table */
44338adfddSSebastian Sanchez struct percpu_ref refcount;
45b4e64397SDennis Dalessandro struct completion comp; /* complete when refcount goes to zero */
465b361328SGustavo A. R. Silva struct rvt_segarray *map[]; /* the segments */
47b4e64397SDennis Dalessandro };
48b4e64397SDennis Dalessandro
49b4e64397SDennis Dalessandro #define RVT_MAX_LKEY_TABLE_BITS 23
50b4e64397SDennis Dalessandro
51b4e64397SDennis Dalessandro struct rvt_lkey_table {
5299f80d2fSMike Marciniszyn /* read mostly fields */
5399f80d2fSMike Marciniszyn u32 max; /* size of the table */
5499f80d2fSMike Marciniszyn u32 shift; /* lkey/rkey shift */
5599f80d2fSMike Marciniszyn struct rvt_mregion __rcu **table;
5699f80d2fSMike Marciniszyn /* writeable fields */
5799f80d2fSMike Marciniszyn /* protect changes in this struct */
5899f80d2fSMike Marciniszyn spinlock_t lock ____cacheline_aligned_in_smp;
59b4e64397SDennis Dalessandro u32 next; /* next unused index (speeds search) */
60b4e64397SDennis Dalessandro u32 gen; /* generation count */
61b4e64397SDennis Dalessandro };
62b4e64397SDennis Dalessandro
63b4e64397SDennis Dalessandro /*
64b4e64397SDennis Dalessandro * These keep track of the copy progress within a memory region.
65b4e64397SDennis Dalessandro * Used by the verbs layer.
66b4e64397SDennis Dalessandro */
67b4e64397SDennis Dalessandro struct rvt_sge {
68b4e64397SDennis Dalessandro struct rvt_mregion *mr;
69b4e64397SDennis Dalessandro void *vaddr; /* kernel virtual address of segment */
70b4e64397SDennis Dalessandro u32 sge_length; /* length of the SGE */
71b4e64397SDennis Dalessandro u32 length; /* remaining length of the segment */
72b4e64397SDennis Dalessandro u16 m; /* current index: mr->map[m] */
73b4e64397SDennis Dalessandro u16 n; /* current index: mr->map[m]->segs[n] */
74b4e64397SDennis Dalessandro };
75b4e64397SDennis Dalessandro
76b4e64397SDennis Dalessandro struct rvt_sge_state {
77b4e64397SDennis Dalessandro struct rvt_sge *sg_list; /* next SGE to be used if any */
78b4e64397SDennis Dalessandro struct rvt_sge sge; /* progress state for the current SGE */
79b4e64397SDennis Dalessandro u32 total_len;
80b4e64397SDennis Dalessandro u8 num_sge;
81b4e64397SDennis Dalessandro };
82b4e64397SDennis Dalessandro
rvt_put_mr(struct rvt_mregion * mr)83b4e64397SDennis Dalessandro static inline void rvt_put_mr(struct rvt_mregion *mr)
84b4e64397SDennis Dalessandro {
85338adfddSSebastian Sanchez percpu_ref_put(&mr->refcount);
86b4e64397SDennis Dalessandro }
87b4e64397SDennis Dalessandro
rvt_get_mr(struct rvt_mregion * mr)88b4e64397SDennis Dalessandro static inline void rvt_get_mr(struct rvt_mregion *mr)
89b4e64397SDennis Dalessandro {
90338adfddSSebastian Sanchez percpu_ref_get(&mr->refcount);
91b4e64397SDennis Dalessandro }
92b4e64397SDennis Dalessandro
rvt_put_ss(struct rvt_sge_state * ss)933b0b3fb3SDennis Dalessandro static inline void rvt_put_ss(struct rvt_sge_state *ss)
943b0b3fb3SDennis Dalessandro {
953b0b3fb3SDennis Dalessandro while (ss->num_sge) {
963b0b3fb3SDennis Dalessandro rvt_put_mr(ss->sge.mr);
973b0b3fb3SDennis Dalessandro if (--ss->num_sge)
983b0b3fb3SDennis Dalessandro ss->sge = *ss->sg_list++;
993b0b3fb3SDennis Dalessandro }
1003b0b3fb3SDennis Dalessandro }
1013b0b3fb3SDennis Dalessandro
rvt_get_sge_length(struct rvt_sge * sge,u32 length)1021198fceaSBrian Welty static inline u32 rvt_get_sge_length(struct rvt_sge *sge, u32 length)
1031198fceaSBrian Welty {
1041198fceaSBrian Welty u32 len = sge->length;
1051198fceaSBrian Welty
1061198fceaSBrian Welty if (len > length)
1071198fceaSBrian Welty len = length;
1081198fceaSBrian Welty if (len > sge->sge_length)
1091198fceaSBrian Welty len = sge->sge_length;
1101198fceaSBrian Welty
1111198fceaSBrian Welty return len;
1121198fceaSBrian Welty }
1131198fceaSBrian Welty
rvt_update_sge(struct rvt_sge_state * ss,u32 length,bool release)1141198fceaSBrian Welty static inline void rvt_update_sge(struct rvt_sge_state *ss, u32 length,
1151198fceaSBrian Welty bool release)
1161198fceaSBrian Welty {
1171198fceaSBrian Welty struct rvt_sge *sge = &ss->sge;
1181198fceaSBrian Welty
1191198fceaSBrian Welty sge->vaddr += length;
1201198fceaSBrian Welty sge->length -= length;
1211198fceaSBrian Welty sge->sge_length -= length;
1221198fceaSBrian Welty if (sge->sge_length == 0) {
1231198fceaSBrian Welty if (release)
1241198fceaSBrian Welty rvt_put_mr(sge->mr);
1251198fceaSBrian Welty if (--ss->num_sge)
1261198fceaSBrian Welty *sge = *ss->sg_list++;
1271198fceaSBrian Welty } else if (sge->length == 0 && sge->mr->lkey) {
1281198fceaSBrian Welty if (++sge->n >= RVT_SEGSZ) {
1291198fceaSBrian Welty if (++sge->m >= sge->mr->mapsz)
1301198fceaSBrian Welty return;
1311198fceaSBrian Welty sge->n = 0;
1321198fceaSBrian Welty }
1331198fceaSBrian Welty sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
1341198fceaSBrian Welty sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
1351198fceaSBrian Welty }
1361198fceaSBrian Welty }
1371198fceaSBrian Welty
rvt_skip_sge(struct rvt_sge_state * ss,u32 length,bool release)1381198fceaSBrian Welty static inline void rvt_skip_sge(struct rvt_sge_state *ss, u32 length,
1391198fceaSBrian Welty bool release)
1401198fceaSBrian Welty {
1411198fceaSBrian Welty struct rvt_sge *sge = &ss->sge;
1421198fceaSBrian Welty
1431198fceaSBrian Welty while (length) {
1441198fceaSBrian Welty u32 len = rvt_get_sge_length(sge, length);
1451198fceaSBrian Welty
1461198fceaSBrian Welty WARN_ON_ONCE(len == 0);
1471198fceaSBrian Welty rvt_update_sge(ss, len, release);
1481198fceaSBrian Welty length -= len;
1491198fceaSBrian Welty }
1501198fceaSBrian Welty }
1511198fceaSBrian Welty
1520208da90SMike Marciniszyn bool rvt_ss_has_lkey(struct rvt_sge_state *ss, u32 lkey);
1530208da90SMike Marciniszyn bool rvt_mr_has_lkey(struct rvt_mregion *mr, u32 lkey);
1540208da90SMike Marciniszyn
155b4e64397SDennis Dalessandro #endif /* DEF_RDMAVT_INCMRH */
156