109938b21SHans Petter Selasky /*-
209938b21SHans Petter Selasky * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
309938b21SHans Petter Selasky *
4478d3005SHans Petter Selasky * Copyright (c) 2014 Mellanox Technologies. All rights reserved.
5478d3005SHans Petter Selasky *
6478d3005SHans Petter Selasky * This software is available to you under a choice of one of two
7478d3005SHans Petter Selasky * licenses. You may choose to be licensed under the terms of the GNU
8478d3005SHans Petter Selasky * General Public License (GPL) Version 2, available from the file
9478d3005SHans Petter Selasky * COPYING in the main directory of this source tree, or the
10478d3005SHans Petter Selasky * OpenIB.org BSD license below:
11478d3005SHans Petter Selasky *
12478d3005SHans Petter Selasky * Redistribution and use in source and binary forms, with or
13478d3005SHans Petter Selasky * without modification, are permitted provided that the following
14478d3005SHans Petter Selasky * conditions are met:
15478d3005SHans Petter Selasky *
16478d3005SHans Petter Selasky * - Redistributions of source code must retain the above
17478d3005SHans Petter Selasky * copyright notice, this list of conditions and the following
18478d3005SHans Petter Selasky * disclaimer.
19478d3005SHans Petter Selasky *
20478d3005SHans Petter Selasky * - Redistributions in binary form must reproduce the above
21478d3005SHans Petter Selasky * copyright notice, this list of conditions and the following
22478d3005SHans Petter Selasky * disclaimer in the documentation and/or other materials
23478d3005SHans Petter Selasky * provided with the distribution.
24478d3005SHans Petter Selasky *
25478d3005SHans Petter Selasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26478d3005SHans Petter Selasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27478d3005SHans Petter Selasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28478d3005SHans Petter Selasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29478d3005SHans Petter Selasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30478d3005SHans Petter Selasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31478d3005SHans Petter Selasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32478d3005SHans Petter Selasky * SOFTWARE.
33478d3005SHans Petter Selasky */
34478d3005SHans Petter Selasky
35478d3005SHans Petter Selasky #ifndef IB_UMEM_ODP_H
36478d3005SHans Petter Selasky #define IB_UMEM_ODP_H
37478d3005SHans Petter Selasky
38478d3005SHans Petter Selasky #include <linux/rbtree.h>
39478d3005SHans Petter Selasky
40478d3005SHans Petter Selasky #include <rdma/ib_umem.h>
41478d3005SHans Petter Selasky #include <rdma/ib_verbs.h>
42478d3005SHans Petter Selasky
43478d3005SHans Petter Selasky struct umem_odp_node {
44478d3005SHans Petter Selasky u64 __subtree_last;
45478d3005SHans Petter Selasky struct rb_node rb;
46478d3005SHans Petter Selasky };
47478d3005SHans Petter Selasky
48478d3005SHans Petter Selasky struct ib_umem_odp {
49478d3005SHans Petter Selasky /*
50478d3005SHans Petter Selasky * An array of the pages included in the on-demand paging umem.
51478d3005SHans Petter Selasky * Indices of pages that are currently not mapped into the device will
52478d3005SHans Petter Selasky * contain NULL.
53478d3005SHans Petter Selasky */
54478d3005SHans Petter Selasky struct page **page_list;
55478d3005SHans Petter Selasky /*
56478d3005SHans Petter Selasky * An array of the same size as page_list, with DMA addresses mapped
57478d3005SHans Petter Selasky * for pages the pages in page_list. The lower two bits designate
58478d3005SHans Petter Selasky * access permissions. See ODP_READ_ALLOWED_BIT and
59478d3005SHans Petter Selasky * ODP_WRITE_ALLOWED_BIT.
60478d3005SHans Petter Selasky */
61478d3005SHans Petter Selasky dma_addr_t *dma_list;
62478d3005SHans Petter Selasky /*
63478d3005SHans Petter Selasky * The umem_mutex protects the page_list and dma_list fields of an ODP
64478d3005SHans Petter Selasky * umem, allowing only a single thread to map/unmap pages. The mutex
65478d3005SHans Petter Selasky * also protects access to the mmu notifier counters.
66478d3005SHans Petter Selasky */
67478d3005SHans Petter Selasky struct mutex umem_mutex;
68478d3005SHans Petter Selasky void *private; /* for the HW driver to use. */
69478d3005SHans Petter Selasky
70478d3005SHans Petter Selasky /* When false, use the notifier counter in the ucontext struct. */
71478d3005SHans Petter Selasky bool mn_counters_active;
72478d3005SHans Petter Selasky int notifiers_seq;
73478d3005SHans Petter Selasky int notifiers_count;
74478d3005SHans Petter Selasky
75478d3005SHans Petter Selasky /* A linked list of umems that don't have private mmu notifier
76478d3005SHans Petter Selasky * counters yet. */
77478d3005SHans Petter Selasky struct list_head no_private_counters;
78478d3005SHans Petter Selasky struct ib_umem *umem;
79478d3005SHans Petter Selasky
80478d3005SHans Petter Selasky /* Tree tracking */
81478d3005SHans Petter Selasky struct umem_odp_node interval_tree;
82478d3005SHans Petter Selasky
83478d3005SHans Petter Selasky struct completion notifier_completion;
84478d3005SHans Petter Selasky int dying;
85478d3005SHans Petter Selasky };
86478d3005SHans Petter Selasky
87478d3005SHans Petter Selasky #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
88478d3005SHans Petter Selasky
89478d3005SHans Petter Selasky int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem);
90478d3005SHans Petter Selasky
91478d3005SHans Petter Selasky void ib_umem_odp_release(struct ib_umem *umem);
92478d3005SHans Petter Selasky
93478d3005SHans Petter Selasky /*
94478d3005SHans Petter Selasky * The lower 2 bits of the DMA address signal the R/W permissions for
95478d3005SHans Petter Selasky * the entry. To upgrade the permissions, provide the appropriate
96478d3005SHans Petter Selasky * bitmask to the map_dma_pages function.
97478d3005SHans Petter Selasky *
98478d3005SHans Petter Selasky * Be aware that upgrading a mapped address might result in change of
99478d3005SHans Petter Selasky * the DMA address for the page.
100478d3005SHans Petter Selasky */
101478d3005SHans Petter Selasky #define ODP_READ_ALLOWED_BIT (1<<0ULL)
102478d3005SHans Petter Selasky #define ODP_WRITE_ALLOWED_BIT (1<<1ULL)
103478d3005SHans Petter Selasky
104478d3005SHans Petter Selasky #define ODP_DMA_ADDR_MASK (~(ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT))
105478d3005SHans Petter Selasky
106478d3005SHans Petter Selasky int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 start_offset, u64 bcnt,
107478d3005SHans Petter Selasky u64 access_mask, unsigned long current_seq);
108478d3005SHans Petter Selasky
109478d3005SHans Petter Selasky void ib_umem_odp_unmap_dma_pages(struct ib_umem *umem, u64 start_offset,
110478d3005SHans Petter Selasky u64 bound);
111478d3005SHans Petter Selasky
112478d3005SHans Petter Selasky void rbt_ib_umem_insert(struct umem_odp_node *node, struct rb_root *root);
113478d3005SHans Petter Selasky void rbt_ib_umem_remove(struct umem_odp_node *node, struct rb_root *root);
114478d3005SHans Petter Selasky typedef int (*umem_call_back)(struct ib_umem *item, u64 start, u64 end,
115478d3005SHans Petter Selasky void *cookie);
116478d3005SHans Petter Selasky /*
117478d3005SHans Petter Selasky * Call the callback on each ib_umem in the range. Returns the logical or of
118478d3005SHans Petter Selasky * the return values of the functions called.
119478d3005SHans Petter Selasky */
120478d3005SHans Petter Selasky int rbt_ib_umem_for_each_in_range(struct rb_root *root, u64 start, u64 end,
121478d3005SHans Petter Selasky umem_call_back cb, void *cookie);
122478d3005SHans Petter Selasky
123478d3005SHans Petter Selasky struct umem_odp_node *rbt_ib_umem_iter_first(struct rb_root *root,
124478d3005SHans Petter Selasky u64 start, u64 last);
125478d3005SHans Petter Selasky struct umem_odp_node *rbt_ib_umem_iter_next(struct umem_odp_node *node,
126478d3005SHans Petter Selasky u64 start, u64 last);
127478d3005SHans Petter Selasky
ib_umem_mmu_notifier_retry(struct ib_umem * item,unsigned long mmu_seq)128478d3005SHans Petter Selasky static inline int ib_umem_mmu_notifier_retry(struct ib_umem *item,
129478d3005SHans Petter Selasky unsigned long mmu_seq)
130478d3005SHans Petter Selasky {
131478d3005SHans Petter Selasky /*
132478d3005SHans Petter Selasky * This code is strongly based on the KVM code from
133478d3005SHans Petter Selasky * mmu_notifier_retry. Should be called with
134478d3005SHans Petter Selasky * the relevant locks taken (item->odp_data->umem_mutex
135478d3005SHans Petter Selasky * and the ucontext umem_mutex semaphore locked for read).
136478d3005SHans Petter Selasky */
137478d3005SHans Petter Selasky
138478d3005SHans Petter Selasky /* Do not allow page faults while the new ib_umem hasn't seen a state
139478d3005SHans Petter Selasky * with zero notifiers yet, and doesn't have its own valid set of
140478d3005SHans Petter Selasky * private counters. */
141478d3005SHans Petter Selasky if (!item->odp_data->mn_counters_active)
142478d3005SHans Petter Selasky return 1;
143478d3005SHans Petter Selasky
144478d3005SHans Petter Selasky if (unlikely(item->odp_data->notifiers_count))
145478d3005SHans Petter Selasky return 1;
146478d3005SHans Petter Selasky if (item->odp_data->notifiers_seq != mmu_seq)
147478d3005SHans Petter Selasky return 1;
148478d3005SHans Petter Selasky return 0;
149478d3005SHans Petter Selasky }
150478d3005SHans Petter Selasky
151478d3005SHans Petter Selasky #else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
152478d3005SHans Petter Selasky
ib_umem_odp_get(struct ib_ucontext * context,struct ib_umem * umem)153478d3005SHans Petter Selasky static inline int ib_umem_odp_get(struct ib_ucontext *context,
154478d3005SHans Petter Selasky struct ib_umem *umem)
155478d3005SHans Petter Selasky {
156478d3005SHans Petter Selasky return -EINVAL;
157478d3005SHans Petter Selasky }
158478d3005SHans Petter Selasky
ib_umem_odp_release(struct ib_umem * umem)159478d3005SHans Petter Selasky static inline void ib_umem_odp_release(struct ib_umem *umem) {}
160478d3005SHans Petter Selasky
161478d3005SHans Petter Selasky #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
162478d3005SHans Petter Selasky
163478d3005SHans Petter Selasky #endif /* IB_UMEM_ODP_H */
164