1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Endpoint Function Driver to implement Non-Transparent Bridge functionality
4  *
5  * Copyright (C) 2020 Texas Instruments
6  * Author: Kishon Vijay Abraham I <kishon@ti.com>
7  */
8 
9 /*
10  * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint
11  * (EP) controller instances (see diagram below) in such a way that
12  * transactions from one EP controller are routed to the other EP controller.
13  * Once PCI NTB function driver configures the SoC with multiple EP instances,
14  * HOST1 and HOST2 can communicate with each other using SoC as a bridge.
15  *
16  *    +-------------+                                   +-------------+
17  *    |             |                                   |             |
18  *    |    HOST1    |                                   |    HOST2    |
19  *    |             |                                   |             |
20  *    +------^------+                                   +------^------+
21  *           |                                                 |
22  *           |                                                 |
23  * +---------|-------------------------------------------------|---------+
24  * |  +------v------+                                   +------v------+  |
25  * |  |             |                                   |             |  |
26  * |  |     EP      |                                   |     EP      |  |
27  * |  | CONTROLLER1 |                                   | CONTROLLER2 |  |
28  * |  |             <----------------------------------->             |  |
29  * |  |             |                                   |             |  |
30  * |  |             |                                   |             |  |
31  * |  |             |  SoC With Multiple EP Instances   |             |  |
32  * |  |             |  (Configured using NTB Function)  |             |  |
33  * |  +-------------+                                   +-------------+  |
34  * +---------------------------------------------------------------------+
35  */
36 
37 #include <linux/delay.h>
38 #include <linux/io.h>
39 #include <linux/module.h>
40 #include <linux/slab.h>
41 
42 #include <linux/pci-epc.h>
43 #include <linux/pci-epf.h>
44 
45 static struct workqueue_struct *kpcintb_workqueue;
46 
47 #define COMMAND_CONFIGURE_DOORBELL	1
48 #define COMMAND_TEARDOWN_DOORBELL	2
49 #define COMMAND_CONFIGURE_MW		3
50 #define COMMAND_TEARDOWN_MW		4
51 #define COMMAND_LINK_UP			5
52 #define COMMAND_LINK_DOWN		6
53 
54 #define COMMAND_STATUS_OK		1
55 #define COMMAND_STATUS_ERROR		2
56 
57 #define LINK_STATUS_UP			BIT(0)
58 
59 #define SPAD_COUNT			64
60 #define DB_COUNT			4
61 #define NTB_MW_OFFSET			2
62 #define DB_COUNT_MASK			GENMASK(15, 0)
63 #define MSIX_ENABLE			BIT(16)
64 #define MAX_DB_COUNT			32
65 #define MAX_MW				4
66 
67 enum epf_ntb_bar {
68 	BAR_CONFIG,
69 	BAR_PEER_SPAD,
70 	BAR_DB_MW1,
71 	BAR_MW2,
72 	BAR_MW3,
73 	BAR_MW4,
74 };
75 
76 struct epf_ntb {
77 	u32 num_mws;
78 	u32 db_count;
79 	u32 spad_count;
80 	struct pci_epf *epf;
81 	u64 mws_size[MAX_MW];
82 	struct config_group group;
83 	struct epf_ntb_epc *epc[2];
84 };
85 
86 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
87 
88 struct epf_ntb_epc {
89 	u8 func_no;
90 	u8 vfunc_no;
91 	bool linkup;
92 	bool is_msix;
93 	int msix_bar;
94 	u32 spad_size;
95 	struct pci_epc *epc;
96 	struct epf_ntb *epf_ntb;
97 	void __iomem *mw_addr[6];
98 	size_t msix_table_offset;
99 	struct epf_ntb_ctrl *reg;
100 	struct pci_epf_bar *epf_bar;
101 	enum pci_barno epf_ntb_bar[6];
102 	struct delayed_work cmd_handler;
103 	enum pci_epc_interface_type type;
104 	const struct pci_epc_features *epc_features;
105 };
106 
107 struct epf_ntb_ctrl {
108 	u32	command;
109 	u32	argument;
110 	u16	command_status;
111 	u16	link_status;
112 	u32	topology;
113 	u64	addr;
114 	u64	size;
115 	u32	num_mws;
116 	u32	mw1_offset;
117 	u32	spad_offset;
118 	u32	spad_count;
119 	u32	db_entry_size;
120 	u32	db_data[MAX_DB_COUNT];
121 	u32	db_offset[MAX_DB_COUNT];
122 } __packed;
123 
124 static struct pci_epf_header epf_ntb_header = {
125 	.vendorid	= PCI_ANY_ID,
126 	.deviceid	= PCI_ANY_ID,
127 	.baseclass_code	= PCI_BASE_CLASS_MEMORY,
128 	.interrupt_pin	= PCI_INTERRUPT_INTA,
129 };
130 
131 /**
132  * epf_ntb_link_up() - Raise link_up interrupt to both the hosts
133  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
134  * @link_up: true or false indicating Link is UP or Down
135  *
136  * Once NTB function in HOST1 and the NTB function in HOST2 invoke
137  * ntb_link_enable(), this NTB function driver will trigger a link event to
138  * the NTB client in both the hosts.
139  */
140 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
141 {
142 	enum pci_epc_interface_type type;
143 	enum pci_epc_irq_type irq_type;
144 	struct epf_ntb_epc *ntb_epc;
145 	struct epf_ntb_ctrl *ctrl;
146 	struct pci_epc *epc;
147 	u8 func_no, vfunc_no;
148 	bool is_msix;
149 	int ret;
150 
151 	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
152 		ntb_epc = ntb->epc[type];
153 		epc = ntb_epc->epc;
154 		func_no = ntb_epc->func_no;
155 		vfunc_no = ntb_epc->vfunc_no;
156 		is_msix = ntb_epc->is_msix;
157 		ctrl = ntb_epc->reg;
158 		if (link_up)
159 			ctrl->link_status |= LINK_STATUS_UP;
160 		else
161 			ctrl->link_status &= ~LINK_STATUS_UP;
162 		irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI;
163 		ret = pci_epc_raise_irq(epc, func_no, vfunc_no, irq_type, 1);
164 		if (ret) {
165 			dev_err(&epc->dev,
166 				"%s intf: Failed to raise Link Up IRQ\n",
167 				pci_epc_interface_string(type));
168 			return ret;
169 		}
170 	}
171 
172 	return 0;
173 }
174 
175 /**
176  * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host
177  *   to access the memory window of other host
178  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
179  * @type: PRIMARY interface or SECONDARY interface
180  * @mw: Index of the memory window (either 0, 1, 2 or 3)
181  *
182  * +-----------------+    +---->+----------------+-----------+-----------------+
183  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
184  * +-----------------+    |     +----------------+           +-----------------+
185  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
186  * +-----------------+----+     +----------------+         | |                 |
187  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
188  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
189  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
190  * +-----------------+    |     |----------------+     | |   |                 |
191  * |       BAR4      |    |     |                |     | |   +-----------------+
192  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
193  * |       BAR5      |    |     |                |   | |     +-----------------+
194  * +-----------------+    +---->-----------------+   | |     |                 |
195  *   EP CONTROLLER 1            |                |   | |     +-----------------+
196  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
197  *                              +----------------+   |       +-----------------+
198  *                      (A)      EP CONTROLLER 2     |       |                 |
199  *                                 (OB SPACE)        |       |                 |
200  *                                                   +------->      MW1        |
201  *                                                           |                 |
202  *                                                           |                 |
203  *                                                   (B)     +-----------------+
204  *                                                           |                 |
205  *                                                           |                 |
206  *                                                           |                 |
207  *                                                           |                 |
208  *                                                           |                 |
209  *                                                           +-----------------+
210  *                                                           PCI Address Space
211  *                                                           (Managed by HOST2)
212  *
213  * This function performs stage (B) in the above diagram (see MW1) i.e., map OB
214  * address space of memory window to PCI address space.
215  *
216  * This operation requires 3 parameters
217  *  1) Address in the outbound address space
218  *  2) Address in the PCI Address space
219  *  3) Size of the address region to be mapped
220  *
221  * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is
222  * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3
223  * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This
224  * is populated in epf_ntb_alloc_peer_mem() in this driver.
225  *
226  * The address and size of the PCI address region that has to be mapped would
227  * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is
228  * connected to HOST2.
229  *
230  * Please note Memory window1 (MW1) and Doorbell registers together will be
231  * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's
232  * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1],
233  * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2].
234  */
235 static int epf_ntb_configure_mw(struct epf_ntb *ntb,
236 				enum pci_epc_interface_type type, u32 mw)
237 {
238 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
239 	struct pci_epf_bar *peer_epf_bar;
240 	enum pci_barno peer_barno;
241 	struct epf_ntb_ctrl *ctrl;
242 	phys_addr_t phys_addr;
243 	u8 func_no, vfunc_no;
244 	struct pci_epc *epc;
245 	u64 addr, size;
246 	int ret = 0;
247 
248 	ntb_epc = ntb->epc[type];
249 	epc = ntb_epc->epc;
250 
251 	peer_ntb_epc = ntb->epc[!type];
252 	peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
253 	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
254 
255 	phys_addr = peer_epf_bar->phys_addr;
256 	ctrl = ntb_epc->reg;
257 	addr = ctrl->addr;
258 	size = ctrl->size;
259 	if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
260 		phys_addr += ctrl->mw1_offset;
261 
262 	if (size > ntb->mws_size[mw]) {
263 		dev_err(&epc->dev,
264 			"%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n",
265 			pci_epc_interface_string(type), mw, size,
266 			ntb->mws_size[mw]);
267 		ret = -EINVAL;
268 		goto err_invalid_size;
269 	}
270 
271 	func_no = ntb_epc->func_no;
272 	vfunc_no = ntb_epc->vfunc_no;
273 
274 	ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, addr, size);
275 	if (ret)
276 		dev_err(&epc->dev,
277 			"%s intf: Failed to map memory window %d address\n",
278 			pci_epc_interface_string(type), mw);
279 
280 err_invalid_size:
281 
282 	return ret;
283 }
284 
285 /**
286  * epf_ntb_teardown_mw() - Teardown the configured OB ATU
287  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
288  * @type: PRIMARY interface or SECONDARY interface
289  * @mw: Index of the memory window (either 0, 1, 2 or 3)
290  *
291  * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
292  * pci_epc_unmap_addr()
293  */
294 static void epf_ntb_teardown_mw(struct epf_ntb *ntb,
295 				enum pci_epc_interface_type type, u32 mw)
296 {
297 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
298 	struct pci_epf_bar *peer_epf_bar;
299 	enum pci_barno peer_barno;
300 	struct epf_ntb_ctrl *ctrl;
301 	phys_addr_t phys_addr;
302 	u8 func_no, vfunc_no;
303 	struct pci_epc *epc;
304 
305 	ntb_epc = ntb->epc[type];
306 	epc = ntb_epc->epc;
307 
308 	peer_ntb_epc = ntb->epc[!type];
309 	peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
310 	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
311 
312 	phys_addr = peer_epf_bar->phys_addr;
313 	ctrl = ntb_epc->reg;
314 	if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
315 		phys_addr += ctrl->mw1_offset;
316 	func_no = ntb_epc->func_no;
317 	vfunc_no = ntb_epc->vfunc_no;
318 
319 	pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr);
320 }
321 
322 /**
323  * epf_ntb_configure_msi() - Map OB address space to MSI address
324  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
325  * @type: PRIMARY interface or SECONDARY interface
326  * @db_count: Number of doorbell interrupts to map
327  *
328  *+-----------------+    +----->+----------------+-----------+-----------------+
329  *|       BAR0      |    |      |   Doorbell 1   +---+------->   MSI ADDRESS   |
330  *+-----------------+    |      +----------------+   |       +-----------------+
331  *|       BAR1      |    |      |   Doorbell 2   +---+       |                 |
332  *+-----------------+----+      +----------------+   |       |                 |
333  *|       BAR2      |           |   Doorbell 3   +---+       |                 |
334  *+-----------------+----+      +----------------+   |       |                 |
335  *|       BAR3      |    |      |   Doorbell 4   +---+       |                 |
336  *+-----------------+    |      |----------------+           |                 |
337  *|       BAR4      |    |      |                |           |                 |
338  *+-----------------+    |      |      MW1       |           |                 |
339  *|       BAR5      |    |      |                |           |                 |
340  *+-----------------+    +----->-----------------+           |                 |
341  *  EP CONTROLLER 1             |                |           |                 |
342  *                              |                |           |                 |
343  *                              +----------------+           +-----------------+
344  *                     (A)       EP CONTROLLER 2             |                 |
345  *                                 (OB SPACE)                |                 |
346  *                                                           |      MW1        |
347  *                                                           |                 |
348  *                                                           |                 |
349  *                                                   (B)     +-----------------+
350  *                                                           |                 |
351  *                                                           |                 |
352  *                                                           |                 |
353  *                                                           |                 |
354  *                                                           |                 |
355  *                                                           +-----------------+
356  *                                                           PCI Address Space
357  *                                                           (Managed by HOST2)
358  *
359  *
360  * This function performs stage (B) in the above diagram (see Doorbell 1,
361  * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
362  * doorbell to MSI address in PCI address space.
363  *
364  * This operation requires 3 parameters
365  *  1) Address reserved for doorbell in the outbound address space
366  *  2) MSI-X address in the PCIe Address space
367  *  3) Number of MSI-X interrupts that has to be configured
368  *
369  * The address in the outbound address space (for the Doorbell) is stored in
370  * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to
371  * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along
372  * with address for MW1.
373  *
374  * pci_epc_map_msi_irq() takes the MSI address from MSI capability register
375  * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI
376  * address.
377  *
378  * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt
379  * in db_data of the peer's control region. This helps the peer to raise
380  * doorbell of the other host by writing db_data to the BAR corresponding to
381  * BAR_DB_MW1.
382  */
383 static int epf_ntb_configure_msi(struct epf_ntb *ntb,
384 				 enum pci_epc_interface_type type, u16 db_count)
385 {
386 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
387 	u32 db_entry_size, db_data, db_offset;
388 	struct pci_epf_bar *peer_epf_bar;
389 	struct epf_ntb_ctrl *peer_ctrl;
390 	enum pci_barno peer_barno;
391 	phys_addr_t phys_addr;
392 	u8 func_no, vfunc_no;
393 	struct pci_epc *epc;
394 	int ret, i;
395 
396 	ntb_epc = ntb->epc[type];
397 	epc = ntb_epc->epc;
398 
399 	peer_ntb_epc = ntb->epc[!type];
400 	peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
401 	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
402 	peer_ctrl = peer_ntb_epc->reg;
403 	db_entry_size = peer_ctrl->db_entry_size;
404 
405 	phys_addr = peer_epf_bar->phys_addr;
406 	func_no = ntb_epc->func_no;
407 	vfunc_no = ntb_epc->vfunc_no;
408 
409 	ret = pci_epc_map_msi_irq(epc, func_no, vfunc_no, phys_addr, db_count,
410 				  db_entry_size, &db_data, &db_offset);
411 	if (ret) {
412 		dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n",
413 			pci_epc_interface_string(type));
414 		return ret;
415 	}
416 
417 	for (i = 0; i < db_count; i++) {
418 		peer_ctrl->db_data[i] = db_data | i;
419 		peer_ctrl->db_offset[i] = db_offset;
420 	}
421 
422 	return 0;
423 }
424 
425 /**
426  * epf_ntb_configure_msix() - Map OB address space to MSI-X address
427  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
428  * @type: PRIMARY interface or SECONDARY interface
429  * @db_count: Number of doorbell interrupts to map
430  *
431  *+-----------------+    +----->+----------------+-----------+-----------------+
432  *|       BAR0      |    |      |   Doorbell 1   +-----------> MSI-X ADDRESS 1 |
433  *+-----------------+    |      +----------------+           +-----------------+
434  *|       BAR1      |    |      |   Doorbell 2   +---------+ |                 |
435  *+-----------------+----+      +----------------+         | |                 |
436  *|       BAR2      |           |   Doorbell 3   +-------+ | +-----------------+
437  *+-----------------+----+      +----------------+       | +-> MSI-X ADDRESS 2 |
438  *|       BAR3      |    |      |   Doorbell 4   +-----+ |   +-----------------+
439  *+-----------------+    |      |----------------+     | |   |                 |
440  *|       BAR4      |    |      |                |     | |   +-----------------+
441  *+-----------------+    |      |      MW1       +     | +-->+ MSI-X ADDRESS 3||
442  *|       BAR5      |    |      |                |     |     +-----------------+
443  *+-----------------+    +----->-----------------+     |     |                 |
444  *  EP CONTROLLER 1             |                |     |     +-----------------+
445  *                              |                |     +---->+ MSI-X ADDRESS 4 |
446  *                              +----------------+           +-----------------+
447  *                     (A)       EP CONTROLLER 2             |                 |
448  *                                 (OB SPACE)                |                 |
449  *                                                           |      MW1        |
450  *                                                           |                 |
451  *                                                           |                 |
452  *                                                   (B)     +-----------------+
453  *                                                           |                 |
454  *                                                           |                 |
455  *                                                           |                 |
456  *                                                           |                 |
457  *                                                           |                 |
458  *                                                           +-----------------+
459  *                                                           PCI Address Space
460  *                                                           (Managed by HOST2)
461  *
462  * This function performs stage (B) in the above diagram (see Doorbell 1,
463  * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
464  * doorbell to MSI-X address in PCI address space.
465  *
466  * This operation requires 3 parameters
467  *  1) Address reserved for doorbell in the outbound address space
468  *  2) MSI-X address in the PCIe Address space
469  *  3) Number of MSI-X interrupts that has to be configured
470  *
471  * The address in the outbound address space (for the Doorbell) is stored in
472  * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to
473  * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along
474  * with address for MW1.
475  *
476  * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and
477  * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected
478  * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the
479  * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix()
480  * gets the MSI-X address and data.
481  *
482  * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt
483  * in db_data of the peer's control region. This helps the peer to raise
484  * doorbell of the other host by writing db_data to the BAR corresponding to
485  * BAR_DB_MW1.
486  */
487 static int epf_ntb_configure_msix(struct epf_ntb *ntb,
488 				  enum pci_epc_interface_type type,
489 				  u16 db_count)
490 {
491 	const struct pci_epc_features *epc_features;
492 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
493 	struct pci_epf_bar *peer_epf_bar, *epf_bar;
494 	struct pci_epf_msix_tbl *msix_tbl;
495 	struct epf_ntb_ctrl *peer_ctrl;
496 	u32 db_entry_size, msg_data;
497 	enum pci_barno peer_barno;
498 	phys_addr_t phys_addr;
499 	u8 func_no, vfunc_no;
500 	struct pci_epc *epc;
501 	size_t align;
502 	u64 msg_addr;
503 	int ret, i;
504 
505 	ntb_epc = ntb->epc[type];
506 	epc = ntb_epc->epc;
507 
508 	epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar];
509 	msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset;
510 
511 	peer_ntb_epc = ntb->epc[!type];
512 	peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
513 	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
514 	phys_addr = peer_epf_bar->phys_addr;
515 	peer_ctrl = peer_ntb_epc->reg;
516 	epc_features = ntb_epc->epc_features;
517 	align = epc_features->align;
518 
519 	func_no = ntb_epc->func_no;
520 	vfunc_no = ntb_epc->vfunc_no;
521 	db_entry_size = peer_ctrl->db_entry_size;
522 
523 	for (i = 0; i < db_count; i++) {
524 		msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align);
525 		msg_data = msix_tbl[i].msg_data;
526 		ret = pci_epc_map_addr(epc, func_no, vfunc_no, phys_addr, msg_addr,
527 				       db_entry_size);
528 		if (ret) {
529 			dev_err(&epc->dev,
530 				"%s intf: Failed to configure MSI-X IRQ\n",
531 				pci_epc_interface_string(type));
532 			return ret;
533 		}
534 		phys_addr = phys_addr + db_entry_size;
535 		peer_ctrl->db_data[i] = msg_data;
536 		peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1);
537 	}
538 	ntb_epc->is_msix = true;
539 
540 	return 0;
541 }
542 
543 /**
544  * epf_ntb_configure_db() - Configure the Outbound Address Space for one host
545  *   to ring the doorbell of other host
546  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
547  * @type: PRIMARY interface or SECONDARY interface
548  * @db_count: Count of the number of doorbells that has to be configured
549  * @msix: Indicates whether MSI-X or MSI should be used
550  *
551  * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for
552  * one HOST to ring the doorbell of other HOST.
553  */
554 static int epf_ntb_configure_db(struct epf_ntb *ntb,
555 				enum pci_epc_interface_type type,
556 				u16 db_count, bool msix)
557 {
558 	struct epf_ntb_epc *ntb_epc;
559 	struct pci_epc *epc;
560 	int ret;
561 
562 	if (db_count > MAX_DB_COUNT)
563 		return -EINVAL;
564 
565 	ntb_epc = ntb->epc[type];
566 	epc = ntb_epc->epc;
567 
568 	if (msix)
569 		ret = epf_ntb_configure_msix(ntb, type, db_count);
570 	else
571 		ret = epf_ntb_configure_msi(ntb, type, db_count);
572 
573 	if (ret)
574 		dev_err(&epc->dev, "%s intf: Failed to configure DB\n",
575 			pci_epc_interface_string(type));
576 
577 	return ret;
578 }
579 
580 /**
581  * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X
582  *   address
583  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
584  * @type: PRIMARY interface or SECONDARY interface
585  *
586  * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address.
587  */
588 static void
589 epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type)
590 {
591 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
592 	struct pci_epf_bar *peer_epf_bar;
593 	enum pci_barno peer_barno;
594 	phys_addr_t phys_addr;
595 	u8 func_no, vfunc_no;
596 	struct pci_epc *epc;
597 
598 	ntb_epc = ntb->epc[type];
599 	epc = ntb_epc->epc;
600 
601 	peer_ntb_epc = ntb->epc[!type];
602 	peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
603 	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
604 	phys_addr = peer_epf_bar->phys_addr;
605 	func_no = ntb_epc->func_no;
606 	vfunc_no = ntb_epc->vfunc_no;
607 
608 	pci_epc_unmap_addr(epc, func_no, vfunc_no, phys_addr);
609 }
610 
611 /**
612  * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host
613  * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY)
614  *
615  * Workqueue function that gets invoked for the two epf_ntb_epc
616  * periodically (once every 5ms) to see if it has received any commands
617  * from NTB host. The host can send commands to configure doorbell or
618  * configure memory window or to update link status.
619  */
620 static void epf_ntb_cmd_handler(struct work_struct *work)
621 {
622 	enum pci_epc_interface_type type;
623 	struct epf_ntb_epc *ntb_epc;
624 	struct epf_ntb_ctrl *ctrl;
625 	u32 command, argument;
626 	struct epf_ntb *ntb;
627 	struct device *dev;
628 	u16 db_count;
629 	bool is_msix;
630 	int ret;
631 
632 	ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work);
633 	ctrl = ntb_epc->reg;
634 	command = ctrl->command;
635 	if (!command)
636 		goto reset_handler;
637 	argument = ctrl->argument;
638 
639 	ctrl->command = 0;
640 	ctrl->argument = 0;
641 
642 	ctrl = ntb_epc->reg;
643 	type = ntb_epc->type;
644 	ntb = ntb_epc->epf_ntb;
645 	dev = &ntb->epf->dev;
646 
647 	switch (command) {
648 	case COMMAND_CONFIGURE_DOORBELL:
649 		db_count = argument & DB_COUNT_MASK;
650 		is_msix = argument & MSIX_ENABLE;
651 		ret = epf_ntb_configure_db(ntb, type, db_count, is_msix);
652 		if (ret < 0)
653 			ctrl->command_status = COMMAND_STATUS_ERROR;
654 		else
655 			ctrl->command_status = COMMAND_STATUS_OK;
656 		break;
657 	case COMMAND_TEARDOWN_DOORBELL:
658 		epf_ntb_teardown_db(ntb, type);
659 		ctrl->command_status = COMMAND_STATUS_OK;
660 		break;
661 	case COMMAND_CONFIGURE_MW:
662 		ret = epf_ntb_configure_mw(ntb, type, argument);
663 		if (ret < 0)
664 			ctrl->command_status = COMMAND_STATUS_ERROR;
665 		else
666 			ctrl->command_status = COMMAND_STATUS_OK;
667 		break;
668 	case COMMAND_TEARDOWN_MW:
669 		epf_ntb_teardown_mw(ntb, type, argument);
670 		ctrl->command_status = COMMAND_STATUS_OK;
671 		break;
672 	case COMMAND_LINK_UP:
673 		ntb_epc->linkup = true;
674 		if (ntb->epc[PRIMARY_INTERFACE]->linkup &&
675 		    ntb->epc[SECONDARY_INTERFACE]->linkup) {
676 			ret = epf_ntb_link_up(ntb, true);
677 			if (ret < 0)
678 				ctrl->command_status = COMMAND_STATUS_ERROR;
679 			else
680 				ctrl->command_status = COMMAND_STATUS_OK;
681 			goto reset_handler;
682 		}
683 		ctrl->command_status = COMMAND_STATUS_OK;
684 		break;
685 	case COMMAND_LINK_DOWN:
686 		ntb_epc->linkup = false;
687 		ret = epf_ntb_link_up(ntb, false);
688 		if (ret < 0)
689 			ctrl->command_status = COMMAND_STATUS_ERROR;
690 		else
691 			ctrl->command_status = COMMAND_STATUS_OK;
692 		break;
693 	default:
694 		dev_err(dev, "%s intf UNKNOWN command: %d\n",
695 			pci_epc_interface_string(type), command);
696 		break;
697 	}
698 
699 reset_handler:
700 	queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler,
701 			   msecs_to_jiffies(5));
702 }
703 
704 /**
705  * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR
706  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
707  *	     address.
708  *
709  *+-----------------+------->+------------------+        +-----------------+
710  *|       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
711  *+-----------------+----+   +------------------+<-------+-----------------+
712  *|       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
713  *+-----------------+    +-->+------------------+<-------+-----------------+
714  *|       BAR2      |            Local Memory            |       BAR2      |
715  *+-----------------+                                    +-----------------+
716  *|       BAR3      |                                    |       BAR3      |
717  *+-----------------+                                    +-----------------+
718  *|       BAR4      |                                    |       BAR4      |
719  *+-----------------+                                    +-----------------+
720  *|       BAR5      |                                    |       BAR5      |
721  *+-----------------+                                    +-----------------+
722  *  EP CONTROLLER 1                                        EP CONTROLLER 2
723  *
724  * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad
725  * region. While BAR1 is the default peer scratchpad BAR, an NTB could have
726  * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
727  * This function can get the exact BAR used for peer scratchpad from
728  * epf_ntb_bar[BAR_PEER_SPAD].
729  *
730  * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function
731  * gets the address of peer scratchpad from
732  * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
733  */
734 static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc)
735 {
736 	struct pci_epf_bar *epf_bar;
737 	enum pci_barno barno;
738 	u8 func_no, vfunc_no;
739 	struct pci_epc *epc;
740 
741 	epc = ntb_epc->epc;
742 	func_no = ntb_epc->func_no;
743 	vfunc_no = ntb_epc->vfunc_no;
744 	barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
745 	epf_bar = &ntb_epc->epf_bar[barno];
746 	pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
747 }
748 
749 /**
750  * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR
751  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
752  * @type: PRIMARY interface or SECONDARY interface
753  *
754  *+-----------------+------->+------------------+        +-----------------+
755  *|       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
756  *+-----------------+----+   +------------------+<-------+-----------------+
757  *|       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
758  *+-----------------+    +-->+------------------+<-------+-----------------+
759  *|       BAR2      |            Local Memory            |       BAR2      |
760  *+-----------------+                                    +-----------------+
761  *|       BAR3      |                                    |       BAR3      |
762  *+-----------------+                                    +-----------------+
763  *|       BAR4      |                                    |       BAR4      |
764  *+-----------------+                                    +-----------------+
765  *|       BAR5      |                                    |       BAR5      |
766  *+-----------------+                                    +-----------------+
767  *  EP CONTROLLER 1                                        EP CONTROLLER 2
768  *
769  * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad
770  * region. While BAR1 is the default peer scratchpad BAR, an NTB could have
771  * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
772  * This function can get the exact BAR used for peer scratchpad from
773  * epf_ntb_bar[BAR_PEER_SPAD].
774  *
775  * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function
776  * gets the address of peer scratchpad from
777  * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
778  */
779 static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb,
780 				     enum pci_epc_interface_type type)
781 {
782 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
783 	struct pci_epf_bar *peer_epf_bar, *epf_bar;
784 	enum pci_barno peer_barno, barno;
785 	u32 peer_spad_offset;
786 	u8 func_no, vfunc_no;
787 	struct pci_epc *epc;
788 	struct device *dev;
789 	int ret;
790 
791 	dev = &ntb->epf->dev;
792 
793 	peer_ntb_epc = ntb->epc[!type];
794 	peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG];
795 	peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
796 
797 	ntb_epc = ntb->epc[type];
798 	barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
799 	epf_bar = &ntb_epc->epf_bar[barno];
800 	func_no = ntb_epc->func_no;
801 	vfunc_no = ntb_epc->vfunc_no;
802 	epc = ntb_epc->epc;
803 
804 	peer_spad_offset = peer_ntb_epc->reg->spad_offset;
805 	epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset;
806 	epf_bar->size = peer_ntb_epc->spad_size;
807 	epf_bar->barno = barno;
808 	epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
809 
810 	ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
811 	if (ret) {
812 		dev_err(dev, "%s intf: peer SPAD BAR set failed\n",
813 			pci_epc_interface_string(type));
814 		return ret;
815 	}
816 
817 	return 0;
818 }
819 
820 /**
821  * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
822  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
823  *	     address.
824  *
825  * +-----------------+------->+------------------+        +-----------------+
826  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
827  * +-----------------+----+   +------------------+<-------+-----------------+
828  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
829  * +-----------------+    +-->+------------------+<-------+-----------------+
830  * |       BAR2      |            Local Memory            |       BAR2      |
831  * +-----------------+                                    +-----------------+
832  * |       BAR3      |                                    |       BAR3      |
833  * +-----------------+                                    +-----------------+
834  * |       BAR4      |                                    |       BAR4      |
835  * +-----------------+                                    +-----------------+
836  * |       BAR5      |                                    |       BAR5      |
837  * +-----------------+                                    +-----------------+
838  *   EP CONTROLLER 1                                        EP CONTROLLER 2
839  *
840  * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
841  * self scratchpad region (removes inbound ATU configuration). While BAR0 is
842  * the default self scratchpad BAR, an NTB could have other BARs for self
843  * scratchpad (because of reserved BARs). This function can get the exact BAR
844  * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
845  *
846  * Please note the self scratchpad region and config region is combined to
847  * a single region and mapped using the same BAR. Also note HOST2's peer
848  * scratchpad is HOST1's self scratchpad.
849  */
850 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc)
851 {
852 	struct pci_epf_bar *epf_bar;
853 	enum pci_barno barno;
854 	u8 func_no, vfunc_no;
855 	struct pci_epc *epc;
856 
857 	epc = ntb_epc->epc;
858 	func_no = ntb_epc->func_no;
859 	vfunc_no = ntb_epc->vfunc_no;
860 	barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
861 	epf_bar = &ntb_epc->epf_bar[barno];
862 	pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
863 }
864 
865 /**
866  * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
867  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
868  *	     address.
869  *
870  * +-----------------+------->+------------------+        +-----------------+
871  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
872  * +-----------------+----+   +------------------+<-------+-----------------+
873  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
874  * +-----------------+    +-->+------------------+<-------+-----------------+
875  * |       BAR2      |            Local Memory            |       BAR2      |
876  * +-----------------+                                    +-----------------+
877  * |       BAR3      |                                    |       BAR3      |
878  * +-----------------+                                    +-----------------+
879  * |       BAR4      |                                    |       BAR4      |
880  * +-----------------+                                    +-----------------+
881  * |       BAR5      |                                    |       BAR5      |
882  * +-----------------+                                    +-----------------+
883  *   EP CONTROLLER 1                                        EP CONTROLLER 2
884  *
885  * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
886  * self scratchpad region. While BAR0 is the default self scratchpad BAR, an
887  * NTB could have other BARs for self scratchpad (because of reserved BARs).
888  * This function can get the exact BAR used for self scratchpad from
889  * epf_ntb_bar[BAR_CONFIG].
890  *
891  * Please note the self scratchpad region and config region is combined to
892  * a single region and mapped using the same BAR. Also note HOST2's peer
893  * scratchpad is HOST1's self scratchpad.
894  */
895 static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc)
896 {
897 	struct pci_epf_bar *epf_bar;
898 	enum pci_barno barno;
899 	u8 func_no, vfunc_no;
900 	struct epf_ntb *ntb;
901 	struct pci_epc *epc;
902 	struct device *dev;
903 	int ret;
904 
905 	ntb = ntb_epc->epf_ntb;
906 	dev = &ntb->epf->dev;
907 
908 	epc = ntb_epc->epc;
909 	func_no = ntb_epc->func_no;
910 	vfunc_no = ntb_epc->vfunc_no;
911 	barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
912 	epf_bar = &ntb_epc->epf_bar[barno];
913 
914 	ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
915 	if (ret) {
916 		dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n",
917 			pci_epc_interface_string(ntb_epc->type));
918 		return ret;
919 	}
920 
921 	return 0;
922 }
923 
924 /**
925  * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
926  *   config + scratchpad region
927  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
928  *
929  * +-----------------+------->+------------------+        +-----------------+
930  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
931  * +-----------------+----+   +------------------+<-------+-----------------+
932  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
933  * +-----------------+    +-->+------------------+<-------+-----------------+
934  * |       BAR2      |            Local Memory            |       BAR2      |
935  * +-----------------+                                    +-----------------+
936  * |       BAR3      |                                    |       BAR3      |
937  * +-----------------+                                    +-----------------+
938  * |       BAR4      |                                    |       BAR4      |
939  * +-----------------+                                    +-----------------+
940  * |       BAR5      |                                    |       BAR5      |
941  * +-----------------+                                    +-----------------+
942  *   EP CONTROLLER 1                                        EP CONTROLLER 2
943  *
944  * Free the Local Memory mentioned in the above diagram. After invoking this
945  * function, any of config + self scratchpad region of HOST1 or peer scratchpad
946  * region of HOST2 should not be accessed.
947  */
948 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
949 {
950 	enum pci_epc_interface_type type;
951 	struct epf_ntb_epc *ntb_epc;
952 	enum pci_barno barno;
953 	struct pci_epf *epf;
954 
955 	epf = ntb->epf;
956 	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
957 		ntb_epc = ntb->epc[type];
958 		barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
959 		if (ntb_epc->reg)
960 			pci_epf_free_space(epf, ntb_epc->reg, barno, type);
961 	}
962 }
963 
964 /**
965  * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
966  *   region
967  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
968  * @type: PRIMARY interface or SECONDARY interface
969  *
970  * +-----------------+------->+------------------+        +-----------------+
971  * |       BAR0      |        |  CONFIG REGION   |        |       BAR0      |
972  * +-----------------+----+   +------------------+<-------+-----------------+
973  * |       BAR1      |    |   |SCRATCHPAD REGION |        |       BAR1      |
974  * +-----------------+    +-->+------------------+<-------+-----------------+
975  * |       BAR2      |            Local Memory            |       BAR2      |
976  * +-----------------+                                    +-----------------+
977  * |       BAR3      |                                    |       BAR3      |
978  * +-----------------+                                    +-----------------+
979  * |       BAR4      |                                    |       BAR4      |
980  * +-----------------+                                    +-----------------+
981  * |       BAR5      |                                    |       BAR5      |
982  * +-----------------+                                    +-----------------+
983  *   EP CONTROLLER 1                                        EP CONTROLLER 2
984  *
985  * Allocate the Local Memory mentioned in the above diagram. The size of
986  * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
987  * is obtained from "spad-count" configfs entry.
988  *
989  * The size of both config region and scratchpad region has to be aligned,
990  * since the scratchpad region will also be mapped as PEER SCRATCHPAD of
991  * other host using a separate BAR.
992  */
993 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb,
994 					 enum pci_epc_interface_type type)
995 {
996 	const struct pci_epc_features *peer_epc_features, *epc_features;
997 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
998 	size_t msix_table_size, pba_size, align;
999 	enum pci_barno peer_barno, barno;
1000 	struct epf_ntb_ctrl *ctrl;
1001 	u32 spad_size, ctrl_size;
1002 	u64 size, peer_size;
1003 	struct pci_epf *epf;
1004 	struct device *dev;
1005 	bool msix_capable;
1006 	u32 spad_count;
1007 	void *base;
1008 
1009 	epf = ntb->epf;
1010 	dev = &epf->dev;
1011 	ntb_epc = ntb->epc[type];
1012 
1013 	epc_features = ntb_epc->epc_features;
1014 	barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
1015 	size = epc_features->bar_fixed_size[barno];
1016 	align = epc_features->align;
1017 
1018 	peer_ntb_epc = ntb->epc[!type];
1019 	peer_epc_features = peer_ntb_epc->epc_features;
1020 	peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
1021 	peer_size = peer_epc_features->bar_fixed_size[peer_barno];
1022 
1023 	/* Check if epc_features is populated incorrectly */
1024 	if ((!IS_ALIGNED(size, align)))
1025 		return -EINVAL;
1026 
1027 	spad_count = ntb->spad_count;
1028 
1029 	ctrl_size = sizeof(struct epf_ntb_ctrl);
1030 	spad_size = spad_count * 4;
1031 
1032 	msix_capable = epc_features->msix_capable;
1033 	if (msix_capable) {
1034 		msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count;
1035 		ctrl_size = ALIGN(ctrl_size, 8);
1036 		ntb_epc->msix_table_offset = ctrl_size;
1037 		ntb_epc->msix_bar = barno;
1038 		/* Align to QWORD or 8 Bytes */
1039 		pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8);
1040 		ctrl_size = ctrl_size + msix_table_size + pba_size;
1041 	}
1042 
1043 	if (!align) {
1044 		ctrl_size = roundup_pow_of_two(ctrl_size);
1045 		spad_size = roundup_pow_of_two(spad_size);
1046 	} else {
1047 		ctrl_size = ALIGN(ctrl_size, align);
1048 		spad_size = ALIGN(spad_size, align);
1049 	}
1050 
1051 	if (peer_size) {
1052 		if (peer_size < spad_size)
1053 			spad_count = peer_size / 4;
1054 		spad_size = peer_size;
1055 	}
1056 
1057 	/*
1058 	 * In order to make sure SPAD offset is aligned to its size,
1059 	 * expand control region size to the size of SPAD if SPAD size
1060 	 * is greater than control region size.
1061 	 */
1062 	if (spad_size > ctrl_size)
1063 		ctrl_size = spad_size;
1064 
1065 	if (!size)
1066 		size = ctrl_size + spad_size;
1067 	else if (size < ctrl_size + spad_size)
1068 		return -EINVAL;
1069 
1070 	base = pci_epf_alloc_space(epf, size, barno, align, type);
1071 	if (!base) {
1072 		dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n",
1073 			pci_epc_interface_string(type));
1074 		return -ENOMEM;
1075 	}
1076 
1077 	ntb_epc->reg = base;
1078 
1079 	ctrl = ntb_epc->reg;
1080 	ctrl->spad_offset = ctrl_size;
1081 	ctrl->spad_count = spad_count;
1082 	ctrl->num_mws = ntb->num_mws;
1083 	ctrl->db_entry_size = align ? align : 4;
1084 	ntb_epc->spad_size = spad_size;
1085 
1086 	return 0;
1087 }
1088 
1089 /**
1090  * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config +
1091  *   scratchpad region for each of PRIMARY and SECONDARY interface
1092  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1093  *
1094  * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for
1095  * config + scratchpad region for a specific interface
1096  */
1097 static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb)
1098 {
1099 	enum pci_epc_interface_type type;
1100 	struct device *dev;
1101 	int ret;
1102 
1103 	dev = &ntb->epf->dev;
1104 
1105 	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1106 		ret = epf_ntb_config_spad_bar_alloc(ntb, type);
1107 		if (ret) {
1108 			dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n",
1109 				pci_epc_interface_string(type));
1110 			return ret;
1111 		}
1112 	}
1113 
1114 	return 0;
1115 }
1116 
1117 /**
1118  * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address
1119  *   space
1120  * @ntb_epc: EPC associated with one of the HOST which holds peers outbound
1121  *   address regions
1122  *
1123  * +-----------------+    +---->+----------------+-----------+-----------------+
1124  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1125  * +-----------------+    |     +----------------+           +-----------------+
1126  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1127  * +-----------------+----+     +----------------+         | |                 |
1128  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1129  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1130  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1131  * +-----------------+    |     |----------------+     | |   |                 |
1132  * |       BAR4      |    |     |                |     | |   +-----------------+
1133  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1134  * |       BAR5      |    |     |                |   | |     +-----------------+
1135  * +-----------------+    +---->-----------------+   | |     |                 |
1136  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1137  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1138  *                              +----------------+   |       +-----------------+
1139  *                      (A)      EP CONTROLLER 2     |       |                 |
1140  *                                 (OB SPACE)        |       |                 |
1141  *                                                   +------->      MW1        |
1142  *                                                           |                 |
1143  *                                                           |                 |
1144  *                                                   (B)     +-----------------+
1145  *                                                           |                 |
1146  *                                                           |                 |
1147  *                                                           |                 |
1148  *                                                           |                 |
1149  *                                                           |                 |
1150  *                                                           +-----------------+
1151  *                                                           PCI Address Space
1152  *                                                           (Managed by HOST2)
1153  *
1154  * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram.
1155  * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3,
1156  * MW4).
1157  */
1158 static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc)
1159 {
1160 	struct pci_epf_bar *epf_bar;
1161 	void __iomem *mw_addr;
1162 	phys_addr_t phys_addr;
1163 	enum epf_ntb_bar bar;
1164 	enum pci_barno barno;
1165 	struct pci_epc *epc;
1166 	size_t size;
1167 
1168 	epc = ntb_epc->epc;
1169 
1170 	for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
1171 		barno = ntb_epc->epf_ntb_bar[bar];
1172 		mw_addr = ntb_epc->mw_addr[barno];
1173 		epf_bar = &ntb_epc->epf_bar[barno];
1174 		phys_addr = epf_bar->phys_addr;
1175 		size = epf_bar->size;
1176 		if (mw_addr) {
1177 			pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size);
1178 			ntb_epc->mw_addr[barno] = NULL;
1179 		}
1180 	}
1181 }
1182 
1183 /**
1184  * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR
1185  * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
1186  *   address
1187  *
1188  * +-----------------+    +---->+----------------+-----------+-----------------+
1189  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1190  * +-----------------+    |     +----------------+           +-----------------+
1191  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1192  * +-----------------+----+     +----------------+         | |                 |
1193  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1194  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1195  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1196  * +-----------------+    |     |----------------+     | |   |                 |
1197  * |       BAR4      |    |     |                |     | |   +-----------------+
1198  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1199  * |       BAR5      |    |     |                |   | |     +-----------------+
1200  * +-----------------+    +---->-----------------+   | |     |                 |
1201  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1202  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1203  *                              +----------------+   |       +-----------------+
1204  *                      (A)      EP CONTROLLER 2     |       |                 |
1205  *                                 (OB SPACE)        |       |                 |
1206  *                                                   +------->      MW1        |
1207  *                                                           |                 |
1208  *                                                           |                 |
1209  *                                                   (B)     +-----------------+
1210  *                                                           |                 |
1211  *                                                           |                 |
1212  *                                                           |                 |
1213  *                                                           |                 |
1214  *                                                           |                 |
1215  *                                                           +-----------------+
1216  *                                                           PCI Address Space
1217  *                                                           (Managed by HOST2)
1218  *
1219  * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above
1220  * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2
1221  * BAR, MW3 BAR and MW4 BAR).
1222  */
1223 static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc)
1224 {
1225 	struct pci_epf_bar *epf_bar;
1226 	enum epf_ntb_bar bar;
1227 	enum pci_barno barno;
1228 	u8 func_no, vfunc_no;
1229 	struct pci_epc *epc;
1230 
1231 	epc = ntb_epc->epc;
1232 
1233 	func_no = ntb_epc->func_no;
1234 	vfunc_no = ntb_epc->vfunc_no;
1235 
1236 	for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
1237 		barno = ntb_epc->epf_ntb_bar[bar];
1238 		epf_bar = &ntb_epc->epf_bar[barno];
1239 		pci_epc_clear_bar(epc, func_no, vfunc_no, epf_bar);
1240 	}
1241 }
1242 
1243 /**
1244  * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory
1245  *   allocated in peers outbound address space
1246  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1247  * @type: PRIMARY interface or SECONDARY interface
1248  *
1249  * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and
1250  * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory.
1251  */
1252 static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb,
1253 				      enum pci_epc_interface_type type)
1254 {
1255 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
1256 
1257 	ntb_epc = ntb->epc[type];
1258 	peer_ntb_epc = ntb->epc[!type];
1259 
1260 	epf_ntb_db_mw_bar_clear(ntb_epc);
1261 	epf_ntb_free_peer_mem(peer_ntb_epc);
1262 }
1263 
1264 /**
1265  * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
1266  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1267  * @type: PRIMARY interface or SECONDARY interface
1268  *
1269  * Configure MSI/MSI-X capability for each interface with number of
1270  * interrupts equal to "db_count" configfs entry.
1271  */
1272 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,
1273 				       enum pci_epc_interface_type type)
1274 {
1275 	const struct pci_epc_features *epc_features;
1276 	bool msix_capable, msi_capable;
1277 	struct epf_ntb_epc *ntb_epc;
1278 	u8 func_no, vfunc_no;
1279 	struct pci_epc *epc;
1280 	struct device *dev;
1281 	u32 db_count;
1282 	int ret;
1283 
1284 	ntb_epc = ntb->epc[type];
1285 	dev = &ntb->epf->dev;
1286 
1287 	epc_features = ntb_epc->epc_features;
1288 	msix_capable = epc_features->msix_capable;
1289 	msi_capable = epc_features->msi_capable;
1290 
1291 	if (!(msix_capable || msi_capable)) {
1292 		dev_err(dev, "MSI or MSI-X is required for doorbell\n");
1293 		return -EINVAL;
1294 	}
1295 
1296 	func_no = ntb_epc->func_no;
1297 	vfunc_no = ntb_epc->vfunc_no;
1298 
1299 	db_count = ntb->db_count;
1300 	if (db_count > MAX_DB_COUNT) {
1301 		dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
1302 		return -EINVAL;
1303 	}
1304 
1305 	ntb->db_count = db_count;
1306 	epc = ntb_epc->epc;
1307 
1308 	if (msi_capable) {
1309 		ret = pci_epc_set_msi(epc, func_no, vfunc_no, db_count);
1310 		if (ret) {
1311 			dev_err(dev, "%s intf: MSI configuration failed\n",
1312 				pci_epc_interface_string(type));
1313 			return ret;
1314 		}
1315 	}
1316 
1317 	if (msix_capable) {
1318 		ret = pci_epc_set_msix(epc, func_no, vfunc_no, db_count,
1319 				       ntb_epc->msix_bar,
1320 				       ntb_epc->msix_table_offset);
1321 		if (ret) {
1322 			dev_err(dev, "MSI configuration failed\n");
1323 			return ret;
1324 		}
1325 	}
1326 
1327 	return 0;
1328 }
1329 
1330 /**
1331  * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space
1332  * @dev: The PCI device.
1333  * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound
1334  *   address
1335  * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be
1336  *   BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4)
1337  * @peer_ntb_epc: EPC associated with HOST whose outbound address space is
1338  *   used by @ntb_epc
1339  * @size: Size of the address region that has to be allocated in peers OB SPACE
1340  *
1341  *
1342  * +-----------------+    +---->+----------------+-----------+-----------------+
1343  * |       BAR0      |    |     |   Doorbell 1   +-----------> MSI|X ADDRESS 1 |
1344  * +-----------------+    |     +----------------+           +-----------------+
1345  * |       BAR1      |    |     |   Doorbell 2   +---------+ |                 |
1346  * +-----------------+----+     +----------------+         | |                 |
1347  * |       BAR2      |          |   Doorbell 3   +-------+ | +-----------------+
1348  * +-----------------+----+     +----------------+       | +-> MSI|X ADDRESS 2 |
1349  * |       BAR3      |    |     |   Doorbell 4   +-----+ |   +-----------------+
1350  * +-----------------+    |     |----------------+     | |   |                 |
1351  * |       BAR4      |    |     |                |     | |   +-----------------+
1352  * +-----------------+    |     |      MW1       +---+ | +-->+ MSI|X ADDRESS 3||
1353  * |       BAR5      |    |     |                |   | |     +-----------------+
1354  * +-----------------+    +---->-----------------+   | |     |                 |
1355  *   EP CONTROLLER 1            |                |   | |     +-----------------+
1356  *                              |                |   | +---->+ MSI|X ADDRESS 4 |
1357  *                              +----------------+   |       +-----------------+
1358  *                      (A)      EP CONTROLLER 2     |       |                 |
1359  *                                 (OB SPACE)        |       |                 |
1360  *                                                   +------->      MW1        |
1361  *                                                           |                 |
1362  *                                                           |                 |
1363  *                                                   (B)     +-----------------+
1364  *                                                           |                 |
1365  *                                                           |                 |
1366  *                                                           |                 |
1367  *                                                           |                 |
1368  *                                                           |                 |
1369  *                                                           +-----------------+
1370  *                                                           PCI Address Space
1371  *                                                           (Managed by HOST2)
1372  *
1373  * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate
1374  * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4).
1375  */
1376 static int epf_ntb_alloc_peer_mem(struct device *dev,
1377 				  struct epf_ntb_epc *ntb_epc,
1378 				  enum epf_ntb_bar bar,
1379 				  struct epf_ntb_epc *peer_ntb_epc,
1380 				  size_t size)
1381 {
1382 	const struct pci_epc_features *epc_features;
1383 	struct pci_epf_bar *epf_bar;
1384 	struct pci_epc *peer_epc;
1385 	phys_addr_t phys_addr;
1386 	void __iomem *mw_addr;
1387 	enum pci_barno barno;
1388 	size_t align;
1389 
1390 	epc_features = ntb_epc->epc_features;
1391 	align = epc_features->align;
1392 
1393 	if (size < 128)
1394 		size = 128;
1395 
1396 	if (align)
1397 		size = ALIGN(size, align);
1398 	else
1399 		size = roundup_pow_of_two(size);
1400 
1401 	peer_epc = peer_ntb_epc->epc;
1402 	mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size);
1403 	if (!mw_addr) {
1404 		dev_err(dev, "%s intf: Failed to allocate OB address\n",
1405 			pci_epc_interface_string(peer_ntb_epc->type));
1406 		return -ENOMEM;
1407 	}
1408 
1409 	barno = ntb_epc->epf_ntb_bar[bar];
1410 	epf_bar = &ntb_epc->epf_bar[barno];
1411 	ntb_epc->mw_addr[barno] = mw_addr;
1412 
1413 	epf_bar->phys_addr = phys_addr;
1414 	epf_bar->size = size;
1415 	epf_bar->barno = barno;
1416 	epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
1417 
1418 	return 0;
1419 }
1420 
1421 /**
1422  * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs
1423  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1424  * @type: PRIMARY interface or SECONDARY interface
1425  *
1426  * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates
1427  * memory in OB address space of HOST2 and configures BAR of HOST1
1428  */
1429 static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb,
1430 				  enum pci_epc_interface_type type)
1431 {
1432 	const struct pci_epc_features *epc_features;
1433 	struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
1434 	struct pci_epf_bar *epf_bar;
1435 	struct epf_ntb_ctrl *ctrl;
1436 	u32 num_mws, db_count;
1437 	enum epf_ntb_bar bar;
1438 	enum pci_barno barno;
1439 	u8 func_no, vfunc_no;
1440 	struct pci_epc *epc;
1441 	struct device *dev;
1442 	size_t align;
1443 	int ret, i;
1444 	u64 size;
1445 
1446 	ntb_epc = ntb->epc[type];
1447 	peer_ntb_epc = ntb->epc[!type];
1448 
1449 	dev = &ntb->epf->dev;
1450 	epc_features = ntb_epc->epc_features;
1451 	align = epc_features->align;
1452 	func_no = ntb_epc->func_no;
1453 	vfunc_no = ntb_epc->vfunc_no;
1454 	epc = ntb_epc->epc;
1455 	num_mws = ntb->num_mws;
1456 	db_count = ntb->db_count;
1457 
1458 	for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) {
1459 		if (bar == BAR_DB_MW1) {
1460 			align = align ? align : 4;
1461 			size = db_count * align;
1462 			size = ALIGN(size, ntb->mws_size[i]);
1463 			ctrl = ntb_epc->reg;
1464 			ctrl->mw1_offset = size;
1465 			size += ntb->mws_size[i];
1466 		} else {
1467 			size = ntb->mws_size[i];
1468 		}
1469 
1470 		ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar,
1471 					     peer_ntb_epc, size);
1472 		if (ret) {
1473 			dev_err(dev, "%s intf: DoorBell mem alloc failed\n",
1474 				pci_epc_interface_string(type));
1475 			goto err_alloc_peer_mem;
1476 		}
1477 
1478 		barno = ntb_epc->epf_ntb_bar[bar];
1479 		epf_bar = &ntb_epc->epf_bar[barno];
1480 
1481 		ret = pci_epc_set_bar(epc, func_no, vfunc_no, epf_bar);
1482 		if (ret) {
1483 			dev_err(dev, "%s intf: DoorBell BAR set failed\n",
1484 				pci_epc_interface_string(type));
1485 			goto err_alloc_peer_mem;
1486 		}
1487 	}
1488 
1489 	return 0;
1490 
1491 err_alloc_peer_mem:
1492 	epf_ntb_db_mw_bar_cleanup(ntb, type);
1493 
1494 	return ret;
1495 }
1496 
1497 /**
1498  * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface
1499  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1500  * @type: PRIMARY interface or SECONDARY interface
1501  *
1502  * Unbind NTB function device from EPC and relinquish reference to pci_epc
1503  * for each of the interface.
1504  */
1505 static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb,
1506 					  enum pci_epc_interface_type type)
1507 {
1508 	struct epf_ntb_epc *ntb_epc;
1509 	struct pci_epc *epc;
1510 	struct pci_epf *epf;
1511 
1512 	if (type < 0)
1513 		return;
1514 
1515 	epf = ntb->epf;
1516 	ntb_epc = ntb->epc[type];
1517 	if (!ntb_epc)
1518 		return;
1519 	epc = ntb_epc->epc;
1520 	pci_epc_remove_epf(epc, epf, type);
1521 	pci_epc_put(epc);
1522 }
1523 
1524 /**
1525  * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
1526  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1527  *
1528  * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
1529  */
1530 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
1531 {
1532 	enum pci_epc_interface_type type;
1533 
1534 	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
1535 		epf_ntb_epc_destroy_interface(ntb, type);
1536 }
1537 
1538 /**
1539  * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface
1540  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1541  * @epc: struct pci_epc to which a particular NTB interface should be associated
1542  * @type: PRIMARY interface or SECONDARY interface
1543  *
1544  * Allocate memory for NTB EPC interface and initialize it.
1545  */
1546 static int epf_ntb_epc_create_interface(struct epf_ntb *ntb,
1547 					struct pci_epc *epc,
1548 					enum pci_epc_interface_type type)
1549 {
1550 	const struct pci_epc_features *epc_features;
1551 	struct pci_epf_bar *epf_bar;
1552 	struct epf_ntb_epc *ntb_epc;
1553 	u8 func_no, vfunc_no;
1554 	struct pci_epf *epf;
1555 	struct device *dev;
1556 
1557 	dev = &ntb->epf->dev;
1558 
1559 	ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL);
1560 	if (!ntb_epc)
1561 		return -ENOMEM;
1562 
1563 	epf = ntb->epf;
1564 	vfunc_no = epf->vfunc_no;
1565 	if (type == PRIMARY_INTERFACE) {
1566 		func_no = epf->func_no;
1567 		epf_bar = epf->bar;
1568 	} else {
1569 		func_no = epf->sec_epc_func_no;
1570 		epf_bar = epf->sec_epc_bar;
1571 	}
1572 
1573 	ntb_epc->linkup = false;
1574 	ntb_epc->epc = epc;
1575 	ntb_epc->func_no = func_no;
1576 	ntb_epc->vfunc_no = vfunc_no;
1577 	ntb_epc->type = type;
1578 	ntb_epc->epf_bar = epf_bar;
1579 	ntb_epc->epf_ntb = ntb;
1580 
1581 	epc_features = pci_epc_get_features(epc, func_no, vfunc_no);
1582 	if (!epc_features)
1583 		return -EINVAL;
1584 	ntb_epc->epc_features = epc_features;
1585 
1586 	ntb->epc[type] = ntb_epc;
1587 
1588 	return 0;
1589 }
1590 
1591 /**
1592  * epf_ntb_epc_create() - Create and initialize NTB EPC interface
1593  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1594  *
1595  * Get a reference to EPC device and bind NTB function device to that EPC
1596  * for each of the interface. It is also a wrapper to
1597  * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface
1598  * and initialize it
1599  */
1600 static int epf_ntb_epc_create(struct epf_ntb *ntb)
1601 {
1602 	struct pci_epf *epf;
1603 	struct device *dev;
1604 	int ret;
1605 
1606 	epf = ntb->epf;
1607 	dev = &epf->dev;
1608 
1609 	ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE);
1610 	if (ret) {
1611 		dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n");
1612 		return ret;
1613 	}
1614 
1615 	ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc,
1616 					   SECONDARY_INTERFACE);
1617 	if (ret) {
1618 		dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n");
1619 		goto err_epc_create;
1620 	}
1621 
1622 	return 0;
1623 
1624 err_epc_create:
1625 	epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE);
1626 
1627 	return ret;
1628 }
1629 
1630 /**
1631  * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of
1632  *   the NTB constructs (scratchpad region, doorbell, memorywindow)
1633  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1634  * @type: PRIMARY interface or SECONDARY interface
1635  *
1636  * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD,
1637  * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4.
1638  */
1639 static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb,
1640 					  enum pci_epc_interface_type type)
1641 {
1642 	const struct pci_epc_features *epc_features;
1643 	struct epf_ntb_epc *ntb_epc;
1644 	enum pci_barno barno;
1645 	enum epf_ntb_bar bar;
1646 	struct device *dev;
1647 	u32 num_mws;
1648 	int i;
1649 
1650 	barno = BAR_0;
1651 	ntb_epc = ntb->epc[type];
1652 	num_mws = ntb->num_mws;
1653 	dev = &ntb->epf->dev;
1654 	epc_features = ntb_epc->epc_features;
1655 
1656 	/* These are required BARs which are mandatory for NTB functionality */
1657 	for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) {
1658 		barno = pci_epc_get_next_free_bar(epc_features, barno);
1659 		if (barno < 0) {
1660 			dev_err(dev, "%s intf: Fail to get NTB function BAR\n",
1661 				pci_epc_interface_string(type));
1662 			return barno;
1663 		}
1664 		ntb_epc->epf_ntb_bar[bar] = barno;
1665 	}
1666 
1667 	/* These are optional BARs which don't impact NTB functionality */
1668 	for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) {
1669 		barno = pci_epc_get_next_free_bar(epc_features, barno);
1670 		if (barno < 0) {
1671 			ntb->num_mws = i;
1672 			dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
1673 		}
1674 		ntb_epc->epf_ntb_bar[bar] = barno;
1675 	}
1676 
1677 	return 0;
1678 }
1679 
1680 /**
1681  * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
1682  * constructs (scratchpad region, doorbell, memorywindow)
1683  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1684  *
1685  * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs
1686  * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2,
1687  * BAR_MW3 and BAR_MW4 for all the interfaces.
1688  */
1689 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
1690 {
1691 	enum pci_epc_interface_type type;
1692 	struct device *dev;
1693 	int ret;
1694 
1695 	dev = &ntb->epf->dev;
1696 	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1697 		ret = epf_ntb_init_epc_bar_interface(ntb, type);
1698 		if (ret) {
1699 			dev_err(dev, "Fail to init EPC bar for %s interface\n",
1700 				pci_epc_interface_string(type));
1701 			return ret;
1702 		}
1703 	}
1704 
1705 	return 0;
1706 }
1707 
1708 /**
1709  * epf_ntb_epc_init_interface() - Initialize NTB interface
1710  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1711  * @type: PRIMARY interface or SECONDARY interface
1712  *
1713  * Wrapper to initialize a particular EPC interface and start the workqueue
1714  * to check for commands from host. This function will write to the
1715  * EP controller HW for configuring it.
1716  */
1717 static int epf_ntb_epc_init_interface(struct epf_ntb *ntb,
1718 				      enum pci_epc_interface_type type)
1719 {
1720 	struct epf_ntb_epc *ntb_epc;
1721 	u8 func_no, vfunc_no;
1722 	struct pci_epc *epc;
1723 	struct pci_epf *epf;
1724 	struct device *dev;
1725 	int ret;
1726 
1727 	ntb_epc = ntb->epc[type];
1728 	epf = ntb->epf;
1729 	dev = &epf->dev;
1730 	epc = ntb_epc->epc;
1731 	func_no = ntb_epc->func_no;
1732 	vfunc_no = ntb_epc->vfunc_no;
1733 
1734 	ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]);
1735 	if (ret) {
1736 		dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n",
1737 			pci_epc_interface_string(type));
1738 		return ret;
1739 	}
1740 
1741 	ret = epf_ntb_peer_spad_bar_set(ntb, type);
1742 	if (ret) {
1743 		dev_err(dev, "%s intf: Peer SPAD BAR init failed\n",
1744 			pci_epc_interface_string(type));
1745 		goto err_peer_spad_bar_init;
1746 	}
1747 
1748 	ret = epf_ntb_configure_interrupt(ntb, type);
1749 	if (ret) {
1750 		dev_err(dev, "%s intf: Interrupt configuration failed\n",
1751 			pci_epc_interface_string(type));
1752 		goto err_peer_spad_bar_init;
1753 	}
1754 
1755 	ret = epf_ntb_db_mw_bar_init(ntb, type);
1756 	if (ret) {
1757 		dev_err(dev, "%s intf: DB/MW BAR init failed\n",
1758 			pci_epc_interface_string(type));
1759 		goto err_db_mw_bar_init;
1760 	}
1761 
1762 	if (vfunc_no <= 1) {
1763 		ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
1764 		if (ret) {
1765 			dev_err(dev, "%s intf: Configuration header write failed\n",
1766 				pci_epc_interface_string(type));
1767 			goto err_write_header;
1768 		}
1769 	}
1770 
1771 	INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler);
1772 	queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work);
1773 
1774 	return 0;
1775 
1776 err_write_header:
1777 	epf_ntb_db_mw_bar_cleanup(ntb, type);
1778 
1779 err_db_mw_bar_init:
1780 	epf_ntb_peer_spad_bar_clear(ntb->epc[type]);
1781 
1782 err_peer_spad_bar_init:
1783 	epf_ntb_config_sspad_bar_clear(ntb->epc[type]);
1784 
1785 	return ret;
1786 }
1787 
1788 /**
1789  * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface
1790  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1791  * @type: PRIMARY interface or SECONDARY interface
1792  *
1793  * Wrapper to cleanup a particular NTB interface.
1794  */
1795 static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb,
1796 					  enum pci_epc_interface_type type)
1797 {
1798 	struct epf_ntb_epc *ntb_epc;
1799 
1800 	if (type < 0)
1801 		return;
1802 
1803 	ntb_epc = ntb->epc[type];
1804 	cancel_delayed_work(&ntb_epc->cmd_handler);
1805 	epf_ntb_db_mw_bar_cleanup(ntb, type);
1806 	epf_ntb_peer_spad_bar_clear(ntb_epc);
1807 	epf_ntb_config_sspad_bar_clear(ntb_epc);
1808 }
1809 
1810 /**
1811  * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
1812  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1813  *
1814  * Wrapper to cleanup all NTB interfaces.
1815  */
1816 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
1817 {
1818 	enum pci_epc_interface_type type;
1819 
1820 	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
1821 		epf_ntb_epc_cleanup_interface(ntb, type);
1822 }
1823 
1824 /**
1825  * epf_ntb_epc_init() - Initialize all NTB interfaces
1826  * @ntb: NTB device that facilitates communication between HOST1 and HOST2
1827  *
1828  * Wrapper to initialize all NTB interface and start the workqueue
1829  * to check for commands from host.
1830  */
1831 static int epf_ntb_epc_init(struct epf_ntb *ntb)
1832 {
1833 	enum pci_epc_interface_type type;
1834 	struct device *dev;
1835 	int ret;
1836 
1837 	dev = &ntb->epf->dev;
1838 
1839 	for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1840 		ret = epf_ntb_epc_init_interface(ntb, type);
1841 		if (ret) {
1842 			dev_err(dev, "%s intf: Failed to initialize\n",
1843 				pci_epc_interface_string(type));
1844 			goto err_init_type;
1845 		}
1846 	}
1847 
1848 	return 0;
1849 
1850 err_init_type:
1851 	epf_ntb_epc_cleanup_interface(ntb, type - 1);
1852 
1853 	return ret;
1854 }
1855 
1856 /**
1857  * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1858  * @epf: NTB endpoint function device
1859  *
1860  * Initialize both the endpoint controllers associated with NTB function device.
1861  * Invoked when a primary interface or secondary interface is bound to EPC
1862  * device. This function will succeed only when EPC is bound to both the
1863  * interfaces.
1864  */
1865 static int epf_ntb_bind(struct pci_epf *epf)
1866 {
1867 	struct epf_ntb *ntb = epf_get_drvdata(epf);
1868 	struct device *dev = &epf->dev;
1869 	int ret;
1870 
1871 	if (!epf->epc) {
1872 		dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1873 		return 0;
1874 	}
1875 
1876 	if (!epf->sec_epc) {
1877 		dev_dbg(dev, "SECONDARY EPC interface not yet bound\n");
1878 		return 0;
1879 	}
1880 
1881 	ret = epf_ntb_epc_create(ntb);
1882 	if (ret) {
1883 		dev_err(dev, "Failed to create NTB EPC\n");
1884 		return ret;
1885 	}
1886 
1887 	ret = epf_ntb_init_epc_bar(ntb);
1888 	if (ret) {
1889 		dev_err(dev, "Failed to create NTB EPC\n");
1890 		goto err_bar_init;
1891 	}
1892 
1893 	ret = epf_ntb_config_spad_bar_alloc_interface(ntb);
1894 	if (ret) {
1895 		dev_err(dev, "Failed to allocate BAR memory\n");
1896 		goto err_bar_alloc;
1897 	}
1898 
1899 	ret = epf_ntb_epc_init(ntb);
1900 	if (ret) {
1901 		dev_err(dev, "Failed to initialize EPC\n");
1902 		goto err_bar_alloc;
1903 	}
1904 
1905 	epf_set_drvdata(epf, ntb);
1906 
1907 	return 0;
1908 
1909 err_bar_alloc:
1910 	epf_ntb_config_spad_bar_free(ntb);
1911 
1912 err_bar_init:
1913 	epf_ntb_epc_destroy(ntb);
1914 
1915 	return ret;
1916 }
1917 
1918 /**
1919  * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1920  * @epf: NTB endpoint function device
1921  *
1922  * Cleanup the initialization from epf_ntb_bind()
1923  */
1924 static void epf_ntb_unbind(struct pci_epf *epf)
1925 {
1926 	struct epf_ntb *ntb = epf_get_drvdata(epf);
1927 
1928 	epf_ntb_epc_cleanup(ntb);
1929 	epf_ntb_config_spad_bar_free(ntb);
1930 	epf_ntb_epc_destroy(ntb);
1931 }
1932 
1933 #define EPF_NTB_R(_name)						\
1934 static ssize_t epf_ntb_##_name##_show(struct config_item *item,		\
1935 				      char *page)			\
1936 {									\
1937 	struct config_group *group = to_config_group(item);		\
1938 	struct epf_ntb *ntb = to_epf_ntb(group);			\
1939 									\
1940 	return sysfs_emit(page, "%d\n", ntb->_name);			\
1941 }
1942 
1943 #define EPF_NTB_W(_name)						\
1944 static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
1945 				       const char *page, size_t len)	\
1946 {									\
1947 	struct config_group *group = to_config_group(item);		\
1948 	struct epf_ntb *ntb = to_epf_ntb(group);			\
1949 	u32 val;							\
1950 									\
1951 	if (kstrtou32(page, 0, &val) < 0)				\
1952 		return -EINVAL;						\
1953 									\
1954 	ntb->_name = val;						\
1955 									\
1956 	return len;							\
1957 }
1958 
1959 #define EPF_NTB_MW_R(_name)						\
1960 static ssize_t epf_ntb_##_name##_show(struct config_item *item,		\
1961 				      char *page)			\
1962 {									\
1963 	struct config_group *group = to_config_group(item);		\
1964 	struct epf_ntb *ntb = to_epf_ntb(group);			\
1965 	int win_no;							\
1966 									\
1967 	sscanf(#_name, "mw%d", &win_no);				\
1968 									\
1969 	return sysfs_emit(page, "%lld\n", ntb->mws_size[win_no - 1]);	\
1970 }
1971 
1972 #define EPF_NTB_MW_W(_name)						\
1973 static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
1974 				       const char *page, size_t len)	\
1975 {									\
1976 	struct config_group *group = to_config_group(item);		\
1977 	struct epf_ntb *ntb = to_epf_ntb(group);			\
1978 	struct device *dev = &ntb->epf->dev;				\
1979 	int win_no;							\
1980 	u64 val;							\
1981 									\
1982 	if (kstrtou64(page, 0, &val) < 0)				\
1983 		return -EINVAL;						\
1984 									\
1985 	if (sscanf(#_name, "mw%d", &win_no) != 1)			\
1986 		return -EINVAL;						\
1987 									\
1988 	if (ntb->num_mws < win_no) {					\
1989 		dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
1990 		return -EINVAL;						\
1991 	}								\
1992 									\
1993 	ntb->mws_size[win_no - 1] = val;				\
1994 									\
1995 	return len;							\
1996 }
1997 
1998 static ssize_t epf_ntb_num_mws_store(struct config_item *item,
1999 				     const char *page, size_t len)
2000 {
2001 	struct config_group *group = to_config_group(item);
2002 	struct epf_ntb *ntb = to_epf_ntb(group);
2003 	u32 val;
2004 
2005 	if (kstrtou32(page, 0, &val) < 0)
2006 		return -EINVAL;
2007 
2008 	if (val > MAX_MW)
2009 		return -EINVAL;
2010 
2011 	ntb->num_mws = val;
2012 
2013 	return len;
2014 }
2015 
2016 EPF_NTB_R(spad_count)
2017 EPF_NTB_W(spad_count)
2018 EPF_NTB_R(db_count)
2019 EPF_NTB_W(db_count)
2020 EPF_NTB_R(num_mws)
2021 EPF_NTB_MW_R(mw1)
2022 EPF_NTB_MW_W(mw1)
2023 EPF_NTB_MW_R(mw2)
2024 EPF_NTB_MW_W(mw2)
2025 EPF_NTB_MW_R(mw3)
2026 EPF_NTB_MW_W(mw3)
2027 EPF_NTB_MW_R(mw4)
2028 EPF_NTB_MW_W(mw4)
2029 
2030 CONFIGFS_ATTR(epf_ntb_, spad_count);
2031 CONFIGFS_ATTR(epf_ntb_, db_count);
2032 CONFIGFS_ATTR(epf_ntb_, num_mws);
2033 CONFIGFS_ATTR(epf_ntb_, mw1);
2034 CONFIGFS_ATTR(epf_ntb_, mw2);
2035 CONFIGFS_ATTR(epf_ntb_, mw3);
2036 CONFIGFS_ATTR(epf_ntb_, mw4);
2037 
2038 static struct configfs_attribute *epf_ntb_attrs[] = {
2039 	&epf_ntb_attr_spad_count,
2040 	&epf_ntb_attr_db_count,
2041 	&epf_ntb_attr_num_mws,
2042 	&epf_ntb_attr_mw1,
2043 	&epf_ntb_attr_mw2,
2044 	&epf_ntb_attr_mw3,
2045 	&epf_ntb_attr_mw4,
2046 	NULL,
2047 };
2048 
2049 static const struct config_item_type ntb_group_type = {
2050 	.ct_attrs	= epf_ntb_attrs,
2051 	.ct_owner	= THIS_MODULE,
2052 };
2053 
2054 /**
2055  * epf_ntb_add_cfs() - Add configfs directory specific to NTB
2056  * @epf: NTB endpoint function device
2057  * @group: A pointer to the config_group structure referencing a group of
2058  *	   config_items of a specific type that belong to a specific sub-system.
2059  *
2060  * Add configfs directory specific to NTB. This directory will hold
2061  * NTB specific properties like db_count, spad_count, num_mws etc.,
2062  */
2063 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
2064 					    struct config_group *group)
2065 {
2066 	struct epf_ntb *ntb = epf_get_drvdata(epf);
2067 	struct config_group *ntb_group = &ntb->group;
2068 	struct device *dev = &epf->dev;
2069 
2070 	config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
2071 
2072 	return ntb_group;
2073 }
2074 
2075 /**
2076  * epf_ntb_probe() - Probe NTB function driver
2077  * @epf: NTB endpoint function device
2078  *
2079  * Probe NTB function driver when endpoint function bus detects a NTB
2080  * endpoint function.
2081  */
2082 static int epf_ntb_probe(struct pci_epf *epf)
2083 {
2084 	struct epf_ntb *ntb;
2085 	struct device *dev;
2086 
2087 	dev = &epf->dev;
2088 
2089 	ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
2090 	if (!ntb)
2091 		return -ENOMEM;
2092 
2093 	epf->header = &epf_ntb_header;
2094 	ntb->epf = epf;
2095 	epf_set_drvdata(epf, ntb);
2096 
2097 	return 0;
2098 }
2099 
2100 static struct pci_epf_ops epf_ntb_ops = {
2101 	.bind	= epf_ntb_bind,
2102 	.unbind	= epf_ntb_unbind,
2103 	.add_cfs = epf_ntb_add_cfs,
2104 };
2105 
2106 static const struct pci_epf_device_id epf_ntb_ids[] = {
2107 	{
2108 		.name = "pci_epf_ntb",
2109 	},
2110 	{},
2111 };
2112 
2113 static struct pci_epf_driver epf_ntb_driver = {
2114 	.driver.name	= "pci_epf_ntb",
2115 	.probe		= epf_ntb_probe,
2116 	.id_table	= epf_ntb_ids,
2117 	.ops		= &epf_ntb_ops,
2118 	.owner		= THIS_MODULE,
2119 };
2120 
2121 static int __init epf_ntb_init(void)
2122 {
2123 	int ret;
2124 
2125 	kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
2126 					    WQ_HIGHPRI, 0);
2127 	ret = pci_epf_register_driver(&epf_ntb_driver);
2128 	if (ret) {
2129 		destroy_workqueue(kpcintb_workqueue);
2130 		pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
2131 		return ret;
2132 	}
2133 
2134 	return 0;
2135 }
2136 module_init(epf_ntb_init);
2137 
2138 static void __exit epf_ntb_exit(void)
2139 {
2140 	pci_epf_unregister_driver(&epf_ntb_driver);
2141 	destroy_workqueue(kpcintb_workqueue);
2142 }
2143 module_exit(epf_ntb_exit);
2144 
2145 MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
2146 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
2147 MODULE_LICENSE("GPL v2");
2148