1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
3 #include <linux/memregion.h>
4 #include <linux/genalloc.h>
5 #include <linux/device.h>
6 #include <linux/module.h>
7 #include <linux/memory.h>
8 #include <linux/slab.h>
9 #include <linux/uuid.h>
10 #include <linux/sort.h>
11 #include <linux/idr.h>
12 #include <cxlmem.h>
13 #include <cxl.h>
14 #include "core.h"
15
16 /**
17 * DOC: cxl core region
18 *
19 * CXL Regions represent mapped memory capacity in system physical address
20 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
21 * Memory ranges, Regions represent the active mapped capacity by the HDM
22 * Decoder Capability structures throughout the Host Bridges, Switches, and
23 * Endpoints in the topology.
24 *
25 * Region configuration has ordering constraints. UUID may be set at any time
26 * but is only visible for persistent regions.
27 * 1. Interleave granularity
28 * 2. Interleave size
29 * 3. Decoder targets
30 */
31
32 static struct cxl_region *to_cxl_region(struct device *dev);
33
34 #define __ACCESS_ATTR_RO(_level, _name) { \
35 .attr = { .name = __stringify(_name), .mode = 0444 }, \
36 .show = _name##_access##_level##_show, \
37 }
38
39 #define ACCESS_DEVICE_ATTR_RO(level, name) \
40 struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name)
41
42 #define ACCESS_ATTR_RO(level, attrib) \
43 static ssize_t attrib##_access##level##_show(struct device *dev, \
44 struct device_attribute *attr, \
45 char *buf) \
46 { \
47 struct cxl_region *cxlr = to_cxl_region(dev); \
48 \
49 if (cxlr->coord[level].attrib == 0) \
50 return -ENOENT; \
51 \
52 return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \
53 } \
54 static ACCESS_DEVICE_ATTR_RO(level, attrib)
55
56 ACCESS_ATTR_RO(0, read_bandwidth);
57 ACCESS_ATTR_RO(0, read_latency);
58 ACCESS_ATTR_RO(0, write_bandwidth);
59 ACCESS_ATTR_RO(0, write_latency);
60
61 #define ACCESS_ATTR_DECLARE(level, attrib) \
62 (&dev_attr_access##level##_##attrib.attr)
63
64 static struct attribute *access0_coordinate_attrs[] = {
65 ACCESS_ATTR_DECLARE(0, read_bandwidth),
66 ACCESS_ATTR_DECLARE(0, write_bandwidth),
67 ACCESS_ATTR_DECLARE(0, read_latency),
68 ACCESS_ATTR_DECLARE(0, write_latency),
69 NULL
70 };
71
72 ACCESS_ATTR_RO(1, read_bandwidth);
73 ACCESS_ATTR_RO(1, read_latency);
74 ACCESS_ATTR_RO(1, write_bandwidth);
75 ACCESS_ATTR_RO(1, write_latency);
76
77 static struct attribute *access1_coordinate_attrs[] = {
78 ACCESS_ATTR_DECLARE(1, read_bandwidth),
79 ACCESS_ATTR_DECLARE(1, write_bandwidth),
80 ACCESS_ATTR_DECLARE(1, read_latency),
81 ACCESS_ATTR_DECLARE(1, write_latency),
82 NULL
83 };
84
85 #define ACCESS_VISIBLE(level) \
86 static umode_t cxl_region_access##level##_coordinate_visible( \
87 struct kobject *kobj, struct attribute *a, int n) \
88 { \
89 struct device *dev = kobj_to_dev(kobj); \
90 struct cxl_region *cxlr = to_cxl_region(dev); \
91 \
92 if (a == &dev_attr_access##level##_read_latency.attr && \
93 cxlr->coord[level].read_latency == 0) \
94 return 0; \
95 \
96 if (a == &dev_attr_access##level##_write_latency.attr && \
97 cxlr->coord[level].write_latency == 0) \
98 return 0; \
99 \
100 if (a == &dev_attr_access##level##_read_bandwidth.attr && \
101 cxlr->coord[level].read_bandwidth == 0) \
102 return 0; \
103 \
104 if (a == &dev_attr_access##level##_write_bandwidth.attr && \
105 cxlr->coord[level].write_bandwidth == 0) \
106 return 0; \
107 \
108 return a->mode; \
109 }
110
111 ACCESS_VISIBLE(0);
112 ACCESS_VISIBLE(1);
113
114 static const struct attribute_group cxl_region_access0_coordinate_group = {
115 .name = "access0",
116 .attrs = access0_coordinate_attrs,
117 .is_visible = cxl_region_access0_coordinate_visible,
118 };
119
get_cxl_region_access0_group(void)120 static const struct attribute_group *get_cxl_region_access0_group(void)
121 {
122 return &cxl_region_access0_coordinate_group;
123 }
124
125 static const struct attribute_group cxl_region_access1_coordinate_group = {
126 .name = "access1",
127 .attrs = access1_coordinate_attrs,
128 .is_visible = cxl_region_access1_coordinate_visible,
129 };
130
get_cxl_region_access1_group(void)131 static const struct attribute_group *get_cxl_region_access1_group(void)
132 {
133 return &cxl_region_access1_coordinate_group;
134 }
135
uuid_show(struct device * dev,struct device_attribute * attr,char * buf)136 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
137 char *buf)
138 {
139 struct cxl_region *cxlr = to_cxl_region(dev);
140 struct cxl_region_params *p = &cxlr->params;
141 ssize_t rc;
142
143 rc = down_read_interruptible(&cxl_region_rwsem);
144 if (rc)
145 return rc;
146 if (cxlr->mode != CXL_DECODER_PMEM)
147 rc = sysfs_emit(buf, "\n");
148 else
149 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
150 up_read(&cxl_region_rwsem);
151
152 return rc;
153 }
154
is_dup(struct device * match,void * data)155 static int is_dup(struct device *match, void *data)
156 {
157 struct cxl_region_params *p;
158 struct cxl_region *cxlr;
159 uuid_t *uuid = data;
160
161 if (!is_cxl_region(match))
162 return 0;
163
164 lockdep_assert_held(&cxl_region_rwsem);
165 cxlr = to_cxl_region(match);
166 p = &cxlr->params;
167
168 if (uuid_equal(&p->uuid, uuid)) {
169 dev_dbg(match, "already has uuid: %pUb\n", uuid);
170 return -EBUSY;
171 }
172
173 return 0;
174 }
175
uuid_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)176 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
177 const char *buf, size_t len)
178 {
179 struct cxl_region *cxlr = to_cxl_region(dev);
180 struct cxl_region_params *p = &cxlr->params;
181 uuid_t temp;
182 ssize_t rc;
183
184 if (len != UUID_STRING_LEN + 1)
185 return -EINVAL;
186
187 rc = uuid_parse(buf, &temp);
188 if (rc)
189 return rc;
190
191 if (uuid_is_null(&temp))
192 return -EINVAL;
193
194 rc = down_write_killable(&cxl_region_rwsem);
195 if (rc)
196 return rc;
197
198 if (uuid_equal(&p->uuid, &temp))
199 goto out;
200
201 rc = -EBUSY;
202 if (p->state >= CXL_CONFIG_ACTIVE)
203 goto out;
204
205 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
206 if (rc < 0)
207 goto out;
208
209 uuid_copy(&p->uuid, &temp);
210 out:
211 up_write(&cxl_region_rwsem);
212
213 if (rc)
214 return rc;
215 return len;
216 }
217 static DEVICE_ATTR_RW(uuid);
218
cxl_rr_load(struct cxl_port * port,struct cxl_region * cxlr)219 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
220 struct cxl_region *cxlr)
221 {
222 return xa_load(&port->regions, (unsigned long)cxlr);
223 }
224
cxl_region_invalidate_memregion(struct cxl_region * cxlr)225 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
226 {
227 if (!cpu_cache_has_invalidate_memregion()) {
228 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
229 dev_info_once(
230 &cxlr->dev,
231 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
232 return 0;
233 } else {
234 dev_err(&cxlr->dev,
235 "Failed to synchronize CPU cache state\n");
236 return -ENXIO;
237 }
238 }
239
240 cpu_cache_invalidate_memregion(IORES_DESC_CXL);
241 return 0;
242 }
243
cxl_region_decode_reset(struct cxl_region * cxlr,int count)244 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
245 {
246 struct cxl_region_params *p = &cxlr->params;
247 int i, rc = 0;
248
249 /*
250 * Before region teardown attempt to flush, and if the flush
251 * fails cancel the region teardown for data consistency
252 * concerns
253 */
254 rc = cxl_region_invalidate_memregion(cxlr);
255 if (rc)
256 return rc;
257
258 for (i = count - 1; i >= 0; i--) {
259 struct cxl_endpoint_decoder *cxled = p->targets[i];
260 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
261 struct cxl_port *iter = cxled_to_port(cxled);
262 struct cxl_dev_state *cxlds = cxlmd->cxlds;
263 struct cxl_ep *ep;
264
265 if (cxlds->rcd)
266 goto endpoint_reset;
267
268 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
269 iter = to_cxl_port(iter->dev.parent);
270
271 for (ep = cxl_ep_load(iter, cxlmd); iter;
272 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
273 struct cxl_region_ref *cxl_rr;
274 struct cxl_decoder *cxld;
275
276 cxl_rr = cxl_rr_load(iter, cxlr);
277 cxld = cxl_rr->decoder;
278 if (cxld->reset)
279 rc = cxld->reset(cxld);
280 if (rc)
281 return rc;
282 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
283 }
284
285 endpoint_reset:
286 rc = cxled->cxld.reset(&cxled->cxld);
287 if (rc)
288 return rc;
289 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
290 }
291
292 /* all decoders associated with this region have been torn down */
293 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
294
295 return 0;
296 }
297
commit_decoder(struct cxl_decoder * cxld)298 static int commit_decoder(struct cxl_decoder *cxld)
299 {
300 struct cxl_switch_decoder *cxlsd = NULL;
301
302 if (cxld->commit)
303 return cxld->commit(cxld);
304
305 if (is_switch_decoder(&cxld->dev))
306 cxlsd = to_cxl_switch_decoder(&cxld->dev);
307
308 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
309 "->commit() is required\n"))
310 return -ENXIO;
311 return 0;
312 }
313
cxl_region_decode_commit(struct cxl_region * cxlr)314 static int cxl_region_decode_commit(struct cxl_region *cxlr)
315 {
316 struct cxl_region_params *p = &cxlr->params;
317 int i, rc = 0;
318
319 for (i = 0; i < p->nr_targets; i++) {
320 struct cxl_endpoint_decoder *cxled = p->targets[i];
321 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
322 struct cxl_region_ref *cxl_rr;
323 struct cxl_decoder *cxld;
324 struct cxl_port *iter;
325 struct cxl_ep *ep;
326
327 /* commit bottom up */
328 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
329 iter = to_cxl_port(iter->dev.parent)) {
330 cxl_rr = cxl_rr_load(iter, cxlr);
331 cxld = cxl_rr->decoder;
332 rc = commit_decoder(cxld);
333 if (rc)
334 break;
335 }
336
337 if (rc) {
338 /* programming @iter failed, teardown */
339 for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
340 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
341 cxl_rr = cxl_rr_load(iter, cxlr);
342 cxld = cxl_rr->decoder;
343 if (cxld->reset)
344 cxld->reset(cxld);
345 }
346
347 cxled->cxld.reset(&cxled->cxld);
348 goto err;
349 }
350 }
351
352 return 0;
353
354 err:
355 /* undo the targets that were successfully committed */
356 cxl_region_decode_reset(cxlr, i);
357 return rc;
358 }
359
commit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)360 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
361 const char *buf, size_t len)
362 {
363 struct cxl_region *cxlr = to_cxl_region(dev);
364 struct cxl_region_params *p = &cxlr->params;
365 bool commit;
366 ssize_t rc;
367
368 rc = kstrtobool(buf, &commit);
369 if (rc)
370 return rc;
371
372 rc = down_write_killable(&cxl_region_rwsem);
373 if (rc)
374 return rc;
375
376 /* Already in the requested state? */
377 if (commit && p->state >= CXL_CONFIG_COMMIT)
378 goto out;
379 if (!commit && p->state < CXL_CONFIG_COMMIT)
380 goto out;
381
382 /* Not ready to commit? */
383 if (commit && p->state < CXL_CONFIG_ACTIVE) {
384 rc = -ENXIO;
385 goto out;
386 }
387
388 /*
389 * Invalidate caches before region setup to drop any speculative
390 * consumption of this address space
391 */
392 rc = cxl_region_invalidate_memregion(cxlr);
393 if (rc)
394 goto out;
395
396 if (commit) {
397 rc = cxl_region_decode_commit(cxlr);
398 if (rc == 0)
399 p->state = CXL_CONFIG_COMMIT;
400 } else {
401 p->state = CXL_CONFIG_RESET_PENDING;
402 up_write(&cxl_region_rwsem);
403 device_release_driver(&cxlr->dev);
404 down_write(&cxl_region_rwsem);
405
406 /*
407 * The lock was dropped, so need to revalidate that the reset is
408 * still pending.
409 */
410 if (p->state == CXL_CONFIG_RESET_PENDING) {
411 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
412 /*
413 * Revert to committed since there may still be active
414 * decoders associated with this region, or move forward
415 * to active to mark the reset successful
416 */
417 if (rc)
418 p->state = CXL_CONFIG_COMMIT;
419 else
420 p->state = CXL_CONFIG_ACTIVE;
421 }
422 }
423
424 out:
425 up_write(&cxl_region_rwsem);
426
427 if (rc)
428 return rc;
429 return len;
430 }
431
commit_show(struct device * dev,struct device_attribute * attr,char * buf)432 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
433 char *buf)
434 {
435 struct cxl_region *cxlr = to_cxl_region(dev);
436 struct cxl_region_params *p = &cxlr->params;
437 ssize_t rc;
438
439 rc = down_read_interruptible(&cxl_region_rwsem);
440 if (rc)
441 return rc;
442 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
443 up_read(&cxl_region_rwsem);
444
445 return rc;
446 }
447 static DEVICE_ATTR_RW(commit);
448
cxl_region_visible(struct kobject * kobj,struct attribute * a,int n)449 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
450 int n)
451 {
452 struct device *dev = kobj_to_dev(kobj);
453 struct cxl_region *cxlr = to_cxl_region(dev);
454
455 /*
456 * Support tooling that expects to find a 'uuid' attribute for all
457 * regions regardless of mode.
458 */
459 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
460 return 0444;
461 return a->mode;
462 }
463
interleave_ways_show(struct device * dev,struct device_attribute * attr,char * buf)464 static ssize_t interleave_ways_show(struct device *dev,
465 struct device_attribute *attr, char *buf)
466 {
467 struct cxl_region *cxlr = to_cxl_region(dev);
468 struct cxl_region_params *p = &cxlr->params;
469 ssize_t rc;
470
471 rc = down_read_interruptible(&cxl_region_rwsem);
472 if (rc)
473 return rc;
474 rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
475 up_read(&cxl_region_rwsem);
476
477 return rc;
478 }
479
480 static const struct attribute_group *get_cxl_region_target_group(void);
481
interleave_ways_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)482 static ssize_t interleave_ways_store(struct device *dev,
483 struct device_attribute *attr,
484 const char *buf, size_t len)
485 {
486 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
487 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
488 struct cxl_region *cxlr = to_cxl_region(dev);
489 struct cxl_region_params *p = &cxlr->params;
490 unsigned int val, save;
491 int rc;
492 u8 iw;
493
494 rc = kstrtouint(buf, 0, &val);
495 if (rc)
496 return rc;
497
498 rc = ways_to_eiw(val, &iw);
499 if (rc)
500 return rc;
501
502 /*
503 * Even for x3, x6, and x12 interleaves the region interleave must be a
504 * power of 2 multiple of the host bridge interleave.
505 */
506 if (!is_power_of_2(val / cxld->interleave_ways) ||
507 (val % cxld->interleave_ways)) {
508 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
509 return -EINVAL;
510 }
511
512 rc = down_write_killable(&cxl_region_rwsem);
513 if (rc)
514 return rc;
515 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
516 rc = -EBUSY;
517 goto out;
518 }
519
520 save = p->interleave_ways;
521 p->interleave_ways = val;
522 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
523 if (rc)
524 p->interleave_ways = save;
525 out:
526 up_write(&cxl_region_rwsem);
527 if (rc)
528 return rc;
529 return len;
530 }
531 static DEVICE_ATTR_RW(interleave_ways);
532
interleave_granularity_show(struct device * dev,struct device_attribute * attr,char * buf)533 static ssize_t interleave_granularity_show(struct device *dev,
534 struct device_attribute *attr,
535 char *buf)
536 {
537 struct cxl_region *cxlr = to_cxl_region(dev);
538 struct cxl_region_params *p = &cxlr->params;
539 ssize_t rc;
540
541 rc = down_read_interruptible(&cxl_region_rwsem);
542 if (rc)
543 return rc;
544 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
545 up_read(&cxl_region_rwsem);
546
547 return rc;
548 }
549
interleave_granularity_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)550 static ssize_t interleave_granularity_store(struct device *dev,
551 struct device_attribute *attr,
552 const char *buf, size_t len)
553 {
554 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
555 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
556 struct cxl_region *cxlr = to_cxl_region(dev);
557 struct cxl_region_params *p = &cxlr->params;
558 int rc, val;
559 u16 ig;
560
561 rc = kstrtoint(buf, 0, &val);
562 if (rc)
563 return rc;
564
565 rc = granularity_to_eig(val, &ig);
566 if (rc)
567 return rc;
568
569 /*
570 * When the host-bridge is interleaved, disallow region granularity !=
571 * root granularity. Regions with a granularity less than the root
572 * interleave result in needing multiple endpoints to support a single
573 * slot in the interleave (possible to support in the future). Regions
574 * with a granularity greater than the root interleave result in invalid
575 * DPA translations (invalid to support).
576 */
577 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
578 return -EINVAL;
579
580 rc = down_write_killable(&cxl_region_rwsem);
581 if (rc)
582 return rc;
583 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
584 rc = -EBUSY;
585 goto out;
586 }
587
588 p->interleave_granularity = val;
589 out:
590 up_write(&cxl_region_rwsem);
591 if (rc)
592 return rc;
593 return len;
594 }
595 static DEVICE_ATTR_RW(interleave_granularity);
596
resource_show(struct device * dev,struct device_attribute * attr,char * buf)597 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
598 char *buf)
599 {
600 struct cxl_region *cxlr = to_cxl_region(dev);
601 struct cxl_region_params *p = &cxlr->params;
602 u64 resource = -1ULL;
603 ssize_t rc;
604
605 rc = down_read_interruptible(&cxl_region_rwsem);
606 if (rc)
607 return rc;
608 if (p->res)
609 resource = p->res->start;
610 rc = sysfs_emit(buf, "%#llx\n", resource);
611 up_read(&cxl_region_rwsem);
612
613 return rc;
614 }
615 static DEVICE_ATTR_RO(resource);
616
mode_show(struct device * dev,struct device_attribute * attr,char * buf)617 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
618 char *buf)
619 {
620 struct cxl_region *cxlr = to_cxl_region(dev);
621
622 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
623 }
624 static DEVICE_ATTR_RO(mode);
625
alloc_hpa(struct cxl_region * cxlr,resource_size_t size)626 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
627 {
628 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
629 struct cxl_region_params *p = &cxlr->params;
630 struct resource *res;
631 u64 remainder = 0;
632
633 lockdep_assert_held_write(&cxl_region_rwsem);
634
635 /* Nothing to do... */
636 if (p->res && resource_size(p->res) == size)
637 return 0;
638
639 /* To change size the old size must be freed first */
640 if (p->res)
641 return -EBUSY;
642
643 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
644 return -EBUSY;
645
646 /* ways, granularity and uuid (if PMEM) need to be set before HPA */
647 if (!p->interleave_ways || !p->interleave_granularity ||
648 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
649 return -ENXIO;
650
651 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
652 if (remainder)
653 return -EINVAL;
654
655 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
656 dev_name(&cxlr->dev));
657 if (IS_ERR(res)) {
658 dev_dbg(&cxlr->dev,
659 "HPA allocation error (%ld) for size:%pap in %s %pr\n",
660 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
661 return PTR_ERR(res);
662 }
663
664 p->res = res;
665 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
666
667 return 0;
668 }
669
cxl_region_iomem_release(struct cxl_region * cxlr)670 static void cxl_region_iomem_release(struct cxl_region *cxlr)
671 {
672 struct cxl_region_params *p = &cxlr->params;
673
674 if (device_is_registered(&cxlr->dev))
675 lockdep_assert_held_write(&cxl_region_rwsem);
676 if (p->res) {
677 /*
678 * Autodiscovered regions may not have been able to insert their
679 * resource.
680 */
681 if (p->res->parent)
682 remove_resource(p->res);
683 kfree(p->res);
684 p->res = NULL;
685 }
686 }
687
free_hpa(struct cxl_region * cxlr)688 static int free_hpa(struct cxl_region *cxlr)
689 {
690 struct cxl_region_params *p = &cxlr->params;
691
692 lockdep_assert_held_write(&cxl_region_rwsem);
693
694 if (!p->res)
695 return 0;
696
697 if (p->state >= CXL_CONFIG_ACTIVE)
698 return -EBUSY;
699
700 cxl_region_iomem_release(cxlr);
701 p->state = CXL_CONFIG_IDLE;
702 return 0;
703 }
704
size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)705 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
706 const char *buf, size_t len)
707 {
708 struct cxl_region *cxlr = to_cxl_region(dev);
709 u64 val;
710 int rc;
711
712 rc = kstrtou64(buf, 0, &val);
713 if (rc)
714 return rc;
715
716 rc = down_write_killable(&cxl_region_rwsem);
717 if (rc)
718 return rc;
719
720 if (val)
721 rc = alloc_hpa(cxlr, val);
722 else
723 rc = free_hpa(cxlr);
724 up_write(&cxl_region_rwsem);
725
726 if (rc)
727 return rc;
728
729 return len;
730 }
731
size_show(struct device * dev,struct device_attribute * attr,char * buf)732 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
733 char *buf)
734 {
735 struct cxl_region *cxlr = to_cxl_region(dev);
736 struct cxl_region_params *p = &cxlr->params;
737 u64 size = 0;
738 ssize_t rc;
739
740 rc = down_read_interruptible(&cxl_region_rwsem);
741 if (rc)
742 return rc;
743 if (p->res)
744 size = resource_size(p->res);
745 rc = sysfs_emit(buf, "%#llx\n", size);
746 up_read(&cxl_region_rwsem);
747
748 return rc;
749 }
750 static DEVICE_ATTR_RW(size);
751
752 static struct attribute *cxl_region_attrs[] = {
753 &dev_attr_uuid.attr,
754 &dev_attr_commit.attr,
755 &dev_attr_interleave_ways.attr,
756 &dev_attr_interleave_granularity.attr,
757 &dev_attr_resource.attr,
758 &dev_attr_size.attr,
759 &dev_attr_mode.attr,
760 NULL,
761 };
762
763 static const struct attribute_group cxl_region_group = {
764 .attrs = cxl_region_attrs,
765 .is_visible = cxl_region_visible,
766 };
767
show_targetN(struct cxl_region * cxlr,char * buf,int pos)768 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
769 {
770 struct cxl_region_params *p = &cxlr->params;
771 struct cxl_endpoint_decoder *cxled;
772 int rc;
773
774 rc = down_read_interruptible(&cxl_region_rwsem);
775 if (rc)
776 return rc;
777
778 if (pos >= p->interleave_ways) {
779 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
780 p->interleave_ways);
781 rc = -ENXIO;
782 goto out;
783 }
784
785 cxled = p->targets[pos];
786 if (!cxled)
787 rc = sysfs_emit(buf, "\n");
788 else
789 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
790 out:
791 up_read(&cxl_region_rwsem);
792
793 return rc;
794 }
795
match_free_decoder(struct device * dev,void * data)796 static int match_free_decoder(struct device *dev, void *data)
797 {
798 struct cxl_decoder *cxld;
799 int *id = data;
800
801 if (!is_switch_decoder(dev))
802 return 0;
803
804 cxld = to_cxl_decoder(dev);
805
806 /* enforce ordered allocation */
807 if (cxld->id != *id)
808 return 0;
809
810 if (!cxld->region)
811 return 1;
812
813 (*id)++;
814
815 return 0;
816 }
817
match_auto_decoder(struct device * dev,void * data)818 static int match_auto_decoder(struct device *dev, void *data)
819 {
820 struct cxl_region_params *p = data;
821 struct cxl_decoder *cxld;
822 struct range *r;
823
824 if (!is_switch_decoder(dev))
825 return 0;
826
827 cxld = to_cxl_decoder(dev);
828 r = &cxld->hpa_range;
829
830 if (p->res && p->res->start == r->start && p->res->end == r->end)
831 return 1;
832
833 return 0;
834 }
835
836 static struct cxl_decoder *
cxl_region_find_decoder(struct cxl_port * port,struct cxl_endpoint_decoder * cxled,struct cxl_region * cxlr)837 cxl_region_find_decoder(struct cxl_port *port,
838 struct cxl_endpoint_decoder *cxled,
839 struct cxl_region *cxlr)
840 {
841 struct device *dev;
842 int id = 0;
843
844 if (port == cxled_to_port(cxled))
845 return &cxled->cxld;
846
847 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
848 dev = device_find_child(&port->dev, &cxlr->params,
849 match_auto_decoder);
850 else
851 dev = device_find_child(&port->dev, &id, match_free_decoder);
852 if (!dev)
853 return NULL;
854 /*
855 * This decoder is pinned registered as long as the endpoint decoder is
856 * registered, and endpoint decoder unregistration holds the
857 * cxl_region_rwsem over unregister events, so no need to hold on to
858 * this extra reference.
859 */
860 put_device(dev);
861 return to_cxl_decoder(dev);
862 }
863
auto_order_ok(struct cxl_port * port,struct cxl_region * cxlr_iter,struct cxl_decoder * cxld)864 static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
865 struct cxl_decoder *cxld)
866 {
867 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
868 struct cxl_decoder *cxld_iter = rr->decoder;
869
870 /*
871 * Allow the out of order assembly of auto-discovered regions.
872 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
873 * in HPA order. Confirm that the decoder with the lesser HPA
874 * starting address has the lesser id.
875 */
876 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
877 dev_name(&cxld->dev), cxld->id,
878 dev_name(&cxld_iter->dev), cxld_iter->id);
879
880 if (cxld_iter->id > cxld->id)
881 return true;
882
883 return false;
884 }
885
886 static struct cxl_region_ref *
alloc_region_ref(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)887 alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
888 struct cxl_endpoint_decoder *cxled)
889 {
890 struct cxl_region_params *p = &cxlr->params;
891 struct cxl_region_ref *cxl_rr, *iter;
892 unsigned long index;
893 int rc;
894
895 xa_for_each(&port->regions, index, iter) {
896 struct cxl_region_params *ip = &iter->region->params;
897
898 if (!ip->res || ip->res->start < p->res->start)
899 continue;
900
901 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
902 struct cxl_decoder *cxld;
903
904 cxld = cxl_region_find_decoder(port, cxled, cxlr);
905 if (auto_order_ok(port, iter->region, cxld))
906 continue;
907 }
908 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
909 dev_name(&port->dev),
910 dev_name(&iter->region->dev), ip->res, p->res);
911
912 return ERR_PTR(-EBUSY);
913 }
914
915 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
916 if (!cxl_rr)
917 return ERR_PTR(-ENOMEM);
918 cxl_rr->port = port;
919 cxl_rr->region = cxlr;
920 cxl_rr->nr_targets = 1;
921 xa_init(&cxl_rr->endpoints);
922
923 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
924 if (rc) {
925 dev_dbg(&cxlr->dev,
926 "%s: failed to track region reference: %d\n",
927 dev_name(&port->dev), rc);
928 kfree(cxl_rr);
929 return ERR_PTR(rc);
930 }
931
932 return cxl_rr;
933 }
934
cxl_rr_free_decoder(struct cxl_region_ref * cxl_rr)935 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
936 {
937 struct cxl_region *cxlr = cxl_rr->region;
938 struct cxl_decoder *cxld = cxl_rr->decoder;
939
940 if (!cxld)
941 return;
942
943 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
944 if (cxld->region == cxlr) {
945 cxld->region = NULL;
946 put_device(&cxlr->dev);
947 }
948 }
949
free_region_ref(struct cxl_region_ref * cxl_rr)950 static void free_region_ref(struct cxl_region_ref *cxl_rr)
951 {
952 struct cxl_port *port = cxl_rr->port;
953 struct cxl_region *cxlr = cxl_rr->region;
954
955 cxl_rr_free_decoder(cxl_rr);
956 xa_erase(&port->regions, (unsigned long)cxlr);
957 xa_destroy(&cxl_rr->endpoints);
958 kfree(cxl_rr);
959 }
960
cxl_rr_ep_add(struct cxl_region_ref * cxl_rr,struct cxl_endpoint_decoder * cxled)961 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
962 struct cxl_endpoint_decoder *cxled)
963 {
964 int rc;
965 struct cxl_port *port = cxl_rr->port;
966 struct cxl_region *cxlr = cxl_rr->region;
967 struct cxl_decoder *cxld = cxl_rr->decoder;
968 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
969
970 if (ep) {
971 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
972 GFP_KERNEL);
973 if (rc)
974 return rc;
975 }
976 cxl_rr->nr_eps++;
977
978 if (!cxld->region) {
979 cxld->region = cxlr;
980 get_device(&cxlr->dev);
981 }
982
983 return 0;
984 }
985
cxl_rr_alloc_decoder(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,struct cxl_region_ref * cxl_rr)986 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
987 struct cxl_endpoint_decoder *cxled,
988 struct cxl_region_ref *cxl_rr)
989 {
990 struct cxl_decoder *cxld;
991
992 cxld = cxl_region_find_decoder(port, cxled, cxlr);
993 if (!cxld) {
994 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
995 dev_name(&port->dev));
996 return -EBUSY;
997 }
998
999 if (cxld->region) {
1000 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
1001 dev_name(&port->dev), dev_name(&cxld->dev),
1002 dev_name(&cxld->region->dev));
1003 return -EBUSY;
1004 }
1005
1006 /*
1007 * Endpoints should already match the region type, but backstop that
1008 * assumption with an assertion. Switch-decoders change mapping-type
1009 * based on what is mapped when they are assigned to a region.
1010 */
1011 dev_WARN_ONCE(&cxlr->dev,
1012 port == cxled_to_port(cxled) &&
1013 cxld->target_type != cxlr->type,
1014 "%s:%s mismatch decoder type %d -> %d\n",
1015 dev_name(&cxled_to_memdev(cxled)->dev),
1016 dev_name(&cxld->dev), cxld->target_type, cxlr->type);
1017 cxld->target_type = cxlr->type;
1018 cxl_rr->decoder = cxld;
1019 return 0;
1020 }
1021
1022 /**
1023 * cxl_port_attach_region() - track a region's interest in a port by endpoint
1024 * @port: port to add a new region reference 'struct cxl_region_ref'
1025 * @cxlr: region to attach to @port
1026 * @cxled: endpoint decoder used to create or further pin a region reference
1027 * @pos: interleave position of @cxled in @cxlr
1028 *
1029 * The attach event is an opportunity to validate CXL decode setup
1030 * constraints and record metadata needed for programming HDM decoders,
1031 * in particular decoder target lists.
1032 *
1033 * The steps are:
1034 *
1035 * - validate that there are no other regions with a higher HPA already
1036 * associated with @port
1037 * - establish a region reference if one is not already present
1038 *
1039 * - additionally allocate a decoder instance that will host @cxlr on
1040 * @port
1041 *
1042 * - pin the region reference by the endpoint
1043 * - account for how many entries in @port's target list are needed to
1044 * cover all of the added endpoints.
1045 */
cxl_port_attach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1046 static int cxl_port_attach_region(struct cxl_port *port,
1047 struct cxl_region *cxlr,
1048 struct cxl_endpoint_decoder *cxled, int pos)
1049 {
1050 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1051 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1052 struct cxl_region_ref *cxl_rr;
1053 bool nr_targets_inc = false;
1054 struct cxl_decoder *cxld;
1055 unsigned long index;
1056 int rc = -EBUSY;
1057
1058 lockdep_assert_held_write(&cxl_region_rwsem);
1059
1060 cxl_rr = cxl_rr_load(port, cxlr);
1061 if (cxl_rr) {
1062 struct cxl_ep *ep_iter;
1063 int found = 0;
1064
1065 /*
1066 * Walk the existing endpoints that have been attached to
1067 * @cxlr at @port and see if they share the same 'next' port
1068 * in the downstream direction. I.e. endpoints that share common
1069 * upstream switch.
1070 */
1071 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1072 if (ep_iter == ep)
1073 continue;
1074 if (ep_iter->next == ep->next) {
1075 found++;
1076 break;
1077 }
1078 }
1079
1080 /*
1081 * New target port, or @port is an endpoint port that always
1082 * accounts its own local decode as a target.
1083 */
1084 if (!found || !ep->next) {
1085 cxl_rr->nr_targets++;
1086 nr_targets_inc = true;
1087 }
1088 } else {
1089 cxl_rr = alloc_region_ref(port, cxlr, cxled);
1090 if (IS_ERR(cxl_rr)) {
1091 dev_dbg(&cxlr->dev,
1092 "%s: failed to allocate region reference\n",
1093 dev_name(&port->dev));
1094 return PTR_ERR(cxl_rr);
1095 }
1096 nr_targets_inc = true;
1097
1098 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
1099 if (rc)
1100 goto out_erase;
1101 }
1102 cxld = cxl_rr->decoder;
1103
1104 rc = cxl_rr_ep_add(cxl_rr, cxled);
1105 if (rc) {
1106 dev_dbg(&cxlr->dev,
1107 "%s: failed to track endpoint %s:%s reference\n",
1108 dev_name(&port->dev), dev_name(&cxlmd->dev),
1109 dev_name(&cxld->dev));
1110 goto out_erase;
1111 }
1112
1113 dev_dbg(&cxlr->dev,
1114 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1115 dev_name(port->uport_dev), dev_name(&port->dev),
1116 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1117 dev_name(&cxled->cxld.dev), pos,
1118 ep ? ep->next ? dev_name(ep->next->uport_dev) :
1119 dev_name(&cxlmd->dev) :
1120 "none",
1121 cxl_rr->nr_eps, cxl_rr->nr_targets);
1122
1123 return 0;
1124 out_erase:
1125 if (nr_targets_inc)
1126 cxl_rr->nr_targets--;
1127 if (cxl_rr->nr_eps == 0)
1128 free_region_ref(cxl_rr);
1129 return rc;
1130 }
1131
cxl_port_detach_region(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1132 static void cxl_port_detach_region(struct cxl_port *port,
1133 struct cxl_region *cxlr,
1134 struct cxl_endpoint_decoder *cxled)
1135 {
1136 struct cxl_region_ref *cxl_rr;
1137 struct cxl_ep *ep = NULL;
1138
1139 lockdep_assert_held_write(&cxl_region_rwsem);
1140
1141 cxl_rr = cxl_rr_load(port, cxlr);
1142 if (!cxl_rr)
1143 return;
1144
1145 /*
1146 * Endpoint ports do not carry cxl_ep references, and they
1147 * never target more than one endpoint by definition
1148 */
1149 if (cxl_rr->decoder == &cxled->cxld)
1150 cxl_rr->nr_eps--;
1151 else
1152 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1153 if (ep) {
1154 struct cxl_ep *ep_iter;
1155 unsigned long index;
1156 int found = 0;
1157
1158 cxl_rr->nr_eps--;
1159 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1160 if (ep_iter->next == ep->next) {
1161 found++;
1162 break;
1163 }
1164 }
1165 if (!found)
1166 cxl_rr->nr_targets--;
1167 }
1168
1169 if (cxl_rr->nr_eps == 0)
1170 free_region_ref(cxl_rr);
1171 }
1172
check_last_peer(struct cxl_endpoint_decoder * cxled,struct cxl_ep * ep,struct cxl_region_ref * cxl_rr,int distance)1173 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1174 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1175 int distance)
1176 {
1177 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1178 struct cxl_region *cxlr = cxl_rr->region;
1179 struct cxl_region_params *p = &cxlr->params;
1180 struct cxl_endpoint_decoder *cxled_peer;
1181 struct cxl_port *port = cxl_rr->port;
1182 struct cxl_memdev *cxlmd_peer;
1183 struct cxl_ep *ep_peer;
1184 int pos = cxled->pos;
1185
1186 /*
1187 * If this position wants to share a dport with the last endpoint mapped
1188 * then that endpoint, at index 'position - distance', must also be
1189 * mapped by this dport.
1190 */
1191 if (pos < distance) {
1192 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1193 dev_name(port->uport_dev), dev_name(&port->dev),
1194 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1195 return -ENXIO;
1196 }
1197 cxled_peer = p->targets[pos - distance];
1198 cxlmd_peer = cxled_to_memdev(cxled_peer);
1199 ep_peer = cxl_ep_load(port, cxlmd_peer);
1200 if (ep->dport != ep_peer->dport) {
1201 dev_dbg(&cxlr->dev,
1202 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1203 dev_name(port->uport_dev), dev_name(&port->dev),
1204 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1205 dev_name(&cxlmd_peer->dev),
1206 dev_name(&cxled_peer->cxld.dev));
1207 return -ENXIO;
1208 }
1209
1210 return 0;
1211 }
1212
cxl_port_setup_targets(struct cxl_port * port,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)1213 static int cxl_port_setup_targets(struct cxl_port *port,
1214 struct cxl_region *cxlr,
1215 struct cxl_endpoint_decoder *cxled)
1216 {
1217 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1218 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1219 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1220 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1221 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1222 struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1223 struct cxl_region_params *p = &cxlr->params;
1224 struct cxl_decoder *cxld = cxl_rr->decoder;
1225 struct cxl_switch_decoder *cxlsd;
1226 u16 eig, peig;
1227 u8 eiw, peiw;
1228
1229 /*
1230 * While root level decoders support x3, x6, x12, switch level
1231 * decoders only support powers of 2 up to x16.
1232 */
1233 if (!is_power_of_2(cxl_rr->nr_targets)) {
1234 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1235 dev_name(port->uport_dev), dev_name(&port->dev),
1236 cxl_rr->nr_targets);
1237 return -EINVAL;
1238 }
1239
1240 cxlsd = to_cxl_switch_decoder(&cxld->dev);
1241 if (cxl_rr->nr_targets_set) {
1242 int i, distance;
1243
1244 /*
1245 * Passthrough decoders impose no distance requirements between
1246 * peers
1247 */
1248 if (cxl_rr->nr_targets == 1)
1249 distance = 0;
1250 else
1251 distance = p->nr_targets / cxl_rr->nr_targets;
1252 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1253 if (ep->dport == cxlsd->target[i]) {
1254 rc = check_last_peer(cxled, ep, cxl_rr,
1255 distance);
1256 if (rc)
1257 return rc;
1258 goto out_target_set;
1259 }
1260 goto add_target;
1261 }
1262
1263 if (is_cxl_root(parent_port)) {
1264 /*
1265 * Root decoder IG is always set to value in CFMWS which
1266 * may be different than this region's IG. We can use the
1267 * region's IG here since interleave_granularity_store()
1268 * does not allow interleaved host-bridges with
1269 * root IG != region IG.
1270 */
1271 parent_ig = p->interleave_granularity;
1272 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1273 /*
1274 * For purposes of address bit routing, use power-of-2 math for
1275 * switch ports.
1276 */
1277 if (!is_power_of_2(parent_iw))
1278 parent_iw /= 3;
1279 } else {
1280 struct cxl_region_ref *parent_rr;
1281 struct cxl_decoder *parent_cxld;
1282
1283 parent_rr = cxl_rr_load(parent_port, cxlr);
1284 parent_cxld = parent_rr->decoder;
1285 parent_ig = parent_cxld->interleave_granularity;
1286 parent_iw = parent_cxld->interleave_ways;
1287 }
1288
1289 rc = granularity_to_eig(parent_ig, &peig);
1290 if (rc) {
1291 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1292 dev_name(parent_port->uport_dev),
1293 dev_name(&parent_port->dev), parent_ig);
1294 return rc;
1295 }
1296
1297 rc = ways_to_eiw(parent_iw, &peiw);
1298 if (rc) {
1299 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1300 dev_name(parent_port->uport_dev),
1301 dev_name(&parent_port->dev), parent_iw);
1302 return rc;
1303 }
1304
1305 iw = cxl_rr->nr_targets;
1306 rc = ways_to_eiw(iw, &eiw);
1307 if (rc) {
1308 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1309 dev_name(port->uport_dev), dev_name(&port->dev), iw);
1310 return rc;
1311 }
1312
1313 /*
1314 * Interleave granularity is a multiple of @parent_port granularity.
1315 * Multiplier is the parent port interleave ways.
1316 */
1317 rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1318 if (rc) {
1319 dev_dbg(&cxlr->dev,
1320 "%s: invalid granularity calculation (%d * %d)\n",
1321 dev_name(&parent_port->dev), parent_ig, parent_iw);
1322 return rc;
1323 }
1324
1325 rc = eig_to_granularity(eig, &ig);
1326 if (rc) {
1327 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1328 dev_name(port->uport_dev), dev_name(&port->dev),
1329 256 << eig);
1330 return rc;
1331 }
1332
1333 if (iw > 8 || iw > cxlsd->nr_targets) {
1334 dev_dbg(&cxlr->dev,
1335 "%s:%s:%s: ways: %d overflows targets: %d\n",
1336 dev_name(port->uport_dev), dev_name(&port->dev),
1337 dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1338 return -ENXIO;
1339 }
1340
1341 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1342 if (cxld->interleave_ways != iw ||
1343 cxld->interleave_granularity != ig ||
1344 cxld->hpa_range.start != p->res->start ||
1345 cxld->hpa_range.end != p->res->end ||
1346 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1347 dev_err(&cxlr->dev,
1348 "%s:%s %s expected iw: %d ig: %d %pr\n",
1349 dev_name(port->uport_dev), dev_name(&port->dev),
1350 __func__, iw, ig, p->res);
1351 dev_err(&cxlr->dev,
1352 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1353 dev_name(port->uport_dev), dev_name(&port->dev),
1354 __func__, cxld->interleave_ways,
1355 cxld->interleave_granularity,
1356 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1357 "enabled" :
1358 "disabled",
1359 cxld->hpa_range.start, cxld->hpa_range.end);
1360 return -ENXIO;
1361 }
1362 } else {
1363 cxld->interleave_ways = iw;
1364 cxld->interleave_granularity = ig;
1365 cxld->hpa_range = (struct range) {
1366 .start = p->res->start,
1367 .end = p->res->end,
1368 };
1369 }
1370 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1371 dev_name(&port->dev), iw, ig);
1372 add_target:
1373 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1374 dev_dbg(&cxlr->dev,
1375 "%s:%s: targets full trying to add %s:%s at %d\n",
1376 dev_name(port->uport_dev), dev_name(&port->dev),
1377 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1378 return -ENXIO;
1379 }
1380 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1381 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1382 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1383 dev_name(port->uport_dev), dev_name(&port->dev),
1384 dev_name(&cxlsd->cxld.dev),
1385 dev_name(ep->dport->dport_dev),
1386 cxl_rr->nr_targets_set);
1387 return -ENXIO;
1388 }
1389 } else
1390 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1391 inc = 1;
1392 out_target_set:
1393 cxl_rr->nr_targets_set += inc;
1394 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1395 dev_name(port->uport_dev), dev_name(&port->dev),
1396 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1397 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1398
1399 return 0;
1400 }
1401
cxl_port_reset_targets(struct cxl_port * port,struct cxl_region * cxlr)1402 static void cxl_port_reset_targets(struct cxl_port *port,
1403 struct cxl_region *cxlr)
1404 {
1405 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1406 struct cxl_decoder *cxld;
1407
1408 /*
1409 * After the last endpoint has been detached the entire cxl_rr may now
1410 * be gone.
1411 */
1412 if (!cxl_rr)
1413 return;
1414 cxl_rr->nr_targets_set = 0;
1415
1416 cxld = cxl_rr->decoder;
1417 cxld->hpa_range = (struct range) {
1418 .start = 0,
1419 .end = -1,
1420 };
1421 }
1422
cxl_region_teardown_targets(struct cxl_region * cxlr)1423 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1424 {
1425 struct cxl_region_params *p = &cxlr->params;
1426 struct cxl_endpoint_decoder *cxled;
1427 struct cxl_dev_state *cxlds;
1428 struct cxl_memdev *cxlmd;
1429 struct cxl_port *iter;
1430 struct cxl_ep *ep;
1431 int i;
1432
1433 /*
1434 * In the auto-discovery case skip automatic teardown since the
1435 * address space is already active
1436 */
1437 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1438 return;
1439
1440 for (i = 0; i < p->nr_targets; i++) {
1441 cxled = p->targets[i];
1442 cxlmd = cxled_to_memdev(cxled);
1443 cxlds = cxlmd->cxlds;
1444
1445 if (cxlds->rcd)
1446 continue;
1447
1448 iter = cxled_to_port(cxled);
1449 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1450 iter = to_cxl_port(iter->dev.parent);
1451
1452 for (ep = cxl_ep_load(iter, cxlmd); iter;
1453 iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1454 cxl_port_reset_targets(iter, cxlr);
1455 }
1456 }
1457
cxl_region_setup_targets(struct cxl_region * cxlr)1458 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1459 {
1460 struct cxl_region_params *p = &cxlr->params;
1461 struct cxl_endpoint_decoder *cxled;
1462 struct cxl_dev_state *cxlds;
1463 int i, rc, rch = 0, vh = 0;
1464 struct cxl_memdev *cxlmd;
1465 struct cxl_port *iter;
1466 struct cxl_ep *ep;
1467
1468 for (i = 0; i < p->nr_targets; i++) {
1469 cxled = p->targets[i];
1470 cxlmd = cxled_to_memdev(cxled);
1471 cxlds = cxlmd->cxlds;
1472
1473 /* validate that all targets agree on topology */
1474 if (!cxlds->rcd) {
1475 vh++;
1476 } else {
1477 rch++;
1478 continue;
1479 }
1480
1481 iter = cxled_to_port(cxled);
1482 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1483 iter = to_cxl_port(iter->dev.parent);
1484
1485 /*
1486 * Descend the topology tree programming / validating
1487 * targets while looking for conflicts.
1488 */
1489 for (ep = cxl_ep_load(iter, cxlmd); iter;
1490 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1491 rc = cxl_port_setup_targets(iter, cxlr, cxled);
1492 if (rc) {
1493 cxl_region_teardown_targets(cxlr);
1494 return rc;
1495 }
1496 }
1497 }
1498
1499 if (rch && vh) {
1500 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1501 cxl_region_teardown_targets(cxlr);
1502 return -ENXIO;
1503 }
1504
1505 return 0;
1506 }
1507
cxl_region_validate_position(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1508 static int cxl_region_validate_position(struct cxl_region *cxlr,
1509 struct cxl_endpoint_decoder *cxled,
1510 int pos)
1511 {
1512 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1513 struct cxl_region_params *p = &cxlr->params;
1514 int i;
1515
1516 if (pos < 0 || pos >= p->interleave_ways) {
1517 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1518 p->interleave_ways);
1519 return -ENXIO;
1520 }
1521
1522 if (p->targets[pos] == cxled)
1523 return 0;
1524
1525 if (p->targets[pos]) {
1526 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1527 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1528
1529 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1530 pos, dev_name(&cxlmd_target->dev),
1531 dev_name(&cxled_target->cxld.dev));
1532 return -EBUSY;
1533 }
1534
1535 for (i = 0; i < p->interleave_ways; i++) {
1536 struct cxl_endpoint_decoder *cxled_target;
1537 struct cxl_memdev *cxlmd_target;
1538
1539 cxled_target = p->targets[i];
1540 if (!cxled_target)
1541 continue;
1542
1543 cxlmd_target = cxled_to_memdev(cxled_target);
1544 if (cxlmd_target == cxlmd) {
1545 dev_dbg(&cxlr->dev,
1546 "%s already specified at position %d via: %s\n",
1547 dev_name(&cxlmd->dev), pos,
1548 dev_name(&cxled_target->cxld.dev));
1549 return -EBUSY;
1550 }
1551 }
1552
1553 return 0;
1554 }
1555
cxl_region_attach_position(struct cxl_region * cxlr,struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled,const struct cxl_dport * dport,int pos)1556 static int cxl_region_attach_position(struct cxl_region *cxlr,
1557 struct cxl_root_decoder *cxlrd,
1558 struct cxl_endpoint_decoder *cxled,
1559 const struct cxl_dport *dport, int pos)
1560 {
1561 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1562 struct cxl_port *iter;
1563 int rc;
1564
1565 if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1566 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1567 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1568 dev_name(&cxlrd->cxlsd.cxld.dev));
1569 return -ENXIO;
1570 }
1571
1572 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1573 iter = to_cxl_port(iter->dev.parent)) {
1574 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1575 if (rc)
1576 goto err;
1577 }
1578
1579 return 0;
1580
1581 err:
1582 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1583 iter = to_cxl_port(iter->dev.parent))
1584 cxl_port_detach_region(iter, cxlr, cxled);
1585 return rc;
1586 }
1587
cxl_region_attach_auto(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1588 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1589 struct cxl_endpoint_decoder *cxled, int pos)
1590 {
1591 struct cxl_region_params *p = &cxlr->params;
1592
1593 if (cxled->state != CXL_DECODER_STATE_AUTO) {
1594 dev_err(&cxlr->dev,
1595 "%s: unable to add decoder to autodetected region\n",
1596 dev_name(&cxled->cxld.dev));
1597 return -EINVAL;
1598 }
1599
1600 if (pos >= 0) {
1601 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1602 dev_name(&cxled->cxld.dev), pos);
1603 return -EINVAL;
1604 }
1605
1606 if (p->nr_targets >= p->interleave_ways) {
1607 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1608 dev_name(&cxled->cxld.dev));
1609 return -ENXIO;
1610 }
1611
1612 /*
1613 * Temporarily record the endpoint decoder into the target array. Yes,
1614 * this means that userspace can view devices in the wrong position
1615 * before the region activates, and must be careful to understand when
1616 * it might be racing region autodiscovery.
1617 */
1618 pos = p->nr_targets;
1619 p->targets[pos] = cxled;
1620 cxled->pos = pos;
1621 p->nr_targets++;
1622
1623 return 0;
1624 }
1625
cmp_interleave_pos(const void * a,const void * b)1626 static int cmp_interleave_pos(const void *a, const void *b)
1627 {
1628 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1629 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1630
1631 return cxled_a->pos - cxled_b->pos;
1632 }
1633
next_port(struct cxl_port * port)1634 static struct cxl_port *next_port(struct cxl_port *port)
1635 {
1636 if (!port->parent_dport)
1637 return NULL;
1638 return port->parent_dport->port;
1639 }
1640
match_switch_decoder_by_range(struct device * dev,void * data)1641 static int match_switch_decoder_by_range(struct device *dev, void *data)
1642 {
1643 struct cxl_switch_decoder *cxlsd;
1644 struct range *r1, *r2 = data;
1645
1646 if (!is_switch_decoder(dev))
1647 return 0;
1648
1649 cxlsd = to_cxl_switch_decoder(dev);
1650 r1 = &cxlsd->cxld.hpa_range;
1651
1652 if (is_root_decoder(dev))
1653 return range_contains(r1, r2);
1654 return (r1->start == r2->start && r1->end == r2->end);
1655 }
1656
find_pos_and_ways(struct cxl_port * port,struct range * range,int * pos,int * ways)1657 static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1658 int *pos, int *ways)
1659 {
1660 struct cxl_switch_decoder *cxlsd;
1661 struct cxl_port *parent;
1662 struct device *dev;
1663 int rc = -ENXIO;
1664
1665 parent = next_port(port);
1666 if (!parent)
1667 return rc;
1668
1669 dev = device_find_child(&parent->dev, range,
1670 match_switch_decoder_by_range);
1671 if (!dev) {
1672 dev_err(port->uport_dev,
1673 "failed to find decoder mapping %#llx-%#llx\n",
1674 range->start, range->end);
1675 return rc;
1676 }
1677 cxlsd = to_cxl_switch_decoder(dev);
1678 *ways = cxlsd->cxld.interleave_ways;
1679
1680 for (int i = 0; i < *ways; i++) {
1681 if (cxlsd->target[i] == port->parent_dport) {
1682 *pos = i;
1683 rc = 0;
1684 break;
1685 }
1686 }
1687 put_device(dev);
1688
1689 return rc;
1690 }
1691
1692 /**
1693 * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1694 * @cxled: endpoint decoder member of given region
1695 *
1696 * The endpoint position is calculated by traversing the topology from
1697 * the endpoint to the root decoder and iteratively applying this
1698 * calculation:
1699 *
1700 * position = position * parent_ways + parent_pos;
1701 *
1702 * ...where @position is inferred from switch and root decoder target lists.
1703 *
1704 * Return: position >= 0 on success
1705 * -ENXIO on failure
1706 */
cxl_calc_interleave_pos(struct cxl_endpoint_decoder * cxled)1707 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1708 {
1709 struct cxl_port *iter, *port = cxled_to_port(cxled);
1710 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1711 struct range *range = &cxled->cxld.hpa_range;
1712 int parent_ways = 0, parent_pos = 0, pos = 0;
1713 int rc;
1714
1715 /*
1716 * Example: the expected interleave order of the 4-way region shown
1717 * below is: mem0, mem2, mem1, mem3
1718 *
1719 * root_port
1720 * / \
1721 * host_bridge_0 host_bridge_1
1722 * | | | |
1723 * mem0 mem1 mem2 mem3
1724 *
1725 * In the example the calculator will iterate twice. The first iteration
1726 * uses the mem position in the host-bridge and the ways of the host-
1727 * bridge to generate the first, or local, position. The second
1728 * iteration uses the host-bridge position in the root_port and the ways
1729 * of the root_port to refine the position.
1730 *
1731 * A trace of the calculation per endpoint looks like this:
1732 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0
1733 * pos = 0 * 2 + 0 pos = 0 * 2 + 1
1734 * pos: 0 pos: 1
1735 *
1736 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1
1737 * pos = 1 * 2 + 0 pos = 1 * 2 + 1
1738 * pos: 2 pos = 3
1739 *
1740 * Note that while this example is simple, the method applies to more
1741 * complex topologies, including those with switches.
1742 */
1743
1744 /* Iterate from endpoint to root_port refining the position */
1745 for (iter = port; iter; iter = next_port(iter)) {
1746 if (is_cxl_root(iter))
1747 break;
1748
1749 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1750 if (rc)
1751 return rc;
1752
1753 pos = pos * parent_ways + parent_pos;
1754 }
1755
1756 dev_dbg(&cxlmd->dev,
1757 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1758 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1759 dev_name(&port->dev), range->start, range->end, pos);
1760
1761 return pos;
1762 }
1763
cxl_region_sort_targets(struct cxl_region * cxlr)1764 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1765 {
1766 struct cxl_region_params *p = &cxlr->params;
1767 int i, rc = 0;
1768
1769 for (i = 0; i < p->nr_targets; i++) {
1770 struct cxl_endpoint_decoder *cxled = p->targets[i];
1771
1772 cxled->pos = cxl_calc_interleave_pos(cxled);
1773 /*
1774 * Record that sorting failed, but still continue to calc
1775 * cxled->pos so that follow-on code paths can reliably
1776 * do p->targets[cxled->pos] to self-reference their entry.
1777 */
1778 if (cxled->pos < 0)
1779 rc = -ENXIO;
1780 }
1781 /* Keep the cxlr target list in interleave position order */
1782 sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1783 cmp_interleave_pos, NULL);
1784
1785 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1786 return rc;
1787 }
1788
cxl_region_attach(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos)1789 static int cxl_region_attach(struct cxl_region *cxlr,
1790 struct cxl_endpoint_decoder *cxled, int pos)
1791 {
1792 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1793 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1794 struct cxl_region_params *p = &cxlr->params;
1795 struct cxl_port *ep_port, *root_port;
1796 struct cxl_dport *dport;
1797 int rc = -ENXIO;
1798
1799 if (cxled->mode != cxlr->mode) {
1800 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1801 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1802 return -EINVAL;
1803 }
1804
1805 if (cxled->mode == CXL_DECODER_DEAD) {
1806 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1807 return -ENODEV;
1808 }
1809
1810 /* all full of members, or interleave config not established? */
1811 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1812 dev_dbg(&cxlr->dev, "region already active\n");
1813 return -EBUSY;
1814 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1815 dev_dbg(&cxlr->dev, "interleave config missing\n");
1816 return -ENXIO;
1817 }
1818
1819 if (p->nr_targets >= p->interleave_ways) {
1820 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1821 p->nr_targets);
1822 return -EINVAL;
1823 }
1824
1825 ep_port = cxled_to_port(cxled);
1826 root_port = cxlrd_to_port(cxlrd);
1827 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1828 if (!dport) {
1829 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1830 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1831 dev_name(cxlr->dev.parent));
1832 return -ENXIO;
1833 }
1834
1835 if (cxled->cxld.target_type != cxlr->type) {
1836 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1837 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1838 cxled->cxld.target_type, cxlr->type);
1839 return -ENXIO;
1840 }
1841
1842 if (!cxled->dpa_res) {
1843 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1844 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1845 return -ENXIO;
1846 }
1847
1848 if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1849 resource_size(p->res)) {
1850 dev_dbg(&cxlr->dev,
1851 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1852 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1853 (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1854 (u64)resource_size(p->res));
1855 return -EINVAL;
1856 }
1857
1858 cxl_region_perf_data_calculate(cxlr, cxled);
1859
1860 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1861 int i;
1862
1863 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1864 if (rc)
1865 return rc;
1866
1867 /* await more targets to arrive... */
1868 if (p->nr_targets < p->interleave_ways)
1869 return 0;
1870
1871 /*
1872 * All targets are here, which implies all PCI enumeration that
1873 * affects this region has been completed. Walk the topology to
1874 * sort the devices into their relative region decode position.
1875 */
1876 rc = cxl_region_sort_targets(cxlr);
1877 if (rc)
1878 return rc;
1879
1880 for (i = 0; i < p->nr_targets; i++) {
1881 cxled = p->targets[i];
1882 ep_port = cxled_to_port(cxled);
1883 dport = cxl_find_dport_by_dev(root_port,
1884 ep_port->host_bridge);
1885 rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1886 dport, i);
1887 if (rc)
1888 return rc;
1889 }
1890
1891 rc = cxl_region_setup_targets(cxlr);
1892 if (rc)
1893 return rc;
1894
1895 /*
1896 * If target setup succeeds in the autodiscovery case
1897 * then the region is already committed.
1898 */
1899 p->state = CXL_CONFIG_COMMIT;
1900
1901 return 0;
1902 }
1903
1904 rc = cxl_region_validate_position(cxlr, cxled, pos);
1905 if (rc)
1906 return rc;
1907
1908 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1909 if (rc)
1910 return rc;
1911
1912 p->targets[pos] = cxled;
1913 cxled->pos = pos;
1914 p->nr_targets++;
1915
1916 if (p->nr_targets == p->interleave_ways) {
1917 rc = cxl_region_setup_targets(cxlr);
1918 if (rc)
1919 return rc;
1920 p->state = CXL_CONFIG_ACTIVE;
1921 }
1922
1923 cxled->cxld.interleave_ways = p->interleave_ways;
1924 cxled->cxld.interleave_granularity = p->interleave_granularity;
1925 cxled->cxld.hpa_range = (struct range) {
1926 .start = p->res->start,
1927 .end = p->res->end,
1928 };
1929
1930 if (p->nr_targets != p->interleave_ways)
1931 return 0;
1932
1933 /*
1934 * Test the auto-discovery position calculator function
1935 * against this successfully created user-defined region.
1936 * A fail message here means that this interleave config
1937 * will fail when presented as CXL_REGION_F_AUTO.
1938 */
1939 for (int i = 0; i < p->nr_targets; i++) {
1940 struct cxl_endpoint_decoder *cxled = p->targets[i];
1941 int test_pos;
1942
1943 test_pos = cxl_calc_interleave_pos(cxled);
1944 dev_dbg(&cxled->cxld.dev,
1945 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
1946 (test_pos == cxled->pos) ? "success" : "fail",
1947 test_pos, cxled->pos);
1948 }
1949
1950 return 0;
1951 }
1952
cxl_region_detach(struct cxl_endpoint_decoder * cxled)1953 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1954 {
1955 struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1956 struct cxl_region *cxlr = cxled->cxld.region;
1957 struct cxl_region_params *p;
1958 int rc = 0;
1959
1960 lockdep_assert_held_write(&cxl_region_rwsem);
1961
1962 if (!cxlr)
1963 return 0;
1964
1965 p = &cxlr->params;
1966 get_device(&cxlr->dev);
1967
1968 if (p->state > CXL_CONFIG_ACTIVE) {
1969 /*
1970 * TODO: tear down all impacted regions if a device is
1971 * removed out of order
1972 */
1973 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1974 if (rc)
1975 goto out;
1976 p->state = CXL_CONFIG_ACTIVE;
1977 }
1978
1979 for (iter = ep_port; !is_cxl_root(iter);
1980 iter = to_cxl_port(iter->dev.parent))
1981 cxl_port_detach_region(iter, cxlr, cxled);
1982
1983 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1984 p->targets[cxled->pos] != cxled) {
1985 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1986
1987 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1988 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1989 cxled->pos);
1990 goto out;
1991 }
1992
1993 if (p->state == CXL_CONFIG_ACTIVE) {
1994 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1995 cxl_region_teardown_targets(cxlr);
1996 }
1997 p->targets[cxled->pos] = NULL;
1998 p->nr_targets--;
1999 cxled->cxld.hpa_range = (struct range) {
2000 .start = 0,
2001 .end = -1,
2002 };
2003
2004 /* notify the region driver that one of its targets has departed */
2005 up_write(&cxl_region_rwsem);
2006 device_release_driver(&cxlr->dev);
2007 down_write(&cxl_region_rwsem);
2008 out:
2009 put_device(&cxlr->dev);
2010 return rc;
2011 }
2012
cxl_decoder_kill_region(struct cxl_endpoint_decoder * cxled)2013 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
2014 {
2015 down_write(&cxl_region_rwsem);
2016 cxled->mode = CXL_DECODER_DEAD;
2017 cxl_region_detach(cxled);
2018 up_write(&cxl_region_rwsem);
2019 }
2020
attach_target(struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled,int pos,unsigned int state)2021 static int attach_target(struct cxl_region *cxlr,
2022 struct cxl_endpoint_decoder *cxled, int pos,
2023 unsigned int state)
2024 {
2025 int rc = 0;
2026
2027 if (state == TASK_INTERRUPTIBLE)
2028 rc = down_write_killable(&cxl_region_rwsem);
2029 else
2030 down_write(&cxl_region_rwsem);
2031 if (rc)
2032 return rc;
2033
2034 down_read(&cxl_dpa_rwsem);
2035 rc = cxl_region_attach(cxlr, cxled, pos);
2036 up_read(&cxl_dpa_rwsem);
2037 up_write(&cxl_region_rwsem);
2038 return rc;
2039 }
2040
detach_target(struct cxl_region * cxlr,int pos)2041 static int detach_target(struct cxl_region *cxlr, int pos)
2042 {
2043 struct cxl_region_params *p = &cxlr->params;
2044 int rc;
2045
2046 rc = down_write_killable(&cxl_region_rwsem);
2047 if (rc)
2048 return rc;
2049
2050 if (pos >= p->interleave_ways) {
2051 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
2052 p->interleave_ways);
2053 rc = -ENXIO;
2054 goto out;
2055 }
2056
2057 if (!p->targets[pos]) {
2058 rc = 0;
2059 goto out;
2060 }
2061
2062 rc = cxl_region_detach(p->targets[pos]);
2063 out:
2064 up_write(&cxl_region_rwsem);
2065 return rc;
2066 }
2067
store_targetN(struct cxl_region * cxlr,const char * buf,int pos,size_t len)2068 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
2069 size_t len)
2070 {
2071 int rc;
2072
2073 if (sysfs_streq(buf, "\n"))
2074 rc = detach_target(cxlr, pos);
2075 else {
2076 struct device *dev;
2077
2078 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
2079 if (!dev)
2080 return -ENODEV;
2081
2082 if (!is_endpoint_decoder(dev)) {
2083 rc = -EINVAL;
2084 goto out;
2085 }
2086
2087 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
2088 TASK_INTERRUPTIBLE);
2089 out:
2090 put_device(dev);
2091 }
2092
2093 if (rc < 0)
2094 return rc;
2095 return len;
2096 }
2097
2098 #define TARGET_ATTR_RW(n) \
2099 static ssize_t target##n##_show( \
2100 struct device *dev, struct device_attribute *attr, char *buf) \
2101 { \
2102 return show_targetN(to_cxl_region(dev), buf, (n)); \
2103 } \
2104 static ssize_t target##n##_store(struct device *dev, \
2105 struct device_attribute *attr, \
2106 const char *buf, size_t len) \
2107 { \
2108 return store_targetN(to_cxl_region(dev), buf, (n), len); \
2109 } \
2110 static DEVICE_ATTR_RW(target##n)
2111
2112 TARGET_ATTR_RW(0);
2113 TARGET_ATTR_RW(1);
2114 TARGET_ATTR_RW(2);
2115 TARGET_ATTR_RW(3);
2116 TARGET_ATTR_RW(4);
2117 TARGET_ATTR_RW(5);
2118 TARGET_ATTR_RW(6);
2119 TARGET_ATTR_RW(7);
2120 TARGET_ATTR_RW(8);
2121 TARGET_ATTR_RW(9);
2122 TARGET_ATTR_RW(10);
2123 TARGET_ATTR_RW(11);
2124 TARGET_ATTR_RW(12);
2125 TARGET_ATTR_RW(13);
2126 TARGET_ATTR_RW(14);
2127 TARGET_ATTR_RW(15);
2128
2129 static struct attribute *target_attrs[] = {
2130 &dev_attr_target0.attr,
2131 &dev_attr_target1.attr,
2132 &dev_attr_target2.attr,
2133 &dev_attr_target3.attr,
2134 &dev_attr_target4.attr,
2135 &dev_attr_target5.attr,
2136 &dev_attr_target6.attr,
2137 &dev_attr_target7.attr,
2138 &dev_attr_target8.attr,
2139 &dev_attr_target9.attr,
2140 &dev_attr_target10.attr,
2141 &dev_attr_target11.attr,
2142 &dev_attr_target12.attr,
2143 &dev_attr_target13.attr,
2144 &dev_attr_target14.attr,
2145 &dev_attr_target15.attr,
2146 NULL,
2147 };
2148
cxl_region_target_visible(struct kobject * kobj,struct attribute * a,int n)2149 static umode_t cxl_region_target_visible(struct kobject *kobj,
2150 struct attribute *a, int n)
2151 {
2152 struct device *dev = kobj_to_dev(kobj);
2153 struct cxl_region *cxlr = to_cxl_region(dev);
2154 struct cxl_region_params *p = &cxlr->params;
2155
2156 if (n < p->interleave_ways)
2157 return a->mode;
2158 return 0;
2159 }
2160
2161 static const struct attribute_group cxl_region_target_group = {
2162 .attrs = target_attrs,
2163 .is_visible = cxl_region_target_visible,
2164 };
2165
get_cxl_region_target_group(void)2166 static const struct attribute_group *get_cxl_region_target_group(void)
2167 {
2168 return &cxl_region_target_group;
2169 }
2170
2171 static const struct attribute_group *region_groups[] = {
2172 &cxl_base_attribute_group,
2173 &cxl_region_group,
2174 &cxl_region_target_group,
2175 &cxl_region_access0_coordinate_group,
2176 &cxl_region_access1_coordinate_group,
2177 NULL,
2178 };
2179
cxl_region_release(struct device * dev)2180 static void cxl_region_release(struct device *dev)
2181 {
2182 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2183 struct cxl_region *cxlr = to_cxl_region(dev);
2184 int id = atomic_read(&cxlrd->region_id);
2185
2186 /*
2187 * Try to reuse the recently idled id rather than the cached
2188 * next id to prevent the region id space from increasing
2189 * unnecessarily.
2190 */
2191 if (cxlr->id < id)
2192 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2193 memregion_free(id);
2194 goto out;
2195 }
2196
2197 memregion_free(cxlr->id);
2198 out:
2199 put_device(dev->parent);
2200 kfree(cxlr);
2201 }
2202
2203 const struct device_type cxl_region_type = {
2204 .name = "cxl_region",
2205 .release = cxl_region_release,
2206 .groups = region_groups
2207 };
2208
is_cxl_region(struct device * dev)2209 bool is_cxl_region(struct device *dev)
2210 {
2211 return dev->type == &cxl_region_type;
2212 }
2213 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2214
to_cxl_region(struct device * dev)2215 static struct cxl_region *to_cxl_region(struct device *dev)
2216 {
2217 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2218 "not a cxl_region device\n"))
2219 return NULL;
2220
2221 return container_of(dev, struct cxl_region, dev);
2222 }
2223
unregister_region(void * _cxlr)2224 static void unregister_region(void *_cxlr)
2225 {
2226 struct cxl_region *cxlr = _cxlr;
2227 struct cxl_region_params *p = &cxlr->params;
2228 int i;
2229
2230 unregister_memory_notifier(&cxlr->memory_notifier);
2231 device_del(&cxlr->dev);
2232
2233 /*
2234 * Now that region sysfs is shutdown, the parameter block is now
2235 * read-only, so no need to hold the region rwsem to access the
2236 * region parameters.
2237 */
2238 for (i = 0; i < p->interleave_ways; i++)
2239 detach_target(cxlr, i);
2240
2241 cxl_region_iomem_release(cxlr);
2242 put_device(&cxlr->dev);
2243 }
2244
2245 static struct lock_class_key cxl_region_key;
2246
cxl_region_alloc(struct cxl_root_decoder * cxlrd,int id)2247 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2248 {
2249 struct cxl_region *cxlr;
2250 struct device *dev;
2251
2252 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2253 if (!cxlr) {
2254 memregion_free(id);
2255 return ERR_PTR(-ENOMEM);
2256 }
2257
2258 dev = &cxlr->dev;
2259 device_initialize(dev);
2260 lockdep_set_class(&dev->mutex, &cxl_region_key);
2261 dev->parent = &cxlrd->cxlsd.cxld.dev;
2262 /*
2263 * Keep root decoder pinned through cxl_region_release to fixup
2264 * region id allocations
2265 */
2266 get_device(dev->parent);
2267 device_set_pm_not_required(dev);
2268 dev->bus = &cxl_bus_type;
2269 dev->type = &cxl_region_type;
2270 cxlr->id = id;
2271
2272 return cxlr;
2273 }
2274
cxl_region_update_coordinates(struct cxl_region * cxlr,int nid)2275 static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
2276 {
2277 int cset = 0;
2278 int rc;
2279
2280 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
2281 if (cxlr->coord[i].read_bandwidth) {
2282 rc = 0;
2283 if (cxl_need_node_perf_attrs_update(nid))
2284 node_set_perf_attrs(nid, &cxlr->coord[i], i);
2285 else
2286 rc = cxl_update_hmat_access_coordinates(nid, cxlr, i);
2287
2288 if (rc == 0)
2289 cset++;
2290 }
2291 }
2292
2293 if (!cset)
2294 return false;
2295
2296 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group());
2297 if (rc)
2298 dev_dbg(&cxlr->dev, "Failed to update access0 group\n");
2299
2300 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group());
2301 if (rc)
2302 dev_dbg(&cxlr->dev, "Failed to update access1 group\n");
2303
2304 return true;
2305 }
2306
cxl_region_perf_attrs_callback(struct notifier_block * nb,unsigned long action,void * arg)2307 static int cxl_region_perf_attrs_callback(struct notifier_block *nb,
2308 unsigned long action, void *arg)
2309 {
2310 struct cxl_region *cxlr = container_of(nb, struct cxl_region,
2311 memory_notifier);
2312 struct cxl_region_params *p = &cxlr->params;
2313 struct cxl_endpoint_decoder *cxled = p->targets[0];
2314 struct cxl_decoder *cxld = &cxled->cxld;
2315 struct memory_notify *mnb = arg;
2316 int nid = mnb->status_change_nid;
2317 int region_nid;
2318
2319 if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
2320 return NOTIFY_DONE;
2321
2322 region_nid = phys_to_target_node(cxld->hpa_range.start);
2323 if (nid != region_nid)
2324 return NOTIFY_DONE;
2325
2326 if (!cxl_region_update_coordinates(cxlr, nid))
2327 return NOTIFY_DONE;
2328
2329 return NOTIFY_OK;
2330 }
2331
2332 /**
2333 * devm_cxl_add_region - Adds a region to a decoder
2334 * @cxlrd: root decoder
2335 * @id: memregion id to create, or memregion_free() on failure
2336 * @mode: mode for the endpoint decoders of this region
2337 * @type: select whether this is an expander or accelerator (type-2 or type-3)
2338 *
2339 * This is the second step of region initialization. Regions exist within an
2340 * address space which is mapped by a @cxlrd.
2341 *
2342 * Return: 0 if the region was added to the @cxlrd, else returns negative error
2343 * code. The region will be named "regionZ" where Z is the unique region number.
2344 */
devm_cxl_add_region(struct cxl_root_decoder * cxlrd,int id,enum cxl_decoder_mode mode,enum cxl_decoder_type type)2345 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2346 int id,
2347 enum cxl_decoder_mode mode,
2348 enum cxl_decoder_type type)
2349 {
2350 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2351 struct cxl_region *cxlr;
2352 struct device *dev;
2353 int rc;
2354
2355 cxlr = cxl_region_alloc(cxlrd, id);
2356 if (IS_ERR(cxlr))
2357 return cxlr;
2358 cxlr->mode = mode;
2359 cxlr->type = type;
2360
2361 dev = &cxlr->dev;
2362 rc = dev_set_name(dev, "region%d", id);
2363 if (rc)
2364 goto err;
2365
2366 rc = device_add(dev);
2367 if (rc)
2368 goto err;
2369
2370 cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback;
2371 cxlr->memory_notifier.priority = CXL_CALLBACK_PRI;
2372 register_memory_notifier(&cxlr->memory_notifier);
2373
2374 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2375 if (rc)
2376 return ERR_PTR(rc);
2377
2378 dev_dbg(port->uport_dev, "%s: created %s\n",
2379 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2380 return cxlr;
2381
2382 err:
2383 put_device(dev);
2384 return ERR_PTR(rc);
2385 }
2386
__create_region_show(struct cxl_root_decoder * cxlrd,char * buf)2387 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2388 {
2389 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2390 }
2391
create_pmem_region_show(struct device * dev,struct device_attribute * attr,char * buf)2392 static ssize_t create_pmem_region_show(struct device *dev,
2393 struct device_attribute *attr, char *buf)
2394 {
2395 return __create_region_show(to_cxl_root_decoder(dev), buf);
2396 }
2397
create_ram_region_show(struct device * dev,struct device_attribute * attr,char * buf)2398 static ssize_t create_ram_region_show(struct device *dev,
2399 struct device_attribute *attr, char *buf)
2400 {
2401 return __create_region_show(to_cxl_root_decoder(dev), buf);
2402 }
2403
__create_region(struct cxl_root_decoder * cxlrd,enum cxl_decoder_mode mode,int id)2404 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2405 enum cxl_decoder_mode mode, int id)
2406 {
2407 int rc;
2408
2409 switch (mode) {
2410 case CXL_DECODER_RAM:
2411 case CXL_DECODER_PMEM:
2412 break;
2413 default:
2414 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2415 return ERR_PTR(-EINVAL);
2416 }
2417
2418 rc = memregion_alloc(GFP_KERNEL);
2419 if (rc < 0)
2420 return ERR_PTR(rc);
2421
2422 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2423 memregion_free(rc);
2424 return ERR_PTR(-EBUSY);
2425 }
2426
2427 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2428 }
2429
create_pmem_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2430 static ssize_t create_pmem_region_store(struct device *dev,
2431 struct device_attribute *attr,
2432 const char *buf, size_t len)
2433 {
2434 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2435 struct cxl_region *cxlr;
2436 int rc, id;
2437
2438 rc = sscanf(buf, "region%d\n", &id);
2439 if (rc != 1)
2440 return -EINVAL;
2441
2442 cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2443 if (IS_ERR(cxlr))
2444 return PTR_ERR(cxlr);
2445
2446 return len;
2447 }
2448 DEVICE_ATTR_RW(create_pmem_region);
2449
create_ram_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2450 static ssize_t create_ram_region_store(struct device *dev,
2451 struct device_attribute *attr,
2452 const char *buf, size_t len)
2453 {
2454 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2455 struct cxl_region *cxlr;
2456 int rc, id;
2457
2458 rc = sscanf(buf, "region%d\n", &id);
2459 if (rc != 1)
2460 return -EINVAL;
2461
2462 cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2463 if (IS_ERR(cxlr))
2464 return PTR_ERR(cxlr);
2465
2466 return len;
2467 }
2468 DEVICE_ATTR_RW(create_ram_region);
2469
region_show(struct device * dev,struct device_attribute * attr,char * buf)2470 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2471 char *buf)
2472 {
2473 struct cxl_decoder *cxld = to_cxl_decoder(dev);
2474 ssize_t rc;
2475
2476 rc = down_read_interruptible(&cxl_region_rwsem);
2477 if (rc)
2478 return rc;
2479
2480 if (cxld->region)
2481 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2482 else
2483 rc = sysfs_emit(buf, "\n");
2484 up_read(&cxl_region_rwsem);
2485
2486 return rc;
2487 }
2488 DEVICE_ATTR_RO(region);
2489
2490 static struct cxl_region *
cxl_find_region_by_name(struct cxl_root_decoder * cxlrd,const char * name)2491 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2492 {
2493 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2494 struct device *region_dev;
2495
2496 region_dev = device_find_child_by_name(&cxld->dev, name);
2497 if (!region_dev)
2498 return ERR_PTR(-ENODEV);
2499
2500 return to_cxl_region(region_dev);
2501 }
2502
delete_region_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)2503 static ssize_t delete_region_store(struct device *dev,
2504 struct device_attribute *attr,
2505 const char *buf, size_t len)
2506 {
2507 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2508 struct cxl_port *port = to_cxl_port(dev->parent);
2509 struct cxl_region *cxlr;
2510
2511 cxlr = cxl_find_region_by_name(cxlrd, buf);
2512 if (IS_ERR(cxlr))
2513 return PTR_ERR(cxlr);
2514
2515 devm_release_action(port->uport_dev, unregister_region, cxlr);
2516 put_device(&cxlr->dev);
2517
2518 return len;
2519 }
2520 DEVICE_ATTR_WO(delete_region);
2521
cxl_pmem_region_release(struct device * dev)2522 static void cxl_pmem_region_release(struct device *dev)
2523 {
2524 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2525 int i;
2526
2527 for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2528 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2529
2530 put_device(&cxlmd->dev);
2531 }
2532
2533 kfree(cxlr_pmem);
2534 }
2535
2536 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2537 &cxl_base_attribute_group,
2538 NULL,
2539 };
2540
2541 const struct device_type cxl_pmem_region_type = {
2542 .name = "cxl_pmem_region",
2543 .release = cxl_pmem_region_release,
2544 .groups = cxl_pmem_region_attribute_groups,
2545 };
2546
is_cxl_pmem_region(struct device * dev)2547 bool is_cxl_pmem_region(struct device *dev)
2548 {
2549 return dev->type == &cxl_pmem_region_type;
2550 }
2551 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2552
to_cxl_pmem_region(struct device * dev)2553 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2554 {
2555 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2556 "not a cxl_pmem_region device\n"))
2557 return NULL;
2558 return container_of(dev, struct cxl_pmem_region, dev);
2559 }
2560 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2561
2562 struct cxl_poison_context {
2563 struct cxl_port *port;
2564 enum cxl_decoder_mode mode;
2565 u64 offset;
2566 };
2567
cxl_get_poison_unmapped(struct cxl_memdev * cxlmd,struct cxl_poison_context * ctx)2568 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2569 struct cxl_poison_context *ctx)
2570 {
2571 struct cxl_dev_state *cxlds = cxlmd->cxlds;
2572 u64 offset, length;
2573 int rc = 0;
2574
2575 /*
2576 * Collect poison for the remaining unmapped resources
2577 * after poison is collected by committed endpoints.
2578 *
2579 * Knowing that PMEM must always follow RAM, get poison
2580 * for unmapped resources based on the last decoder's mode:
2581 * ram: scan remains of ram range, then any pmem range
2582 * pmem: scan remains of pmem range
2583 */
2584
2585 if (ctx->mode == CXL_DECODER_RAM) {
2586 offset = ctx->offset;
2587 length = resource_size(&cxlds->ram_res) - offset;
2588 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2589 if (rc == -EFAULT)
2590 rc = 0;
2591 if (rc)
2592 return rc;
2593 }
2594 if (ctx->mode == CXL_DECODER_PMEM) {
2595 offset = ctx->offset;
2596 length = resource_size(&cxlds->dpa_res) - offset;
2597 if (!length)
2598 return 0;
2599 } else if (resource_size(&cxlds->pmem_res)) {
2600 offset = cxlds->pmem_res.start;
2601 length = resource_size(&cxlds->pmem_res);
2602 } else {
2603 return 0;
2604 }
2605
2606 return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2607 }
2608
poison_by_decoder(struct device * dev,void * arg)2609 static int poison_by_decoder(struct device *dev, void *arg)
2610 {
2611 struct cxl_poison_context *ctx = arg;
2612 struct cxl_endpoint_decoder *cxled;
2613 struct cxl_memdev *cxlmd;
2614 u64 offset, length;
2615 int rc = 0;
2616
2617 if (!is_endpoint_decoder(dev))
2618 return rc;
2619
2620 cxled = to_cxl_endpoint_decoder(dev);
2621 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2622 return rc;
2623
2624 /*
2625 * Regions are only created with single mode decoders: pmem or ram.
2626 * Linux does not support mixed mode decoders. This means that
2627 * reading poison per endpoint decoder adheres to the requirement
2628 * that poison reads of pmem and ram must be separated.
2629 * CXL 3.0 Spec 8.2.9.8.4.1
2630 */
2631 if (cxled->mode == CXL_DECODER_MIXED) {
2632 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2633 return rc;
2634 }
2635
2636 cxlmd = cxled_to_memdev(cxled);
2637 if (cxled->skip) {
2638 offset = cxled->dpa_res->start - cxled->skip;
2639 length = cxled->skip;
2640 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2641 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2642 rc = 0;
2643 if (rc)
2644 return rc;
2645 }
2646
2647 offset = cxled->dpa_res->start;
2648 length = cxled->dpa_res->end - offset + 1;
2649 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2650 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2651 rc = 0;
2652 if (rc)
2653 return rc;
2654
2655 /* Iterate until commit_end is reached */
2656 if (cxled->cxld.id == ctx->port->commit_end) {
2657 ctx->offset = cxled->dpa_res->end + 1;
2658 ctx->mode = cxled->mode;
2659 return 1;
2660 }
2661
2662 return 0;
2663 }
2664
cxl_get_poison_by_endpoint(struct cxl_port * port)2665 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2666 {
2667 struct cxl_poison_context ctx;
2668 int rc = 0;
2669
2670 ctx = (struct cxl_poison_context) {
2671 .port = port
2672 };
2673
2674 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2675 if (rc == 1)
2676 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2677 &ctx);
2678
2679 return rc;
2680 }
2681
2682 struct cxl_dpa_to_region_context {
2683 struct cxl_region *cxlr;
2684 u64 dpa;
2685 };
2686
__cxl_dpa_to_region(struct device * dev,void * arg)2687 static int __cxl_dpa_to_region(struct device *dev, void *arg)
2688 {
2689 struct cxl_dpa_to_region_context *ctx = arg;
2690 struct cxl_endpoint_decoder *cxled;
2691 u64 dpa = ctx->dpa;
2692
2693 if (!is_endpoint_decoder(dev))
2694 return 0;
2695
2696 cxled = to_cxl_endpoint_decoder(dev);
2697 if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2698 return 0;
2699
2700 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start)
2701 return 0;
2702
2703 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa,
2704 dev_name(&cxled->cxld.region->dev));
2705
2706 ctx->cxlr = cxled->cxld.region;
2707
2708 return 1;
2709 }
2710
cxl_dpa_to_region(const struct cxl_memdev * cxlmd,u64 dpa)2711 struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa)
2712 {
2713 struct cxl_dpa_to_region_context ctx;
2714 struct cxl_port *port;
2715
2716 ctx = (struct cxl_dpa_to_region_context) {
2717 .dpa = dpa,
2718 };
2719 port = cxlmd->endpoint;
2720 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port))
2721 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region);
2722
2723 return ctx.cxlr;
2724 }
2725
cxl_is_hpa_in_range(u64 hpa,struct cxl_region * cxlr,int pos)2726 static bool cxl_is_hpa_in_range(u64 hpa, struct cxl_region *cxlr, int pos)
2727 {
2728 struct cxl_region_params *p = &cxlr->params;
2729 int gran = p->interleave_granularity;
2730 int ways = p->interleave_ways;
2731 u64 offset;
2732
2733 /* Is the hpa within this region at all */
2734 if (hpa < p->res->start || hpa > p->res->end) {
2735 dev_dbg(&cxlr->dev,
2736 "Addr trans fail: hpa 0x%llx not in region\n", hpa);
2737 return false;
2738 }
2739
2740 /* Is the hpa in an expected chunk for its pos(-ition) */
2741 offset = hpa - p->res->start;
2742 offset = do_div(offset, gran * ways);
2743 if ((offset >= pos * gran) && (offset < (pos + 1) * gran))
2744 return true;
2745
2746 dev_dbg(&cxlr->dev,
2747 "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa);
2748
2749 return false;
2750 }
2751
cxl_dpa_to_hpa(u64 dpa,struct cxl_region * cxlr,struct cxl_endpoint_decoder * cxled)2752 static u64 cxl_dpa_to_hpa(u64 dpa, struct cxl_region *cxlr,
2753 struct cxl_endpoint_decoder *cxled)
2754 {
2755 u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa;
2756 struct cxl_region_params *p = &cxlr->params;
2757 int pos = cxled->pos;
2758 u16 eig = 0;
2759 u8 eiw = 0;
2760
2761 ways_to_eiw(p->interleave_ways, &eiw);
2762 granularity_to_eig(p->interleave_granularity, &eig);
2763
2764 /*
2765 * The device position in the region interleave set was removed
2766 * from the offset at HPA->DPA translation. To reconstruct the
2767 * HPA, place the 'pos' in the offset.
2768 *
2769 * The placement of 'pos' in the HPA is determined by interleave
2770 * ways and granularity and is defined in the CXL Spec 3.0 Section
2771 * 8.2.4.19.13 Implementation Note: Device Decode Logic
2772 */
2773
2774 /* Remove the dpa base */
2775 dpa_offset = dpa - cxl_dpa_resource_start(cxled);
2776
2777 mask_upper = GENMASK_ULL(51, eig + 8);
2778
2779 if (eiw < 8) {
2780 hpa_offset = (dpa_offset & mask_upper) << eiw;
2781 hpa_offset |= pos << (eig + 8);
2782 } else {
2783 bits_upper = (dpa_offset & mask_upper) >> (eig + 8);
2784 bits_upper = bits_upper * 3;
2785 hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8);
2786 }
2787
2788 /* The lower bits remain unchanged */
2789 hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0);
2790
2791 /* Apply the hpa_offset to the region base address */
2792 hpa = hpa_offset + p->res->start;
2793
2794 if (!cxl_is_hpa_in_range(hpa, cxlr, cxled->pos))
2795 return ULLONG_MAX;
2796
2797 return hpa;
2798 }
2799
cxl_trace_hpa(struct cxl_region * cxlr,const struct cxl_memdev * cxlmd,u64 dpa)2800 u64 cxl_trace_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd,
2801 u64 dpa)
2802 {
2803 struct cxl_region_params *p = &cxlr->params;
2804 struct cxl_endpoint_decoder *cxled = NULL;
2805
2806 for (int i = 0; i < p->nr_targets; i++) {
2807 cxled = p->targets[i];
2808 if (cxlmd == cxled_to_memdev(cxled))
2809 break;
2810 }
2811 if (!cxled || cxlmd != cxled_to_memdev(cxled))
2812 return ULLONG_MAX;
2813
2814 return cxl_dpa_to_hpa(dpa, cxlr, cxled);
2815 }
2816
2817 static struct lock_class_key cxl_pmem_region_key;
2818
cxl_pmem_region_alloc(struct cxl_region * cxlr)2819 static int cxl_pmem_region_alloc(struct cxl_region *cxlr)
2820 {
2821 struct cxl_region_params *p = &cxlr->params;
2822 struct cxl_nvdimm_bridge *cxl_nvb;
2823 struct device *dev;
2824 int i;
2825
2826 guard(rwsem_read)(&cxl_region_rwsem);
2827 if (p->state != CXL_CONFIG_COMMIT)
2828 return -ENXIO;
2829
2830 struct cxl_pmem_region *cxlr_pmem __free(kfree) =
2831 kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL);
2832 if (!cxlr_pmem)
2833 return -ENOMEM;
2834
2835 cxlr_pmem->hpa_range.start = p->res->start;
2836 cxlr_pmem->hpa_range.end = p->res->end;
2837
2838 /* Snapshot the region configuration underneath the cxl_region_rwsem */
2839 cxlr_pmem->nr_mappings = p->nr_targets;
2840 for (i = 0; i < p->nr_targets; i++) {
2841 struct cxl_endpoint_decoder *cxled = p->targets[i];
2842 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2843 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2844
2845 /*
2846 * Regions never span CXL root devices, so by definition the
2847 * bridge for one device is the same for all.
2848 */
2849 if (i == 0) {
2850 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2851 if (!cxl_nvb)
2852 return -ENODEV;
2853 cxlr->cxl_nvb = cxl_nvb;
2854 }
2855 m->cxlmd = cxlmd;
2856 get_device(&cxlmd->dev);
2857 m->start = cxled->dpa_res->start;
2858 m->size = resource_size(cxled->dpa_res);
2859 m->position = i;
2860 }
2861
2862 dev = &cxlr_pmem->dev;
2863 device_initialize(dev);
2864 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2865 device_set_pm_not_required(dev);
2866 dev->parent = &cxlr->dev;
2867 dev->bus = &cxl_bus_type;
2868 dev->type = &cxl_pmem_region_type;
2869 cxlr_pmem->cxlr = cxlr;
2870 cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem);
2871
2872 return 0;
2873 }
2874
cxl_dax_region_release(struct device * dev)2875 static void cxl_dax_region_release(struct device *dev)
2876 {
2877 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2878
2879 kfree(cxlr_dax);
2880 }
2881
2882 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2883 &cxl_base_attribute_group,
2884 NULL,
2885 };
2886
2887 const struct device_type cxl_dax_region_type = {
2888 .name = "cxl_dax_region",
2889 .release = cxl_dax_region_release,
2890 .groups = cxl_dax_region_attribute_groups,
2891 };
2892
is_cxl_dax_region(struct device * dev)2893 static bool is_cxl_dax_region(struct device *dev)
2894 {
2895 return dev->type == &cxl_dax_region_type;
2896 }
2897
to_cxl_dax_region(struct device * dev)2898 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2899 {
2900 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2901 "not a cxl_dax_region device\n"))
2902 return NULL;
2903 return container_of(dev, struct cxl_dax_region, dev);
2904 }
2905 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2906
2907 static struct lock_class_key cxl_dax_region_key;
2908
cxl_dax_region_alloc(struct cxl_region * cxlr)2909 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2910 {
2911 struct cxl_region_params *p = &cxlr->params;
2912 struct cxl_dax_region *cxlr_dax;
2913 struct device *dev;
2914
2915 down_read(&cxl_region_rwsem);
2916 if (p->state != CXL_CONFIG_COMMIT) {
2917 cxlr_dax = ERR_PTR(-ENXIO);
2918 goto out;
2919 }
2920
2921 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2922 if (!cxlr_dax) {
2923 cxlr_dax = ERR_PTR(-ENOMEM);
2924 goto out;
2925 }
2926
2927 cxlr_dax->hpa_range.start = p->res->start;
2928 cxlr_dax->hpa_range.end = p->res->end;
2929
2930 dev = &cxlr_dax->dev;
2931 cxlr_dax->cxlr = cxlr;
2932 device_initialize(dev);
2933 lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2934 device_set_pm_not_required(dev);
2935 dev->parent = &cxlr->dev;
2936 dev->bus = &cxl_bus_type;
2937 dev->type = &cxl_dax_region_type;
2938 out:
2939 up_read(&cxl_region_rwsem);
2940
2941 return cxlr_dax;
2942 }
2943
cxlr_pmem_unregister(void * _cxlr_pmem)2944 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2945 {
2946 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2947 struct cxl_region *cxlr = cxlr_pmem->cxlr;
2948 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2949
2950 /*
2951 * Either the bridge is in ->remove() context under the device_lock(),
2952 * or cxlr_release_nvdimm() is cancelling the bridge's release action
2953 * for @cxlr_pmem and doing it itself (while manually holding the bridge
2954 * lock).
2955 */
2956 device_lock_assert(&cxl_nvb->dev);
2957 cxlr->cxlr_pmem = NULL;
2958 cxlr_pmem->cxlr = NULL;
2959 device_unregister(&cxlr_pmem->dev);
2960 }
2961
cxlr_release_nvdimm(void * _cxlr)2962 static void cxlr_release_nvdimm(void *_cxlr)
2963 {
2964 struct cxl_region *cxlr = _cxlr;
2965 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2966
2967 device_lock(&cxl_nvb->dev);
2968 if (cxlr->cxlr_pmem)
2969 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2970 cxlr->cxlr_pmem);
2971 device_unlock(&cxl_nvb->dev);
2972 cxlr->cxl_nvb = NULL;
2973 put_device(&cxl_nvb->dev);
2974 }
2975
2976 /**
2977 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2978 * @cxlr: parent CXL region for this pmem region bridge device
2979 *
2980 * Return: 0 on success negative error code on failure.
2981 */
devm_cxl_add_pmem_region(struct cxl_region * cxlr)2982 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2983 {
2984 struct cxl_pmem_region *cxlr_pmem;
2985 struct cxl_nvdimm_bridge *cxl_nvb;
2986 struct device *dev;
2987 int rc;
2988
2989 rc = cxl_pmem_region_alloc(cxlr);
2990 if (rc)
2991 return rc;
2992 cxlr_pmem = cxlr->cxlr_pmem;
2993 cxl_nvb = cxlr->cxl_nvb;
2994
2995 dev = &cxlr_pmem->dev;
2996 rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2997 if (rc)
2998 goto err;
2999
3000 rc = device_add(dev);
3001 if (rc)
3002 goto err;
3003
3004 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3005 dev_name(dev));
3006
3007 device_lock(&cxl_nvb->dev);
3008 if (cxl_nvb->dev.driver)
3009 rc = devm_add_action_or_reset(&cxl_nvb->dev,
3010 cxlr_pmem_unregister, cxlr_pmem);
3011 else
3012 rc = -ENXIO;
3013 device_unlock(&cxl_nvb->dev);
3014
3015 if (rc)
3016 goto err_bridge;
3017
3018 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
3019 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
3020
3021 err:
3022 put_device(dev);
3023 err_bridge:
3024 put_device(&cxl_nvb->dev);
3025 cxlr->cxl_nvb = NULL;
3026 return rc;
3027 }
3028
cxlr_dax_unregister(void * _cxlr_dax)3029 static void cxlr_dax_unregister(void *_cxlr_dax)
3030 {
3031 struct cxl_dax_region *cxlr_dax = _cxlr_dax;
3032
3033 device_unregister(&cxlr_dax->dev);
3034 }
3035
devm_cxl_add_dax_region(struct cxl_region * cxlr)3036 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
3037 {
3038 struct cxl_dax_region *cxlr_dax;
3039 struct device *dev;
3040 int rc;
3041
3042 cxlr_dax = cxl_dax_region_alloc(cxlr);
3043 if (IS_ERR(cxlr_dax))
3044 return PTR_ERR(cxlr_dax);
3045
3046 dev = &cxlr_dax->dev;
3047 rc = dev_set_name(dev, "dax_region%d", cxlr->id);
3048 if (rc)
3049 goto err;
3050
3051 rc = device_add(dev);
3052 if (rc)
3053 goto err;
3054
3055 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
3056 dev_name(dev));
3057
3058 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
3059 cxlr_dax);
3060 err:
3061 put_device(dev);
3062 return rc;
3063 }
3064
match_root_decoder_by_range(struct device * dev,void * data)3065 static int match_root_decoder_by_range(struct device *dev, void *data)
3066 {
3067 struct range *r1, *r2 = data;
3068 struct cxl_root_decoder *cxlrd;
3069
3070 if (!is_root_decoder(dev))
3071 return 0;
3072
3073 cxlrd = to_cxl_root_decoder(dev);
3074 r1 = &cxlrd->cxlsd.cxld.hpa_range;
3075 return range_contains(r1, r2);
3076 }
3077
match_region_by_range(struct device * dev,void * data)3078 static int match_region_by_range(struct device *dev, void *data)
3079 {
3080 struct cxl_region_params *p;
3081 struct cxl_region *cxlr;
3082 struct range *r = data;
3083 int rc = 0;
3084
3085 if (!is_cxl_region(dev))
3086 return 0;
3087
3088 cxlr = to_cxl_region(dev);
3089 p = &cxlr->params;
3090
3091 down_read(&cxl_region_rwsem);
3092 if (p->res && p->res->start == r->start && p->res->end == r->end)
3093 rc = 1;
3094 up_read(&cxl_region_rwsem);
3095
3096 return rc;
3097 }
3098
3099 /* Establish an empty region covering the given HPA range */
construct_region(struct cxl_root_decoder * cxlrd,struct cxl_endpoint_decoder * cxled)3100 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
3101 struct cxl_endpoint_decoder *cxled)
3102 {
3103 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3104 struct cxl_port *port = cxlrd_to_port(cxlrd);
3105 struct range *hpa = &cxled->cxld.hpa_range;
3106 struct cxl_region_params *p;
3107 struct cxl_region *cxlr;
3108 struct resource *res;
3109 int rc;
3110
3111 do {
3112 cxlr = __create_region(cxlrd, cxled->mode,
3113 atomic_read(&cxlrd->region_id));
3114 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
3115
3116 if (IS_ERR(cxlr)) {
3117 dev_err(cxlmd->dev.parent,
3118 "%s:%s: %s failed assign region: %ld\n",
3119 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3120 __func__, PTR_ERR(cxlr));
3121 return cxlr;
3122 }
3123
3124 down_write(&cxl_region_rwsem);
3125 p = &cxlr->params;
3126 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
3127 dev_err(cxlmd->dev.parent,
3128 "%s:%s: %s autodiscovery interrupted\n",
3129 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3130 __func__);
3131 rc = -EBUSY;
3132 goto err;
3133 }
3134
3135 set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
3136
3137 res = kmalloc(sizeof(*res), GFP_KERNEL);
3138 if (!res) {
3139 rc = -ENOMEM;
3140 goto err;
3141 }
3142
3143 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
3144 dev_name(&cxlr->dev));
3145 rc = insert_resource(cxlrd->res, res);
3146 if (rc) {
3147 /*
3148 * Platform-firmware may not have split resources like "System
3149 * RAM" on CXL window boundaries see cxl_region_iomem_release()
3150 */
3151 dev_warn(cxlmd->dev.parent,
3152 "%s:%s: %s %s cannot insert resource\n",
3153 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
3154 __func__, dev_name(&cxlr->dev));
3155 }
3156
3157 p->res = res;
3158 p->interleave_ways = cxled->cxld.interleave_ways;
3159 p->interleave_granularity = cxled->cxld.interleave_granularity;
3160 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
3161
3162 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
3163 if (rc)
3164 goto err;
3165
3166 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
3167 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
3168 dev_name(&cxlr->dev), p->res, p->interleave_ways,
3169 p->interleave_granularity);
3170
3171 /* ...to match put_device() in cxl_add_to_region() */
3172 get_device(&cxlr->dev);
3173 up_write(&cxl_region_rwsem);
3174
3175 return cxlr;
3176
3177 err:
3178 up_write(&cxl_region_rwsem);
3179 devm_release_action(port->uport_dev, unregister_region, cxlr);
3180 return ERR_PTR(rc);
3181 }
3182
cxl_add_to_region(struct cxl_port * root,struct cxl_endpoint_decoder * cxled)3183 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
3184 {
3185 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
3186 struct range *hpa = &cxled->cxld.hpa_range;
3187 struct cxl_decoder *cxld = &cxled->cxld;
3188 struct device *cxlrd_dev, *region_dev;
3189 struct cxl_root_decoder *cxlrd;
3190 struct cxl_region_params *p;
3191 struct cxl_region *cxlr;
3192 bool attach = false;
3193 int rc;
3194
3195 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
3196 match_root_decoder_by_range);
3197 if (!cxlrd_dev) {
3198 dev_err(cxlmd->dev.parent,
3199 "%s:%s no CXL window for range %#llx:%#llx\n",
3200 dev_name(&cxlmd->dev), dev_name(&cxld->dev),
3201 cxld->hpa_range.start, cxld->hpa_range.end);
3202 return -ENXIO;
3203 }
3204
3205 cxlrd = to_cxl_root_decoder(cxlrd_dev);
3206
3207 /*
3208 * Ensure that if multiple threads race to construct_region() for @hpa
3209 * one does the construction and the others add to that.
3210 */
3211 mutex_lock(&cxlrd->range_lock);
3212 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
3213 match_region_by_range);
3214 if (!region_dev) {
3215 cxlr = construct_region(cxlrd, cxled);
3216 region_dev = &cxlr->dev;
3217 } else
3218 cxlr = to_cxl_region(region_dev);
3219 mutex_unlock(&cxlrd->range_lock);
3220
3221 rc = PTR_ERR_OR_ZERO(cxlr);
3222 if (rc)
3223 goto out;
3224
3225 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
3226
3227 down_read(&cxl_region_rwsem);
3228 p = &cxlr->params;
3229 attach = p->state == CXL_CONFIG_COMMIT;
3230 up_read(&cxl_region_rwsem);
3231
3232 if (attach) {
3233 /*
3234 * If device_attach() fails the range may still be active via
3235 * the platform-firmware memory map, otherwise the driver for
3236 * regions is local to this file, so driver matching can't fail.
3237 */
3238 if (device_attach(&cxlr->dev) < 0)
3239 dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
3240 p->res);
3241 }
3242
3243 put_device(region_dev);
3244 out:
3245 put_device(cxlrd_dev);
3246 return rc;
3247 }
3248 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
3249
is_system_ram(struct resource * res,void * arg)3250 static int is_system_ram(struct resource *res, void *arg)
3251 {
3252 struct cxl_region *cxlr = arg;
3253 struct cxl_region_params *p = &cxlr->params;
3254
3255 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
3256 return 1;
3257 }
3258
cxl_region_probe(struct device * dev)3259 static int cxl_region_probe(struct device *dev)
3260 {
3261 struct cxl_region *cxlr = to_cxl_region(dev);
3262 struct cxl_region_params *p = &cxlr->params;
3263 int rc;
3264
3265 rc = down_read_interruptible(&cxl_region_rwsem);
3266 if (rc) {
3267 dev_dbg(&cxlr->dev, "probe interrupted\n");
3268 return rc;
3269 }
3270
3271 if (p->state < CXL_CONFIG_COMMIT) {
3272 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
3273 rc = -ENXIO;
3274 goto out;
3275 }
3276
3277 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
3278 dev_err(&cxlr->dev,
3279 "failed to activate, re-commit region and retry\n");
3280 rc = -ENXIO;
3281 goto out;
3282 }
3283
3284 /*
3285 * From this point on any path that changes the region's state away from
3286 * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
3287 */
3288 out:
3289 up_read(&cxl_region_rwsem);
3290
3291 if (rc)
3292 return rc;
3293
3294 switch (cxlr->mode) {
3295 case CXL_DECODER_PMEM:
3296 return devm_cxl_add_pmem_region(cxlr);
3297 case CXL_DECODER_RAM:
3298 /*
3299 * The region can not be manged by CXL if any portion of
3300 * it is already online as 'System RAM'
3301 */
3302 if (walk_iomem_res_desc(IORES_DESC_NONE,
3303 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3304 p->res->start, p->res->end, cxlr,
3305 is_system_ram) > 0)
3306 return 0;
3307 return devm_cxl_add_dax_region(cxlr);
3308 default:
3309 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3310 cxlr->mode);
3311 return -ENXIO;
3312 }
3313 }
3314
3315 static struct cxl_driver cxl_region_driver = {
3316 .name = "cxl_region",
3317 .probe = cxl_region_probe,
3318 .id = CXL_DEVICE_REGION,
3319 };
3320
cxl_region_init(void)3321 int cxl_region_init(void)
3322 {
3323 return cxl_driver_register(&cxl_region_driver);
3324 }
3325
cxl_region_exit(void)3326 void cxl_region_exit(void)
3327 {
3328 cxl_driver_unregister(&cxl_region_driver);
3329 }
3330
3331 MODULE_IMPORT_NS(CXL);
3332 MODULE_IMPORT_NS(DEVMEM);
3333 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);
3334