1 /* $OpenBSD: drm_linux.c,v 1.119 2024/09/30 12:21:17 jsg Exp $ */
2 /*
3 * Copyright (c) 2013 Jonathan Gray <jsg@openbsd.org>
4 * Copyright (c) 2015, 2016 Mark Kettenis <kettenis@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19 #include <sys/types.h>
20 #include <sys/systm.h>
21 #include <sys/param.h>
22 #include <sys/event.h>
23 #include <sys/filedesc.h>
24 #include <sys/kthread.h>
25 #include <sys/stat.h>
26 #include <sys/unistd.h>
27 #include <sys/proc.h>
28 #include <sys/pool.h>
29 #include <sys/fcntl.h>
30
31 #include <dev/pci/ppbreg.h>
32
33 #include <linux/dma-buf.h>
34 #include <linux/mod_devicetable.h>
35 #include <linux/acpi.h>
36 #include <linux/pagevec.h>
37 #include <linux/dma-fence-array.h>
38 #include <linux/dma-fence-chain.h>
39 #include <linux/interrupt.h>
40 #include <linux/err.h>
41 #include <linux/idr.h>
42 #include <linux/scatterlist.h>
43 #include <linux/i2c.h>
44 #include <linux/pci.h>
45 #include <linux/notifier.h>
46 #include <linux/backlight.h>
47 #include <linux/shrinker.h>
48 #include <linux/fb.h>
49 #include <linux/xarray.h>
50 #include <linux/interval_tree.h>
51 #include <linux/kthread.h>
52 #include <linux/processor.h>
53 #include <linux/sync_file.h>
54 #include <linux/suspend.h>
55
56 #include <drm/drm_device.h>
57 #include <drm/drm_connector.h>
58 #include <drm/drm_print.h>
59
60 #if defined(__amd64__) || defined(__i386__)
61 #include "bios.h"
62 #endif
63
64 /* allowed to sleep */
65 void
tasklet_unlock_wait(struct tasklet_struct * ts)66 tasklet_unlock_wait(struct tasklet_struct *ts)
67 {
68 while (test_bit(TASKLET_STATE_RUN, &ts->state))
69 cpu_relax();
70 }
71
72 /* must not sleep */
73 void
tasklet_unlock_spin_wait(struct tasklet_struct * ts)74 tasklet_unlock_spin_wait(struct tasklet_struct *ts)
75 {
76 while (test_bit(TASKLET_STATE_RUN, &ts->state))
77 cpu_relax();
78 }
79
80 void
tasklet_run(void * arg)81 tasklet_run(void *arg)
82 {
83 struct tasklet_struct *ts = arg;
84
85 clear_bit(TASKLET_STATE_SCHED, &ts->state);
86 if (tasklet_trylock(ts)) {
87 if (!atomic_read(&ts->count)) {
88 if (ts->use_callback)
89 ts->callback(ts);
90 else
91 ts->func(ts->data);
92 }
93 tasklet_unlock(ts);
94 }
95 }
96
97 /* 32 bit powerpc lacks 64 bit atomics */
98 #if defined(__powerpc__) && !defined(__powerpc64__)
99 struct mutex atomic64_mtx = MUTEX_INITIALIZER(IPL_HIGH);
100 #endif
101
102 void
set_current_state(int state)103 set_current_state(int state)
104 {
105 int prio = state;
106
107 KASSERT(state != TASK_RUNNING);
108 /* check if already on the sleep list */
109 if (curproc->p_wchan != NULL)
110 return;
111 sleep_setup(curproc, prio, "schto");
112 }
113
114 void
__set_current_state(int state)115 __set_current_state(int state)
116 {
117 struct proc *p = curproc;
118
119 KASSERT(state == TASK_RUNNING);
120 SCHED_LOCK();
121 unsleep(p);
122 p->p_stat = SONPROC;
123 atomic_clearbits_int(&p->p_flag, P_WSLEEP);
124 SCHED_UNLOCK();
125 }
126
127 void
schedule(void)128 schedule(void)
129 {
130 schedule_timeout(MAX_SCHEDULE_TIMEOUT);
131 }
132
133 long
schedule_timeout(long timeout)134 schedule_timeout(long timeout)
135 {
136 unsigned long deadline;
137 int timo = 0;
138
139 KASSERT(!cold);
140
141 if (timeout != MAX_SCHEDULE_TIMEOUT)
142 timo = timeout;
143 if (timeout != MAX_SCHEDULE_TIMEOUT)
144 deadline = jiffies + timeout;
145 sleep_finish(timo, timeout > 0);
146 if (timeout != MAX_SCHEDULE_TIMEOUT)
147 timeout = deadline - jiffies;
148
149 return timeout > 0 ? timeout : 0;
150 }
151
152 long
schedule_timeout_uninterruptible(long timeout)153 schedule_timeout_uninterruptible(long timeout)
154 {
155 tsleep(curproc, PWAIT, "schtou", timeout);
156 return 0;
157 }
158
159 int
wake_up_process(struct proc * p)160 wake_up_process(struct proc *p)
161 {
162 int rv;
163
164 SCHED_LOCK();
165 rv = wakeup_proc(p, 0);
166 SCHED_UNLOCK();
167 return rv;
168 }
169
170 int
autoremove_wake_function(struct wait_queue_entry * wqe,unsigned int mode,int sync,void * key)171 autoremove_wake_function(struct wait_queue_entry *wqe, unsigned int mode,
172 int sync, void *key)
173 {
174 if (wqe->private)
175 wake_up_process(wqe->private);
176 list_del_init(&wqe->entry);
177 return 0;
178 }
179
180 void
prepare_to_wait(wait_queue_head_t * wqh,wait_queue_entry_t * wqe,int state)181 prepare_to_wait(wait_queue_head_t *wqh, wait_queue_entry_t *wqe, int state)
182 {
183 mtx_enter(&wqh->lock);
184 if (list_empty(&wqe->entry))
185 __add_wait_queue(wqh, wqe);
186 mtx_leave(&wqh->lock);
187
188 set_current_state(state);
189 }
190
191 void
finish_wait(wait_queue_head_t * wqh,wait_queue_entry_t * wqe)192 finish_wait(wait_queue_head_t *wqh, wait_queue_entry_t *wqe)
193 {
194 __set_current_state(TASK_RUNNING);
195
196 mtx_enter(&wqh->lock);
197 if (!list_empty(&wqe->entry))
198 list_del_init(&wqe->entry);
199 mtx_leave(&wqh->lock);
200 }
201
202 void
flush_workqueue(struct workqueue_struct * wq)203 flush_workqueue(struct workqueue_struct *wq)
204 {
205 if (cold)
206 return;
207
208 if (wq)
209 taskq_barrier((struct taskq *)wq);
210 }
211
212 bool
flush_work(struct work_struct * work)213 flush_work(struct work_struct *work)
214 {
215 if (cold)
216 return false;
217
218 if (work->tq)
219 taskq_barrier(work->tq);
220 return false;
221 }
222
223 bool
flush_delayed_work(struct delayed_work * dwork)224 flush_delayed_work(struct delayed_work *dwork)
225 {
226 bool ret = false;
227
228 if (cold)
229 return false;
230
231 while (timeout_pending(&dwork->to)) {
232 tsleep(dwork, PWAIT, "fldwto", 1);
233 ret = true;
234 }
235
236 if (dwork->tq)
237 taskq_barrier(dwork->tq);
238 return ret;
239 }
240
241 struct kthread {
242 int (*func)(void *);
243 void *data;
244 struct proc *proc;
245 volatile u_int flags;
246 #define KTHREAD_SHOULDSTOP 0x0000001
247 #define KTHREAD_STOPPED 0x0000002
248 #define KTHREAD_SHOULDPARK 0x0000004
249 #define KTHREAD_PARKED 0x0000008
250 LIST_ENTRY(kthread) next;
251 };
252
253 LIST_HEAD(, kthread) kthread_list = LIST_HEAD_INITIALIZER(kthread_list);
254
255 void
kthread_func(void * arg)256 kthread_func(void *arg)
257 {
258 struct kthread *thread = arg;
259 int ret;
260
261 ret = thread->func(thread->data);
262 thread->flags |= KTHREAD_STOPPED;
263 wakeup(thread);
264 kthread_exit(ret);
265 }
266
267 struct proc *
kthread_run(int (* func)(void *),void * data,const char * name)268 kthread_run(int (*func)(void *), void *data, const char *name)
269 {
270 struct kthread *thread;
271
272 thread = malloc(sizeof(*thread), M_DRM, M_WAITOK);
273 thread->func = func;
274 thread->data = data;
275 thread->flags = 0;
276
277 if (kthread_create(kthread_func, thread, &thread->proc, name)) {
278 free(thread, M_DRM, sizeof(*thread));
279 return ERR_PTR(-ENOMEM);
280 }
281
282 LIST_INSERT_HEAD(&kthread_list, thread, next);
283 return thread->proc;
284 }
285
286 struct kthread_worker *
kthread_create_worker(unsigned int flags,const char * fmt,...)287 kthread_create_worker(unsigned int flags, const char *fmt, ...)
288 {
289 char name[MAXCOMLEN+1];
290 va_list ap;
291
292 struct kthread_worker *w = malloc(sizeof(*w), M_DRM, M_WAITOK);
293 va_start(ap, fmt);
294 vsnprintf(name, sizeof(name), fmt, ap);
295 va_end(ap);
296 w->tq = taskq_create(name, 1, IPL_HIGH, 0);
297
298 return w;
299 }
300
301 void
kthread_destroy_worker(struct kthread_worker * worker)302 kthread_destroy_worker(struct kthread_worker *worker)
303 {
304 taskq_destroy(worker->tq);
305 free(worker, M_DRM, sizeof(*worker));
306
307 }
308
309 void
kthread_init_work(struct kthread_work * work,void (* func)(struct kthread_work *))310 kthread_init_work(struct kthread_work *work, void (*func)(struct kthread_work *))
311 {
312 work->tq = NULL;
313 task_set(&work->task, (void (*)(void *))func, work);
314 }
315
316 bool
kthread_queue_work(struct kthread_worker * worker,struct kthread_work * work)317 kthread_queue_work(struct kthread_worker *worker, struct kthread_work *work)
318 {
319 work->tq = worker->tq;
320 return task_add(work->tq, &work->task);
321 }
322
323 bool
kthread_cancel_work_sync(struct kthread_work * work)324 kthread_cancel_work_sync(struct kthread_work *work)
325 {
326 return task_del(work->tq, &work->task);
327 }
328
329 void
kthread_flush_work(struct kthread_work * work)330 kthread_flush_work(struct kthread_work *work)
331 {
332 if (cold)
333 return;
334
335 if (work->tq)
336 taskq_barrier(work->tq);
337 }
338
339 void
kthread_flush_worker(struct kthread_worker * worker)340 kthread_flush_worker(struct kthread_worker *worker)
341 {
342 if (cold)
343 return;
344
345 if (worker->tq)
346 taskq_barrier(worker->tq);
347 }
348
349 struct kthread *
kthread_lookup(struct proc * p)350 kthread_lookup(struct proc *p)
351 {
352 struct kthread *thread;
353
354 LIST_FOREACH(thread, &kthread_list, next) {
355 if (thread->proc == p)
356 break;
357 }
358 KASSERT(thread);
359
360 return thread;
361 }
362
363 int
kthread_should_park(void)364 kthread_should_park(void)
365 {
366 struct kthread *thread = kthread_lookup(curproc);
367 return (thread->flags & KTHREAD_SHOULDPARK);
368 }
369
370 void
kthread_parkme(void)371 kthread_parkme(void)
372 {
373 struct kthread *thread = kthread_lookup(curproc);
374
375 while (thread->flags & KTHREAD_SHOULDPARK) {
376 thread->flags |= KTHREAD_PARKED;
377 wakeup(thread);
378 tsleep_nsec(thread, PPAUSE, "parkme", INFSLP);
379 thread->flags &= ~KTHREAD_PARKED;
380 }
381 }
382
383 void
kthread_park(struct proc * p)384 kthread_park(struct proc *p)
385 {
386 struct kthread *thread = kthread_lookup(p);
387
388 while ((thread->flags & KTHREAD_PARKED) == 0) {
389 thread->flags |= KTHREAD_SHOULDPARK;
390 wake_up_process(thread->proc);
391 tsleep_nsec(thread, PPAUSE, "park", INFSLP);
392 }
393 }
394
395 void
kthread_unpark(struct proc * p)396 kthread_unpark(struct proc *p)
397 {
398 struct kthread *thread = kthread_lookup(p);
399
400 thread->flags &= ~KTHREAD_SHOULDPARK;
401 wakeup(thread);
402 }
403
404 int
kthread_should_stop(void)405 kthread_should_stop(void)
406 {
407 struct kthread *thread = kthread_lookup(curproc);
408 return (thread->flags & KTHREAD_SHOULDSTOP);
409 }
410
411 void
kthread_stop(struct proc * p)412 kthread_stop(struct proc *p)
413 {
414 struct kthread *thread = kthread_lookup(p);
415
416 while ((thread->flags & KTHREAD_STOPPED) == 0) {
417 thread->flags |= KTHREAD_SHOULDSTOP;
418 kthread_unpark(p);
419 wake_up_process(thread->proc);
420 tsleep_nsec(thread, PPAUSE, "stop", INFSLP);
421 }
422 LIST_REMOVE(thread, next);
423 free(thread, M_DRM, sizeof(*thread));
424 }
425
426 #if NBIOS > 0
427 extern char smbios_board_vendor[];
428 extern char smbios_board_prod[];
429 extern char smbios_board_serial[];
430 #endif
431
432 bool
dmi_match(int slot,const char * str)433 dmi_match(int slot, const char *str)
434 {
435 switch (slot) {
436 case DMI_SYS_VENDOR:
437 if (hw_vendor != NULL &&
438 !strcmp(hw_vendor, str))
439 return true;
440 break;
441 case DMI_PRODUCT_NAME:
442 if (hw_prod != NULL &&
443 !strcmp(hw_prod, str))
444 return true;
445 break;
446 case DMI_PRODUCT_VERSION:
447 if (hw_ver != NULL &&
448 !strcmp(hw_ver, str))
449 return true;
450 break;
451 #if NBIOS > 0
452 case DMI_BOARD_VENDOR:
453 if (strcmp(smbios_board_vendor, str) == 0)
454 return true;
455 break;
456 case DMI_BOARD_NAME:
457 if (strcmp(smbios_board_prod, str) == 0)
458 return true;
459 break;
460 case DMI_BOARD_SERIAL:
461 if (strcmp(smbios_board_serial, str) == 0)
462 return true;
463 break;
464 #else
465 case DMI_BOARD_VENDOR:
466 if (hw_vendor != NULL &&
467 !strcmp(hw_vendor, str))
468 return true;
469 break;
470 case DMI_BOARD_NAME:
471 if (hw_prod != NULL &&
472 !strcmp(hw_prod, str))
473 return true;
474 break;
475 #endif
476 case DMI_NONE:
477 default:
478 return false;
479 }
480
481 return false;
482 }
483
484 static bool
dmi_found(const struct dmi_system_id * dsi)485 dmi_found(const struct dmi_system_id *dsi)
486 {
487 int i, slot;
488
489 for (i = 0; i < nitems(dsi->matches); i++) {
490 slot = dsi->matches[i].slot;
491 if (slot == DMI_NONE)
492 break;
493 if (!dmi_match(slot, dsi->matches[i].substr))
494 return false;
495 }
496
497 return true;
498 }
499
500 const struct dmi_system_id *
dmi_first_match(const struct dmi_system_id * sysid)501 dmi_first_match(const struct dmi_system_id *sysid)
502 {
503 const struct dmi_system_id *dsi;
504
505 for (dsi = sysid; dsi->matches[0].slot != 0 ; dsi++) {
506 if (dmi_found(dsi))
507 return dsi;
508 }
509
510 return NULL;
511 }
512
513 #if NBIOS > 0
514 extern char smbios_bios_date[];
515 extern char smbios_bios_version[];
516 #endif
517
518 const char *
dmi_get_system_info(int slot)519 dmi_get_system_info(int slot)
520 {
521 #if NBIOS > 0
522 switch (slot) {
523 case DMI_BIOS_DATE:
524 return smbios_bios_date;
525 case DMI_BIOS_VERSION:
526 return smbios_bios_version;
527 default:
528 printf("%s slot %d not handled\n", __func__, slot);
529 }
530 #endif
531 return NULL;
532 }
533
534 int
dmi_check_system(const struct dmi_system_id * sysid)535 dmi_check_system(const struct dmi_system_id *sysid)
536 {
537 const struct dmi_system_id *dsi;
538 int num = 0;
539
540 for (dsi = sysid; dsi->matches[0].slot != 0 ; dsi++) {
541 if (dmi_found(dsi)) {
542 num++;
543 if (dsi->callback && dsi->callback(dsi))
544 break;
545 }
546 }
547 return (num);
548 }
549
550 struct vm_page *
alloc_pages(unsigned int gfp_mask,unsigned int order)551 alloc_pages(unsigned int gfp_mask, unsigned int order)
552 {
553 int flags = (gfp_mask & M_NOWAIT) ? UVM_PLA_NOWAIT : UVM_PLA_WAITOK;
554 struct uvm_constraint_range *constraint = &no_constraint;
555 struct pglist mlist;
556
557 if (gfp_mask & M_CANFAIL)
558 flags |= UVM_PLA_FAILOK;
559 if (gfp_mask & M_ZERO)
560 flags |= UVM_PLA_ZERO;
561 if (gfp_mask & __GFP_DMA32)
562 constraint = &dma_constraint;
563
564 TAILQ_INIT(&mlist);
565 if (uvm_pglistalloc(PAGE_SIZE << order, constraint->ucr_low,
566 constraint->ucr_high, PAGE_SIZE, 0, &mlist, 1, flags))
567 return NULL;
568 return TAILQ_FIRST(&mlist);
569 }
570
571 void
__free_pages(struct vm_page * page,unsigned int order)572 __free_pages(struct vm_page *page, unsigned int order)
573 {
574 struct pglist mlist;
575 int i;
576
577 TAILQ_INIT(&mlist);
578 for (i = 0; i < (1 << order); i++)
579 TAILQ_INSERT_TAIL(&mlist, &page[i], pageq);
580 uvm_pglistfree(&mlist);
581 }
582
583 void
__pagevec_release(struct pagevec * pvec)584 __pagevec_release(struct pagevec *pvec)
585 {
586 struct pglist mlist;
587 int i;
588
589 TAILQ_INIT(&mlist);
590 for (i = 0; i < pvec->nr; i++)
591 TAILQ_INSERT_TAIL(&mlist, pvec->pages[i], pageq);
592 uvm_pglistfree(&mlist);
593 pagevec_reinit(pvec);
594 }
595
596 static struct kmem_va_mode kv_physwait = {
597 .kv_map = &phys_map,
598 .kv_wait = 1,
599 };
600
601 void *
kmap(struct vm_page * pg)602 kmap(struct vm_page *pg)
603 {
604 vaddr_t va;
605
606 #if defined (__HAVE_PMAP_DIRECT)
607 va = pmap_map_direct(pg);
608 #else
609 va = (vaddr_t)km_alloc(PAGE_SIZE, &kv_physwait, &kp_none, &kd_waitok);
610 pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), PROT_READ | PROT_WRITE);
611 pmap_update(pmap_kernel());
612 #endif
613 return (void *)va;
614 }
615
616 void
kunmap_va(void * addr)617 kunmap_va(void *addr)
618 {
619 vaddr_t va = (vaddr_t)addr;
620
621 #if defined (__HAVE_PMAP_DIRECT)
622 pmap_unmap_direct(va);
623 #else
624 pmap_kremove(va, PAGE_SIZE);
625 pmap_update(pmap_kernel());
626 km_free((void *)va, PAGE_SIZE, &kv_physwait, &kp_none);
627 #endif
628 }
629
630 vaddr_t kmap_atomic_va;
631 int kmap_atomic_inuse;
632
633 void *
kmap_atomic_prot(struct vm_page * pg,pgprot_t prot)634 kmap_atomic_prot(struct vm_page *pg, pgprot_t prot)
635 {
636 KASSERT(!kmap_atomic_inuse);
637
638 kmap_atomic_inuse = 1;
639 pmap_kenter_pa(kmap_atomic_va, VM_PAGE_TO_PHYS(pg) | prot,
640 PROT_READ | PROT_WRITE);
641 return (void *)kmap_atomic_va;
642 }
643
644 void
kunmap_atomic(void * addr)645 kunmap_atomic(void *addr)
646 {
647 KASSERT(kmap_atomic_inuse);
648
649 pmap_kremove(kmap_atomic_va, PAGE_SIZE);
650 kmap_atomic_inuse = 0;
651 }
652
653 void *
vmap(struct vm_page ** pages,unsigned int npages,unsigned long flags,pgprot_t prot)654 vmap(struct vm_page **pages, unsigned int npages, unsigned long flags,
655 pgprot_t prot)
656 {
657 vaddr_t va;
658 paddr_t pa;
659 int i;
660
661 va = (vaddr_t)km_alloc(PAGE_SIZE * npages, &kv_any, &kp_none,
662 &kd_nowait);
663 if (va == 0)
664 return NULL;
665 for (i = 0; i < npages; i++) {
666 pa = VM_PAGE_TO_PHYS(pages[i]) | prot;
667 pmap_enter(pmap_kernel(), va + (i * PAGE_SIZE), pa,
668 PROT_READ | PROT_WRITE,
669 PROT_READ | PROT_WRITE | PMAP_WIRED);
670 pmap_update(pmap_kernel());
671 }
672
673 return (void *)va;
674 }
675
676 void *
vmap_pfn(unsigned long * pfns,unsigned int npfn,pgprot_t prot)677 vmap_pfn(unsigned long *pfns, unsigned int npfn, pgprot_t prot)
678 {
679 vaddr_t va;
680 paddr_t pa;
681 int i;
682
683 va = (vaddr_t)km_alloc(PAGE_SIZE * npfn, &kv_any, &kp_none,
684 &kd_nowait);
685 if (va == 0)
686 return NULL;
687 for (i = 0; i < npfn; i++) {
688 pa = round_page(pfns[i]) | prot;
689 pmap_enter(pmap_kernel(), va + (i * PAGE_SIZE), pa,
690 PROT_READ | PROT_WRITE,
691 PROT_READ | PROT_WRITE | PMAP_WIRED);
692 pmap_update(pmap_kernel());
693 }
694
695 return (void *)va;
696 }
697
698 void
vunmap(void * addr,size_t size)699 vunmap(void *addr, size_t size)
700 {
701 vaddr_t va = (vaddr_t)addr;
702
703 pmap_remove(pmap_kernel(), va, va + size);
704 pmap_update(pmap_kernel());
705 km_free((void *)va, size, &kv_any, &kp_none);
706 }
707
708 bool
is_vmalloc_addr(const void * p)709 is_vmalloc_addr(const void *p)
710 {
711 vaddr_t min, max, addr;
712
713 min = vm_map_min(kernel_map);
714 max = vm_map_max(kernel_map);
715 addr = (vaddr_t)p;
716
717 if (addr >= min && addr <= max)
718 return true;
719 else
720 return false;
721 }
722
723 void
print_hex_dump(const char * level,const char * prefix_str,int prefix_type,int rowsize,int groupsize,const void * buf,size_t len,bool ascii)724 print_hex_dump(const char *level, const char *prefix_str, int prefix_type,
725 int rowsize, int groupsize, const void *buf, size_t len, bool ascii)
726 {
727 const uint8_t *cbuf = buf;
728 int i;
729
730 for (i = 0; i < len; i++) {
731 if ((i % rowsize) == 0)
732 printf("%s", prefix_str);
733 printf("%02x", cbuf[i]);
734 if ((i % rowsize) == (rowsize - 1))
735 printf("\n");
736 else
737 printf(" ");
738 }
739 }
740
741 void *
memchr_inv(const void * s,int c,size_t n)742 memchr_inv(const void *s, int c, size_t n)
743 {
744 if (n != 0) {
745 const unsigned char *p = s;
746
747 do {
748 if (*p++ != (unsigned char)c)
749 return ((void *)(p - 1));
750 } while (--n != 0);
751 }
752 return (NULL);
753 }
754
755 int
panic_cmp(struct rb_node * a,struct rb_node * b)756 panic_cmp(struct rb_node *a, struct rb_node *b)
757 {
758 panic(__func__);
759 }
760
761 #undef RB_ROOT
762 #define RB_ROOT(head) (head)->rbh_root
763
764 RB_GENERATE(linux_root, rb_node, __entry, panic_cmp);
765
766 /*
767 * This is a fairly minimal implementation of the Linux "idr" API. It
768 * probably isn't very efficient, and definitely isn't RCU safe. The
769 * pre-load buffer is global instead of per-cpu; we rely on the kernel
770 * lock to make this work. We do randomize our IDs in order to make
771 * them harder to guess.
772 */
773
774 int idr_cmp(struct idr_entry *, struct idr_entry *);
775 SPLAY_PROTOTYPE(idr_tree, idr_entry, entry, idr_cmp);
776
777 struct pool idr_pool;
778 struct idr_entry *idr_entry_cache;
779
780 void
idr_init(struct idr * idr)781 idr_init(struct idr *idr)
782 {
783 SPLAY_INIT(&idr->tree);
784 }
785
786 void
idr_destroy(struct idr * idr)787 idr_destroy(struct idr *idr)
788 {
789 struct idr_entry *id;
790
791 while ((id = SPLAY_MIN(idr_tree, &idr->tree))) {
792 SPLAY_REMOVE(idr_tree, &idr->tree, id);
793 pool_put(&idr_pool, id);
794 }
795 }
796
797 void
idr_preload(unsigned int gfp_mask)798 idr_preload(unsigned int gfp_mask)
799 {
800 int flags = (gfp_mask & GFP_NOWAIT) ? PR_NOWAIT : PR_WAITOK;
801
802 KERNEL_ASSERT_LOCKED();
803
804 if (idr_entry_cache == NULL)
805 idr_entry_cache = pool_get(&idr_pool, flags);
806 }
807
808 int
idr_alloc(struct idr * idr,void * ptr,int start,int end,gfp_t gfp_mask)809 idr_alloc(struct idr *idr, void *ptr, int start, int end, gfp_t gfp_mask)
810 {
811 int flags = (gfp_mask & GFP_NOWAIT) ? PR_NOWAIT : PR_WAITOK;
812 struct idr_entry *id;
813 int begin;
814
815 KERNEL_ASSERT_LOCKED();
816
817 if (idr_entry_cache) {
818 id = idr_entry_cache;
819 idr_entry_cache = NULL;
820 } else {
821 id = pool_get(&idr_pool, flags);
822 if (id == NULL)
823 return -ENOMEM;
824 }
825
826 if (end <= 0)
827 end = INT_MAX;
828
829 #ifdef notyet
830 id->id = begin = start + arc4random_uniform(end - start);
831 #else
832 id->id = begin = start;
833 #endif
834 while (SPLAY_INSERT(idr_tree, &idr->tree, id)) {
835 if (id->id == end)
836 id->id = start;
837 else
838 id->id++;
839 if (id->id == begin) {
840 pool_put(&idr_pool, id);
841 return -ENOSPC;
842 }
843 }
844 id->ptr = ptr;
845 return id->id;
846 }
847
848 void *
idr_replace(struct idr * idr,void * ptr,unsigned long id)849 idr_replace(struct idr *idr, void *ptr, unsigned long id)
850 {
851 struct idr_entry find, *res;
852 void *old;
853
854 find.id = id;
855 res = SPLAY_FIND(idr_tree, &idr->tree, &find);
856 if (res == NULL)
857 return ERR_PTR(-ENOENT);
858 old = res->ptr;
859 res->ptr = ptr;
860 return old;
861 }
862
863 void *
idr_remove(struct idr * idr,unsigned long id)864 idr_remove(struct idr *idr, unsigned long id)
865 {
866 struct idr_entry find, *res;
867 void *ptr = NULL;
868
869 find.id = id;
870 res = SPLAY_FIND(idr_tree, &idr->tree, &find);
871 if (res) {
872 SPLAY_REMOVE(idr_tree, &idr->tree, res);
873 ptr = res->ptr;
874 pool_put(&idr_pool, res);
875 }
876 return ptr;
877 }
878
879 void *
idr_find(struct idr * idr,unsigned long id)880 idr_find(struct idr *idr, unsigned long id)
881 {
882 struct idr_entry find, *res;
883
884 find.id = id;
885 res = SPLAY_FIND(idr_tree, &idr->tree, &find);
886 if (res == NULL)
887 return NULL;
888 return res->ptr;
889 }
890
891 void *
idr_get_next(struct idr * idr,int * id)892 idr_get_next(struct idr *idr, int *id)
893 {
894 struct idr_entry *res;
895
896 SPLAY_FOREACH(res, idr_tree, &idr->tree) {
897 if (res->id >= *id) {
898 *id = res->id;
899 return res->ptr;
900 }
901 }
902
903 return NULL;
904 }
905
906 int
idr_for_each(struct idr * idr,int (* func)(int,void *,void *),void * data)907 idr_for_each(struct idr *idr, int (*func)(int, void *, void *), void *data)
908 {
909 struct idr_entry *id;
910 int ret;
911
912 SPLAY_FOREACH(id, idr_tree, &idr->tree) {
913 ret = func(id->id, id->ptr, data);
914 if (ret)
915 return ret;
916 }
917
918 return 0;
919 }
920
921 int
idr_cmp(struct idr_entry * a,struct idr_entry * b)922 idr_cmp(struct idr_entry *a, struct idr_entry *b)
923 {
924 return (a->id < b->id ? -1 : a->id > b->id);
925 }
926
927 SPLAY_GENERATE(idr_tree, idr_entry, entry, idr_cmp);
928
929 void
ida_init(struct ida * ida)930 ida_init(struct ida *ida)
931 {
932 idr_init(&ida->idr);
933 }
934
935 void
ida_destroy(struct ida * ida)936 ida_destroy(struct ida *ida)
937 {
938 idr_destroy(&ida->idr);
939 }
940
941 int
ida_simple_get(struct ida * ida,unsigned int start,unsigned int end,gfp_t gfp_mask)942 ida_simple_get(struct ida *ida, unsigned int start, unsigned int end,
943 gfp_t gfp_mask)
944 {
945 return idr_alloc(&ida->idr, NULL, start, end, gfp_mask);
946 }
947
948 void
ida_simple_remove(struct ida * ida,unsigned int id)949 ida_simple_remove(struct ida *ida, unsigned int id)
950 {
951 idr_remove(&ida->idr, id);
952 }
953
954 int
ida_alloc_min(struct ida * ida,unsigned int min,gfp_t gfp)955 ida_alloc_min(struct ida *ida, unsigned int min, gfp_t gfp)
956 {
957 return idr_alloc(&ida->idr, NULL, min, INT_MAX, gfp);
958 }
959
960 int
ida_alloc_max(struct ida * ida,unsigned int max,gfp_t gfp)961 ida_alloc_max(struct ida *ida, unsigned int max, gfp_t gfp)
962 {
963 return idr_alloc(&ida->idr, NULL, 0, max - 1, gfp);
964 }
965
966 void
ida_free(struct ida * ida,unsigned int id)967 ida_free(struct ida *ida, unsigned int id)
968 {
969 idr_remove(&ida->idr, id);
970 }
971
972 int
xarray_cmp(struct xarray_entry * a,struct xarray_entry * b)973 xarray_cmp(struct xarray_entry *a, struct xarray_entry *b)
974 {
975 return (a->id < b->id ? -1 : a->id > b->id);
976 }
977
978 SPLAY_PROTOTYPE(xarray_tree, xarray_entry, entry, xarray_cmp);
979 struct pool xa_pool;
980 SPLAY_GENERATE(xarray_tree, xarray_entry, entry, xarray_cmp);
981
982 void
xa_init_flags(struct xarray * xa,gfp_t flags)983 xa_init_flags(struct xarray *xa, gfp_t flags)
984 {
985 SPLAY_INIT(&xa->xa_tree);
986 if (flags & XA_FLAGS_LOCK_IRQ)
987 mtx_init(&xa->xa_lock, IPL_TTY);
988 else
989 mtx_init(&xa->xa_lock, IPL_NONE);
990 xa->xa_flags = flags;
991 }
992
993 void
xa_destroy(struct xarray * xa)994 xa_destroy(struct xarray *xa)
995 {
996 struct xarray_entry *id;
997
998 while ((id = SPLAY_MIN(xarray_tree, &xa->xa_tree))) {
999 SPLAY_REMOVE(xarray_tree, &xa->xa_tree, id);
1000 pool_put(&xa_pool, id);
1001 }
1002 }
1003
1004 /* Don't wrap ids. */
1005 int
__xa_alloc(struct xarray * xa,u32 * id,void * entry,struct xarray_range xr,gfp_t gfp)1006 __xa_alloc(struct xarray *xa, u32 *id, void *entry, struct xarray_range xr,
1007 gfp_t gfp)
1008 {
1009 struct xarray_entry *xid;
1010 uint32_t start = xr.start;
1011 uint32_t end = xr.end;
1012
1013 if (start == 0 && (xa->xa_flags & XA_FLAGS_ALLOC1))
1014 start = 1;
1015
1016 if (gfp & GFP_NOWAIT) {
1017 xid = pool_get(&xa_pool, PR_NOWAIT);
1018 } else {
1019 mtx_leave(&xa->xa_lock);
1020 xid = pool_get(&xa_pool, PR_WAITOK);
1021 mtx_enter(&xa->xa_lock);
1022 }
1023
1024 if (xid == NULL)
1025 return -ENOMEM;
1026
1027 xid->id = start;
1028
1029 while (SPLAY_INSERT(xarray_tree, &xa->xa_tree, xid)) {
1030 if (xid->id == end)
1031 xid->id = start;
1032 else
1033 xid->id++;
1034 if (xid->id == start) {
1035 pool_put(&xa_pool, xid);
1036 return -EBUSY;
1037 }
1038 }
1039 xid->ptr = entry;
1040 *id = xid->id;
1041 return 0;
1042 }
1043
1044 /*
1045 * Wrap ids and store next id.
1046 * We walk the entire tree so don't special case wrapping.
1047 * The only caller of this (i915_drm_client.c) doesn't use next id.
1048 */
1049 int
__xa_alloc_cyclic(struct xarray * xa,u32 * id,void * entry,struct xarray_range xr,u32 * next,gfp_t gfp)1050 __xa_alloc_cyclic(struct xarray *xa, u32 *id, void *entry,
1051 struct xarray_range xr, u32 *next, gfp_t gfp)
1052 {
1053 int r = __xa_alloc(xa, id, entry, xr, gfp);
1054 *next = *id + 1;
1055 return r;
1056 }
1057
1058 void *
__xa_erase(struct xarray * xa,unsigned long index)1059 __xa_erase(struct xarray *xa, unsigned long index)
1060 {
1061 struct xarray_entry find, *res;
1062 void *ptr = NULL;
1063
1064 find.id = index;
1065 res = SPLAY_FIND(xarray_tree, &xa->xa_tree, &find);
1066 if (res) {
1067 SPLAY_REMOVE(xarray_tree, &xa->xa_tree, res);
1068 ptr = res->ptr;
1069 pool_put(&xa_pool, res);
1070 }
1071 return ptr;
1072 }
1073
1074 void *
__xa_load(struct xarray * xa,unsigned long index)1075 __xa_load(struct xarray *xa, unsigned long index)
1076 {
1077 struct xarray_entry find, *res;
1078
1079 find.id = index;
1080 res = SPLAY_FIND(xarray_tree, &xa->xa_tree, &find);
1081 if (res == NULL)
1082 return NULL;
1083 return res->ptr;
1084 }
1085
1086 void *
__xa_store(struct xarray * xa,unsigned long index,void * entry,gfp_t gfp)1087 __xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp)
1088 {
1089 struct xarray_entry find, *res;
1090 void *prev;
1091
1092 if (entry == NULL)
1093 return __xa_erase(xa, index);
1094
1095 find.id = index;
1096 res = SPLAY_FIND(xarray_tree, &xa->xa_tree, &find);
1097 if (res != NULL) {
1098 /* index exists */
1099 /* XXX Multislot entries updates not implemented yet */
1100 prev = res->ptr;
1101 res->ptr = entry;
1102 return prev;
1103 }
1104
1105 /* index not found, add new */
1106 if (gfp & GFP_NOWAIT) {
1107 res = pool_get(&xa_pool, PR_NOWAIT);
1108 } else {
1109 mtx_leave(&xa->xa_lock);
1110 res = pool_get(&xa_pool, PR_WAITOK);
1111 mtx_enter(&xa->xa_lock);
1112 }
1113 if (res == NULL)
1114 return XA_ERROR(-ENOMEM);
1115 res->id = index;
1116 res->ptr = entry;
1117 if (SPLAY_INSERT(xarray_tree, &xa->xa_tree, res) != NULL)
1118 return XA_ERROR(-EINVAL);
1119 return NULL; /* no prev entry at index */
1120 }
1121
1122 void *
xa_get_next(struct xarray * xa,unsigned long * index)1123 xa_get_next(struct xarray *xa, unsigned long *index)
1124 {
1125 struct xarray_entry *res;
1126
1127 SPLAY_FOREACH(res, xarray_tree, &xa->xa_tree) {
1128 if (res->id >= *index) {
1129 *index = res->id;
1130 return res->ptr;
1131 }
1132 }
1133
1134 return NULL;
1135 }
1136
1137 int
sg_alloc_table(struct sg_table * table,unsigned int nents,gfp_t gfp_mask)1138 sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
1139 {
1140 table->sgl = mallocarray(nents, sizeof(struct scatterlist),
1141 M_DRM, gfp_mask | M_ZERO);
1142 if (table->sgl == NULL)
1143 return -ENOMEM;
1144 table->nents = table->orig_nents = nents;
1145 sg_mark_end(&table->sgl[nents - 1]);
1146 return 0;
1147 }
1148
1149 void
sg_free_table(struct sg_table * table)1150 sg_free_table(struct sg_table *table)
1151 {
1152 free(table->sgl, M_DRM,
1153 table->orig_nents * sizeof(struct scatterlist));
1154 table->orig_nents = 0;
1155 table->sgl = NULL;
1156 }
1157
1158 size_t
sg_copy_from_buffer(struct scatterlist * sgl,unsigned int nents,const void * buf,size_t buflen)1159 sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
1160 const void *buf, size_t buflen)
1161 {
1162 panic("%s", __func__);
1163 }
1164
1165 int
i2c_master_xfer(struct i2c_adapter * adap,struct i2c_msg * msgs,int num)1166 i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
1167 {
1168 void *cmd = NULL;
1169 int cmdlen = 0;
1170 int err, ret = 0;
1171 int op;
1172
1173 iic_acquire_bus(&adap->ic, 0);
1174
1175 while (num > 2) {
1176 op = (msgs->flags & I2C_M_RD) ? I2C_OP_READ : I2C_OP_WRITE;
1177 err = iic_exec(&adap->ic, op, msgs->addr, NULL, 0,
1178 msgs->buf, msgs->len, 0);
1179 if (err) {
1180 ret = -err;
1181 goto fail;
1182 }
1183 msgs++;
1184 num--;
1185 ret++;
1186 }
1187
1188 if (num > 1) {
1189 cmd = msgs->buf;
1190 cmdlen = msgs->len;
1191 msgs++;
1192 num--;
1193 ret++;
1194 }
1195
1196 op = (msgs->flags & I2C_M_RD) ?
1197 I2C_OP_READ_WITH_STOP : I2C_OP_WRITE_WITH_STOP;
1198 err = iic_exec(&adap->ic, op, msgs->addr, cmd, cmdlen,
1199 msgs->buf, msgs->len, 0);
1200 if (err) {
1201 ret = -err;
1202 goto fail;
1203 }
1204 msgs++;
1205 ret++;
1206
1207 fail:
1208 iic_release_bus(&adap->ic, 0);
1209
1210 return ret;
1211 }
1212
1213 int
__i2c_transfer(struct i2c_adapter * adap,struct i2c_msg * msgs,int num)1214 __i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
1215 {
1216 int ret, retries;
1217
1218 retries = adap->retries;
1219 retry:
1220 if (adap->algo)
1221 ret = adap->algo->master_xfer(adap, msgs, num);
1222 else
1223 ret = i2c_master_xfer(adap, msgs, num);
1224 if (ret == -EAGAIN && retries > 0) {
1225 retries--;
1226 goto retry;
1227 }
1228
1229 return ret;
1230 }
1231
1232 int
i2c_transfer(struct i2c_adapter * adap,struct i2c_msg * msgs,int num)1233 i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
1234 {
1235 int ret;
1236
1237 if (adap->lock_ops)
1238 adap->lock_ops->lock_bus(adap, 0);
1239
1240 ret = __i2c_transfer(adap, msgs, num);
1241
1242 if (adap->lock_ops)
1243 adap->lock_ops->unlock_bus(adap, 0);
1244
1245 return ret;
1246 }
1247
1248 int
i2c_bb_master_xfer(struct i2c_adapter * adap,struct i2c_msg * msgs,int num)1249 i2c_bb_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
1250 {
1251 struct i2c_algo_bit_data *algo = adap->algo_data;
1252 struct i2c_adapter bb;
1253
1254 memset(&bb, 0, sizeof(bb));
1255 bb.ic = algo->ic;
1256 bb.retries = adap->retries;
1257 return i2c_master_xfer(&bb, msgs, num);
1258 }
1259
1260 uint32_t
i2c_bb_functionality(struct i2c_adapter * adap)1261 i2c_bb_functionality(struct i2c_adapter *adap)
1262 {
1263 return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
1264 }
1265
1266 struct i2c_algorithm i2c_bit_algo = {
1267 .master_xfer = i2c_bb_master_xfer,
1268 .functionality = i2c_bb_functionality
1269 };
1270
1271 int
i2c_bit_add_bus(struct i2c_adapter * adap)1272 i2c_bit_add_bus(struct i2c_adapter *adap)
1273 {
1274 adap->algo = &i2c_bit_algo;
1275 adap->retries = 3;
1276
1277 return 0;
1278 }
1279
1280 #if defined(__amd64__) || defined(__i386__)
1281
1282 /*
1283 * This is a minimal implementation of the Linux vga_get/vga_put
1284 * interface. In all likelihood, it will only work for inteldrm(4) as
1285 * it assumes that if there is another active VGA device in the
1286 * system, it is sitting behind a PCI bridge.
1287 */
1288
1289 extern int pci_enumerate_bus(struct pci_softc *,
1290 int (*)(struct pci_attach_args *), struct pci_attach_args *);
1291
1292 pcitag_t vga_bridge_tag;
1293 int vga_bridge_disabled;
1294
1295 int
vga_disable_bridge(struct pci_attach_args * pa)1296 vga_disable_bridge(struct pci_attach_args *pa)
1297 {
1298 pcireg_t bhlc, bc;
1299
1300 if (pa->pa_domain != 0)
1301 return 0;
1302
1303 bhlc = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_BHLC_REG);
1304 if (PCI_HDRTYPE_TYPE(bhlc) != 1)
1305 return 0;
1306
1307 bc = pci_conf_read(pa->pa_pc, pa->pa_tag, PPB_REG_BRIDGECONTROL);
1308 if ((bc & PPB_BC_VGA_ENABLE) == 0)
1309 return 0;
1310 bc &= ~PPB_BC_VGA_ENABLE;
1311 pci_conf_write(pa->pa_pc, pa->pa_tag, PPB_REG_BRIDGECONTROL, bc);
1312
1313 vga_bridge_tag = pa->pa_tag;
1314 vga_bridge_disabled = 1;
1315
1316 return 1;
1317 }
1318
1319 void
vga_get_uninterruptible(struct pci_dev * pdev,int rsrc)1320 vga_get_uninterruptible(struct pci_dev *pdev, int rsrc)
1321 {
1322 if (pdev->pci->sc_bridgetag != NULL)
1323 return;
1324 pci_enumerate_bus(pdev->pci, vga_disable_bridge, NULL);
1325 }
1326
1327 void
vga_put(struct pci_dev * pdev,int rsrc)1328 vga_put(struct pci_dev *pdev, int rsrc)
1329 {
1330 pcireg_t bc;
1331
1332 if (!vga_bridge_disabled)
1333 return;
1334
1335 bc = pci_conf_read(pdev->pc, vga_bridge_tag, PPB_REG_BRIDGECONTROL);
1336 bc |= PPB_BC_VGA_ENABLE;
1337 pci_conf_write(pdev->pc, vga_bridge_tag, PPB_REG_BRIDGECONTROL, bc);
1338
1339 vga_bridge_disabled = 0;
1340 }
1341
1342 #endif
1343
1344 suspend_state_t pm_suspend_target_state;
1345
1346 /*
1347 * ACPI types and interfaces.
1348 */
1349
1350 #ifdef __HAVE_ACPI
1351 #include "acpi.h"
1352 #endif
1353
1354 #if NACPI > 0
1355
1356 #include <dev/acpi/acpireg.h>
1357 #include <dev/acpi/acpivar.h>
1358 #include <dev/acpi/amltypes.h>
1359 #include <dev/acpi/dsdt.h>
1360
1361 struct acpi_fadt acpi_gbl_FADT;
1362
1363 acpi_status
acpi_get_table(const char * sig,int instance,struct acpi_table_header ** hdr)1364 acpi_get_table(const char *sig, int instance,
1365 struct acpi_table_header **hdr)
1366 {
1367 struct acpi_softc *sc = acpi_softc;
1368 struct acpi_q *entry;
1369
1370 KASSERT(instance == 1);
1371
1372 if (sc == NULL)
1373 return AE_NOT_FOUND;
1374
1375 SIMPLEQ_FOREACH(entry, &sc->sc_tables, q_next) {
1376 if (memcmp(entry->q_table, sig, strlen(sig)) == 0) {
1377 *hdr = entry->q_table;
1378 return 0;
1379 }
1380 }
1381
1382 return AE_NOT_FOUND;
1383 }
1384
1385 void
acpi_put_table(struct acpi_table_header * hdr)1386 acpi_put_table(struct acpi_table_header *hdr)
1387 {
1388 }
1389
1390 acpi_status
acpi_get_handle(acpi_handle node,const char * name,acpi_handle * rnode)1391 acpi_get_handle(acpi_handle node, const char *name, acpi_handle *rnode)
1392 {
1393 node = aml_searchname(node, name);
1394 if (node == NULL)
1395 return AE_NOT_FOUND;
1396
1397 *rnode = node;
1398 return 0;
1399 }
1400
1401 acpi_status
acpi_get_name(acpi_handle node,int type,struct acpi_buffer * buffer)1402 acpi_get_name(acpi_handle node, int type, struct acpi_buffer *buffer)
1403 {
1404 KASSERT(buffer->length != ACPI_ALLOCATE_BUFFER);
1405 KASSERT(type == ACPI_FULL_PATHNAME);
1406 strlcpy(buffer->pointer, aml_nodename(node), buffer->length);
1407 return 0;
1408 }
1409
1410 acpi_status
acpi_evaluate_object(acpi_handle node,const char * name,struct acpi_object_list * params,struct acpi_buffer * result)1411 acpi_evaluate_object(acpi_handle node, const char *name,
1412 struct acpi_object_list *params, struct acpi_buffer *result)
1413 {
1414 struct aml_value args[4], res;
1415 union acpi_object *obj;
1416 uint8_t *data;
1417 int i;
1418
1419 KASSERT(params->count <= nitems(args));
1420
1421 for (i = 0; i < params->count; i++) {
1422 args[i].type = params->pointer[i].type;
1423 switch (args[i].type) {
1424 case AML_OBJTYPE_INTEGER:
1425 args[i].v_integer = params->pointer[i].integer.value;
1426 break;
1427 case AML_OBJTYPE_BUFFER:
1428 args[i].length = params->pointer[i].buffer.length;
1429 args[i].v_buffer = params->pointer[i].buffer.pointer;
1430 break;
1431 default:
1432 printf("%s: arg type 0x%02x", __func__, args[i].type);
1433 return AE_BAD_PARAMETER;
1434 }
1435 }
1436
1437 if (name) {
1438 node = aml_searchname(node, name);
1439 if (node == NULL)
1440 return AE_NOT_FOUND;
1441 }
1442 if (aml_evalnode(acpi_softc, node, params->count, args, &res)) {
1443 aml_freevalue(&res);
1444 return AE_ERROR;
1445 }
1446
1447 KASSERT(result->length == ACPI_ALLOCATE_BUFFER);
1448
1449 result->length = sizeof(union acpi_object);
1450 switch (res.type) {
1451 case AML_OBJTYPE_BUFFER:
1452 result->length += res.length;
1453 result->pointer = malloc(result->length, M_DRM, M_WAITOK);
1454 obj = (union acpi_object *)result->pointer;
1455 data = (uint8_t *)(obj + 1);
1456 obj->type = res.type;
1457 obj->buffer.length = res.length;
1458 obj->buffer.pointer = data;
1459 memcpy(data, res.v_buffer, res.length);
1460 break;
1461 default:
1462 printf("%s: return type 0x%02x", __func__, res.type);
1463 aml_freevalue(&res);
1464 return AE_ERROR;
1465 }
1466
1467 aml_freevalue(&res);
1468 return 0;
1469 }
1470
1471 SLIST_HEAD(, notifier_block) drm_linux_acpi_notify_list =
1472 SLIST_HEAD_INITIALIZER(drm_linux_acpi_notify_list);
1473
1474 int
drm_linux_acpi_notify(struct aml_node * node,int notify,void * arg)1475 drm_linux_acpi_notify(struct aml_node *node, int notify, void *arg)
1476 {
1477 struct acpi_bus_event event;
1478 struct notifier_block *nb;
1479
1480 event.device_class = ACPI_VIDEO_CLASS;
1481 event.type = notify;
1482
1483 SLIST_FOREACH(nb, &drm_linux_acpi_notify_list, link)
1484 nb->notifier_call(nb, 0, &event);
1485 return 0;
1486 }
1487
1488 int
register_acpi_notifier(struct notifier_block * nb)1489 register_acpi_notifier(struct notifier_block *nb)
1490 {
1491 SLIST_INSERT_HEAD(&drm_linux_acpi_notify_list, nb, link);
1492 return 0;
1493 }
1494
1495 int
unregister_acpi_notifier(struct notifier_block * nb)1496 unregister_acpi_notifier(struct notifier_block *nb)
1497 {
1498 struct notifier_block *tmp;
1499
1500 SLIST_FOREACH(tmp, &drm_linux_acpi_notify_list, link) {
1501 if (tmp == nb) {
1502 SLIST_REMOVE(&drm_linux_acpi_notify_list, nb,
1503 notifier_block, link);
1504 return 0;
1505 }
1506 }
1507
1508 return -ENOENT;
1509 }
1510
1511 const char *
acpi_format_exception(acpi_status status)1512 acpi_format_exception(acpi_status status)
1513 {
1514 switch (status) {
1515 case AE_NOT_FOUND:
1516 return "not found";
1517 case AE_BAD_PARAMETER:
1518 return "bad parameter";
1519 default:
1520 return "unknown";
1521 }
1522 }
1523
1524 int
acpi_target_system_state(void)1525 acpi_target_system_state(void)
1526 {
1527 return acpi_softc->sc_state;
1528 }
1529
1530 #endif
1531
1532 SLIST_HEAD(,backlight_device) backlight_device_list =
1533 SLIST_HEAD_INITIALIZER(backlight_device_list);
1534
1535 void
backlight_do_update_status(void * arg)1536 backlight_do_update_status(void *arg)
1537 {
1538 backlight_update_status(arg);
1539 }
1540
1541 struct backlight_device *
backlight_device_register(const char * name,void * kdev,void * data,const struct backlight_ops * ops,const struct backlight_properties * props)1542 backlight_device_register(const char *name, void *kdev, void *data,
1543 const struct backlight_ops *ops, const struct backlight_properties *props)
1544 {
1545 struct backlight_device *bd;
1546
1547 bd = malloc(sizeof(*bd), M_DRM, M_WAITOK);
1548 bd->ops = ops;
1549 bd->props = *props;
1550 bd->data = data;
1551
1552 task_set(&bd->task, backlight_do_update_status, bd);
1553
1554 SLIST_INSERT_HEAD(&backlight_device_list, bd, next);
1555 bd->name = name;
1556
1557 return bd;
1558 }
1559
1560 void
backlight_device_unregister(struct backlight_device * bd)1561 backlight_device_unregister(struct backlight_device *bd)
1562 {
1563 SLIST_REMOVE(&backlight_device_list, bd, backlight_device, next);
1564 free(bd, M_DRM, sizeof(*bd));
1565 }
1566
1567 void
backlight_schedule_update_status(struct backlight_device * bd)1568 backlight_schedule_update_status(struct backlight_device *bd)
1569 {
1570 task_add(systq, &bd->task);
1571 }
1572
1573 int
backlight_enable(struct backlight_device * bd)1574 backlight_enable(struct backlight_device *bd)
1575 {
1576 if (bd == NULL)
1577 return 0;
1578
1579 bd->props.power = FB_BLANK_UNBLANK;
1580
1581 return bd->ops->update_status(bd);
1582 }
1583
1584 int
backlight_disable(struct backlight_device * bd)1585 backlight_disable(struct backlight_device *bd)
1586 {
1587 if (bd == NULL)
1588 return 0;
1589
1590 bd->props.power = FB_BLANK_POWERDOWN;
1591
1592 return bd->ops->update_status(bd);
1593 }
1594
1595 struct backlight_device *
backlight_device_get_by_name(const char * name)1596 backlight_device_get_by_name(const char *name)
1597 {
1598 struct backlight_device *bd;
1599
1600 SLIST_FOREACH(bd, &backlight_device_list, next) {
1601 if (strcmp(name, bd->name) == 0)
1602 return bd;
1603 }
1604
1605 return NULL;
1606 }
1607
1608 struct drvdata {
1609 struct device *dev;
1610 void *data;
1611 SLIST_ENTRY(drvdata) next;
1612 };
1613
1614 SLIST_HEAD(,drvdata) drvdata_list = SLIST_HEAD_INITIALIZER(drvdata_list);
1615
1616 void
dev_set_drvdata(struct device * dev,void * data)1617 dev_set_drvdata(struct device *dev, void *data)
1618 {
1619 struct drvdata *drvdata;
1620
1621 SLIST_FOREACH(drvdata, &drvdata_list, next) {
1622 if (drvdata->dev == dev) {
1623 drvdata->data = data;
1624 return;
1625 }
1626 }
1627
1628 if (data == NULL)
1629 return;
1630
1631 drvdata = malloc(sizeof(*drvdata), M_DRM, M_WAITOK);
1632 drvdata->dev = dev;
1633 drvdata->data = data;
1634
1635 SLIST_INSERT_HEAD(&drvdata_list, drvdata, next);
1636 }
1637
1638 void *
dev_get_drvdata(struct device * dev)1639 dev_get_drvdata(struct device *dev)
1640 {
1641 struct drvdata *drvdata;
1642
1643 SLIST_FOREACH(drvdata, &drvdata_list, next) {
1644 if (drvdata->dev == dev)
1645 return drvdata->data;
1646 }
1647
1648 return NULL;
1649 }
1650
1651 void
drm_sysfs_hotplug_event(struct drm_device * dev)1652 drm_sysfs_hotplug_event(struct drm_device *dev)
1653 {
1654 knote_locked(&dev->note, NOTE_CHANGE);
1655 }
1656
1657 void
drm_sysfs_connector_hotplug_event(struct drm_connector * connector)1658 drm_sysfs_connector_hotplug_event(struct drm_connector *connector)
1659 {
1660 knote_locked(&connector->dev->note, NOTE_CHANGE);
1661 }
1662
1663 void
drm_sysfs_connector_status_event(struct drm_connector * connector,struct drm_property * property)1664 drm_sysfs_connector_status_event(struct drm_connector *connector,
1665 struct drm_property *property)
1666 {
1667 STUB();
1668 }
1669
1670 void
drm_sysfs_connector_property_event(struct drm_connector * connector,struct drm_property * property)1671 drm_sysfs_connector_property_event(struct drm_connector *connector,
1672 struct drm_property *property)
1673 {
1674 STUB();
1675 }
1676
1677 struct dma_fence *
dma_fence_get(struct dma_fence * fence)1678 dma_fence_get(struct dma_fence *fence)
1679 {
1680 if (fence)
1681 kref_get(&fence->refcount);
1682 return fence;
1683 }
1684
1685 struct dma_fence *
dma_fence_get_rcu(struct dma_fence * fence)1686 dma_fence_get_rcu(struct dma_fence *fence)
1687 {
1688 if (fence)
1689 kref_get(&fence->refcount);
1690 return fence;
1691 }
1692
1693 struct dma_fence *
dma_fence_get_rcu_safe(struct dma_fence ** dfp)1694 dma_fence_get_rcu_safe(struct dma_fence **dfp)
1695 {
1696 struct dma_fence *fence;
1697 if (dfp == NULL)
1698 return NULL;
1699 fence = *dfp;
1700 if (fence)
1701 kref_get(&fence->refcount);
1702 return fence;
1703 }
1704
1705 void
dma_fence_release(struct kref * ref)1706 dma_fence_release(struct kref *ref)
1707 {
1708 struct dma_fence *fence = container_of(ref, struct dma_fence, refcount);
1709 if (fence->ops && fence->ops->release)
1710 fence->ops->release(fence);
1711 else
1712 free(fence, M_DRM, 0);
1713 }
1714
1715 void
dma_fence_put(struct dma_fence * fence)1716 dma_fence_put(struct dma_fence *fence)
1717 {
1718 if (fence)
1719 kref_put(&fence->refcount, dma_fence_release);
1720 }
1721
1722 int
dma_fence_signal_timestamp_locked(struct dma_fence * fence,ktime_t timestamp)1723 dma_fence_signal_timestamp_locked(struct dma_fence *fence, ktime_t timestamp)
1724 {
1725 struct dma_fence_cb *cur, *tmp;
1726 struct list_head cb_list;
1727
1728 if (fence == NULL)
1729 return -EINVAL;
1730
1731 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1732 return -EINVAL;
1733
1734 list_replace(&fence->cb_list, &cb_list);
1735
1736 fence->timestamp = timestamp;
1737 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
1738
1739 list_for_each_entry_safe(cur, tmp, &cb_list, node) {
1740 INIT_LIST_HEAD(&cur->node);
1741 cur->func(fence, cur);
1742 }
1743
1744 return 0;
1745 }
1746
1747 int
dma_fence_signal(struct dma_fence * fence)1748 dma_fence_signal(struct dma_fence *fence)
1749 {
1750 int r;
1751
1752 if (fence == NULL)
1753 return -EINVAL;
1754
1755 mtx_enter(fence->lock);
1756 r = dma_fence_signal_timestamp_locked(fence, ktime_get());
1757 mtx_leave(fence->lock);
1758
1759 return r;
1760 }
1761
1762 int
dma_fence_signal_locked(struct dma_fence * fence)1763 dma_fence_signal_locked(struct dma_fence *fence)
1764 {
1765 if (fence == NULL)
1766 return -EINVAL;
1767
1768 return dma_fence_signal_timestamp_locked(fence, ktime_get());
1769 }
1770
1771 int
dma_fence_signal_timestamp(struct dma_fence * fence,ktime_t timestamp)1772 dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp)
1773 {
1774 int r;
1775
1776 if (fence == NULL)
1777 return -EINVAL;
1778
1779 mtx_enter(fence->lock);
1780 r = dma_fence_signal_timestamp_locked(fence, timestamp);
1781 mtx_leave(fence->lock);
1782
1783 return r;
1784 }
1785
1786 bool
dma_fence_is_signaled(struct dma_fence * fence)1787 dma_fence_is_signaled(struct dma_fence *fence)
1788 {
1789 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1790 return true;
1791
1792 if (fence->ops->signaled && fence->ops->signaled(fence)) {
1793 dma_fence_signal(fence);
1794 return true;
1795 }
1796
1797 return false;
1798 }
1799
1800 bool
dma_fence_is_signaled_locked(struct dma_fence * fence)1801 dma_fence_is_signaled_locked(struct dma_fence *fence)
1802 {
1803 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1804 return true;
1805
1806 if (fence->ops->signaled && fence->ops->signaled(fence)) {
1807 dma_fence_signal_locked(fence);
1808 return true;
1809 }
1810
1811 return false;
1812 }
1813
1814 ktime_t
dma_fence_timestamp(struct dma_fence * fence)1815 dma_fence_timestamp(struct dma_fence *fence)
1816 {
1817 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
1818 while (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags))
1819 CPU_BUSY_CYCLE();
1820 return fence->timestamp;
1821 } else {
1822 return ktime_get();
1823 }
1824 }
1825
1826 long
dma_fence_wait_timeout(struct dma_fence * fence,bool intr,long timeout)1827 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, long timeout)
1828 {
1829 if (timeout < 0)
1830 return -EINVAL;
1831
1832 if (fence->ops->wait)
1833 return fence->ops->wait(fence, intr, timeout);
1834 else
1835 return dma_fence_default_wait(fence, intr, timeout);
1836 }
1837
1838 long
dma_fence_wait(struct dma_fence * fence,bool intr)1839 dma_fence_wait(struct dma_fence *fence, bool intr)
1840 {
1841 long ret;
1842
1843 ret = dma_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);
1844 if (ret < 0)
1845 return ret;
1846
1847 return 0;
1848 }
1849
1850 void
dma_fence_enable_sw_signaling(struct dma_fence * fence)1851 dma_fence_enable_sw_signaling(struct dma_fence *fence)
1852 {
1853 if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags) &&
1854 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) &&
1855 fence->ops->enable_signaling) {
1856 mtx_enter(fence->lock);
1857 if (!fence->ops->enable_signaling(fence))
1858 dma_fence_signal_locked(fence);
1859 mtx_leave(fence->lock);
1860 }
1861 }
1862
1863 void
dma_fence_init(struct dma_fence * fence,const struct dma_fence_ops * ops,struct mutex * lock,uint64_t context,uint64_t seqno)1864 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
1865 struct mutex *lock, uint64_t context, uint64_t seqno)
1866 {
1867 fence->ops = ops;
1868 fence->lock = lock;
1869 fence->context = context;
1870 fence->seqno = seqno;
1871 fence->flags = 0;
1872 fence->error = 0;
1873 kref_init(&fence->refcount);
1874 INIT_LIST_HEAD(&fence->cb_list);
1875 }
1876
1877 int
dma_fence_add_callback(struct dma_fence * fence,struct dma_fence_cb * cb,dma_fence_func_t func)1878 dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
1879 dma_fence_func_t func)
1880 {
1881 int ret = 0;
1882 bool was_set;
1883
1884 if (WARN_ON(!fence || !func))
1885 return -EINVAL;
1886
1887 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
1888 INIT_LIST_HEAD(&cb->node);
1889 return -ENOENT;
1890 }
1891
1892 mtx_enter(fence->lock);
1893
1894 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags);
1895
1896 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1897 ret = -ENOENT;
1898 else if (!was_set && fence->ops->enable_signaling) {
1899 if (!fence->ops->enable_signaling(fence)) {
1900 dma_fence_signal_locked(fence);
1901 ret = -ENOENT;
1902 }
1903 }
1904
1905 if (!ret) {
1906 cb->func = func;
1907 list_add_tail(&cb->node, &fence->cb_list);
1908 } else
1909 INIT_LIST_HEAD(&cb->node);
1910 mtx_leave(fence->lock);
1911
1912 return ret;
1913 }
1914
1915 bool
dma_fence_remove_callback(struct dma_fence * fence,struct dma_fence_cb * cb)1916 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
1917 {
1918 bool ret;
1919
1920 mtx_enter(fence->lock);
1921
1922 ret = !list_empty(&cb->node);
1923 if (ret)
1924 list_del_init(&cb->node);
1925
1926 mtx_leave(fence->lock);
1927
1928 return ret;
1929 }
1930
1931 static atomic64_t drm_fence_context_count = ATOMIC64_INIT(1);
1932
1933 uint64_t
dma_fence_context_alloc(unsigned int num)1934 dma_fence_context_alloc(unsigned int num)
1935 {
1936 return atomic64_add_return(num, &drm_fence_context_count) - num;
1937 }
1938
1939 struct default_wait_cb {
1940 struct dma_fence_cb base;
1941 struct proc *proc;
1942 };
1943
1944 static void
dma_fence_default_wait_cb(struct dma_fence * fence,struct dma_fence_cb * cb)1945 dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
1946 {
1947 struct default_wait_cb *wait =
1948 container_of(cb, struct default_wait_cb, base);
1949 wake_up_process(wait->proc);
1950 }
1951
1952 long
dma_fence_default_wait(struct dma_fence * fence,bool intr,signed long timeout)1953 dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
1954 {
1955 long ret = timeout ? timeout : 1;
1956 unsigned long end;
1957 int err;
1958 struct default_wait_cb cb;
1959 bool was_set;
1960
1961 KASSERT(timeout <= INT_MAX);
1962
1963 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1964 return ret;
1965
1966 mtx_enter(fence->lock);
1967
1968 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
1969 &fence->flags);
1970
1971 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1972 goto out;
1973
1974 if (!was_set && fence->ops->enable_signaling) {
1975 if (!fence->ops->enable_signaling(fence)) {
1976 dma_fence_signal_locked(fence);
1977 goto out;
1978 }
1979 }
1980
1981 if (timeout == 0) {
1982 ret = 0;
1983 goto out;
1984 }
1985
1986 cb.base.func = dma_fence_default_wait_cb;
1987 cb.proc = curproc;
1988 list_add(&cb.base.node, &fence->cb_list);
1989
1990 end = jiffies + timeout;
1991 for (ret = timeout; ret > 0; ret = MAX(0, end - jiffies)) {
1992 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1993 break;
1994 err = msleep(curproc, fence->lock, intr ? PCATCH : 0,
1995 "dmafence", ret);
1996 if (err == EINTR || err == ERESTART) {
1997 ret = -ERESTARTSYS;
1998 break;
1999 }
2000 }
2001
2002 if (!list_empty(&cb.base.node))
2003 list_del(&cb.base.node);
2004 out:
2005 mtx_leave(fence->lock);
2006
2007 return ret;
2008 }
2009
2010 static bool
dma_fence_test_signaled_any(struct dma_fence ** fences,uint32_t count,uint32_t * idx)2011 dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
2012 uint32_t *idx)
2013 {
2014 int i;
2015
2016 for (i = 0; i < count; ++i) {
2017 struct dma_fence *fence = fences[i];
2018 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
2019 if (idx)
2020 *idx = i;
2021 return true;
2022 }
2023 }
2024 return false;
2025 }
2026
2027 long
dma_fence_wait_any_timeout(struct dma_fence ** fences,uint32_t count,bool intr,long timeout,uint32_t * idx)2028 dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
2029 bool intr, long timeout, uint32_t *idx)
2030 {
2031 struct default_wait_cb *cb;
2032 long ret = timeout;
2033 unsigned long end;
2034 int i, err;
2035
2036 KASSERT(timeout <= INT_MAX);
2037
2038 if (timeout == 0) {
2039 for (i = 0; i < count; i++) {
2040 if (dma_fence_is_signaled(fences[i])) {
2041 if (idx)
2042 *idx = i;
2043 return 1;
2044 }
2045 }
2046 return 0;
2047 }
2048
2049 cb = mallocarray(count, sizeof(*cb), M_DRM, M_WAITOK|M_CANFAIL|M_ZERO);
2050 if (cb == NULL)
2051 return -ENOMEM;
2052
2053 for (i = 0; i < count; i++) {
2054 struct dma_fence *fence = fences[i];
2055 cb[i].proc = curproc;
2056 if (dma_fence_add_callback(fence, &cb[i].base,
2057 dma_fence_default_wait_cb)) {
2058 if (idx)
2059 *idx = i;
2060 goto cb_cleanup;
2061 }
2062 }
2063
2064 end = jiffies + timeout;
2065 for (ret = timeout; ret > 0; ret = MAX(0, end - jiffies)) {
2066 if (dma_fence_test_signaled_any(fences, count, idx))
2067 break;
2068 err = tsleep(curproc, intr ? PCATCH : 0, "dfwat", ret);
2069 if (err == EINTR || err == ERESTART) {
2070 ret = -ERESTARTSYS;
2071 break;
2072 }
2073 }
2074
2075 cb_cleanup:
2076 while (i-- > 0)
2077 dma_fence_remove_callback(fences[i], &cb[i].base);
2078 free(cb, M_DRM, count * sizeof(*cb));
2079 return ret;
2080 }
2081
2082 void
dma_fence_set_deadline(struct dma_fence * f,ktime_t t)2083 dma_fence_set_deadline(struct dma_fence *f, ktime_t t)
2084 {
2085 if (f->ops->set_deadline == NULL)
2086 return;
2087 if (dma_fence_is_signaled(f) == false)
2088 f->ops->set_deadline(f, t);
2089 }
2090
2091 static struct dma_fence dma_fence_stub;
2092 static struct mutex dma_fence_stub_mtx = MUTEX_INITIALIZER(IPL_TTY);
2093
2094 static const char *
dma_fence_stub_get_name(struct dma_fence * fence)2095 dma_fence_stub_get_name(struct dma_fence *fence)
2096 {
2097 return "stub";
2098 }
2099
2100 static const struct dma_fence_ops dma_fence_stub_ops = {
2101 .get_driver_name = dma_fence_stub_get_name,
2102 .get_timeline_name = dma_fence_stub_get_name,
2103 };
2104
2105 struct dma_fence *
dma_fence_get_stub(void)2106 dma_fence_get_stub(void)
2107 {
2108 mtx_enter(&dma_fence_stub_mtx);
2109 if (dma_fence_stub.ops == NULL) {
2110 dma_fence_init(&dma_fence_stub, &dma_fence_stub_ops,
2111 &dma_fence_stub_mtx, 0, 0);
2112 dma_fence_signal_locked(&dma_fence_stub);
2113 }
2114 mtx_leave(&dma_fence_stub_mtx);
2115
2116 return dma_fence_get(&dma_fence_stub);
2117 }
2118
2119 struct dma_fence *
dma_fence_allocate_private_stub(ktime_t ts)2120 dma_fence_allocate_private_stub(ktime_t ts)
2121 {
2122 struct dma_fence *f = malloc(sizeof(*f), M_DRM,
2123 M_ZERO | M_WAITOK | M_CANFAIL);
2124 if (f == NULL)
2125 return NULL;
2126 dma_fence_init(f, &dma_fence_stub_ops, &dma_fence_stub_mtx, 0, 0);
2127 dma_fence_signal_timestamp(f, ts);
2128 return f;
2129 }
2130
2131 static const char *
dma_fence_array_get_driver_name(struct dma_fence * fence)2132 dma_fence_array_get_driver_name(struct dma_fence *fence)
2133 {
2134 return "dma_fence_array";
2135 }
2136
2137 static const char *
dma_fence_array_get_timeline_name(struct dma_fence * fence)2138 dma_fence_array_get_timeline_name(struct dma_fence *fence)
2139 {
2140 return "unbound";
2141 }
2142
2143 static void
irq_dma_fence_array_work(void * arg)2144 irq_dma_fence_array_work(void *arg)
2145 {
2146 struct dma_fence_array *dfa = (struct dma_fence_array *)arg;
2147 dma_fence_signal(&dfa->base);
2148 dma_fence_put(&dfa->base);
2149 }
2150
2151 static void
dma_fence_array_cb_func(struct dma_fence * f,struct dma_fence_cb * cb)2152 dma_fence_array_cb_func(struct dma_fence *f, struct dma_fence_cb *cb)
2153 {
2154 struct dma_fence_array_cb *array_cb =
2155 container_of(cb, struct dma_fence_array_cb, cb);
2156 struct dma_fence_array *dfa = array_cb->array;
2157
2158 if (atomic_dec_and_test(&dfa->num_pending))
2159 timeout_add(&dfa->to, 1);
2160 else
2161 dma_fence_put(&dfa->base);
2162 }
2163
2164 static bool
dma_fence_array_enable_signaling(struct dma_fence * fence)2165 dma_fence_array_enable_signaling(struct dma_fence *fence)
2166 {
2167 struct dma_fence_array *dfa = to_dma_fence_array(fence);
2168 struct dma_fence_array_cb *cb = (void *)(&dfa[1]);
2169 int i;
2170
2171 for (i = 0; i < dfa->num_fences; ++i) {
2172 cb[i].array = dfa;
2173 dma_fence_get(&dfa->base);
2174 if (dma_fence_add_callback(dfa->fences[i], &cb[i].cb,
2175 dma_fence_array_cb_func)) {
2176 dma_fence_put(&dfa->base);
2177 if (atomic_dec_and_test(&dfa->num_pending))
2178 return false;
2179 }
2180 }
2181
2182 return true;
2183 }
2184
2185 static bool
dma_fence_array_signaled(struct dma_fence * fence)2186 dma_fence_array_signaled(struct dma_fence *fence)
2187 {
2188 struct dma_fence_array *dfa = to_dma_fence_array(fence);
2189
2190 return atomic_read(&dfa->num_pending) <= 0;
2191 }
2192
2193 static void
dma_fence_array_release(struct dma_fence * fence)2194 dma_fence_array_release(struct dma_fence *fence)
2195 {
2196 struct dma_fence_array *dfa = to_dma_fence_array(fence);
2197 int i;
2198
2199 for (i = 0; i < dfa->num_fences; ++i)
2200 dma_fence_put(dfa->fences[i]);
2201
2202 free(dfa->fences, M_DRM, 0);
2203 dma_fence_free(fence);
2204 }
2205
2206 struct dma_fence_array *
dma_fence_array_create(int num_fences,struct dma_fence ** fences,u64 context,unsigned seqno,bool signal_on_any)2207 dma_fence_array_create(int num_fences, struct dma_fence **fences, u64 context,
2208 unsigned seqno, bool signal_on_any)
2209 {
2210 struct dma_fence_array *dfa = malloc(sizeof(*dfa) +
2211 (num_fences * sizeof(struct dma_fence_array_cb)),
2212 M_DRM, M_WAITOK|M_CANFAIL|M_ZERO);
2213 if (dfa == NULL)
2214 return NULL;
2215
2216 mtx_init(&dfa->lock, IPL_TTY);
2217 dma_fence_init(&dfa->base, &dma_fence_array_ops, &dfa->lock,
2218 context, seqno);
2219 timeout_set(&dfa->to, irq_dma_fence_array_work, dfa);
2220
2221 dfa->num_fences = num_fences;
2222 atomic_set(&dfa->num_pending, signal_on_any ? 1 : num_fences);
2223 dfa->fences = fences;
2224
2225 return dfa;
2226 }
2227
2228 struct dma_fence *
dma_fence_array_first(struct dma_fence * f)2229 dma_fence_array_first(struct dma_fence *f)
2230 {
2231 struct dma_fence_array *dfa;
2232
2233 if (f == NULL)
2234 return NULL;
2235
2236 if ((dfa = to_dma_fence_array(f)) == NULL)
2237 return f;
2238
2239 if (dfa->num_fences > 0)
2240 return dfa->fences[0];
2241
2242 return NULL;
2243 }
2244
2245 struct dma_fence *
dma_fence_array_next(struct dma_fence * f,unsigned int i)2246 dma_fence_array_next(struct dma_fence *f, unsigned int i)
2247 {
2248 struct dma_fence_array *dfa;
2249
2250 if (f == NULL)
2251 return NULL;
2252
2253 if ((dfa = to_dma_fence_array(f)) == NULL)
2254 return NULL;
2255
2256 if (i < dfa->num_fences)
2257 return dfa->fences[i];
2258
2259 return NULL;
2260 }
2261
2262 const struct dma_fence_ops dma_fence_array_ops = {
2263 .get_driver_name = dma_fence_array_get_driver_name,
2264 .get_timeline_name = dma_fence_array_get_timeline_name,
2265 .enable_signaling = dma_fence_array_enable_signaling,
2266 .signaled = dma_fence_array_signaled,
2267 .release = dma_fence_array_release,
2268 };
2269
2270 int
dma_fence_chain_find_seqno(struct dma_fence ** df,uint64_t seqno)2271 dma_fence_chain_find_seqno(struct dma_fence **df, uint64_t seqno)
2272 {
2273 struct dma_fence_chain *chain;
2274 struct dma_fence *fence;
2275
2276 if (seqno == 0)
2277 return 0;
2278
2279 if ((chain = to_dma_fence_chain(*df)) == NULL)
2280 return -EINVAL;
2281
2282 fence = &chain->base;
2283 if (fence->seqno < seqno)
2284 return -EINVAL;
2285
2286 dma_fence_chain_for_each(*df, fence) {
2287 if ((*df)->context != fence->context)
2288 break;
2289
2290 chain = to_dma_fence_chain(*df);
2291 if (chain->prev_seqno < seqno)
2292 break;
2293 }
2294 dma_fence_put(fence);
2295
2296 return 0;
2297 }
2298
2299 void
dma_fence_chain_init(struct dma_fence_chain * chain,struct dma_fence * prev,struct dma_fence * fence,uint64_t seqno)2300 dma_fence_chain_init(struct dma_fence_chain *chain, struct dma_fence *prev,
2301 struct dma_fence *fence, uint64_t seqno)
2302 {
2303 uint64_t context;
2304
2305 chain->fence = fence;
2306 chain->prev = prev;
2307 mtx_init(&chain->lock, IPL_TTY);
2308
2309 /* if prev is a chain */
2310 if (to_dma_fence_chain(prev) != NULL) {
2311 if (__dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
2312 chain->prev_seqno = prev->seqno;
2313 context = prev->context;
2314 } else {
2315 chain->prev_seqno = 0;
2316 context = dma_fence_context_alloc(1);
2317 seqno = prev->seqno;
2318 }
2319 } else {
2320 chain->prev_seqno = 0;
2321 context = dma_fence_context_alloc(1);
2322 }
2323
2324 dma_fence_init(&chain->base, &dma_fence_chain_ops, &chain->lock,
2325 context, seqno);
2326 }
2327
2328 static const char *
dma_fence_chain_get_driver_name(struct dma_fence * fence)2329 dma_fence_chain_get_driver_name(struct dma_fence *fence)
2330 {
2331 return "dma_fence_chain";
2332 }
2333
2334 static const char *
dma_fence_chain_get_timeline_name(struct dma_fence * fence)2335 dma_fence_chain_get_timeline_name(struct dma_fence *fence)
2336 {
2337 return "unbound";
2338 }
2339
2340 static bool dma_fence_chain_enable_signaling(struct dma_fence *);
2341
2342 static void
dma_fence_chain_timo(void * arg)2343 dma_fence_chain_timo(void *arg)
2344 {
2345 struct dma_fence_chain *chain = (struct dma_fence_chain *)arg;
2346
2347 if (dma_fence_chain_enable_signaling(&chain->base) == false)
2348 dma_fence_signal(&chain->base);
2349 dma_fence_put(&chain->base);
2350 }
2351
2352 static void
dma_fence_chain_cb(struct dma_fence * f,struct dma_fence_cb * cb)2353 dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
2354 {
2355 struct dma_fence_chain *chain =
2356 container_of(cb, struct dma_fence_chain, cb);
2357 timeout_set(&chain->to, dma_fence_chain_timo, chain);
2358 timeout_add(&chain->to, 1);
2359 dma_fence_put(f);
2360 }
2361
2362 static bool
dma_fence_chain_enable_signaling(struct dma_fence * fence)2363 dma_fence_chain_enable_signaling(struct dma_fence *fence)
2364 {
2365 struct dma_fence_chain *chain, *h;
2366 struct dma_fence *f;
2367
2368 h = to_dma_fence_chain(fence);
2369 dma_fence_get(&h->base);
2370 dma_fence_chain_for_each(fence, &h->base) {
2371 chain = to_dma_fence_chain(fence);
2372 if (chain == NULL)
2373 f = fence;
2374 else
2375 f = chain->fence;
2376
2377 dma_fence_get(f);
2378 if (!dma_fence_add_callback(f, &h->cb, dma_fence_chain_cb)) {
2379 dma_fence_put(fence);
2380 return true;
2381 }
2382 dma_fence_put(f);
2383 }
2384 dma_fence_put(&h->base);
2385 return false;
2386 }
2387
2388 static bool
dma_fence_chain_signaled(struct dma_fence * fence)2389 dma_fence_chain_signaled(struct dma_fence *fence)
2390 {
2391 struct dma_fence_chain *chain;
2392 struct dma_fence *f;
2393
2394 dma_fence_chain_for_each(fence, fence) {
2395 chain = to_dma_fence_chain(fence);
2396 if (chain == NULL)
2397 f = fence;
2398 else
2399 f = chain->fence;
2400
2401 if (dma_fence_is_signaled(f) == false) {
2402 dma_fence_put(fence);
2403 return false;
2404 }
2405 }
2406 return true;
2407 }
2408
2409 static void
dma_fence_chain_release(struct dma_fence * fence)2410 dma_fence_chain_release(struct dma_fence *fence)
2411 {
2412 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
2413 struct dma_fence_chain *prev_chain;
2414 struct dma_fence *prev;
2415
2416 for (prev = chain->prev; prev != NULL; prev = chain->prev) {
2417 if (kref_read(&prev->refcount) > 1)
2418 break;
2419 if ((prev_chain = to_dma_fence_chain(prev)) == NULL)
2420 break;
2421 chain->prev = prev_chain->prev;
2422 prev_chain->prev = NULL;
2423 dma_fence_put(prev);
2424 }
2425 dma_fence_put(prev);
2426 dma_fence_put(chain->fence);
2427 dma_fence_free(fence);
2428 }
2429
2430 struct dma_fence *
dma_fence_chain_walk(struct dma_fence * fence)2431 dma_fence_chain_walk(struct dma_fence *fence)
2432 {
2433 struct dma_fence_chain *chain = to_dma_fence_chain(fence), *prev_chain;
2434 struct dma_fence *prev, *new_prev, *tmp;
2435
2436 if (chain == NULL) {
2437 dma_fence_put(fence);
2438 return NULL;
2439 }
2440
2441 while ((prev = dma_fence_get(chain->prev)) != NULL) {
2442 prev_chain = to_dma_fence_chain(prev);
2443 if (prev_chain != NULL) {
2444 if (!dma_fence_is_signaled(prev_chain->fence))
2445 break;
2446 new_prev = dma_fence_get(prev_chain->prev);
2447 } else {
2448 if (!dma_fence_is_signaled(prev))
2449 break;
2450 new_prev = NULL;
2451 }
2452 tmp = atomic_cas_ptr(&chain->prev, prev, new_prev);
2453 dma_fence_put(tmp == prev ? prev : new_prev);
2454 dma_fence_put(prev);
2455 }
2456
2457 dma_fence_put(fence);
2458 return prev;
2459 }
2460
2461 const struct dma_fence_ops dma_fence_chain_ops = {
2462 .get_driver_name = dma_fence_chain_get_driver_name,
2463 .get_timeline_name = dma_fence_chain_get_timeline_name,
2464 .enable_signaling = dma_fence_chain_enable_signaling,
2465 .signaled = dma_fence_chain_signaled,
2466 .release = dma_fence_chain_release,
2467 .use_64bit_seqno = true,
2468 };
2469
2470 bool
dma_fence_is_container(struct dma_fence * fence)2471 dma_fence_is_container(struct dma_fence *fence)
2472 {
2473 return (fence->ops == &dma_fence_chain_ops) ||
2474 (fence->ops == &dma_fence_array_ops);
2475 }
2476
2477 int
dmabuf_read(struct file * fp,struct uio * uio,int fflags)2478 dmabuf_read(struct file *fp, struct uio *uio, int fflags)
2479 {
2480 return (ENXIO);
2481 }
2482
2483 int
dmabuf_write(struct file * fp,struct uio * uio,int fflags)2484 dmabuf_write(struct file *fp, struct uio *uio, int fflags)
2485 {
2486 return (ENXIO);
2487 }
2488
2489 int
dmabuf_ioctl(struct file * fp,u_long com,caddr_t data,struct proc * p)2490 dmabuf_ioctl(struct file *fp, u_long com, caddr_t data, struct proc *p)
2491 {
2492 return (ENOTTY);
2493 }
2494
2495 int
dmabuf_kqfilter(struct file * fp,struct knote * kn)2496 dmabuf_kqfilter(struct file *fp, struct knote *kn)
2497 {
2498 return (EINVAL);
2499 }
2500
2501 int
dmabuf_stat(struct file * fp,struct stat * st,struct proc * p)2502 dmabuf_stat(struct file *fp, struct stat *st, struct proc *p)
2503 {
2504 struct dma_buf *dmabuf = fp->f_data;
2505
2506 memset(st, 0, sizeof(*st));
2507 st->st_size = dmabuf->size;
2508 st->st_mode = S_IFIFO; /* XXX */
2509 return (0);
2510 }
2511
2512 int
dmabuf_close(struct file * fp,struct proc * p)2513 dmabuf_close(struct file *fp, struct proc *p)
2514 {
2515 struct dma_buf *dmabuf = fp->f_data;
2516
2517 fp->f_data = NULL;
2518 KERNEL_LOCK();
2519 dmabuf->ops->release(dmabuf);
2520 KERNEL_UNLOCK();
2521 free(dmabuf, M_DRM, sizeof(struct dma_buf));
2522 return (0);
2523 }
2524
2525 int
dmabuf_seek(struct file * fp,off_t * offset,int whence,struct proc * p)2526 dmabuf_seek(struct file *fp, off_t *offset, int whence, struct proc *p)
2527 {
2528 struct dma_buf *dmabuf = fp->f_data;
2529 off_t newoff;
2530
2531 if (*offset != 0)
2532 return (EINVAL);
2533
2534 switch (whence) {
2535 case SEEK_SET:
2536 newoff = 0;
2537 break;
2538 case SEEK_END:
2539 newoff = dmabuf->size;
2540 break;
2541 default:
2542 return (EINVAL);
2543 }
2544 mtx_enter(&fp->f_mtx);
2545 fp->f_offset = newoff;
2546 mtx_leave(&fp->f_mtx);
2547 *offset = newoff;
2548 return (0);
2549 }
2550
2551 const struct fileops dmabufops = {
2552 .fo_read = dmabuf_read,
2553 .fo_write = dmabuf_write,
2554 .fo_ioctl = dmabuf_ioctl,
2555 .fo_kqfilter = dmabuf_kqfilter,
2556 .fo_stat = dmabuf_stat,
2557 .fo_close = dmabuf_close,
2558 .fo_seek = dmabuf_seek,
2559 };
2560
2561 struct dma_buf *
dma_buf_export(const struct dma_buf_export_info * info)2562 dma_buf_export(const struct dma_buf_export_info *info)
2563 {
2564 struct proc *p = curproc;
2565 struct dma_buf *dmabuf;
2566 struct file *fp;
2567
2568 fp = fnew(p);
2569 if (fp == NULL)
2570 return ERR_PTR(-ENFILE);
2571 fp->f_type = DTYPE_DMABUF;
2572 fp->f_ops = &dmabufops;
2573 dmabuf = malloc(sizeof(struct dma_buf), M_DRM, M_WAITOK | M_ZERO);
2574 dmabuf->priv = info->priv;
2575 dmabuf->ops = info->ops;
2576 dmabuf->size = info->size;
2577 dmabuf->file = fp;
2578 fp->f_data = dmabuf;
2579 INIT_LIST_HEAD(&dmabuf->attachments);
2580 return dmabuf;
2581 }
2582
2583 struct dma_buf *
dma_buf_get(int fd)2584 dma_buf_get(int fd)
2585 {
2586 struct proc *p = curproc;
2587 struct filedesc *fdp = p->p_fd;
2588 struct file *fp;
2589
2590 if ((fp = fd_getfile(fdp, fd)) == NULL)
2591 return ERR_PTR(-EBADF);
2592
2593 if (fp->f_type != DTYPE_DMABUF) {
2594 FRELE(fp, p);
2595 return ERR_PTR(-EINVAL);
2596 }
2597
2598 return fp->f_data;
2599 }
2600
2601 void
dma_buf_put(struct dma_buf * dmabuf)2602 dma_buf_put(struct dma_buf *dmabuf)
2603 {
2604 KASSERT(dmabuf);
2605 KASSERT(dmabuf->file);
2606
2607 FRELE(dmabuf->file, curproc);
2608 }
2609
2610 int
dma_buf_fd(struct dma_buf * dmabuf,int flags)2611 dma_buf_fd(struct dma_buf *dmabuf, int flags)
2612 {
2613 struct proc *p = curproc;
2614 struct filedesc *fdp = p->p_fd;
2615 struct file *fp = dmabuf->file;
2616 int fd, cloexec, error;
2617
2618 cloexec = (flags & O_CLOEXEC) ? UF_EXCLOSE : 0;
2619
2620 fdplock(fdp);
2621 restart:
2622 if ((error = fdalloc(p, 0, &fd)) != 0) {
2623 if (error == ENOSPC) {
2624 fdexpand(p);
2625 goto restart;
2626 }
2627 fdpunlock(fdp);
2628 return -error;
2629 }
2630
2631 fdinsert(fdp, fd, cloexec, fp);
2632 fdpunlock(fdp);
2633
2634 return fd;
2635 }
2636
2637 void
get_dma_buf(struct dma_buf * dmabuf)2638 get_dma_buf(struct dma_buf *dmabuf)
2639 {
2640 FREF(dmabuf->file);
2641 }
2642
2643 enum pci_bus_speed
pcie_get_speed_cap(struct pci_dev * pdev)2644 pcie_get_speed_cap(struct pci_dev *pdev)
2645 {
2646 pci_chipset_tag_t pc;
2647 pcitag_t tag;
2648 int pos ;
2649 pcireg_t xcap, lnkcap = 0, lnkcap2 = 0;
2650 pcireg_t id;
2651 enum pci_bus_speed cap = PCI_SPEED_UNKNOWN;
2652 int bus, device, function;
2653
2654 if (pdev == NULL)
2655 return PCI_SPEED_UNKNOWN;
2656
2657 pc = pdev->pc;
2658 tag = pdev->tag;
2659
2660 if (!pci_get_capability(pc, tag, PCI_CAP_PCIEXPRESS,
2661 &pos, NULL))
2662 return PCI_SPEED_UNKNOWN;
2663
2664 id = pci_conf_read(pc, tag, PCI_ID_REG);
2665 pci_decompose_tag(pc, tag, &bus, &device, &function);
2666
2667 /* we've been informed via and serverworks don't make the cut */
2668 if (PCI_VENDOR(id) == PCI_VENDOR_VIATECH ||
2669 PCI_VENDOR(id) == PCI_VENDOR_RCC)
2670 return PCI_SPEED_UNKNOWN;
2671
2672 lnkcap = pci_conf_read(pc, tag, pos + PCI_PCIE_LCAP);
2673 xcap = pci_conf_read(pc, tag, pos + PCI_PCIE_XCAP);
2674 if (PCI_PCIE_XCAP_VER(xcap) >= 2)
2675 lnkcap2 = pci_conf_read(pc, tag, pos + PCI_PCIE_LCAP2);
2676
2677 lnkcap &= 0x0f;
2678 lnkcap2 &= 0xfe;
2679
2680 if (lnkcap2) { /* PCIE GEN 3.0 */
2681 if (lnkcap2 & 0x02)
2682 cap = PCIE_SPEED_2_5GT;
2683 if (lnkcap2 & 0x04)
2684 cap = PCIE_SPEED_5_0GT;
2685 if (lnkcap2 & 0x08)
2686 cap = PCIE_SPEED_8_0GT;
2687 if (lnkcap2 & 0x10)
2688 cap = PCIE_SPEED_16_0GT;
2689 if (lnkcap2 & 0x20)
2690 cap = PCIE_SPEED_32_0GT;
2691 if (lnkcap2 & 0x40)
2692 cap = PCIE_SPEED_64_0GT;
2693 } else {
2694 if (lnkcap & 0x01)
2695 cap = PCIE_SPEED_2_5GT;
2696 if (lnkcap & 0x02)
2697 cap = PCIE_SPEED_5_0GT;
2698 }
2699
2700 DRM_INFO("probing pcie caps for device %d:%d:%d 0x%04x:0x%04x = %x/%x\n",
2701 bus, device, function, PCI_VENDOR(id), PCI_PRODUCT(id), lnkcap,
2702 lnkcap2);
2703 return cap;
2704 }
2705
2706 enum pcie_link_width
pcie_get_width_cap(struct pci_dev * pdev)2707 pcie_get_width_cap(struct pci_dev *pdev)
2708 {
2709 pci_chipset_tag_t pc = pdev->pc;
2710 pcitag_t tag = pdev->tag;
2711 int pos ;
2712 pcireg_t lnkcap = 0;
2713 pcireg_t id;
2714 int bus, device, function;
2715
2716 if (!pci_get_capability(pc, tag, PCI_CAP_PCIEXPRESS,
2717 &pos, NULL))
2718 return PCIE_LNK_WIDTH_UNKNOWN;
2719
2720 id = pci_conf_read(pc, tag, PCI_ID_REG);
2721 pci_decompose_tag(pc, tag, &bus, &device, &function);
2722
2723 lnkcap = pci_conf_read(pc, tag, pos + PCI_PCIE_LCAP);
2724
2725 DRM_INFO("probing pcie width for device %d:%d:%d 0x%04x:0x%04x = %x\n",
2726 bus, device, function, PCI_VENDOR(id), PCI_PRODUCT(id), lnkcap);
2727
2728 if (lnkcap)
2729 return (lnkcap & 0x3f0) >> 4;
2730 return PCIE_LNK_WIDTH_UNKNOWN;
2731 }
2732
2733 bool
pcie_aspm_enabled(struct pci_dev * pdev)2734 pcie_aspm_enabled(struct pci_dev *pdev)
2735 {
2736 pci_chipset_tag_t pc = pdev->pc;
2737 pcitag_t tag = pdev->tag;
2738 int pos ;
2739 pcireg_t lcsr;
2740
2741 if (!pci_get_capability(pc, tag, PCI_CAP_PCIEXPRESS,
2742 &pos, NULL))
2743 return false;
2744
2745 lcsr = pci_conf_read(pc, tag, pos + PCI_PCIE_LCSR);
2746 if ((lcsr & (PCI_PCIE_LCSR_ASPM_L0S | PCI_PCIE_LCSR_ASPM_L1)) != 0)
2747 return true;
2748
2749 return false;
2750 }
2751
2752 static wait_queue_head_t bit_waitq;
2753 wait_queue_head_t var_waitq;
2754 struct mutex wait_bit_mtx = MUTEX_INITIALIZER(IPL_TTY);
2755
2756 int
wait_on_bit(unsigned long * word,int bit,unsigned mode)2757 wait_on_bit(unsigned long *word, int bit, unsigned mode)
2758 {
2759 int err;
2760
2761 if (!test_bit(bit, word))
2762 return 0;
2763
2764 mtx_enter(&wait_bit_mtx);
2765 while (test_bit(bit, word)) {
2766 err = msleep_nsec(word, &wait_bit_mtx, PWAIT | mode, "wtb",
2767 INFSLP);
2768 if (err) {
2769 mtx_leave(&wait_bit_mtx);
2770 return 1;
2771 }
2772 }
2773 mtx_leave(&wait_bit_mtx);
2774 return 0;
2775 }
2776
2777 int
wait_on_bit_timeout(unsigned long * word,int bit,unsigned mode,int timo)2778 wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode, int timo)
2779 {
2780 int err;
2781
2782 if (!test_bit(bit, word))
2783 return 0;
2784
2785 mtx_enter(&wait_bit_mtx);
2786 while (test_bit(bit, word)) {
2787 err = msleep(word, &wait_bit_mtx, PWAIT | mode, "wtb", timo);
2788 if (err) {
2789 mtx_leave(&wait_bit_mtx);
2790 return 1;
2791 }
2792 }
2793 mtx_leave(&wait_bit_mtx);
2794 return 0;
2795 }
2796
2797 void
wake_up_bit(void * word,int bit)2798 wake_up_bit(void *word, int bit)
2799 {
2800 mtx_enter(&wait_bit_mtx);
2801 wakeup(word);
2802 mtx_leave(&wait_bit_mtx);
2803 }
2804
2805 void
clear_and_wake_up_bit(int bit,void * word)2806 clear_and_wake_up_bit(int bit, void *word)
2807 {
2808 clear_bit(bit, word);
2809 wake_up_bit(word, bit);
2810 }
2811
2812 wait_queue_head_t *
bit_waitqueue(void * word,int bit)2813 bit_waitqueue(void *word, int bit)
2814 {
2815 /* XXX hash table of wait queues? */
2816 return &bit_waitq;
2817 }
2818
2819 wait_queue_head_t *
__var_waitqueue(void * p)2820 __var_waitqueue(void *p)
2821 {
2822 /* XXX hash table of wait queues? */
2823 return &bit_waitq;
2824 }
2825
2826 struct workqueue_struct *system_wq;
2827 struct workqueue_struct *system_highpri_wq;
2828 struct workqueue_struct *system_unbound_wq;
2829 struct workqueue_struct *system_long_wq;
2830 struct taskq *taskletq;
2831
2832 void
drm_linux_init(void)2833 drm_linux_init(void)
2834 {
2835 system_wq = (struct workqueue_struct *)
2836 taskq_create("drmwq", 4, IPL_HIGH, 0);
2837 system_highpri_wq = (struct workqueue_struct *)
2838 taskq_create("drmhpwq", 4, IPL_HIGH, 0);
2839 system_unbound_wq = (struct workqueue_struct *)
2840 taskq_create("drmubwq", 4, IPL_HIGH, 0);
2841 system_long_wq = (struct workqueue_struct *)
2842 taskq_create("drmlwq", 4, IPL_HIGH, 0);
2843
2844 taskletq = taskq_create("drmtskl", 1, IPL_HIGH, 0);
2845
2846 init_waitqueue_head(&bit_waitq);
2847 init_waitqueue_head(&var_waitq);
2848
2849 pool_init(&idr_pool, sizeof(struct idr_entry), 0, IPL_TTY, 0,
2850 "idrpl", NULL);
2851 pool_init(&xa_pool, sizeof(struct xarray_entry), 0, IPL_NONE, 0,
2852 "xapl", NULL);
2853
2854 kmap_atomic_va =
2855 (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_waitok);
2856
2857 #if NACPI > 0
2858 if (acpi_softc) {
2859 memcpy(&acpi_gbl_FADT, acpi_softc->sc_fadt,
2860 sizeof(acpi_gbl_FADT));
2861 }
2862 #endif
2863 }
2864
2865 void
drm_linux_exit(void)2866 drm_linux_exit(void)
2867 {
2868 pool_destroy(&xa_pool);
2869 pool_destroy(&idr_pool);
2870
2871 taskq_destroy(taskletq);
2872
2873 taskq_destroy((struct taskq *)system_long_wq);
2874 taskq_destroy((struct taskq *)system_unbound_wq);
2875 taskq_destroy((struct taskq *)system_highpri_wq);
2876 taskq_destroy((struct taskq *)system_wq);
2877 }
2878
2879 #define PCIE_ECAP_RESIZE_BAR 0x15
2880 #define RBCAP0 0x04
2881 #define RBCTRL0 0x08
2882 #define RBCTRL_BARINDEX_MASK 0x07
2883 #define RBCTRL_BARSIZE_MASK 0x1f00
2884 #define RBCTRL_BARSIZE_SHIFT 8
2885
2886 /* size in MB is 1 << nsize */
2887 int
pci_resize_resource(struct pci_dev * pdev,int bar,int nsize)2888 pci_resize_resource(struct pci_dev *pdev, int bar, int nsize)
2889 {
2890 pcireg_t reg;
2891 uint32_t offset, capid;
2892
2893 KASSERT(bar == 0);
2894
2895 offset = PCI_PCIE_ECAP;
2896
2897 /* search PCI Express Extended Capabilities */
2898 do {
2899 reg = pci_conf_read(pdev->pc, pdev->tag, offset);
2900 capid = PCI_PCIE_ECAP_ID(reg);
2901 if (capid == PCIE_ECAP_RESIZE_BAR)
2902 break;
2903 offset = PCI_PCIE_ECAP_NEXT(reg);
2904 } while (capid != 0);
2905
2906 if (capid == 0) {
2907 printf("%s: could not find resize bar cap!\n", __func__);
2908 return -ENOTSUP;
2909 }
2910
2911 reg = pci_conf_read(pdev->pc, pdev->tag, offset + RBCAP0);
2912
2913 if ((reg & (1 << (nsize + 4))) == 0) {
2914 printf("%s size not supported\n", __func__);
2915 return -ENOTSUP;
2916 }
2917
2918 reg = pci_conf_read(pdev->pc, pdev->tag, offset + RBCTRL0);
2919 if ((reg & RBCTRL_BARINDEX_MASK) != 0) {
2920 printf("%s BAR index not 0\n", __func__);
2921 return -EINVAL;
2922 }
2923
2924 reg &= ~RBCTRL_BARSIZE_MASK;
2925 reg |= (nsize << RBCTRL_BARSIZE_SHIFT) & RBCTRL_BARSIZE_MASK;
2926
2927 pci_conf_write(pdev->pc, pdev->tag, offset + RBCTRL0, reg);
2928
2929 return 0;
2930 }
2931
2932 TAILQ_HEAD(, shrinker) shrinkers = TAILQ_HEAD_INITIALIZER(shrinkers);
2933
2934 int
register_shrinker(struct shrinker * shrinker,const char * format,...)2935 register_shrinker(struct shrinker *shrinker, const char *format, ...)
2936 {
2937 TAILQ_INSERT_TAIL(&shrinkers, shrinker, next);
2938 return 0;
2939 }
2940
2941 void
unregister_shrinker(struct shrinker * shrinker)2942 unregister_shrinker(struct shrinker *shrinker)
2943 {
2944 TAILQ_REMOVE(&shrinkers, shrinker, next);
2945 }
2946
2947 unsigned long
drmbackoff(long npages)2948 drmbackoff(long npages)
2949 {
2950 struct shrink_control sc;
2951 struct shrinker *shrinker;
2952 u_long ret, freed = 0;
2953
2954 shrinker = TAILQ_FIRST(&shrinkers);
2955 while (shrinker && npages > 0) {
2956 sc.nr_to_scan = npages;
2957 ret = shrinker->scan_objects(shrinker, &sc);
2958 if (ret == SHRINK_STOP)
2959 break;
2960 npages -= ret;
2961 freed += ret;
2962 shrinker = TAILQ_NEXT(shrinker, next);
2963 }
2964
2965 return freed;
2966 }
2967
2968 void *
bitmap_zalloc(u_int n,gfp_t flags)2969 bitmap_zalloc(u_int n, gfp_t flags)
2970 {
2971 return kcalloc(BITS_TO_LONGS(n), sizeof(long), flags);
2972 }
2973
2974 void
bitmap_free(void * p)2975 bitmap_free(void *p)
2976 {
2977 kfree(p);
2978 }
2979
2980 int
atomic_dec_and_mutex_lock(volatile int * v,struct rwlock * lock)2981 atomic_dec_and_mutex_lock(volatile int *v, struct rwlock *lock)
2982 {
2983 if (atomic_add_unless(v, -1, 1))
2984 return 0;
2985
2986 rw_enter_write(lock);
2987 if (atomic_dec_return(v) == 0)
2988 return 1;
2989 rw_exit_write(lock);
2990 return 0;
2991 }
2992
2993 int
printk(const char * fmt,...)2994 printk(const char *fmt, ...)
2995 {
2996 int ret, level;
2997 va_list ap;
2998
2999 if (fmt != NULL && *fmt == '\001') {
3000 level = fmt[1];
3001 #ifndef DRMDEBUG
3002 if (level >= KERN_INFO[1] && level <= '9')
3003 return 0;
3004 #endif
3005 fmt += 2;
3006 }
3007
3008 va_start(ap, fmt);
3009 ret = vprintf(fmt, ap);
3010 va_end(ap);
3011
3012 return ret;
3013 }
3014
3015 #define START(node) ((node)->start)
3016 #define LAST(node) ((node)->last)
3017
3018 struct interval_tree_node *
interval_tree_iter_first(struct rb_root_cached * root,unsigned long start,unsigned long last)3019 interval_tree_iter_first(struct rb_root_cached *root, unsigned long start,
3020 unsigned long last)
3021 {
3022 struct interval_tree_node *node;
3023 struct rb_node *rb;
3024
3025 for (rb = rb_first_cached(root); rb; rb = rb_next(rb)) {
3026 node = rb_entry(rb, typeof(*node), rb);
3027 if (LAST(node) >= start && START(node) <= last)
3028 return node;
3029 }
3030 return NULL;
3031 }
3032
3033 void
interval_tree_remove(struct interval_tree_node * node,struct rb_root_cached * root)3034 interval_tree_remove(struct interval_tree_node *node,
3035 struct rb_root_cached *root)
3036 {
3037 rb_erase_cached(&node->rb, root);
3038 }
3039
3040 void
interval_tree_insert(struct interval_tree_node * node,struct rb_root_cached * root)3041 interval_tree_insert(struct interval_tree_node *node,
3042 struct rb_root_cached *root)
3043 {
3044 struct rb_node **iter = &root->rb_root.rb_node;
3045 struct rb_node *parent = NULL;
3046 struct interval_tree_node *iter_node;
3047
3048 while (*iter) {
3049 parent = *iter;
3050 iter_node = rb_entry(*iter, struct interval_tree_node, rb);
3051
3052 if (node->start < iter_node->start)
3053 iter = &(*iter)->rb_left;
3054 else
3055 iter = &(*iter)->rb_right;
3056 }
3057
3058 rb_link_node(&node->rb, parent, iter);
3059 rb_insert_color_cached(&node->rb, root, false);
3060 }
3061
3062 int
syncfile_read(struct file * fp,struct uio * uio,int fflags)3063 syncfile_read(struct file *fp, struct uio *uio, int fflags)
3064 {
3065 return ENXIO;
3066 }
3067
3068 int
syncfile_write(struct file * fp,struct uio * uio,int fflags)3069 syncfile_write(struct file *fp, struct uio *uio, int fflags)
3070 {
3071 return ENXIO;
3072 }
3073
3074 int
syncfile_ioctl(struct file * fp,u_long com,caddr_t data,struct proc * p)3075 syncfile_ioctl(struct file *fp, u_long com, caddr_t data, struct proc *p)
3076 {
3077 return ENOTTY;
3078 }
3079
3080 int
syncfile_kqfilter(struct file * fp,struct knote * kn)3081 syncfile_kqfilter(struct file *fp, struct knote *kn)
3082 {
3083 return EINVAL;
3084 }
3085
3086 int
syncfile_stat(struct file * fp,struct stat * st,struct proc * p)3087 syncfile_stat(struct file *fp, struct stat *st, struct proc *p)
3088 {
3089 memset(st, 0, sizeof(*st));
3090 st->st_mode = S_IFIFO; /* XXX */
3091 return 0;
3092 }
3093
3094 int
syncfile_close(struct file * fp,struct proc * p)3095 syncfile_close(struct file *fp, struct proc *p)
3096 {
3097 struct sync_file *sf = fp->f_data;
3098
3099 dma_fence_put(sf->fence);
3100 fp->f_data = NULL;
3101 free(sf, M_DRM, sizeof(struct sync_file));
3102 return 0;
3103 }
3104
3105 int
syncfile_seek(struct file * fp,off_t * offset,int whence,struct proc * p)3106 syncfile_seek(struct file *fp, off_t *offset, int whence, struct proc *p)
3107 {
3108 off_t newoff;
3109
3110 if (*offset != 0)
3111 return EINVAL;
3112
3113 switch (whence) {
3114 case SEEK_SET:
3115 newoff = 0;
3116 break;
3117 case SEEK_END:
3118 newoff = 0;
3119 break;
3120 default:
3121 return EINVAL;
3122 }
3123 mtx_enter(&fp->f_mtx);
3124 fp->f_offset = newoff;
3125 mtx_leave(&fp->f_mtx);
3126 *offset = newoff;
3127 return 0;
3128 }
3129
3130 const struct fileops syncfileops = {
3131 .fo_read = syncfile_read,
3132 .fo_write = syncfile_write,
3133 .fo_ioctl = syncfile_ioctl,
3134 .fo_kqfilter = syncfile_kqfilter,
3135 .fo_stat = syncfile_stat,
3136 .fo_close = syncfile_close,
3137 .fo_seek = syncfile_seek,
3138 };
3139
3140 void
fd_install(int fd,struct file * fp)3141 fd_install(int fd, struct file *fp)
3142 {
3143 struct proc *p = curproc;
3144 struct filedesc *fdp = p->p_fd;
3145
3146 if (fp->f_type != DTYPE_SYNC)
3147 return;
3148
3149 fdplock(fdp);
3150 /* all callers use get_unused_fd_flags(O_CLOEXEC) */
3151 fdinsert(fdp, fd, UF_EXCLOSE, fp);
3152 fdpunlock(fdp);
3153 }
3154
3155 void
fput(struct file * fp)3156 fput(struct file *fp)
3157 {
3158 if (fp->f_type != DTYPE_SYNC)
3159 return;
3160
3161 FRELE(fp, curproc);
3162 }
3163
3164 int
get_unused_fd_flags(unsigned int flags)3165 get_unused_fd_flags(unsigned int flags)
3166 {
3167 struct proc *p = curproc;
3168 struct filedesc *fdp = p->p_fd;
3169 int error, fd;
3170
3171 KASSERT((flags & O_CLOEXEC) != 0);
3172
3173 fdplock(fdp);
3174 retryalloc:
3175 if ((error = fdalloc(p, 0, &fd)) != 0) {
3176 if (error == ENOSPC) {
3177 fdexpand(p);
3178 goto retryalloc;
3179 }
3180 fdpunlock(fdp);
3181 return -1;
3182 }
3183 fdpunlock(fdp);
3184
3185 return fd;
3186 }
3187
3188 void
put_unused_fd(int fd)3189 put_unused_fd(int fd)
3190 {
3191 struct filedesc *fdp = curproc->p_fd;
3192
3193 fdplock(fdp);
3194 fdremove(fdp, fd);
3195 fdpunlock(fdp);
3196 }
3197
3198 struct dma_fence *
sync_file_get_fence(int fd)3199 sync_file_get_fence(int fd)
3200 {
3201 struct proc *p = curproc;
3202 struct filedesc *fdp = p->p_fd;
3203 struct file *fp;
3204 struct sync_file *sf;
3205 struct dma_fence *f;
3206
3207 if ((fp = fd_getfile(fdp, fd)) == NULL)
3208 return NULL;
3209
3210 if (fp->f_type != DTYPE_SYNC) {
3211 FRELE(fp, p);
3212 return NULL;
3213 }
3214 sf = fp->f_data;
3215 f = dma_fence_get(sf->fence);
3216 FRELE(sf->file, p);
3217 return f;
3218 }
3219
3220 struct sync_file *
sync_file_create(struct dma_fence * fence)3221 sync_file_create(struct dma_fence *fence)
3222 {
3223 struct proc *p = curproc;
3224 struct sync_file *sf;
3225 struct file *fp;
3226
3227 fp = fnew(p);
3228 if (fp == NULL)
3229 return NULL;
3230 fp->f_type = DTYPE_SYNC;
3231 fp->f_ops = &syncfileops;
3232 sf = malloc(sizeof(struct sync_file), M_DRM, M_WAITOK | M_ZERO);
3233 sf->file = fp;
3234 sf->fence = dma_fence_get(fence);
3235 fp->f_data = sf;
3236 return sf;
3237 }
3238
3239 bool
drm_firmware_drivers_only(void)3240 drm_firmware_drivers_only(void)
3241 {
3242 return false;
3243 }
3244
3245
3246 void *
memremap(phys_addr_t phys_addr,size_t size,int flags)3247 memremap(phys_addr_t phys_addr, size_t size, int flags)
3248 {
3249 STUB();
3250 return NULL;
3251 }
3252
3253 void
memunmap(void * addr)3254 memunmap(void *addr)
3255 {
3256 STUB();
3257 }
3258
3259 #include <linux/platform_device.h>
3260
3261 bus_dma_tag_t
dma_tag_lookup(struct device * dev)3262 dma_tag_lookup(struct device *dev)
3263 {
3264 extern struct cfdriver drm_cd;
3265 struct drm_device *drm;
3266 int i;
3267
3268 for (i = 0; i < drm_cd.cd_ndevs; i++) {
3269 drm = drm_cd.cd_devs[i];
3270 if (drm && drm->dev == dev)
3271 return drm->dmat;
3272 }
3273
3274 return ((struct platform_device *)dev)->dmat;
3275 }
3276
3277 LIST_HEAD(, drm_dmamem) dmamem_list = LIST_HEAD_INITIALIZER(dmamem_list);
3278
3279 void *
dma_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,int gfp)3280 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
3281 int gfp)
3282 {
3283 bus_dma_tag_t dmat = dma_tag_lookup(dev);
3284 struct drm_dmamem *mem;
3285
3286 mem = drm_dmamem_alloc(dmat, size, PAGE_SIZE, 1, size,
3287 BUS_DMA_COHERENT, 0);
3288 if (mem == NULL)
3289 return NULL;
3290 *dma_handle = mem->map->dm_segs[0].ds_addr;
3291 LIST_INSERT_HEAD(&dmamem_list, mem, next);
3292 return mem->kva;
3293 }
3294
3295 void
dma_free_coherent(struct device * dev,size_t size,void * cpu_addr,dma_addr_t dma_handle)3296 dma_free_coherent(struct device *dev, size_t size, void *cpu_addr,
3297 dma_addr_t dma_handle)
3298 {
3299 bus_dma_tag_t dmat = dma_tag_lookup(dev);
3300 struct drm_dmamem *mem;
3301
3302 LIST_FOREACH(mem, &dmamem_list, next) {
3303 if (mem->kva == cpu_addr)
3304 break;
3305 }
3306 KASSERT(mem);
3307 KASSERT(mem->size == size);
3308 KASSERT(mem->map->dm_segs[0].ds_addr == dma_handle);
3309
3310 LIST_REMOVE(mem, next);
3311 drm_dmamem_free(dmat, mem);
3312 }
3313
3314 int
dma_get_sgtable(struct device * dev,struct sg_table * sgt,void * cpu_addr,dma_addr_t dma_addr,size_t size)3315 dma_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr,
3316 dma_addr_t dma_addr, size_t size)
3317 {
3318 paddr_t pa;
3319 int ret;
3320
3321 if (!pmap_extract(pmap_kernel(), (vaddr_t)cpu_addr, &pa))
3322 return -EINVAL;
3323
3324 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
3325 if (ret)
3326 return ret;
3327
3328 sg_set_page(sgt->sgl, PHYS_TO_VM_PAGE(pa), size, 0);
3329 return 0;
3330 }
3331
3332 dma_addr_t
dma_map_resource(struct device * dev,phys_addr_t phys_addr,size_t size,enum dma_data_direction dir,u_long attr)3333 dma_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size,
3334 enum dma_data_direction dir, u_long attr)
3335 {
3336 bus_dma_tag_t dmat= dma_tag_lookup(dev);
3337 bus_dmamap_t map;
3338 bus_dma_segment_t seg;
3339
3340 if (bus_dmamap_create(dmat, size, 1, size, 0,
3341 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &map))
3342 return DMA_MAPPING_ERROR;
3343 seg.ds_addr = phys_addr;
3344 seg.ds_len = size;
3345 if (bus_dmamap_load_raw(dmat, map, &seg, 1, size, BUS_DMA_WAITOK)) {
3346 bus_dmamap_destroy(dmat, map);
3347 return DMA_MAPPING_ERROR;
3348 }
3349
3350 return map->dm_segs[0].ds_addr;
3351 }
3352
3353 #ifdef BUS_DMA_FIXED
3354
3355 #include <linux/iommu.h>
3356
3357 size_t
iommu_map_sgtable(struct iommu_domain * domain,u_long iova,struct sg_table * sgt,int prot)3358 iommu_map_sgtable(struct iommu_domain *domain, u_long iova,
3359 struct sg_table *sgt, int prot)
3360 {
3361 bus_dma_segment_t seg;
3362 int error;
3363
3364 error = bus_dmamap_create(domain->dmat, sgt->sgl->length, 1,
3365 sgt->sgl->length, 0, BUS_DMA_WAITOK, &sgt->dmamap);
3366 if (error)
3367 return -ENOMEM;
3368
3369 sgt->dmamap->dm_segs[0].ds_addr = iova;
3370 sgt->dmamap->dm_segs[0].ds_len = sgt->sgl->length;
3371 sgt->dmamap->dm_nsegs = 1;
3372 seg.ds_addr = VM_PAGE_TO_PHYS(sgt->sgl->__page);
3373 seg.ds_len = sgt->sgl->length;
3374 error = bus_dmamap_load_raw(domain->dmat, sgt->dmamap, &seg, 1,
3375 sgt->sgl->length, BUS_DMA_WAITOK | BUS_DMA_FIXED);
3376 if (error)
3377 return -ENOMEM;
3378
3379 return sg_dma_len(sgt->sgl);
3380 }
3381
3382 size_t
iommu_unmap(struct iommu_domain * domain,u_long iova,size_t size)3383 iommu_unmap(struct iommu_domain *domain, u_long iova, size_t size)
3384 {
3385 STUB();
3386 return 0;
3387 }
3388
3389 struct iommu_domain *
iommu_get_domain_for_dev(struct device * dev)3390 iommu_get_domain_for_dev(struct device *dev)
3391 {
3392 STUB();
3393 return NULL;
3394 }
3395
3396 phys_addr_t
iommu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)3397 iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
3398 {
3399 STUB();
3400 return 0;
3401 }
3402
3403 struct iommu_domain *
iommu_domain_alloc(struct bus_type * type)3404 iommu_domain_alloc(struct bus_type *type)
3405 {
3406 return malloc(sizeof(struct iommu_domain), M_DEVBUF, M_WAITOK | M_ZERO);
3407 }
3408
3409 int
iommu_attach_device(struct iommu_domain * domain,struct device * dev)3410 iommu_attach_device(struct iommu_domain *domain, struct device *dev)
3411 {
3412 struct platform_device *pdev = (struct platform_device *)dev;
3413
3414 domain->dmat = pdev->dmat;
3415 return 0;
3416 }
3417
3418 #endif
3419
3420 #include <linux/component.h>
3421
3422 struct component {
3423 struct device *dev;
3424 struct device *adev;
3425 const struct component_ops *ops;
3426 SLIST_ENTRY(component) next;
3427 };
3428
3429 SLIST_HEAD(,component) component_list = SLIST_HEAD_INITIALIZER(component_list);
3430
3431 int
component_add(struct device * dev,const struct component_ops * ops)3432 component_add(struct device *dev, const struct component_ops *ops)
3433 {
3434 struct component *component;
3435
3436 component = malloc(sizeof(*component), M_DEVBUF, M_WAITOK | M_ZERO);
3437 component->dev = dev;
3438 component->ops = ops;
3439 SLIST_INSERT_HEAD(&component_list, component, next);
3440 return 0;
3441 }
3442
3443 int
component_add_typed(struct device * dev,const struct component_ops * ops,int type)3444 component_add_typed(struct device *dev, const struct component_ops *ops,
3445 int type)
3446 {
3447 return component_add(dev, ops);
3448 }
3449
3450 int
component_bind_all(struct device * dev,void * data)3451 component_bind_all(struct device *dev, void *data)
3452 {
3453 struct component *component;
3454 int ret = 0;
3455
3456 SLIST_FOREACH(component, &component_list, next) {
3457 if (component->adev == dev) {
3458 ret = component->ops->bind(component->dev, NULL, data);
3459 if (ret)
3460 break;
3461 }
3462 }
3463
3464 return ret;
3465 }
3466
3467 struct component_match_entry {
3468 int (*compare)(struct device *, void *);
3469 void *data;
3470 };
3471
3472 struct component_match {
3473 struct component_match_entry match[4];
3474 int nmatches;
3475 };
3476
3477 int
component_master_add_with_match(struct device * dev,const struct component_master_ops * ops,struct component_match * match)3478 component_master_add_with_match(struct device *dev,
3479 const struct component_master_ops *ops, struct component_match *match)
3480 {
3481 struct component *component;
3482 int found = 0;
3483 int i, ret;
3484
3485 SLIST_FOREACH(component, &component_list, next) {
3486 for (i = 0; i < match->nmatches; i++) {
3487 struct component_match_entry *m = &match->match[i];
3488 if (m->compare(component->dev, m->data)) {
3489 component->adev = dev;
3490 found = 1;
3491 break;
3492 }
3493 }
3494 }
3495
3496 if (found) {
3497 ret = ops->bind(dev);
3498 if (ret)
3499 return ret;
3500 }
3501
3502 return 0;
3503 }
3504
3505 #ifdef __HAVE_FDT
3506
3507 #include <linux/platform_device.h>
3508 #include <dev/ofw/openfirm.h>
3509 #include <dev/ofw/fdt.h>
3510 #include <machine/fdt.h>
3511
3512 LIST_HEAD(, platform_device) pdev_list = LIST_HEAD_INITIALIZER(pdev_list);
3513
3514 void
platform_device_register(struct platform_device * pdev)3515 platform_device_register(struct platform_device *pdev)
3516 {
3517 int i;
3518
3519 pdev->num_resources = pdev->faa->fa_nreg;
3520 if (pdev->faa->fa_nreg > 0) {
3521 pdev->resource = mallocarray(pdev->faa->fa_nreg,
3522 sizeof(*pdev->resource), M_DEVBUF, M_WAITOK | M_ZERO);
3523 for (i = 0; i < pdev->faa->fa_nreg; i++) {
3524 pdev->resource[i].start = pdev->faa->fa_reg[i].addr;
3525 pdev->resource[i].end = pdev->faa->fa_reg[i].addr +
3526 pdev->faa->fa_reg[i].size - 1;
3527 }
3528 }
3529
3530 pdev->parent = pdev->dev.dv_parent;
3531 pdev->node = pdev->faa->fa_node;
3532 pdev->iot = pdev->faa->fa_iot;
3533 pdev->dmat = pdev->faa->fa_dmat;
3534 LIST_INSERT_HEAD(&pdev_list, pdev, next);
3535 }
3536
3537
3538 struct resource *
platform_get_resource(struct platform_device * pdev,u_int type,u_int num)3539 platform_get_resource(struct platform_device *pdev, u_int type, u_int num)
3540 {
3541 KASSERT(num < pdev->num_resources);
3542 return &pdev->resource[num];
3543 }
3544
3545 void __iomem *
devm_platform_ioremap_resource_byname(struct platform_device * pdev,const char * name)3546 devm_platform_ioremap_resource_byname(struct platform_device *pdev,
3547 const char *name)
3548 {
3549 bus_space_handle_t ioh;
3550 int err, idx;
3551
3552 idx = OF_getindex(pdev->node, name, "reg-names");
3553 if (idx == -1 || idx >= pdev->num_resources)
3554 return ERR_PTR(-EINVAL);
3555
3556 err = bus_space_map(pdev->iot, pdev->resource[idx].start,
3557 pdev->resource[idx].end - pdev->resource[idx].start + 1,
3558 BUS_SPACE_MAP_LINEAR, &ioh);
3559 if (err)
3560 return ERR_PTR(-err);
3561
3562 return bus_space_vaddr(pdev->iot, ioh);
3563 }
3564
3565 #include <dev/ofw/ofw_clock.h>
3566 #include <linux/clk.h>
3567
3568 struct clk *
devm_clk_get(struct device * dev,const char * name)3569 devm_clk_get(struct device *dev, const char *name)
3570 {
3571 struct platform_device *pdev = (struct platform_device *)dev;
3572 struct clk *clk;
3573
3574 clk = malloc(sizeof(*clk), M_DEVBUF, M_WAITOK);
3575 clk->freq = clock_get_frequency(pdev->node, name);
3576 return clk;
3577 }
3578
3579 u_long
clk_get_rate(struct clk * clk)3580 clk_get_rate(struct clk *clk)
3581 {
3582 return clk->freq;
3583 }
3584
3585 #include <linux/gpio/consumer.h>
3586 #include <dev/ofw/ofw_gpio.h>
3587
3588 struct gpio_desc {
3589 uint32_t gpios[4];
3590 };
3591
3592 struct gpio_desc *
devm_gpiod_get_optional(struct device * dev,const char * name,int flags)3593 devm_gpiod_get_optional(struct device *dev, const char *name, int flags)
3594 {
3595 struct platform_device *pdev = (struct platform_device *)dev;
3596 struct gpio_desc *desc;
3597 char fullname[128];
3598 int len;
3599
3600 snprintf(fullname, sizeof(fullname), "%s-gpios", name);
3601
3602 desc = malloc(sizeof(*desc), M_DEVBUF, M_WAITOK | M_ZERO);
3603 len = OF_getpropintarray(pdev->node, fullname, desc->gpios,
3604 sizeof(desc->gpios));
3605 KASSERT(len <= sizeof(desc->gpios));
3606 if (len < 0) {
3607 free(desc, M_DEVBUF, sizeof(*desc));
3608 return NULL;
3609 }
3610
3611 switch (flags) {
3612 case GPIOD_IN:
3613 gpio_controller_config_pin(desc->gpios, GPIO_CONFIG_INPUT);
3614 break;
3615 case GPIOD_OUT_HIGH:
3616 gpio_controller_config_pin(desc->gpios, GPIO_CONFIG_OUTPUT);
3617 gpio_controller_set_pin(desc->gpios, 1);
3618 break;
3619 default:
3620 panic("%s: unimplemented flags 0x%x", __func__, flags);
3621 }
3622
3623 return desc;
3624 }
3625
3626 int
gpiod_get_value_cansleep(const struct gpio_desc * desc)3627 gpiod_get_value_cansleep(const struct gpio_desc *desc)
3628 {
3629 return gpio_controller_get_pin(((struct gpio_desc *)desc)->gpios);
3630 }
3631
3632 struct phy {
3633 int node;
3634 const char *name;
3635 };
3636
3637 struct phy *
devm_phy_optional_get(struct device * dev,const char * name)3638 devm_phy_optional_get(struct device *dev, const char *name)
3639 {
3640 struct platform_device *pdev = (struct platform_device *)dev;
3641 struct phy *phy;
3642 int idx;
3643
3644 idx = OF_getindex(pdev->node, name, "phy-names");
3645 if (idx == -1)
3646 return NULL;
3647
3648 phy = malloc(sizeof(*phy), M_DEVBUF, M_WAITOK);
3649 phy->node = pdev->node;
3650 phy->name = name;
3651
3652 return phy;
3653 }
3654
3655 struct bus_type platform_bus_type;
3656
3657 #include <dev/ofw/ofw_misc.h>
3658
3659 #include <linux/of.h>
3660 #include <linux/platform_device.h>
3661
3662 struct device_node *
__of_devnode(void * arg)3663 __of_devnode(void *arg)
3664 {
3665 struct device *dev = container_of(arg, struct device, of_node);
3666 struct platform_device *pdev = (struct platform_device *)dev;
3667
3668 return (struct device_node *)(uintptr_t)pdev->node;
3669 }
3670
3671 int
__of_device_is_compatible(struct device_node * np,const char * compatible)3672 __of_device_is_compatible(struct device_node *np, const char *compatible)
3673 {
3674 return OF_is_compatible((uintptr_t)np, compatible);
3675 }
3676
3677 int
__of_property_present(struct device_node * np,const char * propname)3678 __of_property_present(struct device_node *np, const char *propname)
3679 {
3680 return OF_getpropbool((uintptr_t)np, (char *)propname);
3681 }
3682
3683 int
__of_property_read_variable_u32_array(struct device_node * np,const char * propname,uint32_t * out_values,size_t sz_min,size_t sz_max)3684 __of_property_read_variable_u32_array(struct device_node *np,
3685 const char *propname, uint32_t *out_values, size_t sz_min, size_t sz_max)
3686 {
3687 int len;
3688
3689 len = OF_getpropintarray((uintptr_t)np, (char *)propname, out_values,
3690 sz_max * sizeof(*out_values));
3691 if (len < 0)
3692 return -EINVAL;
3693 if (len == 0)
3694 return -ENODATA;
3695 if (len < sz_min * sizeof(*out_values) ||
3696 len > sz_max * sizeof(*out_values))
3697 return -EOVERFLOW;
3698 if (sz_min == 1 && sz_max == 1)
3699 return 0;
3700 return len / sizeof(*out_values);
3701 }
3702
3703 int
__of_property_read_variable_u64_array(struct device_node * np,const char * propname,uint64_t * out_values,size_t sz_min,size_t sz_max)3704 __of_property_read_variable_u64_array(struct device_node *np,
3705 const char *propname, uint64_t *out_values, size_t sz_min, size_t sz_max)
3706 {
3707 int len;
3708
3709 len = OF_getpropint64array((uintptr_t)np, (char *)propname, out_values,
3710 sz_max * sizeof(*out_values));
3711 if (len < 0)
3712 return -EINVAL;
3713 if (len == 0)
3714 return -ENODATA;
3715 if (len < sz_min * sizeof(*out_values) ||
3716 len > sz_max * sizeof(*out_values))
3717 return -EOVERFLOW;
3718 if (sz_min == 1 && sz_max == 1)
3719 return 0;
3720 return len / sizeof(*out_values);
3721 }
3722
3723 int
__of_property_match_string(struct device_node * np,const char * propname,const char * str)3724 __of_property_match_string(struct device_node *np,
3725 const char *propname, const char *str)
3726 {
3727 int idx;
3728
3729 idx = OF_getindex((uintptr_t)np, str, propname);
3730 if (idx == -1)
3731 return -ENODATA;
3732 return idx;
3733 }
3734
3735 struct device_node *
__of_parse_phandle(struct device_node * np,const char * propname,int idx)3736 __of_parse_phandle(struct device_node *np, const char *propname, int idx)
3737 {
3738 uint32_t phandles[16] = {};
3739 int len, node;
3740
3741 len = OF_getpropintarray((uintptr_t)np, (char *)propname, phandles,
3742 sizeof(phandles));
3743 if (len < (idx + 1) * sizeof(uint32_t))
3744 return NULL;
3745
3746 node = OF_getnodebyphandle(phandles[idx]);
3747 if (node == 0)
3748 return NULL;
3749
3750 return (struct device_node *)(uintptr_t)node;
3751 }
3752
3753 int
__of_parse_phandle_with_args(struct device_node * np,const char * propname,const char * cellsname,int idx,struct of_phandle_args * args)3754 __of_parse_phandle_with_args(struct device_node *np, const char *propname,
3755 const char *cellsname, int idx, struct of_phandle_args *args)
3756 {
3757 uint32_t phandles[16] = {};
3758 int i, len, node;
3759
3760 len = OF_getpropintarray((uintptr_t)np, (char *)propname, phandles,
3761 sizeof(phandles));
3762 if (len < (idx + 1) * sizeof(uint32_t))
3763 return -ENOENT;
3764
3765 node = OF_getnodebyphandle(phandles[idx]);
3766 if (node == 0)
3767 return -ENOENT;
3768
3769 args->np = (struct device_node *)(uintptr_t)node;
3770 args->args_count = OF_getpropint(node, (char *)cellsname, 0);
3771 for (i = 0; i < args->args_count; i++)
3772 args->args[i] = phandles[i + 1];
3773
3774 return 0;
3775 }
3776
3777 int
of_address_to_resource(struct device_node * np,int idx,struct resource * res)3778 of_address_to_resource(struct device_node *np, int idx, struct resource *res)
3779 {
3780 uint64_t reg[16] = {};
3781 int len;
3782
3783 KASSERT(idx < 8);
3784
3785 len = OF_getpropint64array((uintptr_t)np, "reg", reg, sizeof(reg));
3786 if (len < 0 || idx >= (len / (2 * sizeof(uint64_t))))
3787 return -EINVAL;
3788
3789 res->start = reg[2 * idx];
3790 res->end = reg[2 * idx] + reg[2 * idx + 1] - 1;
3791
3792 return 0;
3793 }
3794
3795 static int
next_node(int node)3796 next_node(int node)
3797 {
3798 int peer = OF_peer(node);
3799
3800 while (node && !peer) {
3801 node = OF_parent(node);
3802 if (node)
3803 peer = OF_peer(node);
3804 }
3805
3806 return peer;
3807 }
3808
3809 static int
find_matching_node(int node,const struct of_device_id * id)3810 find_matching_node(int node, const struct of_device_id *id)
3811 {
3812 int child, match;
3813 int i;
3814
3815 for (child = OF_child(node); child; child = OF_peer(child)) {
3816 match = find_matching_node(child, id);
3817 if (match)
3818 return match;
3819 }
3820
3821 for (i = 0; id[i].compatible; i++) {
3822 if (OF_is_compatible(node, id[i].compatible))
3823 return node;
3824 }
3825
3826 return 0;
3827 }
3828
3829 struct device_node *
__matching_node(struct device_node * np,const struct of_device_id * id)3830 __matching_node(struct device_node *np, const struct of_device_id *id)
3831 {
3832 int node = OF_peer(0);
3833 int match;
3834
3835 if (np)
3836 node = next_node((uintptr_t)np);
3837 while (node) {
3838 match = find_matching_node(node, id);
3839 if (match)
3840 return (struct device_node *)(uintptr_t)match;
3841 node = next_node(node);
3842 }
3843
3844 return NULL;
3845 }
3846
3847 struct platform_device *
of_platform_device_create(struct device_node * np,const char * bus_id,struct device * parent)3848 of_platform_device_create(struct device_node *np, const char *bus_id,
3849 struct device *parent)
3850 {
3851 struct platform_device *pdev;
3852
3853 pdev = malloc(sizeof(*pdev), M_DEVBUF, M_WAITOK | M_ZERO);
3854 pdev->node = (intptr_t)np;
3855 pdev->parent = parent;
3856
3857 LIST_INSERT_HEAD(&pdev_list, pdev, next);
3858
3859 return pdev;
3860 }
3861
3862 struct platform_device *
of_find_device_by_node(struct device_node * np)3863 of_find_device_by_node(struct device_node *np)
3864 {
3865 struct platform_device *pdev;
3866
3867 LIST_FOREACH(pdev, &pdev_list, next) {
3868 if (pdev->node == (intptr_t)np)
3869 return pdev;
3870 }
3871
3872 return NULL;
3873 }
3874
3875 int
of_device_is_available(struct device_node * np)3876 of_device_is_available(struct device_node *np)
3877 {
3878 char status[32];
3879
3880 if (OF_getprop((uintptr_t)np, "status", status, sizeof(status)) > 0 &&
3881 strcmp(status, "disabled") == 0)
3882 return 0;
3883
3884 return 1;
3885 }
3886
3887 int
of_dma_configure(struct device * dev,struct device_node * np,int force_dma)3888 of_dma_configure(struct device *dev, struct device_node *np, int force_dma)
3889 {
3890 struct platform_device *pdev = (struct platform_device *)dev;
3891 bus_dma_tag_t dmat = dma_tag_lookup(pdev->parent);
3892
3893 pdev->dmat = iommu_device_map(pdev->node, dmat);
3894 return 0;
3895 }
3896
3897 struct device_node *
__of_get_compatible_child(void * p,const char * compat)3898 __of_get_compatible_child(void *p, const char *compat)
3899 {
3900 struct device *dev = container_of(p, struct device, of_node);
3901 struct platform_device *pdev = (struct platform_device *)dev;
3902 int child;
3903
3904 for (child = OF_child(pdev->node); child; child = OF_peer(child)) {
3905 if (OF_is_compatible(child, compat))
3906 return (struct device_node *)(uintptr_t)child;
3907 }
3908 return NULL;
3909 }
3910
3911 struct device_node *
__of_get_child_by_name(void * p,const char * name)3912 __of_get_child_by_name(void *p, const char *name)
3913 {
3914 struct device *dev = container_of(p, struct device, of_node);
3915 struct platform_device *pdev = (struct platform_device *)dev;
3916 int child;
3917
3918 child = OF_getnodebyname(pdev->node, name);
3919 if (child == 0)
3920 return NULL;
3921 return (struct device_node *)(uintptr_t)child;
3922 }
3923
3924 int
component_compare_of(struct device * dev,void * data)3925 component_compare_of(struct device *dev, void *data)
3926 {
3927 struct platform_device *pdev = (struct platform_device *)dev;
3928
3929 return (pdev->node == (intptr_t)data);
3930 }
3931
3932 void
drm_of_component_match_add(struct device * master,struct component_match ** matchptr,int (* compare)(struct device *,void *),struct device_node * np)3933 drm_of_component_match_add(struct device *master,
3934 struct component_match **matchptr,
3935 int (*compare)(struct device *, void *),
3936 struct device_node *np)
3937 {
3938 struct component_match *match = *matchptr;
3939
3940 if (match == NULL) {
3941 match = malloc(sizeof(struct component_match),
3942 M_DEVBUF, M_WAITOK | M_ZERO);
3943 *matchptr = match;
3944 }
3945
3946 KASSERT(match->nmatches < nitems(match->match));
3947 match->match[match->nmatches].compare = compare;
3948 match->match[match->nmatches].data = np;
3949 match->nmatches++;
3950 }
3951
3952 #endif
3953