1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved. 4 * Copyright (c) 2010-2012 Broadcom. All rights reserved. 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/sched/signal.h> 10 #include <linux/types.h> 11 #include <linux/errno.h> 12 #include <linux/cdev.h> 13 #include <linux/fs.h> 14 #include <linux/device.h> 15 #include <linux/mm.h> 16 #include <linux/highmem.h> 17 #include <linux/pagemap.h> 18 #include <linux/bug.h> 19 #include <linux/completion.h> 20 #include <linux/list.h> 21 #include <linux/of.h> 22 #include <linux/platform_device.h> 23 #include <linux/compat.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/rcupdate.h> 26 #include <linux/delay.h> 27 #include <linux/slab.h> 28 #include <linux/interrupt.h> 29 #include <linux/io.h> 30 #include <linux/uaccess.h> 31 #include <soc/bcm2835/raspberrypi-firmware.h> 32 33 #include "vchiq_core.h" 34 #include "vchiq_ioctl.h" 35 #include "vchiq_arm.h" 36 #include "vchiq_debugfs.h" 37 #include "vchiq_connected.h" 38 #include "vchiq_pagelist.h" 39 40 #define DEVICE_NAME "vchiq" 41 42 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32) 43 44 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2) 45 46 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0 47 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1 48 49 #define BELL0 0x00 50 #define BELL2 0x08 51 52 #define ARM_DS_ACTIVE BIT(2) 53 54 /* Override the default prefix, which would be vchiq_arm (from the filename) */ 55 #undef MODULE_PARAM_PREFIX 56 #define MODULE_PARAM_PREFIX DEVICE_NAME "." 57 58 #define KEEPALIVE_VER 1 59 #define KEEPALIVE_VER_MIN KEEPALIVE_VER 60 61 /* Run time control of log level, based on KERN_XXX level. */ 62 int vchiq_arm_log_level = VCHIQ_LOG_DEFAULT; 63 int vchiq_susp_log_level = VCHIQ_LOG_ERROR; 64 65 DEFINE_SPINLOCK(msg_queue_spinlock); 66 struct vchiq_state g_state; 67 68 static struct platform_device *bcm2835_camera; 69 static struct platform_device *bcm2835_audio; 70 71 struct vchiq_drvdata { 72 const unsigned int cache_line_size; 73 struct rpi_firmware *fw; 74 }; 75 76 static struct vchiq_drvdata bcm2835_drvdata = { 77 .cache_line_size = 32, 78 }; 79 80 static struct vchiq_drvdata bcm2836_drvdata = { 81 .cache_line_size = 64, 82 }; 83 84 struct vchiq_arm_state { 85 /* Keepalive-related data */ 86 struct task_struct *ka_thread; 87 struct completion ka_evt; 88 atomic_t ka_use_count; 89 atomic_t ka_use_ack_count; 90 atomic_t ka_release_count; 91 92 rwlock_t susp_res_lock; 93 94 struct vchiq_state *state; 95 96 /* 97 * Global use count for videocore. 98 * This is equal to the sum of the use counts for all services. When 99 * this hits zero the videocore suspend procedure will be initiated. 100 */ 101 int videocore_use_count; 102 103 /* 104 * Use count to track requests from videocore peer. 105 * This use count is not associated with a service, so needs to be 106 * tracked separately with the state. 107 */ 108 int peer_use_count; 109 110 /* 111 * Flag to indicate that the first vchiq connect has made it through. 112 * This means that both sides should be fully ready, and we should 113 * be able to suspend after this point. 114 */ 115 int first_connect; 116 }; 117 118 struct vchiq_2835_state { 119 int inited; 120 struct vchiq_arm_state arm_state; 121 }; 122 123 struct vchiq_pagelist_info { 124 struct pagelist *pagelist; 125 size_t pagelist_buffer_size; 126 dma_addr_t dma_addr; 127 enum dma_data_direction dma_dir; 128 unsigned int num_pages; 129 unsigned int pages_need_release; 130 struct page **pages; 131 struct scatterlist *scatterlist; 132 unsigned int scatterlist_mapped; 133 }; 134 135 static void __iomem *g_regs; 136 /* This value is the size of the L2 cache lines as understood by the 137 * VPU firmware, which determines the required alignment of the 138 * offsets/sizes in pagelists. 139 * 140 * Modern VPU firmware looks for a DT "cache-line-size" property in 141 * the VCHIQ node and will overwrite it with the actual L2 cache size, 142 * which the kernel must then respect. That property was rejected 143 * upstream, so we have to use the VPU firmware's compatibility value 144 * of 32. 145 */ 146 static unsigned int g_cache_line_size = 32; 147 static unsigned int g_fragments_size; 148 static char *g_fragments_base; 149 static char *g_free_fragments; 150 static struct semaphore g_free_fragments_sema; 151 152 static DEFINE_SEMAPHORE(g_free_fragments_mutex, 1); 153 154 static int 155 vchiq_blocking_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *data, 156 unsigned int size, enum vchiq_bulk_dir dir); 157 158 static irqreturn_t 159 vchiq_doorbell_irq(int irq, void *dev_id) 160 { 161 struct vchiq_state *state = dev_id; 162 irqreturn_t ret = IRQ_NONE; 163 unsigned int status; 164 165 /* Read (and clear) the doorbell */ 166 status = readl(g_regs + BELL0); 167 168 if (status & ARM_DS_ACTIVE) { /* Was the doorbell rung? */ 169 remote_event_pollall(state); 170 ret = IRQ_HANDLED; 171 } 172 173 return ret; 174 } 175 176 static void 177 cleanup_pagelistinfo(struct vchiq_instance *instance, struct vchiq_pagelist_info *pagelistinfo) 178 { 179 if (pagelistinfo->scatterlist_mapped) { 180 dma_unmap_sg(instance->state->dev, pagelistinfo->scatterlist, 181 pagelistinfo->num_pages, pagelistinfo->dma_dir); 182 } 183 184 if (pagelistinfo->pages_need_release) 185 unpin_user_pages(pagelistinfo->pages, pagelistinfo->num_pages); 186 187 dma_free_coherent(instance->state->dev, pagelistinfo->pagelist_buffer_size, 188 pagelistinfo->pagelist, pagelistinfo->dma_addr); 189 } 190 191 static inline bool 192 is_adjacent_block(u32 *addrs, u32 addr, unsigned int k) 193 { 194 u32 tmp; 195 196 if (!k) 197 return false; 198 199 tmp = (addrs[k - 1] & PAGE_MASK) + 200 (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT); 201 202 return tmp == (addr & PAGE_MASK); 203 } 204 205 /* There is a potential problem with partial cache lines (pages?) 206 * at the ends of the block when reading. If the CPU accessed anything in 207 * the same line (page?) then it may have pulled old data into the cache, 208 * obscuring the new data underneath. We can solve this by transferring the 209 * partial cache lines separately, and allowing the ARM to copy into the 210 * cached area. 211 */ 212 213 static struct vchiq_pagelist_info * 214 create_pagelist(struct vchiq_instance *instance, char *buf, char __user *ubuf, 215 size_t count, unsigned short type) 216 { 217 struct pagelist *pagelist; 218 struct vchiq_pagelist_info *pagelistinfo; 219 struct page **pages; 220 u32 *addrs; 221 unsigned int num_pages, offset, i, k; 222 int actual_pages; 223 size_t pagelist_size; 224 struct scatterlist *scatterlist, *sg; 225 int dma_buffers; 226 dma_addr_t dma_addr; 227 228 if (count >= INT_MAX - PAGE_SIZE) 229 return NULL; 230 231 if (buf) 232 offset = (uintptr_t)buf & (PAGE_SIZE - 1); 233 else 234 offset = (uintptr_t)ubuf & (PAGE_SIZE - 1); 235 num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE); 236 237 if ((size_t)num_pages > (SIZE_MAX - sizeof(struct pagelist) - 238 sizeof(struct vchiq_pagelist_info)) / 239 (sizeof(u32) + sizeof(pages[0]) + 240 sizeof(struct scatterlist))) 241 return NULL; 242 243 pagelist_size = sizeof(struct pagelist) + 244 (num_pages * sizeof(u32)) + 245 (num_pages * sizeof(pages[0]) + 246 (num_pages * sizeof(struct scatterlist))) + 247 sizeof(struct vchiq_pagelist_info); 248 249 /* Allocate enough storage to hold the page pointers and the page 250 * list 251 */ 252 pagelist = dma_alloc_coherent(instance->state->dev, pagelist_size, &dma_addr, 253 GFP_KERNEL); 254 255 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist); 256 257 if (!pagelist) 258 return NULL; 259 260 addrs = pagelist->addrs; 261 pages = (struct page **)(addrs + num_pages); 262 scatterlist = (struct scatterlist *)(pages + num_pages); 263 pagelistinfo = (struct vchiq_pagelist_info *) 264 (scatterlist + num_pages); 265 266 pagelist->length = count; 267 pagelist->type = type; 268 pagelist->offset = offset; 269 270 /* Populate the fields of the pagelistinfo structure */ 271 pagelistinfo->pagelist = pagelist; 272 pagelistinfo->pagelist_buffer_size = pagelist_size; 273 pagelistinfo->dma_addr = dma_addr; 274 pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ? 275 DMA_TO_DEVICE : DMA_FROM_DEVICE; 276 pagelistinfo->num_pages = num_pages; 277 pagelistinfo->pages_need_release = 0; 278 pagelistinfo->pages = pages; 279 pagelistinfo->scatterlist = scatterlist; 280 pagelistinfo->scatterlist_mapped = 0; 281 282 if (buf) { 283 unsigned long length = count; 284 unsigned int off = offset; 285 286 for (actual_pages = 0; actual_pages < num_pages; 287 actual_pages++) { 288 struct page *pg = 289 vmalloc_to_page((buf + 290 (actual_pages * PAGE_SIZE))); 291 size_t bytes = PAGE_SIZE - off; 292 293 if (!pg) { 294 cleanup_pagelistinfo(instance, pagelistinfo); 295 return NULL; 296 } 297 298 if (bytes > length) 299 bytes = length; 300 pages[actual_pages] = pg; 301 length -= bytes; 302 off = 0; 303 } 304 /* do not try and release vmalloc pages */ 305 } else { 306 actual_pages = pin_user_pages_fast((unsigned long)ubuf & PAGE_MASK, num_pages, 307 type == PAGELIST_READ, pages); 308 309 if (actual_pages != num_pages) { 310 vchiq_log_info(vchiq_arm_log_level, 311 "%s - only %d/%d pages locked", 312 __func__, actual_pages, num_pages); 313 314 /* This is probably due to the process being killed */ 315 if (actual_pages > 0) 316 unpin_user_pages(pages, actual_pages); 317 cleanup_pagelistinfo(instance, pagelistinfo); 318 return NULL; 319 } 320 /* release user pages */ 321 pagelistinfo->pages_need_release = 1; 322 } 323 324 /* 325 * Initialize the scatterlist so that the magic cookie 326 * is filled if debugging is enabled 327 */ 328 sg_init_table(scatterlist, num_pages); 329 /* Now set the pages for each scatterlist */ 330 for (i = 0; i < num_pages; i++) { 331 unsigned int len = PAGE_SIZE - offset; 332 333 if (len > count) 334 len = count; 335 sg_set_page(scatterlist + i, pages[i], len, offset); 336 offset = 0; 337 count -= len; 338 } 339 340 dma_buffers = dma_map_sg(instance->state->dev, 341 scatterlist, 342 num_pages, 343 pagelistinfo->dma_dir); 344 345 if (dma_buffers == 0) { 346 cleanup_pagelistinfo(instance, pagelistinfo); 347 return NULL; 348 } 349 350 pagelistinfo->scatterlist_mapped = 1; 351 352 /* Combine adjacent blocks for performance */ 353 k = 0; 354 for_each_sg(scatterlist, sg, dma_buffers, i) { 355 u32 len = sg_dma_len(sg); 356 u32 addr = sg_dma_address(sg); 357 358 /* Note: addrs is the address + page_count - 1 359 * The firmware expects blocks after the first to be page- 360 * aligned and a multiple of the page size 361 */ 362 WARN_ON(len == 0); 363 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK)); 364 WARN_ON(i && (addr & ~PAGE_MASK)); 365 if (is_adjacent_block(addrs, addr, k)) 366 addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT); 367 else 368 addrs[k++] = (addr & PAGE_MASK) | 369 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1); 370 } 371 372 /* Partial cache lines (fragments) require special measures */ 373 if ((type == PAGELIST_READ) && 374 ((pagelist->offset & (g_cache_line_size - 1)) || 375 ((pagelist->offset + pagelist->length) & 376 (g_cache_line_size - 1)))) { 377 char *fragments; 378 379 if (down_interruptible(&g_free_fragments_sema)) { 380 cleanup_pagelistinfo(instance, pagelistinfo); 381 return NULL; 382 } 383 384 WARN_ON(!g_free_fragments); 385 386 down(&g_free_fragments_mutex); 387 fragments = g_free_fragments; 388 WARN_ON(!fragments); 389 g_free_fragments = *(char **)g_free_fragments; 390 up(&g_free_fragments_mutex); 391 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS + 392 (fragments - g_fragments_base) / g_fragments_size; 393 } 394 395 return pagelistinfo; 396 } 397 398 static void 399 free_pagelist(struct vchiq_instance *instance, struct vchiq_pagelist_info *pagelistinfo, 400 int actual) 401 { 402 struct pagelist *pagelist = pagelistinfo->pagelist; 403 struct page **pages = pagelistinfo->pages; 404 unsigned int num_pages = pagelistinfo->num_pages; 405 406 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d", 407 __func__, pagelistinfo->pagelist, actual); 408 409 /* 410 * NOTE: dma_unmap_sg must be called before the 411 * cpu can touch any of the data/pages. 412 */ 413 dma_unmap_sg(instance->state->dev, pagelistinfo->scatterlist, 414 pagelistinfo->num_pages, pagelistinfo->dma_dir); 415 pagelistinfo->scatterlist_mapped = 0; 416 417 /* Deal with any partial cache lines (fragments) */ 418 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) { 419 char *fragments = g_fragments_base + 420 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) * 421 g_fragments_size; 422 int head_bytes, tail_bytes; 423 424 head_bytes = (g_cache_line_size - pagelist->offset) & 425 (g_cache_line_size - 1); 426 tail_bytes = (pagelist->offset + actual) & 427 (g_cache_line_size - 1); 428 429 if ((actual >= 0) && (head_bytes != 0)) { 430 if (head_bytes > actual) 431 head_bytes = actual; 432 433 memcpy_to_page(pages[0], 434 pagelist->offset, 435 fragments, 436 head_bytes); 437 } 438 if ((actual >= 0) && (head_bytes < actual) && 439 (tail_bytes != 0)) 440 memcpy_to_page(pages[num_pages - 1], 441 (pagelist->offset + actual) & 442 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1), 443 fragments + g_cache_line_size, 444 tail_bytes); 445 446 down(&g_free_fragments_mutex); 447 *(char **)fragments = g_free_fragments; 448 g_free_fragments = fragments; 449 up(&g_free_fragments_mutex); 450 up(&g_free_fragments_sema); 451 } 452 453 /* Need to mark all the pages dirty. */ 454 if (pagelist->type != PAGELIST_WRITE && 455 pagelistinfo->pages_need_release) { 456 unsigned int i; 457 458 for (i = 0; i < num_pages; i++) 459 set_page_dirty(pages[i]); 460 } 461 462 cleanup_pagelistinfo(instance, pagelistinfo); 463 } 464 465 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state) 466 { 467 struct device *dev = &pdev->dev; 468 struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev); 469 struct rpi_firmware *fw = drvdata->fw; 470 struct vchiq_slot_zero *vchiq_slot_zero; 471 void *slot_mem; 472 dma_addr_t slot_phys; 473 u32 channelbase; 474 int slot_mem_size, frag_mem_size; 475 int err, irq, i; 476 477 /* 478 * VCHI messages between the CPU and firmware use 479 * 32-bit bus addresses. 480 */ 481 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 482 483 if (err < 0) 484 return err; 485 486 g_cache_line_size = drvdata->cache_line_size; 487 g_fragments_size = 2 * g_cache_line_size; 488 489 /* Allocate space for the channels in coherent memory */ 490 slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE); 491 frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS); 492 493 slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size, 494 &slot_phys, GFP_KERNEL); 495 if (!slot_mem) { 496 dev_err(dev, "could not allocate DMA memory\n"); 497 return -ENOMEM; 498 } 499 500 WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0); 501 502 vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size); 503 if (!vchiq_slot_zero) 504 return -ENOMEM; 505 506 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] = 507 (int)slot_phys + slot_mem_size; 508 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] = 509 MAX_FRAGMENTS; 510 511 g_fragments_base = (char *)slot_mem + slot_mem_size; 512 513 g_free_fragments = g_fragments_base; 514 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) { 515 *(char **)&g_fragments_base[i * g_fragments_size] = 516 &g_fragments_base[(i + 1) * g_fragments_size]; 517 } 518 *(char **)&g_fragments_base[i * g_fragments_size] = NULL; 519 sema_init(&g_free_fragments_sema, MAX_FRAGMENTS); 520 521 err = vchiq_init_state(state, vchiq_slot_zero, dev); 522 if (err) 523 return err; 524 525 g_regs = devm_platform_ioremap_resource(pdev, 0); 526 if (IS_ERR(g_regs)) 527 return PTR_ERR(g_regs); 528 529 irq = platform_get_irq(pdev, 0); 530 if (irq <= 0) 531 return irq; 532 533 err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL, 534 "VCHIQ doorbell", state); 535 if (err) { 536 dev_err(dev, "failed to register irq=%d\n", irq); 537 return err; 538 } 539 540 /* Send the base address of the slots to VideoCore */ 541 channelbase = slot_phys; 542 err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT, 543 &channelbase, sizeof(channelbase)); 544 if (err) { 545 dev_err(dev, "failed to send firmware property: %d\n", err); 546 return err; 547 } 548 549 if (channelbase) { 550 dev_err(dev, "failed to set channelbase (response: %x)\n", 551 channelbase); 552 return -ENXIO; 553 } 554 555 vchiq_log_info(vchiq_arm_log_level, "vchiq_init - done (slots %pK, phys %pad)", 556 vchiq_slot_zero, &slot_phys); 557 558 vchiq_call_connected_callbacks(); 559 560 return 0; 561 } 562 563 static void 564 vchiq_arm_init_state(struct vchiq_state *state, 565 struct vchiq_arm_state *arm_state) 566 { 567 if (arm_state) { 568 rwlock_init(&arm_state->susp_res_lock); 569 570 init_completion(&arm_state->ka_evt); 571 atomic_set(&arm_state->ka_use_count, 0); 572 atomic_set(&arm_state->ka_use_ack_count, 0); 573 atomic_set(&arm_state->ka_release_count, 0); 574 575 arm_state->state = state; 576 arm_state->first_connect = 0; 577 } 578 } 579 580 int 581 vchiq_platform_init_state(struct vchiq_state *state) 582 { 583 struct vchiq_2835_state *platform_state; 584 585 state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL); 586 if (!state->platform_state) 587 return -ENOMEM; 588 589 platform_state = (struct vchiq_2835_state *)state->platform_state; 590 591 platform_state->inited = 1; 592 vchiq_arm_init_state(state, &platform_state->arm_state); 593 594 return 0; 595 } 596 597 static struct vchiq_arm_state *vchiq_platform_get_arm_state(struct vchiq_state *state) 598 { 599 struct vchiq_2835_state *platform_state; 600 601 platform_state = (struct vchiq_2835_state *)state->platform_state; 602 603 WARN_ON_ONCE(!platform_state->inited); 604 605 return &platform_state->arm_state; 606 } 607 608 void 609 remote_event_signal(struct remote_event *event) 610 { 611 /* 612 * Ensure that all writes to shared data structures have completed 613 * before signalling the peer. 614 */ 615 wmb(); 616 617 event->fired = 1; 618 619 dsb(sy); /* data barrier operation */ 620 621 if (event->armed) 622 writel(0, g_regs + BELL2); /* trigger vc interrupt */ 623 } 624 625 int 626 vchiq_prepare_bulk_data(struct vchiq_instance *instance, struct vchiq_bulk *bulk, void *offset, 627 void __user *uoffset, int size, int dir) 628 { 629 struct vchiq_pagelist_info *pagelistinfo; 630 631 pagelistinfo = create_pagelist(instance, offset, uoffset, size, 632 (dir == VCHIQ_BULK_RECEIVE) 633 ? PAGELIST_READ 634 : PAGELIST_WRITE); 635 636 if (!pagelistinfo) 637 return -ENOMEM; 638 639 bulk->data = pagelistinfo->dma_addr; 640 641 /* 642 * Store the pagelistinfo address in remote_data, 643 * which isn't used by the slave. 644 */ 645 bulk->remote_data = pagelistinfo; 646 647 return 0; 648 } 649 650 void 651 vchiq_complete_bulk(struct vchiq_instance *instance, struct vchiq_bulk *bulk) 652 { 653 if (bulk && bulk->remote_data && bulk->actual) 654 free_pagelist(instance, (struct vchiq_pagelist_info *)bulk->remote_data, 655 bulk->actual); 656 } 657 658 int vchiq_dump_platform_state(void *dump_context) 659 { 660 char buf[80]; 661 int len; 662 663 len = snprintf(buf, sizeof(buf), " Platform: 2835 (VC master)"); 664 return vchiq_dump(dump_context, buf, len + 1); 665 } 666 667 #define VCHIQ_INIT_RETRIES 10 668 int vchiq_initialise(struct vchiq_instance **instance_out) 669 { 670 struct vchiq_state *state; 671 struct vchiq_instance *instance = NULL; 672 int i, ret; 673 674 /* 675 * VideoCore may not be ready due to boot up timing. 676 * It may never be ready if kernel and firmware are mismatched,so don't 677 * block forever. 678 */ 679 for (i = 0; i < VCHIQ_INIT_RETRIES; i++) { 680 state = vchiq_get_state(); 681 if (state) 682 break; 683 usleep_range(500, 600); 684 } 685 if (i == VCHIQ_INIT_RETRIES) { 686 vchiq_log_error(vchiq_core_log_level, "%s: videocore not initialized\n", __func__); 687 ret = -ENOTCONN; 688 goto failed; 689 } else if (i > 0) { 690 vchiq_log_warning(vchiq_core_log_level, 691 "%s: videocore initialized after %d retries\n", __func__, i); 692 } 693 694 instance = kzalloc(sizeof(*instance), GFP_KERNEL); 695 if (!instance) { 696 vchiq_log_error(vchiq_core_log_level, 697 "%s: error allocating vchiq instance\n", __func__); 698 ret = -ENOMEM; 699 goto failed; 700 } 701 702 instance->connected = 0; 703 instance->state = state; 704 mutex_init(&instance->bulk_waiter_list_mutex); 705 INIT_LIST_HEAD(&instance->bulk_waiter_list); 706 707 *instance_out = instance; 708 709 ret = 0; 710 711 failed: 712 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, ret); 713 714 return ret; 715 } 716 EXPORT_SYMBOL(vchiq_initialise); 717 718 void free_bulk_waiter(struct vchiq_instance *instance) 719 { 720 struct bulk_waiter_node *waiter, *next; 721 722 list_for_each_entry_safe(waiter, next, 723 &instance->bulk_waiter_list, list) { 724 list_del(&waiter->list); 725 vchiq_log_info(vchiq_arm_log_level, "bulk_waiter - cleaned up %pK for pid %d", 726 waiter, waiter->pid); 727 kfree(waiter); 728 } 729 } 730 731 int vchiq_shutdown(struct vchiq_instance *instance) 732 { 733 int status = 0; 734 struct vchiq_state *state = instance->state; 735 736 if (mutex_lock_killable(&state->mutex)) 737 return -EAGAIN; 738 739 /* Remove all services */ 740 vchiq_shutdown_internal(state, instance); 741 742 mutex_unlock(&state->mutex); 743 744 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status); 745 746 free_bulk_waiter(instance); 747 kfree(instance); 748 749 return status; 750 } 751 EXPORT_SYMBOL(vchiq_shutdown); 752 753 static int vchiq_is_connected(struct vchiq_instance *instance) 754 { 755 return instance->connected; 756 } 757 758 int vchiq_connect(struct vchiq_instance *instance) 759 { 760 int status; 761 struct vchiq_state *state = instance->state; 762 763 if (mutex_lock_killable(&state->mutex)) { 764 vchiq_log_trace(vchiq_core_log_level, "%s: call to mutex_lock failed", __func__); 765 status = -EAGAIN; 766 goto failed; 767 } 768 status = vchiq_connect_internal(state, instance); 769 770 if (!status) 771 instance->connected = 1; 772 773 mutex_unlock(&state->mutex); 774 775 failed: 776 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status); 777 778 return status; 779 } 780 EXPORT_SYMBOL(vchiq_connect); 781 782 static int 783 vchiq_add_service(struct vchiq_instance *instance, 784 const struct vchiq_service_params_kernel *params, 785 unsigned int *phandle) 786 { 787 int status; 788 struct vchiq_state *state = instance->state; 789 struct vchiq_service *service = NULL; 790 int srvstate; 791 792 *phandle = VCHIQ_SERVICE_HANDLE_INVALID; 793 794 srvstate = vchiq_is_connected(instance) 795 ? VCHIQ_SRVSTATE_LISTENING 796 : VCHIQ_SRVSTATE_HIDDEN; 797 798 service = vchiq_add_service_internal(state, params, srvstate, instance, NULL); 799 800 if (service) { 801 *phandle = service->handle; 802 status = 0; 803 } else { 804 status = -EINVAL; 805 } 806 807 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status); 808 809 return status; 810 } 811 812 int 813 vchiq_open_service(struct vchiq_instance *instance, 814 const struct vchiq_service_params_kernel *params, 815 unsigned int *phandle) 816 { 817 int status = -EINVAL; 818 struct vchiq_state *state = instance->state; 819 struct vchiq_service *service = NULL; 820 821 *phandle = VCHIQ_SERVICE_HANDLE_INVALID; 822 823 if (!vchiq_is_connected(instance)) 824 goto failed; 825 826 service = vchiq_add_service_internal(state, params, VCHIQ_SRVSTATE_OPENING, instance, NULL); 827 828 if (service) { 829 *phandle = service->handle; 830 status = vchiq_open_service_internal(service, current->pid); 831 if (status) { 832 vchiq_remove_service(instance, service->handle); 833 *phandle = VCHIQ_SERVICE_HANDLE_INVALID; 834 } 835 } 836 837 failed: 838 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status); 839 840 return status; 841 } 842 EXPORT_SYMBOL(vchiq_open_service); 843 844 int 845 vchiq_bulk_transmit(struct vchiq_instance *instance, unsigned int handle, const void *data, 846 unsigned int size, void *userdata, enum vchiq_bulk_mode mode) 847 { 848 int status; 849 850 while (1) { 851 switch (mode) { 852 case VCHIQ_BULK_MODE_NOCALLBACK: 853 case VCHIQ_BULK_MODE_CALLBACK: 854 status = vchiq_bulk_transfer(instance, handle, 855 (void *)data, NULL, 856 size, userdata, mode, 857 VCHIQ_BULK_TRANSMIT); 858 break; 859 case VCHIQ_BULK_MODE_BLOCKING: 860 status = vchiq_blocking_bulk_transfer(instance, handle, (void *)data, size, 861 VCHIQ_BULK_TRANSMIT); 862 break; 863 default: 864 return -EINVAL; 865 } 866 867 /* 868 * vchiq_*_bulk_transfer() may return -EAGAIN, so we need 869 * to implement a retry mechanism since this function is 870 * supposed to block until queued 871 */ 872 if (status != -EAGAIN) 873 break; 874 875 msleep(1); 876 } 877 878 return status; 879 } 880 EXPORT_SYMBOL(vchiq_bulk_transmit); 881 882 int vchiq_bulk_receive(struct vchiq_instance *instance, unsigned int handle, 883 void *data, unsigned int size, void *userdata, 884 enum vchiq_bulk_mode mode) 885 { 886 int status; 887 888 while (1) { 889 switch (mode) { 890 case VCHIQ_BULK_MODE_NOCALLBACK: 891 case VCHIQ_BULK_MODE_CALLBACK: 892 status = vchiq_bulk_transfer(instance, handle, data, NULL, 893 size, userdata, 894 mode, VCHIQ_BULK_RECEIVE); 895 break; 896 case VCHIQ_BULK_MODE_BLOCKING: 897 status = vchiq_blocking_bulk_transfer(instance, handle, (void *)data, size, 898 VCHIQ_BULK_RECEIVE); 899 break; 900 default: 901 return -EINVAL; 902 } 903 904 /* 905 * vchiq_*_bulk_transfer() may return -EAGAIN, so we need 906 * to implement a retry mechanism since this function is 907 * supposed to block until queued 908 */ 909 if (status != -EAGAIN) 910 break; 911 912 msleep(1); 913 } 914 915 return status; 916 } 917 EXPORT_SYMBOL(vchiq_bulk_receive); 918 919 static int 920 vchiq_blocking_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *data, 921 unsigned int size, enum vchiq_bulk_dir dir) 922 { 923 struct vchiq_service *service; 924 int status; 925 struct bulk_waiter_node *waiter = NULL, *iter; 926 927 service = find_service_by_handle(instance, handle); 928 if (!service) 929 return -EINVAL; 930 931 vchiq_service_put(service); 932 933 mutex_lock(&instance->bulk_waiter_list_mutex); 934 list_for_each_entry(iter, &instance->bulk_waiter_list, list) { 935 if (iter->pid == current->pid) { 936 list_del(&iter->list); 937 waiter = iter; 938 break; 939 } 940 } 941 mutex_unlock(&instance->bulk_waiter_list_mutex); 942 943 if (waiter) { 944 struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk; 945 946 if (bulk) { 947 /* This thread has an outstanding bulk transfer. */ 948 /* FIXME: why compare a dma address to a pointer? */ 949 if ((bulk->data != (dma_addr_t)(uintptr_t)data) || (bulk->size != size)) { 950 /* 951 * This is not a retry of the previous one. 952 * Cancel the signal when the transfer completes. 953 */ 954 spin_lock(&bulk_waiter_spinlock); 955 bulk->userdata = NULL; 956 spin_unlock(&bulk_waiter_spinlock); 957 } 958 } 959 } else { 960 waiter = kzalloc(sizeof(*waiter), GFP_KERNEL); 961 if (!waiter) { 962 vchiq_log_error(vchiq_core_log_level, "%s - out of memory", __func__); 963 return -ENOMEM; 964 } 965 } 966 967 status = vchiq_bulk_transfer(instance, handle, data, NULL, size, 968 &waiter->bulk_waiter, 969 VCHIQ_BULK_MODE_BLOCKING, dir); 970 if ((status != -EAGAIN) || fatal_signal_pending(current) || !waiter->bulk_waiter.bulk) { 971 struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk; 972 973 if (bulk) { 974 /* Cancel the signal when the transfer completes. */ 975 spin_lock(&bulk_waiter_spinlock); 976 bulk->userdata = NULL; 977 spin_unlock(&bulk_waiter_spinlock); 978 } 979 kfree(waiter); 980 } else { 981 waiter->pid = current->pid; 982 mutex_lock(&instance->bulk_waiter_list_mutex); 983 list_add(&waiter->list, &instance->bulk_waiter_list); 984 mutex_unlock(&instance->bulk_waiter_list_mutex); 985 vchiq_log_info(vchiq_arm_log_level, "saved bulk_waiter %pK for pid %d", waiter, 986 current->pid); 987 } 988 989 return status; 990 } 991 992 static int 993 add_completion(struct vchiq_instance *instance, enum vchiq_reason reason, 994 struct vchiq_header *header, struct user_service *user_service, 995 void *bulk_userdata) 996 { 997 struct vchiq_completion_data_kernel *completion; 998 int insert; 999 1000 DEBUG_INITIALISE(g_state.local); 1001 1002 insert = instance->completion_insert; 1003 while ((insert - instance->completion_remove) >= MAX_COMPLETIONS) { 1004 /* Out of space - wait for the client */ 1005 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1006 vchiq_log_trace(vchiq_arm_log_level, "%s - completion queue full", __func__); 1007 DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT); 1008 if (wait_for_completion_interruptible(&instance->remove_event)) { 1009 vchiq_log_info(vchiq_arm_log_level, "service_callback interrupted"); 1010 return -EAGAIN; 1011 } else if (instance->closing) { 1012 vchiq_log_info(vchiq_arm_log_level, "service_callback closing"); 1013 return 0; 1014 } 1015 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1016 } 1017 1018 completion = &instance->completions[insert & (MAX_COMPLETIONS - 1)]; 1019 1020 completion->header = header; 1021 completion->reason = reason; 1022 /* N.B. service_userdata is updated while processing AWAIT_COMPLETION */ 1023 completion->service_userdata = user_service->service; 1024 completion->bulk_userdata = bulk_userdata; 1025 1026 if (reason == VCHIQ_SERVICE_CLOSED) { 1027 /* 1028 * Take an extra reference, to be held until 1029 * this CLOSED notification is delivered. 1030 */ 1031 vchiq_service_get(user_service->service); 1032 if (instance->use_close_delivered) 1033 user_service->close_pending = 1; 1034 } 1035 1036 /* 1037 * A write barrier is needed here to ensure that the entire completion 1038 * record is written out before the insert point. 1039 */ 1040 wmb(); 1041 1042 if (reason == VCHIQ_MESSAGE_AVAILABLE) 1043 user_service->message_available_pos = insert; 1044 1045 insert++; 1046 instance->completion_insert = insert; 1047 1048 complete(&instance->insert_event); 1049 1050 return 0; 1051 } 1052 1053 int 1054 service_callback(struct vchiq_instance *instance, enum vchiq_reason reason, 1055 struct vchiq_header *header, unsigned int handle, void *bulk_userdata) 1056 { 1057 /* 1058 * How do we ensure the callback goes to the right client? 1059 * The service_user data points to a user_service record 1060 * containing the original callback and the user state structure, which 1061 * contains a circular buffer for completion records. 1062 */ 1063 struct user_service *user_service; 1064 struct vchiq_service *service; 1065 bool skip_completion = false; 1066 1067 DEBUG_INITIALISE(g_state.local); 1068 1069 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1070 1071 rcu_read_lock(); 1072 service = handle_to_service(instance, handle); 1073 if (WARN_ON(!service)) { 1074 rcu_read_unlock(); 1075 return 0; 1076 } 1077 1078 user_service = (struct user_service *)service->base.userdata; 1079 1080 if (!instance || instance->closing) { 1081 rcu_read_unlock(); 1082 return 0; 1083 } 1084 1085 /* 1086 * As hopping around different synchronization mechanism, 1087 * taking an extra reference results in simpler implementation. 1088 */ 1089 vchiq_service_get(service); 1090 rcu_read_unlock(); 1091 1092 vchiq_log_trace(vchiq_arm_log_level, 1093 "%s - service %lx(%d,%p), reason %d, header %lx, instance %lx, bulk_userdata %lx", 1094 __func__, (unsigned long)user_service, service->localport, 1095 user_service->userdata, reason, (unsigned long)header, 1096 (unsigned long)instance, (unsigned long)bulk_userdata); 1097 1098 if (header && user_service->is_vchi) { 1099 spin_lock(&msg_queue_spinlock); 1100 while (user_service->msg_insert == 1101 (user_service->msg_remove + MSG_QUEUE_SIZE)) { 1102 spin_unlock(&msg_queue_spinlock); 1103 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1104 DEBUG_COUNT(MSG_QUEUE_FULL_COUNT); 1105 vchiq_log_trace(vchiq_arm_log_level, "%s - msg queue full", __func__); 1106 /* 1107 * If there is no MESSAGE_AVAILABLE in the completion 1108 * queue, add one 1109 */ 1110 if ((user_service->message_available_pos - 1111 instance->completion_remove) < 0) { 1112 int status; 1113 1114 vchiq_log_info(vchiq_arm_log_level, 1115 "Inserting extra MESSAGE_AVAILABLE"); 1116 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1117 status = add_completion(instance, reason, NULL, user_service, 1118 bulk_userdata); 1119 if (status) { 1120 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1121 vchiq_service_put(service); 1122 return status; 1123 } 1124 } 1125 1126 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1127 if (wait_for_completion_interruptible(&user_service->remove_event)) { 1128 vchiq_log_info(vchiq_arm_log_level, "%s interrupted", __func__); 1129 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1130 vchiq_service_put(service); 1131 return -EAGAIN; 1132 } else if (instance->closing) { 1133 vchiq_log_info(vchiq_arm_log_level, "%s closing", __func__); 1134 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1135 vchiq_service_put(service); 1136 return -EINVAL; 1137 } 1138 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1139 spin_lock(&msg_queue_spinlock); 1140 } 1141 1142 user_service->msg_queue[user_service->msg_insert & 1143 (MSG_QUEUE_SIZE - 1)] = header; 1144 user_service->msg_insert++; 1145 1146 /* 1147 * If there is a thread waiting in DEQUEUE_MESSAGE, or if 1148 * there is a MESSAGE_AVAILABLE in the completion queue then 1149 * bypass the completion queue. 1150 */ 1151 if (((user_service->message_available_pos - 1152 instance->completion_remove) >= 0) || 1153 user_service->dequeue_pending) { 1154 user_service->dequeue_pending = 0; 1155 skip_completion = true; 1156 } 1157 1158 spin_unlock(&msg_queue_spinlock); 1159 complete(&user_service->insert_event); 1160 1161 header = NULL; 1162 } 1163 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1164 vchiq_service_put(service); 1165 1166 if (skip_completion) 1167 return 0; 1168 1169 return add_completion(instance, reason, header, user_service, 1170 bulk_userdata); 1171 } 1172 1173 int vchiq_dump(void *dump_context, const char *str, int len) 1174 { 1175 struct dump_context *context = (struct dump_context *)dump_context; 1176 int copy_bytes; 1177 1178 if (context->actual >= context->space) 1179 return 0; 1180 1181 if (context->offset > 0) { 1182 int skip_bytes = min_t(int, len, context->offset); 1183 1184 str += skip_bytes; 1185 len -= skip_bytes; 1186 context->offset -= skip_bytes; 1187 if (context->offset > 0) 1188 return 0; 1189 } 1190 copy_bytes = min_t(int, len, context->space - context->actual); 1191 if (copy_bytes == 0) 1192 return 0; 1193 if (copy_to_user(context->buf + context->actual, str, 1194 copy_bytes)) 1195 return -EFAULT; 1196 context->actual += copy_bytes; 1197 len -= copy_bytes; 1198 1199 /* 1200 * If the terminating NUL is included in the length, then it 1201 * marks the end of a line and should be replaced with a 1202 * carriage return. 1203 */ 1204 if ((len == 0) && (str[copy_bytes - 1] == '\0')) { 1205 char cr = '\n'; 1206 1207 if (copy_to_user(context->buf + context->actual - 1, 1208 &cr, 1)) 1209 return -EFAULT; 1210 } 1211 return 0; 1212 } 1213 1214 int vchiq_dump_platform_instances(void *dump_context) 1215 { 1216 struct vchiq_state *state = vchiq_get_state(); 1217 char buf[80]; 1218 int len; 1219 int i; 1220 1221 if (!state) 1222 return -ENOTCONN; 1223 1224 /* 1225 * There is no list of instances, so instead scan all services, 1226 * marking those that have been dumped. 1227 */ 1228 1229 rcu_read_lock(); 1230 for (i = 0; i < state->unused_service; i++) { 1231 struct vchiq_service *service; 1232 struct vchiq_instance *instance; 1233 1234 service = rcu_dereference(state->services[i]); 1235 if (!service || service->base.callback != service_callback) 1236 continue; 1237 1238 instance = service->instance; 1239 if (instance) 1240 instance->mark = 0; 1241 } 1242 rcu_read_unlock(); 1243 1244 for (i = 0; i < state->unused_service; i++) { 1245 struct vchiq_service *service; 1246 struct vchiq_instance *instance; 1247 int err; 1248 1249 rcu_read_lock(); 1250 service = rcu_dereference(state->services[i]); 1251 if (!service || service->base.callback != service_callback) { 1252 rcu_read_unlock(); 1253 continue; 1254 } 1255 1256 instance = service->instance; 1257 if (!instance || instance->mark) { 1258 rcu_read_unlock(); 1259 continue; 1260 } 1261 rcu_read_unlock(); 1262 1263 len = snprintf(buf, sizeof(buf), 1264 "Instance %pK: pid %d,%s completions %d/%d", 1265 instance, instance->pid, 1266 instance->connected ? " connected, " : 1267 "", 1268 instance->completion_insert - 1269 instance->completion_remove, 1270 MAX_COMPLETIONS); 1271 err = vchiq_dump(dump_context, buf, len + 1); 1272 if (err) 1273 return err; 1274 instance->mark = 1; 1275 } 1276 return 0; 1277 } 1278 1279 int vchiq_dump_platform_service_state(void *dump_context, 1280 struct vchiq_service *service) 1281 { 1282 struct user_service *user_service = 1283 (struct user_service *)service->base.userdata; 1284 char buf[80]; 1285 int len; 1286 1287 len = scnprintf(buf, sizeof(buf), " instance %pK", service->instance); 1288 1289 if ((service->base.callback == service_callback) && user_service->is_vchi) { 1290 len += scnprintf(buf + len, sizeof(buf) - len, ", %d/%d messages", 1291 user_service->msg_insert - user_service->msg_remove, 1292 MSG_QUEUE_SIZE); 1293 1294 if (user_service->dequeue_pending) 1295 len += scnprintf(buf + len, sizeof(buf) - len, 1296 " (dequeue pending)"); 1297 } 1298 1299 return vchiq_dump(dump_context, buf, len + 1); 1300 } 1301 1302 struct vchiq_state * 1303 vchiq_get_state(void) 1304 { 1305 if (!g_state.remote) { 1306 pr_err("%s: g_state.remote == NULL\n", __func__); 1307 return NULL; 1308 } 1309 1310 if (g_state.remote->initialised != 1) { 1311 pr_notice("%s: g_state.remote->initialised != 1 (%d)\n", 1312 __func__, g_state.remote->initialised); 1313 return NULL; 1314 } 1315 1316 return &g_state; 1317 } 1318 1319 /* 1320 * Autosuspend related functionality 1321 */ 1322 1323 static int 1324 vchiq_keepalive_vchiq_callback(struct vchiq_instance *instance, 1325 enum vchiq_reason reason, 1326 struct vchiq_header *header, 1327 unsigned int service_user, void *bulk_user) 1328 { 1329 vchiq_log_error(vchiq_susp_log_level, "%s callback reason %d", __func__, reason); 1330 return 0; 1331 } 1332 1333 static int 1334 vchiq_keepalive_thread_func(void *v) 1335 { 1336 struct vchiq_state *state = (struct vchiq_state *)v; 1337 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1338 1339 int status; 1340 struct vchiq_instance *instance; 1341 unsigned int ka_handle; 1342 int ret; 1343 1344 struct vchiq_service_params_kernel params = { 1345 .fourcc = VCHIQ_MAKE_FOURCC('K', 'E', 'E', 'P'), 1346 .callback = vchiq_keepalive_vchiq_callback, 1347 .version = KEEPALIVE_VER, 1348 .version_min = KEEPALIVE_VER_MIN 1349 }; 1350 1351 ret = vchiq_initialise(&instance); 1352 if (ret) { 1353 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_initialise failed %d", __func__, 1354 ret); 1355 goto exit; 1356 } 1357 1358 status = vchiq_connect(instance); 1359 if (status) { 1360 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_connect failed %d", __func__, 1361 status); 1362 goto shutdown; 1363 } 1364 1365 status = vchiq_add_service(instance, ¶ms, &ka_handle); 1366 if (status) { 1367 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_open_service failed %d", __func__, 1368 status); 1369 goto shutdown; 1370 } 1371 1372 while (1) { 1373 long rc = 0, uc = 0; 1374 1375 if (wait_for_completion_interruptible(&arm_state->ka_evt)) { 1376 vchiq_log_error(vchiq_susp_log_level, "%s interrupted", __func__); 1377 flush_signals(current); 1378 continue; 1379 } 1380 1381 /* 1382 * read and clear counters. Do release_count then use_count to 1383 * prevent getting more releases than uses 1384 */ 1385 rc = atomic_xchg(&arm_state->ka_release_count, 0); 1386 uc = atomic_xchg(&arm_state->ka_use_count, 0); 1387 1388 /* 1389 * Call use/release service the requisite number of times. 1390 * Process use before release so use counts don't go negative 1391 */ 1392 while (uc--) { 1393 atomic_inc(&arm_state->ka_use_ack_count); 1394 status = vchiq_use_service(instance, ka_handle); 1395 if (status) { 1396 vchiq_log_error(vchiq_susp_log_level, 1397 "%s vchiq_use_service error %d", __func__, status); 1398 } 1399 } 1400 while (rc--) { 1401 status = vchiq_release_service(instance, ka_handle); 1402 if (status) { 1403 vchiq_log_error(vchiq_susp_log_level, 1404 "%s vchiq_release_service error %d", __func__, 1405 status); 1406 } 1407 } 1408 } 1409 1410 shutdown: 1411 vchiq_shutdown(instance); 1412 exit: 1413 return 0; 1414 } 1415 1416 int 1417 vchiq_use_internal(struct vchiq_state *state, struct vchiq_service *service, 1418 enum USE_TYPE_E use_type) 1419 { 1420 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1421 int ret = 0; 1422 char entity[16]; 1423 int *entity_uc; 1424 int local_uc; 1425 1426 if (!arm_state) { 1427 ret = -EINVAL; 1428 goto out; 1429 } 1430 1431 if (use_type == USE_TYPE_VCHIQ) { 1432 sprintf(entity, "VCHIQ: "); 1433 entity_uc = &arm_state->peer_use_count; 1434 } else if (service) { 1435 sprintf(entity, "%c%c%c%c:%03d", 1436 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), 1437 service->client_id); 1438 entity_uc = &service->service_use_count; 1439 } else { 1440 vchiq_log_error(vchiq_susp_log_level, "%s null service ptr", __func__); 1441 ret = -EINVAL; 1442 goto out; 1443 } 1444 1445 write_lock_bh(&arm_state->susp_res_lock); 1446 local_uc = ++arm_state->videocore_use_count; 1447 ++(*entity_uc); 1448 1449 vchiq_log_trace(vchiq_susp_log_level, "%s %s count %d, state count %d", __func__, entity, 1450 *entity_uc, local_uc); 1451 1452 write_unlock_bh(&arm_state->susp_res_lock); 1453 1454 if (!ret) { 1455 int status = 0; 1456 long ack_cnt = atomic_xchg(&arm_state->ka_use_ack_count, 0); 1457 1458 while (ack_cnt && !status) { 1459 /* Send the use notify to videocore */ 1460 status = vchiq_send_remote_use_active(state); 1461 if (!status) 1462 ack_cnt--; 1463 else 1464 atomic_add(ack_cnt, &arm_state->ka_use_ack_count); 1465 } 1466 } 1467 1468 out: 1469 vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret); 1470 return ret; 1471 } 1472 1473 int 1474 vchiq_release_internal(struct vchiq_state *state, struct vchiq_service *service) 1475 { 1476 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1477 int ret = 0; 1478 char entity[16]; 1479 int *entity_uc; 1480 1481 if (!arm_state) { 1482 ret = -EINVAL; 1483 goto out; 1484 } 1485 1486 if (service) { 1487 sprintf(entity, "%c%c%c%c:%03d", 1488 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), 1489 service->client_id); 1490 entity_uc = &service->service_use_count; 1491 } else { 1492 sprintf(entity, "PEER: "); 1493 entity_uc = &arm_state->peer_use_count; 1494 } 1495 1496 write_lock_bh(&arm_state->susp_res_lock); 1497 if (!arm_state->videocore_use_count || !(*entity_uc)) { 1498 /* Don't use BUG_ON - don't allow user thread to crash kernel */ 1499 WARN_ON(!arm_state->videocore_use_count); 1500 WARN_ON(!(*entity_uc)); 1501 ret = -EINVAL; 1502 goto unlock; 1503 } 1504 --arm_state->videocore_use_count; 1505 --(*entity_uc); 1506 1507 vchiq_log_trace(vchiq_susp_log_level, "%s %s count %d, state count %d", __func__, entity, 1508 *entity_uc, arm_state->videocore_use_count); 1509 1510 unlock: 1511 write_unlock_bh(&arm_state->susp_res_lock); 1512 1513 out: 1514 vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret); 1515 return ret; 1516 } 1517 1518 void 1519 vchiq_on_remote_use(struct vchiq_state *state) 1520 { 1521 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1522 1523 atomic_inc(&arm_state->ka_use_count); 1524 complete(&arm_state->ka_evt); 1525 } 1526 1527 void 1528 vchiq_on_remote_release(struct vchiq_state *state) 1529 { 1530 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1531 1532 atomic_inc(&arm_state->ka_release_count); 1533 complete(&arm_state->ka_evt); 1534 } 1535 1536 int 1537 vchiq_use_service_internal(struct vchiq_service *service) 1538 { 1539 return vchiq_use_internal(service->state, service, USE_TYPE_SERVICE); 1540 } 1541 1542 int 1543 vchiq_release_service_internal(struct vchiq_service *service) 1544 { 1545 return vchiq_release_internal(service->state, service); 1546 } 1547 1548 struct vchiq_debugfs_node * 1549 vchiq_instance_get_debugfs_node(struct vchiq_instance *instance) 1550 { 1551 return &instance->debugfs_node; 1552 } 1553 1554 int 1555 vchiq_instance_get_use_count(struct vchiq_instance *instance) 1556 { 1557 struct vchiq_service *service; 1558 int use_count = 0, i; 1559 1560 i = 0; 1561 rcu_read_lock(); 1562 while ((service = __next_service_by_instance(instance->state, 1563 instance, &i))) 1564 use_count += service->service_use_count; 1565 rcu_read_unlock(); 1566 return use_count; 1567 } 1568 1569 int 1570 vchiq_instance_get_pid(struct vchiq_instance *instance) 1571 { 1572 return instance->pid; 1573 } 1574 1575 int 1576 vchiq_instance_get_trace(struct vchiq_instance *instance) 1577 { 1578 return instance->trace; 1579 } 1580 1581 void 1582 vchiq_instance_set_trace(struct vchiq_instance *instance, int trace) 1583 { 1584 struct vchiq_service *service; 1585 int i; 1586 1587 i = 0; 1588 rcu_read_lock(); 1589 while ((service = __next_service_by_instance(instance->state, 1590 instance, &i))) 1591 service->trace = trace; 1592 rcu_read_unlock(); 1593 instance->trace = (trace != 0); 1594 } 1595 1596 int 1597 vchiq_use_service(struct vchiq_instance *instance, unsigned int handle) 1598 { 1599 int ret = -EINVAL; 1600 struct vchiq_service *service = find_service_by_handle(instance, handle); 1601 1602 if (service) { 1603 ret = vchiq_use_internal(service->state, service, USE_TYPE_SERVICE); 1604 vchiq_service_put(service); 1605 } 1606 return ret; 1607 } 1608 EXPORT_SYMBOL(vchiq_use_service); 1609 1610 int 1611 vchiq_release_service(struct vchiq_instance *instance, unsigned int handle) 1612 { 1613 int ret = -EINVAL; 1614 struct vchiq_service *service = find_service_by_handle(instance, handle); 1615 1616 if (service) { 1617 ret = vchiq_release_internal(service->state, service); 1618 vchiq_service_put(service); 1619 } 1620 return ret; 1621 } 1622 EXPORT_SYMBOL(vchiq_release_service); 1623 1624 struct service_data_struct { 1625 int fourcc; 1626 int clientid; 1627 int use_count; 1628 }; 1629 1630 void 1631 vchiq_dump_service_use_state(struct vchiq_state *state) 1632 { 1633 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1634 struct service_data_struct *service_data; 1635 int i, found = 0; 1636 /* 1637 * If there's more than 64 services, only dump ones with 1638 * non-zero counts 1639 */ 1640 int only_nonzero = 0; 1641 static const char *nz = "<-- preventing suspend"; 1642 1643 int peer_count; 1644 int vc_use_count; 1645 int active_services; 1646 1647 if (!arm_state) 1648 return; 1649 1650 service_data = kmalloc_array(MAX_SERVICES, sizeof(*service_data), 1651 GFP_KERNEL); 1652 if (!service_data) 1653 return; 1654 1655 read_lock_bh(&arm_state->susp_res_lock); 1656 peer_count = arm_state->peer_use_count; 1657 vc_use_count = arm_state->videocore_use_count; 1658 active_services = state->unused_service; 1659 if (active_services > MAX_SERVICES) 1660 only_nonzero = 1; 1661 1662 rcu_read_lock(); 1663 for (i = 0; i < active_services; i++) { 1664 struct vchiq_service *service_ptr = 1665 rcu_dereference(state->services[i]); 1666 1667 if (!service_ptr) 1668 continue; 1669 1670 if (only_nonzero && !service_ptr->service_use_count) 1671 continue; 1672 1673 if (service_ptr->srvstate == VCHIQ_SRVSTATE_FREE) 1674 continue; 1675 1676 service_data[found].fourcc = service_ptr->base.fourcc; 1677 service_data[found].clientid = service_ptr->client_id; 1678 service_data[found].use_count = service_ptr->service_use_count; 1679 found++; 1680 if (found >= MAX_SERVICES) 1681 break; 1682 } 1683 rcu_read_unlock(); 1684 1685 read_unlock_bh(&arm_state->susp_res_lock); 1686 1687 if (only_nonzero) 1688 vchiq_log_warning(vchiq_susp_log_level, "Too many active services (%d). Only dumping up to first %d services with non-zero use-count", 1689 active_services, found); 1690 1691 for (i = 0; i < found; i++) { 1692 vchiq_log_warning(vchiq_susp_log_level, "----- %c%c%c%c:%d service count %d %s", 1693 VCHIQ_FOURCC_AS_4CHARS(service_data[i].fourcc), 1694 service_data[i].clientid, service_data[i].use_count, 1695 service_data[i].use_count ? nz : ""); 1696 } 1697 vchiq_log_warning(vchiq_susp_log_level, "----- VCHIQ use count %d", peer_count); 1698 vchiq_log_warning(vchiq_susp_log_level, "--- Overall vchiq instance use count %d", 1699 vc_use_count); 1700 1701 kfree(service_data); 1702 } 1703 1704 int 1705 vchiq_check_service(struct vchiq_service *service) 1706 { 1707 struct vchiq_arm_state *arm_state; 1708 int ret = -EINVAL; 1709 1710 if (!service || !service->state) 1711 goto out; 1712 1713 arm_state = vchiq_platform_get_arm_state(service->state); 1714 1715 read_lock_bh(&arm_state->susp_res_lock); 1716 if (service->service_use_count) 1717 ret = 0; 1718 read_unlock_bh(&arm_state->susp_res_lock); 1719 1720 if (ret) { 1721 vchiq_log_error(vchiq_susp_log_level, 1722 "%s ERROR - %c%c%c%c:%d service count %d, state count %d", __func__, 1723 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), service->client_id, 1724 service->service_use_count, arm_state->videocore_use_count); 1725 vchiq_dump_service_use_state(service->state); 1726 } 1727 out: 1728 return ret; 1729 } 1730 1731 void vchiq_platform_conn_state_changed(struct vchiq_state *state, 1732 enum vchiq_connstate oldstate, 1733 enum vchiq_connstate newstate) 1734 { 1735 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1736 char threadname[16]; 1737 1738 vchiq_log_info(vchiq_susp_log_level, "%d: %s->%s", state->id, 1739 get_conn_state_name(oldstate), get_conn_state_name(newstate)); 1740 if (state->conn_state != VCHIQ_CONNSTATE_CONNECTED) 1741 return; 1742 1743 write_lock_bh(&arm_state->susp_res_lock); 1744 if (arm_state->first_connect) { 1745 write_unlock_bh(&arm_state->susp_res_lock); 1746 return; 1747 } 1748 1749 arm_state->first_connect = 1; 1750 write_unlock_bh(&arm_state->susp_res_lock); 1751 snprintf(threadname, sizeof(threadname), "vchiq-keep/%d", 1752 state->id); 1753 arm_state->ka_thread = kthread_create(&vchiq_keepalive_thread_func, 1754 (void *)state, 1755 threadname); 1756 if (IS_ERR(arm_state->ka_thread)) { 1757 vchiq_log_error(vchiq_susp_log_level, 1758 "vchiq: FATAL: couldn't create thread %s", 1759 threadname); 1760 } else { 1761 wake_up_process(arm_state->ka_thread); 1762 } 1763 } 1764 1765 static const struct of_device_id vchiq_of_match[] = { 1766 { .compatible = "brcm,bcm2835-vchiq", .data = &bcm2835_drvdata }, 1767 { .compatible = "brcm,bcm2836-vchiq", .data = &bcm2836_drvdata }, 1768 {}, 1769 }; 1770 MODULE_DEVICE_TABLE(of, vchiq_of_match); 1771 1772 static struct platform_device * 1773 vchiq_register_child(struct platform_device *pdev, const char *name) 1774 { 1775 struct platform_device_info pdevinfo; 1776 struct platform_device *child; 1777 1778 memset(&pdevinfo, 0, sizeof(pdevinfo)); 1779 1780 pdevinfo.parent = &pdev->dev; 1781 pdevinfo.name = name; 1782 pdevinfo.id = PLATFORM_DEVID_NONE; 1783 pdevinfo.dma_mask = DMA_BIT_MASK(32); 1784 1785 child = platform_device_register_full(&pdevinfo); 1786 if (IS_ERR(child)) { 1787 dev_warn(&pdev->dev, "%s not registered\n", name); 1788 child = NULL; 1789 } 1790 1791 return child; 1792 } 1793 1794 static int vchiq_probe(struct platform_device *pdev) 1795 { 1796 struct device_node *fw_node; 1797 const struct of_device_id *of_id; 1798 struct vchiq_drvdata *drvdata; 1799 int err; 1800 1801 of_id = of_match_node(vchiq_of_match, pdev->dev.of_node); 1802 drvdata = (struct vchiq_drvdata *)of_id->data; 1803 if (!drvdata) 1804 return -EINVAL; 1805 1806 fw_node = of_find_compatible_node(NULL, NULL, 1807 "raspberrypi,bcm2835-firmware"); 1808 if (!fw_node) { 1809 dev_err(&pdev->dev, "Missing firmware node\n"); 1810 return -ENOENT; 1811 } 1812 1813 drvdata->fw = devm_rpi_firmware_get(&pdev->dev, fw_node); 1814 of_node_put(fw_node); 1815 if (!drvdata->fw) 1816 return -EPROBE_DEFER; 1817 1818 platform_set_drvdata(pdev, drvdata); 1819 1820 err = vchiq_platform_init(pdev, &g_state); 1821 if (err) 1822 goto failed_platform_init; 1823 1824 vchiq_debugfs_init(); 1825 1826 vchiq_log_info(vchiq_arm_log_level, 1827 "vchiq: platform initialised - version %d (min %d)", 1828 VCHIQ_VERSION, VCHIQ_VERSION_MIN); 1829 1830 /* 1831 * Simply exit on error since the function handles cleanup in 1832 * cases of failure. 1833 */ 1834 err = vchiq_register_chrdev(&pdev->dev); 1835 if (err) { 1836 vchiq_log_warning(vchiq_arm_log_level, 1837 "Failed to initialize vchiq cdev"); 1838 goto error_exit; 1839 } 1840 1841 bcm2835_camera = vchiq_register_child(pdev, "bcm2835-camera"); 1842 bcm2835_audio = vchiq_register_child(pdev, "bcm2835_audio"); 1843 1844 return 0; 1845 1846 failed_platform_init: 1847 vchiq_log_warning(vchiq_arm_log_level, "could not initialize vchiq platform"); 1848 error_exit: 1849 return err; 1850 } 1851 1852 static void vchiq_remove(struct platform_device *pdev) 1853 { 1854 platform_device_unregister(bcm2835_audio); 1855 platform_device_unregister(bcm2835_camera); 1856 vchiq_debugfs_deinit(); 1857 vchiq_deregister_chrdev(); 1858 } 1859 1860 static struct platform_driver vchiq_driver = { 1861 .driver = { 1862 .name = "bcm2835_vchiq", 1863 .of_match_table = vchiq_of_match, 1864 }, 1865 .probe = vchiq_probe, 1866 .remove_new = vchiq_remove, 1867 }; 1868 1869 static int __init vchiq_driver_init(void) 1870 { 1871 int ret; 1872 1873 ret = platform_driver_register(&vchiq_driver); 1874 if (ret) 1875 pr_err("Failed to register vchiq driver\n"); 1876 1877 return ret; 1878 } 1879 module_init(vchiq_driver_init); 1880 1881 static void __exit vchiq_driver_exit(void) 1882 { 1883 platform_driver_unregister(&vchiq_driver); 1884 } 1885 module_exit(vchiq_driver_exit); 1886 1887 MODULE_LICENSE("Dual BSD/GPL"); 1888 MODULE_DESCRIPTION("Videocore VCHIQ driver"); 1889 MODULE_AUTHOR("Broadcom Corporation"); 1890