1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright (c) 2014 Raspberry Pi (Trading) Ltd. All rights reserved. 4 * Copyright (c) 2010-2012 Broadcom. All rights reserved. 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/sched/signal.h> 10 #include <linux/types.h> 11 #include <linux/errno.h> 12 #include <linux/cdev.h> 13 #include <linux/fs.h> 14 #include <linux/device.h> 15 #include <linux/mm.h> 16 #include <linux/highmem.h> 17 #include <linux/pagemap.h> 18 #include <linux/bug.h> 19 #include <linux/completion.h> 20 #include <linux/list.h> 21 #include <linux/of.h> 22 #include <linux/platform_device.h> 23 #include <linux/compat.h> 24 #include <linux/dma-mapping.h> 25 #include <linux/rcupdate.h> 26 #include <linux/delay.h> 27 #include <linux/slab.h> 28 #include <linux/interrupt.h> 29 #include <linux/io.h> 30 #include <linux/uaccess.h> 31 #include <soc/bcm2835/raspberrypi-firmware.h> 32 33 #include "vchiq_core.h" 34 #include "vchiq_ioctl.h" 35 #include "vchiq_arm.h" 36 #include "vchiq_debugfs.h" 37 #include "vchiq_connected.h" 38 #include "vchiq_pagelist.h" 39 40 #define DEVICE_NAME "vchiq" 41 42 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32) 43 44 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2) 45 46 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0 47 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1 48 49 #define BELL0 0x00 50 #define BELL2 0x08 51 52 #define ARM_DS_ACTIVE BIT(2) 53 54 /* Override the default prefix, which would be vchiq_arm (from the filename) */ 55 #undef MODULE_PARAM_PREFIX 56 #define MODULE_PARAM_PREFIX DEVICE_NAME "." 57 58 #define KEEPALIVE_VER 1 59 #define KEEPALIVE_VER_MIN KEEPALIVE_VER 60 61 /* Run time control of log level, based on KERN_XXX level. */ 62 int vchiq_arm_log_level = VCHIQ_LOG_DEFAULT; 63 int vchiq_susp_log_level = VCHIQ_LOG_ERROR; 64 65 DEFINE_SPINLOCK(msg_queue_spinlock); 66 struct vchiq_state g_state; 67 68 static struct platform_device *bcm2835_camera; 69 static struct platform_device *bcm2835_audio; 70 71 struct vchiq_drvdata { 72 const unsigned int cache_line_size; 73 struct rpi_firmware *fw; 74 }; 75 76 static struct vchiq_drvdata bcm2835_drvdata = { 77 .cache_line_size = 32, 78 }; 79 80 static struct vchiq_drvdata bcm2836_drvdata = { 81 .cache_line_size = 64, 82 }; 83 84 struct vchiq_arm_state { 85 /* Keepalive-related data */ 86 struct task_struct *ka_thread; 87 struct completion ka_evt; 88 atomic_t ka_use_count; 89 atomic_t ka_use_ack_count; 90 atomic_t ka_release_count; 91 92 rwlock_t susp_res_lock; 93 94 struct vchiq_state *state; 95 96 /* 97 * Global use count for videocore. 98 * This is equal to the sum of the use counts for all services. When 99 * this hits zero the videocore suspend procedure will be initiated. 100 */ 101 int videocore_use_count; 102 103 /* 104 * Use count to track requests from videocore peer. 105 * This use count is not associated with a service, so needs to be 106 * tracked separately with the state. 107 */ 108 int peer_use_count; 109 110 /* 111 * Flag to indicate that the first vchiq connect has made it through. 112 * This means that both sides should be fully ready, and we should 113 * be able to suspend after this point. 114 */ 115 int first_connect; 116 }; 117 118 struct vchiq_2835_state { 119 int inited; 120 struct vchiq_arm_state arm_state; 121 }; 122 123 struct vchiq_pagelist_info { 124 struct pagelist *pagelist; 125 size_t pagelist_buffer_size; 126 dma_addr_t dma_addr; 127 enum dma_data_direction dma_dir; 128 unsigned int num_pages; 129 unsigned int pages_need_release; 130 struct page **pages; 131 struct scatterlist *scatterlist; 132 unsigned int scatterlist_mapped; 133 }; 134 135 static void __iomem *g_regs; 136 /* This value is the size of the L2 cache lines as understood by the 137 * VPU firmware, which determines the required alignment of the 138 * offsets/sizes in pagelists. 139 * 140 * Modern VPU firmware looks for a DT "cache-line-size" property in 141 * the VCHIQ node and will overwrite it with the actual L2 cache size, 142 * which the kernel must then respect. That property was rejected 143 * upstream, so we have to use the VPU firmware's compatibility value 144 * of 32. 145 */ 146 static unsigned int g_cache_line_size = 32; 147 static unsigned int g_fragments_size; 148 static char *g_fragments_base; 149 static char *g_free_fragments; 150 static struct semaphore g_free_fragments_sema; 151 152 static DEFINE_SEMAPHORE(g_free_fragments_mutex); 153 154 static enum vchiq_status 155 vchiq_blocking_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *data, 156 unsigned int size, enum vchiq_bulk_dir dir); 157 158 static irqreturn_t 159 vchiq_doorbell_irq(int irq, void *dev_id) 160 { 161 struct vchiq_state *state = dev_id; 162 irqreturn_t ret = IRQ_NONE; 163 unsigned int status; 164 165 /* Read (and clear) the doorbell */ 166 status = readl(g_regs + BELL0); 167 168 if (status & ARM_DS_ACTIVE) { /* Was the doorbell rung? */ 169 remote_event_pollall(state); 170 ret = IRQ_HANDLED; 171 } 172 173 return ret; 174 } 175 176 static void 177 cleanup_pagelistinfo(struct vchiq_instance *instance, struct vchiq_pagelist_info *pagelistinfo) 178 { 179 if (pagelistinfo->scatterlist_mapped) { 180 dma_unmap_sg(instance->state->dev, pagelistinfo->scatterlist, 181 pagelistinfo->num_pages, pagelistinfo->dma_dir); 182 } 183 184 if (pagelistinfo->pages_need_release) 185 unpin_user_pages(pagelistinfo->pages, pagelistinfo->num_pages); 186 187 dma_free_coherent(instance->state->dev, pagelistinfo->pagelist_buffer_size, 188 pagelistinfo->pagelist, pagelistinfo->dma_addr); 189 } 190 191 static inline bool 192 is_adjacent_block(u32 *addrs, u32 addr, unsigned int k) 193 { 194 u32 tmp; 195 196 if (!k) 197 return false; 198 199 tmp = (addrs[k - 1] & PAGE_MASK) + 200 (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT); 201 202 return tmp == (addr & PAGE_MASK); 203 } 204 205 /* There is a potential problem with partial cache lines (pages?) 206 * at the ends of the block when reading. If the CPU accessed anything in 207 * the same line (page?) then it may have pulled old data into the cache, 208 * obscuring the new data underneath. We can solve this by transferring the 209 * partial cache lines separately, and allowing the ARM to copy into the 210 * cached area. 211 */ 212 213 static struct vchiq_pagelist_info * 214 create_pagelist(struct vchiq_instance *instance, char *buf, char __user *ubuf, 215 size_t count, unsigned short type) 216 { 217 struct pagelist *pagelist; 218 struct vchiq_pagelist_info *pagelistinfo; 219 struct page **pages; 220 u32 *addrs; 221 unsigned int num_pages, offset, i, k; 222 int actual_pages; 223 size_t pagelist_size; 224 struct scatterlist *scatterlist, *sg; 225 int dma_buffers; 226 dma_addr_t dma_addr; 227 228 if (count >= INT_MAX - PAGE_SIZE) 229 return NULL; 230 231 if (buf) 232 offset = (uintptr_t)buf & (PAGE_SIZE - 1); 233 else 234 offset = (uintptr_t)ubuf & (PAGE_SIZE - 1); 235 num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE); 236 237 if ((size_t)num_pages > (SIZE_MAX - sizeof(struct pagelist) - 238 sizeof(struct vchiq_pagelist_info)) / 239 (sizeof(u32) + sizeof(pages[0]) + 240 sizeof(struct scatterlist))) 241 return NULL; 242 243 pagelist_size = sizeof(struct pagelist) + 244 (num_pages * sizeof(u32)) + 245 (num_pages * sizeof(pages[0]) + 246 (num_pages * sizeof(struct scatterlist))) + 247 sizeof(struct vchiq_pagelist_info); 248 249 /* Allocate enough storage to hold the page pointers and the page 250 * list 251 */ 252 pagelist = dma_alloc_coherent(instance->state->dev, pagelist_size, &dma_addr, 253 GFP_KERNEL); 254 255 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist); 256 257 if (!pagelist) 258 return NULL; 259 260 addrs = pagelist->addrs; 261 pages = (struct page **)(addrs + num_pages); 262 scatterlist = (struct scatterlist *)(pages + num_pages); 263 pagelistinfo = (struct vchiq_pagelist_info *) 264 (scatterlist + num_pages); 265 266 pagelist->length = count; 267 pagelist->type = type; 268 pagelist->offset = offset; 269 270 /* Populate the fields of the pagelistinfo structure */ 271 pagelistinfo->pagelist = pagelist; 272 pagelistinfo->pagelist_buffer_size = pagelist_size; 273 pagelistinfo->dma_addr = dma_addr; 274 pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ? 275 DMA_TO_DEVICE : DMA_FROM_DEVICE; 276 pagelistinfo->num_pages = num_pages; 277 pagelistinfo->pages_need_release = 0; 278 pagelistinfo->pages = pages; 279 pagelistinfo->scatterlist = scatterlist; 280 pagelistinfo->scatterlist_mapped = 0; 281 282 if (buf) { 283 unsigned long length = count; 284 unsigned int off = offset; 285 286 for (actual_pages = 0; actual_pages < num_pages; 287 actual_pages++) { 288 struct page *pg = 289 vmalloc_to_page((buf + 290 (actual_pages * PAGE_SIZE))); 291 size_t bytes = PAGE_SIZE - off; 292 293 if (!pg) { 294 cleanup_pagelistinfo(instance, pagelistinfo); 295 return NULL; 296 } 297 298 if (bytes > length) 299 bytes = length; 300 pages[actual_pages] = pg; 301 length -= bytes; 302 off = 0; 303 } 304 /* do not try and release vmalloc pages */ 305 } else { 306 actual_pages = pin_user_pages_fast((unsigned long)ubuf & PAGE_MASK, num_pages, 307 type == PAGELIST_READ, pages); 308 309 if (actual_pages != num_pages) { 310 vchiq_log_info(vchiq_arm_log_level, 311 "%s - only %d/%d pages locked", 312 __func__, actual_pages, num_pages); 313 314 /* This is probably due to the process being killed */ 315 if (actual_pages > 0) 316 unpin_user_pages(pages, actual_pages); 317 cleanup_pagelistinfo(instance, pagelistinfo); 318 return NULL; 319 } 320 /* release user pages */ 321 pagelistinfo->pages_need_release = 1; 322 } 323 324 /* 325 * Initialize the scatterlist so that the magic cookie 326 * is filled if debugging is enabled 327 */ 328 sg_init_table(scatterlist, num_pages); 329 /* Now set the pages for each scatterlist */ 330 for (i = 0; i < num_pages; i++) { 331 unsigned int len = PAGE_SIZE - offset; 332 333 if (len > count) 334 len = count; 335 sg_set_page(scatterlist + i, pages[i], len, offset); 336 offset = 0; 337 count -= len; 338 } 339 340 dma_buffers = dma_map_sg(instance->state->dev, 341 scatterlist, 342 num_pages, 343 pagelistinfo->dma_dir); 344 345 if (dma_buffers == 0) { 346 cleanup_pagelistinfo(instance, pagelistinfo); 347 return NULL; 348 } 349 350 pagelistinfo->scatterlist_mapped = 1; 351 352 /* Combine adjacent blocks for performance */ 353 k = 0; 354 for_each_sg(scatterlist, sg, dma_buffers, i) { 355 u32 len = sg_dma_len(sg); 356 u32 addr = sg_dma_address(sg); 357 358 /* Note: addrs is the address + page_count - 1 359 * The firmware expects blocks after the first to be page- 360 * aligned and a multiple of the page size 361 */ 362 WARN_ON(len == 0); 363 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK)); 364 WARN_ON(i && (addr & ~PAGE_MASK)); 365 if (is_adjacent_block(addrs, addr, k)) 366 addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT); 367 else 368 addrs[k++] = (addr & PAGE_MASK) | 369 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1); 370 } 371 372 /* Partial cache lines (fragments) require special measures */ 373 if ((type == PAGELIST_READ) && 374 ((pagelist->offset & (g_cache_line_size - 1)) || 375 ((pagelist->offset + pagelist->length) & 376 (g_cache_line_size - 1)))) { 377 char *fragments; 378 379 if (down_interruptible(&g_free_fragments_sema)) { 380 cleanup_pagelistinfo(instance, pagelistinfo); 381 return NULL; 382 } 383 384 WARN_ON(!g_free_fragments); 385 386 down(&g_free_fragments_mutex); 387 fragments = g_free_fragments; 388 WARN_ON(!fragments); 389 g_free_fragments = *(char **)g_free_fragments; 390 up(&g_free_fragments_mutex); 391 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS + 392 (fragments - g_fragments_base) / g_fragments_size; 393 } 394 395 return pagelistinfo; 396 } 397 398 static void 399 free_pagelist(struct vchiq_instance *instance, struct vchiq_pagelist_info *pagelistinfo, 400 int actual) 401 { 402 struct pagelist *pagelist = pagelistinfo->pagelist; 403 struct page **pages = pagelistinfo->pages; 404 unsigned int num_pages = pagelistinfo->num_pages; 405 406 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d", 407 __func__, pagelistinfo->pagelist, actual); 408 409 /* 410 * NOTE: dma_unmap_sg must be called before the 411 * cpu can touch any of the data/pages. 412 */ 413 dma_unmap_sg(instance->state->dev, pagelistinfo->scatterlist, 414 pagelistinfo->num_pages, pagelistinfo->dma_dir); 415 pagelistinfo->scatterlist_mapped = 0; 416 417 /* Deal with any partial cache lines (fragments) */ 418 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) { 419 char *fragments = g_fragments_base + 420 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) * 421 g_fragments_size; 422 int head_bytes, tail_bytes; 423 424 head_bytes = (g_cache_line_size - pagelist->offset) & 425 (g_cache_line_size - 1); 426 tail_bytes = (pagelist->offset + actual) & 427 (g_cache_line_size - 1); 428 429 if ((actual >= 0) && (head_bytes != 0)) { 430 if (head_bytes > actual) 431 head_bytes = actual; 432 433 memcpy_to_page(pages[0], 434 pagelist->offset, 435 fragments, 436 head_bytes); 437 } 438 if ((actual >= 0) && (head_bytes < actual) && 439 (tail_bytes != 0)) 440 memcpy_to_page(pages[num_pages - 1], 441 (pagelist->offset + actual) & 442 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1), 443 fragments + g_cache_line_size, 444 tail_bytes); 445 446 down(&g_free_fragments_mutex); 447 *(char **)fragments = g_free_fragments; 448 g_free_fragments = fragments; 449 up(&g_free_fragments_mutex); 450 up(&g_free_fragments_sema); 451 } 452 453 /* Need to mark all the pages dirty. */ 454 if (pagelist->type != PAGELIST_WRITE && 455 pagelistinfo->pages_need_release) { 456 unsigned int i; 457 458 for (i = 0; i < num_pages; i++) 459 set_page_dirty(pages[i]); 460 } 461 462 cleanup_pagelistinfo(instance, pagelistinfo); 463 } 464 465 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state) 466 { 467 struct device *dev = &pdev->dev; 468 struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev); 469 struct rpi_firmware *fw = drvdata->fw; 470 struct vchiq_slot_zero *vchiq_slot_zero; 471 void *slot_mem; 472 dma_addr_t slot_phys; 473 u32 channelbase; 474 int slot_mem_size, frag_mem_size; 475 int err, irq, i; 476 477 /* 478 * VCHI messages between the CPU and firmware use 479 * 32-bit bus addresses. 480 */ 481 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 482 483 if (err < 0) 484 return err; 485 486 g_cache_line_size = drvdata->cache_line_size; 487 g_fragments_size = 2 * g_cache_line_size; 488 489 /* Allocate space for the channels in coherent memory */ 490 slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE); 491 frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS); 492 493 slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size, 494 &slot_phys, GFP_KERNEL); 495 if (!slot_mem) { 496 dev_err(dev, "could not allocate DMA memory\n"); 497 return -ENOMEM; 498 } 499 500 WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0); 501 502 vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size); 503 if (!vchiq_slot_zero) 504 return -EINVAL; 505 506 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] = 507 (int)slot_phys + slot_mem_size; 508 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] = 509 MAX_FRAGMENTS; 510 511 g_fragments_base = (char *)slot_mem + slot_mem_size; 512 513 g_free_fragments = g_fragments_base; 514 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) { 515 *(char **)&g_fragments_base[i * g_fragments_size] = 516 &g_fragments_base[(i + 1) * g_fragments_size]; 517 } 518 *(char **)&g_fragments_base[i * g_fragments_size] = NULL; 519 sema_init(&g_free_fragments_sema, MAX_FRAGMENTS); 520 521 err = vchiq_init_state(state, vchiq_slot_zero, dev); 522 if (err) 523 return err; 524 525 g_regs = devm_platform_ioremap_resource(pdev, 0); 526 if (IS_ERR(g_regs)) 527 return PTR_ERR(g_regs); 528 529 irq = platform_get_irq(pdev, 0); 530 if (irq <= 0) 531 return irq; 532 533 err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL, 534 "VCHIQ doorbell", state); 535 if (err) { 536 dev_err(dev, "failed to register irq=%d\n", irq); 537 return err; 538 } 539 540 /* Send the base address of the slots to VideoCore */ 541 channelbase = slot_phys; 542 err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT, 543 &channelbase, sizeof(channelbase)); 544 if (err || channelbase) { 545 dev_err(dev, "failed to set channelbase\n"); 546 return err ? : -ENXIO; 547 } 548 549 vchiq_log_info(vchiq_arm_log_level, "vchiq_init - done (slots %pK, phys %pad)", 550 vchiq_slot_zero, &slot_phys); 551 552 vchiq_call_connected_callbacks(); 553 554 return 0; 555 } 556 557 static void 558 vchiq_arm_init_state(struct vchiq_state *state, 559 struct vchiq_arm_state *arm_state) 560 { 561 if (arm_state) { 562 rwlock_init(&arm_state->susp_res_lock); 563 564 init_completion(&arm_state->ka_evt); 565 atomic_set(&arm_state->ka_use_count, 0); 566 atomic_set(&arm_state->ka_use_ack_count, 0); 567 atomic_set(&arm_state->ka_release_count, 0); 568 569 arm_state->state = state; 570 arm_state->first_connect = 0; 571 } 572 } 573 574 int 575 vchiq_platform_init_state(struct vchiq_state *state) 576 { 577 struct vchiq_2835_state *platform_state; 578 579 state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL); 580 if (!state->platform_state) 581 return -ENOMEM; 582 583 platform_state = (struct vchiq_2835_state *)state->platform_state; 584 585 platform_state->inited = 1; 586 vchiq_arm_init_state(state, &platform_state->arm_state); 587 588 return 0; 589 } 590 591 static struct vchiq_arm_state *vchiq_platform_get_arm_state(struct vchiq_state *state) 592 { 593 struct vchiq_2835_state *platform_state; 594 595 platform_state = (struct vchiq_2835_state *)state->platform_state; 596 597 WARN_ON_ONCE(!platform_state->inited); 598 599 return &platform_state->arm_state; 600 } 601 602 void 603 remote_event_signal(struct remote_event *event) 604 { 605 /* 606 * Ensure that all writes to shared data structures have completed 607 * before signalling the peer. 608 */ 609 wmb(); 610 611 event->fired = 1; 612 613 dsb(sy); /* data barrier operation */ 614 615 if (event->armed) 616 writel(0, g_regs + BELL2); /* trigger vc interrupt */ 617 } 618 619 int 620 vchiq_prepare_bulk_data(struct vchiq_instance *instance, struct vchiq_bulk *bulk, void *offset, 621 void __user *uoffset, int size, int dir) 622 { 623 struct vchiq_pagelist_info *pagelistinfo; 624 625 pagelistinfo = create_pagelist(instance, offset, uoffset, size, 626 (dir == VCHIQ_BULK_RECEIVE) 627 ? PAGELIST_READ 628 : PAGELIST_WRITE); 629 630 if (!pagelistinfo) 631 return -ENOMEM; 632 633 bulk->data = pagelistinfo->dma_addr; 634 635 /* 636 * Store the pagelistinfo address in remote_data, 637 * which isn't used by the slave. 638 */ 639 bulk->remote_data = pagelistinfo; 640 641 return 0; 642 } 643 644 void 645 vchiq_complete_bulk(struct vchiq_instance *instance, struct vchiq_bulk *bulk) 646 { 647 if (bulk && bulk->remote_data && bulk->actual) 648 free_pagelist(instance, (struct vchiq_pagelist_info *)bulk->remote_data, 649 bulk->actual); 650 } 651 652 int vchiq_dump_platform_state(void *dump_context) 653 { 654 char buf[80]; 655 int len; 656 657 len = snprintf(buf, sizeof(buf), " Platform: 2835 (VC master)"); 658 return vchiq_dump(dump_context, buf, len + 1); 659 } 660 661 #define VCHIQ_INIT_RETRIES 10 662 int vchiq_initialise(struct vchiq_instance **instance_out) 663 { 664 struct vchiq_state *state; 665 struct vchiq_instance *instance = NULL; 666 int i, ret; 667 668 /* 669 * VideoCore may not be ready due to boot up timing. 670 * It may never be ready if kernel and firmware are mismatched,so don't 671 * block forever. 672 */ 673 for (i = 0; i < VCHIQ_INIT_RETRIES; i++) { 674 state = vchiq_get_state(); 675 if (state) 676 break; 677 usleep_range(500, 600); 678 } 679 if (i == VCHIQ_INIT_RETRIES) { 680 vchiq_log_error(vchiq_core_log_level, "%s: videocore not initialized\n", __func__); 681 ret = -ENOTCONN; 682 goto failed; 683 } else if (i > 0) { 684 vchiq_log_warning(vchiq_core_log_level, 685 "%s: videocore initialized after %d retries\n", __func__, i); 686 } 687 688 instance = kzalloc(sizeof(*instance), GFP_KERNEL); 689 if (!instance) { 690 vchiq_log_error(vchiq_core_log_level, 691 "%s: error allocating vchiq instance\n", __func__); 692 ret = -ENOMEM; 693 goto failed; 694 } 695 696 instance->connected = 0; 697 instance->state = state; 698 mutex_init(&instance->bulk_waiter_list_mutex); 699 INIT_LIST_HEAD(&instance->bulk_waiter_list); 700 701 *instance_out = instance; 702 703 ret = 0; 704 705 failed: 706 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, ret); 707 708 return ret; 709 } 710 EXPORT_SYMBOL(vchiq_initialise); 711 712 void free_bulk_waiter(struct vchiq_instance *instance) 713 { 714 struct bulk_waiter_node *waiter, *next; 715 716 list_for_each_entry_safe(waiter, next, 717 &instance->bulk_waiter_list, list) { 718 list_del(&waiter->list); 719 vchiq_log_info(vchiq_arm_log_level, "bulk_waiter - cleaned up %pK for pid %d", 720 waiter, waiter->pid); 721 kfree(waiter); 722 } 723 } 724 725 enum vchiq_status vchiq_shutdown(struct vchiq_instance *instance) 726 { 727 enum vchiq_status status = VCHIQ_SUCCESS; 728 struct vchiq_state *state = instance->state; 729 730 if (mutex_lock_killable(&state->mutex)) 731 return VCHIQ_RETRY; 732 733 /* Remove all services */ 734 vchiq_shutdown_internal(state, instance); 735 736 mutex_unlock(&state->mutex); 737 738 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status); 739 740 free_bulk_waiter(instance); 741 kfree(instance); 742 743 return status; 744 } 745 EXPORT_SYMBOL(vchiq_shutdown); 746 747 static int vchiq_is_connected(struct vchiq_instance *instance) 748 { 749 return instance->connected; 750 } 751 752 enum vchiq_status vchiq_connect(struct vchiq_instance *instance) 753 { 754 enum vchiq_status status; 755 struct vchiq_state *state = instance->state; 756 757 if (mutex_lock_killable(&state->mutex)) { 758 vchiq_log_trace(vchiq_core_log_level, "%s: call to mutex_lock failed", __func__); 759 status = VCHIQ_RETRY; 760 goto failed; 761 } 762 status = vchiq_connect_internal(state, instance); 763 764 if (status == VCHIQ_SUCCESS) 765 instance->connected = 1; 766 767 mutex_unlock(&state->mutex); 768 769 failed: 770 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status); 771 772 return status; 773 } 774 EXPORT_SYMBOL(vchiq_connect); 775 776 static enum vchiq_status 777 vchiq_add_service(struct vchiq_instance *instance, 778 const struct vchiq_service_params_kernel *params, 779 unsigned int *phandle) 780 { 781 enum vchiq_status status; 782 struct vchiq_state *state = instance->state; 783 struct vchiq_service *service = NULL; 784 int srvstate; 785 786 *phandle = VCHIQ_SERVICE_HANDLE_INVALID; 787 788 srvstate = vchiq_is_connected(instance) 789 ? VCHIQ_SRVSTATE_LISTENING 790 : VCHIQ_SRVSTATE_HIDDEN; 791 792 service = vchiq_add_service_internal(state, params, srvstate, instance, NULL); 793 794 if (service) { 795 *phandle = service->handle; 796 status = VCHIQ_SUCCESS; 797 } else { 798 status = VCHIQ_ERROR; 799 } 800 801 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status); 802 803 return status; 804 } 805 806 enum vchiq_status 807 vchiq_open_service(struct vchiq_instance *instance, 808 const struct vchiq_service_params_kernel *params, 809 unsigned int *phandle) 810 { 811 enum vchiq_status status = VCHIQ_ERROR; 812 struct vchiq_state *state = instance->state; 813 struct vchiq_service *service = NULL; 814 815 *phandle = VCHIQ_SERVICE_HANDLE_INVALID; 816 817 if (!vchiq_is_connected(instance)) 818 goto failed; 819 820 service = vchiq_add_service_internal(state, params, VCHIQ_SRVSTATE_OPENING, instance, NULL); 821 822 if (service) { 823 *phandle = service->handle; 824 status = vchiq_open_service_internal(service, current->pid); 825 if (status != VCHIQ_SUCCESS) { 826 vchiq_remove_service(instance, service->handle); 827 *phandle = VCHIQ_SERVICE_HANDLE_INVALID; 828 } 829 } 830 831 failed: 832 vchiq_log_trace(vchiq_core_log_level, "%s(%p): returning %d", __func__, instance, status); 833 834 return status; 835 } 836 EXPORT_SYMBOL(vchiq_open_service); 837 838 enum vchiq_status 839 vchiq_bulk_transmit(struct vchiq_instance *instance, unsigned int handle, const void *data, 840 unsigned int size, void *userdata, enum vchiq_bulk_mode mode) 841 { 842 enum vchiq_status status; 843 844 while (1) { 845 switch (mode) { 846 case VCHIQ_BULK_MODE_NOCALLBACK: 847 case VCHIQ_BULK_MODE_CALLBACK: 848 status = vchiq_bulk_transfer(instance, handle, 849 (void *)data, NULL, 850 size, userdata, mode, 851 VCHIQ_BULK_TRANSMIT); 852 break; 853 case VCHIQ_BULK_MODE_BLOCKING: 854 status = vchiq_blocking_bulk_transfer(instance, handle, (void *)data, size, 855 VCHIQ_BULK_TRANSMIT); 856 break; 857 default: 858 return VCHIQ_ERROR; 859 } 860 861 /* 862 * vchiq_*_bulk_transfer() may return VCHIQ_RETRY, so we need 863 * to implement a retry mechanism since this function is 864 * supposed to block until queued 865 */ 866 if (status != VCHIQ_RETRY) 867 break; 868 869 msleep(1); 870 } 871 872 return status; 873 } 874 EXPORT_SYMBOL(vchiq_bulk_transmit); 875 876 enum vchiq_status vchiq_bulk_receive(struct vchiq_instance *instance, unsigned int handle, 877 void *data, unsigned int size, void *userdata, 878 enum vchiq_bulk_mode mode) 879 { 880 enum vchiq_status status; 881 882 while (1) { 883 switch (mode) { 884 case VCHIQ_BULK_MODE_NOCALLBACK: 885 case VCHIQ_BULK_MODE_CALLBACK: 886 status = vchiq_bulk_transfer(instance, handle, data, NULL, 887 size, userdata, 888 mode, VCHIQ_BULK_RECEIVE); 889 break; 890 case VCHIQ_BULK_MODE_BLOCKING: 891 status = vchiq_blocking_bulk_transfer(instance, handle, (void *)data, size, 892 VCHIQ_BULK_RECEIVE); 893 break; 894 default: 895 return VCHIQ_ERROR; 896 } 897 898 /* 899 * vchiq_*_bulk_transfer() may return VCHIQ_RETRY, so we need 900 * to implement a retry mechanism since this function is 901 * supposed to block until queued 902 */ 903 if (status != VCHIQ_RETRY) 904 break; 905 906 msleep(1); 907 } 908 909 return status; 910 } 911 EXPORT_SYMBOL(vchiq_bulk_receive); 912 913 static enum vchiq_status 914 vchiq_blocking_bulk_transfer(struct vchiq_instance *instance, unsigned int handle, void *data, 915 unsigned int size, enum vchiq_bulk_dir dir) 916 { 917 struct vchiq_service *service; 918 enum vchiq_status status; 919 struct bulk_waiter_node *waiter = NULL, *iter; 920 921 service = find_service_by_handle(instance, handle); 922 if (!service) 923 return VCHIQ_ERROR; 924 925 vchiq_service_put(service); 926 927 mutex_lock(&instance->bulk_waiter_list_mutex); 928 list_for_each_entry(iter, &instance->bulk_waiter_list, list) { 929 if (iter->pid == current->pid) { 930 list_del(&iter->list); 931 waiter = iter; 932 break; 933 } 934 } 935 mutex_unlock(&instance->bulk_waiter_list_mutex); 936 937 if (waiter) { 938 struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk; 939 940 if (bulk) { 941 /* This thread has an outstanding bulk transfer. */ 942 /* FIXME: why compare a dma address to a pointer? */ 943 if ((bulk->data != (dma_addr_t)(uintptr_t)data) || (bulk->size != size)) { 944 /* 945 * This is not a retry of the previous one. 946 * Cancel the signal when the transfer completes. 947 */ 948 spin_lock(&bulk_waiter_spinlock); 949 bulk->userdata = NULL; 950 spin_unlock(&bulk_waiter_spinlock); 951 } 952 } 953 } else { 954 waiter = kzalloc(sizeof(*waiter), GFP_KERNEL); 955 if (!waiter) { 956 vchiq_log_error(vchiq_core_log_level, "%s - out of memory", __func__); 957 return VCHIQ_ERROR; 958 } 959 } 960 961 status = vchiq_bulk_transfer(instance, handle, data, NULL, size, 962 &waiter->bulk_waiter, 963 VCHIQ_BULK_MODE_BLOCKING, dir); 964 if ((status != VCHIQ_RETRY) || fatal_signal_pending(current) || !waiter->bulk_waiter.bulk) { 965 struct vchiq_bulk *bulk = waiter->bulk_waiter.bulk; 966 967 if (bulk) { 968 /* Cancel the signal when the transfer completes. */ 969 spin_lock(&bulk_waiter_spinlock); 970 bulk->userdata = NULL; 971 spin_unlock(&bulk_waiter_spinlock); 972 } 973 kfree(waiter); 974 } else { 975 waiter->pid = current->pid; 976 mutex_lock(&instance->bulk_waiter_list_mutex); 977 list_add(&waiter->list, &instance->bulk_waiter_list); 978 mutex_unlock(&instance->bulk_waiter_list_mutex); 979 vchiq_log_info(vchiq_arm_log_level, "saved bulk_waiter %pK for pid %d", waiter, 980 current->pid); 981 } 982 983 return status; 984 } 985 986 static enum vchiq_status 987 add_completion(struct vchiq_instance *instance, enum vchiq_reason reason, 988 struct vchiq_header *header, struct user_service *user_service, 989 void *bulk_userdata) 990 { 991 struct vchiq_completion_data_kernel *completion; 992 int insert; 993 994 DEBUG_INITIALISE(g_state.local); 995 996 insert = instance->completion_insert; 997 while ((insert - instance->completion_remove) >= MAX_COMPLETIONS) { 998 /* Out of space - wait for the client */ 999 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1000 vchiq_log_trace(vchiq_arm_log_level, "%s - completion queue full", __func__); 1001 DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT); 1002 if (wait_for_completion_interruptible(&instance->remove_event)) { 1003 vchiq_log_info(vchiq_arm_log_level, "service_callback interrupted"); 1004 return VCHIQ_RETRY; 1005 } else if (instance->closing) { 1006 vchiq_log_info(vchiq_arm_log_level, "service_callback closing"); 1007 return VCHIQ_SUCCESS; 1008 } 1009 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1010 } 1011 1012 completion = &instance->completions[insert & (MAX_COMPLETIONS - 1)]; 1013 1014 completion->header = header; 1015 completion->reason = reason; 1016 /* N.B. service_userdata is updated while processing AWAIT_COMPLETION */ 1017 completion->service_userdata = user_service->service; 1018 completion->bulk_userdata = bulk_userdata; 1019 1020 if (reason == VCHIQ_SERVICE_CLOSED) { 1021 /* 1022 * Take an extra reference, to be held until 1023 * this CLOSED notification is delivered. 1024 */ 1025 vchiq_service_get(user_service->service); 1026 if (instance->use_close_delivered) 1027 user_service->close_pending = 1; 1028 } 1029 1030 /* 1031 * A write barrier is needed here to ensure that the entire completion 1032 * record is written out before the insert point. 1033 */ 1034 wmb(); 1035 1036 if (reason == VCHIQ_MESSAGE_AVAILABLE) 1037 user_service->message_available_pos = insert; 1038 1039 insert++; 1040 instance->completion_insert = insert; 1041 1042 complete(&instance->insert_event); 1043 1044 return VCHIQ_SUCCESS; 1045 } 1046 1047 enum vchiq_status 1048 service_callback(struct vchiq_instance *instance, enum vchiq_reason reason, 1049 struct vchiq_header *header, unsigned int handle, void *bulk_userdata) 1050 { 1051 /* 1052 * How do we ensure the callback goes to the right client? 1053 * The service_user data points to a user_service record 1054 * containing the original callback and the user state structure, which 1055 * contains a circular buffer for completion records. 1056 */ 1057 struct user_service *user_service; 1058 struct vchiq_service *service; 1059 bool skip_completion = false; 1060 1061 DEBUG_INITIALISE(g_state.local); 1062 1063 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1064 1065 rcu_read_lock(); 1066 service = handle_to_service(instance, handle); 1067 if (WARN_ON(!service)) { 1068 rcu_read_unlock(); 1069 return VCHIQ_SUCCESS; 1070 } 1071 1072 user_service = (struct user_service *)service->base.userdata; 1073 1074 if (!instance || instance->closing) { 1075 rcu_read_unlock(); 1076 return VCHIQ_SUCCESS; 1077 } 1078 1079 /* 1080 * As hopping around different synchronization mechanism, 1081 * taking an extra reference results in simpler implementation. 1082 */ 1083 vchiq_service_get(service); 1084 rcu_read_unlock(); 1085 1086 vchiq_log_trace(vchiq_arm_log_level, 1087 "%s - service %lx(%d,%p), reason %d, header %lx, instance %lx, bulk_userdata %lx", 1088 __func__, (unsigned long)user_service, service->localport, 1089 user_service->userdata, reason, (unsigned long)header, 1090 (unsigned long)instance, (unsigned long)bulk_userdata); 1091 1092 if (header && user_service->is_vchi) { 1093 spin_lock(&msg_queue_spinlock); 1094 while (user_service->msg_insert == 1095 (user_service->msg_remove + MSG_QUEUE_SIZE)) { 1096 spin_unlock(&msg_queue_spinlock); 1097 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1098 DEBUG_COUNT(MSG_QUEUE_FULL_COUNT); 1099 vchiq_log_trace(vchiq_arm_log_level, "%s - msg queue full", __func__); 1100 /* 1101 * If there is no MESSAGE_AVAILABLE in the completion 1102 * queue, add one 1103 */ 1104 if ((user_service->message_available_pos - 1105 instance->completion_remove) < 0) { 1106 enum vchiq_status status; 1107 1108 vchiq_log_info(vchiq_arm_log_level, 1109 "Inserting extra MESSAGE_AVAILABLE"); 1110 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1111 status = add_completion(instance, reason, NULL, user_service, 1112 bulk_userdata); 1113 if (status != VCHIQ_SUCCESS) { 1114 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1115 vchiq_service_put(service); 1116 return status; 1117 } 1118 } 1119 1120 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1121 if (wait_for_completion_interruptible(&user_service->remove_event)) { 1122 vchiq_log_info(vchiq_arm_log_level, "%s interrupted", __func__); 1123 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1124 vchiq_service_put(service); 1125 return VCHIQ_RETRY; 1126 } else if (instance->closing) { 1127 vchiq_log_info(vchiq_arm_log_level, "%s closing", __func__); 1128 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1129 vchiq_service_put(service); 1130 return VCHIQ_ERROR; 1131 } 1132 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1133 spin_lock(&msg_queue_spinlock); 1134 } 1135 1136 user_service->msg_queue[user_service->msg_insert & 1137 (MSG_QUEUE_SIZE - 1)] = header; 1138 user_service->msg_insert++; 1139 1140 /* 1141 * If there is a thread waiting in DEQUEUE_MESSAGE, or if 1142 * there is a MESSAGE_AVAILABLE in the completion queue then 1143 * bypass the completion queue. 1144 */ 1145 if (((user_service->message_available_pos - 1146 instance->completion_remove) >= 0) || 1147 user_service->dequeue_pending) { 1148 user_service->dequeue_pending = 0; 1149 skip_completion = true; 1150 } 1151 1152 spin_unlock(&msg_queue_spinlock); 1153 complete(&user_service->insert_event); 1154 1155 header = NULL; 1156 } 1157 DEBUG_TRACE(SERVICE_CALLBACK_LINE); 1158 vchiq_service_put(service); 1159 1160 if (skip_completion) 1161 return VCHIQ_SUCCESS; 1162 1163 return add_completion(instance, reason, header, user_service, 1164 bulk_userdata); 1165 } 1166 1167 int vchiq_dump(void *dump_context, const char *str, int len) 1168 { 1169 struct dump_context *context = (struct dump_context *)dump_context; 1170 int copy_bytes; 1171 1172 if (context->actual >= context->space) 1173 return 0; 1174 1175 if (context->offset > 0) { 1176 int skip_bytes = min_t(int, len, context->offset); 1177 1178 str += skip_bytes; 1179 len -= skip_bytes; 1180 context->offset -= skip_bytes; 1181 if (context->offset > 0) 1182 return 0; 1183 } 1184 copy_bytes = min_t(int, len, context->space - context->actual); 1185 if (copy_bytes == 0) 1186 return 0; 1187 if (copy_to_user(context->buf + context->actual, str, 1188 copy_bytes)) 1189 return -EFAULT; 1190 context->actual += copy_bytes; 1191 len -= copy_bytes; 1192 1193 /* 1194 * If the terminating NUL is included in the length, then it 1195 * marks the end of a line and should be replaced with a 1196 * carriage return. 1197 */ 1198 if ((len == 0) && (str[copy_bytes - 1] == '\0')) { 1199 char cr = '\n'; 1200 1201 if (copy_to_user(context->buf + context->actual - 1, 1202 &cr, 1)) 1203 return -EFAULT; 1204 } 1205 return 0; 1206 } 1207 1208 int vchiq_dump_platform_instances(void *dump_context) 1209 { 1210 struct vchiq_state *state = vchiq_get_state(); 1211 char buf[80]; 1212 int len; 1213 int i; 1214 1215 if (!state) 1216 return -ENOTCONN; 1217 1218 /* 1219 * There is no list of instances, so instead scan all services, 1220 * marking those that have been dumped. 1221 */ 1222 1223 rcu_read_lock(); 1224 for (i = 0; i < state->unused_service; i++) { 1225 struct vchiq_service *service; 1226 struct vchiq_instance *instance; 1227 1228 service = rcu_dereference(state->services[i]); 1229 if (!service || service->base.callback != service_callback) 1230 continue; 1231 1232 instance = service->instance; 1233 if (instance) 1234 instance->mark = 0; 1235 } 1236 rcu_read_unlock(); 1237 1238 for (i = 0; i < state->unused_service; i++) { 1239 struct vchiq_service *service; 1240 struct vchiq_instance *instance; 1241 int err; 1242 1243 rcu_read_lock(); 1244 service = rcu_dereference(state->services[i]); 1245 if (!service || service->base.callback != service_callback) { 1246 rcu_read_unlock(); 1247 continue; 1248 } 1249 1250 instance = service->instance; 1251 if (!instance || instance->mark) { 1252 rcu_read_unlock(); 1253 continue; 1254 } 1255 rcu_read_unlock(); 1256 1257 len = snprintf(buf, sizeof(buf), 1258 "Instance %pK: pid %d,%s completions %d/%d", 1259 instance, instance->pid, 1260 instance->connected ? " connected, " : 1261 "", 1262 instance->completion_insert - 1263 instance->completion_remove, 1264 MAX_COMPLETIONS); 1265 err = vchiq_dump(dump_context, buf, len + 1); 1266 if (err) 1267 return err; 1268 instance->mark = 1; 1269 } 1270 return 0; 1271 } 1272 1273 int vchiq_dump_platform_service_state(void *dump_context, 1274 struct vchiq_service *service) 1275 { 1276 struct user_service *user_service = 1277 (struct user_service *)service->base.userdata; 1278 char buf[80]; 1279 int len; 1280 1281 len = scnprintf(buf, sizeof(buf), " instance %pK", service->instance); 1282 1283 if ((service->base.callback == service_callback) && user_service->is_vchi) { 1284 len += scnprintf(buf + len, sizeof(buf) - len, ", %d/%d messages", 1285 user_service->msg_insert - user_service->msg_remove, 1286 MSG_QUEUE_SIZE); 1287 1288 if (user_service->dequeue_pending) 1289 len += scnprintf(buf + len, sizeof(buf) - len, 1290 " (dequeue pending)"); 1291 } 1292 1293 return vchiq_dump(dump_context, buf, len + 1); 1294 } 1295 1296 struct vchiq_state * 1297 vchiq_get_state(void) 1298 { 1299 if (!g_state.remote) { 1300 pr_err("%s: g_state.remote == NULL\n", __func__); 1301 return NULL; 1302 } 1303 1304 if (g_state.remote->initialised != 1) { 1305 pr_notice("%s: g_state.remote->initialised != 1 (%d)\n", 1306 __func__, g_state.remote->initialised); 1307 return NULL; 1308 } 1309 1310 return &g_state; 1311 } 1312 1313 /* 1314 * Autosuspend related functionality 1315 */ 1316 1317 static enum vchiq_status 1318 vchiq_keepalive_vchiq_callback(struct vchiq_instance *instance, 1319 enum vchiq_reason reason, 1320 struct vchiq_header *header, 1321 unsigned int service_user, void *bulk_user) 1322 { 1323 vchiq_log_error(vchiq_susp_log_level, "%s callback reason %d", __func__, reason); 1324 return 0; 1325 } 1326 1327 static int 1328 vchiq_keepalive_thread_func(void *v) 1329 { 1330 struct vchiq_state *state = (struct vchiq_state *)v; 1331 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1332 1333 enum vchiq_status status; 1334 struct vchiq_instance *instance; 1335 unsigned int ka_handle; 1336 int ret; 1337 1338 struct vchiq_service_params_kernel params = { 1339 .fourcc = VCHIQ_MAKE_FOURCC('K', 'E', 'E', 'P'), 1340 .callback = vchiq_keepalive_vchiq_callback, 1341 .version = KEEPALIVE_VER, 1342 .version_min = KEEPALIVE_VER_MIN 1343 }; 1344 1345 ret = vchiq_initialise(&instance); 1346 if (ret) { 1347 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_initialise failed %d", __func__, 1348 ret); 1349 goto exit; 1350 } 1351 1352 status = vchiq_connect(instance); 1353 if (status != VCHIQ_SUCCESS) { 1354 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_connect failed %d", __func__, 1355 status); 1356 goto shutdown; 1357 } 1358 1359 status = vchiq_add_service(instance, ¶ms, &ka_handle); 1360 if (status != VCHIQ_SUCCESS) { 1361 vchiq_log_error(vchiq_susp_log_level, "%s vchiq_open_service failed %d", __func__, 1362 status); 1363 goto shutdown; 1364 } 1365 1366 while (1) { 1367 long rc = 0, uc = 0; 1368 1369 if (wait_for_completion_interruptible(&arm_state->ka_evt)) { 1370 vchiq_log_error(vchiq_susp_log_level, "%s interrupted", __func__); 1371 flush_signals(current); 1372 continue; 1373 } 1374 1375 /* 1376 * read and clear counters. Do release_count then use_count to 1377 * prevent getting more releases than uses 1378 */ 1379 rc = atomic_xchg(&arm_state->ka_release_count, 0); 1380 uc = atomic_xchg(&arm_state->ka_use_count, 0); 1381 1382 /* 1383 * Call use/release service the requisite number of times. 1384 * Process use before release so use counts don't go negative 1385 */ 1386 while (uc--) { 1387 atomic_inc(&arm_state->ka_use_ack_count); 1388 status = vchiq_use_service(instance, ka_handle); 1389 if (status != VCHIQ_SUCCESS) { 1390 vchiq_log_error(vchiq_susp_log_level, 1391 "%s vchiq_use_service error %d", __func__, status); 1392 } 1393 } 1394 while (rc--) { 1395 status = vchiq_release_service(instance, ka_handle); 1396 if (status != VCHIQ_SUCCESS) { 1397 vchiq_log_error(vchiq_susp_log_level, 1398 "%s vchiq_release_service error %d", __func__, 1399 status); 1400 } 1401 } 1402 } 1403 1404 shutdown: 1405 vchiq_shutdown(instance); 1406 exit: 1407 return 0; 1408 } 1409 1410 int 1411 vchiq_use_internal(struct vchiq_state *state, struct vchiq_service *service, 1412 enum USE_TYPE_E use_type) 1413 { 1414 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1415 int ret = 0; 1416 char entity[16]; 1417 int *entity_uc; 1418 int local_uc; 1419 1420 if (!arm_state) { 1421 ret = -EINVAL; 1422 goto out; 1423 } 1424 1425 if (use_type == USE_TYPE_VCHIQ) { 1426 sprintf(entity, "VCHIQ: "); 1427 entity_uc = &arm_state->peer_use_count; 1428 } else if (service) { 1429 sprintf(entity, "%c%c%c%c:%03d", 1430 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), 1431 service->client_id); 1432 entity_uc = &service->service_use_count; 1433 } else { 1434 vchiq_log_error(vchiq_susp_log_level, "%s null service ptr", __func__); 1435 ret = -EINVAL; 1436 goto out; 1437 } 1438 1439 write_lock_bh(&arm_state->susp_res_lock); 1440 local_uc = ++arm_state->videocore_use_count; 1441 ++(*entity_uc); 1442 1443 vchiq_log_trace(vchiq_susp_log_level, "%s %s count %d, state count %d", __func__, entity, 1444 *entity_uc, local_uc); 1445 1446 write_unlock_bh(&arm_state->susp_res_lock); 1447 1448 if (!ret) { 1449 enum vchiq_status status = VCHIQ_SUCCESS; 1450 long ack_cnt = atomic_xchg(&arm_state->ka_use_ack_count, 0); 1451 1452 while (ack_cnt && (status == VCHIQ_SUCCESS)) { 1453 /* Send the use notify to videocore */ 1454 status = vchiq_send_remote_use_active(state); 1455 if (status == VCHIQ_SUCCESS) 1456 ack_cnt--; 1457 else 1458 atomic_add(ack_cnt, &arm_state->ka_use_ack_count); 1459 } 1460 } 1461 1462 out: 1463 vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret); 1464 return ret; 1465 } 1466 1467 int 1468 vchiq_release_internal(struct vchiq_state *state, struct vchiq_service *service) 1469 { 1470 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1471 int ret = 0; 1472 char entity[16]; 1473 int *entity_uc; 1474 1475 if (!arm_state) { 1476 ret = -EINVAL; 1477 goto out; 1478 } 1479 1480 if (service) { 1481 sprintf(entity, "%c%c%c%c:%03d", 1482 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), 1483 service->client_id); 1484 entity_uc = &service->service_use_count; 1485 } else { 1486 sprintf(entity, "PEER: "); 1487 entity_uc = &arm_state->peer_use_count; 1488 } 1489 1490 write_lock_bh(&arm_state->susp_res_lock); 1491 if (!arm_state->videocore_use_count || !(*entity_uc)) { 1492 /* Don't use BUG_ON - don't allow user thread to crash kernel */ 1493 WARN_ON(!arm_state->videocore_use_count); 1494 WARN_ON(!(*entity_uc)); 1495 ret = -EINVAL; 1496 goto unlock; 1497 } 1498 --arm_state->videocore_use_count; 1499 --(*entity_uc); 1500 1501 vchiq_log_trace(vchiq_susp_log_level, "%s %s count %d, state count %d", __func__, entity, 1502 *entity_uc, arm_state->videocore_use_count); 1503 1504 unlock: 1505 write_unlock_bh(&arm_state->susp_res_lock); 1506 1507 out: 1508 vchiq_log_trace(vchiq_susp_log_level, "%s exit %d", __func__, ret); 1509 return ret; 1510 } 1511 1512 void 1513 vchiq_on_remote_use(struct vchiq_state *state) 1514 { 1515 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1516 1517 atomic_inc(&arm_state->ka_use_count); 1518 complete(&arm_state->ka_evt); 1519 } 1520 1521 void 1522 vchiq_on_remote_release(struct vchiq_state *state) 1523 { 1524 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1525 1526 atomic_inc(&arm_state->ka_release_count); 1527 complete(&arm_state->ka_evt); 1528 } 1529 1530 int 1531 vchiq_use_service_internal(struct vchiq_service *service) 1532 { 1533 return vchiq_use_internal(service->state, service, USE_TYPE_SERVICE); 1534 } 1535 1536 int 1537 vchiq_release_service_internal(struct vchiq_service *service) 1538 { 1539 return vchiq_release_internal(service->state, service); 1540 } 1541 1542 struct vchiq_debugfs_node * 1543 vchiq_instance_get_debugfs_node(struct vchiq_instance *instance) 1544 { 1545 return &instance->debugfs_node; 1546 } 1547 1548 int 1549 vchiq_instance_get_use_count(struct vchiq_instance *instance) 1550 { 1551 struct vchiq_service *service; 1552 int use_count = 0, i; 1553 1554 i = 0; 1555 rcu_read_lock(); 1556 while ((service = __next_service_by_instance(instance->state, 1557 instance, &i))) 1558 use_count += service->service_use_count; 1559 rcu_read_unlock(); 1560 return use_count; 1561 } 1562 1563 int 1564 vchiq_instance_get_pid(struct vchiq_instance *instance) 1565 { 1566 return instance->pid; 1567 } 1568 1569 int 1570 vchiq_instance_get_trace(struct vchiq_instance *instance) 1571 { 1572 return instance->trace; 1573 } 1574 1575 void 1576 vchiq_instance_set_trace(struct vchiq_instance *instance, int trace) 1577 { 1578 struct vchiq_service *service; 1579 int i; 1580 1581 i = 0; 1582 rcu_read_lock(); 1583 while ((service = __next_service_by_instance(instance->state, 1584 instance, &i))) 1585 service->trace = trace; 1586 rcu_read_unlock(); 1587 instance->trace = (trace != 0); 1588 } 1589 1590 enum vchiq_status 1591 vchiq_use_service(struct vchiq_instance *instance, unsigned int handle) 1592 { 1593 enum vchiq_status ret = VCHIQ_ERROR; 1594 struct vchiq_service *service = find_service_by_handle(instance, handle); 1595 1596 if (service) { 1597 ret = vchiq_use_internal(service->state, service, USE_TYPE_SERVICE); 1598 vchiq_service_put(service); 1599 } 1600 return ret; 1601 } 1602 EXPORT_SYMBOL(vchiq_use_service); 1603 1604 enum vchiq_status 1605 vchiq_release_service(struct vchiq_instance *instance, unsigned int handle) 1606 { 1607 enum vchiq_status ret = VCHIQ_ERROR; 1608 struct vchiq_service *service = find_service_by_handle(instance, handle); 1609 1610 if (service) { 1611 ret = vchiq_release_internal(service->state, service); 1612 vchiq_service_put(service); 1613 } 1614 return ret; 1615 } 1616 EXPORT_SYMBOL(vchiq_release_service); 1617 1618 struct service_data_struct { 1619 int fourcc; 1620 int clientid; 1621 int use_count; 1622 }; 1623 1624 void 1625 vchiq_dump_service_use_state(struct vchiq_state *state) 1626 { 1627 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1628 struct service_data_struct *service_data; 1629 int i, found = 0; 1630 /* 1631 * If there's more than 64 services, only dump ones with 1632 * non-zero counts 1633 */ 1634 int only_nonzero = 0; 1635 static const char *nz = "<-- preventing suspend"; 1636 1637 int peer_count; 1638 int vc_use_count; 1639 int active_services; 1640 1641 if (!arm_state) 1642 return; 1643 1644 service_data = kmalloc_array(MAX_SERVICES, sizeof(*service_data), 1645 GFP_KERNEL); 1646 if (!service_data) 1647 return; 1648 1649 read_lock_bh(&arm_state->susp_res_lock); 1650 peer_count = arm_state->peer_use_count; 1651 vc_use_count = arm_state->videocore_use_count; 1652 active_services = state->unused_service; 1653 if (active_services > MAX_SERVICES) 1654 only_nonzero = 1; 1655 1656 rcu_read_lock(); 1657 for (i = 0; i < active_services; i++) { 1658 struct vchiq_service *service_ptr = 1659 rcu_dereference(state->services[i]); 1660 1661 if (!service_ptr) 1662 continue; 1663 1664 if (only_nonzero && !service_ptr->service_use_count) 1665 continue; 1666 1667 if (service_ptr->srvstate == VCHIQ_SRVSTATE_FREE) 1668 continue; 1669 1670 service_data[found].fourcc = service_ptr->base.fourcc; 1671 service_data[found].clientid = service_ptr->client_id; 1672 service_data[found].use_count = service_ptr->service_use_count; 1673 found++; 1674 if (found >= MAX_SERVICES) 1675 break; 1676 } 1677 rcu_read_unlock(); 1678 1679 read_unlock_bh(&arm_state->susp_res_lock); 1680 1681 if (only_nonzero) 1682 vchiq_log_warning(vchiq_susp_log_level, "Too many active services (%d). Only dumping up to first %d services with non-zero use-count", 1683 active_services, found); 1684 1685 for (i = 0; i < found; i++) { 1686 vchiq_log_warning(vchiq_susp_log_level, "----- %c%c%c%c:%d service count %d %s", 1687 VCHIQ_FOURCC_AS_4CHARS(service_data[i].fourcc), 1688 service_data[i].clientid, service_data[i].use_count, 1689 service_data[i].use_count ? nz : ""); 1690 } 1691 vchiq_log_warning(vchiq_susp_log_level, "----- VCHIQ use count %d", peer_count); 1692 vchiq_log_warning(vchiq_susp_log_level, "--- Overall vchiq instance use count %d", 1693 vc_use_count); 1694 1695 kfree(service_data); 1696 } 1697 1698 enum vchiq_status 1699 vchiq_check_service(struct vchiq_service *service) 1700 { 1701 struct vchiq_arm_state *arm_state; 1702 enum vchiq_status ret = VCHIQ_ERROR; 1703 1704 if (!service || !service->state) 1705 goto out; 1706 1707 arm_state = vchiq_platform_get_arm_state(service->state); 1708 1709 read_lock_bh(&arm_state->susp_res_lock); 1710 if (service->service_use_count) 1711 ret = VCHIQ_SUCCESS; 1712 read_unlock_bh(&arm_state->susp_res_lock); 1713 1714 if (ret == VCHIQ_ERROR) { 1715 vchiq_log_error(vchiq_susp_log_level, 1716 "%s ERROR - %c%c%c%c:%d service count %d, state count %d", __func__, 1717 VCHIQ_FOURCC_AS_4CHARS(service->base.fourcc), service->client_id, 1718 service->service_use_count, arm_state->videocore_use_count); 1719 vchiq_dump_service_use_state(service->state); 1720 } 1721 out: 1722 return ret; 1723 } 1724 1725 void vchiq_platform_conn_state_changed(struct vchiq_state *state, 1726 enum vchiq_connstate oldstate, 1727 enum vchiq_connstate newstate) 1728 { 1729 struct vchiq_arm_state *arm_state = vchiq_platform_get_arm_state(state); 1730 char threadname[16]; 1731 1732 vchiq_log_info(vchiq_susp_log_level, "%d: %s->%s", state->id, 1733 get_conn_state_name(oldstate), get_conn_state_name(newstate)); 1734 if (state->conn_state != VCHIQ_CONNSTATE_CONNECTED) 1735 return; 1736 1737 write_lock_bh(&arm_state->susp_res_lock); 1738 if (arm_state->first_connect) { 1739 write_unlock_bh(&arm_state->susp_res_lock); 1740 return; 1741 } 1742 1743 arm_state->first_connect = 1; 1744 write_unlock_bh(&arm_state->susp_res_lock); 1745 snprintf(threadname, sizeof(threadname), "vchiq-keep/%d", 1746 state->id); 1747 arm_state->ka_thread = kthread_create(&vchiq_keepalive_thread_func, 1748 (void *)state, 1749 threadname); 1750 if (IS_ERR(arm_state->ka_thread)) { 1751 vchiq_log_error(vchiq_susp_log_level, 1752 "vchiq: FATAL: couldn't create thread %s", 1753 threadname); 1754 } else { 1755 wake_up_process(arm_state->ka_thread); 1756 } 1757 } 1758 1759 static const struct of_device_id vchiq_of_match[] = { 1760 { .compatible = "brcm,bcm2835-vchiq", .data = &bcm2835_drvdata }, 1761 { .compatible = "brcm,bcm2836-vchiq", .data = &bcm2836_drvdata }, 1762 {}, 1763 }; 1764 MODULE_DEVICE_TABLE(of, vchiq_of_match); 1765 1766 static struct platform_device * 1767 vchiq_register_child(struct platform_device *pdev, const char *name) 1768 { 1769 struct platform_device_info pdevinfo; 1770 struct platform_device *child; 1771 1772 memset(&pdevinfo, 0, sizeof(pdevinfo)); 1773 1774 pdevinfo.parent = &pdev->dev; 1775 pdevinfo.name = name; 1776 pdevinfo.id = PLATFORM_DEVID_NONE; 1777 pdevinfo.dma_mask = DMA_BIT_MASK(32); 1778 1779 child = platform_device_register_full(&pdevinfo); 1780 if (IS_ERR(child)) { 1781 dev_warn(&pdev->dev, "%s not registered\n", name); 1782 child = NULL; 1783 } 1784 1785 return child; 1786 } 1787 1788 static int vchiq_probe(struct platform_device *pdev) 1789 { 1790 struct device_node *fw_node; 1791 const struct of_device_id *of_id; 1792 struct vchiq_drvdata *drvdata; 1793 int err; 1794 1795 of_id = of_match_node(vchiq_of_match, pdev->dev.of_node); 1796 drvdata = (struct vchiq_drvdata *)of_id->data; 1797 if (!drvdata) 1798 return -EINVAL; 1799 1800 fw_node = of_find_compatible_node(NULL, NULL, 1801 "raspberrypi,bcm2835-firmware"); 1802 if (!fw_node) { 1803 dev_err(&pdev->dev, "Missing firmware node\n"); 1804 return -ENOENT; 1805 } 1806 1807 drvdata->fw = devm_rpi_firmware_get(&pdev->dev, fw_node); 1808 of_node_put(fw_node); 1809 if (!drvdata->fw) 1810 return -EPROBE_DEFER; 1811 1812 platform_set_drvdata(pdev, drvdata); 1813 1814 err = vchiq_platform_init(pdev, &g_state); 1815 if (err) 1816 goto failed_platform_init; 1817 1818 vchiq_debugfs_init(); 1819 1820 vchiq_log_info(vchiq_arm_log_level, 1821 "vchiq: platform initialised - version %d (min %d)", 1822 VCHIQ_VERSION, VCHIQ_VERSION_MIN); 1823 1824 /* 1825 * Simply exit on error since the function handles cleanup in 1826 * cases of failure. 1827 */ 1828 err = vchiq_register_chrdev(&pdev->dev); 1829 if (err) { 1830 vchiq_log_warning(vchiq_arm_log_level, 1831 "Failed to initialize vchiq cdev"); 1832 goto error_exit; 1833 } 1834 1835 bcm2835_camera = vchiq_register_child(pdev, "bcm2835-camera"); 1836 bcm2835_audio = vchiq_register_child(pdev, "bcm2835_audio"); 1837 1838 return 0; 1839 1840 failed_platform_init: 1841 vchiq_log_warning(vchiq_arm_log_level, "could not initialize vchiq platform"); 1842 error_exit: 1843 return err; 1844 } 1845 1846 static int vchiq_remove(struct platform_device *pdev) 1847 { 1848 platform_device_unregister(bcm2835_audio); 1849 platform_device_unregister(bcm2835_camera); 1850 vchiq_debugfs_deinit(); 1851 vchiq_deregister_chrdev(); 1852 1853 return 0; 1854 } 1855 1856 static struct platform_driver vchiq_driver = { 1857 .driver = { 1858 .name = "bcm2835_vchiq", 1859 .of_match_table = vchiq_of_match, 1860 }, 1861 .probe = vchiq_probe, 1862 .remove = vchiq_remove, 1863 }; 1864 1865 static int __init vchiq_driver_init(void) 1866 { 1867 int ret; 1868 1869 ret = platform_driver_register(&vchiq_driver); 1870 if (ret) 1871 pr_err("Failed to register vchiq driver\n"); 1872 1873 return ret; 1874 } 1875 module_init(vchiq_driver_init); 1876 1877 static void __exit vchiq_driver_exit(void) 1878 { 1879 platform_driver_unregister(&vchiq_driver); 1880 } 1881 module_exit(vchiq_driver_exit); 1882 1883 MODULE_LICENSE("Dual BSD/GPL"); 1884 MODULE_DESCRIPTION("Videocore VCHIQ driver"); 1885 MODULE_AUTHOR("Broadcom Corporation"); 1886