1.\" 2.\" Copyright (c) 2012-2014 The DragonFly Project. All rights reserved. 3.\" 4.\" This code is derived from software contributed to The DragonFly Project 5.\" by Nuno Antunes <nuno.antunes@gmail.com>. 6.\" 7.\" Redistribution and use in source and binary forms, with or without 8.\" modification, are permitted provided that the following conditions 9.\" are met: 10.\" 11.\" 1. Redistributions of source code must retain the above copyright 12.\" notice, this list of conditions and the following disclaimer. 13.\" 2. Redistributions in binary form must reproduce the above copyright 14.\" notice, this list of conditions and the following disclaimer in 15.\" the documentation and/or other materials provided with the 16.\" distribution. 17.\" 3. Neither the name of The DragonFly Project nor the names of its 18.\" contributors may be used to endorse or promote products derived 19.\" from this software without specific, prior written permission. 20.\" 21.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32.\" SUCH DAMAGE. 33.\" 34.Dd July 16, 2014 35.Dt MSGPORT 9 36.Os 37.Sh NAME 38.Nm lwkt_initport_thread , 39.Nm lwkt_initport_spin , 40.Nm lwkt_initport_serialize , 41.Nm lwkt_initport_panic , 42.Nm lwkt_initport_replyonly_null , 43.Nm lwkt_initport_replyonly , 44.Nm lwkt_initport_putonly , 45.Nm lwkt_sendmsg , 46.Nm lwkt_domsg , 47.Nm lwkt_forwardmsg , 48.Nm lwkt_abortmsg , 49.Nm lwkt_initmsg , 50.Nm lwkt_initmsg_abortable , 51.Nm lwkt_beginmsg , 52.Nm lwkt_replymsg , 53.Nm lwkt_getport , 54.Nm lwkt_waitport , 55.Nm lwkt_waitmsg , 56.Nm lwkt_checkmsg , 57.Nm lwkt_dropmsg 58.Nd LWKT message passing interface 59.Sh SYNOPSIS 60.In sys/msgport.h 61.Ft void 62.Fn lwkt_initport_thread "lwkt_port_t port" "struct thread *td" 63.Ft void 64.Fn lwkt_initport_spin "lwkt_port_t port" "struct thread *td" "boolean_t fixed_cpuid" 65.Ft void 66.Fn lwkt_initport_serialize "lwkt_port_t port" "struct lwkt_serialize *slz" 67.Ft void 68.Fn lwkt_initport_panic "lwkt_port_t port" 69.Ft void 70.Fn lwkt_initport_replyonly_null "lwkt_port_t port" 71.Ft void 72.Fn lwkt_initport_replyonly "lwkt_port_t port" "void (*rportfn)(lwkt_port_t, lwkt_msg_t)" 73.Ft void 74.Fn lwkt_initport_putonly "lwkt_port_t port" "int (*pportfn)(lwkt_port_t, lwkt_msg_t)" 75.Ft void 76.Fn lwkt_sendmsg "lwkt_port_t port" "lwkt_msg_t msg" 77.Ft int 78.Fn lwkt_domsg "lwkt_port_t port" "lwkt_msg_t msg" "int flags" 79.Ft int 80.Fn lwkt_forwardmsg "lwkt_port_t port" "lwkt_msg_t msg" 81.Ft void 82.Fn lwkt_abortmsg "lwkt_msg_t msg" 83.In sys/msgport2.h 84.Ft void 85.Fn lwkt_initmsg "lwkt_msg_t msg" "lwkt_port_t rport" "int flags" 86.Ft void 87.Fn lwkt_initmsg_abortable "lwkt_msg_t msg" "lwkt_port_t rport" "int flags" "void (*abortfn)(lwkt_msg_t)" 88.Ft int 89.Fn lwkt_beginmsg "lwkt_port_t port" "lwkt_msg_t msg" 90.Ft void 91.Fn lwkt_replymsg "lwkt_msg_t msg" "int error" 92.Ft void * 93.Fn lwkt_getport "lwkt_port_t port" 94.Ft void * 95.Fn lwkt_waitport "lwkt_port_t port" "int flags" 96.Ft int 97.Fn lwkt_waitmsg "lwkt_msg_t msg" "int flags" 98.Ft int 99.Fn lwkt_checkmsg "lwkt_msg_t msg" 100.Ft int 101.Fn lwkt_dropmsg "lwkt_msg_t msg" 102.Sh DESCRIPTION 103Light weight kernel threads in 104.Dx 105may use a message passing interface to communicate with each other. 106Messages are sent to message ports. 107All light weight kernel threads have a built-in message port, but you may create 108additional ports if necessary. 109The following types of message ports are available: 110.Bl -bullet 111.It 112thread ports 113.It 114spin ports 115.It 116serializer ports 117.El 118.Pp 119Ports of type 120.Sq thread 121are owned by a single light weight kernel thread. 122When a message is sent to a port of type 123.Sq thread , 124only the owner of that port is allowed to retrieve the message from it. 125When a message is sent to a port of type 126.Sq spin 127or to a port of type 128.Sq serializer , 129multiple threads are allowed to check that port for new 130messages and compete to retrieve them. 131You define the port type when you initialize the port. 132By default, the built-in port of every light weight kernel thread is 133automatically initialized to type 134.Sq thread . 135.Pp 136When a message is sent, the receiver should normally send back a reply. 137The reply is sent to the reply port that is registered on the original message. 138Messages can be replied to synchronously or asynchronously. 139The sender may request a synchronous or asynchronous reply to the message, 140however the target port will ultimately decide how the message will be treated. 141.Sh MESSAGE FUNCTIONS 142Messages must be initialized before being used. 143The 144.Fn lwkt_initmsg 145function initializes a message. 146The 147.Fa rport 148argument identifies the reply port which will be used for asynchronous replies. 149The 150.Fa flags 151argument sets any required flags for this message. 152Flags passed this way will simply be or'ed to any already existing flags on the 153message. 154.Pp 155The 156.Fn lwkt_initmsg_abortable 157function is similar to 158.Fn lwkt_initmsg 159but it takes an additional parameter 160.Fa abortfn 161which defines the abort function for this message. 162.Pp 163The 164.Fn lwkt_sendmsg 165function requests an asynchronous reply, sends the message and returns 166immediately. 167Under normal circumstances, users of this function may always expect the reply 168to be queued to the reply port registered on the message. 169The 170.Fa port 171argument defines the target port to which the 172.Fa msg 173message will be sent. 174.Pp 175The 176.Fn lwkt_domsg 177function requests a synchronous reply, sends the message and does not return 178until the message has been replied to. 179If the target port supports synchronous reply, this function will return that 180reply immediately. 181If not, and this is the most common case, this function will block and wait for the 182reply to arrive and then return it. 183The 184.Fa port 185argument defines the target port to which the 186.Fa msg 187message will be sent. 188.Pp 189The 190.Fn lwkt_replymsg 191function replies to a message that was processed asynchronously by the target 192port. 193This function is used by the thread on the receiving side. 194The 195.Fa msg 196argument is the message being replied to and the 197.Fa error 198argument is the actual response to send back. 199.Pp 200The 201.Fn lwkt_forwardmsg 202simply forwards a message to another port. 203The 204.Fa port 205argument defines the target port to which the 206.Fa msg 207message will be sent. 208.Pp 209If a message has been initialized as abortable, you can use the 210.Fn lwkt_abortmsg 211function to try to abort it. 212The 213.Fa abortfn 214passed upon the initialisation with 215.Fn lwkt_initmsg_abortable 216will be called by this function. 217.Pp 218The 219.Fn lwkt_dropmsg 220will dequeue the specified message from the target port it was sent to and makes 221it look like it was never sent. 222This function can only be used by the thread that owns the target port. 223.Sh PORT FUNCTIONS 224The 225.Fn lwkt_initport_thread 226initializes the specified 227.Fa port 228with the default 229.Sq thread 230port type handlers. 231The 232.Fa td 233argument defines the owner thread of the port and only that thread is allowed to 234receive messages on it. 235.Pp 236The 237.Fn lwkt_initport_spin 238initializes the specified 239.Fa port 240with the default 241.Sq spin 242port type handlers. 243The 244.Fa td 245argument defines the owner thread of the port, for cases where thread built-in 246ports are initialized as 247.Sq spin 248ports. 249If 250.Dv NULL 251is passed, then the port will not have a defined owner, so functions like 252.Fn lwkt_dropmsg 253will not be available for this port. 254If we know that this 255.Sq spin 256port is accessed exclusively by threads on a single CPU, we can set the 257.Fa fixed_cpuid 258argument to true, which will prevent the occurrence of unnecessary IPIs trying 259to wake up threads on other CPUs. 260This function will also initialize the embedded spinlock within the 261.Vt lwkt_port 262structure which will protect subsequent port access. 263.Pp 264The 265.Fn lwkt_initport_serialize 266function initializes the specified 267.Fa port 268with the default 269.Sq serializer 270port type handlers. 271The subsequent port access will be protected by the passed 272.Fa slz 273serializer lock. 274.Pp 275The 276.Fn lwkt_getport 277function checks the specified 278.Fa port 279for available messages, dequeues the first one and returns it. 280If no messages are available then 281.Dv NULL 282is returned instead. 283This function is used by threads on the receiving side. 284.Pp 285The 286.Fn lwkt_waitport 287function checks the specified 288.Fa port 289for available messages, dequeues the first one and returns it. 290If no messages are available then the caller thread will sleep until a message 291arrives on the specified port. 292The 293.Fa flags 294argument defines the flags used for the sleep. 295This function is used by threads on the receiving side. 296.Sh SPECIAL PORT INITIALIZERS 297The 298.Fn lwkt_initport_replyonly 299function initializes a 300.Fa port 301which is used only as reply port and may have a custom reply port handler. 302The reply port handler is specified with the 303.Fa rportfn 304argument. 305All the other handlers will panic the system if they are called. 306This initializer is normally used on ports for freeing resources after the 307messages have fulfilled their purpose. 308.Pp 309The 310.Fn lwkt_initport_replyonly_null 311function initializes a 312.Fa port 313which is used only as reply port. 314The reply port handler will simply mark the message as being done and will not 315attempt to queue it. 316All the other handlers will panic the system if they are called. 317.Pp 318The 319.Fn lwkt_initport_putonly 320function initializes a 321.Fa port 322which is used only as target port. 323The putport handler is specified with the 324.Fa pportfn 325argument. 326All the other handlers will panic the system if they are called. 327.Pp 328The 329.Fn lwkt_initport_panic 330function initializes a 331.Fa port 332which will panic the system if any of its handlers are called. 333This function is sometimes used to initialize a reply-only port which does not 334expect the messages to be replied to, e.g.\& when the messages should be 335consumed by the receiving thread and never replied back. 336.Sh INTERNAL MESSAGE FUNCTIONS 337The following functions are used only by the infrastructure, you should not 338need to use them directly unless in very rare cases. 339.Pp 340The 341.Fn lwkt_beginmsg 342function simply calls the target port's putport handler. 343This function is only called by the 344.Fn lwkt_sendmsg 345and 346.Fn lwkt_replymsg 347functions. 348The putport handler returns 349.Er EASYNC 350for messages processed asynchronously or any other value for messages processed 351synchronously. 352That return value of the putport handler is propagated by this function. 353The 354.Fa port 355argument defines the target port to which the 356.Fa msg 357message will be sent. 358.Pp 359The 360.Fn lwkt_waitmsg 361function puts the caller to sleep until the specified 362.Fa msg 363message has been replied to. 364The 365.Fa flags 366argument defines the flags used for the sleep. 367.Sh IMPLEMENTATION NOTES 368All the default putport handlers (used when a message is sent) currently 369implement asynchronous putports only, i.e.\& all 370.Fn *_putport 371handlers return 372.Er EASYNC . 373You can still have synchronous putport handlers (which are run in the sender's 374context) but you have to implement the function yourself and then override the 375default. 376.Pp 377Port handler functions can be overridden with custom functions if required. 378You can override the default putport handler by either using the 379.Fn lwkt_initport_putonly 380initializer, or by manipulating the mp_putport handler pointer directly on the 381.Vt lwkt_port 382structure. 383.Pp 384There is one such case where the putport handler is overridden in 385.Pa sys/net/netisr.c . 386In that case, the putport handler is overridden to detect a loopback message 387(when the target port belongs to the sending thread). 388This special putport handler turns the sent message into a direct function call 389instead of queueing it to the port. 390.Pp 391The 392.Fn lwkt_replymsg 393function works differently depending on the original message request. 394If the 395message was originally an asynchronous request, the reply will be queued to the 396sender's reply port. 397If the message was originally a synchronous request, then 398this function will just write the error response on the message and wake up the 399waiter without queueing the message to the reply port. 400There is no need to queue in the synchronous request case because the original 401sender had blocked waiting on this specific message with 402.Fn lwkt_domsg . 403.Pp 404As is the case with putport handler, the replyport handler can also be 405overridden. 406You override the default replyport handler by using the 407.Fn lwkt_initport_replyonly 408or the 409.Fn lwkt_initport_replyonly_null 410port initializers, or by manipulating the mp_replyport handler pointer directly 411on the 412.Vt lwkt_port 413structure. 414.Pp 415The sent message structure is reused for replies. 416When a message is replied to, the error response is written on the message 417which is subsequently sent to the reply port. 418.Sh FILES 419The LWKT msgport implementation resides in 420.Pa sys/kern/lwkt_msgport.c . 421.Sh EXAMPLES 422.Bd -literal 423/* 424 * Example 1: per CPU threads. 425 * 426 */ 427 428#include <sys/thread.h> 429#include <sys/msgport.h> 430#include <sys/msgport2.h> 431 432static void my_service_loop(void *dummy); 433lwkt_port_t my_service_portfn(int cpu); 434void my_service_sendmsg(lwkt_msg_t lmsg, int cpu); 435int my_service_domsg(lwkt_msg_t lmsg, int cpu); 436 437/* Array of per-CPU target ports */ 438struct lwkt_port *my_service_ports[MAXCPU]; 439 440/* 441 * Create per-cpu threads for handling msg processing. Remember that 442 * built-in lwkt ports are automatically initialized to type 'thread' 443 * so we don't need to initialize them explicitly. 444 */ 445static void 446my_per_cpu_service_init(void) 447{ 448 int i; 449 thread_t td; 450 451 for (i = 0; i < ncpus; ++i) { 452 lwkt_create(my_service_loop, NULL, &td, 453 NULL, 0, i, "myservice_cpu %d", i); 454 my_service_ports[i] = &td->td_msgport; 455 } 456} 457 458/* 459 * This is the routine executed by the service threads on each CPU. 460 */ 461static void 462my_service_loop(void *dummy __unused) 463{ 464 lwkt_msg_t msg; 465 thread_t td = curthread; 466 int cpu = curthread->td_gd->gd_cpuid; 467 468 while ((msg = lwkt_waitport(&td->td_msgport, 0)) != NULL) { 469 /* Do some work in the receiver thread context. */ 470 kprintf("Received message on CPU %d.\en", cpu); 471 472 /* And finally reply to the message. */ 473 lwkt_replymsg(msg, 0); 474 } 475} 476 477/* 478 * Given a CPU id, return our respective service port. 479 */ 480__inline lwkt_port_t 481my_service_portfn(int cpu) 482{ 483 return my_service_ports[cpu]; 484} 485 486/* 487 * Send an asynchronous message to the service thread on a specific CPU. 488 */ 489void 490my_service_sendmsg(lwkt_msg_t lmsg, int cpu) 491{ 492 KKASSERT(cpu < ncpus); 493 lwkt_sendmsg(my_service_portfn(cpu), lmsg); 494} 495 496/* 497 * Send a synchronous message to the service thread on a specific CPU. 498 */ 499int 500my_service_domsg(lwkt_msg_t lmsg, int cpu) 501{ 502 KKASSERT(cpu < ncpus); 503 return lwkt_domsg(my_service_portfn(cpu), lmsg, 0); 504} 505 506/* 507 * Example use case. Initialize the service threads and send each one a 508 * message. 509 */ 510static void 511mod_load(void) 512{ 513 lwkt_msg lmsg; 514 lwkt_port_t builtin_port = &curthread->td_msgport; 515 int i; 516 517 my_per_cpu_service_init(); 518 for (i=0; i<ncpus; ++i) { 519 kprintf("Sending msg to CPU %d.\en", i); 520 lwkt_initmsg(&lmsg, builtin_port, 0); 521 my_service_domsg(&lmsg, i); 522 } 523} 524 525/* 526 * Example 2: Dynamic allocated message passing with automatic free. 527 * 528 * This scenario is used when resources need to be freed after the 529 * message has been replied to. Features: 530 * - An argument is passed within the message. 531 * - Messages are allocated with kmalloc(). Replying to the message 532 * kfree()s it. 533 */ 534 535#include <sys/thread.h> 536#include <sys/msgport.h> 537#include <sys/msgport2.h> 538 539void my_service_queue(void *arg); 540 541lwkt_port my_autofree_rport; 542lwkt_port_t my_service_port; 543 544/* 545 * Use this function to send messages with a void * argument to our 546 * service thread. 547 */ 548void 549my_service_queue(void *arg) 550{ 551 lwkt_msg_t msg; 552 553 msg = kmalloc(sizeof(*msg), M_TEMP, M_WAITOK); 554 555 /* Set reply port to autofree. */ 556 lwkt_initmsg(msg, &my_autofree_rport, 0); 557 558 /* Attach the argument to the message. */ 559 msg->u.ms_resultp = arg; 560 561 /* Send it. */ 562 lwkt_sendmsg(my_service_port, msg); 563} 564 565/* 566 * This is the routine executed by our service thread. 567 */ 568static void 569my_service_loop(void *dummy __unused) 570{ 571 lwkt_msg_t msg; 572 thread_t td = curthread; 573 574 while ((msg = lwkt_waitport(&td->td_msgport, 0)) != NULL) { 575 /* 576 * Do some work in the receiver thread context. In this 577 * example, the sender wrote his name in the argument he 578 * sent us. We print it here. 579 */ 580 char *arg = msg->u.ms_resultp; 581 kprintf("%s: Hi %s! Got your msg.\en", curthread->td_comm, 582 arg); 583 584 /* And finally reply to the message. */ 585 lwkt_replymsg(msg, 0); 586 } 587} 588 589static void 590my_autofree_reply(lwkt_port_t port, lwkt_msg_t msg) 591{ 592 kfree(msg->u.ms_resultp, M_TEMP); 593 kfree(msg, M_TEMP); 594} 595 596static void 597my_service_init(void) 598{ 599 thread_t tdp; 600 601 /* Initialize our auto free reply port. */ 602 lwkt_initport_replyonly(&my_autofree_rport, my_autofree_reply); 603 604 /* Create our service thread on CPU 0. */ 605 lwkt_create(my_service_loop, NULL, &tdp, NULL, 0, 0, "myservice"); 606 my_service_port = &tdp->td_msgport; 607} 608 609/* 610 * Example use case. Initialize the service and send the current thread 611 * name to the service thread. 612 */ 613static void 614mod_load(void) 615{ 616 void *arg; 617 int len; 618 619 my_service_init(); 620 len = strlen(curthread->td_comm); 621 arg = kmalloc(len + 1, M_TEMP, M_WAITOK); 622 bcopy(curthread->td_comm, arg, len + 1); 623 kprintf("%s: Sending message.\en", curthread->td_comm); 624 my_service_queue(arg); 625} 626.Ed 627.Sh SEE ALSO 628.Xr serializer 9 , 629.Xr sleep 9 , 630.Xr spinlock 9 631.Sh HISTORY 632The LWKT msgport interface first appeared in 633.Dx 1.0 . 634.Sh AUTHORS 635.An -nosplit 636The 637.Nm msgport 638message passing interface implementation was written by 639.An Matthew Dillon . 640This manual page was written by 641.An Nuno Antunes . 642