1 /* $NetBSD: if_shmem.c,v 1.33 2010/12/06 10:48:18 pooka Exp $ */ 2 3 /* 4 * Copyright (c) 2009 Antti Kantee. All Rights Reserved. 5 * 6 * Development of this software was supported by The Nokia Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 18 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: if_shmem.c,v 1.33 2010/12/06 10:48:18 pooka Exp $"); 32 33 #include <sys/param.h> 34 #include <sys/atomic.h> 35 #include <sys/fcntl.h> 36 #include <sys/kmem.h> 37 #include <sys/kthread.h> 38 #include <sys/lock.h> 39 #include <sys/vmem.h> 40 41 #include <net/if.h> 42 #include <net/if_ether.h> 43 44 #include <netinet/in.h> 45 #include <netinet/in_var.h> 46 47 #include <rump/rump.h> 48 #include <rump/rumpuser.h> 49 50 #include "rump_private.h" 51 #include "rump_net_private.h" 52 53 static int shmif_clone(struct if_clone *, int); 54 static int shmif_unclone(struct ifnet *); 55 56 struct if_clone shmif_cloner = 57 IF_CLONE_INITIALIZER("shmif", shmif_clone, shmif_unclone); 58 59 /* 60 * Do r/w prefault for backend pages when attaching the interface. 61 * At least logically thinking improves performance (although no 62 * mlocking is done, so they might go away). 63 */ 64 #define PREFAULT_RW 65 66 /* 67 * A virtual ethernet interface which uses shared memory from a 68 * memory mapped file as the bus. 69 */ 70 71 static int shmif_init(struct ifnet *); 72 static int shmif_ioctl(struct ifnet *, u_long, void *); 73 static void shmif_start(struct ifnet *); 74 static void shmif_stop(struct ifnet *, int); 75 76 #include "shmifvar.h" 77 78 struct shmif_sc { 79 struct ethercom sc_ec; 80 uint8_t sc_myaddr[6]; 81 struct shmif_mem *sc_busmem; 82 int sc_memfd; 83 int sc_kq; 84 int sc_unit; 85 86 char *sc_backfile; 87 size_t sc_backfilelen; 88 89 uint64_t sc_devgen; 90 uint32_t sc_nextpacket; 91 92 kmutex_t sc_mtx; 93 kcondvar_t sc_cv; 94 95 struct lwp *sc_rcvl; 96 bool sc_dying; 97 }; 98 99 static const uint32_t busversion = SHMIF_VERSION; 100 101 static void shmif_rcv(void *); 102 103 #define LOCK_UNLOCKED 0 104 #define LOCK_LOCKED 1 105 #define LOCK_COOLDOWN 1001 106 107 vmem_t *shmif_units; 108 109 /* 110 * This locking needs work and will misbehave severely if: 111 * 1) the backing memory has to be paged in 112 * 2) some lockholder exits while holding the lock 113 */ 114 static void 115 shmif_lockbus(struct shmif_mem *busmem) 116 { 117 int i = 0; 118 119 while (__predict_false(atomic_cas_32(&busmem->shm_lock, 120 LOCK_UNLOCKED, LOCK_LOCKED) == LOCK_LOCKED)) { 121 if (__predict_false(++i > LOCK_COOLDOWN)) { 122 uint64_t sec, nsec; 123 int error; 124 125 sec = 0; 126 nsec = 1000*1000; /* 1ms */ 127 rumpuser_nanosleep(&sec, &nsec, &error); 128 i = 0; 129 } 130 continue; 131 } 132 membar_enter(); 133 } 134 135 static void 136 shmif_unlockbus(struct shmif_mem *busmem) 137 { 138 unsigned int old; 139 140 membar_exit(); 141 old = atomic_swap_32(&busmem->shm_lock, LOCK_UNLOCKED); 142 KASSERT(old == LOCK_LOCKED); 143 } 144 145 static int 146 allocif(int unit, struct shmif_sc **scp) 147 { 148 uint8_t enaddr[ETHER_ADDR_LEN] = { 0xb2, 0xa0, 0x00, 0x00, 0x00, 0x00 }; 149 struct shmif_sc *sc; 150 struct ifnet *ifp; 151 uint32_t randnum; 152 int error; 153 154 randnum = arc4random(); 155 memcpy(&enaddr[2], &randnum, sizeof(randnum)); 156 157 sc = kmem_zalloc(sizeof(*sc), KM_SLEEP); 158 sc->sc_memfd = -1; 159 sc->sc_unit = unit; 160 161 ifp = &sc->sc_ec.ec_if; 162 memcpy(sc->sc_myaddr, enaddr, sizeof(enaddr)); 163 164 sprintf(ifp->if_xname, "shmif%d", unit); 165 ifp->if_softc = sc; 166 ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 167 ifp->if_init = shmif_init; 168 ifp->if_ioctl = shmif_ioctl; 169 ifp->if_start = shmif_start; 170 ifp->if_stop = shmif_stop; 171 ifp->if_mtu = ETHERMTU; 172 173 mutex_init(&sc->sc_mtx, MUTEX_DEFAULT, IPL_NONE); 174 cv_init(&sc->sc_cv, "shmifcv"); 175 176 if_attach(ifp); 177 ether_ifattach(ifp, enaddr); 178 179 aprint_verbose("shmif%d: Ethernet address %s\n", 180 unit, ether_sprintf(enaddr)); 181 182 if (scp) 183 *scp = sc; 184 185 error = 0; 186 if (rump_threads) { 187 error = kthread_create(PRI_NONE, 188 KTHREAD_MPSAFE | KTHREAD_JOINABLE, NULL, 189 shmif_rcv, ifp, &sc->sc_rcvl, "shmif"); 190 } else { 191 printf("WARNING: threads not enabled, shmif NOT working\n"); 192 } 193 194 if (error) { 195 shmif_unclone(ifp); 196 } 197 198 return error; 199 } 200 201 static int 202 initbackend(struct shmif_sc *sc, int memfd) 203 { 204 volatile uint8_t v; 205 volatile uint8_t *p; 206 int error; 207 208 sc->sc_busmem = rumpuser_filemmap(memfd, 0, BUSMEM_SIZE, 209 RUMPUSER_FILEMMAP_TRUNCATE | RUMPUSER_FILEMMAP_SHARED 210 | RUMPUSER_FILEMMAP_READ | RUMPUSER_FILEMMAP_WRITE, &error); 211 if (error) 212 return error; 213 214 if (sc->sc_busmem->shm_magic 215 && sc->sc_busmem->shm_magic != SHMIF_MAGIC) { 216 printf("bus is not magical"); 217 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE); 218 return ENOEXEC; 219 } 220 221 /* Prefault in pages to minimize runtime penalty with buslock */ 222 for (p = (uint8_t *)sc->sc_busmem; 223 p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE; 224 p += PAGE_SIZE) 225 v = *p; 226 227 shmif_lockbus(sc->sc_busmem); 228 /* we're first? initialize bus */ 229 if (sc->sc_busmem->shm_magic == 0) { 230 sc->sc_busmem->shm_magic = SHMIF_MAGIC; 231 sc->sc_busmem->shm_first = BUSMEM_DATASIZE; 232 } 233 234 sc->sc_nextpacket = sc->sc_busmem->shm_last; 235 sc->sc_devgen = sc->sc_busmem->shm_gen; 236 237 #ifdef PREFAULT_RW 238 for (p = (uint8_t *)sc->sc_busmem; 239 p < (uint8_t *)sc->sc_busmem + BUSMEM_SIZE; 240 p += PAGE_SIZE) { 241 v = *p; 242 *p = v; 243 } 244 #endif 245 shmif_unlockbus(sc->sc_busmem); 246 247 sc->sc_kq = rumpuser_writewatchfile_setup(-1, memfd, 0, &error); 248 if (sc->sc_kq == -1) { 249 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE); 250 return error; 251 } 252 253 sc->sc_memfd = memfd; 254 255 return error; 256 } 257 258 static void 259 finibackend(struct shmif_sc *sc) 260 { 261 262 if (sc->sc_backfile == NULL) 263 return; 264 265 if (sc->sc_backfile) { 266 kmem_free(sc->sc_backfile, sc->sc_backfilelen); 267 sc->sc_backfile = NULL; 268 sc->sc_backfilelen = 0; 269 } 270 271 rumpuser_unmap(sc->sc_busmem, BUSMEM_SIZE); 272 rumpuser_close(sc->sc_memfd, NULL); 273 rumpuser_close(sc->sc_kq, NULL); 274 275 sc->sc_memfd = -1; 276 } 277 278 int 279 rump_shmif_create(const char *path, int *ifnum) 280 { 281 struct shmif_sc *sc; 282 int unit, error; 283 int memfd = -1; /* XXXgcc */ 284 285 if (path) { 286 memfd = rumpuser_open(path, O_RDWR | O_CREAT, &error); 287 if (memfd == -1) 288 return error; 289 } 290 291 unit = vmem_xalloc(shmif_units, 1, 0, 0, 0, 0, 0, 292 VM_INSTANTFIT | VM_SLEEP) - 1; 293 294 if ((error = allocif(unit, &sc)) != 0) { 295 if (path) 296 rumpuser_close(memfd, NULL); 297 return error; 298 } 299 300 if (!path) 301 goto out; 302 303 error = initbackend(sc, memfd); 304 if (error) { 305 shmif_unclone(&sc->sc_ec.ec_if); 306 return error; 307 } 308 309 sc->sc_backfilelen = strlen(path)+1; 310 sc->sc_backfile = kmem_alloc(sc->sc_backfilelen, KM_SLEEP); 311 strcpy(sc->sc_backfile, path); 312 313 out: 314 if (ifnum) 315 *ifnum = unit; 316 317 return 0; 318 } 319 320 static int 321 shmif_clone(struct if_clone *ifc, int unit) 322 { 323 int unit2; 324 325 /* 326 * Ok, we know the unit number, but we must still reserve it. 327 * Otherwise the wildcard-side of things might get the same one. 328 * This is slightly offset-happy due to vmem. First, we offset 329 * the range of unit numbers by +1 since vmem cannot deal with 330 * ranges starting from 0. Second, since vmem_xalloc() allocates 331 * from [min,max) (half-*open* interval), we need to add one extra 332 * to the one extra we add to maxaddr. Talk about uuuh. 333 */ 334 unit2 = vmem_xalloc(shmif_units, 1, 0, 0, 0, unit+1, unit+3, 335 VM_SLEEP | VM_INSTANTFIT); 336 KASSERT(unit2-1 == unit); 337 338 return allocif(unit, NULL); 339 } 340 341 static int 342 shmif_unclone(struct ifnet *ifp) 343 { 344 struct shmif_sc *sc = ifp->if_softc; 345 346 shmif_stop(ifp, 1); 347 if_down(ifp); 348 finibackend(sc); 349 350 mutex_enter(&sc->sc_mtx); 351 sc->sc_dying = true; 352 cv_broadcast(&sc->sc_cv); 353 mutex_exit(&sc->sc_mtx); 354 355 if (sc->sc_rcvl) 356 kthread_join(sc->sc_rcvl); 357 sc->sc_rcvl = NULL; 358 359 vmem_xfree(shmif_units, sc->sc_unit+1, 1); 360 361 ether_ifdetach(ifp); 362 if_detach(ifp); 363 364 cv_destroy(&sc->sc_cv); 365 mutex_destroy(&sc->sc_mtx); 366 367 kmem_free(sc, sizeof(*sc)); 368 369 return 0; 370 } 371 372 static int 373 shmif_init(struct ifnet *ifp) 374 { 375 struct shmif_sc *sc = ifp->if_softc; 376 int error = 0; 377 378 if (sc->sc_memfd == -1) 379 return ENXIO; 380 KASSERT(sc->sc_busmem); 381 382 ifp->if_flags |= IFF_RUNNING; 383 384 mutex_enter(&sc->sc_mtx); 385 sc->sc_nextpacket = sc->sc_busmem->shm_last; 386 sc->sc_devgen = sc->sc_busmem->shm_gen; 387 388 cv_broadcast(&sc->sc_cv); 389 mutex_exit(&sc->sc_mtx); 390 391 return error; 392 } 393 394 static int 395 shmif_ioctl(struct ifnet *ifp, u_long cmd, void *data) 396 { 397 struct shmif_sc *sc = ifp->if_softc; 398 struct ifdrv *ifd; 399 char *path; 400 int s, rv, memfd; 401 402 s = splnet(); 403 switch (cmd) { 404 case SIOCGLINKSTR: 405 ifd = data; 406 407 if (sc->sc_backfilelen == 0) { 408 rv = ENOENT; 409 break; 410 } 411 412 ifd->ifd_len = sc->sc_backfilelen; 413 if (ifd->ifd_cmd == IFLINKSTR_QUERYLEN) { 414 rv = 0; 415 break; 416 } 417 418 if (ifd->ifd_cmd != 0) { 419 rv = EINVAL; 420 break; 421 } 422 423 rv = copyoutstr(sc->sc_backfile, ifd->ifd_data, 424 MIN(sc->sc_backfilelen, ifd->ifd_len), NULL); 425 break; 426 case SIOCSLINKSTR: 427 if (ifp->if_flags & IFF_UP) { 428 rv = EBUSY; 429 break; 430 } 431 432 ifd = data; 433 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 434 finibackend(sc); 435 rv = 0; 436 break; 437 } else if (ifd->ifd_cmd != 0) { 438 rv = EINVAL; 439 break; 440 } else if (sc->sc_backfile) { 441 rv = EBUSY; 442 break; 443 } 444 445 if (ifd->ifd_len > MAXPATHLEN) { 446 rv = E2BIG; 447 break; 448 } else if (ifd->ifd_len < 1) { 449 rv = EINVAL; 450 break; 451 } 452 453 path = kmem_alloc(ifd->ifd_len, KM_SLEEP); 454 rv = copyinstr(ifd->ifd_data, path, ifd->ifd_len, NULL); 455 if (rv) { 456 kmem_free(path, ifd->ifd_len); 457 break; 458 } 459 memfd = rumpuser_open(path, O_RDWR | O_CREAT, &rv); 460 if (memfd == -1) { 461 kmem_free(path, ifd->ifd_len); 462 break; 463 } 464 rv = initbackend(sc, memfd); 465 if (rv) { 466 kmem_free(path, ifd->ifd_len); 467 rumpuser_close(memfd, NULL); 468 break; 469 } 470 sc->sc_backfile = path; 471 sc->sc_backfilelen = ifd->ifd_len; 472 473 break; 474 default: 475 rv = ether_ioctl(ifp, cmd, data); 476 if (rv == ENETRESET) 477 rv = 0; 478 break; 479 } 480 splx(s); 481 482 return rv; 483 } 484 485 /* send everything in-context since it's just a matter of mem-to-mem copy */ 486 static void 487 shmif_start(struct ifnet *ifp) 488 { 489 struct shmif_sc *sc = ifp->if_softc; 490 struct shmif_mem *busmem = sc->sc_busmem; 491 struct mbuf *m, *m0; 492 uint32_t dataoff; 493 uint32_t pktsize, pktwrote; 494 bool wrote = false; 495 bool wrap; 496 int error; 497 498 ifp->if_flags |= IFF_OACTIVE; 499 500 for (;;) { 501 struct shmif_pkthdr sp; 502 struct timeval tv; 503 504 IF_DEQUEUE(&ifp->if_snd, m0); 505 if (m0 == NULL) { 506 break; 507 } 508 509 pktsize = 0; 510 for (m = m0; m != NULL; m = m->m_next) { 511 pktsize += m->m_len; 512 } 513 KASSERT(pktsize <= ETHERMTU + ETHER_HDR_LEN); 514 515 getmicrouptime(&tv); 516 sp.sp_len = pktsize; 517 sp.sp_sec = tv.tv_sec; 518 sp.sp_usec = tv.tv_usec; 519 520 shmif_lockbus(busmem); 521 KASSERT(busmem->shm_magic == SHMIF_MAGIC); 522 busmem->shm_last = shmif_nextpktoff(busmem, busmem->shm_last); 523 524 wrap = false; 525 dataoff = shmif_buswrite(busmem, 526 busmem->shm_last, &sp, sizeof(sp), &wrap); 527 pktwrote = 0; 528 for (m = m0; m != NULL; m = m->m_next) { 529 pktwrote += m->m_len; 530 dataoff = shmif_buswrite(busmem, dataoff, 531 mtod(m, void *), m->m_len, &wrap); 532 } 533 KASSERT(pktwrote == pktsize); 534 if (wrap) { 535 busmem->shm_gen++; 536 DPRINTF(("bus generation now %d\n", busmem->shm_gen)); 537 } 538 shmif_unlockbus(busmem); 539 540 m_freem(m0); 541 wrote = true; 542 543 DPRINTF(("shmif_start: send %d bytes at off %d\n", 544 pktsize, busmem->shm_last)); 545 } 546 547 ifp->if_flags &= ~IFF_OACTIVE; 548 549 /* wakeup? */ 550 if (wrote) 551 rumpuser_pwrite(sc->sc_memfd, 552 &busversion, sizeof(busversion), IFMEM_WAKEUP, &error); 553 } 554 555 static void 556 shmif_stop(struct ifnet *ifp, int disable) 557 { 558 struct shmif_sc *sc = ifp->if_softc; 559 560 ifp->if_flags &= ~IFF_RUNNING; 561 membar_producer(); 562 563 /* 564 * wakeup thread. this will of course wake up all bus 565 * listeners, but that's life. 566 */ 567 if (sc->sc_memfd != -1) 568 rumpuser_pwrite(sc->sc_memfd, 569 &busversion, sizeof(busversion), IFMEM_WAKEUP, NULL); 570 } 571 572 573 /* 574 * Check if we have been sleeping too long. Basically, 575 * our in-sc nextpkt must by first <= nextpkt <= last"+1". 576 * We use the fact that first is guaranteed to never overlap 577 * with the last frame in the ring. 578 */ 579 static __inline bool 580 stillvalid_p(struct shmif_sc *sc) 581 { 582 struct shmif_mem *busmem = sc->sc_busmem; 583 unsigned gendiff = busmem->shm_gen - sc->sc_devgen; 584 uint32_t lastoff, devoff; 585 586 KASSERT(busmem->shm_first != busmem->shm_last); 587 588 /* normalize onto a 2x busmem chunk */ 589 devoff = sc->sc_nextpacket; 590 lastoff = shmif_nextpktoff(busmem, busmem->shm_last); 591 592 /* trivial case */ 593 if (gendiff > 1) 594 return false; 595 KASSERT(gendiff <= 1); 596 597 /* Normalize onto 2x busmem chunk */ 598 if (busmem->shm_first >= lastoff) { 599 lastoff += BUSMEM_DATASIZE; 600 if (gendiff == 0) 601 devoff += BUSMEM_DATASIZE; 602 } else { 603 if (gendiff) 604 return false; 605 } 606 607 return devoff >= busmem->shm_first && devoff <= lastoff; 608 } 609 610 static void 611 shmif_rcv(void *arg) 612 { 613 struct ifnet *ifp = arg; 614 struct shmif_sc *sc = ifp->if_softc; 615 struct shmif_mem *busmem; 616 struct mbuf *m = NULL; 617 struct ether_header *eth; 618 uint32_t nextpkt; 619 bool wrap; 620 int error; 621 622 reup: 623 mutex_enter(&sc->sc_mtx); 624 while ((ifp->if_flags & IFF_RUNNING) == 0 && !sc->sc_dying) 625 cv_wait(&sc->sc_cv, &sc->sc_mtx); 626 mutex_exit(&sc->sc_mtx); 627 628 busmem = sc->sc_busmem; 629 630 while (ifp->if_flags & IFF_RUNNING) { 631 struct shmif_pkthdr sp; 632 633 if (m == NULL) { 634 m = m_gethdr(M_WAIT, MT_DATA); 635 MCLGET(m, M_WAIT); 636 } 637 638 DPRINTF(("waiting %d/%d\n", sc->sc_nextpacket, sc->sc_devgen)); 639 KASSERT(m->m_flags & M_EXT); 640 641 shmif_lockbus(busmem); 642 KASSERT(busmem->shm_magic == SHMIF_MAGIC); 643 KASSERT(busmem->shm_gen >= sc->sc_devgen); 644 645 /* need more data? */ 646 if (sc->sc_devgen == busmem->shm_gen && 647 shmif_nextpktoff(busmem, busmem->shm_last) 648 == sc->sc_nextpacket) { 649 shmif_unlockbus(busmem); 650 error = 0; 651 rumpuser_writewatchfile_wait(sc->sc_kq, NULL, &error); 652 if (__predict_false(error)) 653 printf("shmif_rcv: wait failed %d\n", error); 654 membar_consumer(); 655 continue; 656 } 657 658 if (stillvalid_p(sc)) { 659 nextpkt = sc->sc_nextpacket; 660 } else { 661 KASSERT(busmem->shm_gen > 0); 662 nextpkt = busmem->shm_first; 663 if (busmem->shm_first > busmem->shm_last) 664 sc->sc_devgen = busmem->shm_gen - 1; 665 else 666 sc->sc_devgen = busmem->shm_gen; 667 DPRINTF(("dev %p overrun, new data: %d/%d\n", 668 sc, nextpkt, sc->sc_devgen)); 669 } 670 671 /* 672 * If our read pointer is ahead the bus last write, our 673 * generation must be one behind. 674 */ 675 KASSERT(!(nextpkt > busmem->shm_last 676 && sc->sc_devgen == busmem->shm_gen)); 677 678 wrap = false; 679 nextpkt = shmif_busread(busmem, &sp, 680 nextpkt, sizeof(sp), &wrap); 681 KASSERT(sp.sp_len <= ETHERMTU + ETHER_HDR_LEN); 682 nextpkt = shmif_busread(busmem, mtod(m, void *), 683 nextpkt, sp.sp_len, &wrap); 684 685 DPRINTF(("shmif_rcv: read packet of length %d at %d\n", 686 sp.sp_len, nextpkt)); 687 688 sc->sc_nextpacket = nextpkt; 689 shmif_unlockbus(sc->sc_busmem); 690 691 if (wrap) { 692 sc->sc_devgen++; 693 DPRINTF(("dev %p generation now %d\n", 694 sc, sc->sc_devgen)); 695 } 696 697 m->m_len = m->m_pkthdr.len = sp.sp_len; 698 m->m_pkthdr.rcvif = ifp; 699 700 /* if it's from us, don't pass up and reuse storage space */ 701 eth = mtod(m, struct ether_header *); 702 if (memcmp(eth->ether_shost, sc->sc_myaddr, 6) != 0) { 703 KERNEL_LOCK(1, NULL); 704 ifp->if_input(ifp, m); 705 KERNEL_UNLOCK_ONE(NULL); 706 m = NULL; 707 } 708 } 709 m_freem(m); 710 m = NULL; 711 712 if (!sc->sc_dying) 713 goto reup; 714 715 kthread_exit(0); 716 } 717