1 /*- 2 * Copyright (c) 1997, 1998, 1999 3 * Nan Yang Computer Services Limited. All rights reserved. 4 * 5 * Parts copyright (c) 1997, 1998 Cybernet Corporation, NetMAX project. 6 * 7 * Written by Greg Lehey 8 * 9 * This software is distributed under the so-called ``Berkeley 10 * License'': 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by Nan Yang Computer 23 * Services Limited. 24 * 4. Neither the name of the Company nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * This software is provided ``as is'', and any express or implied 29 * warranties, including, but not limited to, the implied warranties of 30 * merchantability and fitness for a particular purpose are disclaimed. 31 * In no event shall the company or contributors be liable for any 32 * direct, indirect, incidental, special, exemplary, or consequential 33 * damages (including, but not limited to, procurement of substitute 34 * goods or services; loss of use, data, or profits; or business 35 * interruption) however caused and on any theory of liability, whether 36 * in contract, strict liability, or tort (including negligence or 37 * otherwise) arising in any way out of the use of this software, even if 38 * advised of the possibility of such damage. 39 * 40 * $Id: vinumrevive.c,v 1.14 2000/12/21 01:55:11 grog Exp grog $ 41 * $FreeBSD: src/sys/dev/vinum/vinumrevive.c,v 1.22.2.5 2001/03/13 02:59:43 grog Exp $ 42 */ 43 44 #include "vinumhdr.h" 45 #include "request.h" 46 47 /* 48 * Revive a block of a subdisk. Return an error 49 * indication. EAGAIN means successful copy, but 50 * that more blocks remain to be copied. EINVAL 51 * means that the subdisk isn't associated with a 52 * plex (which means a programming error if we get 53 * here at all; FIXME). 54 */ 55 56 int 57 revive_block(int sdno) 58 { 59 struct sd *sd; 60 struct plex *plex; 61 struct volume *vol; 62 struct buf *bp; 63 cdev_t dev; 64 int error = EAGAIN; 65 int size; /* size of revive block, bytes */ 66 vinum_off_t plexblkno; /* lblkno in plex */ 67 int psd; /* parity subdisk number */ 68 u_int64_t stripe; /* stripe number */ 69 int paritysd = 0; /* set if this is the parity stripe */ 70 struct rangelock *lock; /* for locking */ 71 vinum_off_t stripeoffset; /* offset in stripe */ 72 73 plexblkno = 0; /* to keep the compiler happy */ 74 sd = &SD[sdno]; 75 lock = NULL; 76 if (sd->plexno < 0) /* no plex? */ 77 return EINVAL; 78 plex = &PLEX[sd->plexno]; /* point to plex */ 79 if (plex->volno >= 0) 80 vol = &VOL[plex->volno]; 81 else 82 vol = NULL; 83 84 if ((sd->revive_blocksize == 0) /* no block size */ 85 ||(sd->revive_blocksize & ((1 << DEV_BSHIFT) - 1))) /* or invalid block size */ 86 sd->revive_blocksize = DEFAULT_REVIVE_BLOCKSIZE; 87 else if (sd->revive_blocksize > MAX_REVIVE_BLOCKSIZE) 88 sd->revive_blocksize = MAX_REVIVE_BLOCKSIZE; 89 size = u64min(sd->revive_blocksize >> DEV_BSHIFT, sd->sectors - sd->revived) << DEV_BSHIFT; 90 sd->reviver = curproc->p_pid; /* note who last had a bash at it */ 91 92 /* Now decide where to read from */ 93 switch (plex->organization) { 94 case plex_concat: 95 plexblkno = sd->revived + sd->plexoffset; /* corresponding address in plex */ 96 break; 97 98 case plex_striped: 99 stripeoffset = sd->revived % plex->stripesize; /* offset from beginning of stripe */ 100 if (stripeoffset + (size >> DEV_BSHIFT) > plex->stripesize) 101 size = (plex->stripesize - stripeoffset) << DEV_BSHIFT; 102 plexblkno = sd->plexoffset /* base */ 103 + (sd->revived - stripeoffset) * plex->subdisks /* offset to beginning of stripe */ 104 + stripeoffset; /* offset from beginning of stripe */ 105 break; 106 107 case plex_raid4: 108 case plex_raid5: 109 stripeoffset = sd->revived % plex->stripesize; /* offset from beginning of stripe */ 110 plexblkno = sd->plexoffset /* base */ 111 + (sd->revived - stripeoffset) * (plex->subdisks - 1) /* offset to beginning of stripe */ 112 +stripeoffset; /* offset from beginning of stripe */ 113 stripe = (sd->revived / plex->stripesize); /* stripe number */ 114 115 /* Make sure we don't go beyond the end of the band. */ 116 size = u64min(size, (plex->stripesize - stripeoffset) << DEV_BSHIFT); 117 if (plex->organization == plex_raid4) 118 psd = plex->subdisks - 1; /* parity subdisk for this stripe */ 119 else 120 psd = plex->subdisks - 1 - stripe % plex->subdisks; /* parity subdisk for this stripe */ 121 paritysd = plex->sdnos[psd] == sdno; /* note if it's the parity subdisk */ 122 123 /* 124 * Now adjust for the strangenesses 125 * in RAID-4 and RAID-5 striping. 126 */ 127 if (sd->plexsdno > psd) /* beyond the parity stripe, */ 128 plexblkno -= plex->stripesize; /* one stripe less */ 129 else if (paritysd) 130 plexblkno -= plex->stripesize * sd->plexsdno; /* go back to the beginning of the band */ 131 break; 132 133 case plex_disorg: /* to keep the compiler happy */ 134 break; 135 } 136 137 if (paritysd) { /* we're reviving a parity block, */ 138 bp = parityrebuild(plex, sd->revived, size, rebuildparity, &lock, NULL); /* do the grunt work */ 139 if (bp == NULL) /* no buffer space */ 140 return ENOMEM; /* chicken out */ 141 } else { /* data block */ 142 bp = getpbuf(&vinum_conf.physbufs); /* Get a buffer */ 143 bp->b_data = Malloc(size); 144 145 /* 146 * Amount to transfer: block size, unless it 147 * would overlap the end. 148 */ 149 bp->b_bcount = size; 150 bp->b_resid = bp->b_bcount; 151 bp->b_bio1.bio_offset = (off_t)plexblkno << DEV_BSHIFT; /* start here */ 152 bp->b_bio1.bio_done = biodone_sync; 153 bp->b_bio1.bio_flags |= BIO_SYNC; 154 if (isstriped(plex)) /* we need to lock striped plexes */ 155 lock = lockrange(plexblkno << DEV_BSHIFT, bp, plex); /* lock it */ 156 if (vol != NULL) /* it's part of a volume, */ 157 /* 158 * First, read the data from the volume. We 159 * don't care which plex, that's bre's job. 160 */ 161 dev = vol->vol_dev; 162 else /* it's an unattached plex */ 163 dev = PLEX[sd->plexno].plex_dev; 164 165 bp->b_cmd = BUF_CMD_READ; 166 vinumstart(dev, &bp->b_bio1, 1); 167 biowait(&bp->b_bio1, "drvrd"); 168 } 169 170 if (bp->b_flags & B_ERROR) 171 error = bp->b_error; 172 else 173 /* Now write to the subdisk */ 174 { 175 dev = SD[sdno].sd_dev; 176 KKASSERT(dev != NULL); 177 bp->b_flags |= B_ORDERED; /* and make this an ordered write */ 178 bp->b_cmd = BUF_CMD_WRITE; 179 bp->b_resid = bp->b_bcount; 180 bp->b_bio1.bio_offset = (off_t)sd->revived << DEV_BSHIFT; /* write it to here */ 181 bp->b_bio1.bio_driver_info = dev; 182 bp->b_bio1.bio_done = biodone_sync; 183 sdio(&bp->b_bio1); /* perform the I/O */ 184 biowait(&bp->b_bio1, "drvwr"); 185 if (bp->b_flags & B_ERROR) 186 error = bp->b_error; 187 else { 188 sd->revived += bp->b_bcount >> DEV_BSHIFT; /* moved this much further down */ 189 if (sd->revived >= sd->sectors) { /* finished */ 190 sd->revived = 0; 191 set_sd_state(sdno, sd_up, setstate_force); /* bring the sd up */ 192 log(LOG_INFO, "vinum: %s is %s\n", sd->name, sd_state(sd->state)); 193 save_config(); /* and save the updated configuration */ 194 error = 0; /* we're done */ 195 } 196 } 197 if (lock) /* we took a lock, */ 198 unlockrange(sd->plexno, lock); /* give it back */ 199 while (sd->waitlist) { /* we have waiting requests */ 200 #if VINUMDEBUG 201 struct request *rq = sd->waitlist; 202 cdev_t dev; 203 204 if (debug & DEBUG_REVIVECONFLICT) { 205 dev = rq->bio->bio_driver_info; 206 log(LOG_DEBUG, 207 "Relaunch revive conflict sd %d: %p\n%s dev %d.%d, offset 0x%jx, length %d\n", 208 rq->sdno, 209 rq, 210 (rq->bio->bio_buf->b_cmd == BUF_CMD_READ) ? "Read" : "Write", 211 major(dev), 212 minor(dev), 213 (uintmax_t)rq->bio->bio_offset, 214 rq->bio->bio_buf->b_bcount); 215 } 216 #endif 217 launch_requests(sd->waitlist, 1); /* do them now */ 218 sd->waitlist = sd->waitlist->next; /* and move on to the next */ 219 } 220 } 221 Free(bp->b_data); 222 relpbuf(bp, &vinum_conf.physbufs); 223 return error; 224 } 225 226 /* 227 * Check or rebuild the parity blocks of a RAID-4 228 * or RAID-5 plex. 229 * 230 * The variables plex->checkblock and 231 * plex->rebuildblock represent the 232 * subdisk-relative address of the stripe we're 233 * looking at, not the plex-relative address. We 234 * store it in the plex and not as a local 235 * variable because this function could be 236 * stopped, and we don't want to repeat the part 237 * we've already done. This is also the reason 238 * why we don't initialize it here except at the 239 * end. It gets initialized with the plex on 240 * creation. 241 * 242 * Each call to this function processes at most 243 * one stripe. We can't loop in this function, 244 * because we're unstoppable, so we have to be 245 * called repeatedly from userland. 246 */ 247 void 248 parityops(struct vinum_ioctl_msg *data) 249 { 250 int plexno; 251 struct plex *plex; 252 int size; /* I/O transfer size, bytes */ 253 struct rangelock *lock; /* lock on stripe */ 254 struct _ioctl_reply *reply; 255 off_t pstripe; /* pointer to our stripe counter */ 256 struct buf *pbp; 257 off_t errorloc; /* offset of parity error */ 258 enum parityop op; /* operation to perform */ 259 260 plexno = data->index; 261 op = data->op; 262 pbp = NULL; 263 reply = (struct _ioctl_reply *) data; 264 reply->error = EAGAIN; /* expect to repeat this call */ 265 plex = &PLEX[plexno]; 266 if (!isparity(plex)) { /* not RAID-4 or RAID-5 */ 267 reply->error = EINVAL; 268 return; 269 } else if (plex->state < plex_flaky) { 270 reply->error = EIO; 271 strcpy(reply->msg, "Plex is not completely accessible\n"); 272 return; 273 } 274 pstripe = data->offset; 275 size = imin(DEFAULT_REVIVE_BLOCKSIZE, /* one block at a time */ 276 plex->stripesize << DEV_BSHIFT); 277 278 errorloc = 0; /* avoid gcc warnings */ 279 pbp = parityrebuild(plex, pstripe, size, op, &lock, &errorloc); /* do the grunt work */ 280 if (pbp == NULL) { /* no buffer space */ 281 reply->error = ENOMEM; 282 return; /* chicken out */ 283 } 284 /* 285 * Now we have a result in the data buffer of 286 * the parity buffer header, which we have kept. 287 * Decide what to do with it. 288 */ 289 reply->msg[0] = '\0'; /* until shown otherwise */ 290 if ((pbp->b_flags & B_ERROR) == 0) { /* no error */ 291 if ((op == rebuildparity) 292 || (op == rebuildandcheckparity)) { 293 pbp->b_cmd = BUF_CMD_WRITE; 294 pbp->b_resid = pbp->b_bcount; 295 pbp->b_bio1.bio_done = biodone_sync; 296 sdio(&pbp->b_bio1); /* write the parity block */ 297 biowait(&pbp->b_bio1, "drvwr"); 298 } 299 if (((op == checkparity) 300 || (op == rebuildandcheckparity)) 301 && (errorloc != -1)) { 302 if (op == checkparity) 303 reply->error = EIO; 304 ksprintf(reply->msg, 305 "Parity incorrect at offset 0x%llx\n", 306 (long long)errorloc); 307 } 308 if (reply->error == EAGAIN) { /* still OK, */ 309 plex->checkblock = pstripe + (pbp->b_bcount >> DEV_BSHIFT); /* moved this much further down */ 310 if (plex->checkblock >= SD[plex->sdnos[0]].sectors) { /* finished */ 311 plex->checkblock = 0; 312 reply->error = 0; 313 } 314 } 315 } 316 if (pbp->b_flags & B_ERROR) 317 reply->error = pbp->b_error; 318 Free(pbp->b_data); 319 relpbuf(pbp, &vinum_conf.physbufs); 320 unlockrange(plexno, lock); 321 } 322 323 /* 324 * Rebuild a parity stripe. Return pointer to 325 * parity bp. On return, 326 * 327 * 1. The band is locked. The caller must unlock 328 * the band and release the buffer header. 329 * 330 * 2. All buffer headers except php have been 331 * released. The caller must release pbp. 332 * 333 * 3. For checkparity and rebuildandcheckparity, 334 * the parity is compared with the current 335 * parity block. If it's different, the 336 * offset of the error is returned to 337 * errorloc. The caller can set the value of 338 * the pointer to NULL if this is called for 339 * rebuilding parity. 340 * 341 * pstripe is the subdisk-relative base address of 342 * the data to be reconstructed, size is the size 343 * of the transfer in bytes. 344 */ 345 struct buf * 346 parityrebuild(struct plex *plex, 347 vinum_off_t pstripe, 348 int size, 349 enum parityop op, 350 struct rangelock **lockp, 351 off_t * errorloc) 352 { 353 int error; 354 int sdno; 355 u_int64_t stripe; /* stripe number */ 356 int *parity_buf; /* buffer address for current parity block */ 357 int *newparity_buf; /* and for new parity block */ 358 int mysize; /* I/O transfer size for this transfer */ 359 int isize; /* mysize in ints */ 360 int i; 361 int psd; /* parity subdisk number */ 362 int newpsd; /* and "subdisk number" of new parity */ 363 struct buf **bpp; /* pointers to our bps */ 364 struct buf *pbp; /* buffer header for parity stripe */ 365 int *sbuf; 366 int bufcount; /* number of buffers we need */ 367 368 stripe = pstripe / plex->stripesize; /* stripe number */ 369 psd = plex->subdisks - 1 - stripe % plex->subdisks; /* parity subdisk for this stripe */ 370 parity_buf = NULL; /* to keep the compiler happy */ 371 error = 0; 372 373 /* 374 * It's possible that the default transfer size 375 * we chose is not a factor of the stripe size. 376 * We *must* limit this operation to a single 377 * stripe, at least for RAID-5 rebuild, since 378 * the parity subdisk changes between stripes, 379 * so in this case we need to perform a short 380 * transfer. Set variable mysize to reflect 381 * this. 382 */ 383 mysize = u64min(size, (plex->stripesize * (stripe + 1) - pstripe) << DEV_BSHIFT); 384 isize = mysize / (sizeof(int)); /* number of ints in the buffer */ 385 bufcount = plex->subdisks + 1; /* sd buffers plus result buffer */ 386 newpsd = plex->subdisks; 387 bpp = (struct buf **) Malloc(bufcount * sizeof(struct buf *)); /* array of pointers to bps */ 388 389 /* First, build requests for all subdisks */ 390 for (sdno = 0; sdno < bufcount; sdno++) { /* for each subdisk */ 391 if ((sdno != psd) || (op != rebuildparity)) { 392 /* Get a buffer header and initialize it. */ 393 bpp[sdno] = getpbuf(&vinum_conf.physbufs); /* Get a buffer */ 394 bpp[sdno]->b_data = Malloc(mysize); 395 if (sdno == psd) 396 parity_buf = (int *) bpp[sdno]->b_data; 397 if (sdno == newpsd) /* the new one? */ 398 bpp[sdno]->b_bio1.bio_driver_info = SD[plex->sdnos[psd]].sd_dev; /* write back to the parity SD */ 399 else 400 bpp[sdno]->b_bio1.bio_driver_info = SD[plex->sdnos[sdno]].sd_dev; /* device number */ 401 KKASSERT(bpp[sdno]->b_bio1.bio_driver_info); 402 bpp[sdno]->b_cmd = BUF_CMD_READ; /* either way, read it */ 403 bpp[sdno]->b_bcount = mysize; 404 bpp[sdno]->b_resid = bpp[sdno]->b_bcount; 405 bpp[sdno]->b_bio1.bio_offset = (off_t)pstripe << DEV_BSHIFT; /* transfer from here */ 406 bpp[sdno]->b_bio1.bio_done = biodone_sync; 407 } 408 } 409 410 /* Initialize result buffer */ 411 pbp = bpp[newpsd]; 412 newparity_buf = (int *) bpp[newpsd]->b_data; 413 bzero(newparity_buf, mysize); 414 415 /* 416 * Now lock the stripe with the first non-parity 417 * bp as locking bp. 418 */ 419 *lockp = lockrange(pstripe * plex->stripesize * (plex->subdisks - 1), 420 bpp[psd ? 0 : 1], 421 plex); 422 423 /* 424 * Then issue requests for all subdisks in 425 * parallel. Don't transfer the parity stripe 426 * if we're rebuilding parity, unless we also 427 * want to check it. 428 */ 429 for (sdno = 0; sdno < plex->subdisks; sdno++) { /* for each real subdisk */ 430 if ((sdno != psd) || (op != rebuildparity)) { 431 sdio(&bpp[sdno]->b_bio1); 432 } 433 } 434 435 /* 436 * Next, wait for the requests to complete. 437 * We wait in the order in which they were 438 * issued, which isn't necessarily the order in 439 * which they complete, but we don't have a 440 * convenient way of doing the latter, and the 441 * delay is minimal. 442 */ 443 for (sdno = 0; sdno < plex->subdisks; sdno++) { /* for each subdisk */ 444 if ((sdno != psd) || (op != rebuildparity)) { 445 biowait(&bpp[sdno]->b_bio1, "drvio"); 446 if (bpp[sdno]->b_flags & B_ERROR) /* can't read, */ 447 error = bpp[sdno]->b_error; 448 else if (sdno != psd) { /* update parity */ 449 sbuf = (int *) bpp[sdno]->b_data; 450 for (i = 0; i < isize; i++) 451 newparity_buf[i] ^= sbuf[i]; /* xor in the buffer */ 452 } 453 } 454 if (sdno != psd) { /* release all bps except parity */ 455 Free(bpp[sdno]->b_data); 456 relpbuf(bpp[sdno], &vinum_conf.physbufs); /* give back our resources */ 457 } 458 } 459 460 /* 461 * If we're checking, compare the calculated 462 * and the read parity block. If they're 463 * different, return the plex-relative offset; 464 * otherwise return -1. 465 */ 466 if ((op == checkparity) 467 || (op == rebuildandcheckparity)) { 468 *errorloc = -1; /* no error yet */ 469 for (i = 0; i < isize; i++) { 470 if (parity_buf[i] != newparity_buf[i]) { 471 *errorloc = (off_t) (pstripe << DEV_BSHIFT) * (plex->subdisks - 1) 472 + i * sizeof(int); 473 break; 474 } 475 } 476 Free(bpp[psd]->b_data); 477 relpbuf(bpp[psd], &vinum_conf.physbufs); /* give back our resources */ 478 } 479 /* release our resources */ 480 Free(bpp); 481 if (error) { 482 pbp->b_flags |= B_ERROR; 483 pbp->b_error = error; 484 } 485 return pbp; 486 } 487 488 /* 489 * Initialize a subdisk by writing zeroes to the 490 * complete address space. If verify is set, 491 * check each transfer for correctness. 492 * 493 * Each call to this function writes (and maybe 494 * checks) a single block. 495 */ 496 int 497 initsd(int sdno, int verify) 498 { 499 struct sd *sd; 500 struct plex *plex; 501 struct buf *bp; 502 int error; 503 int size; /* size of init block, bytes */ 504 int verified; /* set when we're happy with what we wrote */ 505 506 error = 0; 507 sd = &SD[sdno]; 508 if (sd->plexno < 0) /* no plex? */ 509 return EINVAL; 510 plex = &PLEX[sd->plexno]; /* point to plex */ 511 512 if (sd->init_blocksize == 0) { 513 if (plex->stripesize != 0) /* we're striped, don't init more than */ 514 sd->init_blocksize = u64min(DEFAULT_REVIVE_BLOCKSIZE, /* one block at a time */ 515 plex->stripesize << DEV_BSHIFT); 516 else 517 sd->init_blocksize = DEFAULT_REVIVE_BLOCKSIZE; 518 } else if (sd->init_blocksize > MAX_REVIVE_BLOCKSIZE) 519 sd->init_blocksize = MAX_REVIVE_BLOCKSIZE; 520 521 size = u64min(sd->init_blocksize >> DEV_BSHIFT, sd->sectors - sd->initialized) << DEV_BSHIFT; 522 523 bp = getpbuf(&vinum_conf.physbufs); /* Get a buffer */ 524 bp->b_data = Malloc(size); 525 526 verified = 0; 527 while (!verified) { /* until we're happy with it, */ 528 bp->b_bcount = size; 529 bp->b_resid = bp->b_bcount; 530 bp->b_bio1.bio_offset = (off_t)sd->initialized << DEV_BSHIFT; /* write it to here */ 531 bp->b_bio1.bio_driver_info = SD[sdno].sd_dev; 532 bp->b_bio1.bio_done = biodone_sync; 533 KKASSERT(bp->b_bio1.bio_driver_info); 534 bzero(bp->b_data, bp->b_bcount); 535 bp->b_cmd = BUF_CMD_WRITE; 536 sdio(&bp->b_bio1); /* perform the I/O */ 537 biowait(&bp->b_bio1, "drvwr"); 538 if (bp->b_flags & B_ERROR) 539 error = bp->b_error; 540 if ((error == 0) && verify) { /* check that it got there */ 541 bp->b_bcount = size; 542 bp->b_resid = bp->b_bcount; 543 bp->b_bio1.bio_offset = (off_t)sd->initialized << DEV_BSHIFT; /* read from here */ 544 bp->b_bio1.bio_driver_info = SD[sdno].sd_dev; 545 bp->b_bio1.bio_done = biodone_sync; 546 KKASSERT(bp->b_bio1.bio_driver_info); 547 bp->b_cmd = BUF_CMD_READ; /* read it back */ 548 sdio(&bp->b_bio1); 549 biowait(&bp->b_bio1, "drvrd"); 550 /* 551 * XXX Bug fix code. This is hopefully no 552 * longer needed (21 February 2000). 553 */ 554 if (bp->b_flags & B_ERROR) 555 error = bp->b_error; 556 else if ((*bp->b_data != 0) /* first word spammed */ 557 ||(bcmp(bp->b_data, &bp->b_data[1], bp->b_bcount - 1))) { /* or one of the others */ 558 kprintf("vinum: init error on %s, offset 0x%llx sectors\n", 559 sd->name, 560 (long long) sd->initialized); 561 verified = 0; 562 } else 563 verified = 1; 564 } else 565 verified = 1; 566 } 567 Free(bp->b_data); 568 relpbuf(bp, &vinum_conf.physbufs); 569 if (error == 0) { /* did it, */ 570 sd->initialized += size >> DEV_BSHIFT; /* moved this much further down */ 571 if (sd->initialized >= sd->sectors) { /* finished */ 572 sd->initialized = 0; 573 set_sd_state(sdno, sd_initialized, setstate_force); /* bring the sd up */ 574 log(LOG_INFO, "vinum: %s is %s\n", sd->name, sd_state(sd->state)); 575 save_config(); /* and save the updated configuration */ 576 } else /* more to go, */ 577 error = EAGAIN; /* ya'll come back, see? */ 578 } 579 return error; 580 } 581