1 /* $Id: read.c,v 1.15 2011/05/26 20:36:21 kristaps Exp $ */ 2 /* 3 * Copyright (c) 2008, 2009, 2010, 2011 Kristaps Dzonsons <kristaps@bsd.lv> 4 * Copyright (c) 2010, 2011 Ingo Schwarze <schwarze@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 #ifdef HAVE_CONFIG_H 19 #include "config.h" 20 #endif 21 22 #ifdef HAVE_MMAP 23 # include <sys/stat.h> 24 # include <sys/mman.h> 25 #endif 26 27 #include <assert.h> 28 #include <ctype.h> 29 #include <fcntl.h> 30 #include <stdarg.h> 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <string.h> 34 #include <unistd.h> 35 36 #include "mandoc.h" 37 #include "libmandoc.h" 38 #include "mdoc.h" 39 #include "man.h" 40 41 #ifndef MAP_FILE 42 #define MAP_FILE 0 43 #endif 44 45 #define REPARSE_LIMIT 1000 46 47 struct buf { 48 char *buf; /* binary input buffer */ 49 size_t sz; /* size of binary buffer */ 50 }; 51 52 struct mparse { 53 enum mandoclevel file_status; /* status of current parse */ 54 enum mandoclevel wlevel; /* ignore messages below this */ 55 int line; /* line number in the file */ 56 enum mparset inttype; /* which parser to use */ 57 struct man *pman; /* persistent man parser */ 58 struct mdoc *pmdoc; /* persistent mdoc parser */ 59 struct man *man; /* man parser */ 60 struct mdoc *mdoc; /* mdoc parser */ 61 struct roff *roff; /* roff parser (!NULL) */ 62 struct regset regs; /* roff registers */ 63 int reparse_count; /* finite interp. stack */ 64 mandocmsg mmsg; /* warning/error message handler */ 65 void *arg; /* argument to mmsg */ 66 const char *file; 67 }; 68 69 static void resize_buf(struct buf *, size_t); 70 static void mparse_buf_r(struct mparse *, struct buf, int); 71 static void mparse_readfd_r(struct mparse *, int, const char *, int); 72 static void pset(const char *, int, struct mparse *); 73 static void pdesc(struct mparse *, const char *, int); 74 static int read_whole_file(const char *, int, struct buf *, int *); 75 static void mparse_end(struct mparse *); 76 77 static const enum mandocerr mandoclimits[MANDOCLEVEL_MAX] = { 78 MANDOCERR_OK, 79 MANDOCERR_WARNING, 80 MANDOCERR_WARNING, 81 MANDOCERR_ERROR, 82 MANDOCERR_FATAL, 83 MANDOCERR_MAX, 84 MANDOCERR_MAX 85 }; 86 87 static const char * const mandocerrs[MANDOCERR_MAX] = { 88 "ok", 89 90 "generic warning", 91 92 /* related to the prologue */ 93 "no title in document", 94 "document title should be all caps", 95 "unknown manual section", 96 "date missing, using today's date", 97 "cannot parse date, using it verbatim", 98 "prologue macros out of order", 99 "duplicate prologue macro", 100 "macro not allowed in prologue", 101 "macro not allowed in body", 102 103 /* related to document structure */ 104 ".so is fragile, better use ln(1)", 105 "NAME section must come first", 106 "bad NAME section contents", 107 "manual name not yet set", 108 "sections out of conventional order", 109 "duplicate section name", 110 "section not in conventional manual section", 111 112 /* related to macros and nesting */ 113 "skipping obsolete macro", 114 "skipping paragraph macro", 115 "skipping no-space macro", 116 "blocks badly nested", 117 "child violates parent syntax", 118 "nested displays are not portable", 119 "already in literal mode", 120 "line scope broken", 121 122 /* related to missing macro arguments */ 123 "skipping empty macro", 124 "argument count wrong", 125 "missing display type", 126 "list type must come first", 127 "tag lists require a width argument", 128 "missing font type", 129 "skipping end of block that is not open", 130 131 /* related to bad macro arguments */ 132 "skipping argument", 133 "duplicate argument", 134 "duplicate display type", 135 "duplicate list type", 136 "unknown AT&T UNIX version", 137 "bad Boolean value", 138 "unknown font", 139 "unknown standard specifier", 140 "bad width argument", 141 142 /* related to plain text */ 143 "blank line in non-literal context", 144 "tab in non-literal context", 145 "end of line whitespace", 146 "bad comment style", 147 "bad escape sequence", 148 "unterminated quoted string", 149 150 "generic error", 151 152 /* related to tables */ 153 "bad table syntax", 154 "bad table option", 155 "bad table layout", 156 "no table layout cells specified", 157 "no table data cells specified", 158 "ignore data in cell", 159 "data block still open", 160 "ignoring extra data cells", 161 162 "input stack limit exceeded, infinite loop?", 163 "skipping bad character", 164 "escaped character not allowed in a name", 165 "skipping text before the first section header", 166 "skipping unknown macro", 167 "NOT IMPLEMENTED, please use groff: skipping request", 168 "argument count wrong", 169 "skipping end of block that is not open", 170 "missing end of block", 171 "scope open on exit", 172 "uname(3) system call failed", 173 "macro requires line argument(s)", 174 "macro requires body argument(s)", 175 "macro requires argument(s)", 176 "missing list type", 177 "line argument(s) will be lost", 178 "body argument(s) will be lost", 179 180 "generic fatal error", 181 182 "not a manual", 183 "column syntax is inconsistent", 184 "NOT IMPLEMENTED: .Bd -file", 185 "line scope broken, syntax violated", 186 "argument count wrong, violates syntax", 187 "child violates parent syntax", 188 "argument count wrong, violates syntax", 189 "NOT IMPLEMENTED: .so with absolute path or \"..\"", 190 "no document body", 191 "no document prologue", 192 "static buffer exhausted", 193 }; 194 195 static const char * const mandoclevels[MANDOCLEVEL_MAX] = { 196 "SUCCESS", 197 "RESERVED", 198 "WARNING", 199 "ERROR", 200 "FATAL", 201 "BADARG", 202 "SYSERR" 203 }; 204 205 static void 206 resize_buf(struct buf *buf, size_t initial) 207 { 208 209 buf->sz = buf->sz > initial/2 ? 2 * buf->sz : initial; 210 buf->buf = mandoc_realloc(buf->buf, buf->sz); 211 } 212 213 static void 214 pset(const char *buf, int pos, struct mparse *curp) 215 { 216 int i; 217 218 /* 219 * Try to intuit which kind of manual parser should be used. If 220 * passed in by command-line (-man, -mdoc), then use that 221 * explicitly. If passed as -mandoc, then try to guess from the 222 * line: either skip dot-lines, use -mdoc when finding `.Dt', or 223 * default to -man, which is more lenient. 224 * 225 * Separate out pmdoc/pman from mdoc/man: the first persists 226 * through all parsers, while the latter is used per-parse. 227 */ 228 229 if ('.' == buf[0] || '\'' == buf[0]) { 230 for (i = 1; buf[i]; i++) 231 if (' ' != buf[i] && '\t' != buf[i]) 232 break; 233 if ('\0' == buf[i]) 234 return; 235 } 236 237 switch (curp->inttype) { 238 case (MPARSE_MDOC): 239 if (NULL == curp->pmdoc) 240 curp->pmdoc = mdoc_alloc(&curp->regs, curp); 241 assert(curp->pmdoc); 242 curp->mdoc = curp->pmdoc; 243 return; 244 case (MPARSE_MAN): 245 if (NULL == curp->pman) 246 curp->pman = man_alloc(&curp->regs, curp); 247 assert(curp->pman); 248 curp->man = curp->pman; 249 return; 250 default: 251 break; 252 } 253 254 if (pos >= 3 && 0 == memcmp(buf, ".Dd", 3)) { 255 if (NULL == curp->pmdoc) 256 curp->pmdoc = mdoc_alloc(&curp->regs, curp); 257 assert(curp->pmdoc); 258 curp->mdoc = curp->pmdoc; 259 return; 260 } 261 262 if (NULL == curp->pman) 263 curp->pman = man_alloc(&curp->regs, curp); 264 assert(curp->pman); 265 curp->man = curp->pman; 266 } 267 268 /* 269 * Main parse routine for an opened file. This is called for each 270 * opened file and simply loops around the full input file, possibly 271 * nesting (i.e., with `so'). 272 */ 273 static void 274 mparse_buf_r(struct mparse *curp, struct buf blk, int start) 275 { 276 const struct tbl_span *span; 277 struct buf ln; 278 enum rofferr rr; 279 int i, of, rc; 280 int pos; /* byte number in the ln buffer */ 281 int lnn; /* line number in the real file */ 282 unsigned char c; 283 284 memset(&ln, 0, sizeof(struct buf)); 285 286 lnn = curp->line; 287 pos = 0; 288 289 for (i = 0; i < (int)blk.sz; ) { 290 if (0 == pos && '\0' == blk.buf[i]) 291 break; 292 293 if (start) { 294 curp->line = lnn; 295 curp->reparse_count = 0; 296 } 297 298 while (i < (int)blk.sz && (start || '\0' != blk.buf[i])) { 299 300 /* 301 * When finding an unescaped newline character, 302 * leave the character loop to process the line. 303 * Skip a preceding carriage return, if any. 304 */ 305 306 if ('\r' == blk.buf[i] && i + 1 < (int)blk.sz && 307 '\n' == blk.buf[i + 1]) 308 ++i; 309 if ('\n' == blk.buf[i]) { 310 ++i; 311 ++lnn; 312 break; 313 } 314 315 /* 316 * Warn about bogus characters. If you're using 317 * non-ASCII encoding, you're screwing your 318 * readers. Since I'd rather this not happen, 319 * I'll be helpful and drop these characters so 320 * we don't display gibberish. Note to manual 321 * writers: use special characters. 322 */ 323 324 c = (unsigned char) blk.buf[i]; 325 326 if ( ! (isascii(c) && 327 (isgraph(c) || isblank(c)))) { 328 mandoc_msg(MANDOCERR_BADCHAR, curp, 329 curp->line, pos, "ignoring byte"); 330 i++; 331 continue; 332 } 333 334 /* Trailing backslash = a plain char. */ 335 336 if ('\\' != blk.buf[i] || i + 1 == (int)blk.sz) { 337 if (pos >= (int)ln.sz) 338 resize_buf(&ln, 256); 339 ln.buf[pos++] = blk.buf[i++]; 340 continue; 341 } 342 343 /* 344 * Found escape and at least one other character. 345 * When it's a newline character, skip it. 346 * When there is a carriage return in between, 347 * skip that one as well. 348 */ 349 350 if ('\r' == blk.buf[i + 1] && i + 2 < (int)blk.sz && 351 '\n' == blk.buf[i + 2]) 352 ++i; 353 if ('\n' == blk.buf[i + 1]) { 354 i += 2; 355 ++lnn; 356 continue; 357 } 358 359 if ('"' == blk.buf[i + 1] || '#' == blk.buf[i + 1]) { 360 i += 2; 361 /* Comment, skip to end of line */ 362 for (; i < (int)blk.sz; ++i) { 363 if ('\n' == blk.buf[i]) { 364 ++i; 365 ++lnn; 366 break; 367 } 368 } 369 370 /* Backout trailing whitespaces */ 371 for (; pos > 0; --pos) { 372 if (ln.buf[pos - 1] != ' ') 373 break; 374 if (pos > 2 && ln.buf[pos - 2] == '\\') 375 break; 376 } 377 break; 378 } 379 380 /* Some other escape sequence, copy & cont. */ 381 382 if (pos + 1 >= (int)ln.sz) 383 resize_buf(&ln, 256); 384 385 ln.buf[pos++] = blk.buf[i++]; 386 ln.buf[pos++] = blk.buf[i++]; 387 } 388 389 if (pos >= (int)ln.sz) 390 resize_buf(&ln, 256); 391 392 ln.buf[pos] = '\0'; 393 394 /* 395 * A significant amount of complexity is contained by 396 * the roff preprocessor. It's line-oriented but can be 397 * expressed on one line, so we need at times to 398 * readjust our starting point and re-run it. The roff 399 * preprocessor can also readjust the buffers with new 400 * data, so we pass them in wholesale. 401 */ 402 403 of = 0; 404 405 rerun: 406 rr = roff_parseln 407 (curp->roff, curp->line, 408 &ln.buf, &ln.sz, of, &of); 409 410 switch (rr) { 411 case (ROFF_REPARSE): 412 if (REPARSE_LIMIT >= ++curp->reparse_count) 413 mparse_buf_r(curp, ln, 0); 414 else 415 mandoc_msg(MANDOCERR_ROFFLOOP, curp, 416 curp->line, pos, NULL); 417 pos = 0; 418 continue; 419 case (ROFF_APPEND): 420 pos = (int)strlen(ln.buf); 421 continue; 422 case (ROFF_RERUN): 423 goto rerun; 424 case (ROFF_IGN): 425 pos = 0; 426 continue; 427 case (ROFF_ERR): 428 assert(MANDOCLEVEL_FATAL <= curp->file_status); 429 break; 430 case (ROFF_SO): 431 mparse_readfd_r(curp, -1, ln.buf + of, 1); 432 if (MANDOCLEVEL_FATAL <= curp->file_status) 433 break; 434 pos = 0; 435 continue; 436 default: 437 break; 438 } 439 440 /* 441 * If we encounter errors in the recursive parse, make 442 * sure we don't continue parsing. 443 */ 444 445 if (MANDOCLEVEL_FATAL <= curp->file_status) 446 break; 447 448 /* 449 * If input parsers have not been allocated, do so now. 450 * We keep these instanced between parsers, but set them 451 * locally per parse routine since we can use different 452 * parsers with each one. 453 */ 454 455 if ( ! (curp->man || curp->mdoc)) 456 pset(ln.buf + of, pos - of, curp); 457 458 /* 459 * Lastly, push down into the parsers themselves. One 460 * of these will have already been set in the pset() 461 * routine. 462 * If libroff returns ROFF_TBL, then add it to the 463 * currently open parse. Since we only get here if 464 * there does exist data (see tbl_data.c), we're 465 * guaranteed that something's been allocated. 466 * Do the same for ROFF_EQN. 467 */ 468 469 rc = -1; 470 471 if (ROFF_TBL == rr) 472 while (NULL != (span = roff_span(curp->roff))) { 473 rc = curp->man ? 474 man_addspan(curp->man, span) : 475 mdoc_addspan(curp->mdoc, span); 476 if (0 == rc) 477 break; 478 } 479 else if (ROFF_EQN == rr) 480 rc = curp->mdoc ? 481 mdoc_addeqn(curp->mdoc, 482 roff_eqn(curp->roff)) : 483 man_addeqn(curp->man, 484 roff_eqn(curp->roff)); 485 else if (curp->man || curp->mdoc) 486 rc = curp->man ? 487 man_parseln(curp->man, 488 curp->line, ln.buf, of) : 489 mdoc_parseln(curp->mdoc, 490 curp->line, ln.buf, of); 491 492 if (0 == rc) { 493 assert(MANDOCLEVEL_FATAL <= curp->file_status); 494 break; 495 } 496 497 /* Temporary buffers typically are not full. */ 498 499 if (0 == start && '\0' == blk.buf[i]) 500 break; 501 502 /* Start the next input line. */ 503 504 pos = 0; 505 } 506 507 free(ln.buf); 508 } 509 510 static void 511 pdesc(struct mparse *curp, const char *file, int fd) 512 { 513 struct buf blk; 514 int with_mmap; 515 516 /* 517 * Run for each opened file; may be called more than once for 518 * each full parse sequence if the opened file is nested (i.e., 519 * from `so'). Simply sucks in the whole file and moves into 520 * the parse phase for the file. 521 */ 522 523 if ( ! read_whole_file(file, fd, &blk, &with_mmap)) { 524 curp->file_status = MANDOCLEVEL_SYSERR; 525 return; 526 } 527 528 /* Line number is per-file. */ 529 530 curp->line = 1; 531 532 mparse_buf_r(curp, blk, 1); 533 534 #ifdef HAVE_MMAP 535 if (with_mmap) 536 munmap(blk.buf, blk.sz); 537 else 538 #endif 539 free(blk.buf); 540 } 541 542 static int 543 read_whole_file(const char *file, int fd, struct buf *fb, int *with_mmap) 544 { 545 size_t off; 546 ssize_t ssz; 547 548 #ifdef HAVE_MMAP 549 struct stat st; 550 if (-1 == fstat(fd, &st)) { 551 perror(file); 552 return(0); 553 } 554 555 /* 556 * If we're a regular file, try just reading in the whole entry 557 * via mmap(). This is faster than reading it into blocks, and 558 * since each file is only a few bytes to begin with, I'm not 559 * concerned that this is going to tank any machines. 560 */ 561 562 if (S_ISREG(st.st_mode)) { 563 if (st.st_size >= (1U << 31)) { 564 fprintf(stderr, "%s: input too large\n", file); 565 return(0); 566 } 567 *with_mmap = 1; 568 fb->sz = (size_t)st.st_size; 569 fb->buf = mmap(NULL, fb->sz, PROT_READ, 570 MAP_FILE|MAP_SHARED, fd, 0); 571 if (fb->buf != MAP_FAILED) 572 return(1); 573 } 574 #endif 575 576 /* 577 * If this isn't a regular file (like, say, stdin), then we must 578 * go the old way and just read things in bit by bit. 579 */ 580 581 *with_mmap = 0; 582 off = 0; 583 fb->sz = 0; 584 fb->buf = NULL; 585 for (;;) { 586 if (off == fb->sz) { 587 if (fb->sz == (1U << 31)) { 588 fprintf(stderr, "%s: input too large\n", file); 589 break; 590 } 591 resize_buf(fb, 65536); 592 } 593 ssz = read(fd, fb->buf + (int)off, fb->sz - off); 594 if (ssz == 0) { 595 fb->sz = off; 596 return(1); 597 } 598 if (ssz == -1) { 599 perror(file); 600 break; 601 } 602 off += (size_t)ssz; 603 } 604 605 free(fb->buf); 606 fb->buf = NULL; 607 return(0); 608 } 609 610 static void 611 mparse_end(struct mparse *curp) 612 { 613 614 if (MANDOCLEVEL_FATAL <= curp->file_status) 615 return; 616 617 if (curp->mdoc && ! mdoc_endparse(curp->mdoc)) { 618 assert(MANDOCLEVEL_FATAL <= curp->file_status); 619 return; 620 } 621 622 if (curp->man && ! man_endparse(curp->man)) { 623 assert(MANDOCLEVEL_FATAL <= curp->file_status); 624 return; 625 } 626 627 if ( ! (curp->man || curp->mdoc)) { 628 mandoc_msg(MANDOCERR_NOTMANUAL, curp, 1, 0, NULL); 629 curp->file_status = MANDOCLEVEL_FATAL; 630 return; 631 } 632 633 roff_endparse(curp->roff); 634 } 635 636 static void 637 mparse_readfd_r(struct mparse *curp, int fd, const char *file, int re) 638 { 639 const char *svfile; 640 641 if (-1 == fd) 642 if (-1 == (fd = open(file, O_RDONLY, 0))) { 643 perror(file); 644 curp->file_status = MANDOCLEVEL_SYSERR; 645 return; 646 } 647 648 svfile = curp->file; 649 curp->file = file; 650 651 pdesc(curp, file, fd); 652 653 if (0 == re && MANDOCLEVEL_FATAL > curp->file_status) 654 mparse_end(curp); 655 656 if (STDIN_FILENO != fd && -1 == close(fd)) 657 perror(file); 658 659 curp->file = svfile; 660 } 661 662 enum mandoclevel 663 mparse_readfd(struct mparse *curp, int fd, const char *file) 664 { 665 666 mparse_readfd_r(curp, fd, file, 0); 667 return(curp->file_status); 668 } 669 670 struct mparse * 671 mparse_alloc(enum mparset inttype, enum mandoclevel wlevel, mandocmsg mmsg, void *arg) 672 { 673 struct mparse *curp; 674 675 assert(wlevel <= MANDOCLEVEL_FATAL); 676 677 curp = mandoc_calloc(1, sizeof(struct mparse)); 678 679 curp->wlevel = wlevel; 680 curp->mmsg = mmsg; 681 curp->arg = arg; 682 curp->inttype = inttype; 683 684 curp->roff = roff_alloc(&curp->regs, curp); 685 return(curp); 686 } 687 688 void 689 mparse_reset(struct mparse *curp) 690 { 691 692 memset(&curp->regs, 0, sizeof(struct regset)); 693 694 roff_reset(curp->roff); 695 696 if (curp->mdoc) 697 mdoc_reset(curp->mdoc); 698 if (curp->man) 699 man_reset(curp->man); 700 701 curp->file_status = MANDOCLEVEL_OK; 702 curp->mdoc = NULL; 703 curp->man = NULL; 704 } 705 706 void 707 mparse_free(struct mparse *curp) 708 { 709 710 if (curp->pmdoc) 711 mdoc_free(curp->pmdoc); 712 if (curp->pman) 713 man_free(curp->pman); 714 if (curp->roff) 715 roff_free(curp->roff); 716 717 free(curp); 718 } 719 720 void 721 mparse_result(struct mparse *curp, struct mdoc **mdoc, struct man **man) 722 { 723 724 if (mdoc) 725 *mdoc = curp->mdoc; 726 if (man) 727 *man = curp->man; 728 } 729 730 void 731 mandoc_vmsg(enum mandocerr t, struct mparse *m, 732 int ln, int pos, const char *fmt, ...) 733 { 734 char buf[256]; 735 va_list ap; 736 737 va_start(ap, fmt); 738 vsnprintf(buf, sizeof(buf) - 1, fmt, ap); 739 va_end(ap); 740 741 mandoc_msg(t, m, ln, pos, buf); 742 } 743 744 void 745 mandoc_msg(enum mandocerr er, struct mparse *m, 746 int ln, int col, const char *msg) 747 { 748 enum mandoclevel level; 749 750 level = MANDOCLEVEL_FATAL; 751 while (er < mandoclimits[level]) 752 level--; 753 754 if (level < m->wlevel) 755 return; 756 757 if (m->mmsg) 758 (*m->mmsg)(er, level, m->file, ln, col, msg); 759 760 if (m->file_status < level) 761 m->file_status = level; 762 } 763 764 const char * 765 mparse_strerror(enum mandocerr er) 766 { 767 768 return(mandocerrs[er]); 769 } 770 771 const char * 772 mparse_strlevel(enum mandoclevel lvl) 773 { 774 return(mandoclevels[lvl]); 775 } 776